def _populate_batch(self, index, dataset, line_from, lines):
logger.info('Adding batch from line ' + str(line_from))
client = ApiClient(index.account.get_private_apiurl())
index = client.get_index(index.name)
try:
dataset_file = open(dataset_files_path + dataset.filename, 'r')
for i in range(line_from):
dataset_file.readline()
except Exception, e:
logger.error('Failed processing dataset file: %s', e)
return False, 0
def _populate_batch(self, index, dataset, line_from, lines):
logger.info('Adding batch from line ' + str(line_from))
client = ApiClient(index.account.get_private_apiurl())
index = client.get_index(index.name)
try:
dataset_file = open(dataset_files_path + dataset.filename, 'r')
for i in range(line_from):
dataset_file.readline()
except Exception, e:
logger.error('Failed processing dataset file: %s', e)
return False, 0
def monitor(self, population):
client = ApiClient(population.index.account.get_private_apiurl())
index = client.get_index(population.index.name)
if index.has_started():
if population.status == IndexPopulation.Statuses.created:
logger.info('Populating index ' + population.index.code + ' with dataset "' + population.dataset.name + '"')
population.status = IndexPopulation.Statuses.populating
population.populated_size = 0
population.save()
eof, documents_added = self._populate_batch(population.index, population.dataset, population.populated_size, batch_size)
population.populated_size = population.populated_size + documents_added
if eof:
population.status = IndexPopulation.Statuses.finished
population.save()
return True
def monitor(self, population):
client = ApiClient(population.index.account.get_private_apiurl())
index = client.get_index(population.index.name)
if index.has_started():
if population.status == IndexPopulation.Statuses.created:
logger.info('Populating index ' + population.index.code +
' with dataset "' + population.dataset.name + '"')
population.status = IndexPopulation.Statuses.populating
population.populated_size = 0
population.save()
eof, documents_added = self._populate_batch(
population.index, population.dataset,
population.populated_size, batch_size)
population.populated_size = population.populated_size + documents_added
if eof:
population.status = IndexPopulation.Statuses.finished
population.save()
return True
from lib.indextank.client import ApiClient
import sys
import time
if len(sys.argv) != 2:
print "Usage: testapi.py <API_URL>"
api = ApiClient(sys.argv[1])
index = api.get_index("testapi.py")
if index.exists():
print "deleting previous index"
index.delete_index()
print "creating index"
index.create_index()
print "waiting to start..."
while not index.has_started():
time.sleep(1)
print "adding docs"
index.add_document(1, {"text": "a b c1"}, variables={0: 1, 1: 2, 2: 3})
index.add_document(2, {"text": "a b c2"}, variables={0: 2, 1: 2, 2: 2})
index.add_document(3, {"text": "a b c3"}, variables={0: 3, 1: 2, 2: 1})
print "adding functions"
index.add_function(1, "d[0]")
index.add_function(2, "d[2]")
print "checking functions"
from lib.indextank.client import ApiClient
import sys
import time
if len(sys.argv) != 2:
print 'Usage: testapi.py <API_URL>'
api = ApiClient(sys.argv[1])
index = api.get_index('testapi.py')
if index.exists():
print 'deleting previous index'
index.delete_index()
print 'creating index'
index.create_index()
print 'waiting to start...'
while not index.has_started():
time.sleep(1)
print 'adding docs'
index.add_document(1, {'text': 'a b c1'}, variables={0: 1, 1: 2, 2: 3})
index.add_document(2, {'text': 'a b c2'}, variables={0: 2, 1: 2, 2: 2})
index.add_document(3, {'text': 'a b c3'}, variables={0: 3, 1: 2, 2: 1})
print 'adding functions'
index.add_function(1, 'd[0]')
index.add_function(2, 'd[2]')
print 'checking functions'
