def _populate_batch(self, index, dataset, line_from, lines): logger.info('Adding batch from line ' + str(line_from)) client = ApiClient(index.account.get_private_apiurl()) index = client.get_index(index.name) try: dataset_file = open(dataset_files_path + dataset.filename, 'r') for i in range(line_from): dataset_file.readline() except Exception, e: logger.error('Failed processing dataset file: %s', e) return False, 0
def _populate_batch(self, index, dataset, line_from, lines): logger.info('Adding batch from line ' + str(line_from)) client = ApiClient(index.account.get_private_apiurl()) index = client.get_index(index.name) try: dataset_file = open(dataset_files_path + dataset.filename, 'r') for i in range(line_from): dataset_file.readline() except Exception, e: logger.error('Failed processing dataset file: %s', e) return False, 0
def monitor(self, population): client = ApiClient(population.index.account.get_private_apiurl()) index = client.get_index(population.index.name) if index.has_started(): if population.status == IndexPopulation.Statuses.created: logger.info('Populating index ' + population.index.code + ' with dataset "' + population.dataset.name + '"') population.status = IndexPopulation.Statuses.populating population.populated_size = 0 population.save() eof, documents_added = self._populate_batch(population.index, population.dataset, population.populated_size, batch_size) population.populated_size = population.populated_size + documents_added if eof: population.status = IndexPopulation.Statuses.finished population.save() return True
def monitor(self, population): client = ApiClient(population.index.account.get_private_apiurl()) index = client.get_index(population.index.name) if index.has_started(): if population.status == IndexPopulation.Statuses.created: logger.info('Populating index ' + population.index.code + ' with dataset "' + population.dataset.name + '"') population.status = IndexPopulation.Statuses.populating population.populated_size = 0 population.save() eof, documents_added = self._populate_batch( population.index, population.dataset, population.populated_size, batch_size) population.populated_size = population.populated_size + documents_added if eof: population.status = IndexPopulation.Statuses.finished population.save() return True
from lib.indextank.client import ApiClient import sys import time if len(sys.argv) != 2: print "Usage: testapi.py <API_URL>" api = ApiClient(sys.argv[1]) index = api.get_index("testapi.py") if index.exists(): print "deleting previous index" index.delete_index() print "creating index" index.create_index() print "waiting to start..." while not index.has_started(): time.sleep(1) print "adding docs" index.add_document(1, {"text": "a b c1"}, variables={0: 1, 1: 2, 2: 3}) index.add_document(2, {"text": "a b c2"}, variables={0: 2, 1: 2, 2: 2}) index.add_document(3, {"text": "a b c3"}, variables={0: 3, 1: 2, 2: 1}) print "adding functions" index.add_function(1, "d[0]") index.add_function(2, "d[2]") print "checking functions"
from lib.indextank.client import ApiClient import sys import time if len(sys.argv) != 2: print 'Usage: testapi.py <API_URL>' api = ApiClient(sys.argv[1]) index = api.get_index('testapi.py') if index.exists(): print 'deleting previous index' index.delete_index() print 'creating index' index.create_index() print 'waiting to start...' while not index.has_started(): time.sleep(1) print 'adding docs' index.add_document(1, {'text': 'a b c1'}, variables={0: 1, 1: 2, 2: 3}) index.add_document(2, {'text': 'a b c2'}, variables={0: 2, 1: 2, 2: 2}) index.add_document(3, {'text': 'a b c3'}, variables={0: 3, 1: 2, 2: 1}) print 'adding functions' index.add_function(1, 'd[0]') index.add_function(2, 'd[2]') print 'checking functions'