def test_snapshot_exists(self):
# snapshot exists
self.assertFalse(self.indexer.is_snapshot_exist())
# read index config
with open(self.example_dir + '/index_config.yaml',
'r',
encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
# create file index
index_name = 'test_file_index'
self.indexer.create_index(index_name, index_config, sync=True)
self.assertTrue(self.indexer.is_index_exist(index_name))
# read documents
with open(self.example_dir + '/bulk_put.json', 'r',
encoding='utf-8') as file_obj:
test_docs = json.loads(file_obj.read(), encoding='utf-8')
# put documents in bulk
count = self.indexer.put_documents(index_name, test_docs, sync=True)
self.assertEqual(5, count)
# commit
success = self.indexer.commit_index(index_name, sync=True)
self.assertTrue(success)
# search documents
page = self.indexer.search_documents(index_name,
'search',
search_field='text',
page_num=1,
page_len=10)
self.assertEqual(5, page.total)
# create snapshot
self.indexer.create_snapshot(sync=True)
sleep(1) # wait for snapshot file to be created
self.assertTrue(os.path.exists(self.indexer.get_snapshot_file_name()))
with zipfile.ZipFile(self.indexer.get_snapshot_file_name()) as f:
self.assertTrue('raft.bin' in f.namelist())
self.assertTrue('test_file_index_WRITELOCK' in f.namelist())
self.assertEqual(
1,
len([
n for n in f.namelist()
if n.startswith('_test_file_index_') and n.endswith('.toc')
]))
self.assertEqual(
1,
len([
n for n in f.namelist()
if n.startswith('test_file_index_') and n.endswith('.seg')
]))
# snapshot exists
self.assertTrue(True, self.indexer.is_snapshot_exist())
def test_search_documents(self):
# read index config
with open(self.example_dir + '/index_config.yaml',
'r',
encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
# create file index
index_name = 'test_file_index'
self.indexer.create_index(index_name, index_config, sync=True)
self.assertTrue(self.indexer.is_index_exist(index_name))
# read documents
with open(self.example_dir + '/bulk_put.json', 'r',
encoding='utf-8') as file_obj:
test_docs = json.loads(file_obj.read(), encoding='utf-8')
# put documents in bulk
count = self.indexer.put_documents(index_name, test_docs, sync=True)
self.assertEqual(5, count)
# commit
success = self.indexer.commit_index(index_name, sync=True)
self.assertTrue(success)
# search documents
page = self.indexer.search_documents(index_name,
'search',
search_field='text',
page_num=1,
page_len=10)
self.assertEqual(5, page.total)
def test_get_document(self):
# read index config
with open(self.example_dir + '/index_config.yaml',
'r',
encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
# create index
index_name = 'test_file_index'
self.indexer.create_index(index_name, index_config, sync=True)
self.assertTrue(self.indexer.is_index_exist(index_name))
test_doc_id = '1'
with open(self.example_dir + '/doc1.json', 'r',
encoding='utf-8') as file_obj:
test_fields = json.loads(file_obj.read(), encoding='utf-8')
# put document
count = self.indexer.put_document(index_name,
test_doc_id,
test_fields,
sync=True)
self.assertEqual(1, count)
# commit
success = self.indexer.commit_index(index_name, sync=True)
self.assertTrue(success)
# get document
results_page = self.indexer.get_document(index_name, test_doc_id)
self.assertEqual(1, results_page.total)
def test_json_get_writer_auto_commit_limit(self):
file_path = self.example_dir + '/index_config.json'
with open(file_path, 'r', encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
self.assertEqual(100, index_config.get_writer_auto_commit_limit())
def test_json_get_unique_field(self):
file_path = self.example_dir + '/index_config.json'
with open(file_path, 'r', encoding='utf-8') as file_obj:
index_config_dict = json.loads(file_obj.read())
index_config = IndexConfig(index_config_dict)
self.assertEqual('id', index_config.get_doc_id_field())
def test_create_from_json(self):
file_path = self.example_dir + '/index_config.json'
with open(file_path, 'r', encoding='utf-8') as file_obj:
index_config_dict = json.loads(file_obj.read())
index_config = IndexConfig(index_config_dict)
self.assertIsNotNone(index_config)
def test_create_from_yaml(self):
file_path = self.example_dir + '/index_config.yaml'
with open(file_path, 'r', encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
self.assertIsNotNone(index_config)
def test_json_get_writer_multi_segment(self):
file_path = self.example_dir + '/index_config.json'
with open(file_path, 'r', encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
self.assertTrue(index_config.get_writer_multi_segment())
def test_yaml_get_writer_batch_size(self):
file_path = self.example_dir + '/index_config.yaml'
with open(file_path, 'r', encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
self.assertEqual(100, index_config.get_writer_batch_size())
def __create_index(self, index_name):
start_time = time.time()
@after_this_request
def to_do_after_this_request(response):
record_log(request, response, logger=self.__http_logger)
self.__record_metrics(start_time, request, response)
return response
data = {}
status_code = None
try:
mime = mimeparse.parse_mime_type(request.headers.get('Content-Type'))
charset = 'utf-8' if mime[2].get('charset') is None else mime[2].get('charset')
if mime[1] == 'yaml':
index_config_dict = yaml.safe_load(request.data.decode(charset))
elif mime[1] == 'json':
index_config_dict = json.loads(request.data.decode(charset))
else:
raise ValueError('unsupported format')
if index_config_dict is None:
raise ValueError('index config is None')
sync = False
if request.args.get('sync', default='', type=str).lower() in TRUE_STRINGS:
sync = True
index_config = IndexConfig(index_config_dict)
self.__indexer.create_index(index_name, index_config, sync=sync)
if sync:
status_code = HTTPStatus.CREATED
else:
status_code = HTTPStatus.ACCEPTED
except (ConstructorError, JSONDecodeError, ValueError) as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.BAD_REQUEST
self.__logger.error(ex)
except Exception as ex:
data['error'] = '{0}'.format(ex.args[0])
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
self.__logger.error(ex)
finally:
data['time'] = time.time() - start_time
data['status'] = {'code': status_code.value, 'phrase': status_code.phrase,
'description': status_code.description}
output = request.args.get('output', default='json', type=str).lower()
# make response
resp = make_response(data, output)
resp.status_code = status_code
return resp
def test_create_index(self):
# read index config
with open(self.example_dir + '/index_config.yaml',
'r',
encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
# create index
index_name = 'test_file_index'
self.indexer.create_index(index_name, index_config, sync=True)
self.assertTrue(self.indexer.is_index_exist(index_name))
def test_put_documents(self):
# read index config
with open(self.example_dir + '/index_config.yaml',
'r',
encoding='utf-8') as file_obj:
index_config_dict = yaml.safe_load(file_obj.read())
index_config = IndexConfig(index_config_dict)
# create index
index_name = 'test_file_index'
self.indexer.create_index(index_name, index_config, sync=True)
self.assertTrue(self.indexer.is_index_exist(index_name))
with open(self.example_dir + '/bulk_put.json', 'r',
encoding='utf-8') as file_obj:
test_docs = json.loads(file_obj.read(), encoding='utf-8')
# put documents in bulk
count = self.indexer.put_documents(index_name, test_docs, sync=True)
self.assertEqual(5, count)
# commit
success = self.indexer.commit_index(index_name, sync=True)
self.assertTrue(success)
results_page = self.indexer.get_document(index_name, '1')
self.assertEqual(1, results_page.total)
results_page = self.indexer.get_document(index_name, '2')
self.assertEqual(1, results_page.total)
results_page = self.indexer.get_document(index_name, '3')
self.assertEqual(1, results_page.total)
results_page = self.indexer.get_document(index_name, '4')
self.assertEqual(1, results_page.total)
results_page = self.indexer.get_document(index_name, '5')
self.assertEqual(1, results_page.total)
def CreateIndex(self, request, context):
start_time = time.time()
response = CreateIndexResponse()
try:
index_config = IndexConfig(pickle.loads(request.index_config))
index = self.__indexer.create_index(request.index_name, index_config, sync=request.sync)
if request.sync:
if index is None:
response.status.success = False
response.status.message = 'failed to create {0}'.format(request.index_name)
else:
response.index_stats.name = index.indexname
response.index_stats.doc_count = index.doc_count()
response.index_stats.doc_count_all = index.doc_count_all()
response.index_stats.latest_generation = index.latest_generation()
response.index_stats.version = index.version
response.index_stats.storage.folder = index.storage.folder
response.index_stats.storage.supports_mmap = index.storage.supports_mmap
response.index_stats.storage.readonly = index.storage.readonly
response.index_stats.storage.files.extend(index.storage.list())
response.status.success = True
response.status.message = '{0} was successfully created or opened'.format(index.indexname)
else:
response.status.success = True
response.status.message = 'request was successfully accepted to create {0}'.format(request.index_name)
except Exception as ex:
response.status.success = False
response.status.message = str(ex)
finally:
self.__record_metrics(start_time, 'create_index')
return response