def index_bulk_DB_BOTGENDER_TOOL_ZETTAIR():
initial = time.time()
testTool = IndexToolManager(indexName=str(botgender_db_name + '_bulk'))
start = time.time()
bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder,
botgender_truth_txt)
end = time.time()
mylogger.info(f'get_documents_DB_BOTGENDER {end - start}')
mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}')
start = time.time()
testTool.saveToTrecFileZettair(bulk)
end = time.time()
mylogger.info(f'saveToTrecFileZettair {end - start}')
start = time.time()
testTool.zettair_index()
end = time.time()
mylogger.info(f'zettair_index {end - start}')
final = time.time()
mylogger.info(f'index_bulk_DB_BOTGENDER_TOOL_ZETTAIR: {final - initial}')
def index_DB_BOTGENDER_TOOL_ELASTIC():
initial = time.time()
testTool = IndexToolManager(indexName=botgender_db_name)
start = time.time()
bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder,
botgender_truth_txt)
end = time.time()
mylogger.info(f'get_documents_DB_BOTGENDER {end - start}')
mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}')
start = time.time()
for doc in bulk:
testTool.insertElastic(doc.pop('id'), doc)
end = time.time()
mylogger.info(f'for-loop insertElastic {end - start}')
start = time.time()
testTool.refreshElastic()
end = time.time()
mylogger.info(f'refreshElastic {end - start}')
final = time.time()
mylogger.info(f'index_DB_BOTGENDER_TOOL_ELASTIC{final - initial}')
def index_bulk_DB_BOTGENDER_TOOL_ARANGO():
initial = time.time()
testTool = IndexToolManager(indexName=str(botgender_db_name + '_bulk'))
start = time.time()
bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder,
botgender_truth_txt)
end = time.time()
mylogger.info(f'get_documents_DB_BOTGENDER {end - start}')
mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}')
start = time.time()
documentList = testTool.bulkListGeneratorArango(bulk)
end = time.time()
mylogger.info(f'bulkListGeneratorArango {end - start}')
start = time.time()
testTool.bulkImportArango(documentList)
end = time.time()
mylogger.info(f'bulkImportArango {end - start}')
final = time.time()
mylogger.info(f'index_bulk_DB_BOTGENDER_TOOL_ARANGO: {final - initial}')
def index_bulk_DB_BOTGENDER_TOOL_ELASTIC():
initial = time.time()
testTool = IndexToolManager(indexName=str(botgender_db_name + '_bulk'))
start = time.time()
bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder,
botgender_truth_txt)
end = time.time()
mylogger.info(f'get_documents_DB_BOTGENDER {end - start}')
mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}')
# start = time.time()
# bulkBody = testTool.bulkInsertGeneratorElastic(bulk)
# end = time.time()
# mylogger.info(f'bulkInsertGeneratorElastic {end - start}')
start = time.time()
bulkBody = testTool.bulkHelperInsertGeneratorElastic(bulk)
end = time.time()
mylogger.info(f'bulkHelperInsertGeneratorElastic {end - start}')
# start = time.time()
# testTool.bulkElastic(bulkBody)
# end = time.time()
# mylogger.info(f'bulkElastic {end - start}')
start = time.time()
testTool.bulkHelperElastic(bulkBody)
end = time.time()
mylogger.info(f'bulkHelperElastic {end - start}')
start = time.time()
testTool.refreshElastic()
end = time.time()
mylogger.info(f'refreshElastic {end - start}')
final = time.time()
mylogger.info(f'index_bulk_DB_BOTGENDER_TOOL_ELASTIC: {final - initial}')