def index_bulk_DB_BOTGENDER_TOOL_ZETTAIR(): initial = time.time() testTool = IndexToolManager(indexName=str(botgender_db_name + '_bulk')) start = time.time() bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder, botgender_truth_txt) end = time.time() mylogger.info(f'get_documents_DB_BOTGENDER {end - start}') mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}') start = time.time() testTool.saveToTrecFileZettair(bulk) end = time.time() mylogger.info(f'saveToTrecFileZettair {end - start}') start = time.time() testTool.zettair_index() end = time.time() mylogger.info(f'zettair_index {end - start}') final = time.time() mylogger.info(f'index_bulk_DB_BOTGENDER_TOOL_ZETTAIR: {final - initial}')
def index_DB_BOTGENDER_TOOL_ELASTIC(): initial = time.time() testTool = IndexToolManager(indexName=botgender_db_name) start = time.time() bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder, botgender_truth_txt) end = time.time() mylogger.info(f'get_documents_DB_BOTGENDER {end - start}') mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}') start = time.time() for doc in bulk: testTool.insertElastic(doc.pop('id'), doc) end = time.time() mylogger.info(f'for-loop insertElastic {end - start}') start = time.time() testTool.refreshElastic() end = time.time() mylogger.info(f'refreshElastic {end - start}') final = time.time() mylogger.info(f'index_DB_BOTGENDER_TOOL_ELASTIC{final - initial}')
def index_bulk_DB_BOTGENDER_TOOL_ARANGO(): initial = time.time() testTool = IndexToolManager(indexName=str(botgender_db_name + '_bulk')) start = time.time() bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder, botgender_truth_txt) end = time.time() mylogger.info(f'get_documents_DB_BOTGENDER {end - start}') mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}') start = time.time() documentList = testTool.bulkListGeneratorArango(bulk) end = time.time() mylogger.info(f'bulkListGeneratorArango {end - start}') start = time.time() testTool.bulkImportArango(documentList) end = time.time() mylogger.info(f'bulkImportArango {end - start}') final = time.time() mylogger.info(f'index_bulk_DB_BOTGENDER_TOOL_ARANGO: {final - initial}')
def index_bulk_DB_BOTGENDER_TOOL_ELASTIC(): initial = time.time() testTool = IndexToolManager(indexName=str(botgender_db_name + '_bulk')) start = time.time() bulk = testTool.get_documents_DB_BOTGENDER(botgender_xml_folder, botgender_truth_txt) end = time.time() mylogger.info(f'get_documents_DB_BOTGENDER {end - start}') mylogger.info(f'TOTAL documents DB_BOTGENDER {len(bulk)}') # start = time.time() # bulkBody = testTool.bulkInsertGeneratorElastic(bulk) # end = time.time() # mylogger.info(f'bulkInsertGeneratorElastic {end - start}') start = time.time() bulkBody = testTool.bulkHelperInsertGeneratorElastic(bulk) end = time.time() mylogger.info(f'bulkHelperInsertGeneratorElastic {end - start}') # start = time.time() # testTool.bulkElastic(bulkBody) # end = time.time() # mylogger.info(f'bulkElastic {end - start}') start = time.time() testTool.bulkHelperElastic(bulkBody) end = time.time() mylogger.info(f'bulkHelperElastic {end - start}') start = time.time() testTool.refreshElastic() end = time.time() mylogger.info(f'refreshElastic {end - start}') final = time.time() mylogger.info(f'index_bulk_DB_BOTGENDER_TOOL_ELASTIC: {final - initial}')