def test_indexer(self): dictConfig(config.logging_config) logger = logging.getLogger() config.INDEX_CAPACITY = 1000000 table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) store = table.store indexer = table.indexer max_range = 100000 insert_perf = [] key_list = [] for i in range(max_range): key = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = Record(key, value) key_list.append(key) position = store.save(expected_data) indexer.put(expected_data.key, position, store) print("Loading data progress: " + str(i * 100.0 / max_range) + "%", end="\r") print("\n total index files: " + str(len(indexer.index_list))) total_seconds = 0.0 i = 0 for key in key_list: start_time = timeit.default_timer() indexer.get(key, store) elapsed = timeit.default_timer() - start_time insert_perf.append(elapsed * 1000.0) #to ms total_seconds += elapsed print("get test progress: " + str(i * 100.0 / max_range) + "%", end="\r") i += 1 plt.xlim([-1, max_range]) plt.ylim([0, 2]) plt.xlabel("get call") plt.ylabel("ms") plt.plot(insert_perf) plt.title(COG_VERSION + " GET BECHMARK : " + str(max_range), fontsize=12) plt.savefig("get_bench.png") print("\n ops/s: " + str(max_range / total_seconds)) print('\n num index files: ' + str(len(table.indexer.index_list))) table.close()
def test_indexer(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data = ("new super data","super new old stuff") table = Table("testdb","test_table","test_xcvzdfsadx", config, logger) store = table.store indexer = table.indexer position=store.save(expected_data) print("stored") indexer.put(expected_data[0],position,store) print("indexed by indexer") indexer.delete(expected_data[0],store) returned_data=indexer.get(expected_data[0], store) print("indexer retrieved data: "+str(returned_data)) self.assertEqual(None, returned_data) indexer.close() store.close()
def test_indexer_put_get(self): if not os.path.exists("/tmp/"+DIR_NAME+"/test_table/"): os.makedirs("/tmp/"+DIR_NAME+"/test_table/") config.COG_HOME = DIR_NAME print "*** " + config.COG_HOME + "\n" dictConfig(config.logging_config) logger = logging.getLogger() table = Table("testdb","test_table","test_xcvzdfsadx", config, logger) store = table.store indexer = table.indexer.index_list[0] max_range=100 for i in range(max_range): key= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = (key,value) position=store.save(expected_data) indexer.put(expected_data[0],position,store) returned_data=indexer.get(expected_data[0], store) print "indexer retrieved data: "+str(returned_data) self.assertEqual(expected_data, returned_data[1]) print "Test progress: "+str(i*100.0/max_range) c = 0 scanner = indexer.scanner(store) for r in scanner: # print r c += 1 print "Total records scanned: " + str(c)
def test_delete_list(self): dictConfig(config.logging_config) logger = logging.getLogger() fruits = (["apple", "orange", "banana", "pears", "cherry", "mango"]) table = Table("testdb2", "test_table", "test_xcvzdfsadx2", config, logger) print(config.COG_HOME) store = table.store index = table.indexer.index_list[0] for fruit in fruits: print("storing :" + fruit) r = Record('fruits', fruit) print("CHECK IF LIST EXISTS - - - ->") record = index.get(r.key, store) print("CHECK IF LIST EXISTS FOUND -> prev rec: " + str(record) + " get prev pos: " + str(record.store_position)) position = store.save(r, record.store_position, 'l') print("stored new list value at store pos: " + str(position)) index.put(r.key, position, store) print("indexed") index.delete(r.key, store) returned_data = index.get(r.key, store) print("retrieved data: " + str(returned_data)) self.assertTrue(returned_data.is_empty()) index.close() store.close()
def test_delete(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data = Record("new super data", "super new old stuff") table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) store = table.store index = table.indexer.index_list[0] position = store.save(expected_data) print("stored") index.put(expected_data.key, position, store) print("indexed") index.delete(expected_data.key, store) returned_data = index.get(expected_data.key, store) print("retrieved data: " + str(returned_data)) self.assertTrue(returned_data.is_empty()) index.close() store.close()
def test_collision(self): orig_conf = config.INDEX_CAPACITY dictConfig(config.logging_config) logger = logging.getLogger() expected_data_list = [ Record("rocket", "gemini-titan"), Record("rocket2", "saturn V"), Record("rocket0", "V2") ] table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) config.INDEX_CAPACITY = 4 print(config.COG_HOME) store = table.store index = table.indexer.index_list[0] for rec in expected_data_list: position = store.save(rec) index.put(rec.key, position, store) returned_data = index.get(rec.key, store) print("retrieved data: " + str(returned_data)) self.assertTrue(rec.is_equal_val(returned_data)) index.close() store.close() #set original config back config.INDEX_CAPACITY = orig_conf
def test_indexer(self): dictConfig(config.logging_config) logger = logging.getLogger() table = Table("perf_table","perf_ns","instance_1") store = Store(table,config,logger) indexer = Indexer(table,config,logger) max_range=1000000 insert_perf=[] overall_start_time = timeit.default_timer() total_seconds=0.0 for i in range(max_range): key= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = (key,value) start_time = timeit.default_timer() position=store.save(expected_data) indexer.put(expected_data[0],position,store) elapsed = timeit.default_timer() - start_time insert_perf.append(elapsed*1000.0) #to ms total_seconds += elapsed #print "Test progress: "+str(i*100.0/max_range) plt.xlim([-1,max_range]) plt.ylim([0,2]) plt.xlabel("put call") plt.ylabel("ms") plt.plot(insert_perf) plt.savefig("test.png") print "ops/s: "+str(max_range/total_seconds)
def test_put_get_record_update(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data_list = [ Record("rocket", "gemini-titan"), Record("rocket", "saturn V"), Record("rocket", "V2") ] final_expected_data = Record("rocket", "V2") table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) print(config.COG_HOME) store = table.store index = table.indexer.index_list[0] for rec in expected_data_list: position = store.save(rec) index.put(rec.key, position, store) returned_data = index.get(rec.key, store) print("retrieved data: " + str(returned_data)) updated_rec = index.get(final_expected_data.key, store) self.assertTrue(updated_rec.is_equal_val(final_expected_data)) index.close() store.close()
def test_indexer_put_get(self): dictConfig(config.logging_config) logger = logging.getLogger() table = Table("testdb", "test_table", "test_xcvzdfsadx") store = Store(table, config, logger) indexer = Indexer(table, config, logger) max_range = 100 for i in range(max_range): key = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = (key, value) position = store.save(expected_data) indexer.put(expected_data[0], position, store) returned_data = indexer.get(expected_data[0], store) print "indexer retrieved data: " + str(returned_data) self.assertEqual(expected_data, returned_data[1]) print "Test progress: " + str(i * 100.0 / max_range) c = 0 scanner = indexer.scanner(store) for r in scanner: # print r c += 1 print "Total records scanned: " + str(c)
def test_indexer(self): dictConfig(config.logging_config) config.INDEX_CAPACITY = 1000000 logger = logging.getLogger() table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) store = table.store indexer = table.indexer max_range = 100000 insert_perf = [] total_seconds = 0.0 for i in range(max_range): key = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = Record(key, value) start_time = timeit.default_timer() position = store.save(expected_data) indexer.put(expected_data.key, position, store) elapsed = timeit.default_timer() - start_time insert_perf.append(elapsed * 1000.0) #to ms total_seconds += elapsed print("Test progress: " + str(i * 100.0 / max_range) + "%", end="\r") plt.xlim([-1, max_range]) plt.ylim([0, 2]) plt.xlabel("put call") plt.ylabel("ms") plt.plot(insert_perf) plt.savefig("insert_bench.png") print("\n total index files: " + str(len(indexer.index_list))) print("\n ops/s: " + str(max_range / total_seconds)) table.close()
def test_put_get(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data = ("new super data", "super new old stuff") table = Table("testdb", "test_table", "test_xcvzdfsadx") print config.COG_HOME store = Store(table, config, logger) index = Index(table, config, logger) position = store.save(expected_data) print "stored" index.put(expected_data[0], position, store) print "indexed" returned_data = index.get(expected_data[0], store) print "retrieved data: " + str(returned_data) self.assertEqual(expected_data, returned_data[1])
def test_put_get_list(self): dictConfig(config.logging_config) logger = logging.getLogger() fruits = (["apple", "orange", "banana", "pears", "cherry", "mango"]) table = Table("testdb2", "test_table", "test_xcvzdfsadx2", config, logger) store = table.store index = table.indexer.index_list[0] for fruit in fruits: print("storing :" + fruit) r = Record('fruits', fruit, value_type='l') print("CHECK IF LIST EXISTS - - - ->") read_record = index.get(r.key, store) # print("CHECK IF LIST EXISTS FOUND -> prev rec: "+str(record)+" get prev pos: "+str(record.store_position)) if read_record is not None: print("prev record store pos: " + str(read_record.store_position)) r.set_value_link(read_record.store_position) position = store.save(r) print("stored new list value at store pos: " + str(position)) index.put(r.key, position, store) print("indexed") returned_data = index.get(r.key, store) print("retrieved data: " + str(returned_data)) self.assertTrue( returned_data.is_equal_val( Record( 'fruits', ['mango', 'cherry', 'pears', 'banana', 'orange', 'apple' ]))) index.close() store.close()
def test_put_get(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data = ("new super data","super new old stuff") table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) print(config.COG_HOME) store = table.store index = table.indexer.index_list[0] position=store.save(expected_data) print("stored") index.put(expected_data[0],position,store) print("indexed") returned_data=index.get(expected_data[0], store) print("retrieved data: "+str(returned_data)) self.assertEqual(expected_data, returned_data[1]) index.close() store.close()
def test_put_get(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data = ("new super data", "super new old stuff") table = Table("testdb", "test_table", "test_xcvzdfsadx") store = Store(table, config, logger) index = Index(table, config, logger) for i in range(30): print "Index load: " + str(index.get_load()) key = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = (key, value) position = store.save(expected_data) status = index.put(expected_data[0], position, store) if (status != None): returned_data = index.get(expected_data[0], store) self.assertEqual(expected_data, returned_data[1]) else: print "Index has reached its capacity." break c = 0 scanner = index.scanner(store) for r in scanner: print r c += 1 print "Total records scanned: " + str(c)
def test_put_get_string(self): dictConfig(config.logging_config) logger = logging.getLogger() expected_data = Record("rocket", "gemini-titan") table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) print(config.COG_HOME) store = table.store index = table.indexer.index_list[0] position = store.save(expected_data) print("stored") index.put(expected_data.key, position, store) print("indexed") returned_data = index.get(expected_data.key, store) print("retrieved data: " + str(returned_data)) self.assertTrue(expected_data.is_equal_val(returned_data)) index.close() store.close()
def test_indexer(self): dictConfig(config.logging_config) logger = logging.getLogger() config.INDEX_CAPACITY = 10003 table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger) store = table.store indexer = table.indexer max_range = 100000 plt.title("CogDB v" + COG_VERSION + " BENCHMARK Total records:" + str(max_range), fontsize=12) put_perf = [] key_list = [] total_seconds_put = 0.0 annotation = " index size: {}\n index_block_len: {}\n store read buffer: {}\n".format( config.INDEX_CAPACITY, config.INDEX_BLOCK_LEN, config.STORE_READ_BUFFER_SIZE) print(annotation) for i in range(max_range): key = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) value = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(100)) expected_data = Record(key, value) key_list.append(key) start_time = timeit.default_timer() position = store.save(expected_data) indexer.put(expected_data.key, position, store) elapsed = timeit.default_timer() - start_time put_perf.append(elapsed * 1000.0) # to ms total_seconds_put += elapsed print("Loading data progress: " + str(i * 100.0 / max_range) + "%", end="\r") plt.xlim([-1, max_range]) plt.ylim([0, 10]) plt.plot(put_perf, '-r', label="put") annotation += "\n put ops/s: " + str(max_range / total_seconds_put) total_index_file_size = 0 for i in table.indexer.index_list: total_index_file_size += Path(i.name).stat().st_size total_index_file_size = total_index_file_size >> 20 store_file_size = Path(table.store.store).stat().st_size >> 20 annotation += "\n index size: {}Mb \n store size: {}Mb ".format( total_index_file_size, store_file_size) get_perf = [] total_seconds_get = 0.0 i = 0 for key in key_list: start_time = timeit.default_timer() indexer.get(key, store) elapsed = timeit.default_timer() - start_time get_perf.append(elapsed * 1000.0) #to ms total_seconds_get += elapsed print("get test progress: " + str(i * 100.0 / max_range) + "%", end="\r") i += 1 plt.xlim([-1, max_range]) plt.ylim([0, 10]) plt.ylabel("ms") plt.plot(get_perf, '-b', label='get') plt.legend(loc="upper right") annotation += "\n get ops/s: " + str(max_range / total_seconds_get) annotation += '\n num index files: ' + str( len(table.indexer.index_list)) plt.annotate(annotation, xy=(0.05, .5), xycoords='axes fraction') notes_path = "../notes/" if not os.path.exists(notes_path): notes_path = "notes/" plt.savefig("{}bench_{}.png".format(notes_path, max_range)) table.close()