def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
config.INDEX_CAPACITY = 1000000
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
max_range = 100000
insert_perf = []
key_list = []
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
key_list.append(key)
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
print("Loading data progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
print("\n total index files: " + str(len(indexer.index_list)))
total_seconds = 0.0
i = 0
for key in key_list:
start_time = timeit.default_timer()
indexer.get(key, store)
elapsed = timeit.default_timer() - start_time
insert_perf.append(elapsed * 1000.0) #to ms
total_seconds += elapsed
print("get test progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
i += 1
plt.xlim([-1, max_range])
plt.ylim([0, 2])
plt.xlabel("get call")
plt.ylabel("ms")
plt.plot(insert_perf)
plt.title(COG_VERSION + " GET BECHMARK : " + str(max_range),
fontsize=12)
plt.savefig("get_bench.png")
print("\n ops/s: " + str(max_range / total_seconds))
print('\n num index files: ' + str(len(table.indexer.index_list)))
table.close()
def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = ("new super data","super new old stuff")
table = Table("testdb","test_table","test_xcvzdfsadx", config, logger)
store = table.store
indexer = table.indexer
position=store.save(expected_data)
print("stored")
indexer.put(expected_data[0],position,store)
print("indexed by indexer")
indexer.delete(expected_data[0],store)
returned_data=indexer.get(expected_data[0], store)
print("indexer retrieved data: "+str(returned_data))
self.assertEqual(None, returned_data)
indexer.close()
store.close()
def test_indexer_put_get(self):
if not os.path.exists("/tmp/"+DIR_NAME+"/test_table/"):
os.makedirs("/tmp/"+DIR_NAME+"/test_table/")
config.COG_HOME = DIR_NAME
print "*** " + config.COG_HOME + "\n"
dictConfig(config.logging_config)
logger = logging.getLogger()
table = Table("testdb","test_table","test_xcvzdfsadx", config, logger)
store = table.store
indexer = table.indexer.index_list[0]
max_range=100
for i in range(max_range):
key= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
value= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(100))
expected_data = (key,value)
position=store.save(expected_data)
indexer.put(expected_data[0],position,store)
returned_data=indexer.get(expected_data[0], store)
print "indexer retrieved data: "+str(returned_data)
self.assertEqual(expected_data, returned_data[1])
print "Test progress: "+str(i*100.0/max_range)
c = 0
scanner = indexer.scanner(store)
for r in scanner:
# print r
c += 1
print "Total records scanned: " + str(c)
def test_delete_list(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
fruits = (["apple", "orange", "banana", "pears", "cherry", "mango"])
table = Table("testdb2", "test_table", "test_xcvzdfsadx2", config,
logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
for fruit in fruits:
print("storing :" + fruit)
r = Record('fruits', fruit)
print("CHECK IF LIST EXISTS - - - ->")
record = index.get(r.key, store)
print("CHECK IF LIST EXISTS FOUND -> prev rec: " + str(record) +
" get prev pos: " + str(record.store_position))
position = store.save(r, record.store_position, 'l')
print("stored new list value at store pos: " + str(position))
index.put(r.key, position, store)
print("indexed")
index.delete(r.key, store)
returned_data = index.get(r.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(returned_data.is_empty())
index.close()
store.close()
def test_delete(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = Record("new super data", "super new old stuff")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
index = table.indexer.index_list[0]
position = store.save(expected_data)
print("stored")
index.put(expected_data.key, position, store)
print("indexed")
index.delete(expected_data.key, store)
returned_data = index.get(expected_data.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(returned_data.is_empty())
index.close()
store.close()
def test_collision(self):
orig_conf = config.INDEX_CAPACITY
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data_list = [
Record("rocket", "gemini-titan"),
Record("rocket2", "saturn V"),
Record("rocket0", "V2")
]
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
config.INDEX_CAPACITY = 4
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
for rec in expected_data_list:
position = store.save(rec)
index.put(rec.key, position, store)
returned_data = index.get(rec.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(rec.is_equal_val(returned_data))
index.close()
store.close()
#set original config back
config.INDEX_CAPACITY = orig_conf
def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
table = Table("perf_table","perf_ns","instance_1")
store = Store(table,config,logger)
indexer = Indexer(table,config,logger)
max_range=1000000
insert_perf=[]
overall_start_time = timeit.default_timer()
total_seconds=0.0
for i in range(max_range):
key= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
value= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(100))
expected_data = (key,value)
start_time = timeit.default_timer()
position=store.save(expected_data)
indexer.put(expected_data[0],position,store)
elapsed = timeit.default_timer() - start_time
insert_perf.append(elapsed*1000.0) #to ms
total_seconds += elapsed
#print "Test progress: "+str(i*100.0/max_range)
plt.xlim([-1,max_range])
plt.ylim([0,2])
plt.xlabel("put call")
plt.ylabel("ms")
plt.plot(insert_perf)
plt.savefig("test.png")
print "ops/s: "+str(max_range/total_seconds)
def test_put_get_record_update(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data_list = [
Record("rocket", "gemini-titan"),
Record("rocket", "saturn V"),
Record("rocket", "V2")
]
final_expected_data = Record("rocket", "V2")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
for rec in expected_data_list:
position = store.save(rec)
index.put(rec.key, position, store)
returned_data = index.get(rec.key, store)
print("retrieved data: " + str(returned_data))
updated_rec = index.get(final_expected_data.key, store)
self.assertTrue(updated_rec.is_equal_val(final_expected_data))
index.close()
store.close()
def test_indexer_put_get(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
table = Table("testdb", "test_table", "test_xcvzdfsadx")
store = Store(table, config, logger)
indexer = Indexer(table, config, logger)
max_range = 100
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = (key, value)
position = store.save(expected_data)
indexer.put(expected_data[0], position, store)
returned_data = indexer.get(expected_data[0], store)
print "indexer retrieved data: " + str(returned_data)
self.assertEqual(expected_data, returned_data[1])
print "Test progress: " + str(i * 100.0 / max_range)
c = 0
scanner = indexer.scanner(store)
for r in scanner:
# print r
c += 1
print "Total records scanned: " + str(c)
def test_indexer(self):
dictConfig(config.logging_config)
config.INDEX_CAPACITY = 1000000
logger = logging.getLogger()
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
max_range = 100000
insert_perf = []
total_seconds = 0.0
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
start_time = timeit.default_timer()
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
elapsed = timeit.default_timer() - start_time
insert_perf.append(elapsed * 1000.0) #to ms
total_seconds += elapsed
print("Test progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
plt.xlim([-1, max_range])
plt.ylim([0, 2])
plt.xlabel("put call")
plt.ylabel("ms")
plt.plot(insert_perf)
plt.savefig("insert_bench.png")
print("\n total index files: " + str(len(indexer.index_list)))
print("\n ops/s: " + str(max_range / total_seconds))
table.close()
def test_put_get(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = ("new super data", "super new old stuff")
table = Table("testdb", "test_table", "test_xcvzdfsadx")
print config.COG_HOME
store = Store(table, config, logger)
index = Index(table, config, logger)
position = store.save(expected_data)
print "stored"
index.put(expected_data[0], position, store)
print "indexed"
returned_data = index.get(expected_data[0], store)
print "retrieved data: " + str(returned_data)
self.assertEqual(expected_data, returned_data[1])
def test_put_get_list(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
fruits = (["apple", "orange", "banana", "pears", "cherry", "mango"])
table = Table("testdb2", "test_table", "test_xcvzdfsadx2", config,
logger)
store = table.store
index = table.indexer.index_list[0]
for fruit in fruits:
print("storing :" + fruit)
r = Record('fruits', fruit, value_type='l')
print("CHECK IF LIST EXISTS - - - ->")
read_record = index.get(r.key, store)
# print("CHECK IF LIST EXISTS FOUND -> prev rec: "+str(record)+" get prev pos: "+str(record.store_position))
if read_record is not None:
print("prev record store pos: " +
str(read_record.store_position))
r.set_value_link(read_record.store_position)
position = store.save(r)
print("stored new list value at store pos: " + str(position))
index.put(r.key, position, store)
print("indexed")
returned_data = index.get(r.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(
returned_data.is_equal_val(
Record(
'fruits',
['mango', 'cherry', 'pears', 'banana', 'orange', 'apple'
])))
index.close()
store.close()
def test_put_get(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = ("new super data","super new old stuff")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config, logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
position=store.save(expected_data)
print("stored")
index.put(expected_data[0],position,store)
print("indexed")
returned_data=index.get(expected_data[0], store)
print("retrieved data: "+str(returned_data))
self.assertEqual(expected_data, returned_data[1])
index.close()
store.close()
def test_put_get(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = ("new super data", "super new old stuff")
table = Table("testdb", "test_table", "test_xcvzdfsadx")
store = Store(table, config, logger)
index = Index(table, config, logger)
for i in range(30):
print "Index load: " + str(index.get_load())
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = (key, value)
position = store.save(expected_data)
status = index.put(expected_data[0], position, store)
if (status != None):
returned_data = index.get(expected_data[0], store)
self.assertEqual(expected_data, returned_data[1])
else:
print "Index has reached its capacity."
break
c = 0
scanner = index.scanner(store)
for r in scanner:
print r
c += 1
print "Total records scanned: " + str(c)
def test_put_get_string(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = Record("rocket", "gemini-titan")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
position = store.save(expected_data)
print("stored")
index.put(expected_data.key, position, store)
print("indexed")
returned_data = index.get(expected_data.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(expected_data.is_equal_val(returned_data))
index.close()
store.close()
def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
config.INDEX_CAPACITY = 10003
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
max_range = 100000
plt.title("CogDB v" + COG_VERSION + " BENCHMARK Total records:" +
str(max_range),
fontsize=12)
put_perf = []
key_list = []
total_seconds_put = 0.0
annotation = " index size: {}\n index_block_len: {}\n store read buffer: {}\n".format(
config.INDEX_CAPACITY, config.INDEX_BLOCK_LEN,
config.STORE_READ_BUFFER_SIZE)
print(annotation)
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
key_list.append(key)
start_time = timeit.default_timer()
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
elapsed = timeit.default_timer() - start_time
put_perf.append(elapsed * 1000.0) # to ms
total_seconds_put += elapsed
print("Loading data progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
plt.xlim([-1, max_range])
plt.ylim([0, 10])
plt.plot(put_perf, '-r', label="put")
annotation += "\n put ops/s: " + str(max_range / total_seconds_put)
total_index_file_size = 0
for i in table.indexer.index_list:
total_index_file_size += Path(i.name).stat().st_size
total_index_file_size = total_index_file_size >> 20
store_file_size = Path(table.store.store).stat().st_size >> 20
annotation += "\n index size: {}Mb \n store size: {}Mb ".format(
total_index_file_size, store_file_size)
get_perf = []
total_seconds_get = 0.0
i = 0
for key in key_list:
start_time = timeit.default_timer()
indexer.get(key, store)
elapsed = timeit.default_timer() - start_time
get_perf.append(elapsed * 1000.0) #to ms
total_seconds_get += elapsed
print("get test progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
i += 1
plt.xlim([-1, max_range])
plt.ylim([0, 10])
plt.ylabel("ms")
plt.plot(get_perf, '-b', label='get')
plt.legend(loc="upper right")
annotation += "\n get ops/s: " + str(max_range / total_seconds_get)
annotation += '\n num index files: ' + str(
len(table.indexer.index_list))
plt.annotate(annotation, xy=(0.05, .5), xycoords='axes fraction')
notes_path = "../notes/"
if not os.path.exists(notes_path):
notes_path = "notes/"
plt.savefig("{}bench_{}.png".format(notes_path, max_range))
table.close()
