def test_collision(self):
orig_conf = config.INDEX_CAPACITY
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data_list = [
Record("rocket", "gemini-titan"),
Record("rocket2", "saturn V"),
Record("rocket0", "V2")
]
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
config.INDEX_CAPACITY = 4
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
for rec in expected_data_list:
position = store.save(rec)
index.put(rec.key, position, store)
returned_data = index.get(rec.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(rec.is_equal_val(returned_data))
index.close()
store.close()
#set original config back
config.INDEX_CAPACITY = orig_conf
def test_put_get_record_update(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data_list = [
Record("rocket", "gemini-titan"),
Record("rocket", "saturn V"),
Record("rocket", "V2")
]
final_expected_data = Record("rocket", "V2")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
for rec in expected_data_list:
position = store.save(rec)
index.put(rec.key, position, store)
returned_data = index.get(rec.key, store)
print("retrieved data: " + str(returned_data))
updated_rec = index.get(final_expected_data.key, store)
self.assertTrue(updated_rec.is_equal_val(final_expected_data))
index.close()
store.close()
def test_record(self):
cache = Cache("test_cache")
record = Record("rocket",
"saturn-v",
tombstone='0',
store_position=25,
key_link=5,
value_type='l',
value_link=54378)
marshalled_record = record.marshal()
cache.put(0, marshalled_record)
print(cache.get(0))
byte_partial_value = str(10075).encode().rjust(Record.RECORD_LINK_LEN)
cache.partial_update_from_zero_index(0, byte_partial_value)
print(cache.get(0))
unmarshalled_record = Record.unmarshal(cache.get(0))
print(unmarshalled_record)
self.assertTrue(record.is_equal_val(unmarshalled_record))
self.assertEqual(record.key, unmarshalled_record.key)
self.assertEqual(record.value, unmarshalled_record.value)
self.assertEqual(record.tombstone, unmarshalled_record.tombstone)
self.assertEqual(10075, unmarshalled_record.key_link)
self.assertEqual(record.value_type, unmarshalled_record.value_type)
self.assertEqual(record.value_link, unmarshalled_record.value_link)
def test_db(self):
db_path = '/tmp/cogtestdb2'
try:
os.makedirs(db_path)
except OSError:
if not os.path.isdir(db_path):
raise
config.CUSTOM_COG_DB_PATH = db_path
cogdb = Cog()
# create a namespace
cogdb.create_namespace("my_namespace")
# create new table
cogdb.create_table("new_db", "my_namespace")
# put some data
cogdb.put(Record('A', 'val'))
cogdb.put(Record('B', 'val'))
cogdb.put(Record('key3', 'val'))
cogdb.put(Record('key3', 'val_updated'))
self.assertEqual(cogdb.get('key3').value, 'val_updated')
cogdb.close()
def test_put_get_list(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
fruits = (["apple", "orange", "banana", "pears", "cherry", "mango"])
table = Table("testdb2", "test_table", "test_xcvzdfsadx2", config,
logger)
store = table.store
index = table.indexer.index_list[0]
for fruit in fruits:
print("storing :" + fruit)
r = Record('fruits', fruit)
print("CHECK IF LIST EXISTS - - - ->")
record = index.get(r.key, store)
print("CHECK IF LIST EXISTS FOUND -> prev rec: " + str(record) +
" get prev pos: " + str(record.store_position))
position = store.save(r, record.store_position, 'l')
print("stored new list value at store pos: " + str(position))
index.put(r.key, position, store)
print("indexed")
returned_data = index.get(r.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(
returned_data.is_equal_val(
Record(
'fruits',
['mango', 'cherry', 'pears', 'banana', 'orange', 'apple'
])))
index.close()
store.close()
def put_node(self, vertex1, predicate, vertex2):
"""
Graph method
:param vertex1: string
:param predicate:
:param vertex2:
:return:
A - B
A - C
B - C
B - Dput_node
C - D
======
A => [B,C]
B => [A,D]
C => [A,B,D]
D => [B]
"""
# add to node set
predicate_hashed = hash_predicate(predicate)
self.use_table(self.config.GRAPH_EDGE_SET_TABLE_NAME).put(
Record(str(predicate_hashed), predicate))
self.use_table(self.config.GRAPH_NODE_SET_TABLE_NAME).put(
Record(vertex1, ""))
self.use_table(self.config.GRAPH_NODE_SET_TABLE_NAME).put(
Record(vertex2, ""))
self.use_table(predicate_hashed).put_set(
Record(out_nodes(vertex1), vertex2))
self.use_table(predicate_hashed).put_set(
Record(in_nodes(vertex2), vertex1))
def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = Record("new super data", "super new old stuff")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
position = store.save(expected_data)
print("stored")
indexer.put(expected_data.key, position, store)
print("indexed by indexer")
returned_data = indexer.get(expected_data.key, store)
print("indexer retrieved data: " + str(returned_data))
self.assertTrue(expected_data.is_equal_val(returned_data))
indexer.delete(expected_data.key, store)
returned_data = indexer.get(expected_data.key, store)
print("indexer retrieved data after delete: " + str(returned_data))
self.assertTrue(returned_data.is_empty())
indexer.close()
store.close()
def test_put_get(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
tablemeta = TableMeta("testdb", "test_table", "test_xcvzdfsadx", None)
store = Store(tablemeta, config, logger)
index = Index(tablemeta, config, logger, 0)
test_size = 30
for i in range(test_size):
key= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
value= ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(100))
expected_data = Record(key, value)
position=store.save(expected_data)
index.put(expected_data.key,position,store)
returned_data=index.get(expected_data.key, store)
self.assertTrue(expected_data.is_equal_val(returned_data))
c = 0
scanner = index.scanner(store)
for r in scanner:
print(r)
c += 1
self.assertEqual(test_size, c)
index.close()
store.close()
def test_delete(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = Record("new super data", "super new old stuff")
expected_data2 = Record("new super data", "updated value")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
index = table.indexer.index_list[0]
position = store.save(expected_data)
print("stored")
index.put(expected_data.key, position, store)
print("indexed")
position = store.save(expected_data2)
print("stored")
index.put(expected_data2.key, position, store)
print("indexed")
index.delete(expected_data.key, store)
returned_data = index.get(expected_data.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(returned_data == None)
index.close()
store.close()
def test_delete_list(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
fruits = (["apple", "orange", "banana", "pears", "cherry", "mango"])
table = Table("testdb2", "test_table", "test_xcvzdfsadx2", config,
logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
for fruit in fruits:
print("storing :" + fruit)
r = Record('fruits', fruit)
print("CHECK IF LIST EXISTS - - - ->")
record = index.get(r.key, store)
if record is not None:
print("prev record store pos: " + str(record.store_position))
r.set_value_link(record.store_position)
position = store.save(r)
print("stored new list value at store pos: " + str(position))
index.put(r.key, position, store)
print("indexed")
index.delete(r.key, store)
returned_data = index.get(r.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(returned_data == None)
index.close()
store.close()
def test_db(self):
data = Record('testKey','testVal')
cogdb = Cog()
cogdb.create_namespace("test")
cogdb.create_table("db_test", "test")
cogdb.put(data)
self.assertTrue(cogdb.get("testKey").is_equal_val(Record('testKey', 'testVal')))
cogdb.close()
def test_db(self):
data = Record('user100','{"firstname":"Hari","lastname":"seldon"}')
cogdb = Cog()
cogdb.create_namespace("test")
cogdb.create_table("db_test", "test")
cogdb.put(data)
scanner = cogdb.scanner()
res = None
for r in scanner:
res = r
print(res)
self.assertTrue(data.is_equal_val(res))
cogdb.close()
def test_record_list(self):
record = Record("rockets", ["saturn-v", "delta", "atlas", "mercury"],
tombstone='0',
store_position=25,
key_link=5,
value_type='l',
value_link=54378)
unmarshalled_record = Record.unmarshal(record.marshal())
print(unmarshalled_record)
self.assertTrue(record.is_equal_val(unmarshalled_record))
self.assertEqual(record.key, unmarshalled_record.key)
self.assertEqual(record.value, unmarshalled_record.value)
self.assertEqual(record.tombstone, unmarshalled_record.tombstone)
self.assertEqual(record.key_link, unmarshalled_record.key_link)
self.assertEqual(record.value_type, unmarshalled_record.value_type)
self.assertEqual(record.value_link, unmarshalled_record.value_link)
def test_indexer_put_get(self):
if not os.path.exists("/tmp/" + DIR_NAME + "/test_table/"):
os.makedirs("/tmp/" + DIR_NAME + "/test_table/")
config.COG_HOME = DIR_NAME
print("*** " + config.COG_HOME + "\n")
dictConfig(config.logging_config)
logger = logging.getLogger()
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer.index_list[0]
max_range = 100
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
returned_data = indexer.get(expected_data.key, store)
print("indexer retrieved data: " + str(returned_data))
self.assertTrue(expected_data.is_equal_val(returned_data))
print("Test progress: " + str(i * 100.0 / max_range))
c = 0
scanner = indexer.scanner(store)
for r in scanner:
c += 1
self.assertEqual(max_range, c)
indexer.close()
store.close()
table.close()
def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
config.INDEX_CAPACITY = 1000000
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
max_range = 100000
insert_perf = []
key_list = []
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
key_list.append(key)
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
print("Loading data progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
print("\n total index files: " + str(len(indexer.index_list)))
total_seconds = 0.0
i = 0
for key in key_list:
start_time = timeit.default_timer()
indexer.get(key, store)
elapsed = timeit.default_timer() - start_time
insert_perf.append(elapsed * 1000.0) #to ms
total_seconds += elapsed
print("get test progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
i += 1
plt.xlim([-1, max_range])
plt.ylim([0, 2])
plt.xlabel("get call")
plt.ylabel("ms")
plt.plot(insert_perf)
plt.title(COG_VERSION + " GET BECHMARK : " + str(max_range),
fontsize=12)
plt.savefig("get_bench.png")
print("\n ops/s: " + str(max_range / total_seconds))
print('\n num index files: ' + str(len(table.indexer.index_list)))
table.close()
def test_put_get_string(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
expected_data = Record("new super data", "super new old stuff")
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
print(config.COG_HOME)
store = table.store
index = table.indexer.index_list[0]
position = store.save(expected_data)
print("stored")
index.put(expected_data.key, position, store)
print("indexed")
returned_data = index.get(expected_data.key, store)
print("retrieved data: " + str(returned_data))
self.assertTrue(expected_data.is_equal_val(returned_data))
index.close()
store.close()
def test_indexer(self):
dictConfig(config.logging_config)
config.INDEX_CAPACITY = 1000000
logger = logging.getLogger()
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
max_range = 100000
insert_perf = []
total_seconds = 0.0
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
start_time = timeit.default_timer()
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
elapsed = timeit.default_timer() - start_time
insert_perf.append(elapsed * 1000.0) #to ms
total_seconds += elapsed
print("Test progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
plt.xlim([-1, max_range])
plt.ylim([0, 2])
plt.xlabel("put call")
plt.ylabel("ms")
plt.plot(insert_perf)
plt.savefig("insert_bench.png")
print("\n total index files: " + str(len(indexer.index_list)))
print("\n ops/s: " + str(max_range / total_seconds))
table.close()
def test_record(self):
record = Record("rocket",
"saturn-v",
tombstone='0',
store_position=25,
key_link=5,
value_type='s',
value_link=54378)
print(record.marshal())
unmarshalled_record = Record.unmarshal(record.marshal())
print(unmarshalled_record)
self.assertTrue(record.is_equal_val(unmarshalled_record))
self.assertEqual(record.key, unmarshalled_record.key)
self.assertEqual(record.value, unmarshalled_record.value)
self.assertEqual(record.tombstone, unmarshalled_record.tombstone)
self.assertEqual(record.key_link, unmarshalled_record.key_link)
self.assertEqual(record.value_type, unmarshalled_record.value_type)
self.assertEqual(None, unmarshalled_record.value_link)
def test_indexer(self):
dictConfig(config.logging_config)
logger = logging.getLogger()
config.INDEX_CAPACITY = 10003
table = Table("testdb", "test_table", "test_xcvzdfsadx", config,
logger)
store = table.store
indexer = table.indexer
max_range = 100000
plt.title("CogDB v" + COG_VERSION + " BENCHMARK Total records:" +
str(max_range),
fontsize=12)
put_perf = []
key_list = []
total_seconds_put = 0.0
annotation = " index size: {}\n index_block_len: {}\n store read buffer: {}\n".format(
config.INDEX_CAPACITY, config.INDEX_BLOCK_LEN,
config.STORE_READ_BUFFER_SIZE)
print(annotation)
for i in range(max_range):
key = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
value = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(100))
expected_data = Record(key, value)
key_list.append(key)
start_time = timeit.default_timer()
position = store.save(expected_data)
indexer.put(expected_data.key, position, store)
elapsed = timeit.default_timer() - start_time
put_perf.append(elapsed * 1000.0) # to ms
total_seconds_put += elapsed
print("Loading data progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
plt.xlim([-1, max_range])
plt.ylim([0, 10])
plt.plot(put_perf, '-r', label="put")
annotation += "\n put ops/s: " + str(max_range / total_seconds_put)
total_index_file_size = 0
for i in table.indexer.index_list:
total_index_file_size += Path(i.name).stat().st_size
total_index_file_size = total_index_file_size >> 20
store_file_size = Path(table.store.store).stat().st_size >> 20
annotation += "\n index size: {}Mb \n store size: {}Mb ".format(
total_index_file_size, store_file_size)
get_perf = []
total_seconds_get = 0.0
i = 0
for key in key_list:
start_time = timeit.default_timer()
indexer.get(key, store)
elapsed = timeit.default_timer() - start_time
get_perf.append(elapsed * 1000.0) #to ms
total_seconds_get += elapsed
print("get test progress: " + str(i * 100.0 / max_range) + "%",
end="\r")
i += 1
plt.xlim([-1, max_range])
plt.ylim([0, 10])
plt.ylabel("ms")
plt.plot(get_perf, '-b', label='get')
plt.legend(loc="upper right")
annotation += "\n get ops/s: " + str(max_range / total_seconds_get)
annotation += '\n num index files: ' + str(
len(table.indexer.index_list))
plt.annotate(annotation, xy=(0.05, .5), xycoords='axes fraction')
notes_path = "../notes/"
if not os.path.exists(notes_path):
notes_path = "notes/"
plt.savefig("{}bench_{}.png".format(notes_path, max_range))
table.close()