def _test_mnodelog_bigtable(N, M, K): with TemporaryDirectory() as dbpath: with zlmdb.Database(dbpath, maxsize=(5 * 2**30)) as db: schema = Schema.attach(db) data = {} print() # fill table # started = time_ns() with db.begin(write=True) as txn: for i in range(N): rec = MNodeLog() fill_mnodelog(rec) key = (rec.timestamp, rec.node_id) schema.mnode_logs[txn, key] = rec data[key] = rec duration = (time_ns() - started) / 1000000000. rps = int(round(N / duration)) duration = int(round(duration)) print('Inserted {} records in {} seconds [{} records/sec]'.format( N, duration, rps)) skeys = sorted(data.keys()) # random single record selects # if True: started = time_ns() with db.begin() as txn: for i in range(M): key = random.choice(skeys) mnodelog = schema.mnode_logs[txn, key] assert mnodelog duration = (time_ns() - started) / 1000000000. rps = int(round(M / duration)) duration = int(round(duration)) print('Selected {} records in {} seconds [{} records/sec]'. format(M, duration, rps)) # random range counts # if True: started = time_ns() with db.begin() as txn: for i in range(K): # we select a fixed range of (max) 1000 elements: i1 = random.randint(0, len(skeys) - 1) i2 = random.randint(i1, min(len(skeys) - 1, i1 + 1000)) key1 = skeys[i1] key2 = skeys[i2] cnt = schema.mnode_logs.count_range(txn, from_key=key1, to_key=key2) assert cnt == len(skeys[i1:i2]) duration = (time_ns() - started) / 1000000000. rps = int(round(K / duration)) duration = int(round(duration)) print( 'Performed {} range counts in {} seconds [{} queries/sec]'. format(K, duration, rps))
def test_mnodelog_insert(N=1000): with TemporaryDirectory() as dbpath: with zlmdb.Database(dbpath) as db: schema = Schema.attach(db) data = {} # insert test data # with db.begin(write=True) as txn: for i in range(N): rec = MNodeLog() fill_mnodelog(rec) key = (rec.timestamp, rec.node_id) schema.mnode_logs[txn, key] = rec data[key] = rec # do test scans over inserted data # with db.begin() as txn: cnt = schema.mnode_logs.count(txn) assert cnt == N # do a simple full scan and compare to original data # for mnodelog in schema.mnode_logs.select(txn, return_keys=False): key = (mnodelog.timestamp, mnodelog.node_id) _mnodelog = data.get(key, None) # check that we have the record in the original data assert _mnodelog # check that the record data is equal to the original data assert mnodelog.timestamp == _mnodelog.timestamp assert mnodelog.node_id == _mnodelog.node_id assert mnodelog.run_id == _mnodelog.run_id assert mnodelog.state == _mnodelog.state assert mnodelog.ended == _mnodelog.ended assert mnodelog.session == _mnodelog.session assert mnodelog.sent == _mnodelog.sent assert mnodelog.seq == _mnodelog.seq assert mnodelog.routers == _mnodelog.routers assert mnodelog.containers == _mnodelog.containers assert mnodelog.guests == _mnodelog.guests assert mnodelog.proxies == _mnodelog.proxies assert mnodelog.marketmakers == _mnodelog.marketmakers assert mnodelog.cpu_ctx_switches == _mnodelog.cpu_ctx_switches assert mnodelog.cpu_freq == _mnodelog.cpu_freq assert mnodelog.cpu_guest == _mnodelog.cpu_guest assert mnodelog.cpu_guest_nice == _mnodelog.cpu_guest_nice assert mnodelog.cpu_idle == _mnodelog.cpu_idle assert mnodelog.cpu_interrupts == _mnodelog.cpu_interrupts assert mnodelog.cpu_iotwait == _mnodelog.cpu_iotwait assert mnodelog.cpu_irq == _mnodelog.cpu_irq assert mnodelog.cpu_nice == _mnodelog.cpu_nice assert mnodelog.cpu_soft_interrupts == _mnodelog.cpu_soft_interrupts assert mnodelog.cpu_softirq == _mnodelog.cpu_softirq assert mnodelog.cpu_steal == _mnodelog.cpu_steal assert mnodelog.cpu_system == _mnodelog.cpu_system assert mnodelog.cpu_user == _mnodelog.cpu_user assert mnodelog.network_bytes_recv == _mnodelog.network_bytes_recv assert mnodelog.network_bytes_sent == _mnodelog.network_bytes_sent assert mnodelog.network_connection_af_inet == _mnodelog.network_connection_af_inet assert mnodelog.network_connection_af_inet6 == _mnodelog.network_connection_af_inet6 assert mnodelog.network_connection_af_unix == _mnodelog.network_connection_af_unix assert mnodelog.network_dropin == _mnodelog.network_dropin assert mnodelog.network_dropout == _mnodelog.network_dropout assert mnodelog.network_errin == _mnodelog.network_errin assert mnodelog.network_errout == _mnodelog.network_errout assert mnodelog.network_packets_recv == _mnodelog.network_packets_recv assert mnodelog.network_packets_sent == _mnodelog.network_packets_sent assert mnodelog.memory_active == _mnodelog.memory_active assert mnodelog.memory_available == _mnodelog.memory_available assert mnodelog.memory_buffers == _mnodelog.memory_buffers assert mnodelog.memory_cached == _mnodelog.memory_cached assert mnodelog.memory_free == _mnodelog.memory_free assert mnodelog.memory_inactive == _mnodelog.memory_inactive assert mnodelog.memory_percent == _mnodelog.memory_percent assert mnodelog.memory_shared == _mnodelog.memory_shared assert mnodelog.memory_slab == _mnodelog.memory_slab assert mnodelog.memory_total == _mnodelog.memory_total assert mnodelog.memory_used == _mnodelog.memory_used assert mnodelog.disk_busy_time == _mnodelog.disk_busy_time assert mnodelog.disk_read_bytes == _mnodelog.disk_read_bytes assert mnodelog.disk_read_count == _mnodelog.disk_read_count assert mnodelog.disk_read_merged_count == _mnodelog.disk_read_merged_count assert mnodelog.disk_read_time == _mnodelog.disk_read_time assert mnodelog.disk_write_bytes == _mnodelog.disk_write_bytes assert mnodelog.disk_write_count == _mnodelog.disk_write_count assert mnodelog.disk_write_merged_count == _mnodelog.disk_write_merged_count assert mnodelog.disk_write_time == _mnodelog.disk_write_time
def test_mnodelog_queries(N=1000): with TemporaryDirectory() as dbpath: with zlmdb.Database(dbpath) as db: schema = Schema.attach(db) data = {} # insert test data # with db.begin(write=True) as txn: for i in range(N): rec = MNodeLog() fill_mnodelog(rec) key = (rec.timestamp, rec.node_id) schema.mnode_logs[txn, key] = rec data[key] = rec # do test scans over inserted data # with db.begin() as txn: # do some record counting queries # skeys = sorted(data.keys()) for key in skeys: mnodelog = schema.mnode_logs[txn, key] assert mnodelog first_key = (np.datetime64(0, 'ns'), uuid.UUID(bytes=b'\0' * 16)) last_key = (np.datetime64(2**63 - 1, 'ns'), uuid.UUID(bytes=b'\xff' * 16)) cnt = schema.mnode_logs.count_range(txn, from_key=first_key, to_key=last_key) assert cnt == N cnt = schema.mnode_logs.count_range(txn, from_key=skeys[0], to_key=skeys[-1]) assert cnt == N - 1 from_key = skeys[0] to_key = (skeys[-1][0], uuid.UUID(bytes=b'\xff' * 16)) cnt = schema.mnode_logs.count_range(txn, from_key=from_key, to_key=to_key) assert cnt == N K = len(skeys) // 2 cnt = schema.mnode_logs.count_range(txn, from_key=skeys[0], to_key=skeys[K]) assert cnt == N - K K = 10 from_key = skeys[-K] to_key = (skeys[-1][0], uuid.UUID(bytes=b'\xff' * 16)) cnt = schema.mnode_logs.count_range(txn, from_key=from_key, to_key=to_key) assert cnt == K # do some scanning queries # # full scan keys1 = [] for key in schema.mnode_logs.select(txn, return_values=False, reverse=False): keys1.append(key) assert len(keys1) == N # full reverse scan keys2 = [] for key in schema.mnode_logs.select(txn, return_values=False, reverse=True): keys2.append(key) assert len(keys2) == N assert keys1 == list(reversed(keys2)) # scan [from_key, to_key[ keys1 = [] for key in schema.mnode_logs.select(txn, return_values=False, from_key=from_key, to_key=to_key, reverse=False): keys1.append(key) assert len(keys1) == K # reverse scan [from_key, to_key[ keys2 = [] for key in schema.mnode_logs.select(txn, return_values=False, from_key=from_key, to_key=to_key, reverse=True): keys2.append(key) assert len(keys2) == K assert keys1 == list(reversed(keys2)) K = len(skeys) // 2 anchor_key = skeys[K] # scan [from_key, .. keys1 = [] for key in schema.mnode_logs.select(txn, return_values=False, from_key=anchor_key, reverse=False): keys1.append(key) assert len(keys1) == K assert skeys[K:] == keys1 # reverse scan ..., to_key[ keys2 = [] for key in schema.mnode_logs.select(txn, return_values=False, to_key=anchor_key, reverse=True): keys2.append(key) assert len(keys2) == K assert skeys[:K] == list(reversed(keys2)) # scan a range with 2 boundaries # K = 10 from_key = skeys[K] to_key = skeys[-K] _skeys = skeys[K:-K] L = len(_skeys) cnt = schema.mnode_logs.count_range(txn, from_key=from_key, to_key=to_key) assert cnt == L # scan [from_key, to_key[ keys1 = [] for key in schema.mnode_logs.select(txn, return_values=False, from_key=from_key, to_key=to_key, reverse=False): keys1.append(key) assert len(keys1) == L assert _skeys == keys1 # reverse scan [from_key, to_key[ keys2 = [] for key in schema.mnode_logs.select(txn, return_values=False, from_key=from_key, to_key=to_key, reverse=True): keys2.append(key) assert len(keys2) == L assert _skeys == list(reversed(keys2))
def mnodelog(): _mnodelog = MNodeLog() fill_mnodelog(_mnodelog) return _mnodelog