def setup_module(): db_name = "data/pulp/query/json_ds" with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER, idx_dumpers=IDX_DUMPERS) as db: for msg in DATA: #--- # What to index #--- index = {"name": msg["name"]} if "age" in msg: index["age"] = msg["age"] db.append(msg, index) global DB assert DB is None DB = pulp_db.open(db_name, mode="r", msg_dumper=MSG_DUMPER, msg_loader=MSG_LOADER, idx_dumpers=IDX_DUMPERS, idx_loaders=IDX_LOADERS, ) DB.__enter__() assert DB is not None
def setup_module(): db_name = "data/pulp/query/json_ds" with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER, idx_dumpers=IDX_DUMPERS) as db: for msg in DATA: #--- # What to index #--- index = {"name": msg["name"]} if "age" in msg: index["age"] = msg["age"] db.append(msg, index) global DB assert DB is None DB = pulp_db.open( db_name, mode="r", msg_dumper=MSG_DUMPER, msg_loader=MSG_LOADER, idx_dumpers=IDX_DUMPERS, idx_loaders=IDX_LOADERS, ) DB.__enter__() assert DB is not None
def setup_module(): create_db("data/pulp/end_to_end/example_ds", "data/pulp/end_to_end/example.pb", "data/pulp/end_to_end/example.spec", None) global DB, all_ips, all_ports, all_times, all_types DB = pulp_db.open("data/pulp/end_to_end/example_ds", "r") DB.__enter__() all_ips = {x.index['ip'] for x in DATA_SET} all_ports = {x.index['port'] for x in DATA_SET} all_times = {x.index['time'] for x in DATA_SET} all_types = {x.index['type'] for x in DATA_SET}
def create_db(db_name, playback, spec, index_what): global DATA_SET DATA_SET = [] with pulp_db.open(db_name, "w") as db: for msg in parse_pb(playback): index = {} index['ip'] = msg['ip'] index['port'] = msg['port'] index['time'] = msg['time'] index['type'] = next(REGULAR_MSG_TYPES) #size = msg['header_size'] + msg['payload_size'] raw = msg['msg'] DATA_SET.append(element(raw, index)) db.append(raw, index)
IDX_DUMPERS = { "name": dump_str_to_bytes, "age": dump_int, } IDX_LOADERS = { "name": load_str_to_bytes, "age": load_int, } #--- # Write the DB #--- db_name = "data/pulp/query/json_ds" with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER, idx_dumpers=IDX_DUMPERS) as db: for msg in DATA: # What to index index = {"name": msg["name"]} if "age" in msg: index["age"] = msg["age"] # Put the data into the database. # db.append(b'the_bin_json_data', {"name": "foo", "age": 5}) db.append(msg, index) #--- # Read the DB #--- with pulp_db.open(
IDX_DUMPERS = {"name": dump_str_to_bytes, "age": dump_int, } IDX_LOADERS = {"name": load_str_to_bytes, "age": load_int, } #--- # Write the DB #--- db_name = "data/pulp/query/json_ds" with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER, idx_dumpers=IDX_DUMPERS) as db: for msg in DATA: # What to index index = {"name": msg["name"]} if "age" in msg: index["age"] = msg["age"] # Put the data into the database. # db.append(b'the_bin_json_data', {"name": "foo", "age": 5}) db.append(msg, index)
meta, raw = bdata.split(meta_end, 1) h, hint, len, ip, port, time_str, time_num, thing, thing = meta.split(b'/') data = { "raw": raw, "hint": hint, "ip": ip, "time_str": time_str, "time_num": time_num } return data DS_PATH = "example_ds" PB_PATH = "example.pb" # Write playback to db with pulp_db.open(DS_PATH, "w") as db: for msg in parse_pb(PB_PATH): index = {} index['ip'] = msg['ip'] index['port'] = msg['port'] index['time'] = msg['time'] raw = msg['msg'] db.append(raw, index) # Read and query db with pulp_db.open(DS_PATH, "r", msg_loader=decoder) as db: #import pdb #pdb.set_trace()
meta_start = b"*M*E*T*A*S*T*A*R*T*" meta_end = b"*M*E*T*A*E*N*D*" meta, raw = bdata.split(meta_end, 1) h, hint, len, ip, port, time_str, time_num, thing, thing = meta.split(b'/') data = {"raw": raw, "hint": hint, "ip": ip, "time_str": time_str, "time_num": time_num} return data DS_PATH = "example_ds" PB_PATH = "example.pb" # Write playback to db with pulp_db.open(DS_PATH, "w") as db: for msg in parse_pb(PB_PATH): index = {} index['ip'] = msg['ip'] index['port'] = msg['port'] index['time'] = msg['time'] raw = msg['msg'] db.append(raw, index) # Read and query db with pulp_db.open(DS_PATH, "r", msg_loader=decoder) as db: #import pdb #pdb.set_trace()
decoded_msg = decoder(msg['payload']) msg_types = set() for x in range(1, decoded_msg[0]["MsgCount"] + 1): msg_type = decoded_msg[x]["MsgType"] msg_types.add(msg_type) index = {} index['ip'] = msg['ip'] index['port'] = msg['port'] index['time'] = msg['time'] index['msgtype'] = msg_types return raw, index # Write playback to db count = 0 with pulp_db.open(DS_PATH, "w", idx_dumpers=IDX_DUMPERS) as db: for msg in parse_pb(sys.argv[1]): count += 1 if count % 10000 == 0: print("Counted", count) raw, index = parse_msg_select_what_to_store(msg) #continue db.append(raw, index) sys.exit(1) import pdb
def create_db(db_name, data): with pulp_db.open(db_name, "w") as db: for e in data: db.append(e.raw, e.index)
def read_pulp_setup(db_name): x = pulp_db.open(db_name, "r") x.__enter__() return x
raw = msg['msg'] decoded_msg = decoder(msg['payload']) msg_types = set() for x in range(1, decoded_msg[0]["MsgCount"]+1): msg_type = decoded_msg[x]["MsgType"] msg_types.add(msg_type) index = {} index['ip'] = msg['ip'] index['port'] = msg['port'] index['time'] = msg['time'] index['msgtype'] = msg_types return raw, index # Write playback to db count = 0 with pulp_db.open(DS_PATH, "w", idx_dumpers=IDX_DUMPERS) as db: for msg in parse_pb(sys.argv[1]): count += 1 if count%10000 == 0: print("Counted", count) raw, index = parse_msg_select_what_to_store(msg) #continue db.append(raw, index) sys.exit(1) import pdb