def setup_module():
db_name = "data/pulp/query/json_ds"
with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER, idx_dumpers=IDX_DUMPERS) as db:
for msg in DATA:
#---
# What to index
#---
index = {"name": msg["name"]}
if "age" in msg:
index["age"] = msg["age"]
db.append(msg, index)
global DB
assert DB is None
DB = pulp_db.open(db_name,
mode="r",
msg_dumper=MSG_DUMPER,
msg_loader=MSG_LOADER,
idx_dumpers=IDX_DUMPERS,
idx_loaders=IDX_LOADERS,
)
DB.__enter__()
assert DB is not None
def setup_module():
db_name = "data/pulp/query/json_ds"
with pulp_db.open(db_name,
"w",
msg_dumper=MSG_DUMPER,
idx_dumpers=IDX_DUMPERS) as db:
for msg in DATA:
#---
# What to index
#---
index = {"name": msg["name"]}
if "age" in msg:
index["age"] = msg["age"]
db.append(msg, index)
global DB
assert DB is None
DB = pulp_db.open(
db_name,
mode="r",
msg_dumper=MSG_DUMPER,
msg_loader=MSG_LOADER,
idx_dumpers=IDX_DUMPERS,
idx_loaders=IDX_LOADERS,
)
DB.__enter__()
assert DB is not None
def setup_module():
create_db("data/pulp/end_to_end/example_ds",
"data/pulp/end_to_end/example.pb",
"data/pulp/end_to_end/example.spec", None)
global DB, all_ips, all_ports, all_times, all_types
DB = pulp_db.open("data/pulp/end_to_end/example_ds", "r")
DB.__enter__()
all_ips = {x.index['ip'] for x in DATA_SET}
all_ports = {x.index['port'] for x in DATA_SET}
all_times = {x.index['time'] for x in DATA_SET}
all_types = {x.index['type'] for x in DATA_SET}
def setup_module():
create_db("data/pulp/end_to_end/example_ds",
"data/pulp/end_to_end/example.pb",
"data/pulp/end_to_end/example.spec",
None)
global DB, all_ips, all_ports, all_times, all_types
DB = pulp_db.open("data/pulp/end_to_end/example_ds", "r")
DB.__enter__()
all_ips = {x.index['ip'] for x in DATA_SET}
all_ports = {x.index['port'] for x in DATA_SET}
all_times = {x.index['time'] for x in DATA_SET}
all_types = {x.index['type'] for x in DATA_SET}
def create_db(db_name, playback, spec, index_what):
global DATA_SET
DATA_SET = []
with pulp_db.open(db_name, "w") as db:
for msg in parse_pb(playback):
index = {}
index['ip'] = msg['ip']
index['port'] = msg['port']
index['time'] = msg['time']
index['type'] = next(REGULAR_MSG_TYPES)
#size = msg['header_size'] + msg['payload_size']
raw = msg['msg']
DATA_SET.append(element(raw, index))
db.append(raw, index)
def create_db(db_name, playback, spec, index_what):
global DATA_SET
DATA_SET = []
with pulp_db.open(db_name, "w") as db:
for msg in parse_pb(playback):
index = {}
index['ip'] = msg['ip']
index['port'] = msg['port']
index['time'] = msg['time']
index['type'] = next(REGULAR_MSG_TYPES)
#size = msg['header_size'] + msg['payload_size']
raw = msg['msg']
DATA_SET.append(element(raw, index))
db.append(raw, index)
IDX_DUMPERS = {
"name": dump_str_to_bytes,
"age": dump_int,
}
IDX_LOADERS = {
"name": load_str_to_bytes,
"age": load_int,
}
#---
# Write the DB
#---
db_name = "data/pulp/query/json_ds"
with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER,
idx_dumpers=IDX_DUMPERS) as db:
for msg in DATA:
# What to index
index = {"name": msg["name"]}
if "age" in msg:
index["age"] = msg["age"]
# Put the data into the database.
# db.append(b'the_bin_json_data', {"name": "foo", "age": 5})
db.append(msg, index)
#---
# Read the DB
#---
with pulp_db.open(
IDX_DUMPERS = {"name": dump_str_to_bytes,
"age": dump_int,
}
IDX_LOADERS = {"name": load_str_to_bytes,
"age": load_int,
}
#---
# Write the DB
#---
db_name = "data/pulp/query/json_ds"
with pulp_db.open(db_name, "w", msg_dumper=MSG_DUMPER, idx_dumpers=IDX_DUMPERS) as db:
for msg in DATA:
# What to index
index = {"name": msg["name"]}
if "age" in msg:
index["age"] = msg["age"]
# Put the data into the database.
# db.append(b'the_bin_json_data', {"name": "foo", "age": 5})
db.append(msg, index)
meta, raw = bdata.split(meta_end, 1)
h, hint, len, ip, port, time_str, time_num, thing, thing = meta.split(b'/')
data = {
"raw": raw,
"hint": hint,
"ip": ip,
"time_str": time_str,
"time_num": time_num
}
return data
DS_PATH = "example_ds"
PB_PATH = "example.pb"
# Write playback to db
with pulp_db.open(DS_PATH, "w") as db:
for msg in parse_pb(PB_PATH):
index = {}
index['ip'] = msg['ip']
index['port'] = msg['port']
index['time'] = msg['time']
raw = msg['msg']
db.append(raw, index)
# Read and query db
with pulp_db.open(DS_PATH, "r", msg_loader=decoder) as db:
#import pdb
#pdb.set_trace()
meta_start = b"*M*E*T*A*S*T*A*R*T*"
meta_end = b"*M*E*T*A*E*N*D*"
meta, raw = bdata.split(meta_end, 1)
h, hint, len, ip, port, time_str, time_num, thing, thing = meta.split(b'/')
data = {"raw": raw,
"hint": hint,
"ip": ip,
"time_str": time_str,
"time_num": time_num}
return data
DS_PATH = "example_ds"
PB_PATH = "example.pb"
# Write playback to db
with pulp_db.open(DS_PATH, "w") as db:
for msg in parse_pb(PB_PATH):
index = {}
index['ip'] = msg['ip']
index['port'] = msg['port']
index['time'] = msg['time']
raw = msg['msg']
db.append(raw, index)
# Read and query db
with pulp_db.open(DS_PATH, "r", msg_loader=decoder) as db:
#import pdb
#pdb.set_trace()
decoded_msg = decoder(msg['payload'])
msg_types = set()
for x in range(1, decoded_msg[0]["MsgCount"] + 1):
msg_type = decoded_msg[x]["MsgType"]
msg_types.add(msg_type)
index = {}
index['ip'] = msg['ip']
index['port'] = msg['port']
index['time'] = msg['time']
index['msgtype'] = msg_types
return raw, index
# Write playback to db
count = 0
with pulp_db.open(DS_PATH, "w", idx_dumpers=IDX_DUMPERS) as db:
for msg in parse_pb(sys.argv[1]):
count += 1
if count % 10000 == 0:
print("Counted", count)
raw, index = parse_msg_select_what_to_store(msg)
#continue
db.append(raw, index)
sys.exit(1)
import pdb
def create_db(db_name, data):
with pulp_db.open(db_name, "w") as db:
for e in data:
db.append(e.raw, e.index)
def read_pulp_setup(db_name):
x = pulp_db.open(db_name, "r")
x.__enter__()
return x
def create_db(db_name, data):
with pulp_db.open(db_name, "w") as db:
for e in data:
db.append(e.raw, e.index)
def read_pulp_setup(db_name):
x = pulp_db.open(db_name, "r")
x.__enter__()
return x
raw = msg['msg']
decoded_msg = decoder(msg['payload'])
msg_types = set()
for x in range(1, decoded_msg[0]["MsgCount"]+1):
msg_type = decoded_msg[x]["MsgType"]
msg_types.add(msg_type)
index = {}
index['ip'] = msg['ip']
index['port'] = msg['port']
index['time'] = msg['time']
index['msgtype'] = msg_types
return raw, index
# Write playback to db
count = 0
with pulp_db.open(DS_PATH, "w", idx_dumpers=IDX_DUMPERS) as db:
for msg in parse_pb(sys.argv[1]):
count += 1
if count%10000 == 0:
print("Counted", count)
raw, index = parse_msg_select_what_to_store(msg)
#continue
db.append(raw, index)
sys.exit(1)
import pdb
