def index_measurements(config):
"""
Build an index of the measurement results, from the
binary log files generated by fetchmesh.
"""
# index[msm_type][msm_id][prb_id][file][offset]: size
index = Tree()
for file in config.atlas_dir.iterdir():
meta = meta_from_filename(file)
if not isinstance(meta, AtlasResultsMeta):
continue
# Not strictly necessary, but we store the file path relative to the datastore,
# to make things a bit more robust.
relfile = file.relative_to(config.atlas_dir)
# TODO: Helper/property in fetchmesh to get the log file?
logf = file.with_suffix(file.suffix + ".log")
offset = 0
with logf.open("rb") as f:
for (size, msm_id, prb_id) in LogEntry.iter_unpack(f.read()):
index[meta.type][msm_id][prb_id][str(relfile)][offset] = size
offset += size
with config.index_file.open("wb") as f:
pickle.dump(index, f, protocol=4)
def cleanup_files(config):
for file in config.rib_dir.iterdir():
meta = meta_from_filename(file)
if ((not isinstance(meta, IPASNMeta) and not isinstance(meta, RIBMeta))
or meta.datetime < config.span_start
or meta.datetime > config.span_stop):
log.info("Removing %s", file)
file.unlink()
def index_ipasn(config):
log.info("Indexing IPASN files by date")
asnfiles = {}
for file in config.rib_dir.iterdir():
meta = meta_from_filename(file)
if isinstance(meta, IPASNMeta):
asnfiles[meta.datetime.date()] = file
return asnfiles
def convert_ribs(config):
for file in config.rib_dir.iterdir():
m = meta_from_filename(file)
if isinstance(m, RIBMeta):
meta = IPASNMeta(m.collector, m.datetime)
dest = config.rib_dir / meta.filename
log.info("%s", meta)
if not dest.exists():
prefixes = mrtx.parse_mrt_file(str(file), print_progress=True)
mrtx.dump_prefixes_to_file(prefixes, dest, file)
def index_traceroutes(config):
log.info("Indexing traceroute results files by date")
traceroutes = defaultdict(list)
for file in config.atlas_dir.iterdir():
meta = meta_from_filename(file)
if (
isinstance(meta, AtlasResultsMeta)
and meta.type == MeasurementType.Traceroute
):
traceroutes[meta.start_date.date()].append(file)
return traceroutes
def test_ipasn_meta(t, fqdn):
c = Collector.from_fqdn(fqdn)
m = IPASNMeta(c, t)
assert IPASNMeta.from_filename(m.filename) == m
assert meta_from_filename(m.filename) == m
def test_rib_meta(t, fqdn):
c = Collector.from_fqdn(fqdn)
m = RIBMeta(c, t)
assert RIBMeta.from_filename(m.filename) == m
assert meta_from_filename(m.filename) == m
def test_atlas_results_meta(meta):
assert AtlasResultsMeta.from_filename(meta.filename) == meta
assert meta_from_filename(meta.filename) == meta