def test_compare_normalized_with_autoclaved(autoclaved_io):
for fname, bucket_tstamp, yaml_bytes, autoclaved_msmts in autoclaved_io:
log.info("Normalizing %s", fname)
log.info("len autoclaved msmts %d", len(autoclaved_msmts))
g = norm.iter_yaml_msmt_normalized(yaml_bytes, bucket_tstamp)
normalized_msmts = list(g)
assert len(normalized_msmts) == len(autoclaved_msmts)
a_mst = list(i["measurement_start_time"] for i in autoclaved_msmts)
n_mst = list(i["measurement_start_time"] for i in normalized_msmts)
assert a_mst == n_mst
for n in range(len(autoclaved_msmts)):
autoclaved = autoclaved_msmts[n]
normalized = normalized_msmts[n]
log.info(n)
for k, v in sorted(autoclaved.items()):
log.info("K %s", k)
if isinstance(v, dict):
for k1, v1 in sorted(v.items()):
log.info("K1 %s->%s", k, k1)
assert v1 == normalized[k][k1]
else:
assert v == normalized[k]
def test_normalize_yaml_dns_consistency_2018(cans):
can = cans["yaml18"]
canfn = can.as_posix()
day = canfn.split("/")[1]
rfn = canfn.split("/", 1)[1][:-4] # remove testdata/ and .lz4
with lz4frame.open(can) as f:
for n, entry in enumerate(norm.iter_yaml_msmt_normalized(f, day, rfn)):
ujson.dumps(entry) # ensure it's serializable
def load_multiple(fn, touch=True) -> tuple:
"""Load contents of cans. Decompress tar archives if found.
Yields measurements one by one as:
(string of JSON, None) or (None, msmt dict)
"""
if touch:
os.utime(fn) # update access time - used for cache cleanup
# TODO: handle:
# RuntimeError: LZ4F_decompress failed with code: ERROR_decompressionFailed
if fn.endswith(".tar.lz4"):
with lz4frame.open(fn) as f:
tf = tarfile.TarFile(fileobj=f)
while True:
m = tf.next()
if m is None:
break
log.debug("Loading nested %s", m.name)
k = tf.extractfile(m)
if m.name.endswith(".json"):
for line in k:
yield (line, None)
elif m.name.endswith(".yaml"):
continue # FIXME
bucket_tstamp = "FIXME"
for msm in iter_yaml_msmt_normalized(k, bucket_tstamp):
yield (None, msm)
elif fn.endswith(".json.lz4"):
with lz4frame.open(fn) as f:
for line in f:
yield (line, None)
elif fn.endswith(".yaml.lz4"):
with lz4frame.open(fn) as f:
raise Exception("Unsupported format: YAML")
bucket_tstamp = "FIXME"
for msm in iter_yaml_msmt_normalized(f, bucket_tstamp):
metrics.incr("yaml_normalization")
yield (None, msm)
else:
raise RuntimeError(fn)
def test_normalize_yaml_2016(cans):
can = cans["yaml16"]
canfn = can.as_posix()
assert canfn.startswith("testdata/2016-07-07/20160706T000046Z-GB")
day = canfn.split("/")[1]
rfn = canfn.split("/", 1)[1][:-4] # remove testdata/ and .lz4
with lz4frame.open(can) as f:
for n, entry in enumerate(norm.iter_yaml_msmt_normalized(f, day, rfn)):
ujson.dumps(entry) # ensure it's serializable
if n == 0:
with open("fastpath/tests/data/yaml16_0.json") as f:
exp = ujson.load(f)
assert entry == exp
elif n > 20:
break
def test_normalize_yaml_dns_consistency_2017(cans):
can = cans["yaml17"]
canfn = can.as_posix()
day = canfn.split("/")[1]
rfn = canfn.split("/", 1)[1][:-4] # remove testdata/ and .lz4
# s3://ooni-data/autoclaved/jsonl.tar.lz4/2017-12-21/20171220T153044Z-BE-AS5432-dns_consistency-mnKRlHuqk8Eo6XMJt5ZkVQrgReaEXPEWaO9NafgXxSVIhAswTXT7QJc6zhsuttpK-0.1.0-probe.yaml.lz4
# lz4cat <fn> | head -n1 | jq -S . > fastpath/tests/data/yaml17_0.json
with lz4frame.open(can) as f:
for n, entry in enumerate(norm.iter_yaml_msmt_normalized(f, day, rfn)):
ujson.dumps(entry) # ensure it's serializable
if n == 0:
with open("fastpath/tests/data/yaml17_0.json") as f:
exp = ujson.load(f)
assert entry == exp
elif n > 20:
break
def _fetch_measurement(self, fn):
"""Fetch measurements from one collector using SSH/SFTP
:yields: (string of JSON, msmt dict) or (None, msmt dict)
"""
t = time.time()
try:
log.debug("Fetching %s", fn)
fn = os.path.join(self._archive_dir, fn)
with io.BytesIO() as data:
metrics.gauge("fetching", 1)
t = metrics.timer("fetch").start()
# Fetch all data in a blocking call
self.sftp.getfo(fn, data)
metrics.gauge("fetching", 0)
t.stop()
data_len = data.tell()
data.seek(0)
metrics.incr("fetched.count")
metrics.incr("fetched.data", data_len)
metrics.gauge("fetching_bw_KBps",
data_len / (t.ms or 0.000_000_001))
if fn.endswith(".yaml"):
raise Exception("Unsupported format: YAML")
bucket_tstamp = "FIXME"
for msm in normalize.iter_yaml_msmt_normalized(
data, bucket_tstamp):
yield (None, msm)
else:
# JSON documents
while True:
line = data.readline()
if len(line) == 0:
break
yield (line, None)
except Exception as e:
metrics.gauge("fetching", 0)
log.exception(e)
metrics.incr("unhandled_exception")
def load_multiple(fn: str) -> Generator[MsmtTup, None, None]:
"""Load contents of legacy cans and minicans.
Decompress tar archives if found.
Yields measurements one by one as:
(string of JSON, None, uid) or (None, msmt dict, uid)
The uid is either taken from the filename or generated by trivial_id for
legacy cans
"""
# TODO: split this and handle legacy cans and post/minicans independently
if fn.endswith(".tar.lz4"):
# Legacy lz4 cans
with lz4frame.open(fn) as f:
tf = tarfile.TarFile(fileobj=f)
while True:
m = tf.next()
if m is None:
# end of tarball
break
log.debug("Loading nested %s", m.name)
k = tf.extractfile(m)
assert k is not None
if m.name.endswith(".json"):
for line in k:
msm = ujson.loads(line)
msmt_uid = trivial_id(msm)
yield (None, msm, msmt_uid)
elif m.name.endswith(".yaml"):
bucket_tstamp = fn.split("/")[-2]
rfn = f"{bucket_tstamp}/" + fn.split("/")[-1]
for msm in iter_yaml_msmt_normalized(
k, bucket_tstamp, rfn):
metrics.incr("yaml_normalization")
msmt_uid = trivial_id(msm)
yield (None, msm, msmt_uid)
elif fn.endswith(".json.lz4"):
# Legacy lz4 json files
with lz4frame.open(fn) as f:
for line in f:
msm = ujson.loads(line)
msmt_uid = trivial_id(msm)
yield (None, msm, msmt_uid)
elif fn.endswith(".yaml.lz4"):
# Legacy lz4 yaml files
with lz4frame.open(fn) as f:
bucket_tstamp = fn.split("/")[-2]
rfn = f"{bucket_tstamp}/" + fn.split("/")[-1]
for msm in iter_yaml_msmt_normalized(f, bucket_tstamp, rfn):
metrics.incr("yaml_normalization")
msmt_uid = trivial_id(msm)
yield (None, msm, msmt_uid)
elif fn.endswith(".tar.gz"):
# minican with missing gzipping :(
tf = tarfile.open(fn)
while True:
m = tf.next()
if m is None:
# end of tarball
tf.close()
break
log.debug("Loading %s", m.name)
k = tf.extractfile(m)
assert k is not None
if not m.name.endswith(".post"):
log.error("Unexpected filename")
continue
try:
j = ujson.loads(k.read())
except Exception:
log.error(repr(k[:100]), exc_info=1)
continue
fmt = j.get("format", "")
if fmt == "json":
msm = j.get("content", {})
# extract msmt_uid from filename e.g:
# ... /20210614004521.999962_JO_signal_68eb19b439326d60.post
msmt_uid = m.name.rsplit("/", 1)[1]
msmt_uid = msmt_uid[:-5]
yield (None, msm, msmt_uid)
elif fmt == "yaml":
log.info("Skipping YAML")
else:
log.info("Ignoring invalid post")
elif fn.endswith("/index.json.gz"):
pass
else:
raise RuntimeError(f"Unexpected [mini]can filename '{fn}'")
def load_multiple(fn: str) -> Generator[MsmtTup, None, None]:
"""Load contents of cans. Decompress tar archives if found.
Yields measurements one by one as:
(string of JSON, None, None) or (None, msmt dict, None)
"""
# TODO: handle:
# RuntimeError: LZ4F_decompress failed with code: ERROR_decompressionFailed
if fn.endswith(".tar.lz4"):
with lz4frame.open(fn) as f:
tf = tarfile.TarFile(fileobj=f)
while True:
m = tf.next()
if m is None:
# end of tarball
break
log.debug("Loading nested %s", m.name)
k = tf.extractfile(m)
assert k is not None
if m.name.endswith(".json"):
for line in k:
yield (line, None, None)
elif m.name.endswith(".yaml"):
bucket_tstamp = fn.split("/")[-2]
rfn = f"{bucket_tstamp}/" + fn.split("/")[-1]
for msm in iter_yaml_msmt_normalized(
k, bucket_tstamp, rfn):
metrics.incr("yaml_normalization")
yield (None, msm, None)
elif fn.endswith(".json.lz4"):
with lz4frame.open(fn) as f:
for line in f:
yield (line, None, None)
elif fn.endswith(".yaml.lz4"):
with lz4frame.open(fn) as f:
bucket_tstamp = fn.split("/")[-2]
rfn = f"{bucket_tstamp}/" + fn.split("/")[-1]
for msm in iter_yaml_msmt_normalized(f, bucket_tstamp, rfn):
metrics.incr("yaml_normalization")
yield (None, msm, None)
elif fn.endswith(".tar.gz"):
# minican with missing gzipping :(
tf = tarfile.open(fn)
while True:
m = tf.next()
if m is None:
# end of tarball
tf.close()
break
log.debug("Loading %s", m.name)
k = tf.extractfile(m)
assert k is not None
if not m.name.endswith(".post"):
log.error("Unexpected filename")
continue
try:
j = ujson.loads(k.read())
except Exception:
log.error(repr(k[:100]), exc_info=1)
fmt = j.get("format", "")
if fmt == "json":
msm = j.get("content", {})
yield (None, msm, None)
elif fmt == "yaml":
log.info("Skipping YAML")
else:
log.info("Ignoring invalid post")
elif fn.endswith("/index.json.gz"):
pass
else:
raise RuntimeError(f"Unexpected [mini]can filename '{fn}'")