def get_control_dns_measurement(measurement, measurement_id):
measurement_time = parse_ooni_date(
measurement.get("measurement_start_time")).isoformat()
try:
addresses = measurement["test_keys"]["control"]["dns"]["addrs"]
except KeyError:
raise ValueError(
"OONI Control Measurement without test_keys.control.dns.addrs: %s"
% pprint.pformat(measurement, compact=True))
if not addresses:
raise ValueError(
"OONI Control Measurement with empty test_keys.control.dns.addrs: %s"
% pprint.pformat(measurement, compact=True))
records = [] # type: List[dns.ResourceRecord]
last_cname = urlparse(measurement.get("input")).hostname
for address in addresses:
try:
records.append(
dns.ResourceRecord(last_cname, dns.IpAddressData(address)))
except ValueError:
records.append(
dns.ResourceRecord(last_cname, dns.CnameData(address)))
last_cname = address
measurement_time = parse_ooni_date(
measurement.get("measurement_start_time")).isoformat()
return dns.DnsMeasurement(measurement_id="%s:control" % measurement_id,
records=records,
time=measurement_time,
provenance="ooni:%s" % measurement_id,
trust_reason="IN_OONI_CONTROL")
def record_data_from_json(data_json: Dict) -> model.RecordData:
if "ip" in data_json:
return model.IpAddressData(data_json["ip"])
elif "cname" in data_json:
return model.CnameData(data_json["cname"])
else:
raise ValueError("Invalid RecordData json: %s" % json.dumps(data_json))
def get_experiment_dns_measurement(measurement,
measurement_id) -> dns.DnsMeasurement:
measurement_time = parse_ooni_date(
measurement.get("measurement_start_time")).isoformat()
try:
ooni_queries = measurement["test_keys"]["queries"]
except KeyError:
raise ValueError("OONI Measurement without test_keys.queries: %s" %
pprint.pformat(measurement, compact=True))
if not ooni_queries:
raise ValueError("OONI Measurement with empty test_keys.queries: %s" %
pprint.pformat(measurement, compact=True))
records = [] # type: List[dns.ResourceRecord]
for ooni_query in ooni_queries:
last_cname = ooni_query.get("hostname")
if not last_cname:
logging.warning("Missing hostname in query %s", ooni_query)
for ooni_answer in ooni_query.get("answers"):
cname = ooni_answer.get("hostname")
if cname:
if cname == last_cname:
continue
records.append(
dns.ResourceRecord(last_cname, dns.CnameData(cname)))
last_cname = cname
else:
ip_str = ooni_answer.get("ipv4") or ooni_answer.get("ipv6")
if ip_str:
try:
records.append(
dns.ResourceRecord(last_cname,
dns.IpAddressData(ip_str)))
except ValueError:
logging.warning("Measurement %s: invalid IP answer %s",
measurement["id"], ip_str)
measurement_time = parse_ooni_date(
measurement.get("measurement_start_time")).isoformat()
resolver_ip_str = measurement["test_keys"].get("client_resolver")
resolver_ip = ipaddress.ip_address(
resolver_ip_str) if resolver_ip_str else None
return dns.DnsMeasurement(
measurement_id=measurement_id,
records=records,
time=measurement_time,
resolver_ip=resolver_ip,
client_asn=int(measurement.get("probe_asn")[2:]),
client_country=measurement.get("probe_cc"),
provenance="ooni:%s" % measurement_id,
)