def get_control_dns_measurement(measurement, measurement_id): measurement_time = parse_ooni_date( measurement.get("measurement_start_time")).isoformat() try: addresses = measurement["test_keys"]["control"]["dns"]["addrs"] except KeyError: raise ValueError( "OONI Control Measurement without test_keys.control.dns.addrs: %s" % pprint.pformat(measurement, compact=True)) if not addresses: raise ValueError( "OONI Control Measurement with empty test_keys.control.dns.addrs: %s" % pprint.pformat(measurement, compact=True)) records = [] # type: List[dns.ResourceRecord] last_cname = urlparse(measurement.get("input")).hostname for address in addresses: try: records.append( dns.ResourceRecord(last_cname, dns.IpAddressData(address))) except ValueError: records.append( dns.ResourceRecord(last_cname, dns.CnameData(address))) last_cname = address measurement_time = parse_ooni_date( measurement.get("measurement_start_time")).isoformat() return dns.DnsMeasurement(measurement_id="%s:control" % measurement_id, records=records, time=measurement_time, provenance="ooni:%s" % measurement_id, trust_reason="IN_OONI_CONTROL")
def measurement_from_json(measurement_json: Dict) -> model.DnsMeasurement: params = {} for key, value in measurement_json.items(): if key == "time": value = datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S") elif key == "records": value = [record_from_json(r) for r in value] if value is not None: params[key] = value return model.DnsMeasurement(**params)
def get_experiment_dns_measurement(measurement, measurement_id) -> dns.DnsMeasurement: measurement_time = parse_ooni_date( measurement.get("measurement_start_time")).isoformat() try: ooni_queries = measurement["test_keys"]["queries"] except KeyError: raise ValueError("OONI Measurement without test_keys.queries: %s" % pprint.pformat(measurement, compact=True)) if not ooni_queries: raise ValueError("OONI Measurement with empty test_keys.queries: %s" % pprint.pformat(measurement, compact=True)) records = [] # type: List[dns.ResourceRecord] for ooni_query in ooni_queries: last_cname = ooni_query.get("hostname") if not last_cname: logging.warning("Missing hostname in query %s", ooni_query) for ooni_answer in ooni_query.get("answers"): cname = ooni_answer.get("hostname") if cname: if cname == last_cname: continue records.append( dns.ResourceRecord(last_cname, dns.CnameData(cname))) last_cname = cname else: ip_str = ooni_answer.get("ipv4") or ooni_answer.get("ipv6") if ip_str: try: records.append( dns.ResourceRecord(last_cname, dns.IpAddressData(ip_str))) except ValueError: logging.warning("Measurement %s: invalid IP answer %s", measurement["id"], ip_str) measurement_time = parse_ooni_date( measurement.get("measurement_start_time")).isoformat() resolver_ip_str = measurement["test_keys"].get("client_resolver") resolver_ip = ipaddress.ip_address( resolver_ip_str) if resolver_ip_str else None return dns.DnsMeasurement( measurement_id=measurement_id, records=records, time=measurement_time, resolver_ip=resolver_ip, client_asn=int(measurement.get("probe_asn")[2:]), client_country=measurement.get("probe_cc"), provenance="ooni:%s" % measurement_id, )