Ejemplo n.º 1
0
def IPToLocation(ipaddr):
    from ooni.settings import config

    city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat')
    country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
    asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')

    location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}

    def error():
        log.err("Could not find GeoIP data file in %s."
                "Try running ooniresources --update-geoip or"
                " edit your ooniprobe.conf" % config.advanced.geoip_data_dir)

    try:
        country_dat = GeoIP(country_file)
        location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
        if not location['countrycode']:
            location['countrycode'] = 'ZZ'
    except IOError:
        error()

    try:
        city_dat = GeoIP(city_file)
        location['city'] = city_dat.record_by_addr(ipaddr)['city']
    except:
        error()

    try:
        asn_dat = GeoIP(asn_file)
        location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
    except:
        error()

    return location
Ejemplo n.º 2
0
def ip_to_location(ipaddr):
    from ooni.settings import config

    country_file = config.get_data_file_path(
        'resources/maxmind-geoip/GeoIP.dat')
    asn_file = config.get_data_file_path(
        'resources/maxmind-geoip/GeoIPASNum.dat')

    location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
    if not asn_file or not country_file:
        log.err("Could not find GeoIP data file in data directories."
                "Try running ooniresources or"
                " edit your ooniprobe.conf")
        return location

    country_dat = GeoIP(country_file)
    asn_dat = GeoIP(asn_file)

    country_code = country_dat.country_code_by_addr(ipaddr)
    if country_code is not None:
        location['countrycode'] = country_code

    asn = asn_dat.org_by_addr(ipaddr)
    if asn is not None:
        location['asn'] = asn.split(' ')[0]

    return location
Ejemplo n.º 3
0
def IPToLocation(ipaddr):
    from ooni.settings import config

    country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
    asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')

    location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
    if not asn_file or not country_file:
        log.err("Could not find GeoIP data file in data directories."
                "Try running ooniresources or"
                " edit your ooniprobe.conf")
        return location

    country_dat = GeoIP(country_file)
    asn_dat = GeoIP(asn_file)

    country_code = country_dat.country_code_by_addr(ipaddr)
    if country_code is not None:
        location['countrycode'] =  country_code

    asn = asn_dat.org_by_addr(ipaddr)
    if asn is not None:
        location['asn'] = asn.split(' ')[0]

    return location
Ejemplo n.º 4
0
def IPToLocation(ipaddr):
    from ooni.settings import config

    country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
    asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')

    location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}

    def error():
        log.err("Could not find GeoIP data file in data directories."
                "Try running ooniresources or"
                " edit your ooniprobe.conf")

    try:
        country_dat = GeoIP(country_file)
        location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
        if not location['countrycode']:
            location['countrycode'] = 'ZZ'
    except IOError:
        error()

    try:
        asn_dat = GeoIP(asn_file)
        location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
    except:
        error()

    return location
Ejemplo n.º 5
0
def resources_up_to_date():
    if config.get_data_file_path("GeoIP/GeoIP.dat") is None:
        return False

    if config.get_data_file_path("resources/"
                                 "namebench-dns-servers.csv") is None:
        return False

    if config.get_data_file_path("resources/"
                                 "citizenlab-test-lists/"
                                 "global.csv") is None:
        return False

    return True
Ejemplo n.º 6
0
def resources_up_to_date():
    if config.get_data_file_path("GeoIP/GeoIP.dat") is None:
        return False

    if config.get_data_file_path("resources/"
                                 "namebench-dns-servers.csv") is None:
        return False

    if config.get_data_file_path("resources/"
                                 "citizenlab-test-lists/"
                                 "global.csv") is None:
        return False

    return True
Ejemplo n.º 7
0
def database_version():
    from ooni.settings import config

    version = {
        'GeoIP': {
            'sha256': None,
            'timestamp': None,
        },
        'GeoIPASNum': {
            'sha256': None,
            'timestamp': None
        },
        'GeoLiteCity': {
            'sha256': None,
            'timestamp': None
        }
    }

    for key in version.keys():
        geoip_file = config.get_data_file_path("GeoIP/" + key + ".dat")
        if not geoip_file or not os.path.isfile(geoip_file):
            continue
        timestamp = os.stat(geoip_file).st_mtime

        sha256hash = sha256()
        with open(geoip_file) as f:
            while True:
                chunk = f.read(8192)
                if not chunk:
                    break
                sha256hash.update(chunk)

        version[key]['timestamp'] = timestamp
        version[key]['sha256'] = sha256hash.hexdigest()
    return version
Ejemplo n.º 8
0
def database_version():
    from ooni.settings import config

    version = {
        'GeoIP': {
            'sha256': None,
            'timestamp': None,
        },
        'GeoIPASNum': {
            'sha256': None,
            'timestamp': None
        }
    }

    for key in version.keys():
        geoip_file = config.get_data_file_path("GeoIP/" + key + ".dat")
        if not geoip_file or not os.path.isfile(geoip_file):
            continue
        timestamp = os.stat(geoip_file).st_mtime

        sha256hash = sha256()
        with open(geoip_file) as f:
            while True:
                chunk = f.read(8192)
                if not chunk:
                    break
                sha256hash.update(chunk)

        version[key]['timestamp'] = timestamp
        version[key]['sha256'] = sha256hash.hexdigest()
    return version
Ejemplo n.º 9
0
 def __init__(self):
     self.__dict__ = self._borg
     if not self.country:
         try:
             country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
             self.country = GeoIP.open(country_file, GeoIP.GEOIP_STANDARD)
         except:
             raise Exception("Edit the geoip_data_dir line in your config"
                             " file to point to your geoip files")
Ejemplo n.º 10
0
 def __init__(self):
     self.__dict__ = self._borg
     if not self.country:
         try:
             country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
             self.country = GeoIP.open(country_file,
                                       GeoIP.GEOIP_STANDARD)
         except:
             raise Exception("Edit the geoip_data_dir line in your config"
                             " file to point to your geoip files")
Ejemplo n.º 11
0
def generate_global_input(dst):
    filename = os.path.join(dst, "citizenlab-urls-global.txt")

    input_list = config.get_data_file_path("resources/"
                                           "citizenlab-test-lists/"
                                           "test-lists-master/lists/"
                                           "global.csv")

    load_input(input_list, filename)

    return filename
Ejemplo n.º 12
0
def generate_global_input(dst):
    filename = os.path.join(dst, "citizenlab-urls-global.txt")

    input_list = config.get_data_file_path("resources/"
                                           "citizenlab-test-lists/"
                                           "test-lists-master/lists/"
                                           "global.csv")

    load_input(input_list, filename)

    return filename
Ejemplo n.º 13
0
def generate_global_input(dst):
    filename = os.path.join(dst, "citizenlab-urls-global.txt")

    input_list = config.get_data_file_path("resources/"
                                           "citizenlab-test-lists/"
                                           "global.csv")

    if not input_list:
        print("Could not find the global input list")
        print("Perhaps you should run ooniresources")
        raise Exception("Could not find the global input list")

    load_input(input_list, filename)

    return filename
Ejemplo n.º 14
0
def generate_global_input(dst):
    filename = os.path.join(dst, "citizenlab-urls-global.txt")

    input_list = config.get_data_file_path("resources/"
                                           "citizenlab-test-lists/"
                                           "global.csv")

    if not input_list:
        print("Could not find the global input list")
        print("Perhaps you should run ooniresources")
        raise Exception("Could not find the global input list")

    load_input(input_list, filename)

    return filename
Ejemplo n.º 15
0
def generate_country_input(country_code, dst):
    """
    Write to dst/citizenlab-urls-{country_code}.txt
    the list for the given country code.

    Returns:

        the path to the generated input
    """

    country_code = country_code.lower()
    filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)

    input_list = config.get_data_file_path("resources/"
                                           "citizenlab-test-lists/" +
                                           country_code + ".csv")

    if not input_list:
        raise Exception("Could not find list for country %s" % country_code)

    load_input(input_list, filename)

    return filename
Ejemplo n.º 16
0
def generate_country_input(country_code, dst):
    """
    Write to dst/citizenlab-urls-{country_code}.txt
    the list for the given country code.

    Returns:

        the path to the generated input
    """

    country_code = country_code.lower()
    filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)

    input_list = config.get_data_file_path("resources/"
                                           "citizenlab-test-lists/"
                                           + country_code + ".csv")

    if not input_list:
        raise Exception("Could not find list for country %s" % country_code)

    load_input(input_list, filename)

    return filename
Ejemplo n.º 17
0
def generate_country_input(country_code, dst):

    csv_file = config.get_data_file_path("resources/"
                                           "namebench-dns-servers.csv")

    filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
    fw = open(filename, "w")
    geoip_db = GeoIPDB()
    reader = csv.reader(open(csv_file))
    for row in reader:
        if row[2] == 'X-Internal-IP':
            continue
        elif row[2] == 'X-Unroutable':
            continue
        elif row[2] == 'X-Link_local':
            continue
        ipaddr = row[0]
        cc = geoip_db.country.country_code_by_addr(ipaddr)
        if not cc:
            continue
        if cc.lower() == country_code.lower():
            fw.write(ipaddr + "\n")
    fw.close()
    return filename
Ejemplo n.º 18
0
def generate_country_input(country_code, dst):

    csv_file = config.get_data_file_path("resources/"
                                         "namebench-dns-servers.csv")

    filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
    fw = open(filename, "w")
    geoip_db = GeoIPDB()
    reader = csv.reader(open(csv_file))
    for row in reader:
        if row[2] == 'X-Internal-IP':
            continue
        elif row[2] == 'X-Unroutable':
            continue
        elif row[2] == 'X-Link_local':
            continue
        ipaddr = row[0]
        cc = geoip_db.country.country_code_by_addr(ipaddr)
        if not cc:
            continue
        if cc.lower() == country_code.lower():
            fw.write(ipaddr + "\n")
    fw.close()
    return filename