def IPToLocation(ipaddr):
from ooni.settings import config
city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat')
country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
def error():
log.err("Could not find GeoIP data file in %s."
"Try running ooniresources --update-geoip or"
" edit your ooniprobe.conf" % config.advanced.geoip_data_dir)
try:
country_dat = GeoIP(country_file)
location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
if not location['countrycode']:
location['countrycode'] = 'ZZ'
except IOError:
error()
try:
city_dat = GeoIP(city_file)
location['city'] = city_dat.record_by_addr(ipaddr)['city']
except:
error()
try:
asn_dat = GeoIP(asn_file)
location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
except:
error()
return location
def ip_to_location(ipaddr):
from ooni.settings import config
country_file = config.get_data_file_path(
'resources/maxmind-geoip/GeoIP.dat')
asn_file = config.get_data_file_path(
'resources/maxmind-geoip/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
if not asn_file or not country_file:
log.err("Could not find GeoIP data file in data directories."
"Try running ooniresources or"
" edit your ooniprobe.conf")
return location
country_dat = GeoIP(country_file)
asn_dat = GeoIP(asn_file)
country_code = country_dat.country_code_by_addr(ipaddr)
if country_code is not None:
location['countrycode'] = country_code
asn = asn_dat.org_by_addr(ipaddr)
if asn is not None:
location['asn'] = asn.split(' ')[0]
return location
def IPToLocation(ipaddr):
from ooni.settings import config
country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
if not asn_file or not country_file:
log.err("Could not find GeoIP data file in data directories."
"Try running ooniresources or"
" edit your ooniprobe.conf")
return location
country_dat = GeoIP(country_file)
asn_dat = GeoIP(asn_file)
country_code = country_dat.country_code_by_addr(ipaddr)
if country_code is not None:
location['countrycode'] = country_code
asn = asn_dat.org_by_addr(ipaddr)
if asn is not None:
location['asn'] = asn.split(' ')[0]
return location
def IPToLocation(ipaddr):
from ooni.settings import config
country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
def error():
log.err("Could not find GeoIP data file in data directories."
"Try running ooniresources or"
" edit your ooniprobe.conf")
try:
country_dat = GeoIP(country_file)
location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
if not location['countrycode']:
location['countrycode'] = 'ZZ'
except IOError:
error()
try:
asn_dat = GeoIP(asn_file)
location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
except:
error()
return location
def resources_up_to_date():
if config.get_data_file_path("GeoIP/GeoIP.dat") is None:
return False
if config.get_data_file_path("resources/"
"namebench-dns-servers.csv") is None:
return False
if config.get_data_file_path("resources/"
"citizenlab-test-lists/"
"global.csv") is None:
return False
return True
def resources_up_to_date():
if config.get_data_file_path("GeoIP/GeoIP.dat") is None:
return False
if config.get_data_file_path("resources/"
"namebench-dns-servers.csv") is None:
return False
if config.get_data_file_path("resources/"
"citizenlab-test-lists/"
"global.csv") is None:
return False
return True
def database_version():
from ooni.settings import config
version = {
'GeoIP': {
'sha256': None,
'timestamp': None,
},
'GeoIPASNum': {
'sha256': None,
'timestamp': None
},
'GeoLiteCity': {
'sha256': None,
'timestamp': None
}
}
for key in version.keys():
geoip_file = config.get_data_file_path("GeoIP/" + key + ".dat")
if not geoip_file or not os.path.isfile(geoip_file):
continue
timestamp = os.stat(geoip_file).st_mtime
sha256hash = sha256()
with open(geoip_file) as f:
while True:
chunk = f.read(8192)
if not chunk:
break
sha256hash.update(chunk)
version[key]['timestamp'] = timestamp
version[key]['sha256'] = sha256hash.hexdigest()
return version
def database_version():
from ooni.settings import config
version = {
'GeoIP': {
'sha256': None,
'timestamp': None,
},
'GeoIPASNum': {
'sha256': None,
'timestamp': None
}
}
for key in version.keys():
geoip_file = config.get_data_file_path("GeoIP/" + key + ".dat")
if not geoip_file or not os.path.isfile(geoip_file):
continue
timestamp = os.stat(geoip_file).st_mtime
sha256hash = sha256()
with open(geoip_file) as f:
while True:
chunk = f.read(8192)
if not chunk:
break
sha256hash.update(chunk)
version[key]['timestamp'] = timestamp
version[key]['sha256'] = sha256hash.hexdigest()
return version
def __init__(self):
self.__dict__ = self._borg
if not self.country:
try:
country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
self.country = GeoIP.open(country_file, GeoIP.GEOIP_STANDARD)
except:
raise Exception("Edit the geoip_data_dir line in your config"
" file to point to your geoip files")
def __init__(self):
self.__dict__ = self._borg
if not self.country:
try:
country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
self.country = GeoIP.open(country_file,
GeoIP.GEOIP_STANDARD)
except:
raise Exception("Edit the geoip_data_dir line in your config"
" file to point to your geoip files")
def generate_global_input(dst):
filename = os.path.join(dst, "citizenlab-urls-global.txt")
input_list = config.get_data_file_path("resources/"
"citizenlab-test-lists/"
"test-lists-master/lists/"
"global.csv")
load_input(input_list, filename)
return filename
def generate_global_input(dst):
filename = os.path.join(dst, "citizenlab-urls-global.txt")
input_list = config.get_data_file_path("resources/"
"citizenlab-test-lists/"
"test-lists-master/lists/"
"global.csv")
load_input(input_list, filename)
return filename
def generate_global_input(dst):
filename = os.path.join(dst, "citizenlab-urls-global.txt")
input_list = config.get_data_file_path("resources/"
"citizenlab-test-lists/"
"global.csv")
if not input_list:
print("Could not find the global input list")
print("Perhaps you should run ooniresources")
raise Exception("Could not find the global input list")
load_input(input_list, filename)
return filename
def generate_global_input(dst):
filename = os.path.join(dst, "citizenlab-urls-global.txt")
input_list = config.get_data_file_path("resources/"
"citizenlab-test-lists/"
"global.csv")
if not input_list:
print("Could not find the global input list")
print("Perhaps you should run ooniresources")
raise Exception("Could not find the global input list")
load_input(input_list, filename)
return filename
def generate_country_input(country_code, dst):
"""
Write to dst/citizenlab-urls-{country_code}.txt
the list for the given country code.
Returns:
the path to the generated input
"""
country_code = country_code.lower()
filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
input_list = config.get_data_file_path("resources/"
"citizenlab-test-lists/" +
country_code + ".csv")
if not input_list:
raise Exception("Could not find list for country %s" % country_code)
load_input(input_list, filename)
return filename
def generate_country_input(country_code, dst):
"""
Write to dst/citizenlab-urls-{country_code}.txt
the list for the given country code.
Returns:
the path to the generated input
"""
country_code = country_code.lower()
filename = os.path.join(dst, "citizenlab-urls-%s.txt" % country_code)
input_list = config.get_data_file_path("resources/"
"citizenlab-test-lists/"
+ country_code + ".csv")
if not input_list:
raise Exception("Could not find list for country %s" % country_code)
load_input(input_list, filename)
return filename
def generate_country_input(country_code, dst):
csv_file = config.get_data_file_path("resources/"
"namebench-dns-servers.csv")
filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
fw = open(filename, "w")
geoip_db = GeoIPDB()
reader = csv.reader(open(csv_file))
for row in reader:
if row[2] == 'X-Internal-IP':
continue
elif row[2] == 'X-Unroutable':
continue
elif row[2] == 'X-Link_local':
continue
ipaddr = row[0]
cc = geoip_db.country.country_code_by_addr(ipaddr)
if not cc:
continue
if cc.lower() == country_code.lower():
fw.write(ipaddr + "\n")
fw.close()
return filename
def generate_country_input(country_code, dst):
csv_file = config.get_data_file_path("resources/"
"namebench-dns-servers.csv")
filename = os.path.join(dst, "dns-server-%s.txt" % country_code)
fw = open(filename, "w")
geoip_db = GeoIPDB()
reader = csv.reader(open(csv_file))
for row in reader:
if row[2] == 'X-Internal-IP':
continue
elif row[2] == 'X-Unroutable':
continue
elif row[2] == 'X-Link_local':
continue
ipaddr = row[0]
cc = geoip_db.country.country_code_by_addr(ipaddr)
if not cc:
continue
if cc.lower() == country_code.lower():
fw.write(ipaddr + "\n")
fw.close()
return filename