def update_tor_bridge_lines(self, country_code):
from ooni.utils import onion
in_file = self.resources.child("tor-bridges").child(
"tor-bridges-ip-port.csv")
if not in_file.exists():
yield check_for_update(country_code)
data_fname = "tor-bridge-lines.txt"
desc_fname = "tor-bridge-lines.desc"
out_file = self.path.child("data").child(data_fname)
def format_row(row):
host, port, nickname, protocol = row
if protocol.lower() not in onion.pt_names:
return "{}:{}\n".format(host, port)
return "{} {}:{}\n".format(protocol, host, port)
write_txt_from_csv(in_file, out_file, format_row)
desc_file = self.path.child("descriptors").child(desc_fname)
write_descriptor(desc_file, "Tor bridge lines", "tor_bridge_lines",
out_file.path, "file/ip-port")
self._cache_stale = True
# Do an empty defer to fit inside of a event loop clock
yield defer.succeed(None)
def update_tor_bridge_lines(self, country_code):
from ooni.utils import onion
in_file = self.resources.child("tor-bridges").child(
"tor-bridges-ip-port.csv"
)
if not in_file.exists():
yield check_for_update(country_code)
data_fname = "tor-bridge-lines.txt"
desc_fname = "tor-bridge-lines.desc"
out_file = self.path.child("data").child(data_fname)
def format_row(row):
host, port, nickname, protocol = row
if protocol.lower() not in onion.pt_names:
return "{}:{}\n".format(host, port)
return "{} {}:{}\n".format(protocol, host, port)
write_txt_from_csv(in_file, out_file, format_row)
desc_file = self.path.child("descriptors").child(desc_fname)
write_descriptor(
desc_file, "Tor bridge lines",
"tor_bridge_lines", out_file.path,
"file/ip-port"
)
self._cache_stale = True
# Do an empty defer to fit inside of a event loop clock
yield defer.succeed(None)
def task(self):
log.debug("Updating the inputs")
yield probe_ip.lookup()
log.debug("Updating the inputs for country %s" %
probe_ip.geodata['countrycode'])
yield resources.check_for_update(probe_ip.geodata['countrycode'])
yield input_store.update(probe_ip.geodata['countrycode'])
yield probe_ip.resolveGeodata()
def task(self):
log.debug("Updating the inputs")
yield probe_ip.lookup()
log.debug("Updating the inputs for country %s" %
probe_ip.geodata['countrycode'])
yield resources.check_for_update(probe_ip.geodata['countrycode'])
yield input_store.update(probe_ip.geodata['countrycode'])
yield probe_ip.resolveGeodata()
def update_url_lists(self, country_code):
countries = ["global"]
if country_code != "ZZ":
countries.append(country_code)
for cc in countries:
cc = cc.lower()
in_file = self.resources.child("citizenlab-test-lists").child("{0}.csv".format(cc))
if not in_file.exists():
yield check_for_update(country_code)
if not in_file.exists():
log.msg("Could not find input for country "
"{0} in {1}".format(cc, in_file.path))
continue
# XXX maybe move this to some utility function.
# It's duplicated in oonideckgen.
data_fname = "citizenlab-test-lists_{0}.txt".format(cc)
desc_fname = "citizenlab-test-lists_{0}.desc".format(cc)
out_file = self.path.child("data").child(data_fname)
write_txt_from_csv(in_file, out_file,
lambda row: "{}\n".format(row[0])
)
desc_file = self.path.child("descriptors").child(desc_fname)
if cc == "global":
name = "List of globally accessed websites"
else:
# XXX resolve this to a human readable country name
country_name = cc
name = "List of websites for {0}".format(country_name)
write_descriptor(desc_file, name,
"citizenlab_{0}_urls".format(cc),
out_file.path,
"file/url")
self._cache_stale = True
yield defer.succeed(None)
def update_url_lists(self, country_code):
countries = ["global"]
if country_code != "ZZ":
countries.append(country_code)
for cc in countries:
cc = cc.lower()
in_file = self.resources.child("citizenlab-test-lists").child("{0}.csv".format(cc))
if not in_file.exists():
yield check_for_update(country_code)
if not in_file.exists():
log.msg("Could not find input for country "
"{0} in {1}".format(cc, in_file.path))
continue
# XXX maybe move this to some utility function.
# It's duplicated in oonideckgen.
data_fname = "citizenlab-test-lists_{0}.txt".format(cc)
desc_fname = "citizenlab-test-lists_{0}.desc".format(cc)
out_file = self.path.child("data").child(data_fname)
write_txt_from_csv(in_file, out_file,
lambda row: "{}\n".format(row[0])
)
desc_file = self.path.child("descriptors").child(desc_fname)
if cc == "global":
name = "List of globally accessed websites"
else:
# XXX resolve this to a human readable country name
country_name = cc
name = "List of websites for {0}".format(country_name)
write_descriptor(desc_file, name,
"citizenlab_{0}_urls".format(cc),
out_file.path,
"file/url")
self._cache_stale = True
yield defer.succeed(None)
def oonideckgen(reactor):
options = Options()
try:
options.parseOptions()
except usage.UsageError as error_message:
print("%s: %s" % (sys.argv[0], error_message))
print(options)
sys.exit(1)
print("Checking for update of resources")
yield check_for_update()
if not options['output']:
options['output'] = os.getcwd()
if not options['country-code']:
try:
options['country-code'] = yield get_user_country_code()
except errors.ProbeIPUnknown:
print("Could not determine your IP address.")
print("Check your internet connection or specify a country code "
"with -c.")
sys.exit(4)
if len(options['country-code']) != 2:
print("%s: --country-code must be 2 characters" % sys.argv[0])
sys.exit(2)
if os.path.isdir(options['output']):
options['output'] = os.path.join(options['output'], 'web-full.yaml')
options['country-code'] = options['country-code'].lower()
mkdir_p(os.path.dirname(options['output']))
generate_deck(options)
def oonideckgen(reactor):
options = Options()
try:
options.parseOptions()
except usage.UsageError as error_message:
print("%s: %s" % (sys.argv[0], error_message))
print(options)
sys.exit(1)
print("Checking for update of resources")
yield check_for_update()
if not options['output']:
options['output'] = os.getcwd()
if not options['country-code']:
try:
options['country-code'] = yield get_user_country_code()
except errors.ProbeIPUnknown:
print("Could not determine your IP address.")
print("Check your internet connection or specify a country code "
"with -c.")
sys.exit(4)
if len(options['country-code']) != 2:
print("%s: --country-code must be 2 characters" % sys.argv[0])
sys.exit(2)
if os.path.isdir(options['output']):
options['output'] = os.path.join(options['output'], 'web-full.yaml')
options['country-code'] = options['country-code'].lower()
mkdir_p(os.path.dirname(options['output']))
generate_deck(options)
def first_run(self):
"""
On first run we update the resources that are common to every country.
"""
log.debug("Updating the global inputs and resources")
yield resources.check_for_update("ZZ")
def first_run(self):
"""
On first run we update the resources that are common to every country.
"""
log.debug("Updating the global inputs and resources")
yield resources.check_for_update("ZZ")