def update_tor_bridge_lines(self, country_code): from ooni.utils import onion in_file = self.resources.child("tor-bridges").child( "tor-bridges-ip-port.csv") if not in_file.exists(): yield check_for_update(country_code) data_fname = "tor-bridge-lines.txt" desc_fname = "tor-bridge-lines.desc" out_file = self.path.child("data").child(data_fname) def format_row(row): host, port, nickname, protocol = row if protocol.lower() not in onion.pt_names: return "{}:{}\n".format(host, port) return "{} {}:{}\n".format(protocol, host, port) write_txt_from_csv(in_file, out_file, format_row) desc_file = self.path.child("descriptors").child(desc_fname) write_descriptor(desc_file, "Tor bridge lines", "tor_bridge_lines", out_file.path, "file/ip-port") self._cache_stale = True # Do an empty defer to fit inside of a event loop clock yield defer.succeed(None)
def update_tor_bridge_lines(self, country_code): from ooni.utils import onion in_file = self.resources.child("tor-bridges").child( "tor-bridges-ip-port.csv" ) if not in_file.exists(): yield check_for_update(country_code) data_fname = "tor-bridge-lines.txt" desc_fname = "tor-bridge-lines.desc" out_file = self.path.child("data").child(data_fname) def format_row(row): host, port, nickname, protocol = row if protocol.lower() not in onion.pt_names: return "{}:{}\n".format(host, port) return "{} {}:{}\n".format(protocol, host, port) write_txt_from_csv(in_file, out_file, format_row) desc_file = self.path.child("descriptors").child(desc_fname) write_descriptor( desc_file, "Tor bridge lines", "tor_bridge_lines", out_file.path, "file/ip-port" ) self._cache_stale = True # Do an empty defer to fit inside of a event loop clock yield defer.succeed(None)
def task(self): log.debug("Updating the inputs") yield probe_ip.lookup() log.debug("Updating the inputs for country %s" % probe_ip.geodata['countrycode']) yield resources.check_for_update(probe_ip.geodata['countrycode']) yield input_store.update(probe_ip.geodata['countrycode']) yield probe_ip.resolveGeodata()
def update_url_lists(self, country_code): countries = ["global"] if country_code != "ZZ": countries.append(country_code) for cc in countries: cc = cc.lower() in_file = self.resources.child("citizenlab-test-lists").child("{0}.csv".format(cc)) if not in_file.exists(): yield check_for_update(country_code) if not in_file.exists(): log.msg("Could not find input for country " "{0} in {1}".format(cc, in_file.path)) continue # XXX maybe move this to some utility function. # It's duplicated in oonideckgen. data_fname = "citizenlab-test-lists_{0}.txt".format(cc) desc_fname = "citizenlab-test-lists_{0}.desc".format(cc) out_file = self.path.child("data").child(data_fname) write_txt_from_csv(in_file, out_file, lambda row: "{}\n".format(row[0]) ) desc_file = self.path.child("descriptors").child(desc_fname) if cc == "global": name = "List of globally accessed websites" else: # XXX resolve this to a human readable country name country_name = cc name = "List of websites for {0}".format(country_name) write_descriptor(desc_file, name, "citizenlab_{0}_urls".format(cc), out_file.path, "file/url") self._cache_stale = True yield defer.succeed(None)
def oonideckgen(reactor): options = Options() try: options.parseOptions() except usage.UsageError as error_message: print("%s: %s" % (sys.argv[0], error_message)) print(options) sys.exit(1) print("Checking for update of resources") yield check_for_update() if not options['output']: options['output'] = os.getcwd() if not options['country-code']: try: options['country-code'] = yield get_user_country_code() except errors.ProbeIPUnknown: print("Could not determine your IP address.") print("Check your internet connection or specify a country code " "with -c.") sys.exit(4) if len(options['country-code']) != 2: print("%s: --country-code must be 2 characters" % sys.argv[0]) sys.exit(2) if os.path.isdir(options['output']): options['output'] = os.path.join(options['output'], 'web-full.yaml') options['country-code'] = options['country-code'].lower() mkdir_p(os.path.dirname(options['output'])) generate_deck(options)
def first_run(self): """ On first run we update the resources that are common to every country. """ log.debug("Updating the global inputs and resources") yield resources.check_for_update("ZZ")