Beispiel #1
0
    def update_tor_bridge_lines(self, country_code):
        from ooni.utils import onion
        in_file = self.resources.child("tor-bridges").child(
            "tor-bridges-ip-port.csv")
        if not in_file.exists():
            yield check_for_update(country_code)

        data_fname = "tor-bridge-lines.txt"
        desc_fname = "tor-bridge-lines.desc"
        out_file = self.path.child("data").child(data_fname)

        def format_row(row):
            host, port, nickname, protocol = row
            if protocol.lower() not in onion.pt_names:
                return "{}:{}\n".format(host, port)
            return "{} {}:{}\n".format(protocol, host, port)

        write_txt_from_csv(in_file, out_file, format_row)
        desc_file = self.path.child("descriptors").child(desc_fname)
        write_descriptor(desc_file, "Tor bridge lines", "tor_bridge_lines",
                         out_file.path, "file/ip-port")
        self._cache_stale = True

        # Do an empty defer to fit inside of a event loop clock
        yield defer.succeed(None)
Beispiel #2
0
    def update_tor_bridge_lines(self, country_code):
        from ooni.utils import onion
        in_file = self.resources.child("tor-bridges").child(
            "tor-bridges-ip-port.csv"
        )
        if not in_file.exists():
            yield check_for_update(country_code)

        data_fname = "tor-bridge-lines.txt"
        desc_fname = "tor-bridge-lines.desc"
        out_file = self.path.child("data").child(data_fname)

        def format_row(row):
            host, port, nickname, protocol = row
            if protocol.lower() not in onion.pt_names:
                return "{}:{}\n".format(host, port)
            return "{} {}:{}\n".format(protocol, host, port)

        write_txt_from_csv(in_file, out_file, format_row)
        desc_file = self.path.child("descriptors").child(desc_fname)
        write_descriptor(
            desc_file, "Tor bridge lines",
            "tor_bridge_lines", out_file.path,
            "file/ip-port"
        )
        self._cache_stale = True

        # Do an empty defer to fit inside of a event loop clock
        yield defer.succeed(None)
Beispiel #3
0
 def task(self):
     log.debug("Updating the inputs")
     yield probe_ip.lookup()
     log.debug("Updating the inputs for country %s" %
               probe_ip.geodata['countrycode'])
     yield resources.check_for_update(probe_ip.geodata['countrycode'])
     yield input_store.update(probe_ip.geodata['countrycode'])
     yield probe_ip.resolveGeodata()
Beispiel #4
0
 def task(self):
     log.debug("Updating the inputs")
     yield probe_ip.lookup()
     log.debug("Updating the inputs for country %s" %
               probe_ip.geodata['countrycode'])
     yield resources.check_for_update(probe_ip.geodata['countrycode'])
     yield input_store.update(probe_ip.geodata['countrycode'])
     yield probe_ip.resolveGeodata()
Beispiel #5
0
    def update_url_lists(self, country_code):
        countries = ["global"]
        if country_code != "ZZ":
            countries.append(country_code)

        for cc in countries:
            cc = cc.lower()
            in_file = self.resources.child("citizenlab-test-lists").child("{0}.csv".format(cc))
            if not in_file.exists():
                yield check_for_update(country_code)

            if not in_file.exists():
                log.msg("Could not find input for country "
                        "{0} in {1}".format(cc, in_file.path))
                continue

            # XXX maybe move this to some utility function.
            # It's duplicated in oonideckgen.
            data_fname = "citizenlab-test-lists_{0}.txt".format(cc)
            desc_fname = "citizenlab-test-lists_{0}.desc".format(cc)

            out_file = self.path.child("data").child(data_fname)
            write_txt_from_csv(in_file, out_file,
                lambda row: "{}\n".format(row[0])
            )

            desc_file = self.path.child("descriptors").child(desc_fname)
            if cc == "global":
                name = "List of globally accessed websites"
            else:
                # XXX resolve this to a human readable country name
                country_name = cc
                name = "List of websites for {0}".format(country_name)
            write_descriptor(desc_file, name,
                             "citizenlab_{0}_urls".format(cc),
                             out_file.path,
                             "file/url")

        self._cache_stale = True
        yield defer.succeed(None)
Beispiel #6
0
    def update_url_lists(self, country_code):
        countries = ["global"]
        if country_code != "ZZ":
            countries.append(country_code)

        for cc in countries:
            cc = cc.lower()
            in_file = self.resources.child("citizenlab-test-lists").child("{0}.csv".format(cc))
            if not in_file.exists():
                yield check_for_update(country_code)

            if not in_file.exists():
                log.msg("Could not find input for country "
                        "{0} in {1}".format(cc, in_file.path))
                continue

            # XXX maybe move this to some utility function.
            # It's duplicated in oonideckgen.
            data_fname = "citizenlab-test-lists_{0}.txt".format(cc)
            desc_fname = "citizenlab-test-lists_{0}.desc".format(cc)

            out_file = self.path.child("data").child(data_fname)
            write_txt_from_csv(in_file, out_file,
                lambda row: "{}\n".format(row[0])
            )

            desc_file = self.path.child("descriptors").child(desc_fname)
            if cc == "global":
                name = "List of globally accessed websites"
            else:
                # XXX resolve this to a human readable country name
                country_name = cc
                name = "List of websites for {0}".format(country_name)
            write_descriptor(desc_file, name,
                             "citizenlab_{0}_urls".format(cc),
                             out_file.path,
                             "file/url")

        self._cache_stale = True
        yield defer.succeed(None)
Beispiel #7
0
def oonideckgen(reactor):
    options = Options()
    try:
        options.parseOptions()
    except usage.UsageError as error_message:
        print("%s: %s" % (sys.argv[0], error_message))
        print(options)
        sys.exit(1)

    print("Checking for update of resources")
    yield check_for_update()

    if not options['output']:
        options['output'] = os.getcwd()

    if not options['country-code']:
        try:
            options['country-code'] = yield get_user_country_code()
        except errors.ProbeIPUnknown:
            print("Could not determine your IP address.")
            print("Check your internet connection or specify a country code "
                  "with -c.")
            sys.exit(4)

    if len(options['country-code']) != 2:
        print("%s: --country-code must be 2 characters" % sys.argv[0])
        sys.exit(2)

    if os.path.isdir(options['output']):
        options['output'] = os.path.join(options['output'], 'web-full.yaml')

    options['country-code'] = options['country-code'].lower()

    mkdir_p(os.path.dirname(options['output']))

    generate_deck(options)
def oonideckgen(reactor):
    options = Options()
    try:
        options.parseOptions()
    except usage.UsageError as error_message:
        print("%s: %s" % (sys.argv[0], error_message))
        print(options)
        sys.exit(1)

    print("Checking for update of resources")
    yield check_for_update()

    if not options['output']:
        options['output'] = os.getcwd()

    if not options['country-code']:
        try:
            options['country-code'] = yield get_user_country_code()
        except errors.ProbeIPUnknown:
            print("Could not determine your IP address.")
            print("Check your internet connection or specify a country code "
                  "with -c.")
            sys.exit(4)

    if len(options['country-code']) != 2:
        print("%s: --country-code must be 2 characters" % sys.argv[0])
        sys.exit(2)

    if os.path.isdir(options['output']):
        options['output'] = os.path.join(options['output'], 'web-full.yaml')

    options['country-code'] = options['country-code'].lower()

    mkdir_p(os.path.dirname(options['output']))

    generate_deck(options)
Beispiel #9
0
 def first_run(self):
     """
     On first run we update the resources that are common to every country.
     """
     log.debug("Updating the global inputs and resources")
     yield resources.check_for_update("ZZ")
Beispiel #10
0
 def first_run(self):
     """
     On first run we update the resources that are common to every country.
     """
     log.debug("Updating the global inputs and resources")
     yield resources.check_for_update("ZZ")