def _LoadConfigFromPath(self, path): """Load a configuration file describing data sources that may be available.""" conf_file = util.FindDataFile(path) config = ConfigParser.ConfigParser() config.read(conf_file) for section in config.sections(): if section not in self.source_config: self.source_config[section] = { 'name': None, 'search_paths': set(), # Store whether or not this data source contains personal data 'full_hostnames': True, 'synthetic': False, 'include_duplicates': False, 'max_mtime_days': MAX_FILE_MTIME_AGE_DAYS } for (key, value) in config.items(section): if key == 'name': self.source_config[section]['name'] = value elif key == 'full_hostnames' and int(value) == 0: self.source_config[section]['full_hostnames'] = False elif key == 'max_mtime_days': self.source_config[section]['max_mtime_days'] = int(value) elif key == 'include_duplicates': self.source_config[section]['include_duplicates'] = bool( value) elif key == 'synthetic': self.source_config[section]['synthetic'] = bool(value) else: self.source_config[section]['search_paths'].add(value)
def ProcessConfigurationFile(options): """Process configuration file, merge configuration with OptionParser. Args: options: optparse.OptionParser() object Returns: options: optparse.OptionParser() object global_ns: A list of global nameserver tuples. regional_ns: A list of regional nameservers tuples. Raises: ValueError: If we are unable to find a usable configuration file. """ config = ConfigParser.ConfigParser() full_path = util.FindDataFile(options.config) config.read(full_path) if not config or not config.has_section('general'): raise ValueError('Could not find usable configuration in %s (%s)' % (full_path, options.config)) general = dict(config.items('general')) if options.only or options.system_only: global_ns = [] regional_ns = [] else: global_ns = config.items('global') regional_ns = config.items('regional') + config.items('private') # -U implies -u if options.site_url: options.upload_results = True for option in general: if not getattr(options, option, None): if 'timeout' in option: value = float(general[option]) elif 'count' in option or 'num' in option or 'hide' in option: value = int(general[option]) else: value = general[option] setattr(options, option, value) for key in ('input_file', 'output_file', 'csv_file', 'input_source'): value = getattr(options, key, None) if value: setattr(options, key, os.path.expanduser(value)) options.version = version.VERSION return (options, global_ns, regional_ns)
def GetNameServerData(filename='config/servers.csv'): server_file = util.FindDataFile(filename) ns_data = _ParseNameServerListing(open(server_file)) # Add the system servers for later reference. for i, ip in enumerate(sys_nameservers.GetCurrentNameServers()): ns = nameserver.NameServer(ip, name='SYS%s-%s' % (i, ip), system_position=i) ns_data.append(ns) for i, ip in enumerate(sys_nameservers.GetAssignedNameServers()): ns = nameserver.NameServer(ip, name='DHCP%s-%s' % (i, ip), dhcp_position=i) ns_data.append(ns) return ns_data
def _FindBestFileForSource(self, source, min_file_size=None, max_mtime_age_days=None): """Find the best file (newest over X size) to use for a given source type. Args: source: source type min_file_size: What the minimum allowable file size is for this source (int) max_mtime_age_days: Maximum days old the file can be for this source (int) Returns: A file path. """ found = [] for path in self._GetSourceSearchPaths(source): if not os.path.isabs(path): path = util.FindDataFile(path) for filename in glob.glob(path): if min_file_size and os.path.getsize(filename) < min_file_size: self.msg('Skipping %s (only %sb)' % (filename, os.path.getsize(filename))) else: try: fp = open(filename, 'rb') fp.close() found.append(filename) except IOError: self.msg('Skipping %s (could not open)' % filename) if found: newest = sorted(found, key=os.path.getmtime)[-1] age_days = (time.time() - os.path.getmtime(newest)) / 86400 if max_mtime_age_days and age_days > max_mtime_age_days: pass # self.msg('Skipping %s (%2.0fd old)' % (newest, age_days)) else: return newest else: return None
def MergeConfigurationFileOptions(options): """Process configuration file, merge configuration with OptionParser. Args: options: optparse.OptionParser() object Returns: options: optparse.OptionParser() object Raises: ValueError: If we are unable to find a usable configuration file. """ config = ConfigParser.ConfigParser() full_path = util.FindDataFile(options.config) config.read(full_path) if not config or not config.has_section('general'): raise ValueError('Could not find usable configuration in %s (%s)' % (full_path, options.config)) general = dict(config.items('general')) # -U implies -u if options.site_url: options.upload_results = True for option in general: if not getattr(options, option, None): if 'timeout' in option: value = float(general[option]) elif 'count' in option or 'num' in option or 'hide' in option: value = int(general[option]) else: value = general[option] setattr(options, option, value) for key in ('input_file', 'output_file', 'csv_file', 'input_source'): value = getattr(options, key, None) if value: setattr(options, key, os.path.expanduser(value)) # This makes it easier to pass around later. Lazy-hack. options.version = version.VERSION return options
def GetLatestSanityChecks(): """Get the latest copy of the sanity checks config.""" h = httplib2.Http(tempfile.gettempdir(), timeout=10) http_version_usable = False use_config = None content = None try: unused_resp, content = h.request(SANITY_REFERENCE_URL, 'GET') except: print '* Unable to fetch latest reference: %s' % util.GetLastExceptionString( ) http_config = ConfigParser.ConfigParser() if content and '[base]' in content: fp = StringIO.StringIO(content) try: http_config.readfp(fp) http_version_usable = True except: pass ref_file = util.FindDataFile('config/hostname_reference.cfg') local_config = ConfigParser.ConfigParser() local_config.read(ref_file) if http_version_usable: if int(http_config.get('base', 'version')) > int( local_config.get('base', 'version')): print '- Using %s' % SANITY_REFERENCE_URL use_config = http_config if not use_config: use_config = local_config return (use_config.items('sanity'), use_config.items('sanity-secondary'), use_config.items('censorship'))
def CreateReport(self, format='ascii', output_fp=None, csv_path=None, sharing_url=None, sharing_state=None): """Create a Report in a given format. Args: format: string (ascii, html, etc.) which defines what template to load. output_fp: A File object to send the output to (optional) csv_path: A string pathname to the CSV output to link to (optional) sharing_url: A string URL where the results have been shared to. (optional) sharing_state: A string showing what the shared result state is (optional) Returns: A rendered template (string) """ # First generate all of the charts necessary. if format == 'ascii': lowest_latency = self._LowestLatencyAsciiChart() mean_duration = self._MeanRequestAsciiChart() else: lowest_latency = None mean_duration = None sorted_averages = sorted(self.ComputeAverages(), key=operator.itemgetter(1)) runs_data = [(x[0].name, x[2]) for x in sorted_averages] mean_duration_url = charts.PerRunDurationBarGraph(runs_data) min_duration_url = charts.MinimumDurationBarGraph( self.FastestNameServerResult()) distribution_url_200 = charts.DistributionLineGraph( self.DigestedResults(), scale=200) distribution_url = charts.DistributionLineGraph( self.DigestedResults(), scale=self.config.timeout * 1000) # Now generate all of the required textual information. ns_summary = self._GenerateNameServerSummary() best_ns = self.BestOverallNameServer() recommended = [ns_summary[0]] for row in sorted(ns_summary, key=operator.itemgetter('duration_min')): if row['ip'] != ns_summary[0]['ip']: recommended.append(row) if len(recommended) == 3: break compare_title = 'Undecided' compare_subtitle = 'Not enough servers to compare.' compare_reference = None for ns_record in ns_summary: if ns_record.get('is_reference'): if ns_record == ns_summary[0]: compare_reference = ns_record compare_title = 'N/A' compare_subtitle = '' elif len(ns_record['durations'][0]) >= MIN_RELEVANT_COUNT: compare_reference = ns_record compare_title = '%0.1f%%' % ns_summary[0]['diff'] compare_subtitle = 'Faster' else: compare_subtitle = 'Too few tests (needs %s)' % ( MIN_RELEVANT_COUNT) break # Fragile, makes assumption about the CSV being in the same path as the HTML file if csv_path: csv_link = os.path.basename(csv_path) else: csv_link = None template_name = '%s.tmpl' % format template_path = util.FindDataFile( os.path.join('templates', template_name)) filtered_config = self.FilteredConfig() template_dir = os.path.dirname(template_path) env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) template = env.get_template(template_name) sys_nameservers = nameserver_list.InternalNameServers() if sys_nameservers: system_primary = sys_nameservers[0] else: system_primary = None rendered = template.render(best_ns=best_ns, system_primary=system_primary, timestamp=datetime.datetime.now(), lowest_latency=lowest_latency, version=self.config.version, compare_subtitle=compare_subtitle, compare_title=compare_title, compare_reference=compare_reference, sharing_url=sharing_url, sharing_state=sharing_state, config=filtered_config, mean_duration=mean_duration, ns_summary=ns_summary, mean_duration_url=mean_duration_url, min_duration_url=min_duration_url, distribution_url=distribution_url, distribution_url_200=distribution_url_200, recommended=recommended, csv_link=csv_link) if output_fp: output_fp.write(rendered) output_fp.close() else: return rendered
def _ReadConfigFile(conf_file): """Read a local config file.""" ref_file = util.FindDataFile(conf_file) local_config = ConfigParser.ConfigParser() local_config.read(ref_file) return local_config
'\.bor|' '\.corp|' '\.hot$|' '\.local' '\.pro[md]z*\.|' '^0|' '^\w+d[cs]\.|' '^\w+nt\.|' '^\w+sv\.|' 'dmz|' 'internal|' 'intranet|' 'intra|' '\.\w{5,}$', re.IGNORECASE) _SUFFIX_RULES_PATH = util.FindDataFile('data/effective_tld_names.dat') _LOADED_SUFFIX_RULES = set() _LOADED_SUFFIX_EXCEPTIONS = set() def _public_suffix_rules(): """Return a tuple of cached public suffix rules and exceptions.""" if not _LOADED_SUFFIX_RULES: for line in codecs.open(_SUFFIX_RULES_PATH, 'r', 'utf8'): line = line.strip() if not line or line.startswith('//'): continue elif line.startswith('!'): _LOADED_SUFFIX_EXCEPTIONS.add(line[1:]) # Make sure exceptions can fall back even if omitted from the data _LOADED_SUFFIX_RULES.add(line[line.index('.') + 1:])
'^0|\.pro[md]z*\.|\.corp|\.bor|\.hot$|internal|dmz|' '\._[ut][dc]p\.|intra|\.\w$|\.\w{5,}$', re.IGNORECASE) # Used to decide if a hostname should be censored later. PRIVATE_RE = re.compile( '^\w+dc\.|^\w+ds\.|^\w+sv\.|^\w+nt\.|\.corp|internal|' 'intranet|\.local', re.IGNORECASE) # ^.*[\w-]+\.[\w-]+\.[\w-]+\.[a-zA-Z]+\.$|^[\w-]+\.[\w-]{3,}\.[a-zA-Z]+\.$ FQDN_RE = re.compile( '^.*\..*\..*\..*\.$|^.*\.[\w-]*\.\w{3,4}\.$|^[\w-]+\.[\w-]{4,}\.\w+\.') IP_RE = re.compile('^[0-9.]+$') KNOWN_SECOND_DOMAINS = [ x.rstrip() for x in open(util.FindDataFile( 'data/second_level_domains.txt')).readlines() ] def ExtractIPsFromString(ip_string): """Return a tuple of ip addressed held in a string.""" ips = [] # IPV6 If this regexp is too loose, see Regexp-IPv6 in CPAN for inspiration. ips.extend( re.findall('[\dabcdef:]+:[\dabcdef:]+', ip_string, re.IGNORECASE)) for ip in re.findall('\d+\.\d+\.\d+\.+\d+', ip_string): # Remove any leading zeros ips.append(re.sub('\.0(\d+)', '.\\1', ip)) return ips
def ReadCountryData(filename='data/countries.csv'): """Read country data file, yielding rows of information.""" country_file = util.FindDataFile(filename) for row in csv.DictReader(open(country_file), fieldnames=['name', 'code', 'coords']): yield row