def _LoadConfigFromPath(self, path):
"""Load a configuration file describing data sources that may be available."""
conf_file = util.FindDataFile(path)
config = ConfigParser.ConfigParser()
config.read(conf_file)
for section in config.sections():
if section not in self.source_config:
self.source_config[section] = {
'name': None,
'search_paths': set(),
# Store whether or not this data source contains personal data
'full_hostnames': True,
'synthetic': False,
'include_duplicates': False,
'max_mtime_days': MAX_FILE_MTIME_AGE_DAYS
}
for (key, value) in config.items(section):
if key == 'name':
self.source_config[section]['name'] = value
elif key == 'full_hostnames' and int(value) == 0:
self.source_config[section]['full_hostnames'] = False
elif key == 'max_mtime_days':
self.source_config[section]['max_mtime_days'] = int(value)
elif key == 'include_duplicates':
self.source_config[section]['include_duplicates'] = bool(
value)
elif key == 'synthetic':
self.source_config[section]['synthetic'] = bool(value)
else:
self.source_config[section]['search_paths'].add(value)
def ProcessConfigurationFile(options):
"""Process configuration file, merge configuration with OptionParser.
Args:
options: optparse.OptionParser() object
Returns:
options: optparse.OptionParser() object
global_ns: A list of global nameserver tuples.
regional_ns: A list of regional nameservers tuples.
Raises:
ValueError: If we are unable to find a usable configuration file.
"""
config = ConfigParser.ConfigParser()
full_path = util.FindDataFile(options.config)
config.read(full_path)
if not config or not config.has_section('general'):
raise ValueError('Could not find usable configuration in %s (%s)' %
(full_path, options.config))
general = dict(config.items('general'))
if options.only or options.system_only:
global_ns = []
regional_ns = []
else:
global_ns = config.items('global')
regional_ns = config.items('regional') + config.items('private')
# -U implies -u
if options.site_url:
options.upload_results = True
for option in general:
if not getattr(options, option, None):
if 'timeout' in option:
value = float(general[option])
elif 'count' in option or 'num' in option or 'hide' in option:
value = int(general[option])
else:
value = general[option]
setattr(options, option, value)
for key in ('input_file', 'output_file', 'csv_file', 'input_source'):
value = getattr(options, key, None)
if value:
setattr(options, key, os.path.expanduser(value))
options.version = version.VERSION
return (options, global_ns, regional_ns)
def GetNameServerData(filename='config/servers.csv'):
server_file = util.FindDataFile(filename)
ns_data = _ParseNameServerListing(open(server_file))
# Add the system servers for later reference.
for i, ip in enumerate(sys_nameservers.GetCurrentNameServers()):
ns = nameserver.NameServer(ip,
name='SYS%s-%s' % (i, ip),
system_position=i)
ns_data.append(ns)
for i, ip in enumerate(sys_nameservers.GetAssignedNameServers()):
ns = nameserver.NameServer(ip,
name='DHCP%s-%s' % (i, ip),
dhcp_position=i)
ns_data.append(ns)
return ns_data
def _FindBestFileForSource(self,
source,
min_file_size=None,
max_mtime_age_days=None):
"""Find the best file (newest over X size) to use for a given source type.
Args:
source: source type
min_file_size: What the minimum allowable file size is for this source (int)
max_mtime_age_days: Maximum days old the file can be for this source (int)
Returns:
A file path.
"""
found = []
for path in self._GetSourceSearchPaths(source):
if not os.path.isabs(path):
path = util.FindDataFile(path)
for filename in glob.glob(path):
if min_file_size and os.path.getsize(filename) < min_file_size:
self.msg('Skipping %s (only %sb)' %
(filename, os.path.getsize(filename)))
else:
try:
fp = open(filename, 'rb')
fp.close()
found.append(filename)
except IOError:
self.msg('Skipping %s (could not open)' % filename)
if found:
newest = sorted(found, key=os.path.getmtime)[-1]
age_days = (time.time() - os.path.getmtime(newest)) / 86400
if max_mtime_age_days and age_days > max_mtime_age_days:
pass
# self.msg('Skipping %s (%2.0fd old)' % (newest, age_days))
else:
return newest
else:
return None
def MergeConfigurationFileOptions(options):
"""Process configuration file, merge configuration with OptionParser.
Args:
options: optparse.OptionParser() object
Returns:
options: optparse.OptionParser() object
Raises:
ValueError: If we are unable to find a usable configuration file.
"""
config = ConfigParser.ConfigParser()
full_path = util.FindDataFile(options.config)
config.read(full_path)
if not config or not config.has_section('general'):
raise ValueError('Could not find usable configuration in %s (%s)' %
(full_path, options.config))
general = dict(config.items('general'))
# -U implies -u
if options.site_url:
options.upload_results = True
for option in general:
if not getattr(options, option, None):
if 'timeout' in option:
value = float(general[option])
elif 'count' in option or 'num' in option or 'hide' in option:
value = int(general[option])
else:
value = general[option]
setattr(options, option, value)
for key in ('input_file', 'output_file', 'csv_file', 'input_source'):
value = getattr(options, key, None)
if value:
setattr(options, key, os.path.expanduser(value))
# This makes it easier to pass around later. Lazy-hack.
options.version = version.VERSION
return options
def GetLatestSanityChecks():
"""Get the latest copy of the sanity checks config."""
h = httplib2.Http(tempfile.gettempdir(), timeout=10)
http_version_usable = False
use_config = None
content = None
try:
unused_resp, content = h.request(SANITY_REFERENCE_URL, 'GET')
except:
print '* Unable to fetch latest reference: %s' % util.GetLastExceptionString(
)
http_config = ConfigParser.ConfigParser()
if content and '[base]' in content:
fp = StringIO.StringIO(content)
try:
http_config.readfp(fp)
http_version_usable = True
except:
pass
ref_file = util.FindDataFile('config/hostname_reference.cfg')
local_config = ConfigParser.ConfigParser()
local_config.read(ref_file)
if http_version_usable:
if int(http_config.get('base', 'version')) > int(
local_config.get('base', 'version')):
print '- Using %s' % SANITY_REFERENCE_URL
use_config = http_config
if not use_config:
use_config = local_config
return (use_config.items('sanity'), use_config.items('sanity-secondary'),
use_config.items('censorship'))
def CreateReport(self,
format='ascii',
output_fp=None,
csv_path=None,
sharing_url=None,
sharing_state=None):
"""Create a Report in a given format.
Args:
format: string (ascii, html, etc.) which defines what template to load.
output_fp: A File object to send the output to (optional)
csv_path: A string pathname to the CSV output to link to (optional)
sharing_url: A string URL where the results have been shared to. (optional)
sharing_state: A string showing what the shared result state is (optional)
Returns:
A rendered template (string)
"""
# First generate all of the charts necessary.
if format == 'ascii':
lowest_latency = self._LowestLatencyAsciiChart()
mean_duration = self._MeanRequestAsciiChart()
else:
lowest_latency = None
mean_duration = None
sorted_averages = sorted(self.ComputeAverages(),
key=operator.itemgetter(1))
runs_data = [(x[0].name, x[2]) for x in sorted_averages]
mean_duration_url = charts.PerRunDurationBarGraph(runs_data)
min_duration_url = charts.MinimumDurationBarGraph(
self.FastestNameServerResult())
distribution_url_200 = charts.DistributionLineGraph(
self.DigestedResults(), scale=200)
distribution_url = charts.DistributionLineGraph(
self.DigestedResults(), scale=self.config.timeout * 1000)
# Now generate all of the required textual information.
ns_summary = self._GenerateNameServerSummary()
best_ns = self.BestOverallNameServer()
recommended = [ns_summary[0]]
for row in sorted(ns_summary, key=operator.itemgetter('duration_min')):
if row['ip'] != ns_summary[0]['ip']:
recommended.append(row)
if len(recommended) == 3:
break
compare_title = 'Undecided'
compare_subtitle = 'Not enough servers to compare.'
compare_reference = None
for ns_record in ns_summary:
if ns_record.get('is_reference'):
if ns_record == ns_summary[0]:
compare_reference = ns_record
compare_title = 'N/A'
compare_subtitle = ''
elif len(ns_record['durations'][0]) >= MIN_RELEVANT_COUNT:
compare_reference = ns_record
compare_title = '%0.1f%%' % ns_summary[0]['diff']
compare_subtitle = 'Faster'
else:
compare_subtitle = 'Too few tests (needs %s)' % (
MIN_RELEVANT_COUNT)
break
# Fragile, makes assumption about the CSV being in the same path as the HTML file
if csv_path:
csv_link = os.path.basename(csv_path)
else:
csv_link = None
template_name = '%s.tmpl' % format
template_path = util.FindDataFile(
os.path.join('templates', template_name))
filtered_config = self.FilteredConfig()
template_dir = os.path.dirname(template_path)
env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
template = env.get_template(template_name)
sys_nameservers = nameserver_list.InternalNameServers()
if sys_nameservers:
system_primary = sys_nameservers[0]
else:
system_primary = None
rendered = template.render(best_ns=best_ns,
system_primary=system_primary,
timestamp=datetime.datetime.now(),
lowest_latency=lowest_latency,
version=self.config.version,
compare_subtitle=compare_subtitle,
compare_title=compare_title,
compare_reference=compare_reference,
sharing_url=sharing_url,
sharing_state=sharing_state,
config=filtered_config,
mean_duration=mean_duration,
ns_summary=ns_summary,
mean_duration_url=mean_duration_url,
min_duration_url=min_duration_url,
distribution_url=distribution_url,
distribution_url_200=distribution_url_200,
recommended=recommended,
csv_link=csv_link)
if output_fp:
output_fp.write(rendered)
output_fp.close()
else:
return rendered
def _ReadConfigFile(conf_file):
"""Read a local config file."""
ref_file = util.FindDataFile(conf_file)
local_config = ConfigParser.ConfigParser()
local_config.read(ref_file)
return local_config
'\.bor|'
'\.corp|'
'\.hot$|'
'\.local'
'\.pro[md]z*\.|'
'^0|'
'^\w+d[cs]\.|'
'^\w+nt\.|'
'^\w+sv\.|'
'dmz|'
'internal|'
'intranet|'
'intra|'
'\.\w{5,}$', re.IGNORECASE)
_SUFFIX_RULES_PATH = util.FindDataFile('data/effective_tld_names.dat')
_LOADED_SUFFIX_RULES = set()
_LOADED_SUFFIX_EXCEPTIONS = set()
def _public_suffix_rules():
"""Return a tuple of cached public suffix rules and exceptions."""
if not _LOADED_SUFFIX_RULES:
for line in codecs.open(_SUFFIX_RULES_PATH, 'r', 'utf8'):
line = line.strip()
if not line or line.startswith('//'):
continue
elif line.startswith('!'):
_LOADED_SUFFIX_EXCEPTIONS.add(line[1:])
# Make sure exceptions can fall back even if omitted from the data
_LOADED_SUFFIX_RULES.add(line[line.index('.') + 1:])
'^0|\.pro[md]z*\.|\.corp|\.bor|\.hot$|internal|dmz|'
'\._[ut][dc]p\.|intra|\.\w$|\.\w{5,}$', re.IGNORECASE)
# Used to decide if a hostname should be censored later.
PRIVATE_RE = re.compile(
'^\w+dc\.|^\w+ds\.|^\w+sv\.|^\w+nt\.|\.corp|internal|'
'intranet|\.local', re.IGNORECASE)
# ^.*[\w-]+\.[\w-]+\.[\w-]+\.[a-zA-Z]+\.$|^[\w-]+\.[\w-]{3,}\.[a-zA-Z]+\.$
FQDN_RE = re.compile(
'^.*\..*\..*\..*\.$|^.*\.[\w-]*\.\w{3,4}\.$|^[\w-]+\.[\w-]{4,}\.\w+\.')
IP_RE = re.compile('^[0-9.]+$')
KNOWN_SECOND_DOMAINS = [
x.rstrip() for x in open(util.FindDataFile(
'data/second_level_domains.txt')).readlines()
]
def ExtractIPsFromString(ip_string):
"""Return a tuple of ip addressed held in a string."""
ips = []
# IPV6 If this regexp is too loose, see Regexp-IPv6 in CPAN for inspiration.
ips.extend(
re.findall('[\dabcdef:]+:[\dabcdef:]+', ip_string, re.IGNORECASE))
for ip in re.findall('\d+\.\d+\.\d+\.+\d+', ip_string):
# Remove any leading zeros
ips.append(re.sub('\.0(\d+)', '.\\1', ip))
return ips
def ReadCountryData(filename='data/countries.csv'):
"""Read country data file, yielding rows of information."""
country_file = util.FindDataFile(filename)
for row in csv.DictReader(open(country_file),
fieldnames=['name', 'code', 'coords']):
yield row
