def open(url):
"""Opens a URL, overriding the normal webbrowser.open methods for sanity."""
try:
webbrowser.open(url, new=1, autoraise=True)
# If the user is missing the osascript binary - see
# http://code.google.com/p/namebench/issues/detail?id=88
except:
output('Failed to open: [%s]: %s' %
(url, util.GetLastExceptionString()))
if os.path.exists('/usr/bin/open'):
try:
output('trying open: %s' % url)
p = subprocess.Popen(('open', url))
p.wait()
except:
output('open did not seem to work: %s' %
util.GetLastExceptionString())
elif sys.platform[:3] == 'win':
try:
output('trying default Windows controller: %s' % url)
controller = webbrowser.get('windows-default')
controller.open_new(url)
except:
output('WindowsController did not work: %s' %
util.GetLastExceptionString())
def CheckHealth(self,
primary_checks,
secondary_checks,
cache_dir=None,
censor_tests=None):
"""Filter out unhealthy or slow replica servers."""
if len(self) == 1:
return None
if cache_dir:
self.cache_dir = cache_dir
cpath = self._SecondaryCachePath()
try:
cached = self.InvokeSecondaryCache()
except:
self.msg('Failed to use secondary cache in [%s]: %s' %
(cpath, util.GetLastExceptionString()))
cached = False
if not cached:
self.msg(
'Building initial DNS cache for %s nameservers (%s threads)' %
(len(self), self.thread_count))
self.PingNameServers()
if len(self.enabled) > int(self.num_servers * NS_CACHE_SLACK):
self.DisableDistantServers()
self.RunHealthCheckThreads(primary_checks)
if len(self.enabled) > self.num_servers:
self._DemoteSecondaryGlobalNameServers()
self.DisableUnwantedServers(target_count=int(self.num_servers *
NS_CACHE_SLACK),
delete_unwanted=True)
if not cached:
try:
self._UpdateSecondaryCache(cpath)
except:
self.msg('Failed to save secondary cache in [%s]: %s' %
(cpath, util.GetLastExceptionString()))
if not self.skip_cache_collusion_checks:
self.CheckCacheCollusion()
self.DisableUnwantedServers(self.num_servers)
self.RunFinalHealthCheckThreads(secondary_checks)
if censor_tests:
self.RunCensorshipCheckThreads(censor_tests)
else:
# If we aren't doing censorship checks, quiet any possible false positives.
self._RemoveGlobalWarnings()
if not self.enabled:
raise TooFewNameservers(
'None of the nameservers tested are healthy')
def GetFromGoogleLocAPI():
"""Use the Google Loc JSON API from Google Gears.
Returns:
A dictionary containing geolocation information
NOTE: This is in violation of the Gears Terms of Service. See:
http://code.google.com/p/gears/wiki/GeolocationAPI
"""
h = httplib2.Http(tempfile.gettempdir(), timeout=10)
url = 'http://www.google.com/loc/json'
post_data = {
'request_address': 'true',
'version': '1.1.0',
'source': 'namebench'
}
unused_resp, content = h.request(url, 'POST', simplejson.dumps(post_data))
try:
data = simplejson.loads(content)['location']
return {
'region_name': data['address'].get('region'),
'country_name': data['address'].get('country'),
'country_code': data['address'].get('country_code'),
'city': data['address'].get('city'),
'latitude': data['latitude'],
'longitude': data['longitude'],
'source': 'gloc'
}
except:
print '* Failed to use GoogleLocAPI: %s (content: %s)' % (
util.GetLastExceptionString(), content)
return {}
def TimedRequest(self,
type_string,
record_string,
timeout=None,
rdataclass=None):
"""Make a DNS request, returning the reply and duration it took.
Args:
type_string: DNS record type to query (string)
record_string: DNS record name to query (string)
timeout: optional timeout (float)
rdataclass: optional result class (defaults to rdataclass.IN)
Returns:
A tuple of (response, duration in ms [float], error_msg)
In the case of a DNS response timeout, the response object will be None.
"""
if not rdataclass:
rdataclass = dns.rdataclass.IN
else:
rdataclass = dns.rdataclass.from_text(rdataclass)
request_type = dns.rdatatype.from_text(type_string)
record = dns.name.from_text(record_string, None)
request = None
self.request_count += 1
# Sometimes it takes great effort just to craft a UDP packet.
try:
request = self.CreateRequest(record, request_type, rdataclass)
except ValueError, exc:
if not request:
return (None, 0, util.GetLastExceptionString())
def GetTxtRecordWithDuration(self, record, retries_left=2):
(response, duration, _) = self.TimedRequest('TXT', record, timeout=self.health_timeout)
if response and response.answer:
return (response.answer[0].items[0].to_text().lstrip('"').rstrip('"'), duration)
elif not response and retries_left:
print "* Failed to lookup %s (retries left: %s): %s" % (record, retries_left, util.GetLastExceptionString())
return self.GetTxtRecordWithDuration(record, retries_left=retries_left-1)
else:
return (None, duration)
def __init__(self, ip=None):
if not ip:
try:
ip = socket.gethostbyname(MY_RESOLVER_HOST)
except:
print "Could not resolve %s: %s" % (MY_RESOLVER_HOST, util.GetLastExceptionString())
# TODO(tstromberg): Find a more elegant fallback solution.
ip = '127.0.0.1'
super(MyResolverInfo, self).__init__(ip=ip)
def GetGeoData():
"""Get geodata from any means necessary. Sanitize as necessary."""
try:
json_data = GetFromGoogleLocAPI()
if not json_data:
json_data = GetFromMaxmindJSAPI()
# Make our data less accurate. We don't need any more than that.
json_data['latitude'] = '%.3f' % float(json_data['latitude'])
json_data['longitude'] = '%.3f' % float(json_data['longitude'])
return json_data
except:
print 'Failed to get Geodata: %s' % util.GetLastExceptionString()
return {}
def GetAutoUpdatingConfigFile(conf_file):
"""Get the latest copy of the config file"""
local_config = _ReadConfigFile(conf_file)
download_latest = int(local_config.get('config', 'download_latest'))
local_version = int(local_config.get('config', 'version'))
if download_latest == 0:
return _ExpandConfigSections(local_config)
h = httplib2.Http(tempfile.gettempdir(), timeout=10)
url = '%s/%s' % (TRUNK_URL, conf_file)
content = None
try:
_, content = h.request(url, 'GET')
remote_config = ConfigParser.ConfigParser()
except:
print '* Unable to fetch remote %s: %s' % (
conf_file, util.GetLastExceptionString())
return _ExpandConfigSections(local_config)
if content and '[config]' in content:
fp = StringIO.StringIO(content)
try:
remote_config.readfp(fp)
except:
print '* Unable to read remote %s: %s' % (
conf_file, util.GetLastExceptionString())
return _ExpandConfigSections(local_config)
if remote_config and remote_config.has_section('config') and int(
remote_config.get('config', 'version')) > local_version:
print '- Using %s' % url
return _ExpandConfigSections(remote_config)
else:
return _ExpandConfigSections(local_config)
def UploadJsonResults(self,
json_data,
hide_results=False,
fail_quickly=False):
"""Data is generated by reporter.CreateJsonData."""
url = self.url + '/submit'
if not url or not url.startswith('http'):
return (False, 'error')
h = httplib2.Http()
post_data = {
'client_id': self._CalculateDuplicateCheckId(),
'submit_id': random.randint(0, 2**32),
'hidden': bool(hide_results),
'data': json_data
}
try:
resp, content = h.request(url, 'POST', urllib.urlencode(post_data))
try:
data = simplejson.loads(content)
for note in data['notes']:
print ' * %s' % note
return (''.join((self.url, data['url'])), data['state'])
except:
self.msg('BAD RESPONSE from %s: [%s]:\n %s' %
(url, resp, content))
print "DATA:"
print post_data
# See http://code.google.com/p/httplib2/issues/detail?id=62
except AttributeError:
self.msg('%s refused connection' % url)
except:
self.msg('Error uploading results: %s' %
util.GetLastExceptionString())
# We haven't returned, something is up.
if not fail_quickly:
self.msg('Problem talking to %s, will retry after %ss' %
(url, RETRY_WAIT))
time.sleep(RETRY_WAIT)
self.UploadJsonResults(json_data,
hide_results=hide_results,
fail_quickly=True)
return (False, 'error')
def GetFromIpInfoIO():
h = httplib2.Http(tempfile.gettempdir(), timeout=10)
url = 'http://ipinfo.io/json'
try:
unused_resp, content = h.request(url, 'GET')
data = simplejson.loads(content)
return {
'region_name': data['region'],
'country_name': data['country'],
'country_code': data['country'],
'city': data['city'],
'latitude': data['loc'].split(',')[0],
'longitude': data['loc'].split(',')[1],
'source': 'ipinfo'
}
except:
print '* Failed to use GoogleLocAPI: %s (content: %s)' % (
util.GetLastExceptionString(), content)
return {}
def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None):
"""Make a DNS Get, returning the reply and duration it took.
Args:
type_string: DNS record type to query (string)
record_string: DNS record name to query (string)
timeout: optional timeout (float)
rdataclass: optional result class (defaults to rdataclass.IN)
Returns:
A tuple of (response, duration in ms [float], error_msg)
In the case of a DNS response timeout, the response object will be None.
"""
if not rdataclass:
rdataclass = dns.rdataclass.IN
else:
rdataclass = dns.rdataclass.from_text(rdataclass)
request_type = dns.rdatatype.from_text(type_string)
record = dns.name.from_text(record_string, None)
request = None
self.request_count += 1
try:
request = self.CreateRequest(record, request_type, rdataclass)
except ValueError:
if not request:
return (None, 0, util.GetLastExceptionString())
if not timeout:
timeout = self.timeout
error_msg = None
exc = None
duration = None
try:
start_time = self.timer()
response = self.Query(request, timeout)
duration = self.timer() - start_time
except (dns.exception.Timeout), exc:
response = None
def GetIndexHosts(self):
"""Get a list of 'index' hosts for standardized testing."""
url = self.url + '/index_hosts'
h = httplib2.Http(tempfile.gettempdir(), timeout=10)
content = None
try:
unused_resp, content = h.request(url, 'GET')
hosts = []
for record_type, host in simplejson.loads(content):
hosts.append((str(record_type), str(host)))
return hosts
except simplejson.decoder.JSONDecodeError:
self.msg('Failed to decode: "%s"' % content)
return []
except AttributeError:
self.msg('%s refused connection' % url)
return []
except:
self.msg('* Failed to fetch %s: %s' %
(url, util.GetLastExceptionString()))
return []
def GetLatestSanityChecks():
"""Get the latest copy of the sanity checks config."""
h = httplib2.Http(tempfile.gettempdir(), timeout=10)
http_version_usable = False
use_config = None
content = None
try:
unused_resp, content = h.request(SANITY_REFERENCE_URL, 'GET')
except:
print '* Unable to fetch latest reference: %s' % util.GetLastExceptionString(
)
http_config = ConfigParser.ConfigParser()
if content and '[base]' in content:
fp = StringIO.StringIO(content)
try:
http_config.readfp(fp)
http_version_usable = True
except:
pass
ref_file = util.FindDataFile('config/hostname_reference.cfg')
local_config = ConfigParser.ConfigParser()
local_config.read(ref_file)
if http_version_usable:
if int(http_config.get('base', 'version')) > int(
local_config.get('base', 'version')):
print '- Using %s' % SANITY_REFERENCE_URL
use_config = http_config
if not use_config:
use_config = local_config
return (use_config.items('sanity'), use_config.items('sanity-secondary'),
use_config.items('censorship'))
if not timeout:
timeout = self.timeout
error_msg = None
exc = None
duration = None
try:
start_time = self.timer()
response = self.Query(request, timeout)
duration = self.timer() - start_time
except (dns.exception.Timeout), exc:
response = None
except (dns.query.BadResponse, dns.message.TrailingJunk,
dns.query.UnexpectedSource), exc:
error_msg = util.GetLastExceptionString()
response = None
# This is pretty normal if someone runs namebench offline.
except socket.error:
response = None
if ':' in self.ip:
error_msg = 'socket error: IPv6 may not be available.'
else:
error_msg = util.GetLastExceptionString()
# Pass these exceptions up the food chain
except (KeyboardInterrupt, SystemExit, SystemError), exc:
raise exc
except:
error_msg = util.GetLastExceptionString()
print "* Unusual error with %s:%s on %s: %s" % (
type_string, record_string, self, error_msg)
start_time = self.timer()
response = self.Query(request, timeout)
duration = self.timer() - start_time
except (dns.exception.Timeout), exc:
response = None
except (dns.query.BadResponse, dns.message.TrailingJunk,
dns.query.UnexpectedSource), exc:
error_msg = util.GetLastExceptionString()
response = None
# This is pretty normal if someone runs namebench offline.
except socket.error:
response = None
if ':' in self.ip:
error_msg = 'socket error: IPv6 may not be available.'
else:
error_msg = util.GetLastExceptionString()
# Pass these exceptions up the food chain
except (KeyboardInterrupt, SystemExit, SystemError), exc:
raise exc
except:
error_msg = util.GetLastExceptionString()
print "* Unusual error with %s:%s on %s: %s" % (type_string, record_string, self, error_msg)
response = None
if not response:
self.failure_count += 1
if not duration:
duration = self.timer() - start_time
if exc and not error_msg:
def GetReverseIp(self, ip, retries_left=2):
"""Request a hostname for a given IP address."""
try:
print "reverse: %s -> %s" % (ip, self)
answer = dns.resolver.query(dns.reversename.from_address(ip), 'PTR')
except dns.resolver.NXDOMAIN:
return ip
except:
if retries_left:
print "* Failed to get hostname for %s (retries left: %s): %s" % (ip, retries_left, util.GetLastExceptionString())
return self.GetReverseIp(ip, retries_left=retries_left-1)
else:
return ip
if answer:
return answer[0].to_text().rstrip('.')
else:
return ip
class NameServer(health_checks.NameServerHealthChecks, provider_extensions.NameServerProvider):
"""Hold information about a particular nameserver."""
def __init__(self, ip, hostname=None, name=None, tags=None, provider=None,
instance=None, location=None, latitude=None, longitude=None, asn=None,
network_owner=None, dhcp_position=None, system_position=None):
self.ip = ip
self.name = name
self.dhcp_position = dhcp_position
self.system_position = system_position
if tags:
self.tags = set(tags)
else:
self.tags = set()
self.provider = provider
self.instance = instance
self.location = location
if self.location:
self.country_code = location.split('/')[0]
self.tags.add('country_%s' % self.country_code.lower())
else:
self.country_code = None
self.latitude = latitude
self.longitude = longitude
self.asn = asn
self.network_owner = network_owner
self._hostname = hostname
self.timeout = 5
self.health_timeout = 5
self.ping_timeout = 1
self.ResetTestStatus()
self._version = None
self._node_ids = set()
self.timer = BEST_TIMER_FUNCTION
if ':' in self.ip:
self.tags.add('ipv6')
elif '.' in self.ip:
self.tags.add('ipv4')
if self.dhcp_position is not None:
self.tags.add('dhcp')
if self.system_position is not None:
self.tags.add('system')
if ip.endswith('.0') or ip.endswith('.255'):
self.DisableWithMessage("IP appears to be a broadcast address.")
elif self.is_bad:
self.DisableWithMessage("Known bad address.")
def AddNetworkTags(self, domain, provider, asn, country_code):
if self.hostname:
my_domain = addr_util.GetDomainFromHostname(self.hostname)
hostname = self.hostname.lower()
else:
my_domain = 'UNKNOWN'
hostname = ''
if provider:
provider = provider.lower()
if domain and my_domain == domain:
self.tags.add('isp')
if asn and self.asn == asn:
self.tags.add('network')
if provider and 'isp' not in self.tags:
if (provider in self.name.lower() or provider in self.hostname.lower()
or (self.network_owner and provider in self.network_owner.lower())):
self.tags.add('isp')
elif provider and self.country_code == country_code and my_domain != domain:
if (provider in self.name.lower() or provider in hostname
or (self.network_owner and provider in self.network_owner.lower())):
self.tags.add('likely-isp')
def ResetTestStatus(self):
"""Reset testing status of this host."""
self.warnings = set()
self.shared_with = set()
if self.is_disabled:
self.tags.remove('disabled')
self.checks = []
self.failed_test_count = 0
self.share_check_count = 0
self.cache_checks = []
self.is_slower_replica = False
self.ResetErrorCounts()
def ResetErrorCounts(self):
"""NOTE: This gets called by benchmark.Run()!"""
self.request_count = 0
self.failure_count = 0
self.error_map = {}
@property
def is_keeper(self):
return bool(self.MatchesTags(['preferred', 'dhcp', 'system', 'specified']))
@property
def is_bad(self):
if not self.is_keeper and self.MatchesTags(['rejected', 'blacklist']):
return True
@property
def is_hidden(self):
return self.HasTag('hidden')
@property
def is_disabled(self):
return self.HasTag('disabled')
@property
def check_average(self):
# If we only have a ping result, sort by it. Otherwise, use all non-ping results.
if len(self.checks) == 1:
return self.checks[0][3]
else:
return util.CalculateListAverage([x[3] for x in self.checks[1:]])
@property
def fastest_check_duration(self):
if self.checks:
return min([x[3] for x in self.checks])
else:
return 0.0
@property
def check_duration(self):
return sum([x[3] for x in self.checks])
@property
def warnings_string(self):
if self.is_disabled:
return 'DISABLED: %s' % self.disabled_msg
else:
return ', '.join(map(str, self.warnings))
@property
def errors(self):
return ['%s (%s requests)' % (_[0], _[1]) for _ in self.error_map.items() if _[0] != 'Timeout']
@property
def error_count(self):
return sum([_[1] for _ in self.error_map.items() if _[0] != 'Timeout'])
@property
def timeout_count(self):
return self.error_map.get('Timeout', 0)
@property
def notes(self):
"""Return a list of notes about this nameserver object."""
my_notes = []
if self.system_position == 0:
my_notes.append('The current preferred DNS server')
elif self.system_position:
my_notes.append('A backup DNS server for this system')
if self.dhcp_position is not None:
my_notes.append('Assigned by your network DHCP server')
if self.is_failure_prone:
my_notes.append('%s of %s queries failed' % (self.failure_count, self.request_count))
if self.HasTag('blacklist'):
my_notes.append('BEWARE: IP appears in DNS server blacklist')
if self.is_disabled:
my_notes.append(self.disabled_msg)
else:
my_notes.extend(self.warnings)
if self.errors:
my_notes.extend(self.errors)
return my_notes
@property
def hostname(self):
if self._hostname is None and not self.is_disabled:
self.UpdateHostname()
return self._hostname
def UpdateHostname(self):
if not self.is_disabled:
self._hostname = self.GetReverseIp(self.ip)
return self._hostname
@property
def version(self):
if self._version is None and not self.is_disabled:
self.GetVersion()
if not self._version:
return None
# Only return meaningful data
if (re.search('\d', self._version) or
(re.search('recursive|ns|server|bind|unbound', self._version, re.I)
and 'ontact' not in self._version and '...' not in self._version)):
return self._version
else:
return None
@property
def external_ips(self):
"""Return a set of external ips seen on this system."""
# We use a slightly different pattern here because we want to
# append to our results each time this is called.
self._external_ips.add(self.GetMyResolverInfoWithDuration())
# Only return non-blank entries
return [x for x in self._external_ips if x]
@property
def node_ids(self):
"""Return a set of node_ids seen on this system."""
if self.is_disabled:
return []
# Only return non-blank entries
return [x for x in self._node_ids if x]
def UpdateNodeIds(self):
node_id = self.GetNodeIdWithDuration()[0]
self._node_ids.add(node_id)
return node_id
@property
def partial_node_ids(self):
partials = []
for node_id in self._node_ids:
node_bits = node_id.split('.')
if len(node_bits) >= 3:
partials.append('.'.join(node_bits[0:-2]))
else:
partials.append('.'.join(node_bits))
return partials
@property
def name_and_node(self):
if self.node_ids:
return '%s [%s]' % (self.name, ', '.join(self.partial_node_ids))
else:
return self.name
@property
def is_failure_prone(self):
if self.failure_rate >= FAILURE_PRONE_RATE:
return True
else:
return False
@property
def failure_rate(self):
if not self.failure_count or not self.request_count:
return 0
else:
return (float(self.failure_count) / float(self.request_count)) * 100
def __str__(self):
return '%s [%s:%s]' % (self.name, self.ip, ','.join(self.tags))
def __repr__(self):
return self.__str__()
def HasTag(self, tag):
"""Matches one tag."""
return tag in self.tags
def MatchesTags(self, tags):
"""Matches many tags."""
return self.tags.intersection(tags)
def AddFailure(self, message, fatal=False):
"""Add a failure for this nameserver. This will effectively disable it's use."""
respect_fatal = True
if self.is_keeper:
max_count = MAX_KEEPER_FAILURES
else:
max_count = MAX_NORMAL_FAILURES
self.failed_test_count += 1
if self.is_keeper:
respect_fatal = False
# Be quiet if this is simply a 'preferred' ipv6 host.
if self.HasTag('preferred') and self.HasTag('ipv6') and len(self.checks) <= 1:
self.tags.add('disabled')
else:
print "\n* %s failed test #%s/%s: %s" % (self, self.failed_test_count, max_count, message)
if fatal and respect_fatal:
self.DisableWithMessage(message)
elif self.failed_test_count >= max_count:
self.DisableWithMessage("Failed %s tests, last: %s" % (self.failed_test_count, message))
def AddWarning(self, message, penalty=True):
"""Add a warning to a host."""
if not isinstance(message, str):
print "Tried to add %s to %s (not a string)" % (message, self)
return None
self.warnings.add(message)
if penalty and len(self.warnings) >= MAX_WARNINGS:
self.AddFailure('Too many warnings (%s), probably broken.' % len(self.warnings), fatal=True)
def DisableWithMessage(self, message):
self.tags.add('disabled')
if self.is_keeper and not self.HasTag('ipv6'):
print "\nDISABLING %s: %s\n" % (self, message)
else:
self.tags.add('hidden')
self.disabled_msg = message
def CreateRequest(self, record, request_type, return_type):
"""Function to work around any dnspython make_query quirks."""
return dns.message.make_query(record, request_type, return_type)
def Query(self, request, timeout):
# print "%s -> %s" % (request, self)
return dns.query.udp(request, self.ip, timeout, 53)
def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None):
"""Make a DNS Get, returning the reply and duration it took.
Args:
type_string: DNS record type to query (string)
record_string: DNS record name to query (string)
timeout: optional timeout (float)
rdataclass: optional result class (defaults to rdataclass.IN)
Returns:
A tuple of (response, duration in ms [float], error_msg)
In the case of a DNS response timeout, the response object will be None.
"""
if not rdataclass:
rdataclass = dns.rdataclass.IN
else:
rdataclass = dns.rdataclass.from_text(rdataclass)
request_type = dns.rdatatype.from_text(type_string)
record = dns.name.from_text(record_string, None)
request = None
self.request_count += 1
try:
request = self.CreateRequest(record, request_type, rdataclass)
except ValueError:
if not request:
return (None, 0, util.GetLastExceptionString())
if not timeout:
timeout = self.timeout
error_msg = None
exc = None
duration = None
try:
start_time = self.timer()
response = self.Query(request, timeout)
duration = self.timer() - start_time
except (dns.exception.Timeout), exc:
response = None
except (dns.query.BadResponse, dns.message.TrailingJunk,
dns.query.UnexpectedSource), exc:
error_msg = util.GetLastExceptionString()
response = None
