def open(url): """Opens a URL, overriding the normal webbrowser.open methods for sanity.""" try: webbrowser.open(url, new=1, autoraise=True) # If the user is missing the osascript binary - see # http://code.google.com/p/namebench/issues/detail?id=88 except: output('Failed to open: [%s]: %s' % (url, util.GetLastExceptionString())) if os.path.exists('/usr/bin/open'): try: output('trying open: %s' % url) p = subprocess.Popen(('open', url)) p.wait() except: output('open did not seem to work: %s' % util.GetLastExceptionString()) elif sys.platform[:3] == 'win': try: output('trying default Windows controller: %s' % url) controller = webbrowser.get('windows-default') controller.open_new(url) except: output('WindowsController did not work: %s' % util.GetLastExceptionString())
def CheckHealth(self, primary_checks, secondary_checks, cache_dir=None, censor_tests=None): """Filter out unhealthy or slow replica servers.""" if len(self) == 1: return None if cache_dir: self.cache_dir = cache_dir cpath = self._SecondaryCachePath() try: cached = self.InvokeSecondaryCache() except: self.msg('Failed to use secondary cache in [%s]: %s' % (cpath, util.GetLastExceptionString())) cached = False if not cached: self.msg( 'Building initial DNS cache for %s nameservers (%s threads)' % (len(self), self.thread_count)) self.PingNameServers() if len(self.enabled) > int(self.num_servers * NS_CACHE_SLACK): self.DisableDistantServers() self.RunHealthCheckThreads(primary_checks) if len(self.enabled) > self.num_servers: self._DemoteSecondaryGlobalNameServers() self.DisableUnwantedServers(target_count=int(self.num_servers * NS_CACHE_SLACK), delete_unwanted=True) if not cached: try: self._UpdateSecondaryCache(cpath) except: self.msg('Failed to save secondary cache in [%s]: %s' % (cpath, util.GetLastExceptionString())) if not self.skip_cache_collusion_checks: self.CheckCacheCollusion() self.DisableUnwantedServers(self.num_servers) self.RunFinalHealthCheckThreads(secondary_checks) if censor_tests: self.RunCensorshipCheckThreads(censor_tests) else: # If we aren't doing censorship checks, quiet any possible false positives. self._RemoveGlobalWarnings() if not self.enabled: raise TooFewNameservers( 'None of the nameservers tested are healthy')
def GetFromGoogleLocAPI(): """Use the Google Loc JSON API from Google Gears. Returns: A dictionary containing geolocation information NOTE: This is in violation of the Gears Terms of Service. See: http://code.google.com/p/gears/wiki/GeolocationAPI """ h = httplib2.Http(tempfile.gettempdir(), timeout=10) url = 'http://www.google.com/loc/json' post_data = { 'request_address': 'true', 'version': '1.1.0', 'source': 'namebench' } unused_resp, content = h.request(url, 'POST', simplejson.dumps(post_data)) try: data = simplejson.loads(content)['location'] return { 'region_name': data['address'].get('region'), 'country_name': data['address'].get('country'), 'country_code': data['address'].get('country_code'), 'city': data['address'].get('city'), 'latitude': data['latitude'], 'longitude': data['longitude'], 'source': 'gloc' } except: print '* Failed to use GoogleLocAPI: %s (content: %s)' % ( util.GetLastExceptionString(), content) return {}
def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None): """Make a DNS request, returning the reply and duration it took. Args: type_string: DNS record type to query (string) record_string: DNS record name to query (string) timeout: optional timeout (float) rdataclass: optional result class (defaults to rdataclass.IN) Returns: A tuple of (response, duration in ms [float], error_msg) In the case of a DNS response timeout, the response object will be None. """ if not rdataclass: rdataclass = dns.rdataclass.IN else: rdataclass = dns.rdataclass.from_text(rdataclass) request_type = dns.rdatatype.from_text(type_string) record = dns.name.from_text(record_string, None) request = None self.request_count += 1 # Sometimes it takes great effort just to craft a UDP packet. try: request = self.CreateRequest(record, request_type, rdataclass) except ValueError, exc: if not request: return (None, 0, util.GetLastExceptionString())
def GetTxtRecordWithDuration(self, record, retries_left=2): (response, duration, _) = self.TimedRequest('TXT', record, timeout=self.health_timeout) if response and response.answer: return (response.answer[0].items[0].to_text().lstrip('"').rstrip('"'), duration) elif not response and retries_left: print "* Failed to lookup %s (retries left: %s): %s" % (record, retries_left, util.GetLastExceptionString()) return self.GetTxtRecordWithDuration(record, retries_left=retries_left-1) else: return (None, duration)
def __init__(self, ip=None): if not ip: try: ip = socket.gethostbyname(MY_RESOLVER_HOST) except: print "Could not resolve %s: %s" % (MY_RESOLVER_HOST, util.GetLastExceptionString()) # TODO(tstromberg): Find a more elegant fallback solution. ip = '127.0.0.1' super(MyResolverInfo, self).__init__(ip=ip)
def GetGeoData(): """Get geodata from any means necessary. Sanitize as necessary.""" try: json_data = GetFromGoogleLocAPI() if not json_data: json_data = GetFromMaxmindJSAPI() # Make our data less accurate. We don't need any more than that. json_data['latitude'] = '%.3f' % float(json_data['latitude']) json_data['longitude'] = '%.3f' % float(json_data['longitude']) return json_data except: print 'Failed to get Geodata: %s' % util.GetLastExceptionString() return {}
def GetAutoUpdatingConfigFile(conf_file): """Get the latest copy of the config file""" local_config = _ReadConfigFile(conf_file) download_latest = int(local_config.get('config', 'download_latest')) local_version = int(local_config.get('config', 'version')) if download_latest == 0: return _ExpandConfigSections(local_config) h = httplib2.Http(tempfile.gettempdir(), timeout=10) url = '%s/%s' % (TRUNK_URL, conf_file) content = None try: _, content = h.request(url, 'GET') remote_config = ConfigParser.ConfigParser() except: print '* Unable to fetch remote %s: %s' % ( conf_file, util.GetLastExceptionString()) return _ExpandConfigSections(local_config) if content and '[config]' in content: fp = StringIO.StringIO(content) try: remote_config.readfp(fp) except: print '* Unable to read remote %s: %s' % ( conf_file, util.GetLastExceptionString()) return _ExpandConfigSections(local_config) if remote_config and remote_config.has_section('config') and int( remote_config.get('config', 'version')) > local_version: print '- Using %s' % url return _ExpandConfigSections(remote_config) else: return _ExpandConfigSections(local_config)
def UploadJsonResults(self, json_data, hide_results=False, fail_quickly=False): """Data is generated by reporter.CreateJsonData.""" url = self.url + '/submit' if not url or not url.startswith('http'): return (False, 'error') h = httplib2.Http() post_data = { 'client_id': self._CalculateDuplicateCheckId(), 'submit_id': random.randint(0, 2**32), 'hidden': bool(hide_results), 'data': json_data } try: resp, content = h.request(url, 'POST', urllib.urlencode(post_data)) try: data = simplejson.loads(content) for note in data['notes']: print ' * %s' % note return (''.join((self.url, data['url'])), data['state']) except: self.msg('BAD RESPONSE from %s: [%s]:\n %s' % (url, resp, content)) print "DATA:" print post_data # See http://code.google.com/p/httplib2/issues/detail?id=62 except AttributeError: self.msg('%s refused connection' % url) except: self.msg('Error uploading results: %s' % util.GetLastExceptionString()) # We haven't returned, something is up. if not fail_quickly: self.msg('Problem talking to %s, will retry after %ss' % (url, RETRY_WAIT)) time.sleep(RETRY_WAIT) self.UploadJsonResults(json_data, hide_results=hide_results, fail_quickly=True) return (False, 'error')
def GetFromIpInfoIO(): h = httplib2.Http(tempfile.gettempdir(), timeout=10) url = 'http://ipinfo.io/json' try: unused_resp, content = h.request(url, 'GET') data = simplejson.loads(content) return { 'region_name': data['region'], 'country_name': data['country'], 'country_code': data['country'], 'city': data['city'], 'latitude': data['loc'].split(',')[0], 'longitude': data['loc'].split(',')[1], 'source': 'ipinfo' } except: print '* Failed to use GoogleLocAPI: %s (content: %s)' % ( util.GetLastExceptionString(), content) return {}
def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None): """Make a DNS Get, returning the reply and duration it took. Args: type_string: DNS record type to query (string) record_string: DNS record name to query (string) timeout: optional timeout (float) rdataclass: optional result class (defaults to rdataclass.IN) Returns: A tuple of (response, duration in ms [float], error_msg) In the case of a DNS response timeout, the response object will be None. """ if not rdataclass: rdataclass = dns.rdataclass.IN else: rdataclass = dns.rdataclass.from_text(rdataclass) request_type = dns.rdatatype.from_text(type_string) record = dns.name.from_text(record_string, None) request = None self.request_count += 1 try: request = self.CreateRequest(record, request_type, rdataclass) except ValueError: if not request: return (None, 0, util.GetLastExceptionString()) if not timeout: timeout = self.timeout error_msg = None exc = None duration = None try: start_time = self.timer() response = self.Query(request, timeout) duration = self.timer() - start_time except (dns.exception.Timeout), exc: response = None
def GetIndexHosts(self): """Get a list of 'index' hosts for standardized testing.""" url = self.url + '/index_hosts' h = httplib2.Http(tempfile.gettempdir(), timeout=10) content = None try: unused_resp, content = h.request(url, 'GET') hosts = [] for record_type, host in simplejson.loads(content): hosts.append((str(record_type), str(host))) return hosts except simplejson.decoder.JSONDecodeError: self.msg('Failed to decode: "%s"' % content) return [] except AttributeError: self.msg('%s refused connection' % url) return [] except: self.msg('* Failed to fetch %s: %s' % (url, util.GetLastExceptionString())) return []
def GetLatestSanityChecks(): """Get the latest copy of the sanity checks config.""" h = httplib2.Http(tempfile.gettempdir(), timeout=10) http_version_usable = False use_config = None content = None try: unused_resp, content = h.request(SANITY_REFERENCE_URL, 'GET') except: print '* Unable to fetch latest reference: %s' % util.GetLastExceptionString( ) http_config = ConfigParser.ConfigParser() if content and '[base]' in content: fp = StringIO.StringIO(content) try: http_config.readfp(fp) http_version_usable = True except: pass ref_file = util.FindDataFile('config/hostname_reference.cfg') local_config = ConfigParser.ConfigParser() local_config.read(ref_file) if http_version_usable: if int(http_config.get('base', 'version')) > int( local_config.get('base', 'version')): print '- Using %s' % SANITY_REFERENCE_URL use_config = http_config if not use_config: use_config = local_config return (use_config.items('sanity'), use_config.items('sanity-secondary'), use_config.items('censorship'))
if not timeout: timeout = self.timeout error_msg = None exc = None duration = None try: start_time = self.timer() response = self.Query(request, timeout) duration = self.timer() - start_time except (dns.exception.Timeout), exc: response = None except (dns.query.BadResponse, dns.message.TrailingJunk, dns.query.UnexpectedSource), exc: error_msg = util.GetLastExceptionString() response = None # This is pretty normal if someone runs namebench offline. except socket.error: response = None if ':' in self.ip: error_msg = 'socket error: IPv6 may not be available.' else: error_msg = util.GetLastExceptionString() # Pass these exceptions up the food chain except (KeyboardInterrupt, SystemExit, SystemError), exc: raise exc except: error_msg = util.GetLastExceptionString() print "* Unusual error with %s:%s on %s: %s" % ( type_string, record_string, self, error_msg)
start_time = self.timer() response = self.Query(request, timeout) duration = self.timer() - start_time except (dns.exception.Timeout), exc: response = None except (dns.query.BadResponse, dns.message.TrailingJunk, dns.query.UnexpectedSource), exc: error_msg = util.GetLastExceptionString() response = None # This is pretty normal if someone runs namebench offline. except socket.error: response = None if ':' in self.ip: error_msg = 'socket error: IPv6 may not be available.' else: error_msg = util.GetLastExceptionString() # Pass these exceptions up the food chain except (KeyboardInterrupt, SystemExit, SystemError), exc: raise exc except: error_msg = util.GetLastExceptionString() print "* Unusual error with %s:%s on %s: %s" % (type_string, record_string, self, error_msg) response = None if not response: self.failure_count += 1 if not duration: duration = self.timer() - start_time if exc and not error_msg:
def GetReverseIp(self, ip, retries_left=2): """Request a hostname for a given IP address.""" try: print "reverse: %s -> %s" % (ip, self) answer = dns.resolver.query(dns.reversename.from_address(ip), 'PTR') except dns.resolver.NXDOMAIN: return ip except: if retries_left: print "* Failed to get hostname for %s (retries left: %s): %s" % (ip, retries_left, util.GetLastExceptionString()) return self.GetReverseIp(ip, retries_left=retries_left-1) else: return ip if answer: return answer[0].to_text().rstrip('.') else: return ip
class NameServer(health_checks.NameServerHealthChecks, provider_extensions.NameServerProvider): """Hold information about a particular nameserver.""" def __init__(self, ip, hostname=None, name=None, tags=None, provider=None, instance=None, location=None, latitude=None, longitude=None, asn=None, network_owner=None, dhcp_position=None, system_position=None): self.ip = ip self.name = name self.dhcp_position = dhcp_position self.system_position = system_position if tags: self.tags = set(tags) else: self.tags = set() self.provider = provider self.instance = instance self.location = location if self.location: self.country_code = location.split('/')[0] self.tags.add('country_%s' % self.country_code.lower()) else: self.country_code = None self.latitude = latitude self.longitude = longitude self.asn = asn self.network_owner = network_owner self._hostname = hostname self.timeout = 5 self.health_timeout = 5 self.ping_timeout = 1 self.ResetTestStatus() self._version = None self._node_ids = set() self.timer = BEST_TIMER_FUNCTION if ':' in self.ip: self.tags.add('ipv6') elif '.' in self.ip: self.tags.add('ipv4') if self.dhcp_position is not None: self.tags.add('dhcp') if self.system_position is not None: self.tags.add('system') if ip.endswith('.0') or ip.endswith('.255'): self.DisableWithMessage("IP appears to be a broadcast address.") elif self.is_bad: self.DisableWithMessage("Known bad address.") def AddNetworkTags(self, domain, provider, asn, country_code): if self.hostname: my_domain = addr_util.GetDomainFromHostname(self.hostname) hostname = self.hostname.lower() else: my_domain = 'UNKNOWN' hostname = '' if provider: provider = provider.lower() if domain and my_domain == domain: self.tags.add('isp') if asn and self.asn == asn: self.tags.add('network') if provider and 'isp' not in self.tags: if (provider in self.name.lower() or provider in self.hostname.lower() or (self.network_owner and provider in self.network_owner.lower())): self.tags.add('isp') elif provider and self.country_code == country_code and my_domain != domain: if (provider in self.name.lower() or provider in hostname or (self.network_owner and provider in self.network_owner.lower())): self.tags.add('likely-isp') def ResetTestStatus(self): """Reset testing status of this host.""" self.warnings = set() self.shared_with = set() if self.is_disabled: self.tags.remove('disabled') self.checks = [] self.failed_test_count = 0 self.share_check_count = 0 self.cache_checks = [] self.is_slower_replica = False self.ResetErrorCounts() def ResetErrorCounts(self): """NOTE: This gets called by benchmark.Run()!""" self.request_count = 0 self.failure_count = 0 self.error_map = {} @property def is_keeper(self): return bool(self.MatchesTags(['preferred', 'dhcp', 'system', 'specified'])) @property def is_bad(self): if not self.is_keeper and self.MatchesTags(['rejected', 'blacklist']): return True @property def is_hidden(self): return self.HasTag('hidden') @property def is_disabled(self): return self.HasTag('disabled') @property def check_average(self): # If we only have a ping result, sort by it. Otherwise, use all non-ping results. if len(self.checks) == 1: return self.checks[0][3] else: return util.CalculateListAverage([x[3] for x in self.checks[1:]]) @property def fastest_check_duration(self): if self.checks: return min([x[3] for x in self.checks]) else: return 0.0 @property def check_duration(self): return sum([x[3] for x in self.checks]) @property def warnings_string(self): if self.is_disabled: return 'DISABLED: %s' % self.disabled_msg else: return ', '.join(map(str, self.warnings)) @property def errors(self): return ['%s (%s requests)' % (_[0], _[1]) for _ in self.error_map.items() if _[0] != 'Timeout'] @property def error_count(self): return sum([_[1] for _ in self.error_map.items() if _[0] != 'Timeout']) @property def timeout_count(self): return self.error_map.get('Timeout', 0) @property def notes(self): """Return a list of notes about this nameserver object.""" my_notes = [] if self.system_position == 0: my_notes.append('The current preferred DNS server') elif self.system_position: my_notes.append('A backup DNS server for this system') if self.dhcp_position is not None: my_notes.append('Assigned by your network DHCP server') if self.is_failure_prone: my_notes.append('%s of %s queries failed' % (self.failure_count, self.request_count)) if self.HasTag('blacklist'): my_notes.append('BEWARE: IP appears in DNS server blacklist') if self.is_disabled: my_notes.append(self.disabled_msg) else: my_notes.extend(self.warnings) if self.errors: my_notes.extend(self.errors) return my_notes @property def hostname(self): if self._hostname is None and not self.is_disabled: self.UpdateHostname() return self._hostname def UpdateHostname(self): if not self.is_disabled: self._hostname = self.GetReverseIp(self.ip) return self._hostname @property def version(self): if self._version is None and not self.is_disabled: self.GetVersion() if not self._version: return None # Only return meaningful data if (re.search('\d', self._version) or (re.search('recursive|ns|server|bind|unbound', self._version, re.I) and 'ontact' not in self._version and '...' not in self._version)): return self._version else: return None @property def external_ips(self): """Return a set of external ips seen on this system.""" # We use a slightly different pattern here because we want to # append to our results each time this is called. self._external_ips.add(self.GetMyResolverInfoWithDuration()) # Only return non-blank entries return [x for x in self._external_ips if x] @property def node_ids(self): """Return a set of node_ids seen on this system.""" if self.is_disabled: return [] # Only return non-blank entries return [x for x in self._node_ids if x] def UpdateNodeIds(self): node_id = self.GetNodeIdWithDuration()[0] self._node_ids.add(node_id) return node_id @property def partial_node_ids(self): partials = [] for node_id in self._node_ids: node_bits = node_id.split('.') if len(node_bits) >= 3: partials.append('.'.join(node_bits[0:-2])) else: partials.append('.'.join(node_bits)) return partials @property def name_and_node(self): if self.node_ids: return '%s [%s]' % (self.name, ', '.join(self.partial_node_ids)) else: return self.name @property def is_failure_prone(self): if self.failure_rate >= FAILURE_PRONE_RATE: return True else: return False @property def failure_rate(self): if not self.failure_count or not self.request_count: return 0 else: return (float(self.failure_count) / float(self.request_count)) * 100 def __str__(self): return '%s [%s:%s]' % (self.name, self.ip, ','.join(self.tags)) def __repr__(self): return self.__str__() def HasTag(self, tag): """Matches one tag.""" return tag in self.tags def MatchesTags(self, tags): """Matches many tags.""" return self.tags.intersection(tags) def AddFailure(self, message, fatal=False): """Add a failure for this nameserver. This will effectively disable it's use.""" respect_fatal = True if self.is_keeper: max_count = MAX_KEEPER_FAILURES else: max_count = MAX_NORMAL_FAILURES self.failed_test_count += 1 if self.is_keeper: respect_fatal = False # Be quiet if this is simply a 'preferred' ipv6 host. if self.HasTag('preferred') and self.HasTag('ipv6') and len(self.checks) <= 1: self.tags.add('disabled') else: print "\n* %s failed test #%s/%s: %s" % (self, self.failed_test_count, max_count, message) if fatal and respect_fatal: self.DisableWithMessage(message) elif self.failed_test_count >= max_count: self.DisableWithMessage("Failed %s tests, last: %s" % (self.failed_test_count, message)) def AddWarning(self, message, penalty=True): """Add a warning to a host.""" if not isinstance(message, str): print "Tried to add %s to %s (not a string)" % (message, self) return None self.warnings.add(message) if penalty and len(self.warnings) >= MAX_WARNINGS: self.AddFailure('Too many warnings (%s), probably broken.' % len(self.warnings), fatal=True) def DisableWithMessage(self, message): self.tags.add('disabled') if self.is_keeper and not self.HasTag('ipv6'): print "\nDISABLING %s: %s\n" % (self, message) else: self.tags.add('hidden') self.disabled_msg = message def CreateRequest(self, record, request_type, return_type): """Function to work around any dnspython make_query quirks.""" return dns.message.make_query(record, request_type, return_type) def Query(self, request, timeout): # print "%s -> %s" % (request, self) return dns.query.udp(request, self.ip, timeout, 53) def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None): """Make a DNS Get, returning the reply and duration it took. Args: type_string: DNS record type to query (string) record_string: DNS record name to query (string) timeout: optional timeout (float) rdataclass: optional result class (defaults to rdataclass.IN) Returns: A tuple of (response, duration in ms [float], error_msg) In the case of a DNS response timeout, the response object will be None. """ if not rdataclass: rdataclass = dns.rdataclass.IN else: rdataclass = dns.rdataclass.from_text(rdataclass) request_type = dns.rdatatype.from_text(type_string) record = dns.name.from_text(record_string, None) request = None self.request_count += 1 try: request = self.CreateRequest(record, request_type, rdataclass) except ValueError: if not request: return (None, 0, util.GetLastExceptionString()) if not timeout: timeout = self.timeout error_msg = None exc = None duration = None try: start_time = self.timer() response = self.Query(request, timeout) duration = self.timer() - start_time except (dns.exception.Timeout), exc: response = None except (dns.query.BadResponse, dns.message.TrailingJunk, dns.query.UnexpectedSource), exc: error_msg = util.GetLastExceptionString() response = None