def record_failure(self, e,): stats.failures += 1 ndb.report_metric('ServiceScanFailure', str(e)) if (isinstance(e, SSLScanTimeoutException)): stats.failure_timeouts += 1 return if (isinstance(e, SSLAlertException)): stats.failure_ssl_alert += 1 return if (isinstance(e, ValueError)): stats.failure_other += 1 return err = self.get_errno(e) if err == errno.ECONNREFUSED or err == errno.EINVAL: stats.failure_conn_refused += 1 elif err == errno.EHOSTUNREACH or err == errno.ENETUNREACH: stats.failure_no_route += 1 elif err == errno.ECONNRESET: stats.failure_conn_reset += 1 elif err == -2 or err == -3 or err == -5 or err == 8: stats.failure_dns += 1 else: stats.failure_other += 1 logging.error("Unknown error scanning '%s'\n" % self.sid) traceback.print_exc(file=sys.stdout)
def report_observation_with_db(ndb, service, fp): """Insert or update an Observation record in the notary database.""" cur_time = int(time.time()) obs = ndb.get_observations(service) most_recent_time_by_key = {} most_recent_key = None most_recent_time = 0 # calculate the most recently seen key for (service, key, start, end) in obs: if key not in most_recent_time_by_key or end > most_recent_time_by_key[key]: most_recent_time_by_key[key] = end for k in most_recent_time_by_key: if most_recent_time_by_key[k] > most_recent_time: most_recent_key = k most_recent_time = most_recent_time_by_key[k] ndb.close_session() if most_recent_key == fp: # this key matches the most recently seen key before this observation. # just update the observation 'end' time. ndb.update_observation_end_time(service, fp, most_recent_time, cur_time) ndb.report_metric('ServiceScanKeyUpdated', service) else: # the key has changed or no observations exist yet for this service. # add a new entry for this key with start and end set to the current time ndb.insert_observation(service, fp, cur_time, cur_time) if most_recent_key != None: # if there was a previous key, set its 'end' timespan value to be # the current time minus one second (ending just before the new key) ndb.update_observation_end_time(service, most_recent_key, most_recent_time, cur_time -1) ndb.report_metric('ServiceScanPrevKeyUpdated', service)
def record_failure(self, e): stats.failures += 1 ndb.report_metric("ServiceScanFailure", str(e)) if type(e) == type(self.timeout_exc): stats.failure_timeouts += 1 return if type(e) == type(self.alert_exc): stats.failure_ssl_alert += 1 return err = self.get_errno(e) if err == errno.ECONNREFUSED or err == errno.EINVAL: stats.failure_conn_refused += 1 elif err == errno.EHOSTUNREACH or err == errno.ENETUNREACH: stats.failure_no_route += 1 elif err == errno.ECONNRESET: stats.failure_conn_reset += 1 elif err == -2 or err == -3 or err == -5 or err == 8: stats.failure_dns += 1 else: stats.failure_other += 1 print "Unknown error scanning '%s'\n" % self.sid traceback.print_exc(file=sys.stdout)
res_list = [] stats = GlobalStats() rate = args.scans timeout_sec = args.timeout f = args.service_id_file start_time = time.time() localtime = time.asctime(time.localtime(start_time)) # read all service names to start; # otherwise the database can lock up # if we're accepting data piped from another process all_sids = [line.rstrip() for line in f] print "Starting scan of %s service-ids at: %s" % (len(all_sids), localtime) print "INFO: *** Timeout = %s sec Scans-per-second = %s" % (timeout_sec, rate) ndb.report_metric("ServiceScanStart", "ServiceCount: " + str(len(all_sids))) for sid in all_sids: try: # ignore non SSL services # TODO: use a regex instead if sid.split(",")[1] == notary_common.SSL_TYPE: stats.num_started += 1 t = ScanThread(sid, stats, timeout_sec) t.start() if (stats.num_started % rate) == 0: time.sleep(1) record_observations_in_db(res_list) res_list = [] so_far = int(time.time() - start_time)
stats = GlobalStats() rate = args.scans timeout_sec = args.timeout f = args.service_id_file start_time = time.time() localtime = time.asctime( time.localtime(start_time) ) # read all service names to start; # otherwise the database can lock up # if we're accepting data piped from another process all_sids = [ line.rstrip() for line in f ] print "Starting scan of %s service-ids at: %s" % (len(all_sids), localtime) print "INFO: *** Timeout = %s sec Scans-per-second = %s" % \ (timeout_sec, rate) ndb.report_metric('ServiceScanStart', "ServiceCount: " + str(len(all_sids))) for sid in all_sids: try: # ignore non SSL services # TODO: use a regex instead if sid.split(",")[1] == notary_common.SSL_TYPE: stats.num_started += 1 t = ScanThread(sid,stats,timeout_sec,args.sni) t.start() if (stats.num_started % rate) == 0: time.sleep(1) record_observations_in_db(res_list) res_list = [] so_far = int(time.time() - start_time)