def dns_retrieval_func(collector):
allresults = dict()
probe_ids = collector.measro.get('probe_ids')
raw_file_path = collector.measro.get('raw_file_path')
all_ids = collector.measro.get('running_msm_ids')
running_msm_ids = set(deepcopy(all_ids))
attempts = defaultdict(int)
max_attempts = max([1, (collector.timeout / collector.spin_time)])
things_left = len(running_msm_ids)
while things_left > 0:
for msm_id in all_ids:
if msm_id in running_msm_ids:
attempts[msm_id] += 1
kwargs = {'msm_id': msm_id, 'probe_ids': probe_ids}
is_success, results = rac.AtlasResultsRequest(
**kwargs).create()
if attempts[msm_id] < max_attempts:
meas = rac.Measurement(id=msm_id)
if meas.status_id in range(
5, 8): # give up if there was an error
running_msm_ids.remove(
msm_id
) # we don't need to check it again if it's err'd
allresults[msm_id] = {'result': None, 'err': results}
elif len(results) == len(
probe_ids
): # finish if all results have been obtained
running_msm_ids.remove(
msm_id
) # we don't need to check it again if it's done
allresults[msm_id] = {'result': results, 'err': None}
else:
allresults[msm_id] = {'result': results, 'err': None}
else:
running_msm_ids.remove(
msm_id) # if we've made the max attempts, give up
allresults[msm_id] = {'result': results, 'err': None}
things_left = len(running_msm_ids)
if things_left > 0:
collector.time_elapsed += collector.spin_time
sleep(collector.spin_time)
# format for output to collector
res = list()
errs = list()
for msm_id in all_ids:
if msm_id in allresults:
res.append(allresults[msm_id]['result'])
errs.append(allresults[msm_id]['err'])
else:
res.append(None)
errs.append({'err': 'no result...'})
with open(raw_file_path, "w+") as f:
json.dump(res, f)
return True, res, errs
def get_meas_status(measid):
"""
:param measid: int
:return: str
"""
measurement = rac.Measurement(id=measid)
if measurement.status_id < 3:
return 'ongoing'
elif measurement.status_id == 3:
return 'done'
else:
return 'dead'
def get_ripe_measurement(measurement_id: int,
ripe_slow_down_sema: mp.Semaphore,
max_retries: int = -1):
"""Call the RIPE measurement entry point to get the ripe measurement with measurement_id"""
retries = 0
while True:
try:
ripe_slow_down_sema.acquire()
return ripe_atlas.Measurement(id=measurement_id)
except ripe_atlas_exceptions.APIResponseError:
if retries >= max_retries >= 0:
raise
logging.exception(
'Ripe get Measurement (id {}) error!'.format(measurement_id))
retries += 1
time.sleep(5 + (random.randrange(0, 500) / 100) * retries)
if retries % 5 == 0:
time.sleep(30)
def run(self):
"""Fetch the given measures and save them in the SQL table."""
self.sql = mysql.connect(
host='localhost',
database='atlas',
user='******',
password='******'
)
logger.debug("Thread fetch_results_thread started")
while True:
time.sleep(5)
self.cur1 = self.sql.cursor()
self.cur1.execute('SELECT COUNT(*) FROM measurements WHERE state="REQUESTED"')
num = self.cur1.fetchone()[0]
self.cur1.close()
logger.debug("Measurements to be fetched: {}".format(num))
if num == 0:
return;
self.cur1 = self.sql.cursor()
self.cur1.execute("SELECT msm FROM measurements WHERE state='REQUESTED'")
for i in range(self.cur1.rowcount):
msm = self.cur1.fetchone()[0]
logger.debug("Fetch metadata for {}".format(msm))
count = 0
while True:
measurement = atlas.Measurement(id=msm)
logger.debug("Current state for {}: {}".format(msm, measurement.status))
if (measurement.status == "Stopped" or
measurement.status == "Failed" or
measurement.status == "No suitable probes" or
measurement.status == "Scheduling denied"):
break
count += 1
if count % 10 == 0:
logger.debug(("Waiting for measurement {} "
"to complete "
"(current state: {})").format(msm, measurement.status))
time.sleep(5)
if (measurement.status == "Failed" or
measurement.status == "No suitable probes" or
measurement.status == "Scheduling denied"):
logger.error("Measurement from {} failed".format(msm))
self.cur2 = self.sql.cursor()
self.cur2.execute('''
UPDATE measurements
SET
state = 'FAILED'
WHERE msm = %s''', [msm])
self.sql.commit()
self.cur2.close()
continue
logger.debug("Fetch actual results for {}".format(msm))
success, response = atlas.AtlasLatestRequest(msm_id=msm).create()
if success:
logger.debug("Measure successfully fetched for {}".format(msm))
response = response[0]
if response["sent"] == 0 or response["rcvd"] == 0:
logger.error("Measurement from {} to {} failed: no ping sent or received".format(
response["src_addr"], response["dst_addr"]))
self.cur2 = self.sql.cursor()
self.cur2.execute('''
UPDATE measurements
SET
state = 'FAILED',
msm_timestamp = FROM_UNIXTIME(%s),
results_raw_json = %s
WHERE msm = %s''', [
response["timestamp"],
json.dumps(response),
msm
])
self.sql.commit()
self.cur2.close()
continue
stats = {}
stats['loss'] = (response["sent"] - response["rcvd"]) * 100. / response["sent"]
rtts = [y['rtt']
for y in response['result']
if 'rtt' in y]
if rtts:
stats['mean'] = statistics.mean(rtts)
stats['median'] = statistics.median(rtts)
stats['stdev'] = len(rtts) > 1 and statistics.stdev(rtts) or 0
stats['min'] = min(rtts)
stats['max'] = max(rtts)
self.cur2 = self.sql.cursor()
self.cur2.execute('''
UPDATE measurements
SET
state = 'FETCHED',
msm_timestamp = FROM_UNIXTIME(%s),
sent = %s,
rcvd = %s,
min = %s,
avg = %s,
mean = %s,
median = %s,
stdev = %s,
max = %s,
results_raw_json = %s
WHERE msm = %s''', [
response["timestamp"],
response["sent"],
response["rcvd"],
round(stats['min'], 2),
round(response["avg"], 2),
round(stats['mean'], 2),
round(stats['median'], 2),
round(stats['stdev'], 2),
round(stats['max'], 2),
json.dumps(response),
msm
])
self.sql.commit()
logger.debug("Results {} successfully fetched".format(msm))
else:
raise RuntimeError(
"Unable to fetch results for measure {}: {}".format(msm, response))
self.cur1.close()