def __get_measurements_for_nodes(measurement_ids: [int],
ripe_slow_down_sema: mp.Semaphore,
near_nodes: [RipeAtlasProbe],
allowed_measurement_age: int) \
-> typing.Generator[typing.Tuple[int, typing.List[RipeMeasurementResult]], None, None]:
"""Loads all results for all measurements if they are less than a year ago"""
node_dct = {}
for node in near_nodes:
node_dct[node.probe_id] = node.id
for measurement_id in measurement_ids:
allowed_start_time = int(time.time()) - allowed_measurement_age
params = {
'msm_id': measurement_id,
'start': allowed_start_time,
'probe_ids': [node.probe_id for node in near_nodes][:1000]
}
ripe_slow_down_sema.acquire()
success, result_list = ripe_atlas.AtlasResultsRequest(
**params).create()
retries = 0
while not success and retries < 5:
if not success and 'error' in result_list and 'status' in result_list[
'error'] and 'code' in result_list['error'] and result_list[
'error']['status'] == 406 and result_list['error'][
'code'] == 104:
retries = 5
break
logging.debug('AtlasResultsRequest error! {}'.format(result_list))
time.sleep(10 + (random.randrange(0, 500) / 100))
ripe_slow_down_sema.acquire()
success, result_list = ripe_atlas.AtlasResultsRequest(
**params).create()
if not success:
retries += 1
if retries > 4:
logging.error('AtlasResultsRequest error! {}'.format(result_list))
continue
measurements = []
for res in result_list:
ripe_measurement = RipeMeasurementResult.create_from_dict(res)
ripe_measurement.probe_id = node_dct[str(res['prb_id'])]
measurements.append(ripe_measurement)
yield measurement_id, measurements
def _get_measurement_response(self, measurement_id: int, ripe_slowdown_sema: mp.Semaphore,
additional_probes: ['RipeAtlasProbe']) \
-> typing.Optional[RipeMeasurementResult]:
def sleep_time(amount: float = 10):
"""Sleep for ten seconds"""
time.sleep(amount)
sleep_time(amount=360)
while True:
ripe_slowdown_sema.acquire()
res = ripe_helper.get_ripe_measurement(measurement_id,
ripe_slowdown_sema)
if res is not None:
if res.status_id == 4:
break
elif res.status_id in [6, 7]:
raise ProbeError()
elif res.status_id in [0, 1, 2]:
sleep_time()
else:
sleep_time()
ripe_slowdown_sema.acquire()
success, m_results = ripe_atlas.AtlasResultsRequest(
**{
'msm_id': measurement_id
}).create()
while not success:
logging.error('ResultRequest error {}'.format(m_results))
sleep_time(amount=(10 + (random.randrange(0, 500) / 100)))
ripe_slowdown_sema.acquire()
success, m_results = ripe_atlas.AtlasResultsRequest(
**{
'msm_id': measurement_id
}).create()
if not m_results or not isinstance(m_results, list):
return None
min_result = min(m_results, key=operator.itemgetter('min'))
measurement_result = RipeMeasurementResult.create_from_dict(min_result)
if self.probe_id == str(min_result['prb_id']):
measurement_result.probe_id = self.id
else:
probe = [
probe for probe in additional_probes
if probe.probe_id == str(min_result['prb_id'])
][0]
measurement_result.probe_id = probe.id
return measurement_result
def handle_meas(self):
useless = ['avg', 'max', 'min', 'step']
try:
msm_id = self.latest_meas_pull['id']
success, self.latest_result_pull = rac.AtlasResultsRequest(msm_id=msm_id).create()
if success:
for ind, res in enumerate(self.latest_result_pull):
if ind < self.result_ind:
print("continuing!"+str(ind)+','+str(self.result_ind))
continue
for k in useless:
try:
del res[k]
except KeyError:
print("missing key "+k)
with open(self.outf, 'a+') as f:
json.dump(res, f)
f.write("\n")
coll.insert_many(self.latest_result_pull)
with open(self.outf+'inds.json', 'w+') as f:
json.dump({'meas_ind': self.meas_ind, 'result_ind': self.result_ind}, f)
else:
print("failed on "+str(msm_id))
except Exception as e:
with open("result_pull_fails.json", 'a+') as f:
json.dump({'meas': self.latest_meas_pull, 'e': str(e)}, f)
f.write("\n")
print("failed to pull", end=" ")
print(e)
sys.stdout.flush()
def get_results(measid, probe_id=None, start=None, stop=None, fmt=None):
"""
:param measid: int
:param probe_id: int
:return: (bool, dict)
"""
kwargs = {
"msm_id": measid,
}
if probe_id is not None:
kwargs['probe_ids'] = probe_id
if start is not None:
kwargs['start'] = start
if stop is not None:
kwargs['stop'] = stop
if fmt is not None:
kwargs['format'] = fmt
is_success, results = rac.AtlasResultsRequest(**kwargs).create()
if is_success:
if len(results) > 0:
if 'result' in results[0].keys():
return is_success, df.byteify(results[0])
print results
logger.debug('failed to get results from ' + str(measid))
return False, results
def dns_retrieval_func(collector):
allresults = dict()
probe_ids = collector.measro.get('probe_ids')
raw_file_path = collector.measro.get('raw_file_path')
all_ids = collector.measro.get('running_msm_ids')
running_msm_ids = set(deepcopy(all_ids))
attempts = defaultdict(int)
max_attempts = max([1, (collector.timeout / collector.spin_time)])
things_left = len(running_msm_ids)
while things_left > 0:
for msm_id in all_ids:
if msm_id in running_msm_ids:
attempts[msm_id] += 1
kwargs = {'msm_id': msm_id, 'probe_ids': probe_ids}
is_success, results = rac.AtlasResultsRequest(
**kwargs).create()
if attempts[msm_id] < max_attempts:
meas = rac.Measurement(id=msm_id)
if meas.status_id in range(
5, 8): # give up if there was an error
running_msm_ids.remove(
msm_id
) # we don't need to check it again if it's err'd
allresults[msm_id] = {'result': None, 'err': results}
elif len(results) == len(
probe_ids
): # finish if all results have been obtained
running_msm_ids.remove(
msm_id
) # we don't need to check it again if it's done
allresults[msm_id] = {'result': results, 'err': None}
else:
allresults[msm_id] = {'result': results, 'err': None}
else:
running_msm_ids.remove(
msm_id) # if we've made the max attempts, give up
allresults[msm_id] = {'result': results, 'err': None}
things_left = len(running_msm_ids)
if things_left > 0:
collector.time_elapsed += collector.spin_time
sleep(collector.spin_time)
# format for output to collector
res = list()
errs = list()
for msm_id in all_ids:
if msm_id in allresults:
res.append(allresults[msm_id]['result'])
errs.append(allresults[msm_id]['err'])
else:
res.append(None)
errs.append({'err': 'no result...'})
with open(raw_file_path, "w+") as f:
json.dump(res, f)
return True, res, errs