def test_probe_ids_query_params(self):
"""Tests probe_ids as query params for different entries"""
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"probe_ids": [1, 2, 3]
})
query_filters = request.http_method_args["params"]
self.assertEqual(
query_filters["probe_ids"], "1,2,3"
)
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"probe_ids": "15, 2,3"
})
query_filters = request.http_method_args["params"]
self.assertEqual(
query_filters["probe_ids"], "15, 2,3"
)
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"probe_ids": 15
})
query_filters = request.http_method_args["params"]
self.assertEqual(
query_filters["probe_ids"], 15
)
def test_stop_time_query_params(self):
"""Tests stop time as query params for different entries"""
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"stop": "2011-11-27",
})
query_filters = request.http_method_args["params"]
self.assertEqual(query_filters["stop"], 1322352000)
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"stop": "2011-11-27 01:01",
})
query_filters = request.http_method_args["params"]
self.assertEqual(query_filters["stop"], 1322355660)
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"stop": 1322352000,
})
query_filters = request.http_method_args["params"]
self.assertEqual(query_filters["stop"], 1322352000)
request = AtlasResultsRequest(**{
"msm_id": 1000002,
"stop": datetime(2011, 11, 27)
})
query_filters = request.http_method_args["params"]
self.assertEqual(query_filters["stop"], 1322352000)
def request_results(response, timeout):
msm_data = list()
for measurement_num in response:
kwargs = {
"msm_id": measurement_num
}
# Check measurement status. Move on if measurement stopped or timeout expired.
is_success, results = AtlasStatusRequest(**kwargs).create()
while not is_success \
or results['status']['name'] == 'Ongoing' \
or (timeout is not None and
time.time() - results['start_time'] < timeout):
time.sleep(10)
is_success, results = AtlasStatusRequest(**kwargs).create()
# Get measurement data
is_success, results = AtlasResultsRequest(**kwargs).create()
while not is_success:
time.sleep(10)
is_success, results = AtlasResultsRequest(**kwargs).create()
msm_data.extend(results)
return msm_data
def create_list(kwargs):
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
l_soa = []
l_time = []
l_dt = []
l_hover = []
count = 0
while count < len(results) - 1:
my_error = DnsResult(results[count],
on_error=DnsResult.ACTION_IGNORE)
if not my_error.is_error:
timestamp = results[count]['timestamp']
dt = datetime.datetime.fromtimestamp(timestamp)
dt = dt.strftime("%m/%d/%Y , %H:%M:%S")
og_soa_serial = results[count]['result']['answers'][0][
'SERIAL']
soa_serial = str(og_soa_serial)
soa_serial = datetime.datetime.strptime(
soa_serial, "%Y%m%d%H")
soa_serial = datetime.datetime.timestamp(soa_serial)
l_soa.append(soa_serial)
l_time.append(timestamp)
l_dt.append(dt)
l_hover.append(og_soa_serial)
count += 1
return l_soa, l_time, l_dt, l_hover
def measurement_parser(kwargs):
"""
receives a ripe atlas measurement id with a start time and possibly a stop time
and returns a dictionary containing the nodes reached as keys and a list of probes that
contacted the node as values
:param kwargs:
:return dictionary keys = dns nodes, values = list of probes that contacted
the dns node:
"""
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
for result in results:
probe = result['prb_id']
try:
answer = result['result']['abuf'] + "=="
content = base64.b64decode(answer)
msg = dns.message.from_wire(content)
soa_serial = msg.answer[0].to_text().split()[
6] # if it is a soa query the soa will be in msg.answer[0]
time = result['timestamp']
response_time = (result['result']['rt'])
for opt in msg.options:
if opt.otype == dns.edns.NSID:
print(
f"{probe} -> NSID: {str(opt.data)} -> SOA: {soa_serial} : {time} -> RT: {response_time}"
)
d[str(opt.data.decode("utf-8"))].append(probe)
except:
next
return d
def get_measurement_results(measurement_ids, polling_interval):
## Retrieve measurement results
# Copy the list
pending_measurements = list(measurement_ids)
results = dict()
i = 0
logger.debug('Polling through ids')
cur_length = len(pending_measurements)
logger.info('Initial Pending Domains Length: ' + str(cur_length))
# Keep polling while there are pending results
while pending_measurements:
logger.debug('Polling iteration: ' + str(i))
for m_id in pending_measurements:
logger.debug('Polling id: ' + str(m_id))
is_success, response = AtlasResultsRequest(msm_id=m_id).create()
logger.debug('Polling success: ' + str(is_success))
logger.debug('Polling response: ' + str(response))
if is_success and response:
logger.debug('Good response...')
pending_measurements.remove(m_id)
results[m_id] = response
if len(pending_measurements) != cur_length:
cur_length = len(pending_measurements)
logger.info('New Pending Domains Length: ' + str(cur_length))
# Minor optimization to skip sleeping if unnecessary
if pending_measurements:
time.sleep(polling_interval)
i += 1
return results
def collect_measurement_results(monitoring_goal, query_type, details,
start_date, stop_date):
"""Collects measurement results from RIPE Atlas."""
msm_ids, msm_attributes = database.get_measurements(
monitoring_goal, query_type, None)
if start_date is None:
stop_date = datetime.utcnow()
start_date = stop_date - timedelta(minutes=60)
elif start_date is not None and stop_date is None:
stop_date = datetime.utcnow()
msm_results = {}
for msm_id in msm_ids:
kwargs = {"msm_id": msm_id, "start": start_date, "stop": stop_date}
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
msm_results[msm_id] = results
if len(msm_results) > 0:
if monitoring_goal == 'pubdelay' or monitoring_goal == 'propdelay':
analysis.get_state_publication_and_propagation(
msm_results, msm_attributes, start_date, stop_date, details)
else:
analysis.get_state_trust_chain(msm_results, msm_attributes,
start_date, stop_date, details)
def fetch_results(args):
kwargs = {
"msm_id": args.id,
"start": datetime.fromisoformat(args.start),
"stop": datetime.fromisoformat(args.stop),
}
return AtlasResultsRequest(**kwargs).create()
def startLive(self, endTime=None):
WINDOW = 5 * 60
currentTS = int(
(datetime.utcnow() - datetime.utcfromtimestamp(0)).total_seconds())
while endTime is None or endTime > currentTS:
try:
kwargs = {
"msm_id": 7000,
"start": datetime.utcfromtimestamp(currentTS - 2 * WINDOW),
"stop":
datetime.utcfromtimestamp((currentTS - WINDOW) - 1),
}
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
for ent in results:
timestamp = ent["timestamp"]
timestamp = timestamp * 1000 #convert to milliseconds
self.producer.send(self.topicName,
ent,
timestamp_ms=timestamp)
else:
logging.error("Fetch Failed! {}".format(kwargs))
time.sleep(WINDOW)
currentTS += WINDOW
except Exception as e:
logging.error("Error: ", e)
def dump_dns(ids):
"""
fetches and calculates and dumps to console the DNS measurements for all countries given in IDs dict
@param ids: dict of measurement ids
@type ids: dict
@return: None
"""
print("cc, num_probes, avg_rtt")
for cc, mid in ids.items():
is_success, results = AtlasResultsRequest(msm_id=mid[1]).create()
rt_list = []
if is_success:
for r in results:
try:
rt = r.get("result")["rt"]
if rt > 0:
rt_list.append(rt)
except:
pass
if len(rt_list)>0:
average = sum(rt_list) // len(rt_list)
else:
average = 0
print("%s, %d, %d" % (cc, len(rt_list), average))
def _get_connect_and_disconnect_events(self):
"""
Return all of the connect and disconnect events from Atlas.
These events are a list of dict, each of which looks like
this:
{'asn': '3292',
'controller': 'ctr-nue16',
'event': 'connect',
'msm_id': 7000,
'prb_id': 22964,
'prefix': '83.88.0.0/13',
'timestamp': 1463848675,
'type': 'connection'},
This list is sorted by timestamp.
"""
results = AtlasResultsRequest(
# 7000 is the undocumented magical measurement ID which
# returns the connect & disconnect events
msm_id=7000,
start=self.start_time,
stop=self.stop_time,
probe_ids=self.probe_ids).create()[1]
return sorted(results, key=lambda x: x['timestamp'])
def get_measurement(self, kwargs):
'''
Downloads the requested ripe measurement and returns the results.
Input:
a) kwargs: A dict containing the keys required as defined by ripe.cousteau. Supported keys are "mesm_id","start","stop" and "probe_ids".
Output:
a) results: A list containing the traceroutes as returned by ripe.atlas.cousteau.
'''
kwargs['key'] = self.ripe_key
is_success, results = AtlasResultsRequest(**kwargs).create()
if (is_success):
if (len(results)):
if (results[0]['type'] == 'traceroute'
and results[0]['af'] == 4):
return results
else:
print(
'TraIXroute only supports Traceroute and IPv4 measurements. Exiting.'
)
else:
print('Empty measurement returned. Exiting.')
else:
print('Ripe measurement not found. Exiting.')
print(
'Check also your RIPE Atlas authentication key in \'config\' file in \"ripe_auth_key\".'
)
sys.exit(0)
def get_tcp_stats(msm_id, start, stop, res):
# not doing anything with start/stop
print >> sys.stderr, "loading msm: %s" % msm_id
is_success, results = AtlasResultsRequest(msm_id=msm_id).create()
print >> sys.stderr, "loaded msm: %s (result:%s, entries:%s)" % (
msm_id, is_success, len(results))
if is_success:
for r in results:
try:
prb = r['prb_id']
res.setdefault(prb, {'tcp': []})
for rr in r['result']:
if 'result' in rr:
for rrr in rr['result']:
if 'rtt' in rrr:
res[prb]['tcp'].append(rrr['rtt'])
#print rrr
else:
res[prb]['tcp'].append(None)
#print rrr
else:
# u'result': [{u'hop': 255, u'error': u'connect failed: Network is unreachable'}]
res[prb]['tcp'].append(None)
except:
print >> sys.stderr, "err> %s" % r
else:
print >> sys.stderr, "fetch failure"
def get_results(mid):
from datetime import datetime
from ripe.atlas.cousteau import AtlasResultsRequest
kwargs = {
"msm_id": mid,
}
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
# print(results)
pass
else:
raise Exception("Unsuccessful when getting results.")
from ripe.atlas.sagan import Result
ret = {}
for r in results:
rp = Result.get(r)
print "probe_id=%d probe_ip=%s sent=%d recv=%d" % (
rp.probe_id, rp.origin, rp.packets_sent, rp.packets_received)
if rp.probe_id in ret:
raise Exception("Probe ID %d is already in result." % rp.probe_id)
else:
ret[rp.
probe_id] = False if rp.packets_sent > 0 and rp.packets_received == 0 else True
return ret
def fetch(msm_id, **kwargs):
kwargs['msm_id'] = msm_id
is_success, results = AtlasResultsRequest(**kwargs).create()
if (is_success):
if (len(results) > 0):
for l in results:
yield (l)
def _get_results(self, probe_ids):
return AtlasResultsRequest(
# 7000 is the undocumented magical measurement ID which
# returns the connect & disconnect events
msm_id=7000,
start=self.start_time,
stop=self.stop_time,
probe_ids=probe_ids).create()[1]
def fetch_result(msm):
kwargs = {'msm_id': msm['id']}
(is_success, response) = AtlasResultsRequest(**kwargs).create()
if is_success:
return msm['cc'], response
return None
def downloader(item):
print(item)
collection, startTime, endTime, probeIDs = item
mongodb = mongoClient()
msmIDs = []
files = ['data/anchorMsmIdsv4.txt', 'data/builtinMsmIdsv4.txt']
#files=['data/builtinMsmIdsv4.txt']
for file in files:
with open(file, 'r') as fp:
for line in fp:
l = int(line.rstrip('\n').split(':')[1])
msmIDs.append(l)
'''
if self.USE_STREAM:
try:
#Read Stream
atlas_stream = AtlasStream()
atlas_stream.connect()
# Probe's connection status results
channel = "result"
atlas_stream.bind_channel(channel, self.onTracerouteResponse)
#for msm in msmIDs:
#print(msm)
stream_parameters = {"msm": msm,"startTime":startTime,"endTime":endTime}
atlas_stream.start_stream(stream_type="result", **stream_parameters)
atlas_stream.timeout()
# Shut down everything
atlas_stream.disconnect()
except:
print('Unexpected Event. Quiting.')
atlas_stream.disconnect()
else:
'''
#startTime=datetime.fromtimestamp(1461911358.0)
#endTime=datetime.fromtimestamp(1461912358.5)
for msm in msmIDs:
try:
kwargs = {
"msm_id": msm,
"start": startTime,
"stop": endTime,
"probe_ids": probeIDs
}
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
if len(results) > 0:
mongodb.insertTraceroutes(collection, results)
except:
traceback.print_exc()
def getResults(test_id, start_time, stop_time, probe_list):
is_success = False
kwargs = {
"msm_id": test_id,
"start": start_time,
"stop": stop_time,
"probe_ids": probe_list
}
is_success, results = AtlasResultsRequest(**kwargs).create()
return (is_success, results)
def delta_update(m: Measurement) -> Measurement:
# Create results directory if it does not exists.
if not os.path.exists(results_dir):
os.makedirs(results_dir)
if not os.path.isfile(os.path.join(
results_dir,
str(m.mid) + ".json")) or not m.last_updated:
# Download everything
is_success, result = AtlasResultsRequest(**{"msm_id": m.mid}).create()
if is_success and len(result) > 0:
json.dump(
result,
open(os.path.join(results_dir,
str(m.mid) + ".json"), "w"))
m.last_updated = int(datetime.utcnow().timestamp())
else:
# Download and add to existing file the delta update
is_success, result = AtlasResultsRequest(
**{
"msm_id": m.mid,
"start": datetime.fromtimestamp(m.last_updated)
}).create()
print("Downloaded result length:", len(result))
if is_success and len(result) > 0:
local_result = json.load(
open(os.path.join(results_dir,
str(m.mid) + ".json"), "r"))
print("local result length: ", str(len(local_result)))
local_result.extend(result)
print("Updated result length: ", str(len(local_result)))
json.dump(
local_result,
open(os.path.join(results_dir,
str(m.mid) + ".json"), "w"))
if is_success:
m.last_updated = (int(datetime.utcnow().timestamp()))
return m
def get_resolver_info(probe_ids, measurement):
kwargs = {
"msm_id": measurement,
"start": datetime.utcnow() - timedelta(minutes=90),
# "stop": datetime(2017, 4, 2),
}
if probe_ids:
kwargs["probe_ids"] = probe_ids
is_success, results = AtlasResultsRequest(**kwargs).create()
if not is_success:
_LOGGER.error("Request failed: %s %s", probe_ids, measurement)
return []
return parse_result(results)
def _get_request(self):
kwargs = {"msm_id": self.arguments.measurement_id}
if self.arguments.probes:
kwargs["probe_ids"] = ",".join(
[str(_) for _ in self.arguments.probes])
if self.arguments.start_time:
kwargs["start"] = self.arguments.start_time
if self.arguments.stop_time:
kwargs["stop"] = self.arguments.stop_time
if "start" in kwargs or "stop" in kwargs:
return AtlasResultsRequest(**kwargs)
return AtlasLatestRequest(**kwargs)
def wait_on_measurement(measurement_id):
measurement_result = None
err = False
kwargs = {"msm_id": int(measurement_id)}
# Wait to get a result
while True:
time.sleep(10)
print("Checking measurement status...")
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success and len(results) != 0:
print(results)
break
def downloadData(start,
end,
msmId=7000,
timeWindow=timedelta(minutes=24 * 60)):
errors = []
# Get measurments results
currDate = start
while currDate + timeWindow <= end:
path = "data/%s/%s" % (currDate.year, currDate.month)
try:
print("%s: measurement id %s" % (currDate, msmId))
if not os.path.exists(path):
os.makedirs(path)
if os.path.exists("%s/%s_msmId%s.json.gz" %
(path, currDate, msmId)):
continue
kwargs = {
"msm_id": msmId,
"start": currDate,
"stop": currDate + timeWindow,
}
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
# Output file
fi = gzip.open(
"%s/%s_msmId%s.json.gz" % (path, currDate, msmId), "wb")
print("Storing data for %s measurement id %s" %
(currDate, msmId))
json.dump(results, fi)
fi.close()
else:
errors.append("%s: msmId=%s" % (currDate, msmId))
print "error: %s: msmId=%s" % (currDate, msmId)
except ValueError:
errors.append("%s: msmId=%s" % (currDate, msmId))
print "error: %s: msmId=%s" % (currDate, msmId)
finally:
currDate += timeWindow
if errors:
print("Errors with the following parameters:")
print(errors)
def run(self, measurement_id, probe_id):
"""
# self.config['<key>'] is useful for retrieving API keys, etc.
# not needed for this action, so everything is a parameter
"""
is_success, m_results = AtlasResultsRequest(
msm_id=measurement_id,
# start=start,
# stop=stop,
probe_ids=[probe_id]).create()
action_results = {"last_result": m_results[-1]}
return (is_success, action_results)
def test_url_path_and_params(self):
request = AtlasResultsRequest(
**{
"msm_id": 1000002,
"start": "2011-11-27",
"stop": "2011-11-27 01",
"probe_ids": [1, 2, 3],
}
)
self.assertEqual(request.url_path, "/api/v2/measurements/1000002/results")
query_filters = request.http_method_args["params"]
self.assertEqual(set(query_filters.keys()), set(["stop", "start", "probe_ids"]))
self.assertEqual(query_filters["start"], 1322352000)
self.assertEqual(query_filters["stop"], 1322355600)
self.assertEqual(query_filters["probe_ids"], "1,2,3")
def getASNResults(asn, testid, start_time, stop_time, probe_list):
# Get the all the measurments of interest for these probes
if len(probe_list) > 0:
kwargs = {
"msm_id": testid,
"start": start_time,
"stop": stop_time,
"probe_ids": probe_list
}
try:
is_success, results = AtlasResultsRequest(**kwargs).create()
except:
e = sys.exc_info()
print('Error AtlasResultsRequest', str(e))
return (0, 0, 0, 0, 0)
if is_success:
try:
rtt_avg_list = []
rtt_list = []
for res in results:
rtt_avg_list.append(res['avg']) # append the average to the list of averages
result_rtt_list = res['result']
for rtt in result_rtt_list:
rtt_list.append(rtt)
# Convert to numpy array
arr = numpy.array(rtt_avg_list)
rtt_average = numpy.average(arr)
rtt_mean = numpy.mean(arr)
rtt_median = numpy.median(arr)
rtt_std = numpy.std(arr)
rtt_var = numpy.var(arr)
except:
rtt_average = 0
rtt_mean = 0
rtt_median = 0
rtt_std = 0
rtt_var = 0
e = sys.exc_info()
print('Error AtlasRequestResults2', str(e))
else:
print('AtlasRequestResultsNotSuccessful')
rtt_average = 0
rtt_mean = 0
rtt_median = 0
rtt_std = 0
rtt_var = 0
rtt_list = []
return(rtt_average, rtt_mean, rtt_median, rtt_std, rtt_var, rtt_list)
def fetch_ripe_result(result_id, output_file):
# fetch the result
kwargs = {
"msm_id": result_id,
}
# while not_fetched:
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
probenumber = 1
print(len(results))
for res in results:
print(probenumber, ":vantage address: ", res['src_addr'])
probenumber += 1
with open(output_file, 'w') as outfile:
json.dump(results, outfile)
def wait_on_measurement(measurement_id):
kwargs = {"msm_id": int(measurement_id)}
# Wait to get a result
while True:
time.sleep(10)
log_message("Checking measurement status...")
is_success, results = AtlasResultsRequest(**kwargs).create()
# Return first (and should be only) result for this ID
if is_success and len(results) != 0:
return results[0], False
if not is_success:
return _, True
def startPeriod(self, startTS, endTS):
kwargs = {
"msm_id": 7000,
"start": datetime.utcfromtimestamp(startTS),
"stop": datetime.utcfromtimestamp(endTS),
}
is_success, results = AtlasResultsRequest(**kwargs).create()
if is_success:
for ent in results:
timestamp = ent["timestamp"]
timestamp = timestamp * 1000 #convert to milliseconds
self.producer.send(self.topicName, ent, timestamp_ms=timestamp)
else:
logging.error("Fetch Failed! {}".format(kwargs))