def collect(self):
uri1 = "https://atlas.ripe.net/api/v2/measurements/"
uri2 = "/latest?format=json"
uri3 = "/status-check?format=json"
uri4 = "?format=json"
r = requests.get(uri1 + msmid + uri2)
response = json.loads(
requests.get(uri1 + msmid +
uri2.format(msmid)).content.decode('UTF-8'))
chk_stats = json.loads(
requests.get(uri1 + msmid +
uri3.format(msmid)).content.decode('UTF-8'))
prb_stats = json.loads(
requests.get(uri1 + msmid +
uri4.format(msmid)).content.decode('UTF-8'))
# return error if endpoint is unreachable
if r.status_code == 404:
sys.exit("Endpoint unreachable: Error 404 ")
# DNS msm
if (msm == 'dns'):
for item in response:
af = item.get('af')
probes = item.get('prb_id')
src_addr = item.get('from')
dns_rt = DictQuery(item).get('resultset/result/rt')
dns_ip = DictQuery(item).get('resultset/dst_addr')
dns_port = DictQuery(item).get('resultset/dst_port')
dns_response_enc = DictQuery(item).get('resultset/result/abuf')
print(dns_response_enc)
dns_response_dec = str(
dns.message.from_wire(base64.b64decode(
dns_response_enc[0])))
dns_results = re.findall(r"[0-9]+(?:\.[0-9]+){1,3}",
dns_response_dec)
dns_domain = re.findall(
r"\b(?:[a-z0-9]+(?:-[a-z0-9]+)*\.)+[a-z]{2,}\b",
dns_response_dec, re.M)
# adding metric to Prometheus
try:
metric = Metric(metrics, 'DNS Measurement', 'summary')
metric.add_sample(metrics,
value=float(dns_rt[0]),
labels={
'prb_id': repr(probes),
'src_addr': src_addr,
'af': repr(af),
'dns_ip': dns_ip[0],
'dns_response': dns_results[0]
})
yield metric
pass
except None:
continue
# SSL msm
elif (msm == 'ssl'):
for item in response:
af = item.get('af')
probes = item.get('prb_id')
src_addr = item.get('from')
round_trip = item.get('rt')
check_type = item.get('type')
dst_name = item.get('dst_name')
try:
metric = Metric(metrics, 'Probes SSL', 'summary')
metric.add_sample(metrics,
value=float(round_trip),
labels={
'prb_id': repr(probes),
'src_addr': src_addr,
'dst_name': dst_name,
'af': repr(af),
'type': check_type
})
yield metric
pass
except:
continue
# Traceroute hopcount msm
elif (msm == 'traceroute'):
for item in response:
probes = item.get('prb_id')
src_addr = item.get('from')
af = item.get('af')
hops = DictQuery(item).get("result/hop")
check_type = item.get('type')
prb_id = Probe(id=probes)
try:
metric = Metric(metrics, 'Probes Traceroute Hopcount',
'summary')
metric.add_sample(metrics,
value=int(hops[-1]),
labels={
'prb_id': repr(probes),
'src_addr': src_addr,
'af': repr(af),
'type': check_type
})
yield metric
pass
except:
continue
# Ping msm
elif (msm == 'ping'):
for item in response:
af = item.get('af')
probes = item.get('prb_id')
src_addr = item.get('from')
last_latency = DictQuery(item).get("result/rtt")
check_type = item.get('type')
proto = item.get('proto')
dst_name = item.get('dst_name')
prb_id = Probe(id=probes)
try:
metric = Metric(metrics, 'Probes ID', 'summary')
metric.add_sample(metrics,
value=float(last_latency[-1]),
labels={
'prb_id': repr(probes),
'src_addr': src_addr,
'asn_v4': repr(prb_id.asn_v4),
'dst_name': dst_name,
'af': repr(af),
'proto': proto,
'type': check_type
})
yield metric
pass
except:
continue
else:
sys.exit("No measurements type defined")
def test_probe_repr_request(self):
"""Unittest for Probe representation request"""
if self.server == "":
pytest.skip("No ATLAS_SERVER defined")
Probe(id=6001)
def test_probe_repr_request(self):
"""Unittest for Probe representation request"""
if self.server == "":
raise SkipTest
Probe(id=6001)
def test_raw_renderer(self):
"""Test case where user specified report with raw rendering."""
json_results = []
for result in self.mocked_results:
json_results.append(json.dumps(result, separators=(",", ":")))
expected_output = "\n".join(json_results) + "\n"
probes = [
Probe(id=202,
meta_data={
"country_code": "GR",
"asn_v4": 3337,
"asn_v6": "4445"
}),
Probe(id=677,
meta_data={
"country_code": "DE",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=2225,
meta_data={
"country_code": "DE",
"asn_v4": 3332,
"asn_v6": "4444"
}),
Probe(id=165,
meta_data={
"country_code": "NL",
"asn_v4": 3334,
"asn_v6": "4444"
}),
Probe(id=1216,
meta_data={
"country_code": "GR",
"asn_v4": 3335,
"asn_v6": "4444"
}),
Probe(id=270,
meta_data={
"country_code": "GR",
"asn_v4": 3340,
"asn_v6": "4444"
}),
Probe(id=579,
meta_data={
"country_code": "GR",
"asn_v4": 3336,
"asn_v6": "4444"
}),
Probe(id=945,
meta_data={
"country_code": "GR",
"asn_v4": 3334,
"asn_v6": "4444"
}),
Probe(id=879,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
]
with capture_sys_output() as (stdout, stderr):
path = 'ripe.atlas.cousteau.AtlasRequest.get'
with mock.patch(path) as mock_get:
mock_get.side_effect = [(True, {
"creation_time": 1,
"start_time": 1,
"type": {
"name": "ping"
},
"description": ""
}), (True, self.mocked_results)]
mpath = 'ripe.atlas.tools.helpers.rendering.Probe.get_many'
with mock.patch(mpath) as mock_get_many:
mock_get_many.return_value = probes
self.cmd.init_args(["1", "--renderer", "raw"])
self.cmd.run()
self.assertEquals(stdout.getvalue(), expected_output)
traceroute_dict[measurement][
"target_address"] = target_address # set target geo address
probes = {
} # initialise all the probes dictionaries within this measurement
i = 0
for result in results: # get all the results of the pings from landmarks to target
print("Reading measurement data, ", measurement, "from probes ", i,
" one moment")
print("this is result ", i, "of measuremnt ", measurement)
result = TracerouteResult(result)
print(result)
if not result.is_error: # if no error in handling/parsing this result
p = result.probe_id
probe = Probe(
id=p
) # Get all the properties of the individual probe used in this individual measurement
print('Probe ', p)
a = probe.geometry[
'coordinates'] # Create a list of Coordinates
probe_x = a[0] # Probes X coordinate
probe_y = a[1] # Probes Y coordinate
probe_id = result.probe_id # get the probe_id for this individual measurement
if probe_id == '1000492':
print("HERE ITS IS *************************", probe_id,
probe_x, probe_y)
traceroute_dict[measurement][probe_id] = {}
traceroute_dict[measurement][probe_id]['probe_x'] = probe_x
traceroute_dict[measurement][probe_id]['probe_y'] = probe_y
traceroute_dict[measurement][probe_id][
'probe_asn'] = probe.asn_v4
#"start": datetime(2015, 05, 19), # just testing date filtering
#"stop": datetime(2015, 05, 20), # just testing date filtering
#"probe_ids": [1000070] # the first probe in the measurement
}
is_success, results = AtlasLatestRequest(**kwargs).create()
probes = []
measurements = []
for probe in results:
probes.append(probe['prb_id'])
# now carry out a traceroute to each probe(target) from every other probe (landmark)
print(results)
for target in results:
target_probe_id = target['prb_id']
probe = Probe(id=target_probe_id)
target_ip = probe.address_v4
t = str(target_ip)
desc = "2 Traceroute to " + t + " at " + str(target_probe_id)
print(desc)
# Remove all landmarks that dont have an public IP address.
#target_ip = None # TODO REMOVE This Safeguard before running again (dont want to create more unneccessary traceroute on ATLAS)
if target_ip == None:
continue
traceroute = Traceroute(af=4,
target=target_ip,
def test_asns_filter(self):
"""Test case where user specified probe asns filters.."""
expected_output = (
"20 bytes from probe #165 194.85.27.7 to hsi.cablecom.ch (62.2.16.24): ttl=48 times:87.825, 87.611, 91.0, \n"
"20 bytes from probe #945 92.111.237.94 to hsi.cablecom.ch (62.2.16.24): ttl=56 times:61.665, 23.833, 23.269, \n"
)
probes = [
Probe(id=202,
meta_data={
"country_code": "GR",
"asn_v4": 3337,
"asn_v6": "4445"
}),
Probe(id=677,
meta_data={
"country_code": "DE",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=2225,
meta_data={
"country_code": "DE",
"asn_v4": 3332,
"asn_v6": "4444"
}),
Probe(id=165,
meta_data={
"country_code": "NL",
"asn_v4": 3334,
"asn_v6": "4444"
}),
Probe(id=1216,
meta_data={
"country_code": "GR",
"asn_v4": 3335,
"asn_v6": "4444"
}),
Probe(id=270,
meta_data={
"country_code": "GR",
"asn_v4": 3340,
"asn_v6": "4444"
}),
Probe(id=579,
meta_data={
"country_code": "GR",
"asn_v4": 3336,
"asn_v6": "4444"
}),
Probe(id=945,
meta_data={
"country_code": "GR",
"asn_v4": 3334,
"asn_v6": "4444"
}),
Probe(id=879,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
]
with capture_sys_output() as (stdout, stderr):
path = 'ripe.atlas.cousteau.AtlasRequest.get'
with mock.patch(path) as mock_get:
mock_get.side_effect = [(True, self.mocked_results)]
mpath = 'ripe.atlas.tools.helpers.rendering.Probe.get_many'
with mock.patch(mpath) as mock_get_many:
mock_get_many.return_value = probes
self.cmd.init_args(["1", "--probe-asns", "3334"])
self.cmd.run()
self.assertEquals(stdout.getvalue(), expected_output)
def test_valid_case_with_aggr(self):
"""Test case where we have result with aggregation."""
expected_output = (
"RTT_MEDIAN: 40-50\n"
" 20 bytes from probe #202 178.190.51.206 to hsi.cablecom.ch (62.2.16.24): ttl=52 times:40.024, 40.399, 39.29, \n"
" 20 bytes from probe #677 78.128.9.202 to hsi.cablecom.ch (62.2.16.24): ttl=54 times:40.715, 40.259, 40.317, \n"
"RTT_MEDIAN: 10-20\n"
" 20 bytes from probe #2225 46.126.90.165 to hsi.cablecom.ch (62.2.16.24): ttl=56 times:10.858, 12.632, 20.53, 32.775, 47.509, 62.745, 78.54, 93.272, 109.738,\n"
"RTT_MEDIAN: 50-100\n"
" 20 bytes from probe #165 194.85.27.7 to hsi.cablecom.ch (62.2.16.24): ttl=48 times:87.825, 87.611, 91.0, \n"
"RTT_MEDIAN: 20-30\n"
" 20 bytes from probe #1216 109.190.83.40 to hsi.cablecom.ch (62.2.16.24): ttl=54 times:27.429, 25.672, 25.681, \n"
" 20 bytes from probe #270 188.192.110.111 to hsi.cablecom.ch (62.2.16.24): ttl=51 times:28.527, 26.586, 26.393, \n"
" 20 bytes from probe #579 195.88.195.170 to hsi.cablecom.ch (62.2.16.24): ttl=51 times:23.201, 22.981, 22.863, \n"
" 20 bytes from probe #945 92.111.237.94 to hsi.cablecom.ch (62.2.16.24): ttl=56 times:61.665, 23.833, 23.269, \n"
"RTT_MEDIAN: 30-40\n"
" 20 bytes from probe #879 94.254.125.2 to hsi.cablecom.ch (62.2.16.24): ttl=53 times:34.32, 34.446, 34.376, \n"
)
probes = [
Probe(id=202,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=677,
meta_data={
"country_code": "DE",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=2225,
meta_data={
"country_code": "DE",
"asn_v4": 3332,
"asn_v6": "4444"
}),
Probe(id=165,
meta_data={
"country_code": "NL",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=1216,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=270,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=579,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=945,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=879,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
]
with capture_sys_output() as (stdout, stderr):
path = 'ripe.atlas.cousteau.AtlasRequest.get'
with mock.patch(path) as mock_get:
mock_get.side_effect = [(True, self.mocked_results)]
mpath = 'ripe.atlas.tools.helpers.rendering.Probe.get_many'
with mock.patch(mpath) as mock_get_many:
mock_get_many.return_value = probes
self.cmd.init_args(["--aggregate-by", "rtt-median", "1"])
self.cmd.run()
expected_set = set(expected_output.split("\n"))
returned_set = set(stdout.getvalue().split("\n"))
self.assertEquals(returned_set, expected_set)
def test_valid_case_no_aggr(self):
"""Test case where we have result no aggregation."""
probes = [
Probe(id=202,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=677,
meta_data={
"country_code": "DE",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=2225,
meta_data={
"country_code": "DE",
"asn_v4": 3332,
"asn_v6": "4444"
}),
Probe(id=165,
meta_data={
"country_code": "NL",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=1216,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=270,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=579,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=945,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
Probe(id=879,
meta_data={
"country_code": "GR",
"asn_v4": 3333,
"asn_v6": "4444"
}),
]
with capture_sys_output() as (stdout, stderr):
path = 'ripe.atlas.cousteau.AtlasRequest.get'
with mock.patch(path) as mock_get:
mock_get.side_effect = [(True, self.mocked_results)]
mpath = 'ripe.atlas.tools.helpers.rendering.Probe.get_many'
with mock.patch(mpath) as mock_get_many:
mock_get_many.return_value = probes
self.cmd.init_args(["1"])
self.cmd.run()
self.assertEquals(stdout.getvalue(),
self.expected_output_no_aggr)
import sys
import json
import urllib
from ripe.atlas.cousteau import Probe, ProbeRequest
filters = {"country_code": "LB"}
probes = ProbeRequest(**filters)
for probe in probes:
print(probe["id"])
i = probe["id"]
p = Probe(id=i)
print(p.tags)
# Print total count of found probes
#print(probe["count"])
def runTraceroute(target, country, asn):
global noProbesASes
msms = []
probeList = []
probesids = ""
prolen = 0
filters = {"country_code": country, "asn_v4": asn}
print(filters)
probes = ProbeRequest(**filters)
for pb in probes:
probe = Probe(id=pb["id"])
lon = float(probe.geometry['coordinates'][0])
lat = float(probe.geometry['coordinates'][1])
print(pb["country_code"], country, pb["id"], pb["asn_v4"], lat, lon)
if pb["country_code"] != country:
print('Country filter didnt match: ', pb['id'], pb['country_code'],
country)
continue
try:
ip = str(ipaddress.IPv4Address(probe.address_v4))
except:
continue
probeList.append([pb["id"], lat, lon])
if len(probeList) == 0:
print('No probes found.')
return msms
selectedProbes = selectProbes(probeList)
for prbid in selectedProbes:
prolen += 1
probesids = probesids + str(prbid) + ","
#print(probesids)
if prolen == 0:
print('No probes found: ' + str(prolen))
return msms
try:
probesids = probesids[:-1]
measurement = None
#print(probesids)
if typeRun == "traceroute":
traceroute = Traceroute(
af=4,
target=target,
description="Traceroute " + str(target),
protocol="ICMP",
)
measurement = traceroute
elif typeRun == "ping":
ping = Ping(af=4, target=target, description="Ping " + str(target))
measurement = ping
else:
print('Need correct type of measurement specified.')
exit(1)
source = AtlasSource(type="probes", value=probesids, requested=prolen)
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=API_KEY_CREATE_UDM,
measurements=[measurement],
sources=[source],
is_oneoff=True)
try:
(is_success, response) = atlas_request.create()
except:
traceback.print_exc()
if is_success:
msms = response['measurements']
else:
print(response)
except:
traceback.print_exc()
print("ERROR in traceroute.")
return msms