def main():
try:
target = sys.argv[1]
except:
print('Please provide a host to traceroute')
exit(1)
probes = load_json()
sources = AtlasSource(
type='probes',
value=','.join(str(x) for x in random.sample(probes, num_probes)),
requested=num_probes)
tr4 = Traceroute(af=4,
target=target,
description='Transient tr4 test',
protocol='ICMP')
tr6 = Traceroute(af=6,
target=target,
description='Transient tr6 test',
protocol='ICMP')
atlas_request = AtlasCreateRequest(key=API_KEY,
measurements=[tr4, tr6],
sources=[sources],
is_oneoff=True)
response = atlas_request.create()
print(response)
def run_traceroute(probe_id,destination_ip, ip_version):
global q
af = 4 if ip_version == 'v4' else 6
traceroute = Traceroute(
af=af,
target=destination_ip,
description="testing",
protocol="ICMP",
)
source = AtlasSource(type="probes", value=probe_id, requested=1)
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True
)
(is_success, response) = atlas_request.create()
if not is_success:
raise Exception('Error creating the measurement.')
__fetch_result(response['measurements'][0])
return q.get()
def create_measurements(probes, target):
'''
Create two new ICMP traceroute measurements, one for the IPv4 target address
and one for the IPv6.
'''
probes.remove(target['id']) # Don't waste credits tracerouting to itself
num_probes = len(probes)
sources = AtlasSource(
type='probes',
value=','.join(str(x) for x in random.sample(probes, num_probes)),
requested=num_probes)
tr4 = Traceroute(af=4,
target=target['address_v4'],
description='Transient tr4 test',
protocol='ICMP')
tr6 = Traceroute(af=6,
target=target['address_v6'],
description='Transient tr6 test',
protocol='ICMP')
atlas_request = AtlasCreateRequest(key=API_KEY,
measurements=[tr4, tr6],
sources=[sources],
is_oneoff=True)
if dry_run:
response = False
else:
response = atlas_request.create()
return response
def _create_measurement_impl(self, target_probe):
args = self._args
log = self._log
probe = target_probe
if 'address_v4' not in probe:
log.error('probe has no ipv4 addr')
return None
target_id = probe['id']
target_ip = probe['address_v4']
log.notice('Creating measurement to probe', target_id, target_ip)
desc = '{} {} to {}'.format(args.test_name_prefix, args.src_probe,
target_id)
ping = Ping(af=4, target=target_ip, description=desc)
source = AtlasSource(type='probes',
value='{}'.format(args.src_probe),
requested=1)
req = AtlasCreateRequest(start_time=datetime.utcnow(),
key=args.api,
measurements=[ping],
sources=[source],
is_oneoff=True)
is_success, resp = req.create()
if not is_success:
log.warn('Error creating measurement:', str(resp))
return None
else:
msm_id = resp['measurements'][0]
return msm_id
log(is_success, resp)
def startTraceroute(self, query_params):
# localhost:8080/api/v1/traceroute?dest=X.X.X.X&desc=description&proto=TCP
dest = query_params["dest"][0]
desc = query_params["desc"][0]
proto = query_params["proto"][0]
traceroute = Traceroute(
af=4,
target=dest,
description=desc,
protocol=proto,
)
source = AtlasSource(type="area",
value="WW",
requested=5,
tags={"include": ["system-ipv4-works"]})
start_time = datetime.utcnow() + timedelta(0, 2)
atlas_request = AtlasCreateRequest(start_time=start_time,
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
print(response)
print("Success ", is_success)
self.wfile.write(json.dumps(response, ensure_ascii=False).encode())
def create_measurement(cc):
"""
Creates DNS and Traceroute measurements for given country code
@param cc: Two characters (ISO) country code
@type cc: str
@return: tuple of measurement ids
@rtype: tuple
"""
global API_CREATE_KEY
traceroute = Traceroute(af=4, target="193.0.14.129", protocol="ICMP",
description="Traceroute from %s to K Root" % cc)
dns = Dns(af=4, target="193.0.14.129", query_argument="com.", query_type="NS", query_class="IN",
description="DNS Response time fromt %s to K Root" % cc)
source = AtlasSource(value=cc,
requested=50,
type="country",
action="add")
request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=API_CREATE_KEY,
measurements=[traceroute, dns],
sources=[source],
is_oneoff=True)
(is_success, response) = request.create()
if is_success:
print("- Created measurement for %s" % cc)
return list(response['measurements'])
else:
print("- Failed to create measurement for %s: %s" % (cc, response))
return None
def create_ripe_measurement(prefix_list):
measurement_count = 0
measurement_limit = 10
ATLAS_API_KEY = "secret"
for prefix, ip_list in prefix_list.iteritems():
for ip in ip_list:
ipAddr = ip
count = 0
descr = "Prefix: " + prefix + "Flapped " + str(count) + " times"
ping = Ping(af=4, target=ipAddr, description=descr)
traceroute = Traceroute(
af=4,
target=ipAddr,
description=descr,
protocol="ICMP",
)
source = AtlasSource(type="area", value="WW", requested=5)
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=[ping, traceroute],
sources=[source],
is_oneoff=True
)
(is_success, response) = atlas_request.create()
measurement_count += 1
if measurement_count > measurement_limit:
break
def test_create_request():
"""Unittest for Atlas create request"""
raise SkipTest("Skip create request")
source = AtlasSource(**{"type": "area", "value": "WW", "requested": 38})
ping = Ping(**{
"target": "www.google.fr",
"af": 4,
"description": "testing",
"prefer_anchors": True
})
dns = Dns(**{
"target": "k.root-servers.net", "af": 4,
"description": "testing new wrapper", "query_type": "SOA",
"query_class": "IN", "query_argument": "nl", "retry": 6
})
stop = datetime.utcnow() + timedelta(minutes=220)
request = AtlasCreateRequest(
**{
"stop_time": stop,
"key": "testing",
"server": "testing",
"measurements": [ping, dns],
"sources": [source]
}
)
result = namedtuple('Result', 'success response')
(result.success, result.response) = request.create()
assert (result.success)
def submit_traceroute(asn):
traceroute_time = int(time.time())
traceroute = Traceroute(
af=4,
target="208.45.214.0",
description="{}_{}".format(asn, traceroute_time),
protocol="ICMP",
)
source = AtlasSource(type="asn", value=asn, requested=1)
atlas_request = AtlasCreateRequest(start_time=datetime.datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
if not is_success:
return None, traceroute_time
measurement_id = response["measurements"][0]
log_message(
"Measurement ID of {} for ongoing traceroute".format(measurement_id))
return measurement_id, traceroute_time
def initRequest(country, source, m):
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=m,
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
print(str(country) + ": " + str(response))
def create_measurement(tr_list, source_list):
ATLAS_API_KEY = "e2bb393f-b198-4bd0-96e3-fb5a5bc7fa22"
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=tr_list,
sources=source_list,
is_oneoff=True)
(is_success, response) = atlas_request.create()
print is_success
print response
def send_ripe_measurement_request(website, ATLAS_API_KEY):
traceroute1 = Traceroute(
af=4,
target=website,
description="testing",
protocol="ICMP",
)
source1 = AtlasSource(
type="country",
value="CA", #, canada "CA", "PL", "AU", "AR"],
requested=3,
tags={"include": ["system-ipv4-works"]})
source2 = AtlasSource(
type="country",
value="ZA", #, South Africa "CA", "PL", "AU", "AR"],
requested=3,
tags={"include": ["system-ipv4-works"]})
source3 = AtlasSource(
type="country",
value="DE", #, Germany "CA", "PL", "AU", "AR"],
requested=3,
tags={"include": ["system-ipv4-works"]})
source4 = AtlasSource(
type="country",
value="NG", #, Nigeria "CA", "PL", "AU", "AR"],
requested=3,
tags={"include": ["system-ipv4-works"]})
source5 = AtlasSource(
type="country",
value="BR", #, Brazil "CA", "PL", "AU", "AR"],
requested=3,
tags={"include": ["system-ipv4-works"]})
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=[traceroute1],
sources=[source1, source2, source3, source4, source5],
is_oneoff=True)
is_success, response = atlas_request.create()
result_id = []
if is_success:
result_id = response['measurements']
print("Measurement for", website, "sent. Result ID is", result_id[0])
return result_id
else:
print("Measurement not successful")
return []
def getTracerouteResults(self, query_params):
dest = query_params["dest"][0]
myresults = getMyResults()
id = 0
for result in myresults["results"]:
target = result["target"]
if target == dest:
id = result["id"]
if id == 0:
# start a new measurement
traceroute = Traceroute(
af=4,
target=dest,
description="auto traceroute to %s" % dest,
protocol="TCP",
)
source = AtlasSource(type="area",
value="WW",
requested=5,
tags={"include": ["system-ipv4-works"]})
start_time = datetime.utcnow() + timedelta(0, 1)
atlas_request = AtlasCreateRequest(start_time=start_time,
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
if is_success:
id = response['measurements'][0]
# {'measurements': [23976423, 23976424]}
print("Run traceroute back for id %d" % id)
rt = RunReverseTraceroute(id)
rt.start()
rt.join(timeout=60)
responses = get_json(id)
# source = "https://atlas.ripe.net/api/v2/measurements/%d/results/"%id
# responses = requests.get(source).json()
# for i in responses:
# dst_addr = i['dst_addr']
# i['dst_addr'] = self.getCity(dst_addr)
# i['from'] = self.getCity(i['from'])
# results = i['result']
# for j in results:
# for k in j['result']:
# if k.get('from'):
# k['from'] = self.getCity(k['from'])
self.wfile.write(json.dumps(responses, ensure_ascii=False).encode())
def measure(msm_spec):
traceroute = Traceroute(**msm_spec['definitions'])
source = AtlasSource(**msm_spec['probes'])
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
return response['measurements'][0]
def oneofftrace(probes_def, dst, **kwargs):
probe_list = []
if isinstance(probes_def, int):
probe_list.append(probes_def)
elif isinstance(probes_def, list):
probe_list = probes_def
else:
raise ValueError(
"Probes definition needs to be of type int or list, not %s" %
(type(probes_def)))
default_defs = {
'target': dst,
'type': 'traceroute',
'protocol': 'ICMP',
'resolve_on_probe': True,
'is_oneoff': True
}
defs = dict(default_defs.items() + kwargs.items())
# handle 'af'
if not 'af' in defs:
if ':' in dst:
defs['af'] = 6
else: #default to 4
defs['af'] = 4
# handle 'descr'
if not 'description' in defs:
defs['description'] = 'trace to %s (IPv%d)' % (dst, defs['af'])
data = {
'definitions': defs,
'probes': {
'requested': len(probe_list),
'type': 'probes',
'value': ','.join(map(str, probe_list))
}
}
traceroute = Traceroute(**data['definitions'])
source = AtlasSource(**data['probes'])
atlas_request = AtlasCreateRequest(
# start_time = datetime.utcnow(), # this was generating "Measurements must have the start time in future"
key=KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
if 'measurements' in response:
return response['measurements'][0]
else:
print >> sys.stderr, response
def submit_traceroute(asn):
global PREFIX
global PEERING
traceroute_time = int(time.time())
prefix_no_subnet = str(PREFIX.split('/')[0])
if PEERING:
split_prefix = prefix_no_subnet.split('.')
prefix_no_subnet = "{}.{}.{}.1".format(split_prefix[0],
split_prefix[1],
split_prefix[2])
traceroute = Traceroute(af=4,
target=prefix_no_subnet,
description="{}_{}_{}".format(
asn, traceroute_time,
"{}".format(prefix_no_subnet)),
protocol="ICMP",
max_hops=255)
source = AtlasSource(
type="asn",
value=str(asn),
requested=1,
tags={"include": ["system-ipv4-works", "system-ipv4-stable-1d"]})
atlas_request = AtlasCreateRequest(start_time=datetime.datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
if not is_success:
response = str(response)
log_message("Traceroute failed: {}".format(response))
# Handle probe not found in ASN
if "Your selected ASN is not covered by our network." in response:
return -1, traceroute_time
# Handle all other cases
else:
return None, traceroute_time
measurement_id = response["measurements"][0]
log_message(
"Measurement ID of {} for ongoing traceroute".format(measurement_id))
return measurement_id, traceroute_time
def create_request(self,
measurements,
source,
is_oneoff=True,
packet_interval=5000):
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=self.atlas_api_key,
measurements=measurements,
sources=[source],
packet_interval=packet_interval,
is_oneoff=is_oneoff)
(is_success, response) = atlas_request.create()
return is_success, response
def create_measurement(self, kwargs):
'''
Creates the request for a measurement to ripe.
Input:
a) kwargs: The user input arguments in a dictionary.
Output:
a) response: The response from ripe.
'''
default_defs = {
'target': 'www.inspire.edu.gr',
'type': 'traceroute',
'protocol': 'ICMP',
'resolve_on_probe': True,
'af': 4,
'description': 'traIXroute (IPv4)'
}
default_probes = {
'type': 'probes',
'value': '23385',
'requested': 1
}
default_defs.update(kwargs[0].items())
default_probes.update(kwargs[1].items())
if(default_defs['type'] != 'traceroute' or default_defs['af'] != 4):
print('TraIXroute only supports Traceroute and IPv4 measurements. Exiting.')
traceroute = Traceroute(** default_defs)
source = AtlasSource(** default_probes)
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=self.ripe_key,
measurements=[traceroute],
sources=[source],
is_oneoff=True
)
(is_success, response) = atlas_request.create()
if(is_success):
print('Please wait for ripe to complete the measurements and run traIXroute again with: ripe -r \'{\"msm_id\":' + str(
response['measurements'][0]) + '}\'')
else:
print('Please revise the arguments you have given. Ripe does not accept these arguments or it may be unavailable at this time.')
sys.exit(0)
def atlas_api_call():
#ATLAS_API_KEY = ""
q = Target.objects.all().last()
if (q.specification.one_off is True):
traceroute = Traceroute(
af=4,
target=str(q.target),
description=str(q.description),
protocol=str(q.specification.protocol),
)
else:
traceroute = Traceroute(
af=4,
target=str(q.target),
description=str(q.description),
protocol=str(q.specification.protocol),
interval=int(q.specification.interval),
)
source = []
for i in range(0, q.probes.all().count()):
l = q.probes.all()[i]
#new[i]='source'+i
source.append(
AtlasSource(type="country",
value=str(l.country),
requested=int(l.number),
tags={"exclude": ["system-anchor"]}))
if (q.specification.one_off is True):
atlas_request = AtlasCreateRequest(start_time=int(time.time()),
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=source,
is_oneoff=q.specification.one_off)
else:
atlas_request = AtlasCreateRequest(
start_time=int(time.time()),
stop_time=int(time.mktime(q.specification.stop_time.timetuple())),
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=source,
is_oneoff=q.specification.one_off)
(is_success, response) = atlas_request.create()
print response
return response
def create_measurement(self, kwargs):
'''
Creates the request for a measurement to ripe.
Input:
a) kwargs: The user input arguments in a dictionary.
Output:
a) response: The response from ripe.
'''
default_defs = {
'target': 'www.inspire.edu.gr',
'type': 'traceroute',
'protocol': 'ICMP',
'resolve_on_probe': True,
'af': 4,
'description': 'traIXroute (IPv4)'
}
default_probes = {'type': 'probes', 'value': '23385', 'requested': 1}
default_defs.update(kwargs[0].items())
default_probes.update(kwargs[1].items())
if (default_defs['type'] != 'traceroute' or default_defs['af'] != 4):
print(
'TraIXroute only supports Traceroute and IPv4 measurements. Exiting.'
)
traceroute = Traceroute(**default_defs)
source = AtlasSource(**default_probes)
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=self.ripe_key,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
if (is_success):
print(
'Please wait for ripe to complete the measurements and run traIXroute again with: ripe -r \'{\"msm_id\":'
+ str(response['measurements'][0]) + '}\'')
else:
print(
'Please revise the arguments you have given. Ripe does not accept these arguments or it may be unavailable at this time.'
)
sys.exit(0)
def setUp(self):
self.create_source = AtlasSource(
**{"type": "area", "value": "WW", "requested": 3}
)
self.measurement = Ping(**{
"target": "testing", "af": 6,
"description": "testing"
})
self.request = AtlasCreateRequest(**{
"start_time": datetime(2015, 10, 16),
"stop_time": 1445040000,
"key": "path_to_key",
"measurements": [self.measurement],
"sources": [self.create_source],
"is_oneoff": True,
})
class TestAtlasRequestCustomHeaders(TestCase):
def setUp(self):
self.create_source = AtlasSource(
**{"type": "area", "value": "WW", "requested": 3}
)
self.measurement = Ping(**{
"target": "testing", "af": 6,
"description": "testing"
})
self.request = AtlasCreateRequest(**{
"start_time": datetime(2015, 10, 16),
"stop_time": 1445040000,
"key": "path_to_key",
"measurements": [self.measurement],
"sources": [self.create_source],
"is_oneoff": True,
"headers": {"hello": "world"},
})
def test_custom_headers(self):
expected_headers = {
"Content-Type": "application/json",
"hello": "world",
"Accept": "application/json",
"User-Agent": "RIPE ATLAS Cousteau v{0}".format(__version__)
}
self.assertEqual(self.request.get_headers(), expected_headers)
def run_measurement(tgt):
m = [
Ping(af=4, target=addr, description="ROV tgt %d" % mid)
for (mid, addr) in enumerate(tgt)
]
source = AtlasSource(type="area", value="WW", requested=1000)
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=m,
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
print "tgt=%s is_success=%s response=%s" % (str(tgt), str(is_success),
str(response))
def measure_from_template( template_file, template_vars ):
env = Environment()
template = env.get_template( template_file )
msm_spec = template.render(**{template_vars})
traceroute = Traceroute( ** msm_spec['definitions'] )
source = AtlasSource( ** msm_spec['probes'] )
atlas_request = AtlasCreateRequest(
start_time = datetime.utcnow(),
key = KEY,
measurements = [traceroute],
sources = [source],
is_oneoff = True
)
(is_success, response) = atlas_request.create()
return response['measurements'][0]
def submit(self, key):
# create the measurements
start_time = datetime.utcnow() + timedelta(0, 0, 0, 0,
1) # start in a minute
self.request = AtlasCreateRequest(
key=key,
measurements=self.measurements,
sources=self.probe_sources,
# start_time = start_time,
is_oneoff=self.is_oneoff)
(is_success, response) = self.request.create()
if is_success:
self.measurement_ids.extend(response["measurements"])
print("SUCCESS: measurement created: %s" %
response["measurements"])
else:
raise Exception("failed to create measurement: %s" % response)
def create(self):
creation_class = self.CREATION_CLASSES[self.arguments.type]
return AtlasCreateRequest(
server=conf["ripe-ncc"]["endpoint"].replace("https://", ""),
key=self.arguments.auth,
measurements=[creation_class(**self._get_measurement_kwargs())],
sources=[AtlasSource(**self._get_source_kwargs())],
is_oneoff=self._is_oneoff).create()
def ping(self, source):
ping = Ping(af=4, target=self.target, description="RTT measurement")
# add 1 second to make sure time in request is not in the past
atlas_request = AtlasCreateRequest(start_time=datetime.utcnow() +
timedelta(seconds=1),
key=ATLAS_API_KEY,
measurements=[ping],
sources=[source],
is_oneoff=True)
is_success, response = atlas_request.create()
if not is_success:
logging.warning(response)
return [] # return empty list representing no results
msm_id = response['measurements'][0]
return self.wait_for_all_results(msm_id, source.requested)
def submit_traceroute(asn, probe):
global PREFIX
traceroute_time = int(time.time())
prefix_no_subnet = str(PREFIX.split('/')[0])
o1, o2, o3, o4 = prefix_no_subnet.split('.')
prefix_loopback = "{}.{}.{}.1".format(o1, o2, o3)
traceroute = Traceroute(af=4,
target=prefix_loopback,
description="{}_{}_{}".format(
asn, traceroute_time,
"{}".format(prefix_no_subnet)),
protocol="TCP",
port=39876,
max_hops=60)
source = AtlasSource(type="probes", value=str(probe), requested=1)
atlas_request = AtlasCreateRequest(start_time=datetime.datetime.utcnow(),
key=ATLAS_API_KEY,
measurements=[traceroute],
sources=[source],
is_oneoff=True)
(is_success, response) = atlas_request.create()
if not is_success:
response = str(response)
log_message("Traceroute failed: {}".format(response))
# Handle probe not found in ASN
if "Your selected ASN is not covered by our network." in response:
return -1, traceroute_time
# Handle all other cases
else:
return None, traceroute_time
measurement_id = response["measurements"][0]
log_message(
"Measurement ID of {} for ongoing traceroute".format(measurement_id))
return measurement_id, traceroute_time
def schedule_measurement(dest, probes, tag, stage):
msm_status = defaultdict(list)
traceroute = Traceroute(
af=4,
target=dest,
description="%s_%s: Traceroute to %s" % (tag, stage, dest),
protocol="ICMP",
is_public=True,
paris=16,
can_visualize=False,
)
trace_origin = AtlasSource(
requested=len(probes),
type="probes",
value=",".join(probes)
)
try:
atlas_req = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=authkey,
measurements=[traceroute],
sources=[trace_origin],
is_oneoff=True
)
(is_success, response) = atlas_req.create()
# Check if we got a positive result
if is_success:
print response
for m in response['measurements']:
msm_status['list'].append(m)
else:
msm_status['failed'].append(dest)
# This sleep is important to give time to the scheduled measurements to complete before trying more.
time.sleep(3)
except (urllib2.HTTPError, urllib2.URLError) as e:
# Other kind of error
msm_status['failed'].append(dest)
print "Fatal Error: %s " % e
return msm_status
def test_create_delete_request(self):
"""Unittest for Atlas create and delete request"""
if self.server == "":
raise SkipTest
source = AtlasSource(**{"type": "area", "value": "WW", "requested": 38})
ping = Ping(**{
"target": "www.google.fr",
"af": 4,
"description": "testing",
"prefer_anchors": True
})
dns = Dns(**{
"target": "k.root-servers.net", "af": 4,
"description": "testing new wrapper", "query_type": "SOA",
"query_class": "IN", "query_argument": "nl", "retry": 6
})
stop = datetime.utcnow() + timedelta(minutes=220)
request = AtlasCreateRequest(
**{
"stop_time": stop,
"key": self.create_key,
"server": self.server,
"measurements": [ping, dns],
"sources": [source]
}
)
result = namedtuple('Result', 'success response')
(result.success, result.response) = request.create()
self.delete_msm = result.response["measurements"][0]
print result.response
self.assertTrue(result.success)
# Unittest for Atlas delete request
if self.server == "":
raise SkipTest
kwargs = {"msm_id": self.delete_msm, "key": self.delete_key, "server": self.server}
request = AtlasStopRequest(**kwargs)
result = namedtuple('Result', 'success response')
(result.success, result.response) = request.create()
print result.response
self.assertTrue(result.success)
class TestAtlasCreateRequest(unittest.TestCase):
def setUp(self):
create_source = AtlasSource(
**{"type": "area", "value": "WW", "requested": 3}
)
measurement = Ping(**{
"target": "testing", "af": 6,
"description": "testing"
})
self.request = AtlasCreateRequest(**{
"start_time": datetime.utcnow(),
"stop_time": datetime.utcnow() + timedelta(hours=2),
"key": "path_to_key",
"measurements": [measurement],
"sources": [create_source]
})
def test_construct_post_data(self):
self.request._construct_post_data()
validate(self.request.post_data, post_data_create_schema)
def measure(port, hostname, afs):
for af in afs:
if (hostname, port, af) in msm_state:
msm_state[(hostname, port, af)]['state'] = 'active'
else:
# the mapping between API and cousteau drives me nuts!
msm_def['definitions'][0]['port'] = port
msm_def['definitions'][0]['target'] = hostname
msm_def['definitions'][0]['af'] = af
traceroute = Traceroute(**msm_def['definitions'][0])
source = AtlasSource(**msm_def['probes'][0])
start = int(time.time()) + 60
atlas_request = AtlasCreateRequest(key=KEY,
measurements=[traceroute],
sources=[source],
start_time=start)
(is_success, response) = atlas_request.create()
msm_id = None
if is_success:
msm_id = response['measurements'][0]
json.dump(
{
'hostname': hostname,
'port': port,
'af': af,
'action': 'started',
'ts': start,
'msm_id': msm_id
}, log_fh)
log_fh.write("\n")
msm_state[(hostname, port, af)] = {
'msm_id': msm_id,
'state': 'new'
}
print("new msm created for %s %s %s (id: %s)" %
(hostname, port, af, msm_id),
file=sys.stderr)
else:
print("msm start failed for %s %s %s: %s" %
(hostname, port, af, response),
file=sys.stderr)
def test_post_method_with_bill_to(self):
"""Tests POST reuest method without any time specified"""
request = AtlasCreateRequest(**{
"key": "path_to_key",
"measurements": [self.measurement],
"sources": [self.create_source],
"bill_to": "billing@address"
})
self.maxDiff = None
expected_args = {
"json": {
"definitions": [{
"af": 6, "description": "testing",
"target": "testing", "type": "ping"
}],
"is_oneoff": False,
"probes": [{"requested": 3, "type": "area", "value": "WW"}],
"bill_to": "billing@address",
},
"params": {"key": "path_to_key"},
"verify": True,
"headers": {
"User-Agent": "RIPE ATLAS Cousteau v{0}".format(__version__),
"Content-Type": "application/json",
"Accept": "application/json"
},
"proxies": {},
}
with mock.patch("ripe.atlas.cousteau.request.AtlasRequest.http_method") as mock_get:
request._construct_post_data()
mock_get.return_value = True
request.post()
self.assertEqual(request.http_method_args, expected_args)
def schedule_measurement(dest, probes, tag, stage):
msm_status = defaultdict(list)
traceroute = Traceroute(
af=4,
target=dest,
description="%s_%s: Traceroute to %s" % (tag, stage, dest),
protocol="ICMP",
is_public=True,
paris=16,
can_visualize=False,
)
trace_origin = AtlasSource(requested=len(probes),
type="probes",
value=",".join(probes))
try:
atlas_req = AtlasCreateRequest(start_time=datetime.utcnow(),
key=authkey,
measurements=[traceroute],
sources=[trace_origin],
is_oneoff=True)
(is_success, response) = atlas_req.create()
# Check if we got a positive result
if is_success:
print response
for m in response['measurements']:
msm_status['list'].append(m)
else:
msm_status['failed'].append(dest)
# This sleep is important to give time to the scheduled measurements to complete before trying more.
time.sleep(3)
except (urllib2.HTTPError, urllib2.URLError) as e:
# Other kind of error
msm_status['failed'].append(dest)
print "Fatal Error: %s " % e
return msm_status
def create_measurements(api_key, measurements, sources):
success_list = []
request_ids = []
logger.info('Chunking measurements')
chunk_size = 100
measurement_chunks = [measurements[x:x+chunk_size] for x in range(0, len(measurements), chunk_size)]
for i, measurement_chunk in enumerate(measurement_chunks):
# Check if there are simultaneous (concurrently running) measurements before running the next chunk
concurrent_status = '1,2'
concurrent_measurements = [x['id'] for x in MyMeasurementRequest(**{'status': concurrent_status, 'key': api_key})]
while concurrent_measurements:
logger.info('Found ' + str(len(concurrent_measurements)) + ' concurrent measurements. Sleeping')
logger.debug(concurrent_measurements)
time.sleep(60)
concurrent_measurements = [x['id'] for x in MyMeasurementRequest(**{'status': concurrent_status, 'key': api_key})]
logger.info('-Creating measurements for measurement chunk ' + str(i))
for msm in measurement_chunk:
logger.debug('Creating: ' + str(msm))
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=api_key,
is_oneoff=True,
measurements=[msm],
sources=sources,
)
success, response = atlas_request.create()
logger.debug('Creating success: ' + str(success))
logger.debug('Creating response: ' + str(response))
if not success:
logger.error('Unsuccessful creating measurements for ' + str(msm))
logger.error('Failure response:' + str(response))
# return False, request_ids
else:
request_ids.extend(response['measurements'])
success_list.append(msm)
return success_list, request_ids
def _create_measurement_impl(self, inmsm):
args = self._args
log = self._log
target_ip = inmsm.target
probes = [str(p) for p in inmsm.probes]
desc = '[real02] all-pairs ping {} (nonce={})'.format(
target_ip, random.randint(0, 1000000000))
log(desc)
ping = Ping(af=4, target=target_ip, description=desc)
source = AtlasSource(type='probes',
requested=len(probes),
value='{}'.format(','.join(probes)))
req = AtlasCreateRequest(start_time=datetime.utcnow(),
key=args.api,
measurements=[ping],
sources=[source],
is_oneoff=True)
success, resp = req.create()
if not success:
log.warn('Error creating measurement:', str(resp))
return None
msm_id = resp['measurements'][0]
return msm_id
def setUp(self):
create_source = AtlasSource(
**{"type": "area", "value": "WW", "requested": 3}
)
measurement = Ping(**{
"target": "testing", "af": 6,
"description": "testing"
})
self.request = AtlasCreateRequest(**{
"start_time": datetime.utcnow(),
"stop_time": datetime.utcnow() + timedelta(hours=2),
"key": "path_to_key",
"measurements": [measurement],
"sources": [create_source]
})
ATLAS_API_KEY = f.readline().rstrip("\n")
# Execute some measurements
# print(probe_sample)
msm_list = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
for (asn, probe_list) in probe_sample:
print(probe_list)
source = AtlasSource(type='probes', value=",".join(probe_list),
requested=len(probe_list))
for family in [4, 6]:
for dest in sites:
description = "RIPE72: Dual stack trace from %s to %s" % (asn, dest)
traceroute = Traceroute(af=family, target=dest,
description=description,
protocol='ICMP')
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=ATLAS_API_KEY,
is_oneoff=True,
sources=[source],
measurements=[traceroute]
)
(is_success, response) = atlas_request.create()
if is_success:
msm_list[asn][dest][family] = [m for m in response[
'measurements']]
with open('dual-stack-msm.json', 'wb') as f:
json.dump(msm_list, f)
def test_url_build(self):
request = AtlasCreateRequest(**{
"measurements": [self.measurement], "sources": [self.create_source]
})
request.build_url()
self.assertNotEquals(getattr(request, "url", None), None)
def test_create_delete_request(self):
"""Unittest for Atlas create and delete request"""
if self.server == "":
raise SkipTest
source = AtlasSource(**{"type": "area", "value": "WW", "requested": 38})
ping = Ping(**{
"target": "www.ripe.net",
"af": 4,
"description": "Cousteau testing",
"prefer_anchors": True
})
traceroute = Traceroute(**{
"target": "www.google.fr",
"af": 4, "protocol": "UDP",
"description": "Cousteau testing",
"dont_fragment": True
})
dns = Dns(**{
"target": "k.root-servers.net", "af": 4,
"description": "Cousteau testing", "query_type": "SOA",
"query_class": "IN", "query_argument": "nl", "retry": 6
})
ntp = Ntp(**{
"target": "www.ripe.net",
"af": 4,
"description": "Cousteau testing",
"timeout": 1000
})
ssl = Sslcert(**{
"target": "www.ripe.net",
"af": 4,
"description": "Cousteau testing",
})
http = Http(**{
"target": "www.ripe.net",
"af": 4,
"description": "Cousteau testing",
})
stop = datetime.utcnow() + timedelta(minutes=220)
request = AtlasCreateRequest(
**{
"verify": False,
"stop_time": stop,
"key": self.create_key,
"server": self.server,
"measurements": [ping, traceroute, dns, ntp, ssl, http],
"sources": [source]
}
)
result = namedtuple('Result', 'success response')
(result.success, result.response) = request.create()
print(result.response)
self.assertTrue(result.success)
self.delete_msm = result.response["measurements"][0]
self.assertTrue(result.success)
# Unittest for Atlas delete request
if self.server == "":
raise SkipTest
kwargs = {"verify": False, "msm_id": self.delete_msm, "key": self.delete_key, "server": self.server}
request = AtlasStopRequest(**kwargs)
result = namedtuple('Result', 'success response')
(result.success, result.response) = request.create()
print(result.response)
self.assertTrue(result.success)
class TestAtlasCreateRequest(TestCase):
def setUp(self):
self.create_source = AtlasSource(
**{"type": "area", "value": "WW", "requested": 3}
)
self.measurement = Ping(**{
"target": "testing", "af": 6,
"description": "testing"
})
self.request = AtlasCreateRequest(**{
"start_time": datetime(2015, 10, 16),
"stop_time": 1445040000,
"key": "path_to_key",
"measurements": [self.measurement],
"sources": [self.create_source],
"is_oneoff": True,
})
def test_construct_post_data(self):
"""Tests construction of past data"""
self.request._construct_post_data()
validate(self.request.post_data, post_data_create_schema)
def test_post_method(self):
"""Tests POST reuest method"""
self.maxDiff = None
expected_args = {
"json": {
"definitions": [{
"af": 6, "description": "testing",
"target": "testing", "type": "ping"
}],
"is_oneoff": True,
"probes": [{"requested": 3, "type": "area", "value": "WW"}],
"start_time": 1444953600,
"stop_time": 1445040000
},
"params": {"key": "path_to_key"},
"verify": True,
"headers": {
"User-Agent": "RIPE ATLAS Cousteau v{0}".format(__version__),
"Content-Type": "application/json",
"Accept": "application/json"
},
"proxies": {},
}
with mock.patch("ripe.atlas.cousteau.request.AtlasRequest.http_method") as mock_get:
self.request._construct_post_data()
mock_get.return_value = True
self.request.post()
self.assertEqual(self.request.http_method_args, expected_args)
def test_post_method_without_times(self):
"""Tests POST reuest method with a bill to address specified"""
request = AtlasCreateRequest(**{
"key": "path_to_key",
"measurements": [self.measurement],
"sources": [self.create_source],
})
self.maxDiff = None
expected_args = {
"json": {
"definitions": [{
"af": 6, "description": "testing",
"target": "testing", "type": "ping"
}],
"is_oneoff": False,
"probes": [{"requested": 3, "type": "area", "value": "WW"}],
},
"params": {"key": "path_to_key"},
"verify": True,
"headers": {
"User-Agent": "RIPE ATLAS Cousteau v{0}".format(__version__),
"Content-Type": "application/json",
"Accept": "application/json"
},
"proxies": {},
}
with mock.patch("ripe.atlas.cousteau.request.AtlasRequest.http_method") as mock_get:
request._construct_post_data()
mock_get.return_value = True
request.post()
self.assertEqual(request.http_method_args, expected_args)
def test_post_method_with_bill_to(self):
"""Tests POST reuest method without any time specified"""
request = AtlasCreateRequest(**{
"key": "path_to_key",
"measurements": [self.measurement],
"sources": [self.create_source],
"bill_to": "billing@address"
})
self.maxDiff = None
expected_args = {
"json": {
"definitions": [{
"af": 6, "description": "testing",
"target": "testing", "type": "ping"
}],
"is_oneoff": False,
"probes": [{"requested": 3, "type": "area", "value": "WW"}],
"bill_to": "billing@address",
},
"params": {"key": "path_to_key"},
"verify": True,
"headers": {
"User-Agent": "RIPE ATLAS Cousteau v{0}".format(__version__),
"Content-Type": "application/json",
"Accept": "application/json"
},
"proxies": {},
}
with mock.patch("ripe.atlas.cousteau.request.AtlasRequest.http_method") as mock_get:
request._construct_post_data()
mock_get.return_value = True
request.post()
self.assertEqual(request.http_method_args, expected_args)
def runTraceroute(target,country,asn):
global noProbesASes
msms=[]
probeList=[]
probesids=""
prolen=0
filters = {"country_code": country,"asn_v4":asn}
print(filters)
probes = ProbeRequest(**filters)
for pb in probes:
probe = Probe(id=pb["id"])
lon=float(probe.geometry['coordinates'][0])
lat=float(probe.geometry['coordinates'][1])
print(pb["country_code"],country,pb["id"],pb["asn_v4"],lat,lon)
if pb["country_code"] != country:
print('Country filter didnt match: ',pb['id'],pb['country_code'],country)
continue
try:
ip=str(ipaddress.IPv4Address(probe.address_v4))
except:
continue
probeList.append([pb["id"],lat,lon])
if len(probeList)==0:
print('No probes found.')
return msms
selectedProbes=selectProbes(probeList)
for prbid in selectedProbes:
prolen+=1
probesids=probesids+str(prbid)+","
#print(probesids)
if prolen==0:
print('No probes found: '+str(prolen))
return msms
try:
probesids=probesids[:-1]
measurement=None
#print(probesids)
if typeRun=="traceroute":
traceroute = Traceroute(
af=4,
target=target,
description="Traceroute "+str(target),
protocol="ICMP",
)
measurement=traceroute
elif typeRun=="ping":
ping = Ping(af=4, target=target, description="Ping "+str(target))
measurement=ping
else:
print('Need correct type of measurement specified.')
exit(1)
source = AtlasSource(type="probes", value=probesids,requested=prolen)
atlas_request = AtlasCreateRequest(
start_time=datetime.utcnow(),
key=API_KEY_CREATE_UDM,
measurements=[measurement],
sources=[source],
is_oneoff=True
)
try:
(is_success, response) = atlas_request.create()
except:
traceback.print_exc()
if is_success:
msms=response['measurements']
else:
print(response)
except:
traceback.print_exc()
print("ERROR in traceroute.")
return msms
def find_psboxes(IPs, verbose, recovery=False):
"""
Finds the closest box to each IP in <IPs>, displays the results on the screen and stores them in a file in the
'output' folder and whose naming-scheme is '<timestamp_of_creation_time>_psbox.txt'
:param IPs: a list containing all the IPs a closest box should be found to
:param verbose: if true, an error message gets displayed when an internal problem occurs; otherwise not
:param recovery: if true, the recovery mode will be enabled (for more info, please see the docs in the folder
'doc')
:return: a dictionary whose values are the IPs and the keys are the corresponding closest boxes. If there
is no entry for a given IP, no box has been found
"""
currentTime = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
measurementIDs = set()
IPsToMeasurementIDs = dict()
IPsAlreadyAnalysed = set()
if recovery: # recovery mode enabled
# recover ID-to-AS mapping that has been done so far - begin
try:
ASMap = open('../logs/ID_To_AS.log', 'r')
except IOError:
if verbose:
print("error: Could not open/create file '../logs/ID_To_AS.log'\n")
return None
for line in ASMap:
line = line.rstrip('\r\n')
if line:
data = line.split('\t')
probeToASMap[data[0]] = data[1]
ASMap.close()
# recover ID-to-AS mapping that has been done so far - end
# recover IPs that have been analysed so far and the corresponding output-file - begin
try:
logFile = open('../logs/current_ping_measurementIDs.log', 'r')
except IOError:
if verbose:
print("error: Could not open file '../logs/current_ping_measurementIDs.log'\n")
return None
cnt = 0
timeStamp = ''
for line in logFile:
line = line.rstrip('\r\n')
if line:
if cnt == 0:
timeStamp = line
else:
data = line.split('\t')
IPsToMeasurementIDs[data[-2]] = data[:-2]
measurementIDs.update(data[:-2])
additionalInfoAboutMeasurements[data[-2]] = data[-1]
IPsAlreadyAnalysed.add(data[-2])
cnt += 1
logFile.close()
# recover IPs that have been analysed so far and the corresponding output-file - end
if not recovery:
try:
ASMap = open('../logs/ID_To_AS.log', 'w') # clear content of ID-to-AS log
except IOError:
if verbose:
print("error: Could not open/create file '../logs/ID_To_AS.log'\n")
return None
ASMap.close()
# open/create output-file - begin
try:
if recovery:
output = open('../output/' + str(timeStamp) + '_psbox.txt', 'a', 1)
else:
output = open('../output/' + currentTime + '_psbox.txt', 'w', 1)
except IOError:
if verbose:
if recovery:
print("error: Could not open/create file '../output/" + str(timeStamp) + "_psbox.txt'\n")
else:
print("error: Could not open/create file '../output/" + currentTime + "_psbox.txt'\n")
return None
# open/create output-file - end
# open/create log-file - begin
try:
if recovery:
logFile = open('../logs/current_ping_measurementIDs.log', 'a', 1)
else:
logFile = open('../logs/current_ping_measurementIDs.log', 'w', 1)
logFile.write(currentTime + '\n')
except IOError:
if verbose:
print("error: Could not open/create file '../logs/current_ping_measurementIDs.log'\n")
return None
# open/create log-file - end
# open file containing RIPE Atlas boxes and load data - begin
try:
with open('../lib/probelist.txt', 'r') as plFile:
probeList = list() # load list with all currently connected RIPE probes
for line in plFile:
line = line.rstrip('\r\n')
if line:
probeData = line.split('\t')
probeList.append((probeData[0], probeData[3]))
except IOError:
if verbose:
print("error: Could not open file '../lib/probelist.txt'\n")
output.close()
logFile.close()
return None
# open file containing RIPE Atlas boxes and load data - end
targetIPs = list(IPs)
IPToASMap = IPToAS.mapIPtoAS(targetIPs, '../lib/GeoIPASNum2.csv', True)
if IPToASMap is None:
output.close()
logFile.close()
return None
encounteredASes = dict()
# launching measurements to find closest box - start
for IP in IPToASMap:
if IP in IPsAlreadyAnalysed:
continue
IPsAlreadyAnalysed.add(IP)
if verbose:
print('Starting to do measurements for IP: ' + IP + '...\n')
AS = IPToASMap[IP]
if AS == 'NA_MAP':
additionalInfoAboutMeasurements[IP] = '[NO_AS]'
idx = random.sample(range(len(probeList)), 100)
selectedProbes = [probeList[i][0] for i in idx]
for i in idx:
probeToASMap[probeList[i][0]] = probeList[i][1]
try:
with open('../logs/ID_To_AS.log', 'a', 0) as ASMap:
for i in idx:
ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
except IOError:
if verbose:
print("error: Could not open/create file '../logs/ID_To_AS.log'\n")
output.close()
logFile.close()
return None
probes = [selectedProbes[i:i + 500] for i in range(len(selectedProbes), 500)]
elif AS not in encounteredASes: # check whether we have already retrieved probes for this AS
# check whether there are probes in IP's AS
probes = list(ProbeRequest(asn=IPToASMap[IP]))
# TODO: what if problem in executing request?
# if not, look at the neighbour ASes
if not probes:
neighbours = pa.findASNeighbourhood(IPToASMap[IP], True)
if neighbours is None:
output.close()
logFile.close()
return None
for neighbour in neighbours:
probes = list(ProbeRequest(asn=neighbour))
# TODO: what if problem in executing request?
if probes: # we have found neighbouring probes
probes = pa.parseProbeListOutput(probes, True, probeToASMap)
if probes is None:
output.close()
logFile.close()
return None
encounteredASes[AS] = probes
else:
encounteredASes[AS] = ''
# pinging neighbours - start
if AS != 'NA_MAP':
probes = encounteredASes[AS]
if not probes: # if no probes in neighbourhood, use randomly selected probes
additionalInfoAboutMeasurements[IP] = '[RANDOM]'
idx = random.sample(range(len(probeList)), 100)
selectedProbes = [probeList[i][0] for i in idx]
for i in idx:
probeToASMap[probeList[i][0]] = probeList[i][1]
try:
with open('../logs/ID_To_AS.log', 'a', 0) as ASMap:
for i in idx:
ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
except IOError:
if verbose:
print("error: Could not open/create file '../logs/ID_To_AS.log'\n")
output.close()
logFile.close()
return None
probes = [list(map(int, selectedProbes[i:i + 500])) for i in range(len(selectedProbes), 500)]
elif AS != 'NA_MAP':
additionalInfoAboutMeasurements[IP] = '[OK]'
giveUp = False
for probesToUse in probes:
for _ in range(5): # Perform at most 5 tries before giving up.
description = "Ping target={target}".format(target=IP)
ping = Ping(af=4, target=IP, description=description, protocol="ICMP", packets=10)
source = AtlasSource(type="probes", value=','.join(map(str, probesToUse)), requested=len(probesToUse))
request = AtlasCreateRequest(key=API_KEY, measurements=[ping], sources=[source], is_oneoff=True)
(is_success, response) = request.create()
if is_success and 'measurements' in response:
if IP not in IPsToMeasurementIDs:
IPsToMeasurementIDs[IP] = [response['measurements'][0]]
else:
IPsToMeasurementIDs[IP].append(response['measurements'][0])
measurementIDs.add(response['measurements'][0])
break
else:
time.sleep(180)
else:
giveUp = True
IPsToMeasurementIDs.pop(IP, None) # delete this entry; should not be analyzed
break
if giveUp:
break
if IPsToMeasurementIDs[IP]:
logFile.write('\t'.join(map(str, IPsToMeasurementIDs[IP])) + '\t' + IP + '\t'
+ additionalInfoAboutMeasurements[IP] + '\n')
# pinging neighbours - end
# launching measurements to find closest box - end
logFile.close()
# waiting for ping-measurements to finish
if verbose:
print('Waiting for ping measurements to finish...\n')
status = cm.checkMeasurements(measurementIDs, True)
if status is None:
return None
while not status:
time.sleep(180)
status = cm.checkMeasurements(measurementIDs, True)
if status is None:
output.close()
return None
if verbose:
print('Computing closest RIPE Atlas box...\n')
results = getSmallestPingProbe(IPsToMeasurementIDs, output)
output.close()
# os.remove('../logs/current_ping_measurementIDs.log')
# if os.path.exists('../logs/ID_To_AS.log'):
# os.remove('../logs/ID_To_AS.log')
return results