def find_psboxes(IPs, verbose, recovery):
"""
Finds the closest box to each IP in <IPs>, displays the results on the screen and stores them in a file in the
'output' folder and whose naming-scheme is '<timestamp_of_creation_time>_psbox.txt'
:param IPs: a list containing all the IPs a closest box should be found to
:param verbose: if true, an error-message gets displayed when an internal problem occurs; otherwise not
:param recovery: if true, the recovery-modus will be enabled (for more infos, please see the docs in the folder
'doc')
:return: a dictionary whose values are the IPs and the keys are the corresponding closest boxes. If there
is no entry for a given IP, no box has been found
"""
currentTime = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H-%M-%S')
measurementIDs = set()
IPsToMeasurementIDs = dict()
IPsAlreadyAnalysed = set()
if recovery: # recovery-mode enabled
# recover ID-to-AS mapping that has been done so far - begin
try:
ASMap = open('../logs/ID_To_AS.log', 'r')
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
return None
for line in ASMap:
l = line.rstrip('\r\n')
if l:
data = l.split('\t')
probeToASMap[data[0]] = data[1]
ASMap.close()
# recover ID-to-AS mapping that has been done so far - end
# recover IPs that have been analysed so far and the corresponding output-file - begin
try:
logFile = open('../logs/current_ping_measurementIDs.log', 'r')
except IOError:
if verbose:
print "error: Could not open file '../logs/current_ping_measurementIDs.log'\n"
return None
cnt = 0
timeStamp = ''
for line in logFile:
l = line.rstrip('\r\n')
if l:
if cnt == 0:
timeStamp = l
else:
data = l.split('\t')
IPsToMeasurementIDs[data[data.__len__() -
2]] = data[:data.__len__() - 2]
measurementIDs.update(data[:data.__len__() - 2])
additionalInfoAboutMeasurements[data[
data.__len__() - 2]] = data[data.__len__() - 1]
IPsAlreadyAnalysed.add(data[data.__len__() - 2])
cnt += 1
logFile.close()
# recover IPs that have been analysed so far and the corresponding output-file - end
if not recovery:
try:
ASMap = open('../logs/ID_To_AS.log',
'w') # clear content of ID-to-AS log
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
return None
ASMap.close()
# open/create output-file - begin
try:
if recovery:
output = open('../output/' + timeStamp + '_psbox.txt', 'a', 1)
else:
output = open('../output/' + currentTime + '_psbox.txt', 'w', 1)
except IOError:
if verbose:
if recovery:
print "error: Could not open/create file '../output/" + timeStamp + "_psbox.txt'\n"
else:
print "error: Could not open/create file '../output/" + currentTime + "_psbox.txt'\n"
return None
# open/create output-file - end
# open/create log-file - begin
try:
if recovery:
logFile = open('../logs/current_ping_measurementIDs.log', 'a', 1)
else:
logFile = open('../logs/current_ping_measurementIDs.log', 'w', 1)
logFile.write(currentTime + '\n')
except IOError:
if verbose:
print "error: Could not open/create file '../logs/current_ping_measurementIDs.log'\n"
return None
# open/create log-file - end
# open file containing RIPE Atlas boxes and load data - begin
try:
plFile = open('../lib/probelist.txt', 'r')
except IOError:
if verbose:
print "error: Could not open file '../lib/probelist.txt'\n"
output.close()
logFile.close()
return None
probeList = list() # load list with all currently connected RIPE probes
for line in plFile:
l = line.rstrip('\r\n')
if l:
probeData = l.split('\t')
probeList.append((probeData[0], probeData[3]))
plFile.close()
# open file containing RIPE Atlas boxes and load data - end
targetIPs = list(IPs)
IPToASMap = IPToAS.mapIPtoAS(targetIPs, '../lib/GeoIPASNum2.csv', True)
if IPToASMap == None:
output.close()
logFile.close()
return None
encounteredASes = dict()
# launching measurements to find closest box - start
for IP in IPToASMap:
if IP in IPsAlreadyAnalysed:
continue
IPsAlreadyAnalysed.add(IP)
if verbose:
print 'Starting to do measurements for IP: ' + IP + '...\n'
AS = IPToASMap[IP]
if AS == 'NA_MAP':
additionalInfoAboutMeasurements[IP] = '[NO_AS]'
idx = random.sample(range(0, probeList.__len__()), 100)
selectedProbes = [probeList[i][0] for i in idx]
for i in idx:
probeToASMap[probeList[i][0]] = probeList[i][1]
try:
ASMap = open('../logs/ID_To_AS.log', 'a', 0)
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
output.close()
logFile.close()
return None
for i in idx:
ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
ASMap.close()
probes = [
selectedProbes[i:i + 500]
for i in range(0, selectedProbes.__len__(), 500)
]
elif not AS in encounteredASes: # check whether we have already retrieved probes for this AS
# check whether there are probes in IP's AS
nbOfConsecutiveFailures = 0
giveUp = False
while True:
try:
probeListInfo = subprocess.check_output(
['../contrib/probe-list.pl', '--asn', IPToASMap[IP]])
nbOfConsecutiveFailures = 0
break
except subprocess.CalledProcessError:
nbOfConsecutiveFailures += 1
time.sleep(120)
# if download-attempt fails for 5 consecutive times, abord
if nbOfConsecutiveFailures == 5:
giveUp = True
break
if giveUp:
break # proceed to closest-box analysis
# if not, look at the neighbour-ASes
if not probeListInfo:
neighbours = pa.findASNeighbourhood(IPToASMap[IP], True)
if neighbours == None:
output.close()
logFile.close()
return None
giveUp = False
nbOfConsecutiveFailures = 0
for neighbour in neighbours:
while True:
try:
probeListInfo += subprocess.check_output([
'../contrib/probe-list.pl', '--asn', neighbour
])
nbOfConsecutiveFailures = 0
break
except subprocess.CalledProcessError:
nbOfConsecutiveFailures += 1
time.sleep(120)
# if download-attempt fails for 5 consecutive times, abord
if nbOfConsecutiveFailures == 5:
giveUp = True
break
if giveUp:
break
if giveUp:
continue
if probeListInfo: # we have found neighbour-probes
probes = pa.parseProbeListOutput(probeListInfo, True,
probeToASMap)
if probes == None:
output.close()
logFile.close()
return None
encounteredASes[AS] = probes
else:
encounteredASes[AS] = ''
# pinging neighbours - start
if AS != 'NA_MAP':
probes = encounteredASes[AS]
if not probes: # if no probes in neighbourhood, use randomly selected probes
additionalInfoAboutMeasurements[IP] = '[RANDOM]'
idx = random.sample(range(0, probeList.__len__()), 100)
selectedProbes = [probeList[i][0] for i in idx]
for i in idx:
probeToASMap[probeList[i][0]] = probeList[i][1]
try:
ASMap = open('../logs/ID_To_AS.log', 'a', 0)
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
output.close()
logFile.close()
return None
for i in idx:
ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
ASMap.close()
probes = [
selectedProbes[i:i + 500]
for i in range(0, selectedProbes.__len__(), 500)
]
elif AS != 'NA_MAP':
additionalInfoAboutMeasurements[IP] = '[OK]'
nbOfConsecutiveFailures = 0
giveUp = False
for probeSet in probes:
probesToUse = ','.join(probeSet)
while True:
try:
udmCreateInfo = subprocess.check_output([
'../contrib/udm-create.pl', '--api', API_KEY, '--type',
'ping', '--target', IP, '--probe-list', probesToUse,
'--packets', '10'
])
udmCreateInfo = udmCreateInfo.rstrip('\r\n')
nbOfConsecutiveFailures = 0
if udmCreateInfo:
if IP not in IPsToMeasurementIDs:
IPsToMeasurementIDs[IP] = [udmCreateInfo]
else:
IPsToMeasurementIDs[IP].append(udmCreateInfo)
measurementIDs.add(udmCreateInfo)
break
except subprocess.CalledProcessError, e: # maybe too many measurements running?
nbOfConsecutiveFailures += 1
time.sleep(180)
# if 5 consecutive measurement-attempts fail, give up
if nbOfConsecutiveFailures == 5:
IPsToMeasurementIDs.pop(
IP,
None) # delete this entry; should not be analyzed
giveUp = True
break
if giveUp:
break
if giveUp:
break
if IPsToMeasurementIDs[IP]:
logFile.write('\t'.join(IPsToMeasurementIDs[IP]) + '\t' + IP +
'\t' + additionalInfoAboutMeasurements[IP] + '\n')
def retrieve_traceroute_results(filename, verbose):
"""
Downloads and parses traceroute-results for the measurement-IDs indicated in file '../logs/<filename>' and write
results to file '../output/<timestamp>_scheduled_traceroutes.log'
The input-file has to be located in the input-folder.
For the exact format to be followed by the input-file, please have a look at the documentation
The download of user-defined-measurement-results is done via the Atlas-toolbox
:param filename: the name of the file containing measurement-IDs
:param verbose: if true, the progress will be written to the standard-output
"""
try:
udmFile = open(filename, 'r')
except IOError:
print "error: Could not open '" + filename + "'!\n"
return None
measurementsToAnalyse = list()
IPsToAnalyse = set()
for udmLine in udmFile:
udmL = udmLine.rstrip('\r\n')
data = udmL.split('\t')
udms = data[:data.__len__() - 1]
dstIP = data[data.__len__() - 1]
for udm in udms:
resultInfo = subprocess.check_output(['../contrib/udm-result.pl', '--udm', data[0]])
if not resultInfo.rstrip('\r\n'):
continue
resultInfo = resultInfo.split('\n')
for line in resultInfo:
if line:
if not line.startswith('\t'): # first line of a result
l = line.lstrip().split('\t')
nbHop = int(l[5])
probeID = l[1]
srcIP = l[2]
timestamp = l[0]
if verbose:
print 'Analysing traceroute from ' + srcIP + ' to ' + dstIP + '...\n'
currentMeasurement = TracerouteMeasurement()
currentMeasurement.addProbeID(probeID)
currentMeasurement.addNbHops(nbHop)
currentMeasurement.addTimestamp(timestamp)
IPsToAnalyse.add(srcIP)
currentMeasurement.addIPInfo(srcIP, 'init')
else:
l = line.lstrip().split('\t')
ip = l[1]
rtt = l[3]
hopIndex = l[0]
if rtt != '*':
if ip != '*':
currentMeasurement.addIPInfo(ip, rtt)
IPsToAnalyse.add(ip)
else:
currentMeasurement.addIPInfo('NA_TR', rtt)
else:
if ip == '*':
currentMeasurement.addIPInfo('NA_TR', '')
else:
currentMeasurement.addIPInfo(ip, '')
IPsToAnalyse.add(ip)
#finished analysing - store results for probe
if int(hopIndex) >= nbHop:
measurementsToAnalyse.append(currentMeasurement)
# We will do the IP2AS-mapping and store the results to a file
IPToASMapping = parseIP.mapIPtoAS(IPsToAnalyse, '../lib/GeoIPASNum2.csv', verbose)
if IPToASMapping == None:
return None
if verbose:
print 'Saving results to file...\n'
currentTime = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
for measurement in measurementsToAnalyse:
measurement.saveToFile(IPToASMapping, currentTime)
udmFile.close()
def find_psboxes(IPs, verbose, recovery):
"""
Finds the closest box to each IP in <IPs>, displays the results on the screen and stores them in a file in the
'output' folder and whose naming-scheme is '<timestamp_of_creation_time>_psbox.txt'
:param IPs: a list containing all the IPs a closest box should be found to
:param verbose: if true, an error-message gets displayed when an internal problem occurs; otherwise not
:param recovery: if true, the recovery-modus will be enabled (for more infos, please see the docs in the folder
'doc')
:return: a dictionary whose values are the IPs and the keys are the corresponding closest boxes. If there
is no entry for a given IP, no box has been found
"""
currentTime = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
measurementIDs = set()
IPsToMeasurementIDs = dict()
IPsAlreadyAnalysed = set()
if recovery: # recovery-mode enabled
# recover ID-to-AS mapping that has been done so far - begin
try:
ASMap = open('../logs/ID_To_AS.log', 'r')
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
return None
for line in ASMap:
l = line.rstrip('\r\n')
if l:
data = l.split('\t')
probeToASMap[data[0]] = data[1]
ASMap.close()
# recover ID-to-AS mapping that has been done so far - end
# recover IPs that have been analysed so far and the corresponding output-file - begin
try:
logFile = open('../logs/current_ping_measurementIDs.log', 'r')
except IOError:
if verbose:
print "error: Could not open file '../logs/current_ping_measurementIDs.log'\n"
return None
cnt = 0
timeStamp = ''
for line in logFile:
l = line.rstrip('\r\n')
if l:
if cnt == 0:
timeStamp = l
else:
data = l.split('\t')
IPsToMeasurementIDs[data[data.__len__() - 2]] = data[:data.__len__() - 2]
measurementIDs.update(data[:data.__len__() - 2])
additionalInfoAboutMeasurements[data[data.__len__() - 2]] = data[data.__len__() - 1]
IPsAlreadyAnalysed.add(data[data.__len__() - 2])
cnt += 1
logFile.close()
# recover IPs that have been analysed so far and the corresponding output-file - end
if not recovery:
try:
ASMap = open('../logs/ID_To_AS.log', 'w') # clear content of ID-to-AS log
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
return None
ASMap.close()
# open/create output-file - begin
try:
if recovery:
output = open('../output/' + timeStamp + '_psbox.txt', 'a', 1)
else:
output = open('../output/' + currentTime + '_psbox.txt', 'w', 1)
except IOError:
if verbose:
if recovery:
print "error: Could not open/create file '../output/" + timeStamp + "_psbox.txt'\n"
else:
print "error: Could not open/create file '../output/" + currentTime + "_psbox.txt'\n"
return None
# open/create output-file - end
# open/create log-file - begin
try:
if recovery:
logFile = open('../logs/current_ping_measurementIDs.log', 'a', 1)
else:
logFile = open('../logs/current_ping_measurementIDs.log', 'w', 1)
logFile.write(currentTime + '\n')
except IOError:
if verbose:
print "error: Could not open/create file '../logs/current_ping_measurementIDs.log'\n"
return None
# open/create log-file - end
# open file containing RIPE Atlas boxes and load data - begin
try:
plFile = open('../lib/probelist.txt', 'r')
except IOError:
if verbose:
print "error: Could not open file '../lib/probelist.txt'\n"
output.close()
logFile.close()
return None
probeList = list() # load list with all currently connected RIPE probes
for line in plFile:
l = line.rstrip('\r\n')
if l:
probeData = l.split('\t')
probeList.append((probeData[0], probeData[3]))
plFile.close()
# open file containing RIPE Atlas boxes and load data - end
targetIPs = list(IPs)
IPToASMap = IPToAS.mapIPtoAS(targetIPs, '../lib/GeoIPASNum2.csv', True)
if IPToASMap == None:
output.close()
logFile.close()
return None
encounteredASes = dict()
# launching measurements to find closest box - start
for IP in IPToASMap:
if IP in IPsAlreadyAnalysed:
continue
IPsAlreadyAnalysed.add(IP)
if verbose:
print 'Starting to do measurements for IP: ' + IP + '...\n'
AS = IPToASMap[IP]
if AS == 'NA_MAP':
additionalInfoAboutMeasurements[IP] = '[NO_AS]'
idx = random.sample(range(0, probeList.__len__()), 100)
selectedProbes = [probeList[i][0] for i in idx]
for i in idx:
probeToASMap[probeList[i][0]] = probeList[i][1]
try:
ASMap = open('../logs/ID_To_AS.log', 'a', 0)
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
output.close()
logFile.close()
return None
for i in idx:
ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
ASMap.close()
probes = [selectedProbes[i:i + 500] for i in range(0, selectedProbes.__len__(), 500)]
elif not AS in encounteredASes: # check whether we have already retrieved probes for this AS
# check whether there are probes in IP's AS
nbOfConsecutiveFailures = 0
giveUp = False
while True:
try:
probeListInfo = subprocess.check_output(['../contrib/probe-list.pl', '--asn', IPToASMap[IP]])
nbOfConsecutiveFailures = 0
break
except subprocess.CalledProcessError:
nbOfConsecutiveFailures += 1
time.sleep(120)
# if download-attempt fails for 5 consecutive times, abord
if nbOfConsecutiveFailures == 5:
giveUp = True
break
if giveUp:
break # proceed to closest-box analysis
# if not, look at the neighbour-ASes
if not probeListInfo:
neighbours = pa.findASNeighbourhood(IPToASMap[IP], True)
if neighbours == None:
output.close()
logFile.close()
return None
giveUp = False
nbOfConsecutiveFailures = 0
for neighbour in neighbours:
while True:
try:
probeListInfo += subprocess.check_output(['../contrib/probe-list.pl', '--asn', neighbour])
nbOfConsecutiveFailures = 0
break
except subprocess.CalledProcessError:
nbOfConsecutiveFailures += 1
time.sleep(120)
# if download-attempt fails for 5 consecutive times, abord
if nbOfConsecutiveFailures == 5:
giveUp = True
break
if giveUp:
break
if giveUp:
continue
if probeListInfo: # we have found neighbour-probes
probes = pa.parseProbeListOutput(probeListInfo, True, probeToASMap)
if probes == None:
output.close()
logFile.close()
return None
encounteredASes[AS] = probes
else:
encounteredASes[AS] = ''
# pinging neighbours - start
if AS != 'NA_MAP':
probes = encounteredASes[AS]
if not probes: # if no probes in neighbourhood, use randomly selected probes
additionalInfoAboutMeasurements[IP] = '[RANDOM]'
idx = random.sample(range(0, probeList.__len__()), 100)
selectedProbes = [probeList[i][0] for i in idx]
for i in idx:
probeToASMap[probeList[i][0]] = probeList[i][1]
try:
ASMap = open('../logs/ID_To_AS.log', 'a', 0)
except IOError:
if verbose:
print "error: Could not open/create file '../logs/ID_To_AS.log'\n"
output.close()
logFile.close()
return None
for i in idx:
ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
ASMap.close()
probes = [selectedProbes[i:i + 500] for i in range(0, selectedProbes.__len__(), 500)]
elif AS != 'NA_MAP':
additionalInfoAboutMeasurements[IP] = '[OK]'
nbOfConsecutiveFailures = 0
giveUp = False
for probeSet in probes:
probesToUse = ','.join(probeSet)
while True:
try:
udmCreateInfo = subprocess.check_output(['../contrib/udm-create.pl', '--api', API_KEY, '--type', 'ping', '--target',
IP, '--probe-list', probesToUse, '--packets', '10'])
udmCreateInfo = udmCreateInfo.rstrip('\r\n')
nbOfConsecutiveFailures = 0
if udmCreateInfo:
if IP not in IPsToMeasurementIDs:
IPsToMeasurementIDs[IP] = [udmCreateInfo]
else:
IPsToMeasurementIDs[IP].append(udmCreateInfo)
measurementIDs.add(udmCreateInfo)
break
except subprocess.CalledProcessError, e: # maybe too many measurements running?
nbOfConsecutiveFailures += 1
time.sleep(180)
# if 5 consecutive measurement-attempts fail, give up
if nbOfConsecutiveFailures == 5:
IPsToMeasurementIDs.pop(IP, None) # delete this entry; should not be analyzed
giveUp = True
break
if giveUp:
break
if giveUp:
break
if IPsToMeasurementIDs[IP]:
logFile.write('\t'.join(IPsToMeasurementIDs[IP]) + '\t' + IP + '\t'
+ additionalInfoAboutMeasurements[IP] + '\n')
def retrieve_traceroute_results(filename, verbose):
"""
Downloads and parses traceroute-results for the measurement-IDs indicated in file '../logs/<filename>' and write
results to file '../output/<timestamp>_scheduled_traceroutes.log'
The input-file has to be located in the input-folder.
For the exact format to be followed by the input-file, please have a look at the documentation
The download of user-defined-measurement-results is done via the Atlas-toolbox
:param filename: the name of the file containing measurement-IDs
:param verbose: if true, the progress will be written to the standard-output
"""
try:
udmFile = open(filename, 'r')
except IOError:
print "error: Could not open '" + filename + "'!\n"
return None
measurementsToAnalyse = list()
IPsToAnalyse = set()
for udmLine in udmFile:
udmL = udmLine.rstrip('\r\n')
data = udmL.split('\t')
udms = data[:data.__len__() - 1]
dstIP = data[data.__len__() - 1]
for udm in udms:
resultInfo = subprocess.check_output(
['../contrib/udm-result.pl', '--udm', data[0]])
if not resultInfo.rstrip('\r\n'):
continue
resultInfo = resultInfo.split('\n')
for line in resultInfo:
if line:
if not line.startswith('\t'): # first line of a result
l = line.lstrip().split('\t')
nbHop = int(l[5])
probeID = l[1]
srcIP = l[2]
timestamp = l[0]
if verbose:
print 'Analysing traceroute from ' + srcIP + ' to ' + dstIP + '...\n'
currentMeasurement = TracerouteMeasurement()
currentMeasurement.addProbeID(probeID)
currentMeasurement.addNbHops(nbHop)
currentMeasurement.addTimestamp(timestamp)
IPsToAnalyse.add(srcIP)
currentMeasurement.addIPInfo(srcIP, 'init')
else:
l = line.lstrip().split('\t')
ip = l[1]
rtt = l[3]
hopIndex = l[0]
if rtt != '*':
if ip != '*':
currentMeasurement.addIPInfo(ip, rtt)
IPsToAnalyse.add(ip)
else:
currentMeasurement.addIPInfo('NA_TR', rtt)
else:
if ip == '*':
currentMeasurement.addIPInfo('NA_TR', '')
else:
currentMeasurement.addIPInfo(ip, '')
IPsToAnalyse.add(ip)
#finished analysing - store results for probe
if int(hopIndex) >= nbHop:
measurementsToAnalyse.append(currentMeasurement)
# We will do the IP2AS-mapping and store the results to a file
IPToASMapping = parseIP.mapIPtoAS(IPsToAnalyse, '../lib/GeoIPASNum2.csv',
verbose)
if IPToASMapping == None:
return None
if verbose:
print 'Saving results to file...\n'
currentTime = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H-%M-%S')
for measurement in measurementsToAnalyse:
measurement.saveToFile(IPToASMapping, currentTime)
udmFile.close()