Beispiel #1
0
def find_psboxes(IPs, verbose, recovery):
    """
    Finds the closest box to each IP in <IPs>, displays the results on the screen and stores them in a file in the
    'output' folder and whose naming-scheme is '<timestamp_of_creation_time>_psbox.txt'
    :param IPs:      a list containing all the IPs a closest box should be found to
    :param verbose:  if true, an error-message gets displayed when an internal problem occurs, otherwise not
    :param recovery: if true, the recovery-modus will be enabled (for more infos, please see the docs in the folder
                     'doc')
    :return:         a dictionary whose values are the IPs and the keys are the corresponding cloesest boxes. If there
                     is not entry for a given IP, no box has been found
    """

    currentTime = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
    measurementIDs = set()
    IPsToMeasurementIDs = dict()
    IPsAlreadyAnalysed = set()

    if recovery: # recovery-mode enabled
        # recover ID-to-AS mapping that has been done so far - begin
        try:
            ASMap = open('DisNETPerf/logs/ID_To_AS.log', 'r')
        except IOError:
            if verbose:
                print( "error: Could not open/create file 'DisNETPerf/logs/ID_To_AS.log'\n")
            return None

        for line in ASMap:
            l = line.rstrip('\r\n')
            if l:
                data = l.split('\t')
                probeToASMap[data[0]] = data[1]
        ASMap.close()
        # recover ID-to-AS mapping that has been done so far - end

        # recover IPs that have been analysed so far and the corresponding output-file - begin
        try:
            logFile = open('DisNETPerf/logs/current_ping_measurementIDs.log', 'r')
        except IOError:
            if verbose:
                print( "error: Could not open file 'DisNETPerf/logs/current_ping_measurementIDs.log'\n")
            return None

        cnt = 0
        timeStamp = ''
        for line in logFile:
            l = line.rstrip('\r\n')
            if l:
                if cnt == 0:
                    timeStamp = l
                else:
                    data = l.split('\t')
                    IPsToMeasurementIDs[data[data.__len__() - 2]] = data[:data.__len__() - 2]
                    measurementIDs.update(data[:data.__len__() - 2])
                    additionalInfoAboutMeasurements[data[data.__len__() - 2]] = data[data.__len__() - 1]
                    IPsAlreadyAnalysed.add(data[data.__len__() - 2])
                cnt += 1
        logFile.close()
        # recover IPs that have been analysed so far and the corresponding output-file - end

    if not recovery:
        try:
            ASMap = open('DisNETPerf/logs/ID_To_AS.log', 'w') # clear content of ID-to-AS log
        except IOError:
            if verbose:
                print( "error: Could not open/create file 'DisNETPerf/logs/ID_To_AS.log'\n")
            return None
        ASMap.close()

    # open/create output-file - begin
    try:
        if recovery:
            output = open('DisNETPerf/output/' + timeStamp + '_psbox.txt', 'a', 1)
        else:
            output = open('DisNETPerf/output/' + currentTime + '_psbox.txt', 'w', 1)
    except IOError:
        if verbose:
            if recovery:
                print( "error: Could not open/create file 'DisNETPerf/output/" + timeStamp  + "_psbox.txt'\n")
            else:
                print( "error: Could not open/create file 'DisNETPerf/output/" + currentTime  + "_psbox.txt'\n")
        return None
    # open/create output-file - end

    # open/create log-file - begin
    try:
        if recovery:
            logFile = open('DisNETPerf/logs/current_ping_measurementIDs.log', 'a', 1)
        else:
            logFile = open('DisNETPerf/logs/current_ping_measurementIDs.log', 'w', 1)
            logFile.write(currentTime + '\n')
    except IOError:
        if verbose:
            print( "error: Could not open/create file 'DisNETPerf/logs/current_ping_measurementIDs.log'\n")
        return None
    # open/create log-file - end

    # open file containing RIPE Atlas boxes and load data - begin
    try:
        plFile = open('DisNETPerf/lib/probelist.txt', 'r')
    except IOError:
        if verbose:
            print( "error: Could not open file 'DisNETPerf/lib/probelist.txt'\n")
        output.close()
        logFile.close()
        return None

    probeList = list() # load list with all currently connected RIPE probes
    for line in plFile:
        l = line.rstrip('\r\n')
        if l:
            probeData = l.split('\t')
            probeList.append((probeData[0], probeData[3]))
    plFile.close()
    # open file containing RIPE Atlas boxes and load data - end

    targetIPs = list(IPs)

    IPToASMap = IPToAS.mapIPtoAS(targetIPs, 'DisNETPerf/lib/GeoIPASNum2.csv', True)

    if IPToASMap == None:
        output.close()
        logFile.close()
        return None

    encounteredASes = dict()

    # launching measurements to find closest box - start
    for IP in IPToASMap:
        if IP in IPsAlreadyAnalysed:
            continue
        IPsAlreadyAnalysed.add(IP)

        if verbose:
            print( 'Starting to do measurements for IP: ' + IP + '...\n')
        AS = IPToASMap[IP]

        if AS == 'NA':
            additionalInfoAboutMeasurements[IP] = '[NO_AS]'
            idx = random.sample(range(0, probeList.__len__()), 100)
            selectedProbes = [probeList[i][0] for i in idx]

            try:
                ASMap = open('DisNETPerf/logs/ID_To_AS.log', 'a', 0)
            except IOError:
                if verbose:
                    print( "error: Could not open/create file 'DisNETPerf/logs/ID_To_AS.log'\n")
                output.close()
                logFile.close()
                return None
            for i in idx:
                ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
            ASMap.close()

            probes = [selectedProbes[i:i + 500] for i in range(0, selectedProbes.__len__(), 500)]

        elif not AS in encounteredASes: # check whether we have already retrieved probes for this AS
            # check whether there are probes in IP's AS
            nbOfConsecutiveFailures = 0
            giveUp = False
            while True:
                try:
                    probeListInfo = subprocess.check_output(['DisNETPerf/contrib/probe-list.pl', '--asn', IPToASMap[IP]])
                    nbOfConsecutiveFailures = 0
                    break
                except subprocess.CalledProcessError:
                    nbOfConsecutiveFailures += 1
                    time.sleep(120)

                    # if download-attempt fails for 5 consecutive times, abord
                    if nbOfConsecutiveFailures == 5:
                        giveUp = True
                        break
            if giveUp:
                break # proceed to closest-box analysis

            # if not, look at the neighbour-ASes
            if not probeListInfo:
                neighbours = pa.findASNeighbourhood(IPToASMap[IP], True)
                if neighbours == None:
                    output.close()
                    logFile.close()
                    return None

                giveUp = False
                nbOfConsecutiveFailures = 0
                for neighbour in neighbours:
                    while True:
                        try:
                            probeListInfo += subprocess.check_output(['DisNETPerf/contrib/probe-list.pl', '--asn', neighbour])
                            nbOfConsecutiveFailures = 0
                            break
                        except subprocess.CalledProcessError:
                            nbOfConsecutiveFailures += 1
                            time.sleep(120)

                            # if download-attempt fails for 5 consecutive times, abord
                            if nbOfConsecutiveFailures == 5:
                                giveUp = True
                                break
                    if giveUp:
                        break

                if giveUp:
                    continue

            if probeListInfo: # we have found neighbour-probes
                probes = pa.parseProbeListOutput(probeListInfo, True, probeToASMap)
                if probes == None:
                    output.close()
                    logFile.close()
                    return None

                encounteredASes[AS] = probes
            else:
                encounteredASes[AS] = ''

        # pinging neighbours - start
        if AS != 'NA':
            probes = encounteredASes[AS]

        if not probes: # if no probes in neighbourhood, use randomly selected probes
            additionalInfoAboutMeasurements[IP] = '[RANDOM]'

            idx = random.sample(range(0, probeList.__len__()), 100)
            selectedProbes = [probeList[i][0] for i in idx]

            try:
                ASMap = open('DisNETPerf/logs/ID_To_AS.log', 'a')
            except IOError:
                if verbose:
                    print( "error: Could not open/create file 'DisNETPerf/logs/ID_To_AS.log'\n")
                output.close()
                logFile.close()
                return None
            for i in idx:
                ASMap.write(probeList[i][0] + '\t' + probeList[i][1] + '\n')
            ASMap.close()

            probes = [selectedProbes[i:i + 500] for i in range(0, selectedProbes.__len__(), 500)]
        elif AS != 'NA':
            additionalInfoAboutMeasurements[IP] = '[OK]'

        nbOfConsecutiveFailures = 0
        giveUp = False

        for probeSet in probes:
            probesToUse = ','.join(probeSet)
            while True:
                try:
                    udmCreateInfo = subprocess.check_output(['DisNETPerf/contrib/udm-create.pl', '--api', API_KEY, '--type',  'ping', '--target',
                                                         IP, '--probe-list', probesToUse, '--packets', '10'])
                    udmCreateInfo = udmCreateInfo.decode("utf-8").rstrip('\r\n')
                    nbOfConsecutiveFailures = 0

                    if udmCreateInfo:
                        if IP not in IPsToMeasurementIDs:
                            IPsToMeasurementIDs[IP] = [udmCreateInfo]
                        else:
                            IPsToMeasurementIDs[IP].append(udmCreateInfo)
                        measurementIDs.add(udmCreateInfo)
                    break
                except (subprocess.CalledProcessError) as e: # maybe too many measurements running?
                    nbOfConsecutiveFailures += 1
                    time.sleep(180)

                    # if 5 consecutive measurement-attempts fail, give up
                    if nbOfConsecutiveFailures == 5:
                        IPsToMeasurementIDs.pop(IP, None) # delete this entry; should not be analyzed
                        giveUp = True
                        break
            if giveUp:
                break
        if giveUp:
            break

        if IPsToMeasurementIDs[IP]:
            logFile.write('\t'.join(IPsToMeasurementIDs[IP]) + '\t' + IP + '\t'
                                    + additionalInfoAboutMeasurements[IP] + '\n')
        # pinging neighbours - end

    # launching measurements to find closest box - end
    logFile.close()

    # waiting for ping-measurements to finish
    if verbose:
        print( 'Waiting for ping-measurements to finish...\n')
    status = cm.checkMeasurements(measurementIDs, True)
    if status == None:
        return None

    while not status:
        time.sleep(60)
        status = cm.checkMeasurements(measurementIDs, True)

        if status == None:
            output.close()
            return None

    if verbose:
        print( 'Computing closest RIPE Atlas box...\n')

    results = getSmallestPingProbe(IPsToMeasurementIDs, output)

    output.close()
    # os.remove('../logs/current_ping_measurementIDs.log')
    # if os.path.exists('../logs/ID_To_AS.log'):
    #     os.remove('../logs/ID_To_AS.log')

    return results
Beispiel #2
0
def retrieve_traceroute_results(filename, verbose):
    """
    Downloads and parses traceroute-results for the measurement-IDs indicated in file '../logs/<filename>' and write
    results to file '../output/<timestamp>_scheduled_traceroutes.log'
    The input-file has to be located in the input-folder.
    For the exact format to be followed by the input-file, please have a look at the documentation
    The download of user-defined-measurement-results is done via the Atlas-toolbox
    :param filename:    the name of the file containing measurement-IDs
    :param verbose:     if true, the progress will be written to the standard-output
    """
    try:
        udmFile = open(filename, 'r')
    except IOError:
        print ("error: Could not open '" + filename + "'!\n")
        return None

    measurementsToAnalyse = list()
    IPsToAnalyse = set()

    for udmLine in udmFile:
        udmL = udmLine.rstrip('\r\n')
        data = udmL.split('\t')
        udms = data[:data.__len__() - 1]
        dstIP = data[data.__len__() - 1]

        for udm in udms:
            resultInfo = subprocess.check_output(['udm-result.pl', '--udm', data[0]]).decode("utf-8")
            if not resultInfo.rstrip('\r\n'):
                continue
            resultInfo = resultInfo.split('\n')

            for line in resultInfo:
                if line:
                    if not line.startswith('\t'):     # first line of a result
                        l = line.lstrip().split('\t')
                        nbHop = int(l[5])
                        probeID = l[1]
                        srcIP = l[2]
                        timestamp = l[0]

                        if verbose:
                            print ("Analysing traceroute from ' + srcIP + ' to ' + dstIP + '...\n")

                        currentMeasurement = TracerouteMeasurement()
                        currentMeasurement.addProbeID(probeID)
                        currentMeasurement.addNbHops(nbHop)
                        currentMeasurement.addTimestamp(timestamp)

                        IPsToAnalyse.add(srcIP)
                        currentMeasurement.addIPInfo(srcIP, 'init')
                    else:
                        l = line.lstrip().split('\t')
                        ip = l[1]
                        rtt = l[3]
                        hopIndex = l[0]
                        if rtt != '*':
                            if ip != '*':
                                currentMeasurement.addIPInfo(ip, rtt)
                                IPsToAnalyse.add(ip)
                            else:
                                currentMeasurement.addIPInfo('NA_TR', rtt)
                        else:
                            if ip == '*':
                                currentMeasurement.addIPInfo('NA_TR', '')
                            else:
                                currentMeasurement.addIPInfo(ip, '')
                                IPsToAnalyse.add(ip)

                        #finished analysing - store results for probe
                        if int(hopIndex) >= nbHop:
                            measurementsToAnalyse.append(currentMeasurement)

    # We will do the IP2AS-mapping and store the results to a file
    IPToASMapping = parseIP.mapIPtoAS(IPsToAnalyse, 'DisNETPerf/lib/GeoIPASNum2.csv', verbose)
    if IPToASMapping == None:
        return None

    if verbose:
        print ("Saving results to file...\n")

    currentTime = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H-%M-%S')
    for measurement in measurementsToAnalyse:
        measurement.saveToFile(IPToASMapping, currentTime)
    udmFile.close()