示例#1
0
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder):
    '''Create a list of all files that must be fetched unless done already.'''

    # For AN 20091112, etc, some of the ortho images are stored at the
    # beginning of the next day's flight. Need to sort this out, and
    # it is tricky. More comments within the code.
    fetchNextDay = True

    separateByLat = (options.type == 'ortho'
                     and isInSeparateByLatTable(options.yyyymmdd))
    if separateByLat:
        # Here we won't fetch the next day, we will just separate by latitude within
        # a given day
        fetchNextDay = False

    orthoOrFireball = ((options.type == 'ortho')
                       or (options.type == 'fireball'))

    if fetchNextDay:
        # Normally we fetch for next day only for ortho or fireball. However,
        # for one single special flight, we do it for jpeg too, as then
        # the jpegs are also split.
        if orthoOrFireball or \
           ((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)):
            fetchNextDay = True
        else:
            fetchNextDay = False

    # If we need to parse the next flight day as well, as expected in some runs,
    # we will fetch two html files, but create a single index out of them.
    dayVals = [0]
    if fetchNextDay:
        dayVals.append(1)

    indexPath = icebridge_common.htmlIndexFile(outputFolder)

    currIndexPath = indexPath
    parsedIndexPath = icebridge_common.csvIndexFile(outputFolder)

    if options.refetchIndex:
        os.system('rm -f ' + indexPath)
        os.system('rm -f ' + parsedIndexPath)

    if icebridge_common.fileNonEmpty(parsedIndexPath):
        logger.info('Already have the index file ' + parsedIndexPath +
                    ', keeping it.')
        return parsedIndexPath

    frameDict = {}
    urlDict = {}

    # We need the list of jpeg frames. Sometimes when fetching ortho images,
    # and we have to fetch from the next day, don't fetch unless
    # in the jpeg index.
    if len(dayVals) > 1 and options.type != 'jpeg':
        jpegFolder = icebridge_common.getJpegFolder(
            os.path.dirname(outputFolder))
        jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
        (jpegFrameDict,
         jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)

    orthoStamp = {}
    if options.type == 'fireball':
        # This is a bugfix. Ensure that the fireball DEM has not just
        # the same frame number, but also same timestamp as the ortho.
        orthoFolder = icebridge_common.getOrthoFolder(
            os.path.dirname(outputFolder))
        orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
        (orthoFrameDict,
         orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
        for frame in sorted(orthoFrameDict.keys()):
            filename = orthoFrameDict[frame]
            [imageDateString,
             imageTimeString] = icebridge_common.parseTimeStamps(filename)
            orthoStamp[frame] = imageTimeString

    for dayVal in dayVals:

        if len(dayVals) > 1:
            currIndexPath = indexPath + '.day' + str(dayVal)
            if options.refetchIndex:
                os.system('rm -f ' + currIndexPath)

        # Find folderUrl which contains all of the files
        if options.type in LIDAR_TYPES:
            options.allFrames = True  # For lidar, always get all the frames!

            # For lidar, the data can come from one of three sources.
            # Unfortunately sometimes there is more than one source, and then
            # we need to pick by latitude.
            folderUrls = []
            lidar_types = []
            for lidar in LIDAR_TYPES:
                folderUrl = getFolderUrl(
                    options.yyyymmdd,
                    options.year,
                    options.month,
                    options.day,
                    dayVal,  # note here the dayVal
                    options.site,
                    lidar)
                logger.info('Checking lidar URL: ' + folderUrl)
                if checkIfUrlExists(folderUrl):
                    logger.info('Found match with lidar type: ' + lidar)
                    folderUrls.append(folderUrl)
                    lidar_types.append(lidar)

            if len(folderUrls) == 0:
                logger.info(
                    'WARNING: Could not find any lidar data for the given date!'
                )

            elif len(folderUrls) == 1:
                # Unique solution
                folderUrl = folderUrls[0]
                options.type = lidar_types[0]

            elif len(folderUrls) >= 2:
                # Multiple solutions. Pick the good one by latitude.
                logger.info("Multiples URLs to search: " +
                            " ".join(folderUrls))
                count = -1
                isGood = False
                for folderUrl in folderUrls:
                    count += 1
                    (localFrameDict, localUrlDict) = \
                                     fetchAndParseIndexFileAux(isSouth,
                                                               separateByLat, dayVal,
                                                               baseCurlCmd, folderUrl,
                                                               currIndexPath,
                                                               lidar_types[count])
                    for frame in sorted(localFrameDict.keys()):
                        filename = localFrameDict[frame]
                        xmlFile = icebridge_common.xmlFile(filename)
                        url = os.path.join(folderUrl, xmlFile)

                        # Download the file
                        curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile
                        logger.info(curlCmd)
                        p = subprocess.Popen(curlCmd, shell=True)
                        os.waitpid(p.pid, 0)

                        latitude = icebridge_common.parseLatitude(xmlFile)
                        if os.path.exists(xmlFile): os.remove(xmlFile)

                        if hasGoodLat(latitude, isSouth):
                            isGood = True
                            options.type = lidar_types[count]
                            logger.info("Good latitude " + str(latitude) +
                                        ", will use " + folderUrl +
                                        " of type " + lidar_types[count])
                        else:
                            logger.info("Bad latitude " + str(latitude) +
                                        ", will not use " + folderUrl +
                                        " of type " + lidar_types[count])

                        # Stop at first file no matter what
                        break

                    if isGood:
                        break

                if not isGood:
                    if options.type in LIDAR_TYPES and options.ignoreMissingLidar:
                        logger.info("No lidar. None of these URLs are good: " +
                                    " ".join(folderUrls))
                    else:
                        raise Exception("None of these URLs are good: " +
                                        " ".join(folderUrls))

        else:  # Other cases are simpler
            folderUrl = getFolderUrl(
                options.yyyymmdd,
                options.year,
                options.month,
                options.day,
                dayVal,  # note here the dayVal
                options.site,
                options.type)

        logger.info('Fetching from URL: ' + folderUrl)
        (localFrameDict, localUrlDict) = \
                         fetchAndParseIndexFileAux(isSouth,
                                                   separateByLat, dayVal,
                                                   baseCurlCmd, folderUrl,
                                                   currIndexPath, options.type)

        # Append to the main index
        for frame in sorted(localFrameDict.keys()):

            if options.type == 'fireball':
                # This is a bugfix. Ensure that the fireball DEM has not just
                # the same frame number, but also same timestamp as the ortho.
                # Otherwise we may accidentally getting one from next day.
                [imageDateString, imageTimeString] = \
                                  icebridge_common.parseTimeStamps(localFrameDict[frame])
                if frame not in orthoStamp:
                    #logger.info("Missing ortho for fireball: " + localFrameDict[frame])
                    continue
                if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000:
                    # Apparently a tolerance is needed. Use 10 seconds, so the number 1000.
                    #logger.info("Will not use fireball DEM whose timestamp differs from ortho.")
                    #logger.info("Fireball is: " + localFrameDict[frame])
                    #logger.info("Ortho is:    " + orthoFrameDict[frame])
                    continue

            # Fetch from next day, unless already have a value. And don't fetch
            # frames not in the jpeg index.
            if len(dayVals) > 1 and options.type != 'jpeg':
                if not frame in jpegFrameDict.keys(): continue
                if frame in frameDict.keys(): continue

            frameDict[frame] = localFrameDict[frame]
            urlDict[frame] = localUrlDict[frame]

    # Write the combined index file
    icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict)

    return parsedIndexPath
def pairLidarFiles(lidarFolder, skipValidate, logger):
    '''For each pair of lidar files generate a double size point cloud.
       We can use these later since they do not have any gaps between adjacent files.'''

    logger.info('Generating lidar pairs...')

    # Create the output folder
    pairedFolder = icebridge_common.getPairedLidarFolder(lidarFolder)
    os.system('mkdir -p ' + pairedFolder)

    convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)
    if not os.path.exists(convLidarFile):
        raise Exception("Missing file: " + convLidarFile)

    (lidarDict, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile)
    lidarExt = ''
    for frame in lidarDict:
        lidarExt = icebridge_common.fileExtension(lidarDict[frame])

    numLidarFiles = len(lidarDict.keys())

    pairedDict = {}

    # Loop through all pairs of csv files in the folder
    badFiles = False
    lidarKeys = sorted(lidarDict.keys())
    for i in range(len(lidarKeys) - 1):

        thisFile = lidarDict[lidarKeys[i]]
        nextFile = lidarDict[lidarKeys[i + 1]]

        date2, time2 = icebridge_common.parseTimeStamps(nextFile)

        # Record the name with the second file
        # - More useful because the time for the second file represents the middle of the file.
        outputName = icebridge_common.lidar_pair_prefix(
        ) + date2 + '_' + time2 + lidarExt

        pairedDict[lidarKeys[i]] = outputName

        # Handle paths
        path1 = os.path.join(lidarFolder, thisFile)
        path2 = os.path.join(lidarFolder, nextFile)
        outputPath = os.path.join(pairedFolder, outputName)

        if not os.path.exists(path1) or not os.path.exists(path2):
            logger.info("Cannot create " + outputPath +
                        " as we are missing its inputs")
            # If the inputs are missing, but the output is there, most likely it is corrupt.
            # Wipe it. Hopefully a subsequent fetch and convert step will bring it back.
            if os.path.exists(outputPath):
                logger.info("Wiping: " + outputPath)
                os.system('rm -f ' + outputPath)  # will not throw
                badFiles = True
            continue

        # Skip existing valid files
        if skipValidate:
            if os.path.exists(outputPath):
                logger.info("File exists, skipping: " + outputPath)
                continue
        else:
            if icebridge_common.isValidLidarCSV(outputPath):
                #logger.info("File exists and is valid, skipping: " + outputPath)
                continue

        # Concatenate the two files
        cmd1 = 'cat ' + path1 + ' > ' + outputPath
        cmd2 = 'tail -n +2 -q ' + path2 + ' >> ' + outputPath
        logger.info(cmd1)
        p = subprocess.Popen(cmd1, stdout=subprocess.PIPE, shell=True)
        out, err = p.communicate()
        logger.info(cmd2)
        p = subprocess.Popen(cmd2, stdout=subprocess.PIPE, shell=True)
        out, err = p.communicate()

        if not icebridge_common.isValidLidarCSV(outputPath):
            logger.error('Failed to generate merged LIDAR file, will wipe: ' +
                         outputPath)
            os.system('rm -f ' + outputPath)  # will not throw
            badFiles = True

    pairedLidarFile = icebridge_common.getPairedIndexFile(pairedFolder)

    willWritePairedFile = False
    if not os.path.exists(pairedLidarFile):
        willWritePairedFile = True
    else:
        # Bugfix: Sometimes the written converted file has the wrong size, maybe
        # something got interrupted earlier.
        (lidarDictIn,
         dummyUrlDict) = icebridge_common.readIndexFile(pairedLidarFile)
        if lidarDictIn != pairedDict:
            willWritePairedFile = True

    if willWritePairedFile:
        logger.info("Writing: " + pairedLidarFile)
        icebridge_common.writeIndexFile(pairedLidarFile, pairedDict, {})

    return (not badFiles)
def convertLidarDataToCsv(lidarFolder, startFrame, stopFrame, skipValidate,
                          logger):
    '''Make sure all lidar data is available in a readable text format.
       Returns false if any files failed to convert.'''

    logger.info('Converting LIDAR files...')

    lidarIndexPath = icebridge_common.csvIndexFile(lidarFolder)
    (frameDict, urlDict) = icebridge_common.readIndexFile(lidarIndexPath)

    if not skipValidate:
        validFilesList = icebridge_common.validFilesList(
            os.path.dirname(lidarFolder), startFrame, stopFrame)
        validFilesSet = set()
        validFilesSet = icebridge_common.updateValidFilesListFromDisk(
            validFilesList, validFilesSet)
        numInitialValidFiles = len(validFilesSet)

    convDict = {}

    # Loop through all files in the folder
    badFiles = False
    for frame in sorted(frameDict.keys()):

        f = frameDict[frame]
        extension = icebridge_common.fileExtension(f)

        # Only interested in a few file types
        if (extension != '.qi') and (extension != '.hdf5') and (extension !=
                                                                '.h5'):
            convDict[frame] = f  # these are already in plain text
            continue

        convDict[frame] = os.path.splitext(f)[0] + '.csv'
        outputPath = os.path.join(lidarFolder, convDict[frame])

        # Handle paths
        fullPath = os.path.join(lidarFolder, f)
        if not os.path.exists(fullPath):
            logger.info("Cannot convert missing file: " + fullPath)
            continue

        # If the input is invalid, wipe both it, its xml, and the output
        # Hopefully there will be a subsquent fetch step where it will get
        # refetched.
        if not icebridge_common.hasValidChkSum(fullPath, logger):
            logger.info("Will wipe invalid file: " + fullPath)
            xmlFile = icebridge_common.xmlFile(fullPath)
            os.system('rm -f ' + fullPath)  # will not throw
            os.system('rm -f ' + xmlFile)  # will not throw
            os.system('rm -f ' + outputPath)  # will not throw
            badFiles = True
            continue

        # Skip existing valid files
        if skipValidate:
            if os.path.exists(outputPath):
                logger.info("File exists, skipping: " + outputPath)
                continue
        else:
            if outputPath in validFilesSet and os.path.exists(outputPath):
                #logger.info('Previously validated: ' + outputPath) # verbose
                continue
            if icebridge_common.isValidLidarCSV(outputPath):
                #logger.info("File exists and is valid, skipping: " + outputPath)
                continue

        # Call the conversion
        logger.info("Process " + fullPath)
        extract_icebridge_ATM_points.main([fullPath])

        # Check the result
        if not icebridge_common.isValidLidarCSV(outputPath):
            logger.error('Failed to parse LIDAR file, will wipe: ' +
                         outputPath)
            os.system('rm -f ' + outputPath)  # will not throw
            badFiles = True
        else:
            if not skipValidate:
                validFilesSet.add(outputPath)  # mark it as validated

    convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)

    willWriteConvFile = False
    if not os.path.exists(convLidarFile):
        willWriteConvFile = True
    else:
        # Bugfix: Sometimes the written converted file has the wrong size, maybe
        # something got interrupted earlier.
        (lidarDictIn,
         dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile)
        if lidarDictIn != convDict:
            willWriteConvFile = True

    if willWriteConvFile:
        logger.info("Writing: " + convLidarFile)
        icebridge_common.writeIndexFile(convLidarFile, convDict, {})

    if not skipValidate:
        # Write to disk the list of validated files, but only if new
        # validations happened.  First re-read that list, in case a
        # different process modified it in the meantime, such as if two
        # managers are running at the same time.
        numFinalValidFiles = len(validFilesSet)
        if numInitialValidFiles != numFinalValidFiles:
            validFilesSet = icebridge_common.updateValidFilesListFromDisk(
                validFilesList, validFilesSet)
            icebridge_common.writeValidFilesList(validFilesList, validFilesSet)

    return (not badFiles)
def fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder):
    '''Create a list of all files that must be fetched unless done already.'''

    # For AN 20091112, etc, some of the ortho images are stored at the
    # beginning of the next day's flight. Need to sort this out, and
    # it is tricky. More comments within the code. 
    fetchNextDay = True
    
    separateByLat = (options.type == 'ortho' and isInSeparateByLatTable(options.yyyymmdd))
    if separateByLat:
        # Here we won't fetch the next day, we will just separate by latitude within
        # a given day
        fetchNextDay = False

    orthoOrFireball = ( (options.type == 'ortho') or (options.type == 'fireball') )

    if fetchNextDay:
        # Normally we fetch for next day only for ortho or fireball. However,
        # for one single special flight, we do it for jpeg too, as then
        # the jpegs are also split. 
       if orthoOrFireball or \
          ((options.type == 'jpeg') and twoFlightsInOneDay(options.site, options.yyyymmdd)):
           fetchNextDay = True
       else:
           fetchNextDay = False
           
    # If we need to parse the next flight day as well, as expected in some runs,
    # we will fetch two html files, but create a single index out of them.
    dayVals = [0]
    if fetchNextDay:
        dayVals.append(1)

    indexPath       = icebridge_common.htmlIndexFile(outputFolder)
    
    currIndexPath   = indexPath
    parsedIndexPath = icebridge_common.csvIndexFile(outputFolder)

    if options.refetchIndex:
        os.system('rm -f ' + indexPath)
        os.system('rm -f ' + parsedIndexPath)

    if icebridge_common.fileNonEmpty(parsedIndexPath):
        logger.info('Already have the index file ' + parsedIndexPath + ', keeping it.')
        return parsedIndexPath
    
    frameDict  = {}
    urlDict    = {}

    # We need the list of jpeg frames. Sometimes when fetching ortho images,
    # and we have to fetch from the next day, don't fetch unless
    # in the jpeg index.
    if len(dayVals) > 1 and options.type != 'jpeg':
        jpegFolder = icebridge_common.getJpegFolder(os.path.dirname(outputFolder))
        jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder)
        (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath)

    orthoStamp = {}
    if options.type == 'fireball':
        # This is a bugfix. Ensure that the fireball DEM has not just
        # the same frame number, but also same timestamp as the ortho.
        orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(outputFolder))
        orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder)
        (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath)
        for frame in sorted(orthoFrameDict.keys()):
            filename = orthoFrameDict[frame]
            [imageDateString, imageTimeString] = icebridge_common.parseTimeStamps(filename)
            orthoStamp[frame] = imageTimeString
            
    for dayVal in dayVals:

        if len(dayVals) > 1:
            currIndexPath = indexPath + '.day' + str(dayVal)
            if options.refetchIndex:
                os.system('rm -f ' + currIndexPath)
            
        # Find folderUrl which contains all of the files
        if options.type in LIDAR_TYPES:
            options.allFrames = True # For lidar, always get all the frames!
            
            # For lidar, the data can come from one of three sources.
            # Unfortunately sometimes there is more than one source, and then
            # we need to pick by latitude.
            folderUrls = []
            lidar_types = []
            for lidar in LIDAR_TYPES:
                folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month,
                                         options.day, dayVal, # note here the dayVal
                                         options.site, lidar)
                logger.info('Checking lidar URL: ' + folderUrl)
                if checkIfUrlExists(folderUrl, baseCurlCmd):
                    logger.info('Found match with lidar type: ' + lidar)
                    folderUrls.append(folderUrl)
                    lidar_types.append(lidar)

            if len(folderUrls) == 0:
                logger.info('WARNING: Could not find any lidar data for the given date!')

            elif len(folderUrls) == 1:
                # Unique solution
                folderUrl = folderUrls[0]
                options.type = lidar_types[0]

            elif len(folderUrls) >= 2:
                # Multiple solutions. Pick the good one by latitude.
                logger.info("Multiples URLs to search: " + " ".join(folderUrls))
                count = -1
                isGood = False
                for folderUrl in folderUrls:
                    count += 1
                    (localFrameDict, localUrlDict) = \
                                     fetchAndParseIndexFileAux(isSouth,
                                                               separateByLat, dayVal,
                                                               baseCurlCmd, folderUrl,
                                                               currIndexPath,
                                                               lidar_types[count])
                    for frame in sorted(localFrameDict.keys()):
                        filename = localFrameDict[frame]
                        xmlFile  = icebridge_common.xmlFile(filename)
                        url      = os.path.join(folderUrl, xmlFile)
                                        
                        # Download the file
                        curlCmd = baseCurlCmd + ' ' + url + ' > ' + xmlFile
                        logger.info(curlCmd)
                        p = subprocess.Popen(curlCmd, shell=True, universal_newlines=True)
                        os.waitpid(p.pid, 0)
                        
                        latitude = icebridge_common.parseLatitude(xmlFile)
                        if os.path.exists(xmlFile): os.remove(xmlFile)

                        if hasGoodLat(latitude, isSouth):
                            isGood = True
                            options.type = lidar_types[count]
                            logger.info("Good latitude " + str(latitude) + ", will use " +
                                        folderUrl + " of type " + lidar_types[count])
                        else:
                            logger.info("Bad latitude " + str(latitude) + ", will not use " +
                                        folderUrl + " of type " + lidar_types[count])
                            
                        # Stop at first file no matter what
                        break

                    if isGood:
                        break

                if not isGood:
                    if options.type in LIDAR_TYPES and options.ignoreMissingLidar:
                        logger.info("No lidar. None of these URLs are good: " +
                                    " ".join(folderUrls))
                    else:
                        raise Exception("None of these URLs are good: " +
                                        " ".join(folderUrls))
                    
        else: # Other cases are simpler
            folderUrl = getFolderUrl(options.yyyymmdd, options.year, options.month,
                                     options.day, dayVal, # note here the dayVal
                                     options.site, options.type)

        logger.info('Fetching from URL: ' + folderUrl)
        (localFrameDict, localUrlDict) = \
                         fetchAndParseIndexFileAux(isSouth,
                                                   separateByLat, dayVal,
                                                   baseCurlCmd, folderUrl,
                                                   currIndexPath, options.type)

        # Append to the main index
        for frame in sorted(localFrameDict.keys()):

            if options.type == 'fireball':
                # This is a bugfix. Ensure that the fireball DEM has not just
                # the same frame number, but also same timestamp as the ortho.
                # Otherwise we may accidentally getting one from next day.
                [imageDateString, imageTimeString] = \
                                  icebridge_common.parseTimeStamps(localFrameDict[frame])
                if frame not in orthoStamp:
                    #logger.info("Missing ortho for fireball: " + localFrameDict[frame])
                    continue
                if abs(int(imageTimeString) - int(orthoStamp[frame])) > 1000:
                    # Apparently a tolerance is needed. Use 10 seconds, so the number 1000.
                    #logger.info("Will not use fireball DEM whose timestamp differs from ortho.")
                    #logger.info("Fireball is: " + localFrameDict[frame])
                    #logger.info("Ortho is:    " + orthoFrameDict[frame])
                    continue
                
            # Fetch from next day, unless already have a value. And don't fetch
            # frames not in the jpeg index.
            if len(dayVals) > 1 and options.type != 'jpeg':
                if not frame in jpegFrameDict.keys(): continue
                if frame in frameDict.keys(): continue
                
            frameDict[frame] = localFrameDict[frame]
            urlDict[frame]   = localUrlDict[frame]
        
    # Write the combined index file
    icebridge_common.writeIndexFile(parsedIndexPath, frameDict, urlDict)
            
    return parsedIndexPath
def pairLidarFiles(lidarFolder, skipValidate, logger):
    '''For each pair of lidar files generate a double size point cloud.
       We can use these later since they do not have any gaps between adjacent files.'''
    
    logger.info('Generating lidar pairs...')

    # Create the output folder
    pairedFolder = icebridge_common.getPairedLidarFolder(lidarFolder)
    os.system('mkdir -p ' + pairedFolder)

    convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)
    if not os.path.exists(convLidarFile):
        raise Exception("Missing file: " + convLidarFile)

    (lidarDict, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile)
    lidarExt = ''
    for frame in lidarDict:
        lidarExt = icebridge_common.fileExtension(lidarDict[frame])

    numLidarFiles = len(lidarDict.keys())

    pairedDict = {}
    
    # Loop through all pairs of csv files in the folder    
    badFiles = False
    lidarKeys = sorted(lidarDict.keys())
    for i in range(len(lidarKeys)-1):
        
        thisFile = lidarDict[lidarKeys[i  ]]
        nextFile = lidarDict[lidarKeys[i+1]]

        date2, time2 = icebridge_common.parseTimeStamps(nextFile)
        
        # Record the name with the second file
        # - More useful because the time for the second file represents the middle of the file.
        outputName = icebridge_common.lidar_pair_prefix() + date2 +'_'+ time2 + lidarExt

        pairedDict[lidarKeys[i]] = outputName
        
        # Handle paths
        path1      = os.path.join(lidarFolder, thisFile)
        path2      = os.path.join(lidarFolder, nextFile)
        outputPath = os.path.join(pairedFolder, outputName)

        if not os.path.exists(path1) or not os.path.exists(path2):
            logger.info("Cannot create " + outputPath + " as we are missing its inputs")
            # If the inputs are missing, but the output is there, most likely it is corrupt.
            # Wipe it. Hopefully a subsequent fetch and convert step will bring it back.
            if os.path.exists(outputPath):
                logger.info("Wiping: " + outputPath)
                os.system('rm -f ' + outputPath) # will not throw
                badFiles = True
            continue
        
        # Skip existing valid files
        if skipValidate:
            if os.path.exists(outputPath):
                logger.info("File exists, skipping: " + outputPath)
                continue
        else:
            if icebridge_common.isValidLidarCSV(outputPath):
                #logger.info("File exists and is valid, skipping: " + outputPath)
                continue

        # Concatenate the two files
        cmd1 = 'cat ' + path1 + ' > ' + outputPath
        cmd2 = 'tail -n +2 -q ' + path2 + ' >> ' + outputPath
        logger.info(cmd1)
        p        = subprocess.Popen(cmd1, stdout=subprocess.PIPE, shell=True,
                                    universal_newlines=True)
        out, err = p.communicate()
        logger.info(cmd2)
        p        = subprocess.Popen(cmd2, stdout=subprocess.PIPE, shell=True,
                                    universal_newlines=True)
        out, err = p.communicate()

        if not icebridge_common.isValidLidarCSV(outputPath):
            logger.error('Failed to generate merged LIDAR file, will wipe: ' + outputPath)
            os.system('rm -f ' + outputPath) # will not throw
            badFiles = True

    pairedLidarFile = icebridge_common.getPairedIndexFile(pairedFolder)
    
    willWritePairedFile = False
    if not os.path.exists(pairedLidarFile):
        willWritePairedFile = True
    else: 
        # Bugfix: Sometimes the written converted file has the wrong size, maybe
        # something got interrupted earlier.
        (lidarDictIn, dummyUrlDict) = icebridge_common.readIndexFile(pairedLidarFile)
        if lidarDictIn != pairedDict:
            willWritePairedFile = True
    
    if willWritePairedFile:
        logger.info("Writing: " + pairedLidarFile)
        icebridge_common.writeIndexFile(pairedLidarFile, pairedDict, {})

    return (not badFiles)
def convertLidarDataToCsv(lidarFolder, startFrame, stopFrame, 
                          skipValidate, logger):
    '''Make sure all lidar data is available in a readable text format.
       Returns false if any files failed to convert.'''

    logger.info('Converting LIDAR files...')

    lidarIndexPath = icebridge_common.csvIndexFile(lidarFolder)
    (frameDict, urlDict) = icebridge_common.readIndexFile(lidarIndexPath)

    if not skipValidate:
        validFilesList = icebridge_common.validFilesList(os.path.dirname(lidarFolder),
                                                         startFrame, stopFrame)
        validFilesSet = set()
        validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet)
        numInitialValidFiles = len(validFilesSet)

    convDict = {}
    
    # Loop through all files in the folder
    badFiles = False
    for frame in sorted(frameDict.keys()):

        f = frameDict[frame]
        extension = icebridge_common.fileExtension(f)
        
        # Only interested in a few file types
        if (extension != '.qi') and (extension != '.hdf5') and (extension != '.h5'):
            convDict[frame] = f # these are already in plain text
            continue

        convDict[frame] = os.path.splitext(f)[0] + '.csv'
        outputPath = os.path.join(lidarFolder, convDict[frame])

        # Handle paths
        fullPath = os.path.join(lidarFolder, f)
        if not os.path.exists(fullPath):
            logger.info("Cannot convert missing file: " + fullPath)
            continue

        # If the input is invalid, wipe both it, its xml, and the output
        # Hopefully there will be a subsquent fetch step where it will get
        # refetched.
        if not icebridge_common.hasValidChkSum(fullPath, logger):
            logger.info("Will wipe invalid file: " + fullPath)
            xmlFile = icebridge_common.xmlFile(fullPath)
            os.system('rm -f ' + fullPath) # will not throw
            os.system('rm -f ' + xmlFile) # will not throw
            os.system('rm -f ' + outputPath) # will not throw
            badFiles = True
            continue

        # Skip existing valid files
        if skipValidate:
            if os.path.exists(outputPath):
                logger.info("File exists, skipping: " + outputPath)
                continue
        else:
            if outputPath in validFilesSet and os.path.exists(outputPath):
                #logger.info('Previously validated: ' + outputPath) # verbose
                continue
            if icebridge_common.isValidLidarCSV(outputPath):
                #logger.info("File exists and is valid, skipping: " + outputPath)
                continue
        
        # Call the conversion
        logger.info("Process " + fullPath)
        extract_icebridge_ATM_points.main([fullPath])
        
        # Check the result
        if not icebridge_common.isValidLidarCSV(outputPath):
            logger.error('Failed to parse LIDAR file, will wipe: ' + outputPath)
            os.system('rm -f ' + outputPath) # will not throw            
            badFiles = True
        else:
            if not skipValidate:
                validFilesSet.add(outputPath) # mark it as validated
            
    convLidarFile = icebridge_common.getConvertedLidarIndexFile(lidarFolder)

    willWriteConvFile = False
    if not os.path.exists(convLidarFile):
        willWriteConvFile = True
    else: 
        # Bugfix: Sometimes the written converted file has the wrong size, maybe
        # something got interrupted earlier.
        (lidarDictIn, dummyUrlDict) = icebridge_common.readIndexFile(convLidarFile)
        if lidarDictIn != convDict:
            willWriteConvFile = True
            
    if willWriteConvFile:
        logger.info("Writing: " + convLidarFile)
        icebridge_common.writeIndexFile(convLidarFile, convDict, {})
        
    if not skipValidate:
        # Write to disk the list of validated files, but only if new
        # validations happened.  First re-read that list, in case a
        # different process modified it in the meantime, such as if two
        # managers are running at the same time.
        numFinalValidFiles = len(validFilesSet)
        if numInitialValidFiles != numFinalValidFiles:
            validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList,
                                                                          validFilesSet)
            icebridge_common.writeValidFilesList(validFilesList, validFilesSet)

    return (not badFiles)