def extract_hdf5_points(inputPath): '''Extract data from an Icebridge hdf5 file. Use this ASP format code to read: --csv-format 1:lat,2:lon,3:height_above_datum ''' outputPath = inputPath.replace('.h5', '.csv') tempLat = inputPath + '_lat.txt' tempLon = inputPath + '_lon.txt' tempAlt = inputPath + '_alt.txt' lookInLibexec = True execPath = asp_system_utils.which('h5dump', lookInLibexec) cmd = execPath + ' -o '+tempLat+' -m "%.7f" --noindex -w 20 --dataset=latitude ' + inputPath os.system(cmd) cmd = execPath + ' -o '+tempLon+' -m "%.7f" --noindex -w 20 --dataset=longitude ' + inputPath os.system(cmd) cmd = execPath + ' -o '+tempAlt+' -m "%.7f" --noindex -w 20 --dataset=elevation ' + inputPath os.system(cmd) execPath = asp_system_utils.which('paste', lookInLibexec) cmd = execPath + ' ' + tempLat + ' ' + tempLon + ' ' + tempAlt + ' > ' + outputPath os.system(cmd) if os.path.exists(tempLat): os.remove(tempLat) if os.path.exists(tempLon): os.remove(tempLon) if os.path.exists(tempAlt): os.remove(tempAlt) if os.path.exists(outputPath): print 'Wrote file: ' + outputPath else: print 'Failed to write file ' + outputPath + '!!!'
def corrFireball(demFolder, corrDemFolder, isNorth): '''Correct Fireball DEMs''' logger = logging.getLogger(__name__) logger.info('Correcting Fireball DEMs ...') # Loop through all the input images os.system('mkdir -p ' + corrDemFolder) demFiles = os.listdir(demFolder) for demFile in demFiles: inputPath = os.path.join(demFolder, demFile) if not icebridge_common.isDEM(inputPath): continue # Make sure the timestamp and frame number are in the output file name outputPath = os.path.join(corrDemFolder, os.path.basename(inputPath)) # Skip existing files if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue execPath = asp_system_utils.which('correct_icebridge_l3_dem') cmd = (('%s %s %s %d') % (execPath, inputPath, outputPath, isNorth)) logger.info(cmd) os.system(cmd) if not icebridge_common.isValidImage(outputPath): raise Exception('Failed to convert dem file: ' + demFile)
def extract_qi_points(inputPath): '''Extract data from a n Icebridge qi file. Use this ASP format code to read: --csv-format 1:lat,2:lon,3:height_above_datum ''' # Extract the data from the name name = os.path.basename(inputPath) start = name.find('_') stop = name.rfind('_') dateString = name[start+1:stop] year = int(dateString[0:4]) month = int(dateString[4:6]) # Check if the file is little endian endianFlag = ' -L ' if (year < 2010) or ((year == 2010) and (month < 10)): endianFlag = ' ' # Earlier dates are big endian, later dates are little endian. outputPath = inputPath.replace('.qi', '.csv') lookInLibexec = True execPath = asp_system_utils.which('qi2txt', lookInLibexec) cmd = execPath + ' -S' + endianFlag + inputPath + ' > ' + outputPath print(cmd) os.system(cmd) if os.path.exists(outputPath): print ('Wrote file: ' + outputPath) else: print ('Failed to write file ' + outputPath + '!!!')
def extract_hdf5_points(inputPath): '''Extract data from an Icebridge hdf5 file. Use this ASP format code to read: --csv-format 1:lat,2:lon,3:height_above_datum ''' outputPath = inputPath.replace('.h5', '.csv') tempLat = inputPath + '_lat.txt' tempLon = inputPath + '_lon.txt' tempAlt = inputPath + '_alt.txt' lookInLibexec = True execPath = asp_system_utils.which('h5dump', lookInLibexec) # Extract the three values of interest one at a time cmd = execPath + ' -o '+tempLat+' -m "%.7f" --noindex -w 20 --dataset=latitude ' + \ inputPath + ' >/dev/null' print(cmd) os.system(cmd) cmd = execPath + ' -o '+tempLon+' -m "%.7f" --noindex -w 20 --dataset=longitude ' + \ inputPath + ' >/dev/null' print(cmd) os.system(cmd) cmd = execPath + ' -o '+tempAlt+' -m "%.7f" --noindex -w 20 --dataset=elevation ' + \ inputPath + ' >/dev/null' print(cmd) os.system(cmd) # Merge the three one column files into a single three column file execPath = asp_system_utils.which('paste', lookInLibexec) cmd = execPath + ' ' + tempLat + ' ' + tempLon + ' ' + tempAlt + ' > ' + outputPath print(cmd) os.system(cmd) if os.path.exists(tempLat): os.remove(tempLat) if os.path.exists(tempLon): os.remove(tempLon) if os.path.exists(tempAlt): os.remove(tempAlt) # Remove any trailing commas in lines and strip all lines without commas. cmd = "sed -i 's/,$//;/,/!d' " + outputPath print(cmd) os.system(cmd) if os.path.exists(outputPath): print('Wrote: ' + outputPath) else: print('Failed to write: ' + outputPath)
def convertCoordinate(input_srs_string, output_srs_string, x, y): '''Convert a single 2D coordinate between proj.4 coordinate systems.''' cmd = [asp_system_utils.which('gdaltransform'), '-s_srs', input_srs_string, '-t_srs', output_srs_string] p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding = 'utf8') input_str = str(x) + " " + str(y) out, err = p.communicate(input = input_str) parts = out.split() return (float(parts[0]), float(parts[1]))
def convertCoords(x, y, projStringIn, projStringOut): '''Convert coordinates from one projection to another''' # Using subprocess32 to access the timeout argument which is not always present in subprocess cmd = [asp_system_utils.which('gdaltransform'), '-s_srs', projStringIn, '-t_srs', projStringOut] #print(" ".join(cmd)) p = subprocess32.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=False) textOutput, err = p.communicate( ('%f %f\n' % (x, y)), timeout=0.5 ) parts = textOutput.split() return ( float(parts[0]), float(parts[1]) )
def fetchIndices(options, logger): '''Fetch the csv indices of available files.''' logger.info("Fetch indices from NSIDC.") pythonPath = asp_system_utils.which('python') cmd = ( (pythonPath + ' ' + icebridge_common.fullPath('full_processing_script.py') + \ ' --camera-calibration-folder %s --reference-dem-folder %s --site %s ' + \ '--yyyymmdd %s --stop-after-index-fetch --no-nav ' ) % \ (options.inputCalFolder, options.refDemFolder, options.site, options.yyyymmdd)) logger.info(cmd) start_time() os.system(cmd) stop_time("fetch index", logger)
def convertCoords(x, y, projStringIn, projStringOut): '''Convert coordinates from one projection to another''' # Using subprocess32 to access the timeout argument which is not always present in subprocess cmd = [asp_system_utils.which('gdaltransform'), '-s_srs', projStringIn, '-t_srs', projStringOut] #print(" ".join(cmd)) p = subprocess32.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=False, universal_newlines=True) textOutput, err = p.communicate( ('%f %f\n' % (x, y)), timeout=0.5 ) parts = textOutput.split() return ( float(parts[0]), float(parts[1]) )
def isValidImage(filename): if not os.path.exists(filename): return False gdalinfoPath = asp_system_utils.which("gdalinfo") cmd = gdalinfoPath + ' ' + filename p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: return False return True
def convertCoordinate(input_srs_string, output_srs_string, x, y): '''Convert a single 2D coordinate between proj.4 coordinate systems.''' cmd = [asp_system_utils.which('gdaltransform'), '-s_srs', input_srs_string, '-t_srs', output_srs_string] try: # Fancier way, if installed. p = subprocess32.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=False, universal_newlines=True) text, err = p.communicate( ('%f %f\n' % (x, y)), timeout=0.5 ) except: # Simpler way. p = subprocess.Popen(cmd, stdin=subprocess.PIPE,stdout=subprocess.PIPE) text = p.communicate("%f %f" % (x, y))[0] parts = text.split() return (float(parts[0]), float(parts[1]))
def label_images(inputFolder, outputFolder, minFrame, maxFrame, trainingPath, numProcesses): '''Apply the labeling algorithm to a single image.''' # Format the input path # Run the label tool toolPath = asp_system_utils.which('batch_process_mp.py') NO_SPLITTING = 1 # Plenty of RAM to load these images cmd = ('%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' % (toolPath, inputFolder, outputFolder, minFrame, maxFrame, trainingPath, NO_SPLITTING, numProcesses)) print cmd os.system(cmd)
def doesImageHaveGeoData(imagePath): '''Returns true if a file has geo data associated with it''' if not os.path.exists(imagePath): raise Exception('Image file ' + imagePath + ' not found!') # Call command line tool silently cmd = [asp_system_utils.which('gdalinfo'), imagePath, '-proj4'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) textOutput, err = p.communicate() # For now we just do a very simple check if "Coordinate System is `'" in textOutput: return False else: return True
def cameraFromOrthoWrapper(inputPath, orthoPath, inputCamFile, outputCamFile, \ refDemPath, numThreads): '''Generate a camera model from a single ortho file''' logger = logging.getLogger(__name__) # Call ortho2pinhole command ortho2pinhole = asp_system_utils.which("ortho2pinhole") cmd = (('%s %s %s %s %s --reference-dem %s --threads %d') % (ortho2pinhole, inputPath, orthoPath, inputCamFile, outputCamFile, refDemPath, numThreads)) logger.info(cmd) os.system(cmd) if not os.path.exists(outputCamFile): # This function is getting called from a pool, so just log the failure. logger.error('Failed to convert ortho file: ' + orthoFile)
def label_images(inputFolder, outputFolder, minFrame, maxFrame, trainingPath, numProcesses): '''Apply the labeling algorithm to a single image.''' # Format the input path # Run the label tool toolPath = asp_system_utils.which('batch_process_mp.py') NO_SPLITTING = 1 # Plenty of RAM to load these images cmd = ( '%s %s --output_dir %s --min_frame %d --max_frame %d srgb %s --splits %d --parallel %d' % (toolPath, inputFolder, outputFolder, minFrame, maxFrame, trainingPath, NO_SPLITTING, numProcesses)) print cmd os.system(cmd)
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder = True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame]) except Exception as e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90 ' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90 ' cmd = ('%s %s -colorspace Gray %s%s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE|re.MULTILINE|re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) if badFiles: logger.error("Converstion of JPEGs failed. If any files were corrupted, " + "they were removed, and need to be re-fetched.") return (not badFiles)
def getImageGeoInfo(imagePath, getStats=True): """Obtains some image geo information from gdalinfo in dictionary format""" if not os.path.exists(imagePath): raise Exception('Error: input file ' + imagePath + ' does not exist!') outputDict = {} # Call command line tool silently cmd = [asp_system_utils.which('gdalinfo'), imagePath, '-proj4'] if getStats: cmd.append('-stats') p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) textOutput, err = p.communicate() # Get the size in pixels imageSizeLine = asp_string_utils.getLineAfterText(textOutput, 'Size is ') sizeVals = imageSizeLine.split(',') outputDict['image_size'] = (int(sizeVals[0]), int(sizeVals[1])) # Get origin location and pixel size originLine = asp_string_utils.getLineAfterText(textOutput, 'Origin = ') pixelSizeLine = asp_string_utils.getLineAfterText(textOutput, 'Pixel Size = ') originVals = asp_string_utils.getNumbersInParentheses(originLine) pixelSizeVals = asp_string_utils.getNumbersInParentheses(pixelSizeLine) outputDict['origin'] = originVals outputDict['pixel_size'] = pixelSizeVals # Get bounding box in projected coordinates upperLeftLine = asp_string_utils.getLineAfterText(textOutput, 'Upper Left') lowerRightLine = asp_string_utils.getLineAfterText(textOutput, 'Lower Right') (minX, maxY) = asp_string_utils.getNumbersInParentheses(upperLeftLine) (maxX, minY) = asp_string_utils.getNumbersInParentheses(lowerRightLine) outputDict['projection_bounds'] = (minX, maxX, minY, maxY) outputDict['projection_center'] = ( (minX+maxX)/2.0, (minY+maxY)/2.0 ) # Get some proj4 values outputDict['standard_parallel_1'] = getGdalInfoTagValue(textOutput, 'standard_parallel_1') outputDict['central_meridian'] = getGdalInfoTagValue(textOutput, 'central_meridian') # Get the projection type projStart = textOutput.find('PROJ.4 string is:') nextLine = textOutput.find("'", projStart)+1 endLine = textOutput.find("'", nextLine) outputDict['proj_string'] = textOutput[nextLine:endLine] outputDict['projection'] = 'UNKNOWN' if '+proj=eqc' in textOutput: outputDict['projection'] = 'EQUIRECTANGULAR' elif '+proj=ster' in textOutput: outputDict['projection'] = 'POLAR STEREOGRAPHIC' # Extract this variable which ASP inserts into its point cloud files try: pointOffsetLine = asp_string_utils.getLineAfterText(textOutput, 'POINT_OFFSET=') # Tag name must be synced with C++ code offsetValues = pointOffsetLine.split(' ') outputDict['point_offset'] = (float(offsetValues[0]), float(offsetValues[1]), float(offsetValues[2])) except: pass # In most cases this line will not be present # TODO: Currently this does not find much information, and there # is another function in image_utils dedicated to returning statistics. if getStats: # List of dictionaries per band outputDict['band_info'] = [] # Populate band information band = 1 while (True): # Loop until we run out of bands bandString = 'Band ' + str(band) + ' Block=' bandLoc = textOutput.find(bandString) if bandLoc < 0: # Ran out of bands break # Found the band, read pertinent information bandInfo = {} # Get the type string bandLine = asp_string_utils.getLineAfterText(textOutput, bandString) typePos = bandLine.find('Type=') commaPos = bandLine.find(',') typeName = bandLine[typePos+5:commaPos-1] bandInfo['type'] = typeName outputDict['band_info'] = bandInfo band = band + 1 # Move on to the next band return outputDict
def doFetch(options, outputFolder): # Verify that required files exist home = os.path.expanduser("~") if not (os.path.exists(home + '/.netrc') and os.path.exists(home + '/.urs_cookies')): logger.error( 'Missing a required authentication file! See instructions here:\n' + ' https://nsidc.org/support/faq/what-options-are-available-bulk-downloading-data-https-earthdata-login-enabled' ) return -1 curlPath = asp_system_utils.which("curl") curlOpts = ' -n -L ' cookiePaths = ' -b ~/.urs_cookies -c ~/.urs_cookies ' baseCurlCmd = curlPath + curlOpts + cookiePaths logger.info('Creating output folder: ' + outputFolder) os.system('mkdir -p ' + outputFolder) isSouth = (options.site == 'AN') parsedIndexPath = fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder) if not icebridge_common.fileNonEmpty(parsedIndexPath): # Some dirs are weird, both images, dems, and ortho. # Just accept whatever there is, but with a warning. logger.info('Warning: Missing index file: ' + parsedIndexPath) # Store file information in a dictionary # - Keep track of the earliest and latest frame logger.info('Reading file list from ' + parsedIndexPath) try: (frameDict, urlDict) = readIndexFile(parsedIndexPath) except: # We probably ran into old format index file. Must refetch. logger.info('Could not read index file. Try again.') options.refetchIndex = True parsedIndexPath = fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder) (frameDict, urlDict) = readIndexFile(parsedIndexPath) allFrames = sorted(frameDict.keys()) firstFrame = icebridge_common.getLargestFrame() # start big lastFrame = icebridge_common.getSmallestFrame() # start small for frameNumber in allFrames: if frameNumber < firstFrame: firstFrame = frameNumber if frameNumber > lastFrame: lastFrame = frameNumber if options.allFrames: options.startFrame = firstFrame options.stopFrame = lastFrame # There is always a chance that not all requested frames are available. # That is particularly true for Fireball DEMs. Instead of failing, # just download what is present and give a warning. if options.startFrame not in frameDict: logger.info("Warning: Frame " + str(options.startFrame) + \ " is not found in this flight.") if options.stopFrame and (options.stopFrame not in frameDict): logger.info("Warning: Frame " + str(options.stopFrame) + \ " is not found in this flight.") allFilesToFetch = [ ] # Files that we will fetch, relative to the current dir. allUrlsToFetch = [] # Full url of each file. # Loop through all found frames within the provided range currentFileCount = 0 lastFrame = "" if len(allFrames) > 0: lastFrame = allFrames[len(allFrames) - 1] hasTfw = (options.type == 'dem') hasXml = ((options.type in LIDAR_TYPES) or (options.type == 'ortho') or hasTfw) numFetched = 0 for frame in allFrames: if (frame >= options.startFrame) and (frame <= options.stopFrame): filename = frameDict[frame] # Some files have an associated xml file. DEMs also have a tfw file. currFilesToFetch = [filename] if hasXml: currFilesToFetch.append(icebridge_common.xmlFile(filename)) if hasTfw: currFilesToFetch.append(icebridge_common.tfwFile(filename)) for filename in currFilesToFetch: url = os.path.join(urlDict[frame], filename) outputPath = os.path.join(outputFolder, filename) allFilesToFetch.append(outputPath) allUrlsToFetch.append(url) if options.maxNumToFetch > 0 and len( allFilesToFetch) > options.maxNumToFetch: allFilesToFetch = allFilesToFetch[0:options.maxNumToFetch] allUrlsToFetch = allUrlsToFetch[0:options.maxNumToFetch] icebridge_common.fetchFilesInBatches(baseCurlCmd, MAX_IN_ONE_CALL, options.dryRun, outputFolder, allFilesToFetch, allUrlsToFetch, logger) # Verify that all files were fetched and are in good shape failedFiles = [] for outputPath in allFilesToFetch: if options.skipValidate: continue if not icebridge_common.fileNonEmpty(outputPath): logger.info('Missing file: ' + outputPath) failedFiles.append(outputPath) continue if icebridge_common.hasImageExtension(outputPath): if not icebridge_common.isValidImage(outputPath): logger.info('Found an invalid image. Will wipe it: ' + outputPath) if os.path.exists(outputPath): os.remove(outputPath) failedFiles.append(outputPath) continue else: logger.info('Valid image: ' + outputPath) # Sanity check: XML files must have the right latitude. if icebridge_common.fileExtension(outputPath) == '.xml': if os.path.exists(outputPath): latitude = icebridge_common.parseLatitude(outputPath) isGood = hasGoodLat(latitude, isSouth) if not isGood: logger.info("Wiping XML file " + outputPath + " with bad latitude " + \ str(latitude)) os.remove(outputPath) imageFile = icebridge_common.xmlToImage(outputPath) if os.path.exists(imageFile): logger.info("Wiping TIF file " + imageFile + " with bad latitude " + \ str(latitude)) os.remove(imageFile) # Verify the chcksum if hasXml and len(outputPath) >= 4 and outputPath[-4:] != '.xml' \ and outputPath[-4:] != '.tfw': isGood = icebridge_common.hasValidChkSum(outputPath) if not isGood: xmlFile = icebridge_common.xmlFile(outputPath) logger.info('Found invalid data. Will wipe it: ' + outputPath + ' ' + xmlFile) if os.path.exists(outputPath): os.remove(outputPath) if os.path.exists(xmlFile): os.remove(xmlFile) failedFiles.append(outputPath) failedFiles.append(xmlFile) continue else: logger.info('Valid chksum: ' + outputPath) if hasTfw and icebridge_common.fileExtension(outputPath) == '.tfw': isGood = icebridge_common.isValidTfw(outputPath) if not isGood: xmlFile = icebridge_common.xmlFile(outputPath) logger.info('Found invalid data. Will wipe it: ' + outputPath + ' ' + xmlFile) if os.path.exists(outputPath): os.remove(outputPath) if os.path.exists(xmlFile): os.remove(xmlFile) failedFiles.append(outputPath) failedFiles.append(xmlFile) continue else: logger.info('Valid tfw file: ' + outputPath) numFailed = len(failedFiles) if numFailed > 0: logger.info("Number of files that could not be processed: " + str(numFailed)) return numFailed
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder=True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ((frame >= startFrame) and (frame <= stopFrame)): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile( inputPath, orthoFrameDict[frame]) except Exception, e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90' cmd = ('%s %s -colorspace Gray %s %s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE | re.MULTILINE | re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw
def doFetch(options, outputFolder): '''The main fetch function. Returns the number of failures.''' # Verify that required files exist home = os.path.expanduser("~") if not (os.path.exists(home + '/.netrc') and os.path.exists(home + '/.urs_cookies')): logger.error( 'Missing a required authentication file! See instructions here:\n' + ' https://nsidc.org/support/faq/what-options-are-available-bulk-' + 'downloading-data-https-earthdata-login-enabled') return -1 curlPath = asp_system_utils.which("curl") curlOpts = ' -n -L ' cookiePaths = ' -b ~/.urs_cookies -c ~/.urs_cookies ' baseCurlCmd = curlPath + curlOpts + cookiePaths logger.info('Creating output folder: ' + outputFolder) os.system('mkdir -p ' + outputFolder) isSouth = (options.site == 'AN') if options.type == 'nav': # Nav fetching is much less complicated return fetchNavData(options, outputFolder) parsedIndexPath = fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder) if not icebridge_common.fileNonEmpty(parsedIndexPath): # Some dirs are weird, both images, fireball dems, and ortho. # Just accept whatever there is, but with a warning. logger.info('Warning: Missing index file: ' + parsedIndexPath) # Store file information in a dictionary # - Keep track of the earliest and latest frame logger.info('Reading file list from ' + parsedIndexPath) try: (frameDict, urlDict) = icebridge_common.readIndexFile(parsedIndexPath) except: # We probably ran into old format index file. Must refetch. logger.info('Could not read index file. Try again.') options.refetchIndex = True parsedIndexPath = fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder) (frameDict, urlDict) = icebridge_common.readIndexFile(parsedIndexPath) if options.stopAfterIndexFetch: return 0 isLidar = (options.type in LIDAR_TYPES) allFrames = sorted(frameDict.keys()) if not isLidar: # The lidar frames use a totally different numbering than the image/ortho/dem frames firstFrame = icebridge_common.getLargestFrame() # start big lastFrame = icebridge_common.getSmallestFrame() # start small for frameNumber in allFrames: if frameNumber < firstFrame: firstFrame = frameNumber if frameNumber > lastFrame: lastFrame = frameNumber if options.allFrames: options.startFrame = firstFrame options.stopFrame = lastFrame if isLidar: # Based on image frames, determine which lidar frames to fetch. if options.ignoreMissingLidar and len(frameDict.keys()) == 0: # Nothing we can do if this run has no lidar and we are told to continue logger.info("Warning: missing lidar, but continuing.") lidarsToFetch = set() else: lidarsToFetch = lidarFilesInRange(frameDict, outputFolder, options.startFrame, options.stopFrame) # There is always a chance that not all requested frames are available. # That is particularly true for Fireball DEMs. Instead of failing, # just download what is present and give a warning. if options.startFrame not in frameDict and not isLidar: logger.info("Warning: Frame " + str(options.startFrame) + " is not found in this flight.") if options.stopFrame and (options.stopFrame not in frameDict) and not isLidar: logger.info("Warning: Frame " + str(options.stopFrame) + " is not found in this flight.") allFilesToFetch = [ ] # Files that we will fetch, relative to the current dir. allUrlsToFetch = [] # Full url of each file. # Loop through all found frames within the provided range currentFileCount = 0 lastFrame = "" if len(allFrames) > 0: lastFrame = allFrames[len(allFrames) - 1] hasTfw = (options.type == 'fireball') hasXml = (isLidar or (options.type == 'ortho') or hasTfw) numFetched = 0 skipCount = 0 for frame in allFrames: # Skip frame outside of range if isLidar: if frameDict[frame] not in lidarsToFetch: continue else: if ((frame < options.startFrame) or (frame > options.stopFrame)): continue # Handle the frame skip option if options.frameSkip > 0: if skipCount < options.frameSkip: skipCount += 1 continue skipCount = 0 filename = frameDict[frame] # Some files have an associated xml file. Fireball DEMs also have a tfw file. currFilesToFetch = [filename] if hasXml: currFilesToFetch.append(icebridge_common.xmlFile(filename)) if hasTfw: currFilesToFetch.append(icebridge_common.tfwFile(filename)) for filename in currFilesToFetch: url = os.path.join(urlDict[frame], filename) outputPath = os.path.join(outputFolder, filename) allFilesToFetch.append(outputPath) allUrlsToFetch.append(url) # Restrict lidar fetch amount according to the parameter if (isLidar and options.maxNumLidarToFetch > 0 and len(allFilesToFetch) > options.maxNumLidarToFetch): # Ensure an even number, to fetch both the lidar file and its xml if options.maxNumLidarToFetch % 2 == 1: options.maxNumLidarToFetch += 1 allFilesToFetch = allFilesToFetch[0:options.maxNumLidarToFetch] allUrlsToFetch = allUrlsToFetch[0:options.maxNumLidarToFetch] icebridge_common.fetchFilesInBatches(baseCurlCmd, MAX_IN_ONE_CALL, options.dryRun, outputFolder, allFilesToFetch, allUrlsToFetch, logger) # Fetch from disk the set of already validated files, if any validFilesList = icebridge_common.validFilesList( os.path.dirname(outputFolder), options.startFrame, options.stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Verify that all files were fetched and are in good shape failedFiles = [] for outputPath in allFilesToFetch: if options.skipValidate: continue if not icebridge_common.fileNonEmpty(outputPath): logger.info('Missing file: ' + outputPath) failedFiles.append(outputPath) continue if icebridge_common.hasImageExtension(outputPath): if False: # This check is just so slow. Turn it off for now. # This will impact only the validation of jpegs, # as the other files can be validated via the checksum. # Jpegs will be validated when converting them to 1 band images if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # verbose continue else: if not icebridge_common.isValidImage(outputPath): logger.info('Found an invalid image. Will wipe it: ' + outputPath) if os.path.exists(outputPath): os.remove(outputPath) failedFiles.append(outputPath) continue else: logger.info('Valid image: ' + outputPath) validFilesSet.add(outputPath) # mark it as validated # Sanity check: XML files must have the right latitude. if icebridge_common.fileExtension(outputPath) == '.xml': if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) #verbose continue else: if os.path.exists(outputPath): try: latitude = icebridge_common.parseLatitude(outputPath) logger.info('Valid file: ' + outputPath) validFilesSet.add(outputPath) # mark it as validated except: # Corrupted file logger.info("Failed to parse latitude, will wipe: " + outputPath) if os.path.exists(outputPath): os.remove(outputPath) failedFiles.append(outputPath) # On a second thought, don't wipe files with wrong latitude, as # next time we run fetch we will have to fetch them again. # Hopefully they will be ignored. #isGood = hasGoodLat(latitude, isSouth) #if not isGood: # logger.info("Wiping XML file " + outputPath + " with bad latitude " + \ # str(latitude)) # os.remove(outputPath) # imageFile = icebridge_common.xmlToImage(outputPath) # if os.path.exists(imageFile): # logger.info("Wiping TIF file " + imageFile + " with bad latitude " + \ # str(latitude)) # os.remove(imageFile) # Verify the chcksum if hasXml and len(outputPath) >= 4 and outputPath[-4:] != '.xml' \ and outputPath[-4:] != '.tfw': if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # verbose continue else: isGood = icebridge_common.hasValidChkSum(outputPath, logger) if not isGood: xmlFile = icebridge_common.xmlFile(outputPath) logger.info('Found invalid data. Will wipe: ' + outputPath + ' ' + xmlFile) if os.path.exists(outputPath): os.remove(outputPath) if os.path.exists(xmlFile): os.remove(xmlFile) failedFiles.append(outputPath) failedFiles.append(xmlFile) continue else: logger.info('Valid file: ' + outputPath) validFilesSet.add(outputPath) if hasTfw and icebridge_common.fileExtension(outputPath) == '.tfw': if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) continue else: isGood = icebridge_common.isValidTfw(outputPath, logger) if not isGood: xmlFile = icebridge_common.xmlFile(outputPath) logger.info('Found invalid tfw. Will wipe: ' + outputPath + ' ' + xmlFile) if os.path.exists(outputPath): os.remove(outputPath) if os.path.exists(xmlFile): os.remove(xmlFile) failedFiles.append(outputPath) failedFiles.append(xmlFile) continue else: logger.info('Valid tfw file: ' + outputPath) validFilesSet.add(outputPath) # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = \ icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) numFailed = len(failedFiles) if numFailed > 0: logger.info("Number of files that could not be processed: " + str(numFailed)) return numFailed
def main(argsIn): # Command line parsing try: usage = "usage: camera_models_from_nav.py <image_folder> <ortho_folder> <cal_folder> <nav_folder> <output_folder> [options]" parser = optparse.OptionParser(usage=usage) parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=999999, type='int', help='The frame number to finish processing with.') parser.add_option("--input-calibration-camera", dest="inputCalCamera", default="", help="Use this input calibrated camera.") parser.add_option( '--camera-mounting', dest='cameraMounting', default=0, type='int', help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) (options, args) = parser.parse_args(argsIn) if len(args) < 5: print('Error: Missing arguments.') print(usage) return -1 imageFolder = os.path.abspath(args[0]) orthoFolder = os.path.abspath(args[1]) calFolder = os.path.abspath(args[2]) navFolder = os.path.abspath(args[3]) outputFolder = os.path.abspath(args[4]) except optparse.OptionError as msg: raise Usage(msg) runDir = os.path.dirname(orthoFolder) os.system("mkdir -p " + runDir) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(runDir, logLevel, 'camera_models_from_nav_log') if not os.path.exists(orthoFolder): logger.error('Ortho folder ' + orthoFolder + ' does not exist!') return -1 # Find the nav file # - There should only be one or two nav files per flight. fileList = os.listdir(navFolder) fileList = [x for x in fileList if '.out' in x] if len(fileList) == 0: logger.error('No nav files in: ' + navFolder) return -1 navPath = os.path.join(navFolder, fileList[0]) parsedNavPath = navPath.replace('.out', '.txt') if not asp_file_utils.fileIsNonZero(navPath): logger.error('Nav file ' + navPath + ' is invalid!') return -1 # Create the output file only if it is empty or does not exist isNonEmpty = asp_file_utils.fileIsNonZero(parsedNavPath) if not isNonEmpty: # Initialize the output file as being empty logger.info("Create empty file: " + parsedNavPath) open(parsedNavPath, 'w').close() # Append to the output parsed nav file for fileName in fileList: # Convert the nav file from binary to text navPath = os.path.join(navFolder, fileName) with open(navPath, 'r') as f: try: text = f.readline() if 'HTML' in text: # Sometimes the server is down, and instead of the binary nav file # we are given an html file with an error message. logger.info("Have invalid nav file: " + navPath) return -1 # Die in this case! except UnicodeDecodeError as e: # Got a binary file, that means likely we are good pass cmd = asp_system_utils.which( 'sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath logger.info(cmd) if not isNonEmpty: os.system(cmd) cameraPath = options.inputCalCamera if cameraPath == "": # No input camera file provided, look one up. It does not matter much, # as later ortho2pinhole will insert the correct intrinsics. goodFile = False fileList = os.listdir(calFolder) fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))] if not fileList: logger.error('Unable to find any camera files in ' + calFolder) return -1 for fileName in fileList: cameraPath = os.path.join(calFolder, fileName) # Check if this path is valid with open(cameraPath, 'r') as f: for line in f: if 'fu' in line: goodFile = True break if goodFile: break # Get the ortho list orthoFiles = icebridge_common.getTifs(orthoFolder) logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.') # Look up the frame numbers for each ortho file infoDict = {} for ortho in orthoFiles: if ('gray' in ortho) or ('sub' in ortho): continue frame = icebridge_common.getFrameNumberFromFilename(ortho) if frame < options.startFrame or frame > options.stopFrame: continue infoDict[frame] = [ortho, ''] # Get the image file list try: imageFiles = icebridge_common.getTifs(imageFolder) except Exception as e: raise Exception( "Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. " + str(e)) logger.info('Found ' + str(len(imageFiles)) + ' image files.') # Update the second part of each dictionary object for image in imageFiles: if ('gray' in image) or ('sub' in image): continue frame = icebridge_common.getFrameNumberFromFilename(image) if frame < options.startFrame or frame > options.stopFrame: continue if frame not in infoDict: logger.info('Image missing ortho file: ' + image) # don't throw here, that will mess the whole batch, we will recover # the missing one later. continue infoDict[frame][1] = image os.system('mkdir -p ' + outputFolder) orthoListFile = os.path.join( outputFolder, 'ortho_file_list_' + str(options.startFrame) + "_" + str(options.stopFrame) + '.csv') # Open the output file for writing logger.info("Writing: " + orthoListFile) with open(orthoListFile, 'w') as outputFile: # Loop through frames in order for key in sorted(infoDict): # Write the ortho name and the output camera name to the file (ortho, image) = infoDict[key] if not image: #raise Exception('Ortho missing image file: ' +ortho) continue camera = image.replace('.tif', '.tsai') outputFile.write(ortho + ', ' + camera + '\n') # Check if we already have all of the output camera files. haveAllFiles = True with open(orthoListFile, 'r') as inputFile: for line in inputFile: parts = line.split(',') camPath = os.path.join(outputFolder, parts[1].strip()) if not asp_file_utils.fileIsNonZero(camPath): logger.info('Missing file -> ' + camPath) haveAllFiles = False break # Call the C++ tool to generate a camera model for each ortho file if not haveAllFiles: cmd = ( 'nav2cam --input-cam %s --nav-file %s --cam-list %s --output-folder %s --camera-mounting %d' % (cameraPath, parsedNavPath, orthoListFile, outputFolder, options.cameraMounting)) logger.info(cmd) os.system(cmd) else: logger.info("All nav files were already generated.") # Generate a kml file for the nav camera files kmlPath = os.path.join(outputFolder, 'nav_cameras.kml') # This is a hack. If we are invoked from a Pleiades node, do not # create this kml file, as nodes will just overwrite each other. # This job may happen anyway earlier or later when on the head node. if not 'PBS_NODEFILE' in os.environ: try: tempPath = os.path.join(outputFolder, 'list.txt') logger.info('Generating nav camera kml file: ' + kmlPath) os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath) orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole') cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath logger.info(cmd) asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False) os.remove(tempPath) except Exception as e: logger.info("Warning: " + str(e)) logger.info('Finished generating camera models from nav!') return 0
def doFetch(options, outputFolder): '''The main fetch function. Returns the number of failures.''' # Verify that required files exist home = os.path.expanduser("~") if not (os.path.exists(home+'/.netrc') and os.path.exists(home+'/.urs_cookies')): logger.error('Missing a required authentication file! See instructions here:\n' + ' https://nsidc.org/support/faq/what-options-are-available-bulk-' + 'downloading-data-https-earthdata-login-enabled') return -1 curlPath = asp_system_utils.which("curl") curlOpts = ' -n -L ' cookiePaths = ' -b ~/.urs_cookies -c ~/.urs_cookies ' baseCurlCmd = curlPath + curlOpts + cookiePaths logger.info('Creating output folder: ' + outputFolder) os.system('mkdir -p ' + outputFolder) isSouth = (options.site == 'AN') if options.type == 'nav': # Nav fetching is much less complicated return fetchNavData(options, outputFolder) parsedIndexPath = fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder) if not icebridge_common.fileNonEmpty(parsedIndexPath): # Some dirs are weird, both images, fireball dems, and ortho. # Just accept whatever there is, but with a warning. logger.info('Warning: Missing index file: ' + parsedIndexPath) # Store file information in a dictionary # - Keep track of the earliest and latest frame logger.info('Reading file list from ' + parsedIndexPath) try: (frameDict, urlDict) = icebridge_common.readIndexFile(parsedIndexPath) except: # We probably ran into old format index file. Must refetch. logger.info('Could not read index file. Try again.') options.refetchIndex = True parsedIndexPath = fetchAndParseIndexFile(options, isSouth, baseCurlCmd, outputFolder) (frameDict, urlDict) = icebridge_common.readIndexFile(parsedIndexPath) if options.stopAfterIndexFetch: return 0 isLidar = (options.type in LIDAR_TYPES) allFrames = sorted(frameDict.keys()) if not isLidar: # The lidar frames use a totally different numbering than the image/ortho/dem frames firstFrame = icebridge_common.getLargestFrame() # start big lastFrame = icebridge_common.getSmallestFrame() # start small for frameNumber in allFrames: if frameNumber < firstFrame: firstFrame = frameNumber if frameNumber > lastFrame: lastFrame = frameNumber if options.allFrames: options.startFrame = firstFrame options.stopFrame = lastFrame if isLidar: # Based on image frames, determine which lidar frames to fetch. if options.ignoreMissingLidar and len(frameDict.keys()) == 0: # Nothing we can do if this run has no lidar and we are told to continue logger.info("Warning: missing lidar, but continuing.") lidarsToFetch = set() else: lidarsToFetch = lidarFilesInRange(frameDict, outputFolder, options.startFrame, options.stopFrame) # There is always a chance that not all requested frames are available. # That is particularly true for Fireball DEMs. Instead of failing, # just download what is present and give a warning. if options.startFrame not in frameDict and not isLidar: logger.info("Warning: Frame " + str(options.startFrame) + " is not found in this flight.") if options.stopFrame and (options.stopFrame not in frameDict) and not isLidar: logger.info("Warning: Frame " + str(options.stopFrame) + " is not found in this flight.") allFilesToFetch = [] # Files that we will fetch, relative to the current dir. allUrlsToFetch = [] # Full url of each file. # Loop through all found frames within the provided range currentFileCount = 0 lastFrame = "" if len(allFrames) > 0: lastFrame = allFrames[len(allFrames)-1] hasTfw = (options.type == 'fireball') hasXml = ( isLidar or (options.type == 'ortho') or hasTfw ) numFetched = 0 skipCount = 0 for frame in allFrames: # Skip frame outside of range if isLidar: if frameDict[frame] not in lidarsToFetch: continue else: if ((frame < options.startFrame) or (frame > options.stopFrame) ): continue # Handle the frame skip option if options.frameSkip > 0: if skipCount < options.frameSkip: skipCount += 1 continue skipCount = 0 filename = frameDict[frame] # Some files have an associated xml file. Fireball DEMs also have a tfw file. currFilesToFetch = [filename] if hasXml: currFilesToFetch.append(icebridge_common.xmlFile(filename)) if hasTfw: currFilesToFetch.append(icebridge_common.tfwFile(filename)) for filename in currFilesToFetch: url = os.path.join(urlDict[frame], filename) outputPath = os.path.join(outputFolder, filename) allFilesToFetch.append(outputPath) allUrlsToFetch.append(url) # Restrict lidar fetch amount according to the parameter if (isLidar and options.maxNumLidarToFetch > 0 and len(allFilesToFetch) > options.maxNumLidarToFetch): # Ensure an even number, to fetch both the lidar file and its xml if options.maxNumLidarToFetch % 2 == 1: options.maxNumLidarToFetch += 1 allFilesToFetch = allFilesToFetch[0:options.maxNumLidarToFetch] allUrlsToFetch = allUrlsToFetch [0:options.maxNumLidarToFetch] icebridge_common.fetchFilesInBatches(baseCurlCmd, MAX_IN_ONE_CALL, options.dryRun, outputFolder, allFilesToFetch, allUrlsToFetch, logger) # Fetch from disk the set of already validated files, if any validFilesList = icebridge_common.validFilesList(os.path.dirname(outputFolder), options.startFrame, options.stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Verify that all files were fetched and are in good shape failedFiles = [] for outputPath in allFilesToFetch: if options.skipValidate: continue if not icebridge_common.fileNonEmpty(outputPath): logger.info('Missing file: ' + outputPath) failedFiles.append(outputPath) continue if icebridge_common.hasImageExtension(outputPath): if False: # This check is just so slow. Turn it off for now. # This will impact only the validation of jpegs, # as the other files can be validated via the checksum. # Jpegs will be validated when converting them to 1 band images if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # verbose continue else: if not icebridge_common.isValidImage(outputPath): logger.info('Found an invalid image. Will wipe it: ' + outputPath) if os.path.exists(outputPath): os.remove(outputPath) failedFiles.append(outputPath) continue else: logger.info('Valid image: ' + outputPath) validFilesSet.add(outputPath) # mark it as validated # Sanity check: XML files must have the right latitude. if icebridge_common.fileExtension(outputPath) == '.xml': if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) #verbose continue else: if os.path.exists(outputPath): try: latitude = icebridge_common.parseLatitude(outputPath) logger.info('Valid file: ' + outputPath) validFilesSet.add(outputPath) # mark it as validated except: # Corrupted file logger.info("Failed to parse latitude, will wipe: " + outputPath) if os.path.exists(outputPath): os.remove(outputPath) failedFiles.append(outputPath) # On a second thought, don't wipe files with wrong latitude, as # next time we run fetch we will have to fetch them again. # Hopefully they will be ignored. #isGood = hasGoodLat(latitude, isSouth) #if not isGood: # logger.info("Wiping XML file " + outputPath + " with bad latitude " + \ # str(latitude)) # os.remove(outputPath) # imageFile = icebridge_common.xmlToImage(outputPath) # if os.path.exists(imageFile): # logger.info("Wiping TIF file " + imageFile + " with bad latitude " + \ # str(latitude)) # os.remove(imageFile) # Verify the chcksum if hasXml and len(outputPath) >= 4 and outputPath[-4:] != '.xml' \ and outputPath[-4:] != '.tfw': if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # verbose continue else: isGood = icebridge_common.hasValidChkSum(outputPath, logger) if not isGood: xmlFile = icebridge_common.xmlFile(outputPath) logger.info('Found invalid data. Will wipe: ' + outputPath + ' ' + xmlFile) if os.path.exists(outputPath): os.remove(outputPath) if os.path.exists(xmlFile): os.remove(xmlFile) failedFiles.append(outputPath) failedFiles.append(xmlFile) continue else: logger.info('Valid file: ' + outputPath) validFilesSet.add(outputPath) if hasTfw and icebridge_common.fileExtension(outputPath) == '.tfw': if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) continue else: isGood = icebridge_common.isValidTfw(outputPath, logger) if not isGood: xmlFile = icebridge_common.xmlFile(outputPath) logger.info('Found invalid tfw. Will wipe: ' + outputPath + ' ' + xmlFile) if os.path.exists(outputPath): os.remove(outputPath) if os.path.exists(xmlFile): os.remove(xmlFile) failedFiles.append(outputPath) failedFiles.append(xmlFile) continue else: logger.info('Valid tfw file: ' + outputPath) validFilesSet.add(outputPath) # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = \ icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) numFailed = len(failedFiles) if numFailed > 0: logger.info("Number of files that could not be processed: " + str(numFailed)) return numFailed
except optparse.OptionError, msg: raise Usage(msg) if len(args) != 1: raise Exception(usage) # The input is a pdf file inFile = args[0] base, ext = os.path.splitext(inFile) if ext != ".pdf": raise Exception("The input file must be in pdf format.") if not os.path.isfile(inFile): raise Exception("Missing input file: " + inFile + ".") # Convert from pdf to text pdfToTextExec = asp_system_utils.which("pdftotext") cmd = pdfToTextExec + ' ' + inFile print(cmd) p = subprocess.Popen(cmd, shell=True) os.waitpid(p.pid, 0) txtFile = base + '.txt' if not os.path.isfile(txtFile): raise Exception("Missing file: " + txtFile + ".") tsaiFile = base + '.tsai' # Read the lines lines = [] fin = open(txtFile, 'r') while True:
def getImageGeoInfo(imagePath, getStats=True): """Obtains some image geo information from gdalinfo in dictionary format""" if not os.path.exists(imagePath): raise Exception('Error: input file ' + imagePath + ' does not exist!') outputDict = {} # Call command line tool silently cmd = [asp_system_utils.which('gdalinfo'), imagePath, '-proj4'] if getStats: cmd.append('-stats') p = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) textOutput, err = p.communicate() # Get the size in pixels imageSizeLine = asp_string_utils.getLineAfterText(textOutput, 'Size is ') sizeVals = imageSizeLine.split(',') outputDict['image_size'] = (int(sizeVals[0]), int(sizeVals[1])) # Get origin location and pixel size originLine = asp_string_utils.getLineAfterText(textOutput, 'Origin = ') pixelSizeLine = asp_string_utils.getLineAfterText(textOutput, 'Pixel Size = ') originVals = asp_string_utils.getNumbersInParentheses(originLine) pixelSizeVals = asp_string_utils.getNumbersInParentheses(pixelSizeLine) outputDict['origin'] = originVals outputDict['pixel_size'] = pixelSizeVals # Get bounding box in projected coordinates upperLeftLine = asp_string_utils.getLineAfterText(textOutput, 'Upper Left') lowerRightLine = asp_string_utils.getLineAfterText(textOutput, 'Lower Right') (minX, maxY) = asp_string_utils.getNumbersInParentheses(upperLeftLine) (maxX, minY) = asp_string_utils.getNumbersInParentheses(lowerRightLine) outputDict['projection_bounds'] = (minX, maxX, minY, maxY) outputDict['projection_center'] = ( (minX+maxX)/2.0, (minY+maxY)/2.0 ) # Get some proj4 values outputDict['standard_parallel_1'] = getGdalInfoTagValue(textOutput, 'standard_parallel_1') outputDict['central_meridian'] = getGdalInfoTagValue(textOutput, 'central_meridian') # Get the projection type projStart = textOutput.find('PROJ.4 string is:') nextLine = textOutput.find("'", projStart)+1 endLine = textOutput.find("'", nextLine) outputDict['proj_string'] = textOutput[nextLine:endLine] outputDict['projection'] = 'UNKNOWN' if '+proj=eqc' in textOutput: outputDict['projection'] = 'EQUIRECTANGULAR' elif ('+proj=ster' in textOutput) or ('+proj=stere' in textOutput): outputDict['projection'] = 'POLAR STEREOGRAPHIC' outputDict['lonlat_bounds'] = outputDict['projection_bounds'] if '+proj=longlat' not in outputDict['proj_string']: longlatString = getLonLatProjString(outputDict['proj_string']) ul = convertCoordinate(outputDict['proj_string'], longlatString, minX, maxY) br = convertCoordinate(outputDict['proj_string'], longlatString, maxX, minY) outputDict['lonlat_bounds'] = (ul[0], br[0], br[1], ul[1]) # Extract this variable which ASP inserts into its point cloud files try: pointOffsetLine = asp_string_utils.getLineAfterText(textOutput, 'POINT_OFFSET=') # Tag name must be synced with C++ code offsetValues = pointOffsetLine.split(' ') outputDict['point_offset'] = (float(offsetValues[0]), float(offsetValues[1]), float(offsetValues[2])) except: pass # In most cases this line will not be present # TODO: Currently this does not find much information, and there # is another function in image_utils dedicated to returning statistics. if getStats: # List of dictionaries per band outputDict['band_info'] = [] # Populate band information band = 1 while (True): # Loop until we run out of bands bandString = 'Band ' + str(band) + ' Block=' bandLoc = textOutput.find(bandString) if bandLoc < 0: # Ran out of bands break # Found the band, read pertinent information bandInfo = {} # Get the type string bandLine = asp_string_utils.getLineAfterText(textOutput, bandString) typePos = bandLine.find('Type=') commaPos = bandLine.find(',') typeName = bandLine[typePos+5:commaPos-1] bandInfo['type'] = typeName outputDict['band_info'] = bandInfo band = band + 1 # Move on to the next band return outputDict
os.system(cmd) else: logger.info("All nav files were already generated.") # Generate a kml file for the nav camera files kmlPath = os.path.join(outputFolder, 'nav_cameras.kml') # This is a hack. If we are invoked from a Pleiades node, do not # create this kml file, as nodes will just overwrite each other. # This job may happen anyway earlier or later when on the head node. if not 'PBS_NODEFILE' in os.environ: try: tempPath = os.path.join(outputFolder, 'list.txt') logger.info('Generating nav camera kml file: ' + kmlPath) os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath) orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole') cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath logger.info(cmd) asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False) os.remove(tempPath) except Exception, e: logger.info("Warning: " + str(e)) logger.info('Finished generating camera models from nav!') return 0
def main(argsIn): try: usage = '''usage: process_calibration_file.py file.pdf''' parser = optparse.OptionParser(usage=usage) (options, args) = parser.parse_args(argsIn) except optparse.OptionError as msg: raise Usage(msg) if len(args) != 1: raise Exception(usage) # The input is a pdf file inFile = args[0] base, ext = os.path.splitext(inFile) if ext != ".pdf": raise Exception("The input file must be in pdf format.") if not os.path.isfile(inFile): raise Exception("Missing input file: " + inFile + ".") # Convert from pdf to text pdfToTextExec = asp_system_utils.which("pdftotext") cmd = pdfToTextExec + ' ' + inFile print(cmd) p = subprocess.Popen(cmd, shell=True, universal_newlines=True) os.waitpid(p.pid, 0) txtFile = base + '.txt' if not os.path.isfile(txtFile): raise Exception("Missing file: " + txtFile + ".") tsaiFile = base + '.tsai' # Read the lines lines = [] fin = open(txtFile, 'r') while True: line = fin.readline() if not line: break line = line.strip() if line == "": continue lines.append(line) fin.close() width, height, f, xp, yp, pitch_x, pitch_y, k1, k2, k3, p1, p2, b1, b2 = \ (-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1) # Strip the mm count = 0 for line in lines: while True: m = re.match("^(.*?\d)mm(.*?)$", line) if not m: break line = m.group(1) + m.group(2) lines[count] = line count += 1 #print("line is ", line) # Parse the lines count = 0 for line in lines: m = re.match("^.*?Resolution\s*=\s*(\d+)\s*x\s*(\d+)", line) if m and width == -1 and height == -1: width = m.group(1) height = m.group(2) m = re.match("^.*?Pixel width\s*=\s*(.*?),\s*Pixel height\s*=\s*(.*?)\s*$", line) if m and pitch_x == -1 and pitch_y == -1: pitch_x = m.group(1) pitch_y = m.group(2) if pitch_x != pitch_y: raise Exception ("Expecting pixel width and height to be same.") m = re.match(".*?c\s*=", line) if m and f == -1: f = lines[count+3] m = re.match(".*?xp\s*=", line) if m and xp == -1: xp = lines[count+1] m = re.match(".*?yp\s*=", line) if m and yp == -1: yp = lines[count+1] m = re.match(".*?K1\s*=\s*(.*?)\s*$", line) if m and k1 == -1: k1 = m.group(1) m = re.match(".*?K2\s*=\s*(.*?)\s*$", line) if m and k2 == -1: k2 = m.group(1) m = re.match(".*?K3\s*=\s*(.*?)\s*$", line) if m and k3 == -1: k3 = m.group(1) m = re.match(".*?P1\s*=\s*(.*?)\s*$", line) if m and p1 == -1: p1 = m.group(1) m = re.match(".*?P2\s*=\s*(.*?)\s*$", line) if m and p2 == -1: p2 = m.group(1) m = re.match(".*?B1\s*=\s*(.*?)\s*$", line) if m and b1 == -1: b1 = m.group(1) m = re.match(".*?B2\s*=\s*(.*?)\s*$", line) if m and b2 == -1: b2 = m.group(1) count += 1 cu = float(width) * float(pitch_x) / 2.0 cv = float(height) * float(pitch_y) / 2.0 print ("Writing: " + tsaiFile) fout = open(tsaiFile, 'w') fout.write("VERSION_3\n") fout.write("fu = " + f + "\n") fout.write("fv = " + f + "\n") fout.write("cu = " + repr(cu) + "\n") fout.write("cv = " + repr(cv) + "\n") fout.write("u_direction = 1 0 0\n") fout.write("v_direction = 0 1 0\n") fout.write("w_direction = 0 0 1\n") fout.write("C = 0 0 0\n") fout.write("R = 1 0 0 0 1 0 0 0 1\n") fout.write("pitch = " + pitch_x + "\n") fout.write("Photometrix\n") fout.write("xp = " + xp + "\n") fout.write("yp = " + yp + "\n") fout.write("k1 = " + k1 + "\n") fout.write("k2 = " + k2 + "\n") fout.write("k3 = " + k3 + "\n") fout.write("p1 = " + p1 + "\n") fout.write("p2 = " + p2 + "\n") fout.write("b1 = " + b1 + "\n") fout.write("b2 = " + b2 + "\n") fout.close()
def cameraFromOrthoWrapper(inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCamera, numThreads): '''Generate a camera model from a single ortho file''' # Make multiple calls with different options until we get one that works well IP_METHOD = [1, 0, 2, 1, 1, 2, 0] # IP method FORCE_SIMPLE = [0, 0, 0, 0, 0, 0, 1] # If all else fails use simple mode LOCAL_NORM = [False, False, False, False, True, True, False] # If true, image tiles are individually normalized with method 1 and 2 IP_PER_TILE = [0, 0, 0, 1024, 0, 0, 0] numAttempts = len(IP_METHOD) MIN_IP = 15 # Require more IP to make sure we don't get bogus camera models DESIRED_IP = 200 # If we don't hit this number, try other methods before taking the best one. # The max distance in meters the ortho2pinhole solution is allowed to move from the input # navigation estimate. MAX_TRANSLATION = 7 bestIpCount = 0 tempFilePath = outputCamFile + '_temp' # Used to hold the current best result matchPath = outputCamFile + '.match' # Used to hold the match file if it exists tempMatchPath = matchPath + '_temp' os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others numPoints = 0 # must be initialized for i in range(0,numAttempts): # Get parameters for this attempt ipMethod = IP_METHOD[i] localNorm = LOCAL_NORM[i] if FORCE_SIMPLE[i]: # Always turn this on for the final attempt! simpleCamera = True # Call ortho2pinhole command ortho2pinhole = asp_system_utils.which("ortho2pinhole") cmd = (('%s %s %s %s %s --reference-dem %s --crop-reference-dem --threads %d ' \ '--ip-detect-method %d --minimum-ip %d --max-translation %f') % (ortho2pinhole, inputPath, orthoPath, inputCamFile, outputCamFile, refDemPath, numThreads, ipMethod, MIN_IP, MAX_TRANSLATION) ) if localNorm: cmd += ' --skip-image-normalization' if estimatedCameraPath is not None: cmd += ' --camera-estimate ' + estimatedCameraPath if simpleCamera: cmd += ' --short-circuit' if IP_PER_TILE[i] > 0: cmd += ' --ip-per-tile ' + str(IP_PER_TILE[i]) # Use a print statement as the logger fails from multiple processes print(cmd) os.system('rm -f ' + matchPath) # Needs to be gone p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) textOutput, err = p.communicate() p.wait() print(textOutput) if not os.path.exists(outputCamFile): # Keep trying if no output file produced continue if simpleCamera: break # Never need more than one attempt with simpleCamera! # Check the number of IP used m = re.findall(r"Using (\d+) points to create the camera model.", textOutput) if len(m) != 1: # An unknown error occurred, move on. continue numPoints = int(m[0]) if numPoints >= DESIRED_IP: # Got a lot of points, quit break if numPoints > bestIpCount: # Got some points but not many, try other options bestIpCount = numPoints # to see if we can beat this result. shutil.move(outputCamFile, tempFilePath) if os.path.exists(matchPath): shutil.move(matchPath, tempMatchPath) if (not simpleCamera) and (numPoints < DESIRED_IP): # If we never got the desired # of points shutil.move(tempFilePath, outputCamFile) # Use the camera file with the most points found if os.path.exists(tempMatchPath): shutil.move(tempMatchPath, matchPath) print ('Best number of ortho points = ' + str(bestIpCount)) else: print ('Best number of ortho points = ' + str(numPoints)) os.system('rm -f ' + tempFilePath ) # Clean up these files os.system('rm -f ' + tempMatchPath) os.system("rm -f core.*") # these keep on popping up os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs if not os.path.exists(outputCamFile): # This function is getting called from a pool, so just log the failure. print('Failed to convert ortho file: ' + orthoPath) # I saw this being recommended, to dump all print statements in the current task sys.stdout.flush()
def correctFireballDems(fireballFolder, corrFireballFolder, startFrame, stopFrame, isNorth, skipValidate, logger): '''Fix the header problem in Fireball DEMs''' logger.info('Correcting Fireball DEMs ...') # Read the existing DEMs fireballIndexPath = icebridge_common.csvIndexFile(fireballFolder) if not os.path.exists(fireballIndexPath): raise Exception("Error: Missing fireball index file: " + fireballIndexPath + ".") (fireballFrameDict, fireballUrlDict) = \ icebridge_common.readIndexFile(fireballIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(fireballFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Loop through all the input images os.system('mkdir -p ' + corrFireballFolder) badFiles = False for frame in sorted(fireballFrameDict.keys()): # Skip if outside the frame range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue inputPath = fireballFrameDict[frame] if not icebridge_common.isDEM(inputPath): continue outputPath = os.path.join(corrFireballFolder, os.path.basename(inputPath)) # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very vebose continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) validFilesSet.add(outputPath) # mark it as validated continue # Run the correction script execPath = asp_system_utils.which('correct_icebridge_l3_dem') cmd = (('%s %s %s %d') % (execPath, inputPath, outputPath, isNorth)) logger.info(cmd) # TODO: Run this as a subprocess and check the return code os.system(cmd) # Check if the output file is good if not icebridge_common.isValidImage(outputPath): logger.error('Failed to convert dem file, wiping: ' + inputPath + ' ' + outputPath) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw badFiles = True else: if not skipValidate: validFilesSet.add(outputPath) # mark it as validated if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
def correctFireballDems(fireballFolder, corrFireballFolder, startFrame, stopFrame, isNorth, skipValidate, logger): '''Fix the header problem in Fireball DEMs''' logger.info('Correcting Fireball DEMs ...') # Read the existing DEMs fireballIndexPath = icebridge_common.csvIndexFile(fireballFolder) if not os.path.exists(fireballIndexPath): raise Exception("Error: Missing fireball index file: " + fireballIndexPath + ".") (fireballFrameDict, fireballUrlDict) = \ icebridge_common.readIndexFile(fireballIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(fireballFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Loop through all the input images os.system('mkdir -p ' + corrFireballFolder) badFiles = False for frame in sorted(fireballFrameDict.keys()): # Skip if outside the frame range if not ((frame >= startFrame) and (frame <= stopFrame)): continue inputPath = fireballFrameDict[frame] if not icebridge_common.isDEM(inputPath): continue outputPath = os.path.join(corrFireballFolder, os.path.basename(inputPath)) # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very vebose continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) validFilesSet.add(outputPath) # mark it as validated continue # Run the correction script execPath = asp_system_utils.which('correct_icebridge_l3_dem') cmd = (('%s %s %s %d') % (execPath, inputPath, outputPath, isNorth)) logger.info(cmd) # TODO: Run this as a subprocess and check the return code os.system(cmd) # Check if the output file is good if not icebridge_common.isValidImage(outputPath): logger.error('Failed to convert dem file, wiping: ' + inputPath + ' ' + outputPath) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw badFiles = True else: if not skipValidate: validFilesSet.add(outputPath) # mark it as validated if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) return (not badFiles)
open(parsedNavPath, 'w').close() # Append to the output parsed nav file for fileName in fileList: # Convert the nav file from binary to text navPath = os.path.join(navFolder, fileName) with open(navPath, 'r') as f: text = f.readline() if 'HTML' in text: # Sometimes the server is down, and instead of the binary nav file # we are given an html file with an error message. logger.info("Have invalid nav file: " + navPath) return -1 # Die in this case! cmd = asp_system_utils.which('sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath logger.info(cmd) if not isNonEmpty: os.system(cmd) cameraPath = options.inputCalCamera if cameraPath == "": # No input camera file provided, look one up. It does not matter much, # as later ortho2pinhole will insert the correct intrinsics. goodFile = False fileList = os.listdir(calFolder) fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))] if not fileList: logger.error('Unable to find any camera files in ' + calFolder) return -1 for fileName in fileList:
def cameraFromOrthoWrapper(inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCamera, numThreads): '''Generate a camera model from a single ortho file''' # Make multiple calls with different options until we get one that works well IP_METHOD = [1, 0, 2, 1, 1, 2, 0] # IP method FORCE_SIMPLE = [0, 0, 0, 0, 0, 0, 1] # If all else fails use simple mode LOCAL_NORM = [ False, False, False, False, True, True, False ] # If true, image tiles are individually normalized with method 1 and 2 IP_PER_TILE = [0, 0, 0, 1024, 0, 0, 0] numAttempts = len(IP_METHOD) MIN_IP = 15 # Require more IP to make sure we don't get bogus camera models DESIRED_IP = 200 # If we don't hit this number, try other methods before taking the best one. # The max distance in meters the ortho2pinhole solution is allowed to move from the input # navigation estimate. MAX_TRANSLATION = 7 bestIpCount = 0 tempFilePath = outputCamFile + '_temp' # Used to hold the current best result matchPath = outputCamFile + '.match' # Used to hold the match file if it exists tempMatchPath = matchPath + '_temp' os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others numPoints = 0 # must be initialized for i in range(0, numAttempts): # Get parameters for this attempt ipMethod = IP_METHOD[i] localNorm = LOCAL_NORM[i] if FORCE_SIMPLE[i]: # Always turn this on for the final attempt! simpleCamera = True # Call ortho2pinhole command ortho2pinhole = asp_system_utils.which("ortho2pinhole") cmd = (('%s %s %s %s %s --reference-dem %s --crop-reference-dem --threads %d ' \ '--ip-detect-method %d --minimum-ip %d --max-translation %f') % (ortho2pinhole, inputPath, orthoPath, inputCamFile, outputCamFile, refDemPath, numThreads, ipMethod, MIN_IP, MAX_TRANSLATION) ) if localNorm: cmd += ' --skip-image-normalization' if estimatedCameraPath is not None: cmd += ' --camera-estimate ' + estimatedCameraPath if simpleCamera: cmd += ' --short-circuit' if IP_PER_TILE[i] > 0: cmd += ' --ip-per-tile ' + str(IP_PER_TILE[i]) # Use a print statement as the logger fails from multiple processes print(cmd) os.system('rm -f ' + matchPath) # Needs to be gone p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) textOutput, err = p.communicate() p.wait() print(textOutput) if not os.path.exists( outputCamFile): # Keep trying if no output file produced continue if simpleCamera: break # Never need more than one attempt with simpleCamera! # Check the number of IP used m = re.findall(r"Using (\d+) points to create the camera model.", textOutput) if len(m) != 1: # An unknown error occurred, move on. continue numPoints = int(m[0]) if numPoints >= DESIRED_IP: # Got a lot of points, quit break if numPoints > bestIpCount: # Got some points but not many, try other options bestIpCount = numPoints # to see if we can beat this result. shutil.move(outputCamFile, tempFilePath) if os.path.exists(matchPath): shutil.move(matchPath, tempMatchPath) if (not simpleCamera) and ( numPoints < DESIRED_IP): # If we never got the desired # of points shutil.move( tempFilePath, outputCamFile) # Use the camera file with the most points found if os.path.exists(tempMatchPath): shutil.move(tempMatchPath, matchPath) print 'Best number of ortho points = ' + str(bestIpCount) else: print 'Best number of ortho points = ' + str(numPoints) os.system('rm -f ' + tempFilePath) # Clean up these files os.system('rm -f ' + tempMatchPath) os.system("rm -f core.*") # these keep on popping up os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs if not os.path.exists(outputCamFile): # This function is getting called from a pool, so just log the failure. print('Failed to convert ortho file: ' + orthoPath) # I saw this being recommended, to dump all print statements in the current task sys.stdout.flush()