def getMeanDemDiff(dems, outputPrefix): '''Get the mean distance between the input DEMs to the main DEM.''' mainDem = dems[0] meanDiff = 0.0 meanCount = 0.0 for i in range(1,len(dems)): thisDem = dems[i] if not thisDem: continue diffPrefix = outputPrefix + '_' + str(i) diffPath = diffPrefix + '-diff.tif' cmd = ('geodiff --absolute %s %s -o %s' % (mainDem, thisDem, diffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, True, False) try: # Read in and examine the results results = icebridge_common.readGeodiffOutput(diffPath) print("Found inter-DEM diff " + str(i) + " = " + str(results['Mean'])) meanDiff = meanDiff + results['Mean'] meanCount = meanCount + 1.0 except: print('No overlap with DEM ' + thisDem) if meanCount < 1: # Handle degenerate cases return 0 meanDiff = meanDiff / meanCount print('Mean of DEM diffs = ' + str(meanDiff)) return meanDiff
def getMeanDemDiff(dems, outputPrefix): '''Get the mean distance between the input DEMs to the main DEM.''' mainDem = dems[0] meanDiff = 0.0 meanCount = 0.0 for i in range(1, len(dems)): thisDem = dems[i] if not thisDem: continue diffPrefix = outputPrefix + '_' + str(i) diffPath = diffPrefix + '-diff.tif' cmd = ('geodiff --absolute %s %s -o %s' % (mainDem, thisDem, diffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, True, False) try: # Read in and examine the results results = icebridge_common.readGeodiffOutput(diffPath) print("Found inter-DEM diff " + str(i) + " = " + str(results['Mean'])) meanDiff = meanDiff + results['Mean'] meanCount = meanCount + 1.0 except: print('No overlap with DEM ' + thisDem) if meanCount < 1: # Handle degenerate cases return 0 meanDiff = meanDiff / meanCount print('Mean of DEM diffs = ' + str(meanDiff)) return meanDiff
def processPair(imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder, options): '''Processes a single image pair''' suppressOutput = False redo = False # Just set the options and call the pair python tool. # We can try out bundle adjustment for intrinsic parameters here. cmd = ('python process_icebridge_pair.py --lidar-overlay --align-max-displacement 100 %s %s %s %s %s %s %s' % (imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder, options)) asp_system_utils.executeCommand(cmd, None, suppressOutput, redo)
def processPair(imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder, options): '''Processes a single image pair''' suppressOutput = False redo = False # Just set the options and call the pair python tool. # We can try out bundle adjustment for intrinsic parameters here. cmd = ( 'python process_icebridge_pair.py --lidar-overlay --align-max-displacement 100 %s %s %s %s %s %s %s' % (imageA, imageB, cameraA, cameraB, lidarFolder, outputFolder, options)) asp_system_utils.executeCommand(cmd, None, suppressOutput, redo)
def fetchCameraFolder(run, logger): '''Fetch a camera folder from the archive if it exists. Returns True if we got the file. If more than one, return the latest by modification time.''' logger.info('Fetching camera folder for ' + str(run)) # Tar up the camera files and send them at the same time using the shiftc command cameraFolder = run.getCameraFolder() fileName = run.getCameraTarName() # There could be multiple camera folders, fetch the latest by modification time. strippedName = fileName m = re.match('^(.*?)\.tar', strippedName) if m: strippedName = m.group(1) m = re.match('^(.*?)' + run.suffix, strippedName) if m: strippedName = m.group(1) # Look for the tarred camera files in each remote folder # - TODO: This may not get the latest if cameras are in each folder! found_rcf = '' for rcf in [REMOTE_CAMERA_FOLDER_OLEG, REMOTE_CAMERA_FOLDER_SCOTT]: cmd = 'ssh lfe "ls -dt ' + stripHost( rcf) + '/' + strippedName + '*.tar"' logger.info(cmd) (out, err, status) = asp_system_utils.executeCommand(cmd, outputPath=None, suppressOutput=True, redo=True, noThrow=True) out = out.strip() vals = out.split() if len(vals) >= 1: # Pick the first one, which is the newest out = vals[0] if out != '': found_rcf = rcf break if found_rcf == '': logger.info('Did not find camera file for run.') return False fileName = os.path.basename(out) lfePath = os.path.join(found_rcf, fileName) cmd = 'shiftc --wait -d -r --extract-tar ' + lfePath + ' .' # Don't try too hard below, if failed first time that means there are no cameras # to fetch logger.info(cmd) status = os.system(cmd) if status != 0: logger.info('Did not find camera file for run.') return False else: logger.info('Finished retrieving cameras from lfe.') return True
def packAndSendSummaryFolder(run, folder, logger): '''Archive the summary folder in case we want to look at it later''' logger.info('Archiving summary folder for run ' + str(run)) # Create a local tar file # - Some fiddling to make the packed folders convenient fileName = run.getSummaryTarName() cmd = 'tar -chf '+ fileName +' -C '+ folder +'/.. ' + os.path.basename(folder) logger.info(cmd) (out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None, suppressOutput = True, redo = True, noThrow = True) # This tends to print a very verbose message ans = out + '\n' + err vals = ans.split('\n') if len(vals) < 10: logger.info(ans) else: vals = vals[0:10] logger.info("\n".join(vals)) logger.info("Above output truncated.") # Delete any existing copy of the file on lfe lfePath = os.path.join(REMOTE_SUMMARY_FOLDER, fileName) cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null" logger.info(cmd) os.system(cmd) # Send the file to lfe using shiftc cmd = 'shiftc --wait -d -r ' + fileName + ' ' + lfePath logger.info(cmd) status = os.system(cmd) if status != 0: raise Exception('Failed to pack/send summary folder for run ' + str(run)) logger.info('Finished sending summary to lfe.') if icebridge_common.getUser() != 'oalexan1': # Wipe the copy on lunokhod l2Path = os.path.join(L_SUMMARY_FOLDER, fileName) cmd = "ssh " + LUNOKHOD + " 'rm -f "+ stripHost(l2Path) +"' 2>/dev/null" logger.info(cmd) os.system(cmd) # Make target directory on lunokhod cmd = "ssh " + LUNOKHOD + " 'mkdir -p " + os.path.dirname(stripHost(l2Path)) + \ "' 2>/dev/null" logger.info(cmd) os.system(cmd) # Send a copy of the file to Lunokhod for convenience cmd = 'scp ' + fileName + ' ' + l2Path + ' 2>/dev/null' logger.info(cmd) os.system(cmd) # Clean up the local tar file os.system('rm -f ' + fileName)
def fetchCameraFolder(run, logger): '''Fetch a camera folder from the archive if it exists. Returns True if we got the file. If more than one, return the latest by modification time.''' logger.info('Fetching camera folder for ' + str(run)) # Tar up the camera files and send them at the same time using the shiftc command cameraFolder = run.getCameraFolder() fileName = run.getCameraTarName() # There could be multiple camera folders, fetch the latest by modification time. strippedName = fileName m = re.match('^(.*?)\.tar', strippedName) if m: strippedName = m.group(1) m = re.match('^(.*?)' + run.suffix, strippedName) if m: strippedName = m.group(1) # Look for the tarred camera files in each remote folder # - TODO: This may not get the latest if cameras are in each folder! found_rcf = '' for rcf in [REMOTE_CAMERA_FOLDER_OLEG, REMOTE_CAMERA_FOLDER_SCOTT]: cmd = 'ssh lfe "ls -dt ' + stripHost(rcf) + '/' + strippedName + '*.tar"' logger.info(cmd) (out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None, suppressOutput = True, redo = True, noThrow = True) out = out.strip() vals = out.split() if len(vals) >= 1: # Pick the first one, which is the newest out = vals[0] if out != '': found_rcf = rcf break if found_rcf == '': logger.info('Did not find camera file for run.') return False fileName = os.path.basename(out) lfePath = os.path.join(found_rcf, fileName) cmd = 'shiftc --wait -d -r --extract-tar ' + lfePath + ' .' # Don't try too hard below, if failed first time that means there are no cameras # to fetch logger.info(cmd) status = os.system(cmd) if status != 0: logger.info('Did not find camera file for run.') return False else: logger.info('Finished retrieving cameras from lfe.') return True
def convertToTsai(path, demPath): '''Given the json metadata, generate a .tsai camera file''' # Get the other file paths tifPath = path.replace('metadata.json', 'basic_panchromatic_dn.tif') outputPath = tifPath.replace('.tif', '.tsai') # Get the image size (for the center coordinate) width, height = asp_image_utils.getImageSize(tifPath) # Set up the required coordinate string with open(path, 'r') as f: contents = f.read() parts = json.loads(contents) coords = parts['geometry']['coordinates'][0] coordString = (('%f %f %f %f %f %f %f %f') % (coords[0][0], coords[0][1], coords[3][0], coords[3][1], coords[2][0], coords[2][1], coords[1][0], coords[1][1])) cmd = (('cam_gen %s --reference-dem %s --focal-length 553846.153846 --optical-center %f %f --pixel-pitch 1 --refine-camera --lon-lat-values "%s" -o %s') % (tifPath, demPath, width/2.0, height/2.0, coordString, outputPath)) asp_system_utils.executeCommand(cmd, outputPath=outputPath)
def convertToTsai(path, demPath): '''Given the json metadata, generate a .tsai camera file''' # Get the other file paths tifPath = path.replace('metadata.json', 'basic_panchromatic_dn.tif') outputPath = tifPath.replace('.tif', '.tsai') # Get the image size (for the center coordinate) width, height = asp_image_utils.getImageSize(tifPath) # Set up the required coordinate string with open(path, 'r') as f: contents = f.read() parts = json.loads(contents) coords = parts['geometry']['coordinates'][0] coordString = (('%f %f %f %f %f %f %f %f') % (coords[0][0], coords[0][1], coords[3][0], coords[3][1], coords[2][0], coords[2][1], coords[1][0], coords[1][1])) cmd = (( 'cam_gen %s --reference-dem %s --focal-length 553846.153846 --optical-center %f %f --pixel-pitch 1 --refine-camera --lon-lat-values "%s" -o %s' ) % (tifPath, demPath, width / 2.0, height / 2.0, coordString, outputPath)) asp_system_utils.executeCommand(cmd, outputPath=outputPath)
def getActiveJobs(user): '''Returns a list of the currently active jobs and their status''' # Run qstat command to get a list of all active jobs by this user cmd = ['qstat', '-u', user] (textOutput, err, status) = \ asp_system_utils.executeCommand(cmd, outputPath=None, suppressOutput = True, redo=True, noThrow=True, numAttempts = 1000, sleepTime = 60 ) lines = textOutput.split('\n') # Strip header lines NUM_HEADER_LINES = 3 JOBID_INDEX = 0 NAME_INDEX = 3 STATUS_INDEX = 7 if len(lines) <= NUM_HEADER_LINES: return [] lines = lines[NUM_HEADER_LINES:] # Pick out all the job names and put them in a list jobs = [] for line in lines: parts = line.split() if len(parts) < STATUS_INDEX: continue jobID = cleanJobID(parts[JOBID_INDEX]) jobName = parts[NAME_INDEX] status = parts[STATUS_INDEX] jobs.append((jobID, jobName, status)) return jobs
def pushByType(run, options, logger, dataType): # Fetch the ortho index from NSIDC if missing outputFolder = run.getFolder() logger.info("Output folder is " + outputFolder) os.system("mkdir -p " + outputFolder) # Current directory. It is important to go from /u to the real dir which is /nobackup... unpackDir = os.path.realpath(os.getcwd()) logger.info("Unpack directory is " + unpackDir) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): fetchIndices(options, logger) logger.info("Reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) # Fetch unarchived folder if missing if dataType == 'DEM': unarchivedFolder = run.getAssemblyFolder() elif dataType == 'ORTHO': unarchivedFolder = run.getProcessFolder() else: raise Exception("Unknown data type: " + dataType) logger.info("Unarchived data folder is " + unarchivedFolder) # Especially for ortho, force-fetch each time, as there is no good way # of checking if we fetched well before. start_time() if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType): return stop_time("fetching archived data by type: " + dataType, logger) # Make the output directory at NSIDC m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd) if m: outDir = options.site + "_" + m.group(1) + "." + m.group(2) + "." + m.group(3) else: raise Exception("Could not parse: " + options.yyyymmdd) # Keep the output directory locally here localDirPath = os.path.join(outputFolder, dataType, outDir) os.system("mkdir -p " + localDirPath) logger.info("Storing the renamed " + dataType + " files in " + localDirPath) logger.info("Directory name at NSIDC: " + outDir) # Read the DEMs and orthos, and copy them to outDir according to the final convention if dataType == 'DEM': dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True) else: dataFiles = glob.glob(os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif')) for dataFile in dataFiles: # Here we use the convention from archive_functions.py for DEMs and from how we store orthos. if dataType == 'DEM': m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \ "\.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) else: m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \ "out-ortho.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) if frameNumber < options.startFrame or frameNumber > options.stopFrame: continue # For each data file, copy from the ortho its meta info if not frameNumber in orthoFrameDict.keys(): # Bugfix: Ortho fetching failed, try again fetchIndices(options, logger) logger.info("Re-reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if not frameNumber in orthoFrameDict.keys(): # This time there is nothing we can do raise Exception("Cannot find ortho for frame: " + str(frameNumber)) orthoFile = orthoFrameDict[frameNumber] [dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile) # It is always possible that the ortho file date will be the next day # after the current flight date, if the flight goes after midnight. # So it is not unreasonable that options.yyyymmdd != dateString. if dataType == 'DEM': outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber)) else: # TODO: Need to think more of the naming convention. outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber)) cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(localDirPath, outFile) logger.info(cmd) os.system(cmd) # Push the directory to NSIDC remoteDirPath = os.path.join(os.path.basename(os.path.dirname(localDirPath)), os.path.basename(localDirPath)) remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath) logger.info("Storing at NSIDC in: " + remoteDirPath) cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \ ' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo logger.info(cmd) start_time() (output, err, status) = asp_system_utils.executeCommand(cmd, suppressOutput = True) #status = os.system(cmd) logger.info("LFTP output and error: " + output + ' ' + err) logger.info("LFTP status: " + str(status)) #if status != 0: # raise Exception("Problem pushing") stop_time("push to NSIDC", logger)
def fetchProcessedByType(run, unpackFolder, logger, dataType): '''Fetch from lfe the latest archive of processed DEMs or orthos by modification time. For now we ignore the timestamp.''' logger.info('Fetching processed ' + dataType + ' data for ' + str(run)) # Fetch the latest file by modification time as returned by 'ls'. # A given file can be either in Scott's or Oleg's archive. # It can be in both, and it can be multiple instances. # Need to list all and fetch the latest. if dataType == 'DEM': fileName = run.getOutputTarName(useWildCard = True) lfePaths = os.path.join(stripHost(get_remote_output_folder('oalexan1')), fileName) + " " + \ os.path.join(stripHost(get_remote_output_folder('smcmich1')), fileName) elif dataType == 'ORTHO': fileName = run.getOrthoTarName(useWildCard = True) lfePaths = os.path.join(stripHost(get_remote_ortho_folder('oalexan1')), fileName) + " " + \ os.path.join(stripHost(get_remote_ortho_folder('smcmich1')), fileName) else: raise Exception("Unknown data type: " + dataType) # Get the list cmd = 'ssh lfe "ls -altd ' + lfePaths + '"' logger.info(cmd) (out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None, suppressOutput = True, redo = True, noThrow = True) # Parse the answer and keep the latest. out = out.strip() vals = out.split('\n') for val in vals: logger.info("Found: " + val) lfePath = "" for val in vals: if len(val) >= 1 and val[0] == 'l': continue # this is a symlink arr = val.split() lfePath = arr[-1] break # found what we needed if lfePath == "": logger.info("Could not locate: " + fileName) return False cmd = 'shiftc --wait -d -r --verify --extract-tar ' + 'lfe:' + lfePath + ' ' + unpackFolder logger.info(cmd) status = os.system(cmd) # Try tar if shift failed if status != 0: logger.info('Did not sucessfully fetch archived processed data of type: ' + dataType) logger.info('Will try with tar.''') tar_cmd = 'ssh lfe "cd ' + os.path.dirname(lfePath) + "; tar xfv " + \ os.path.basename(lfePath) + " -C " + os.path.realpath(unpackFolder) + '"' logger.info(tar_cmd) status = os.system(tar_cmd) if status != 0: logger.info('Failed using tar as well.') return False return True
# Generate a kml file for the nav camera files kmlPath = os.path.join(outputFolder, 'nav_cameras.kml') # This is a hack. If we are invoked from a Pleiades node, do not # create this kml file, as nodes will just overwrite each other. # This job may happen anyway earlier or later when on the head node. if not 'PBS_NODEFILE' in os.environ: try: tempPath = os.path.join(outputFolder, 'list.txt') logger.info('Generating nav camera kml file: ' + kmlPath) os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath) orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole') cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath logger.info(cmd) asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False) os.remove(tempPath) except Exception, e: logger.info("Warning: " + str(e)) logger.info('Finished generating camera models from nav!') return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))
def submitJob(jobName, queueName, maxHours, logger, minutesInDevelQueue, groupId, nodeType, commandPath, args, logPrefix, priority=None, pythonPath='/u/oalexan1/.local/lib/python2.7/site-packages'): '''Submits a job to the PBS system. Any such job must invoke icebridge_common.switchWorkDir().''' if len(queueName) > MAX_PBS_NAME_LENGTH: raise Exception('Job name "'+queueName+'" exceeds the maximum length of ' + str(MAX_PBS_NAME_LENGTH)) numCpus = getNumCores(nodeType) # Cores or CPUs? hourString = '"'+str(maxHours)+':00:00"' # We must create and execute a shell script, to be able to explicitely # direct the output and error to files, without counting # on PBS to manage that, as that one overfloes. # All our outputs and errors will go to the "verbose" files, # while PBS will write its summaries, etc, to the other files. shellScriptPath = logPrefix + '_script.sh' verboseOutputPath = logPrefix + '_verbose_output.log' verboseErrorsPath = logPrefix + '_verbose_errors.log' outputPath = logPrefix + '_output.log' errorsPath = logPrefix + '_errors.log' workDir = os.getcwd() numAttempts = 1000 # For debugging if minutesInDevelQueue > 0: queueName = 'devel' hourString = '00:' + str(minutesInDevelQueue).zfill(2) + ':00' #numAttempts = 1 # For some reason qsub errors out in the devel queue # The "-m eb" option sends the user an email when the process begins and when it ends. # The -r n ensures the job does not restart if it runs out of memory. # Debug the environment #for v in os.environ.keys(): # logger.info("env is " + v + '=' + str(os.environ[v])) # We empty PYTHONSTARTUP and LD_LIBRARY_PATH so that python can function # properly on the nodes. priorityString = '' if priority: priorityString = ' -p ' + str(priority) + ' ' # Generate the shell command shellCommand = ( "%s %s > %s 2> %s\n" % (commandPath, args, verboseOutputPath, verboseErrorsPath) ) with open(shellScriptPath, 'w') as f: f.write("#!/bin/bash\n") f.write(shellCommand) # Make it executable os.system("chmod a+rx " + shellScriptPath) # Run it pbsCommand = ('qsub -r y -q %s -N %s %s -l walltime=%s -W group_list=%s -j oe -e %s -o %s -S /bin/bash -V -C %s -l select=1:ncpus=%d:model=%s -- /usr/bin/env OIB_WORK_DIR=%s PYTHONPATH=%s PYTHONSTARTUP="" LD_LIBRARY_PATH="" %s' % (queueName, jobName, priorityString, hourString, groupId, errorsPath, outputPath, workDir, numCpus, nodeType, workDir, pythonPath, shellScriptPath)) #~/repo/python_env/lib/python2.7/site-packages/ #/u/oalexan1/.local/lib/python2.7/site-packages logger.info(pbsCommand) outputPath = None suppressOutput = True redo = True noThrow = True (out, err, status) = \ asp_system_utils.executeCommand(pbsCommand, outputPath, suppressOutput, redo, noThrow, numAttempts, SLEEP_TIME) if status != 0: logger.info(out) logger.info(err) logger.info("Status is: " + str(status)) jobID = '' else: jobID = cleanJobID(out) logger.info("Submitted job named " + str(jobName) + " with id " + str(jobID)) return jobID
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument('--camera-mounting', dest='cameraMounting', default=0, type=int, help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput): WEIGHT_EXP = 1.3 # This will run as multiple processes. Hence have to catch all exceptions. try: demFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.alignFileName(), processFolder, options.bundleLength) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) if demFile == "": print("Could not find DEM for frame: " + str(frame)) return # The names for the final results finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM') finalBlend = finalOutputPrefix + '.tif' finalDiff = finalOutputPrefix + "-diff.csv" fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint') fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif' finalFireballOutput = fireballOutputPrefix + '-DEM.tif' fireballDiffPath = fireballOutputPrefix + "-diff.csv" # This is turned off for now. Find the diff between neighboring # aligned DEMs before blending. prevDemFile, prevBatchFolder = \ icebridge_common.frameToFile(frame-1, icebridge_common.alignFileName(), processFolder, options.bundleLength) prevDiffPrefix = os.path.join(batchFolder, 'out-prev') prevDiffFile = prevDiffPrefix + '-diff.tif' if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile): os.system("rm -f " + prevDiffFile) if os.path.exists(prevDemFile) and os.path.exists(demFile): if os.path.exists(prevDiffFile): print("File exists: " + prevDiffFile) else: cmd = ('geodiff --absolute %s %s -o %s' % (prevDemFile, demFile, prevDiffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo) if not redo: set1Exists = False if (os.path.exists(finalBlend) and os.path.exists(finalDiff)): print("Files exist: " + finalBlend + " " + finalDiff + ".") set1Exists = True set2Exists = True if fireballDEM != "": if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)): print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".") set2Exists = True else: set2Exists = False if set1Exists and set2Exists: return # We will blend the dems with frame offsets within frameOffsets[0:index] filesToWipe = [] bestMean = 1.0e+100 bestBlend = '' bestVals = '' bestDiff = '' # Look at frames with these offsets when blending frameOffsets = [0, 1, -1, 2, -2] for index in range(len(frameOffsets)): # Find all the DEMs up to the current index dems = [] currDemFile = "" for val in range(0, index + 1): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) if currDemFile == "": # The last DEM was not present. Hence this iteration will add nothing new. continue # Compute the mean distance between the DEMs # TODO: Make sure this gets cleaned up! meanWorkPrefix = os.path.join(batchFolder, 'bd') meanDiff = getMeanDemDiff(dems, meanWorkPrefix) # If the mean error between DEMs is creater than this, # use a less aggressive blending method. MEAN_DIFF_BLEND_THRESHOLD = 1.0 demString = " ".join(dems) outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index)) # See if we have a pre-existing DEM to use as footprint footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif') blendOutput = outputPrefix + '-tile-0.tif' if os.path.exists(footprintDEM): cmd = ( 'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix)) else: cmd = ( 'dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s' % (WEIGHT_EXP, demString, threadText, outputPrefix)) if meanDiff > MEAN_DIFF_BLEND_THRESHOLD: cmd += ' --propagate-nodata --use-centerline-weights ' print(cmd) # Execute the blend command. # - Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True print(cmd) asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo) filesToWipe.append(blendOutput) diffPath = outputPrefix + "-diff.csv" filesToWipe.append(diffPath) # Compute post-blending error to lidar cmd = ( 'geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, blendOutput, lidarFile, outputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(diffPath) print("Current mean error to lidar is " + str(results['Mean'])) if bestMean > float(results['Mean']): bestMean = float(results['Mean']) bestBlend = blendOutput bestVals = demString bestDiff = diffPath except Exception, e: pass logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Update the filenames of the output files print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals) cmd = "mv " + bestBlend + " " + finalBlend print(cmd) asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo) cmd = "mv " + bestDiff + " " + finalDiff print(cmd) asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo) # Generate a thumbnail of the final DEM hillOutput = finalOutputPrefix + '_HILLSHADE.tif' cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # Generate a low resolution compressed thumbnail of the hillshade for debugging thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif' cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) os.system("rm -f " + hillOutput) # Remove this file to keep down the file count # Do another blend, to this DEM's footprint, but not using it if fireballDEM != "": # Find all the DEMs dems = [] for val in range(0, len(frameOffsets)): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) demString = " ".join(dems) cmd = ( 'dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix)) #filesToWipe.append(fireballBlendOutput) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo) #filesToWipe.append(fireballDiffPath) cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(fireballDiffPath) print("Mean error to lidar in fireball footprint is " + str(results['Mean'])) cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput print(cmd) asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo) except Exception, e: pass # Generate a thumbnail of the final DEM #hillOutput = fireballOutputPrefix+'_HILLSHADE.tif' #cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput #print(cmd) #asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) ## Generate a low resolution compressed thumbnail of the hillshade for debugging #thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif' #cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' #asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) #os.system("rm -f " + hillOutput) # Remove this file to keep down the file count logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 10 usage = '''blend_dems.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") parser.add_argument("--compute-diff-to-prev-dem", action="store_true", dest="computeDiffToPrev", default=False, help="Compute the absolute difference between the current DEM " + "and the one before it.") parser.add_argument("--blend-to-fireball-footprint", action="store_true", dest="blendToFireball", default=False, help="Create additional blended DEMs having the same " + \ "footprint as Fireball DEMs.") # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_blend_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) logger.info(str(argsIn)) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems(options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(orthoFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No ortho files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in orthoFrameDict: logger.info("Error: Missing ortho file for frame: " + str(frame) + ".") continue orthoFile = orthoFrameDict[frame] try: lidarFile = icebridge_common.findMatchingLidarFile(orthoFile, lidarFolder) except: # Skip if no lidar file matches this frame continue fireballDEM = "" if options.blendToFireball: if frame in fireballFrameDict: fireballDEM = fireballFrameDict[frame] else: logger.info("No fireball DEM for frame: " + str(frame)) args = (frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runBlend, args)) else: runBlend(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive = False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
def pushByType(run, options, logger, dataType): # Fetch the ortho index from NSIDC if missing outputFolder = run.getFolder() logger.info("Output folder is " + outputFolder) os.system("mkdir -p " + outputFolder) # Current directory. It is important to go from /u to the real dir which is /nobackup... unpackDir = os.path.realpath(os.getcwd()) logger.info("Unpack directory is " + unpackDir) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): fetchIndices(options, logger) logger.info("Reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) # Fetch unarchived folder if missing if dataType == 'DEM': unarchivedFolder = run.getAssemblyFolder() elif dataType == 'ORTHO': unarchivedFolder = run.getProcessFolder() else: raise Exception("Unknown data type: " + dataType) logger.info("Unarchived data folder is " + unarchivedFolder) # Especially for ortho, force-fetch each time, as there is no good way # of checking if we fetched well before. start_time() if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType): return stop_time("fetching archived data by type: " + dataType, logger) # Make the output directory at NSIDC m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd) if m: outDir = options.site + "_" + m.group(1) + "." + m.group( 2) + "." + m.group(3) else: raise Exception("Could not parse: " + options.yyyymmdd) # Keep the output directory locally here localDirPath = os.path.join(outputFolder, dataType, outDir) os.system("mkdir -p " + localDirPath) logger.info("Storing the renamed " + dataType + " files in " + localDirPath) logger.info("Directory name at NSIDC: " + outDir) # Read the DEMs and orthos, and copy them to outDir according to the final convention if dataType == 'DEM': dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True) else: dataFiles = glob.glob( os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif')) for dataFile in dataFiles: # Here we use the convention from archive_functions.py for DEMs and from how we store orthos. if dataType == 'DEM': m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \ "\.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) else: m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \ "out-ortho.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) if frameNumber < options.startFrame or frameNumber > options.stopFrame: continue # For each data file, copy from the ortho its meta info if not frameNumber in orthoFrameDict.keys(): # Bugfix: Ortho fetching failed, try again fetchIndices(options, logger) logger.info("Re-reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if not frameNumber in orthoFrameDict.keys(): # This time there is nothing we can do raise Exception("Cannot find ortho for frame: " + str(frameNumber)) orthoFile = orthoFrameDict[frameNumber] [dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile) # It is always possible that the ortho file date will be the next day # after the current flight date, if the flight goes after midnight. # So it is not unreasonable that options.yyyymmdd != dateString. if dataType == 'DEM': outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber)) else: # TODO: Need to think more of the naming convention. outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber)) cmd = "/bin/cp -fv " + dataFile + " " + os.path.join( localDirPath, outFile) logger.info(cmd) os.system(cmd) # Push the directory to NSIDC remoteDirPath = os.path.join( os.path.basename(os.path.dirname(localDirPath)), os.path.basename(localDirPath)) remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath) logger.info("Storing at NSIDC in: " + remoteDirPath) cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \ ' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo logger.info(cmd) start_time() (output, err, status) = asp_system_utils.executeCommand(cmd, suppressOutput=True) #status = os.system(cmd) logger.info("LFTP output and error: " + output + ' ' + err) logger.info("LFTP status: " + str(status)) #if status != 0: # raise Exception("Problem pushing") stop_time("push to NSIDC", logger)
def main(argsIn): try: usage = '''usage: regenerate_summary_images.py <options> ''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--data-folder", dest="dataFolder", help="Where all the inputs and outputs are stored.") parser.add_argument("--work-folder", dest="workFolder", help="Where working files are stored.") parser.add_argument("--site", dest="site", help="Site code.") parser.add_argument("--yyyymmdd", dest="yyyymmdd", help="Date.") #parser.add_argument("--dem-tarball", dest="demTarball", default=os.getcwd(), # help="Where all the inputs and outputs are stored.") #parser.add_argument("--ortho-tarball", dest="orthoTarball", default=None, # help="Where to unpack the data.") #parser.add_argument("--summary-tarball", dest="summaryTarball", default=None, # help="Where to unpack the data.") #parser.add_argument("--unpack-dir", dest="unpackDir", default=None, # help="Where to unpack the data.") parser.add_argument("--node-type", dest="nodeType", default='san', help="Node type to use (wes[mfe], san, ivy, has, bro)") #parser.add_argument("--skip-archive-summary", action="store_true", # dest="skipArchiveSummary", default=False, # help="Skip archiving the summary.") # Debug option parser.add_argument('--minutes-in-devel-queue', dest='minutesInDevelQueue', type=int, default=0, help="If positive, submit to the devel queue for this many minutes.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) # Check if we are on the right machine (host, err, status) = asp_system_utils.executeCommand(['uname', '-n'], suppressOutput = True) host = host.strip() if 'pfe' in host and options.nodeType not in PFE_NODES: raise Exception("From machine " + host + " can only launch on: " + " ".join(PFE_NODES)) if 'mfe' in host and options.nodeType != 'wes': raise Exception("From machine " + host + " can only launch on: wes") # Make sure our paths will work when called from PBS options.dataFolder= os.path.abspath(options.dataFolder) #os.system('mkdir -p ' + options.unpackDir) # TODO: Check folders! run = run_helper.RunHelper(options.site, options.yyyymmdd, options.workFolder) runFolder = os.path.join(options.workFolder, str(run)) os.system('mkdir -p ' + runFolder) logFolder = os.path.join(runFolder, 'logs') # Set up logging in the run directory os.system('mkdir -p ' + logFolder) logLevel = logging.INFO logger = icebridge_common.setUpLogger(logFolder, logLevel, icebridge_common.manager_log_prefix()) logger.info("Logging in: " + logFolder) checkRequiredTools() # Make sure all the needed tools can be found before we start logger.info("Disabling core dumps.") # these just take a lot of room os.system("ulimit -c 0") os.system("umask 022") # enforce files be readable by others # See how many hours we used so far. I think this counter gets updated once a day. (out, err, status) = asp_system_utils.executeCommand("acct_ytd", outputPath = None, suppressOutput = True, redo = True, noThrow = True) logger.info("Hours used so far:\n" + out + '\n' + err) try: # Fetch and extract the tarball files from Lou localDemFolder = os.path.join(options.dataFolder, run.name()+'_dems') localOrthoFolder = os.path.join(options.dataFolder, run.name()+'_orthos') demSummaryFolder = os.path.join(options.dataFolder, run.name()+'_dem_summaries') orthoSummaryFolder = os.path.join(options.dataFolder, run.name()+'_ortho_summaries') missingDemFiles = [] missingOrthoFiles = [] for f in os.listdir(localDemFolder): if 'temp' in f: raise Exception('Bad file: ' + f) if ('IODEM3' in f) and (f[-4:] == '.tif'): inputPath = os.path.join(localDemFolder, f) outputPath = os.path.join(demSummaryFolder, f.replace('DEM.tif', 'DEM_browse.tif')) if not os.path.exists(outputPath): missingDemFiles.append((inputPath, outputPath)) for f in os.listdir(localOrthoFolder): if 'temp' in f: raise Exception('Bad file: ' + f) if ('IODIM3' in f) and (f[-4:] == '.tif'): inputPath = os.path.join(localOrthoFolder, f) outputPath = os.path.join(orthoSummaryFolder, f.replace('ORTHO.tif', 'ORTHO.jpg')) if not os.path.exists(outputPath): missingOrthoFiles.append((inputPath, outputPath)) #print 'Fetching and unpacking tarballs...' #fetchTarball(options.demTarball, localDemFolder) #fetchTarball(options.orthoTarball, localOrthoFolder) #fetchTarball(options.summaryTarball, localSummaryFolder) # If the summary tarball unpacked to []/summary/summary, # work with the lower level folder from now on. #localSummaryFolder = descendIfNeeded(localSummaryFolder) # Make a list of all input files that are missing their summary file, and # the desired output path for that file. #missingDemFiles = getMissingSummaryFiles(localDemFolder, localSummaryFolder, isOrtho=False) #missingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, localSummaryFolder, isOrtho=True ) # Divide this list into chunks and for each chunk generate a file containing all of # the gdal_translate commands that need to be executed. print 'Writing command files...' commandFileLength = getParallelParams(options.nodeType)[2] commandFilePrefix = os.path.join(runFolder, 'convert_commands_') print 'Clearing existing command files.' os.system('rm ' + commandFilePrefix + '*') commandFileList = writeCommandFiles(missingDemFiles, missingOrthoFiles, commandFilePrefix, commandFileLength) #raise Exception('DEBUG') # Get the location to store the logs pbsLogFolder = run.getPbsLogFolder() logger.info("Storing logs in: " + pbsLogFolder) os.system('mkdir -p ' + pbsLogFolder) # Call multi_process_command_runner.py through PBS for each chunk. start_time() (baseName, jobIDs) = submitBatchJobs(commandFileList, options, pbsLogFolder, run, logger) # Wait for everything to finish. pbs_functions.waitForJobCompletion(jobIDs, logger, baseName) stop_time("pbs_jobs", logger) # Check that we now have all of the summary files. # - Both of these should now be empty. #newMissingDemFiles = getMissingSummaryFiles(localDemFolder, demSummaryFolder, isOrtho=False) #newMissingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, orthoSummaryFolder, isOrtho=True ) numDemsMissing = 0 numOrthosMissing = 0 for pair in missingDemFiles: if not os.path.exists(pair[1]): numDemsMissing += 1 for pair in missingOrthoFiles: if not os.path.exists(pair[1]): numOrthosMissing += 1 resultText = ('After regeneration, missing %d DEM summaries and %d ORTHO summaries' % (numDemsMissing, numOrthosMissing)) logger.info(resultText) runWasSuccess = ((numDemsMissing == 0) and (numOrthosMissing == 0)) # If successful, create a new tarball and send it to Lou. #if runWasSuccess and (not options.skipArchiveSummary): # start_time() # archive_functions.packAndSendSummaryFolder(run, localSummaryFolder, logger) # stop_time("archive summary", logger) except Exception as e: resultText = 'Caught exception: ' + str(e) + '\n' + traceback.format_exc() runWasSuccess = False # Send a summary email. emailAddress = getEmailAddress(icebridge_common.getUser()) logger.info("Sending email to: " + emailAddress) if runWasSuccess: sendEmail(emailAddress, 'OIB summary regen passed', resultText) else: sendEmail(emailAddress, '"OIB summary regen failed', resultText) # TODO: Add automated delete command! #if options.wipeProcessed: # processedFolder = run.getProcessFolder() # logger.info("Will delete: " + processedFolder) # os.system("rm -rf " + processedFolder) logger.info('==== regenerate_summary_images script has finished for run: ' + str(run) + ' ====')
def submitJob(jobName, queueName, maxHours, logger, minutesInDevelQueue, groupId, nodeType, commandPath, args, logPrefix, priority=None, pythonPath='/u/oalexan1/.local/lib/python2.7/site-packages'): '''Submits a job to the PBS system. Any such job must invoke icebridge_common.switchWorkDir().''' if len(queueName) > MAX_PBS_NAME_LENGTH: raise Exception('Job name "' + queueName + '" exceeds the maximum length of ' + str(MAX_PBS_NAME_LENGTH)) numCpus = getNumCores(nodeType) # Cores or CPUs? hourString = '"' + str(maxHours) + ':00:00"' # We must create and execute a shell script, to be able to explicitely # direct the output and error to files, without counting # on PBS to manage that, as that one overfloes. # All our outputs and errors will go to the "verbose" files, # while PBS will write its summaries, etc, to the other files. shellScriptPath = logPrefix + '_script.sh' verboseOutputPath = logPrefix + '_verbose_output.log' verboseErrorsPath = logPrefix + '_verbose_errors.log' outputPath = logPrefix + '_output.log' errorsPath = logPrefix + '_errors.log' workDir = os.getcwd() numAttempts = 1000 # For debugging if minutesInDevelQueue > 0: queueName = 'devel' hourString = '00:' + str(minutesInDevelQueue).zfill(2) + ':00' #numAttempts = 1 # For some reason qsub errors out in the devel queue # The "-m eb" option sends the user an email when the process begins and when it ends. # The -r n ensures the job does not restart if it runs out of memory. # Debug the environment #for v in os.environ.keys(): # logger.info("env is " + v + '=' + str(os.environ[v])) # We empty PYTHONSTARTUP and LD_LIBRARY_PATH so that python can function # properly on the nodes. priorityString = '' if priority: priorityString = ' -p ' + str(priority) + ' ' # Generate the shell command shellCommand = ("%s %s > %s 2> %s\n" % (commandPath, args, verboseOutputPath, verboseErrorsPath)) with open(shellScriptPath, 'w') as f: f.write("#!/bin/bash\n") f.write(shellCommand) # Make it executable os.system("chmod a+rx " + shellScriptPath) # Run it pbsCommand = ( 'qsub -r y -q %s -N %s %s -l walltime=%s -W group_list=%s -j oe -e %s -o %s -S /bin/bash -V -C %s -l select=1:ncpus=%d:model=%s -- /usr/bin/env OIB_WORK_DIR=%s PYTHONPATH=%s PYTHONSTARTUP="" LD_LIBRARY_PATH="" %s' % (queueName, jobName, priorityString, hourString, groupId, errorsPath, outputPath, workDir, numCpus, nodeType, workDir, pythonPath, shellScriptPath)) #~/repo/python_env/lib/python2.7/site-packages/ #/u/oalexan1/.local/lib/python2.7/site-packages logger.info(pbsCommand) outputPath = None suppressOutput = True redo = True noThrow = True (out, err, status) = \ asp_system_utils.executeCommand(pbsCommand, outputPath, suppressOutput, redo, noThrow, numAttempts, SLEEP_TIME) if status != 0: logger.info(out) logger.info(err) logger.info("Status is: " + str(status)) jobID = '' else: jobID = cleanJobID(out) logger.info("Submitted job named " + str(jobName) + " with id " + str(jobID)) return jobID
def runBlend(frame, processFolder, lidarFile, fireballDEM, bundleLength, threadText, redo, suppressOutput): # This will run as multiple processes. Hence have to catch all exceptions: try: demFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.alignFileName(), processFolder, bundleLength) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) if demFile == "": print("Could not find DEM for frame: " + str(frame)) return # The names for the final results finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM') finalBlend = finalOutputPrefix + '.tif' finalDiff = finalOutputPrefix + "-diff.csv" fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint') fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif' finalFireballOutput = fireballOutputPrefix + '-DEM.tif' fireballDiffPath = fireballOutputPrefix + "-diff.csv" if not redo: set1Exists = False if (os.path.exists(finalBlend) and os.path.exists(finalDiff)): print("Files exist: " + finalBlend + " " + finalDiff + ".") set1Exists = True set2Exists = True if fireballDEM != "": if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)): print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".") set2Exists = True else: set2Exists = False if set1Exists and set2Exists: return # We will blend the dems with frame offsets within frameOffsets[0:index] filesToWipe = [] bestMean = 1.0e+100 bestBlend = '' bestVals = '' bestDiff = '' # Look at frames with these offsets when blending frameOffsets = [0, 1, -1, 2, -2] for index in range(len(frameOffsets)): # Find all the DEMs up to the current index dems = [] currDemFile = "" for val in range(0, index + 1): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, bundleLength) if currDemFile == "": continue dems.append(currDemFile) if currDemFile == "": # The last DEM was not present. Hence this iteration will add nothing new. continue demString = " ".join(dems) outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index)) # See if we have a pre-existing DEM to use as footprint footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif') blendOutput = outputPrefix + '-tile-0.tif' if os.path.exists(footprintDEM): cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' % (footprintDEM, demString, threadText, outputPrefix)) #filesToWipe.append(footprintDEM) # no longer needed else: cmd = ('dem_mosaic --first-dem-as-reference %s %s -o %s' % (demString, threadText, outputPrefix)) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo) filesToWipe.append(blendOutput) diffPath = outputPrefix + "-diff.csv" filesToWipe.append(diffPath) cmd = ( 'geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, blendOutput, lidarFile, outputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo) # Read in and examine the results results = icebridge_common.readGeodiffOutput(diffPath) print("Current mean error to lidar is " + str(results['Mean'])) if bestMean > float(results['Mean']): bestMean = float(results['Mean']) bestBlend = blendOutput bestVals = demString bestDiff = diffPath logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Update the filenames of the output files print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals) cmd = "mv " + bestBlend + " " + finalBlend print(cmd) asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo) cmd = "mv " + bestDiff + " " + finalDiff print(cmd) asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo) # Generate a thumbnail of the final DEM hillOutput = finalOutputPrefix + '_HILLSHADE.tif' cmd = 'hillshade ' + finalBlend + ' -o ' + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # Generate a low resolution compressed thumbnail of the hillshade for debugging thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif' cmd = 'gdal_translate ' + hillOutput + ' ' + thumbOutput + ' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) os.remove(hillOutput) # Remove this file to keep down the file count # Do another blend, to this DEM's footprint, but not using it if fireballDEM != "": # Find all the DEMs dems = [] for val in range(0, len(frameOffsets)): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, bundleLength) if currDemFile == "": continue dems.append(currDemFile) demString = " ".join(dems) cmd = ('dem_mosaic --this-dem-as-reference %s %s %s -o %s' % (fireballDEM, demString, threadText, fireballOutputPrefix)) #filesToWipe.append(fireballBlendOutput) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo) #filesToWipe.append(fireballDiffPath) cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo) # Read in and examine the results results = icebridge_common.readGeodiffOutput(fireballDiffPath) print("Mean error to lidar in fireball footprint is " + str(results['Mean'])) cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput print(cmd) asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo) # Generate a thumbnail of the final DEM #hillOutput = fireballOutputPrefix+'_HILLSHADE.tif' #cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput #print(cmd) #asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) ## Generate a low resolution compressed thumbnail of the hillshade for debugging #thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif' #cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' #asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) #os.remove(hillOutput) # Remove this file to keep down the file count logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Done with dealing with the fireball footprint # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) except Exception as e: print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc())) sys.stdout.flush()
def main(argsIn): try: usage = '''usage: push_to_nsidc.py <options> ''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument( "--yyyymmdd", dest="yyyymmdd", default="", help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", default="", help="Name of the location of the images (AN, GR, or AL)") parser.add_argument( "--site_yyyymmdd", dest="site_yyyymmdd", default="", help= "A value like GR_20150330, which will be split into site and yyyymmdd by this script." ) parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on.') parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None, help="The folder containing camera calibration.") parser.add_argument( "--reference-dem-folder", dest="refDemFolder", default=None, help="The folder containing DEMs that created orthoimages.") parser.add_argument("--login-info", dest="loginInfo", default=None, help="user,password destination.nsidc.org.") parser.add_argument("--done-file", dest="doneFile", default=None, help="List of runs that were done by now.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) # parse --site_yyyymmdd. Sometimes it is easier to pass this than # to pass separately --site and --yyyymmdd. m = re.match('^(\w+)_(\w+)', options.site_yyyymmdd) if m: options.site = m.group(1) options.yyyymmdd = m.group(2) else: options.site_yyyymmdd = options.site + "_" + options.yyyymmdd # Read the done file and exit if the current flight is done done = set() if options.doneFile != "": with open(options.doneFile, 'r') as f: for val in f: val = val.strip() done.add(val) if options.site_yyyymmdd in done: print("Skipping done flight: " + options.site_yyyymmdd) return 0 run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd()) # Set up logging in the run directory. Log outside of the run dir, # as that one we will wipe logFolder = os.path.abspath( os.path.join(run.getFolder(), '..', 'push_logs')) os.system('mkdir -p ' + logFolder) logLevel = logging.INFO logger = icebridge_common.setUpLogger(logFolder, logLevel, "push") logger.info("Logging in: " + logFolder) # Check the lftp version. On some machines it is too old. (out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'], suppressOutput=True) m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out) if not m: raise Exception('Expecting LFTP version 4.5.') else: logger.info("Found an acceptable version of LFTP.") pushByType(run, options, logger, 'DEM') #pushByType(run, options, logger, 'ORTHO') # need to wait for format decision # Wipe at the end cmd = "rm -rf " + run.getFolder() logger.info(cmd) os.system(cmd)
def main(argsIn): # Command line parsing try: usage = "usage: camera_models_from_nav.py <image_folder> <ortho_folder> <cal_folder> <nav_folder> <output_folder> [options]" parser = optparse.OptionParser(usage=usage) parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=999999, type='int', help='The frame number to finish processing with.') parser.add_option("--input-calibration-camera", dest="inputCalCamera", default="", help="Use this input calibrated camera.") parser.add_option( '--camera-mounting', dest='cameraMounting', default=0, type='int', help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) (options, args) = parser.parse_args(argsIn) if len(args) < 5: print('Error: Missing arguments.') print(usage) return -1 imageFolder = os.path.abspath(args[0]) orthoFolder = os.path.abspath(args[1]) calFolder = os.path.abspath(args[2]) navFolder = os.path.abspath(args[3]) outputFolder = os.path.abspath(args[4]) except optparse.OptionError as msg: raise Usage(msg) runDir = os.path.dirname(orthoFolder) os.system("mkdir -p " + runDir) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(runDir, logLevel, 'camera_models_from_nav_log') if not os.path.exists(orthoFolder): logger.error('Ortho folder ' + orthoFolder + ' does not exist!') return -1 # Find the nav file # - There should only be one or two nav files per flight. fileList = os.listdir(navFolder) fileList = [x for x in fileList if '.out' in x] if len(fileList) == 0: logger.error('No nav files in: ' + navFolder) return -1 navPath = os.path.join(navFolder, fileList[0]) parsedNavPath = navPath.replace('.out', '.txt') if not asp_file_utils.fileIsNonZero(navPath): logger.error('Nav file ' + navPath + ' is invalid!') return -1 # Create the output file only if it is empty or does not exist isNonEmpty = asp_file_utils.fileIsNonZero(parsedNavPath) if not isNonEmpty: # Initialize the output file as being empty logger.info("Create empty file: " + parsedNavPath) open(parsedNavPath, 'w').close() # Append to the output parsed nav file for fileName in fileList: # Convert the nav file from binary to text navPath = os.path.join(navFolder, fileName) with open(navPath, 'r') as f: try: text = f.readline() if 'HTML' in text: # Sometimes the server is down, and instead of the binary nav file # we are given an html file with an error message. logger.info("Have invalid nav file: " + navPath) return -1 # Die in this case! except UnicodeDecodeError as e: # Got a binary file, that means likely we are good pass cmd = asp_system_utils.which( 'sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath logger.info(cmd) if not isNonEmpty: os.system(cmd) cameraPath = options.inputCalCamera if cameraPath == "": # No input camera file provided, look one up. It does not matter much, # as later ortho2pinhole will insert the correct intrinsics. goodFile = False fileList = os.listdir(calFolder) fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))] if not fileList: logger.error('Unable to find any camera files in ' + calFolder) return -1 for fileName in fileList: cameraPath = os.path.join(calFolder, fileName) # Check if this path is valid with open(cameraPath, 'r') as f: for line in f: if 'fu' in line: goodFile = True break if goodFile: break # Get the ortho list orthoFiles = icebridge_common.getTifs(orthoFolder) logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.') # Look up the frame numbers for each ortho file infoDict = {} for ortho in orthoFiles: if ('gray' in ortho) or ('sub' in ortho): continue frame = icebridge_common.getFrameNumberFromFilename(ortho) if frame < options.startFrame or frame > options.stopFrame: continue infoDict[frame] = [ortho, ''] # Get the image file list try: imageFiles = icebridge_common.getTifs(imageFolder) except Exception as e: raise Exception( "Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. " + str(e)) logger.info('Found ' + str(len(imageFiles)) + ' image files.') # Update the second part of each dictionary object for image in imageFiles: if ('gray' in image) or ('sub' in image): continue frame = icebridge_common.getFrameNumberFromFilename(image) if frame < options.startFrame or frame > options.stopFrame: continue if frame not in infoDict: logger.info('Image missing ortho file: ' + image) # don't throw here, that will mess the whole batch, we will recover # the missing one later. continue infoDict[frame][1] = image os.system('mkdir -p ' + outputFolder) orthoListFile = os.path.join( outputFolder, 'ortho_file_list_' + str(options.startFrame) + "_" + str(options.stopFrame) + '.csv') # Open the output file for writing logger.info("Writing: " + orthoListFile) with open(orthoListFile, 'w') as outputFile: # Loop through frames in order for key in sorted(infoDict): # Write the ortho name and the output camera name to the file (ortho, image) = infoDict[key] if not image: #raise Exception('Ortho missing image file: ' +ortho) continue camera = image.replace('.tif', '.tsai') outputFile.write(ortho + ', ' + camera + '\n') # Check if we already have all of the output camera files. haveAllFiles = True with open(orthoListFile, 'r') as inputFile: for line in inputFile: parts = line.split(',') camPath = os.path.join(outputFolder, parts[1].strip()) if not asp_file_utils.fileIsNonZero(camPath): logger.info('Missing file -> ' + camPath) haveAllFiles = False break # Call the C++ tool to generate a camera model for each ortho file if not haveAllFiles: cmd = ( 'nav2cam --input-cam %s --nav-file %s --cam-list %s --output-folder %s --camera-mounting %d' % (cameraPath, parsedNavPath, orthoListFile, outputFolder, options.cameraMounting)) logger.info(cmd) os.system(cmd) else: logger.info("All nav files were already generated.") # Generate a kml file for the nav camera files kmlPath = os.path.join(outputFolder, 'nav_cameras.kml') # This is a hack. If we are invoked from a Pleiades node, do not # create this kml file, as nodes will just overwrite each other. # This job may happen anyway earlier or later when on the head node. if not 'PBS_NODEFILE' in os.environ: try: tempPath = os.path.join(outputFolder, 'list.txt') logger.info('Generating nav camera kml file: ' + kmlPath) os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath) orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole') cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath logger.info(cmd) asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False) os.remove(tempPath) except Exception as e: logger.info("Warning: " + str(e)) logger.info('Finished generating camera models from nav!') return 0
def packAndSendSummaryFolder(run, folder, logger, backupToL2=False): '''Archive the summary folder in case we want to look at it later''' logger.info('Archiving summary folder for run ' + str(run)) cwd = os.getcwd() os.chdir(run.parentFolder) fileName = run.getSummaryTarName() # Create a local tar file. # Turn this approach off, new approach below. # - Some fiddling to make the packed folders convenient cmd = 'tar -chf '+ fileName +' -C '+ folder +'/.. ' + os.path.basename(folder) logger.info(cmd) (out, err, status) = asp_system_utils.executeCommand(cmd, outputPath = None, suppressOutput = True, redo = True, noThrow = True) # This tends to print a very verbose message ans = out + '\n' + err vals = ans.split('\n') if len(vals) < 10: logger.info(ans) else: vals = vals[0:10] logger.info("\n".join(vals)) logger.info("Above output truncated.") # Use shiftc to create a local copy, and we want to include log files too #runFolder = str(run) #sumName = os.path.basename(run.getSummaryFolder()) #cmd = 'shiftc --wait -d -r --dereference --include=\'^.*?(' \ # + icebridge_common.logFilePrefix() + '|' \ # + runFolder + '/' + sumName + '|' \ # + icebridge_common.manager_log_prefix() \ # + ')\' --create-tar ' + runFolder \ # + ' ' + fileName #logger.info(cmd) #os.system(cmd) # Delete any existing copy of the file on lfe lfePath = os.path.join(REMOTE_SUMMARY_FOLDER, fileName) cmd = "ssh lfe 'rm -f " + stripHost(lfePath) + "' 2>/dev/null" logger.info(cmd) os.system(cmd) # Send the file to lfe using shiftc cmd = 'shiftc --wait -d -r ' + fileName + ' ' + lfePath robust_shiftc(cmd, logger) logger.info('Finished sending summary to lfe.') if backupToL2: # Wipe the copy on lunokhod l2Path = os.path.join(L_SUMMARY_FOLDER, fileName) cmd = "ssh " + LUNOKHOD + " 'rm -f "+ stripHost(l2Path) +"' 2>/dev/null" logger.info(cmd) os.system(cmd) # Make target directory on lunokhod cmd = "ssh " + LUNOKHOD + " 'mkdir -p " + os.path.dirname(stripHost(l2Path)) + \ "' 2>/dev/null" logger.info(cmd) os.system(cmd) # Send a copy of the file to Lunokhod for convenience cmd = 'scp ' + fileName + ' ' + l2Path + ' 2>/dev/null' logger.info(cmd) os.system(cmd) # Clean up the local tar file cmd = 'rm -f ' + fileName logger.info(cmd) os.system(cmd) os.chdir(cwd)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/gen_ortho.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 3. usage = '''gen_ortho.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument( '--camera-mounting', dest='cameraMounting', default=0, type=int, help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(jpegFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No jpeg files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in jpegFrameDict: logger.info("Error: Missing jpeg file for frame: " + str(frame) + ".") continue # Find the right image currImage = jpegFrameDict[frame] args = (frame, processFolder, currImage, options.bundleLength, options.cameraMounting, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runOrtho, args)) else: runOrtho(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
print("Skipping done flight: " + options.site_yyyymmdd) return 0 run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd()) # Set up logging in the run directory. Log outside of the run dir, # as that one we will wipe logFolder = os.path.abspath( os.path.join(run.getFolder(), '..', 'push_logs')) os.system('mkdir -p ' + logFolder) logLevel = logging.INFO logger = icebridge_common.setUpLogger(logFolder, logLevel, "push") logger.info("Logging in: " + logFolder) # Check the lftp version. On some machines it is too old. (out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'], suppressOutput=True) m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out) if not m: raise Exception('Expecting LFTP version 4.5.') else: logger.info("Found an acceptable version of LFTP.") pushByType(run, options, logger, 'DEM') #pushByType(run, options, logger, 'ORTHO') # need to wait for format decision # Wipe at the end cmd = "rm -rf " + run.getFolder() logger.info(cmd) os.system(cmd)
def main(argsIn): try: usage = '''usage: regenerate_summary_images.py <options> ''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument( "--data-folder", dest="dataFolder", help="Where all the inputs and outputs are stored.") parser.add_argument("--work-folder", dest="workFolder", help="Where working files are stored.") parser.add_argument("--site", dest="site", help="Site code.") parser.add_argument("--yyyymmdd", dest="yyyymmdd", help="Date.") #parser.add_argument("--dem-tarball", dest="demTarball", default=os.getcwd(), # help="Where all the inputs and outputs are stored.") #parser.add_argument("--ortho-tarball", dest="orthoTarball", default=None, # help="Where to unpack the data.") #parser.add_argument("--summary-tarball", dest="summaryTarball", default=None, # help="Where to unpack the data.") #parser.add_argument("--unpack-dir", dest="unpackDir", default=None, # help="Where to unpack the data.") parser.add_argument( "--node-type", dest="nodeType", default='san', help="Node type to use (wes[mfe], san, ivy, has, bro)") #parser.add_argument("--skip-archive-summary", action="store_true", # dest="skipArchiveSummary", default=False, # help="Skip archiving the summary.") # Debug option parser.add_argument( '--minutes-in-devel-queue', dest='minutesInDevelQueue', type=int, default=0, help="If positive, submit to the devel queue for this many minutes." ) options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) # Check if we are on the right machine (host, err, status) = asp_system_utils.executeCommand(['uname', '-n'], suppressOutput=True) host = host.strip() if 'pfe' in host and options.nodeType not in PFE_NODES: raise Exception("From machine " + host + " can only launch on: " + " ".join(PFE_NODES)) if 'mfe' in host and options.nodeType != 'wes': raise Exception("From machine " + host + " can only launch on: wes") # Make sure our paths will work when called from PBS options.dataFolder = os.path.abspath(options.dataFolder) #os.system('mkdir -p ' + options.unpackDir) # TODO: Check folders! run = run_helper.RunHelper(options.site, options.yyyymmdd, options.workFolder) runFolder = os.path.join(options.workFolder, str(run)) os.system('mkdir -p ' + runFolder) logFolder = os.path.join(runFolder, 'logs') # Set up logging in the run directory os.system('mkdir -p ' + logFolder) logLevel = logging.INFO logger = icebridge_common.setUpLogger( logFolder, logLevel, icebridge_common.manager_log_prefix()) logger.info("Logging in: " + logFolder) checkRequiredTools( ) # Make sure all the needed tools can be found before we start logger.info("Disabling core dumps.") # these just take a lot of room os.system("ulimit -c 0") os.system("umask 022") # enforce files be readable by others # See how many hours we used so far. I think this counter gets updated once a day. (out, err, status) = asp_system_utils.executeCommand("acct_ytd", outputPath=None, suppressOutput=True, redo=True, noThrow=True) logger.info("Hours used so far:\n" + out + '\n' + err) try: # Fetch and extract the tarball files from Lou localDemFolder = os.path.join(options.dataFolder, run.name() + '_dems') localOrthoFolder = os.path.join(options.dataFolder, run.name() + '_orthos') demSummaryFolder = os.path.join(options.dataFolder, run.name() + '_dem_summaries') orthoSummaryFolder = os.path.join(options.dataFolder, run.name() + '_ortho_summaries') missingDemFiles = [] missingOrthoFiles = [] for f in os.listdir(localDemFolder): if 'temp' in f: raise Exception('Bad file: ' + f) if ('IODEM3' in f) and (f[-4:] == '.tif'): inputPath = os.path.join(localDemFolder, f) outputPath = os.path.join( demSummaryFolder, f.replace('DEM.tif', 'DEM_browse.tif')) if not os.path.exists(outputPath): missingDemFiles.append((inputPath, outputPath)) for f in os.listdir(localOrthoFolder): if 'temp' in f: raise Exception('Bad file: ' + f) if ('IODIM3' in f) and (f[-4:] == '.tif'): inputPath = os.path.join(localOrthoFolder, f) outputPath = os.path.join(orthoSummaryFolder, f.replace('ORTHO.tif', 'ORTHO.jpg')) if not os.path.exists(outputPath): missingOrthoFiles.append((inputPath, outputPath)) #print 'Fetching and unpacking tarballs...' #fetchTarball(options.demTarball, localDemFolder) #fetchTarball(options.orthoTarball, localOrthoFolder) #fetchTarball(options.summaryTarball, localSummaryFolder) # If the summary tarball unpacked to []/summary/summary, # work with the lower level folder from now on. #localSummaryFolder = descendIfNeeded(localSummaryFolder) # Make a list of all input files that are missing their summary file, and # the desired output path for that file. #missingDemFiles = getMissingSummaryFiles(localDemFolder, localSummaryFolder, isOrtho=False) #missingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, localSummaryFolder, isOrtho=True ) # Divide this list into chunks and for each chunk generate a file containing all of # the gdal_translate commands that need to be executed. print 'Writing command files...' commandFileLength = getParallelParams(options.nodeType)[2] commandFilePrefix = os.path.join(runFolder, 'convert_commands_') print 'Clearing existing command files.' os.system('rm ' + commandFilePrefix + '*') commandFileList = writeCommandFiles(missingDemFiles, missingOrthoFiles, commandFilePrefix, commandFileLength) #raise Exception('DEBUG') # Get the location to store the logs pbsLogFolder = run.getPbsLogFolder() logger.info("Storing logs in: " + pbsLogFolder) os.system('mkdir -p ' + pbsLogFolder) # Call multi_process_command_runner.py through PBS for each chunk. start_time() (baseName, jobIDs) = submitBatchJobs(commandFileList, options, pbsLogFolder, run, logger) # Wait for everything to finish. pbs_functions.waitForJobCompletion(jobIDs, logger, baseName) stop_time("pbs_jobs", logger) # Check that we now have all of the summary files. # - Both of these should now be empty. #newMissingDemFiles = getMissingSummaryFiles(localDemFolder, demSummaryFolder, isOrtho=False) #newMissingOrthoFiles = getMissingSummaryFiles(localOrthoFolder, orthoSummaryFolder, isOrtho=True ) numDemsMissing = 0 numOrthosMissing = 0 for pair in missingDemFiles: if not os.path.exists(pair[1]): numDemsMissing += 1 for pair in missingOrthoFiles: if not os.path.exists(pair[1]): numOrthosMissing += 1 resultText = ( 'After regeneration, missing %d DEM summaries and %d ORTHO summaries' % (numDemsMissing, numOrthosMissing)) logger.info(resultText) runWasSuccess = ((numDemsMissing == 0) and (numOrthosMissing == 0)) # If successful, create a new tarball and send it to Lou. #if runWasSuccess and (not options.skipArchiveSummary): # start_time() # archive_functions.packAndSendSummaryFolder(run, localSummaryFolder, logger) # stop_time("archive summary", logger) except Exception as e: resultText = 'Caught exception: ' + str( e) + '\n' + traceback.format_exc() runWasSuccess = False # Send a summary email. emailAddress = getEmailAddress(icebridge_common.getUser()) logger.info("Sending email to: " + emailAddress) if runWasSuccess: sendEmail(emailAddress, 'OIB summary regen passed', resultText) else: sendEmail(emailAddress, '"OIB summary regen failed', resultText) # TODO: Add automated delete command! #if options.wipeProcessed: # processedFolder = run.getProcessFolder() # logger.info("Will delete: " + processedFolder) # os.system("rm -rf " + processedFolder) logger.info( '==== regenerate_summary_images script has finished for run: ' + str(run) + ' ====')
def runBlend(frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput): WEIGHT_EXP = 1.3 # This will run as multiple processes. Hence have to catch all exceptions. try: demFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.alignFileName(), processFolder, options.bundleLength) lidarCsvFormatString = icebridge_common.getLidarCsvFormat(lidarFile) if demFile == "": print("Could not find DEM for frame: " + str(frame)) return # The names for the final results finalOutputPrefix = os.path.join(batchFolder, 'out-blend-DEM') finalBlend = finalOutputPrefix + '.tif' finalDiff = finalOutputPrefix + "-diff.csv" fireballOutputPrefix = os.path.join(batchFolder, 'out-blend-fb-footprint') fireballBlendOutput = fireballOutputPrefix + '-tile-0.tif' finalFireballOutput = fireballOutputPrefix + '-DEM.tif' fireballDiffPath = fireballOutputPrefix + "-diff.csv" # This is turned off for now. Find the diff between neighboring # aligned DEMs before blending. prevDemFile, prevBatchFolder = \ icebridge_common.frameToFile(frame-1, icebridge_common.alignFileName(), processFolder, options.bundleLength) prevDiffPrefix = os.path.join(batchFolder, 'out-prev') prevDiffFile = prevDiffPrefix + '-diff.tif' if options.computeDiffToPrev and redo and os.path.exists(prevDiffFile): os.system("rm -f " + prevDiffFile) if os.path.exists(prevDemFile) and os.path.exists(demFile): if os.path.exists(prevDiffFile): print("File exists: " + prevDiffFile) else: cmd = ('geodiff --absolute %s %s -o %s' % (prevDemFile, demFile, prevDiffPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, prevDiffFile, suppressOutput, redo) if not redo: set1Exists = False if (os.path.exists(finalBlend) and os.path.exists(finalDiff)): print("Files exist: " + finalBlend + " " + finalDiff + ".") set1Exists = True set2Exists = True if fireballDEM != "": if (os.path.exists(finalFireballOutput) and os.path.exists(fireballDiffPath)): print("Files exist: " + finalFireballOutput + " " + fireballDiffPath + ".") set2Exists = True else: set2Exists = False if set1Exists and set2Exists: return # We will blend the dems with frame offsets within frameOffsets[0:index] filesToWipe = [] bestMean = 1.0e+100 bestBlend = '' bestVals = '' bestDiff = '' # Look at frames with these offsets when blending frameOffsets = [0, 1, -1, 2, -2] for index in range(len(frameOffsets)): # Find all the DEMs up to the current index dems = [] currDemFile = "" for val in range(0, index+1): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) if currDemFile == "": # The last DEM was not present. Hence this iteration will add nothing new. continue # Compute the mean distance between the DEMs # TODO: Make sure this gets cleaned up! meanWorkPrefix = os.path.join(batchFolder, 'bd') meanDiff = getMeanDemDiff(dems, meanWorkPrefix) # If the mean error between DEMs is creater than this, # use a less aggressive blending method. MEAN_DIFF_BLEND_THRESHOLD = 1.0 demString = " ".join(dems) outputPrefix = os.path.join(batchFolder, 'out-blend-' + str(index)) # See if we have a pre-existing DEM to use as footprint footprintDEM = os.path.join(batchFolder, 'out-trans-footprint-DEM.tif') blendOutput = outputPrefix + '-tile-0.tif' if os.path.exists(footprintDEM): cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, footprintDEM, demString, threadText, outputPrefix)) else: cmd = ('dem_mosaic --weights-exponent %f --first-dem-as-reference %s %s -o %s' % (WEIGHT_EXP, demString, threadText, outputPrefix)) if meanDiff > MEAN_DIFF_BLEND_THRESHOLD: cmd += ' --propagate-nodata --use-centerline-weights ' print(cmd) # Execute the blend command. # - Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True print(cmd) asp_system_utils.executeCommand(cmd, blendOutput, suppressOutput, localRedo) filesToWipe.append(blendOutput) diffPath = outputPrefix + "-diff.csv" filesToWipe.append(diffPath) # Compute post-blending error to lidar cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, blendOutput, lidarFile, outputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, diffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(diffPath) print("Current mean error to lidar is " + str(results['Mean'])) if bestMean > float(results['Mean']): bestMean = float(results['Mean']) bestBlend = blendOutput bestVals = demString bestDiff = diffPath except Exception as e: pass logFiles = glob.glob(outputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Update the filenames of the output files print("Best mean error to lidar is " + str(bestMean) + " when blending " + bestVals) cmd = "mv " + bestBlend + " " + finalBlend print(cmd) asp_system_utils.executeCommand(cmd, finalBlend, suppressOutput, redo) cmd = "mv " + bestDiff + " " + finalDiff print(cmd) asp_system_utils.executeCommand(cmd, finalDiff, suppressOutput, redo) # Generate a thumbnail of the final DEM hillOutput = finalOutputPrefix+'_HILLSHADE.tif' cmd = 'hillshade ' + finalBlend +' -o ' + hillOutput asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) # Generate a low resolution compressed thumbnail of the hillshade for debugging thumbOutput = finalOutputPrefix + '_HILLSHADE_browse.tif' cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) os.system("rm -f " + hillOutput) # Remove this file to keep down the file count # Do another blend, to this DEM's footprint, but not using it if fireballDEM != "": # Find all the DEMs dems = [] for val in range(0, len(frameOffsets)): offset = frameOffsets[val] currDemFile, currBatchFolder = \ icebridge_common.frameToFile(frame + offset, icebridge_common.alignFileName(), processFolder, options.bundleLength) if currDemFile == "": continue dems.append(currDemFile) demString = " ".join(dems) cmd = ('dem_mosaic --weights-exponent %f --this-dem-as-reference %s %s %s -o %s' % (WEIGHT_EXP, fireballDEM, demString, threadText, fireballOutputPrefix)) #filesToWipe.append(fireballBlendOutput) print(cmd) # Sometimes there is junk left from a previous interrupted run. So if we # got so far, recreate all files. localRedo = True asp_system_utils.executeCommand(cmd, fireballBlendOutput, suppressOutput, localRedo) #filesToWipe.append(fireballDiffPath) cmd = ('geodiff --absolute --csv-format %s %s %s -o %s' % (lidarCsvFormatString, fireballBlendOutput, lidarFile, fireballOutputPrefix)) print(cmd) asp_system_utils.executeCommand(cmd, fireballDiffPath, suppressOutput, redo) # Read in and examine the results try: results = icebridge_common.readGeodiffOutput(fireballDiffPath) print("Mean error to lidar in fireball footprint is " + str(results['Mean'])) cmd = "mv " + fireballBlendOutput + " " + finalFireballOutput print(cmd) asp_system_utils.executeCommand(cmd, finalFireballOutput, suppressOutput, redo) except Exception as e: pass # Generate a thumbnail of the final DEM #hillOutput = fireballOutputPrefix+'_HILLSHADE.tif' #cmd = 'hillshade ' + finalFireballOutput +' -o ' + hillOutput #print(cmd) #asp_system_utils.executeCommand(cmd, hillOutput, suppressOutput, redo) ## Generate a low resolution compressed thumbnail of the hillshade for debugging #thumbOutput = fireballOutputPrefix + '_HILLSHADE_browse.tif' #cmd = 'gdal_translate '+hillOutput+' '+thumbOutput+' -of GTiff -outsize 40% 40% -b 1 -co "COMPRESS=JPEG"' #asp_system_utils.executeCommand(cmd, thumbOutput, suppressOutput, redo) #os.system("rm -f " + hillOutput) # Remove this file to keep down the file count logFiles = glob.glob(fireballOutputPrefix + "*" + "-log-" + "*") filesToWipe += logFiles # Done with dealing with the fireball footprint # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.system("rm -f " + fileName) # TODO: Handle this cleanup better! os.system('rm -f ' + meanWorkPrefix + '*') except Exception as e: print('Blending failed!\n' + str(e) + ". " + str(traceback.print_exc())) sys.stdout.flush()
def runOrtho(frame, processFolder, imageFile, bundleLength, cameraMounting, threadText, redo, suppressOutput): os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others # This will run as multiple processes. Hence have to catch all exceptions: projBounds = () try: # Retrieve the aligned camera file alignCamFile, batchFolder = \ icebridge_common.frameToFile(frame, icebridge_common.alignedBundleStr() + '*' + str(frame) + '.tsai', processFolder, bundleLength) if alignCamFile == "": print("Could not find aligned camera for frame: " + str(frame)) return # To ensure we mapproject the image fully, mosaic the several DEMs # around it. Keep the closest 5. Try to grab more first to account # for skipped frames. frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4] dems = [] for offset in frameOffsets: # Find the DEM file for the desired frame demFile, batchFolder = icebridge_common.frameToFile(frame + offset, icebridge_common.blendFileName(), processFolder, bundleLength) # If the central DEM is missing, we are out of luck if offset == 0 and demFile == "": print("Could not find blended DEM for frame: " + str(frame + offset)) return if offset == 0: demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False) projBounds = demGeoInfo['projection_bounds'] # minX maxX minY maxY if demFile == "": # Missing DEM continue if len(dems) >= 5: break # too many already dems.append(demFile) demList = " ".join(dems) # Call this one more time, to get the current batch folder currDemFile, batchFolder = icebridge_common.frameToFile(frame, icebridge_common.blendFileName(), processFolder, bundleLength) # The names for the final results finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName()) finalOrthoPreview = os.path.join(batchFolder, icebridge_common.orthoPreviewFileName()) if (not redo) and os.path.exists(finalOrtho): print("File exists: " + finalOrtho + ".") else: filesToWipe = [] # If the center dem spans say 1 km, there's no way the # ortho can span more than 5 km, unless something is # seriously out of whack, such as alignment failing for # some neighbours. In the best case, if the center dem is # 1 km by 1 km, the obtained ortho will likely be 1.4 km # by 1 km, as an image extends beyond its stereo dem with # a neighbor. factor = float(2.0) projWinStr = "" if len(projBounds) >= 4: # projBounds is in the format minX maxX minY maxY widX = float(projBounds[1]) - float(projBounds[0]) widY = float(projBounds[3]) - float(projBounds[2]) projBounds = ( float(projBounds[0]) - factor*widX, # minX float(projBounds[1]) + factor*widX, # maxX float(projBounds[2]) - factor*widY, # minY float(projBounds[3]) + factor*widY # maxY ) projWinStr = ("--t_projwin %f %f %f %f" % \ (projBounds[0], projBounds[2], projBounds[1], projBounds[3])) # See if we have a pre-existing DEM to use as footprint mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic') mosaicOutput = mosaicPrefix + '-tile-0.tif' cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' % (demList, threadText, projWinStr, mosaicPrefix)) filesToWipe.append(mosaicOutput) # no longer needed # Generate the DEM mosaic print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo) # Borow some pixels from the footprint DEM,just to grow a bit the real estate finalFootprintDEM = os.path.join(batchFolder, icebridge_common.footprintFileName()) if os.path.exists(finalFootprintDEM): mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2') mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif' cmd = ('dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s' % (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2)) print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo, noThrow = True) if os.path.exists(mosaicOutput2): cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput print(cmd) os.system(cmd) # TODO: Look at more aggressive hole-filling. But need a testcase. filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*') # First mapproject to create a tif image with 4 channels. # Then pull 3 channels and compress them as jpeg, while keeping the # image a geotiff. tempOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName() + "_tmp.tif") # There is no need for this file to exist unless it is stray junk if os.path.exists(tempOrtho): os.remove(tempOrtho) # If needed, generate a temporary camera file to correct a mounting rotation. # - When the camera mount is rotated 90 degrees stereo is run on a corrected version # but ortho needs to work on the original uncorrected jpeg image. tempCamFile = alignCamFile + '_temp_rot.tsai' tempCamFile = createRotatedCameraFile(alignCamFile, tempCamFile, cameraMounting) # Run mapproject. The grid size is auto-determined. cmd = ('mapproject --no-geoheader-info %s %s %s %s %s' % (mosaicOutput, imageFile, tempCamFile, tempOrtho, threadText)) print(cmd) asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo) # Set temporary files to be cleaned up filesToWipe.append(tempOrtho) if tempCamFile != alignCamFile: filesToWipe.append(tempCamFile) # This makes the images smaller than Rose's by a factor of about 4, # even though both types are jpeg compressed. Rose's images filtered # through this command also get compressed by a factor of 4. # I conclude that the jpeg compression used by Rose was not as # aggressive as the one used in gdal_translate, but there is no # apparent knob to control that. cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho print(cmd) asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo) # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) if (not redo) and os.path.exists(finalOrthoPreview): print("File exists: " + finalOrthoPreview + ".") else: cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \ ' ' + finalOrthoPreview print(cmd) asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo) except Exception as e: print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc())) os.system("rm -f core.*") # these keep on popping up # To ensure we print promptly what we did so far sys.stdout.flush()
#medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth, # gsdFrameSkip) #outputResolution = icebridge_common.gsdToDemRes(medianGsd) #logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution)) outputResolution = 0.4 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += camera+' ' cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString logger.info('Running orbitviz on input files...') # Suppress (potentially long) output asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo) # Set up options for process_icebridge_batch extraOptions = '' if options.numThreads: extraOptions += ' --num-threads ' + str(options.numThreads) if options.solve_intr: extraOptions += ' --solve-intrinsics ' if options.isSouth: extraOptions += ' --south ' if options.maxDisplacement: extraOptions += ' --max-displacement ' + str(options.maxDisplacement) if options.fireballFolder: extraOptions += ' --fireball-folder ' + str(options.fireballFolder) if options.cleanup: extraOptions += ' --cleanup '
for (image, camera) in zip(imageFiles, cameraFiles): frameNumber = getFrameNumberFromFilename(image) if (getFrameNumberFromFilename(camera) != frameNumber): print 'Error: input files do not align!' print (image, camera) return -1 imageString += image +' ' # Build strings for the bundle_adjust step cameraString += camera+' ' # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in zip(imageFiles, cameraFiles): vizString += image +' ' + camera+' ' cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o '+ orbitvizBefore +' '+ vizString asp_system_utils.executeCommand(cmd, orbitvizBefore, suppressOutput, redo) if options.globalBundleAdjust: # TODO: Intrinsics??? # Bundle adjust all of the cameras # - Could use an overlap of 4 but there is very little overlap at that point. # - If we start dealing with crossover paths we can use the KML overlap method. print 'Setting up bundle adjustment...' baFolder = os.path.join(outputFolder, 'group_bundle') baPrefix = os.path.join(baFolder, 'out') cmd = ('bundle_adjust '+ imageString + cameraString + ' --overlap-limit 3 --local-pinhole --solve-intrinsics -o '+ baPrefix) suppressOutput = False redo = False baOutFile = baPrefix +'-'+ os.path.basename(cameraFiles[-1]) asp_system_utils.executeCommand(cmd, baOutFile, suppressOutput, redo)
gsdFrameSkip = 1 medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth, gsdFrameSkip) outputResolution = medianGsd * GSD_RESOLUTION_MULTIPLIER logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution)) #return 0 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += camera + ' ' cmd = 'orbitviz_pinhole --hide-labels -o ' + orbitvizBefore + ' ' + vizString logger.info('Running orbitviz on input files...') asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo) # Suppress (potentially long) output #raise Exception('DEBUG') # Set up options for process_icebridge_batch extraOptions = '' if options.numThreads: extraOptions += ' --num-threads ' + str(options.numThreads) if options.numProcessesPerBatch: extraOptions += ' --num-processes-per-batch ' + str( options.numProcessesPerBatch) if options.solve_intr: extraOptions += ' --solve-intrinsics ' if options.isSouth: extraOptions += ' --south ' if options.maxDisplacement: extraOptions += ' --max-displacement ' + str(options.maxDisplacement)
def runOrtho(frame, processFolder, imageFile, bundleLength, threadText, redo, suppressOutput): os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others # This will run as multiple processes. Hence have to catch all exceptions: projBounds = () try: alignCamFile, batchFolder = \ icebridge_common.frameToFile(frame, icebridge_common.alignedBundleStr() + '*' + str(frame) + '.tsai', processFolder, bundleLength) if alignCamFile == "": print("Could not find aligned camera for frame: " + str(frame)) return # To ensure we mapproject the image fully, mosaic the several DEMs # around it. Keep the closest 5. Try to grab more first to account # for skipped frames. frameOffsets = [0, 1, -1, 2, -2, -3, 3, -4, 4] dems = [] for offset in frameOffsets: demFile, batchFolder = icebridge_common.frameToFile( frame + offset, icebridge_common.blendFileName(), processFolder, bundleLength) # If the central DEM is missing, we are out of luck if offset == 0 and demFile == "": print("Could not find DEM for frame: " + str(frame + offset)) return if offset == 0: demGeoInfo = asp_geo_utils.getImageGeoInfo(demFile, getStats=False) projBounds = demGeoInfo[ 'projection_bounds'] # minX maxX minY maxY if demFile == "": # Missing DEM continue if len(dems) >= 5: break # too many already dems.append(demFile) demList = " ".join(dems) # Call this one more time, to get the current batch folder currDemFile, batchFolder = icebridge_common.frameToFile( frame, icebridge_common.blendFileName(), processFolder, bundleLength) # The names for the final results finalOrtho = os.path.join(batchFolder, icebridge_common.orthoFileName()) finalOrthoPreview = os.path.join( batchFolder, icebridge_common.orthoPreviewFileName()) if (not redo) and os.path.exists(finalOrtho): print("File exists: " + finalOrtho + ".") else: filesToWipe = [] # If the center dem spans say 1 km, there's no way the # ortho can span more than 5 km, unless something is # seriously out of whack, such as alignment failing for # some neighbours. In the best case, if the center dem is # 1 km by 1 km, the obtained ortho will likely be 1.4 km # by 1 km, as an image extends beyond its stereo dem with # a neighbor. factor = float(2.0) projWinStr = "" if len(projBounds) >= 4: # projBounds is in the format minX maxX minY maxY widX = float(projBounds[1]) - float(projBounds[0]) widY = float(projBounds[3]) - float(projBounds[2]) projBounds = ( float(projBounds[0]) - factor * widX, # minX float(projBounds[1]) + factor * widX, # maxX float(projBounds[2]) - factor * widY, # minY float(projBounds[3]) + factor * widY # maxY ) projWinStr = ("--t_projwin %f %f %f %f" % \ (projBounds[0], projBounds[2], projBounds[1], projBounds[3])) # See if we have a pre-existing DEM to use as footprint mosaicPrefix = os.path.join(batchFolder, 'out-temp-mosaic') mosaicOutput = mosaicPrefix + '-tile-0.tif' cmd = ('dem_mosaic --hole-fill-length 500 %s %s %s -o %s' % (demList, threadText, projWinStr, mosaicPrefix)) filesToWipe.append(mosaicOutput) # no longer needed print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput, suppressOutput, localRedo) # Borow some pixels from the footprint DEM,just to grow a bit the real estate finalFootprintDEM = os.path.join( batchFolder, icebridge_common.footprintFileName()) if os.path.exists(finalFootprintDEM): mosaicPrefix2 = os.path.join(batchFolder, 'out-temp-mosaic2') mosaicOutput2 = mosaicPrefix2 + '-tile-0.tif' cmd = ( 'dem_mosaic --priority-blending-length 50 %s %s %s %s -o %s' % (mosaicOutput, finalFootprintDEM, threadText, projWinStr, mosaicPrefix2)) print(cmd) localRedo = True # The file below should not exist unless there was a crash asp_system_utils.executeCommand(cmd, mosaicOutput2, suppressOutput, localRedo, noThrow=True) if os.path.exists(mosaicOutput2): cmd = "mv -f " + mosaicOutput2 + " " + mosaicOutput print(cmd) os.system(cmd) # TODO: Look at more aggressive hole-filling. But need a testcase. filesToWipe += glob.glob(mosaicPrefix + '*' + '-log-' + '*') # First mapproject to create a tif image with 4 channels. # Then pull 3 channels and compress them as jpeg, while keeping the # image a geotiff. tempOrtho = os.path.join( batchFolder, icebridge_common.orthoFileName() + "_tmp.tif") # There is no need for this file to exist unless it is stray junk if os.path.exists(tempOrtho): os.remove(tempOrtho) # Run mapproject. The grid size is auto-determined. cmd = ( 'mapproject --no-geoheader-info %s %s %s %s %s' % (mosaicOutput, imageFile, alignCamFile, tempOrtho, threadText)) print(cmd) asp_system_utils.executeCommand(cmd, tempOrtho, suppressOutput, redo) filesToWipe.append(tempOrtho) # This makes the images smaller than Rose's by a factor of about 4, # even though both types are jpeg compressed. Rose's images filtered # through this command also get compressed by a factor of 4. # I conclude that the jpeg compression used by Rose was not as # aggressive as the one used in gdal_translate, but there is no # apparent knob to control that. cmd = "gdal_translate -b 1 -b 2 -b 3 -co compress=jpeg " + tempOrtho + " " + finalOrtho print(cmd) asp_system_utils.executeCommand(cmd, finalOrtho, suppressOutput, redo) # Clean up extra files for fileName in filesToWipe: if os.path.exists(fileName): print("Removing: " + fileName) os.remove(fileName) if (not redo) and os.path.exists(finalOrthoPreview): print("File exists: " + finalOrthoPreview + ".") else: cmd = 'gdal_translate -scale -outsize 25% 25% -of jpeg ' + finalOrtho + \ ' ' + finalOrthoPreview print(cmd) asp_system_utils.executeCommand(cmd, finalOrthoPreview, suppressOutput, redo) except Exception as e: print('Ortho creation failed!\n' + str(e) + ". " + str(traceback.print_exc())) os.system("rm -f core.*") # these keep on popping up # To ensure we print promptly what we did so far sys.stdout.flush()
def main(argsIn): try: # Sample usage: # python full_processing_script.py \ # --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \ # --start-frame 350 --stop-frame 353 --skip-validate \ # --camera-calibration-folder camera_calib \ # --reference-dem-folder ref_dem_folder # An output folder will be crated automatically (with a name like # AN_20091016), or its name can be specified via the --output-folder # option. usage = '''full_processing_script.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument("--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None, help="The file to use to find which camera was used for which " + \ "flight. By default it is in the same directory as this script " + \ "and named camera_lookup.txt.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") # TODO: Compute this automatically?? parser.add_argument('--overlap-limit', dest='overlapLimit', default=2, type=int, help="The number of images to treat as overlapping for " + \ "bundle adjustment.") parser.add_argument('--stereo-arguments', dest='stereoArgs', # set --min-xcorr-level 0 to do the left-to-right # and right-to-left consistency check at the lowest level. default='--stereo-algorithm 2 --min-xcorr-level 0', help='Extra arguments to pass to stereo.') parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on.') parser.add_argument('--frames-file', dest='framesFile', default="", help='Specific frames to run ortho2pinhole on within this frame range.') parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None, type=int, help="The maximum number of lidar files to fetch. " + \ "This is used in debugging.") parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None, help="The folder containing camera calibration.") parser.add_argument("--input-calibration-camera", dest="inputCalCamera", default="", help="Instead of looking up the calibrated camera in the calibration folder, use this one.") parser.add_argument("--output-calibration-camera", dest="outputCalCamera", default="", help="If specified, float the intrinsics and write the optimized model here.") parser.add_argument("--output-model-type", dest="outputModelType", default="RPC", help="Generate a distortion model of type RPC, RPC5, or RPC6.") parser.add_argument("--reference-dem-folder", dest="refDemFolder", default=None, help="The folder containing DEMs that created orthoimages.") parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. " + \ "fault is no additional folder") parser.add_argument("--simple-cameras", action="store_true", dest="simpleCameras", default=False, help="Don't use orthoimages to refine the camera models.") # This option is only needed when generating camera models from the nav files. parser.add_argument('--camera-mounting', default=0, dest='cameraMounting', type=int, help='0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.') # Performance options parser.add_argument('--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-ortho-processes', dest='numOrthoProcesses', default=-1, type=int, help='The number of simultaneous ortho processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') # Action control parser.add_argument("--skip-fetch", action="store_true", dest="noFetch", default=False, help="Skip data fetching.") parser.add_argument("--skip-convert", action="store_true", dest="noConvert", default=False, help="Skip data conversion.") parser.add_argument("--stop-after-fetch", action="store_true", dest="stopAfterFetch", default=False, help="Stop program after data fetching.") parser.add_argument("--stop-after-convert", action="store_true", dest="stopAfterConvert", default=False, help="Stop program after data conversion.") parser.add_argument("--skip-validate", action="store_true", dest="skipValidate", default=False, help="Skip input data validation.") parser.add_argument("--ignore-missing-lidar", action="store_true", dest="ignoreMissingLidar", default=False, help="Keep going if the lidar is missing.") parser.add_argument("--log-batches", action="store_true", dest="logBatches", default=False, help="Log the required batch commands without running them.") parser.add_argument('--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_argument('--many-ip', action='store_true', default=False, dest='manyip', help='If to use a lot of IP in bundle adjustment from the beginning.') parser.add_argument("--dry-run", action="store_true", dest="dryRun", default=False, help="Set up the input directories but do not fetch/process any imagery.") parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False, help="Try fetching again if some files turned out invalid " + \ "during conversions.") parser.add_argument("--refetch-index", action="store_true", dest="refetchIndex", default=False, help="Force refetch of the index file.") parser.add_argument("--refetch-nav", action="store_true", dest="refetchNav", default=False, help="Force refetch of the nav file.") parser.add_argument("--stop-after-index-fetch", action="store_true", dest="stopAfterIndexFetch", default=False, help="Stop after fetching the indices.") parser.add_argument("--no-nav", action="store_true", dest="noNavFetch", default=False, help="Don't fetch or convert the nav data.") parser.add_argument("--no-lidar-convert", action="store_true", dest="noLidarConvert", default=False, help="Skip lidar files in the conversion step.") parser.add_argument("--no-ortho-convert", action="store_true", dest="noOrthoConvert", default=False, help="Skip generating camera models in the conversion step.") parser.add_argument("--skip-fast-conversions", action="store_true", dest="skipFastConvert", default=False, help="Skips all non-ortho conversions.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if options.numOrthoProcesses < 0: options.numOrthoProcesses = options.numProcesses isSouth = icebridge_common.checkSite(options.site) # Turned off elevation limits here since they are being set from LIDAR data. ## Add the site based elevation limits to the stereoArgs option #altLimits = icebridge_common.getElevationLimits(options.site) #options.stereoArgs = (' %s --elevation-limit %f %f ' # % (options.stereoArgs, altLimits[0], altLimits[1])) options.stereoArgs = (' %s ' % (options.stereoArgs)) if options.cameraLookupFile is None: options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt') if not os.path.isfile(options.cameraLookupFile): raise Exception("Can't find camera file: " + options.cameraLookupFile) if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder(options.site, options.yyyymmdd) if options.stopAfterIndexFetch: options.stopAfterFetch = True os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Record everything logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_processing_log_frames_' + \ str(options.startFrame) + "_" + str(options.stopFrame)) # Make sure we later know what we were doing logger.info("full_processing_script.py " + " ".join(argsIn)) (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput = True) logger.info("Running on machine: " + out) logger.info("Work dir is " + os.getcwd()) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Perform some input checks and initializations # These are not needed unless cameras are initialized if options.inputCalFolder is None or not os.path.exists(options.inputCalFolder): raise Exception("Missing camera calibration folder.") if options.refDemFolder is None or not os.path.exists(options.refDemFolder): raise Exception("Missing reference DEM folder.") refDemName = icebridge_common.getReferenceDemName(options.site) refDemPath = os.path.join(options.refDemFolder, refDemName) if not os.path.exists(refDemPath): raise Exception("Missing reference DEM: " + refDemPath) # TODO: CLEAN UP!!! # Set up the output folders cameraFolder = icebridge_common.getCameraFolder(options.outputFolder) imageFolder = icebridge_common.getImageFolder(options.outputFolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) fireballFolder = icebridge_common.getFireballFolder(options.outputFolder) corrFireballFolder = icebridge_common.getCorrFireballFolder(options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) navFolder = icebridge_common.getNavFolder(options.outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder) processedFolder = icebridge_common.getProcessedFolder(options.outputFolder) if options.outputCalCamera != "": # Prepare to solve for intrinsics. Note that this modifies some things along the way. (options, cameraFolder, navCameraFolder, processedFolder) = \ solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder, logger) # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processedFolder = os.path.join(processedFolder, options.processingSubfolder) logger.info('Will write to processing subfolder: ' + options.processingSubfolder) # If something failed in the first attempt either in fetch or in # convert, we will wipe bad files, and try to refetch/re-convert. numAttempts = 1 if options.reFetch and (not options.noFetch): numAttempts = 2 for attempt in range(numAttempts): if numAttempts > 1: logger.info("Fetch/convert attempt: " + str(attempt+1)) ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder, jpegFolder, orthoFolder, fireballFolder, corrFireballFolder, lidarFolder, processedFolder, navFolder, navCameraFolder, refDemPath, logger) if ans == 0: break if options.stopAfterFetch or options.dryRun or options.stopAfterConvert: logger.info('Fetch/convert finished!') return 0 # Call the processing routine processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, corrFireballFolder, processedFolder, isSouth, refDemPath) if options.outputCalCamera != "": # Finish solving for intrinscs. solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, processedFolder, isSouth, logger)
def main(argsIn): try: # Sample usage: # python ~/projects/StereoPipeline/src/asp/IceBridge/blend_dems.py --site GR \ # --yyyymmdd 20120315 --start-frame 2490 --stop-frame 2491 --bundle-length 2 \ # --num-threads 8 --num-processes 10 usage = '''blend_dems.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument( '--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on. This frame will also be processed.') parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. "+\ "The default is no additional folder.") parser.add_argument( "--compute-diff-to-prev-dem", action="store_true", dest="computeDiffToPrev", default=False, help="Compute the absolute difference between the current DEM " + "and the one before it.") parser.add_argument("--blend-to-fireball-footprint", action="store_true", dest="blendToFireball", default=False, help="Create additional blended DEMs having the same " + \ "footprint as Fireball DEMs.") # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() os.system("ulimit -c 0") # disable core dumps os.system("rm -f core.*") # these keep on popping up os.system("umask 022") # enforce files be readable by others if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_blend_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) logger.info(str(argsIn)) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if options.blendToFireball: fireballFrameDict = icebridge_common.getCorrectedFireballDems( options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) threadText = '' if options.numThreads: threadText = '--threads ' + str(options.numThreads) redo = False suppressOutput = True taskHandles = [] if options.numProcesses > 1: pool = multiprocessing.Pool(options.numProcesses) # Bound the frames sortedFrames = sorted(orthoFrameDict.keys()) if len(sortedFrames) > 0: if options.startFrame < sortedFrames[0]: options.startFrame = sortedFrames[0] if options.stopFrame > sortedFrames[-1] + 1: options.stopFrame = sortedFrames[-1] + 1 else: # No ortho files, that means nothing to do options.startFrame = 0 options.stopFrame = 0 for frame in range(options.startFrame, options.stopFrame): if not frame in orthoFrameDict: logger.info("Error: Missing ortho file for frame: " + str(frame) + ".") continue orthoFile = orthoFrameDict[frame] try: lidarFile = icebridge_common.findMatchingLidarFile( orthoFile, lidarFolder) except: # Skip if no lidar file matches this frame continue fireballDEM = "" if options.blendToFireball: if frame in fireballFrameDict: fireballDEM = fireballFrameDict[frame] else: logger.info("No fireball DEM for frame: " + str(frame)) args = (frame, processFolder, lidarFile, fireballDEM, options, threadText, redo, suppressOutput) # Run things sequentially if only one process, to make it easy to debug if options.numProcesses > 1: taskHandles.append(pool.apply_async(runBlend, args)) else: runBlend(*args) if options.numProcesses > 1: icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q', sleepTime=20) icebridge_common.stopTaskPool(pool)
for (image, camera) in zip(imageFiles, cameraFiles): frameNumber = getFrameNumberFromFilename(image) if (getFrameNumberFromFilename(camera) != frameNumber): print 'Error: input files do not align!' print(image, camera) return -1 imageString += image + ' ' # Build strings for the bundle_adjust step cameraString += camera + ' ' # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in zip(imageFiles, cameraFiles): vizString += image + ' ' + camera + ' ' cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o ' + orbitvizBefore + ' ' + vizString asp_system_utils.executeCommand(cmd, orbitvizBefore, suppressOutput, redo) if options.globalBundleAdjust: # TODO: Intrinsics??? # Bundle adjust all of the cameras # - Could use an overlap of 4 but there is very little overlap at that point. # - If we start dealing with crossover paths we can use the KML overlap method. print 'Setting up bundle adjustment...' baFolder = os.path.join(outputFolder, 'group_bundle') baPrefix = os.path.join(baFolder, 'out') cmd = ('bundle_adjust ' + imageString + cameraString + ' --overlap-limit 3 --local-pinhole --solve-intrinsics -o ' + baPrefix) suppressOutput = False redo = False baOutFile = baPrefix + '-' + os.path.basename(cameraFiles[-1])
def main(argsIn): try: usage = '''usage: push_to_nsidc.py <options> ''' parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--yyyymmdd", dest="yyyymmdd", default="", help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument("--site", dest="site", default = "", help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--site_yyyymmdd", dest="site_yyyymmdd", default = "", help="A value like GR_20150330, which will be split into site and yyyymmdd by this script.") parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on.') parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None, help="The folder containing camera calibration.") parser.add_argument("--reference-dem-folder", dest="refDemFolder", default=None, help="The folder containing DEMs that created orthoimages.") parser.add_argument("--login-info", dest="loginInfo", default=None, help="user,password destination.nsidc.org.") parser.add_argument("--done-file", dest="doneFile", default=None, help="List of runs that were done by now.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) # parse --site_yyyymmdd. Sometimes it is easier to pass this than # to pass separately --site and --yyyymmdd. m = re.match('^(\w+)_(\w+)', options.site_yyyymmdd) if m: options.site = m.group(1) options.yyyymmdd = m.group(2) else: options.site_yyyymmdd = options.site + "_" + options.yyyymmdd # Read the done file and exit if the current flight is done done = set() if options.doneFile != "": with open(options.doneFile, 'r') as f: for val in f: val = val.strip() done.add(val) if options.site_yyyymmdd in done: print("Skipping done flight: " + options.site_yyyymmdd) return 0 run = run_helper.RunHelper(options.site, options.yyyymmdd, os.getcwd()) # Set up logging in the run directory. Log outside of the run dir, # as that one we will wipe logFolder = os.path.abspath(os.path.join(run.getFolder(), '..', 'push_logs')) os.system('mkdir -p ' + logFolder) logLevel = logging.INFO logger = icebridge_common.setUpLogger(logFolder, logLevel, "push") logger.info("Logging in: " + logFolder) # Check the lftp version. On some machines it is too old. (out, err, status) = asp_system_utils.executeCommand(['lftp', '--version'], suppressOutput = True) m = re.match('^.*?LFTP\s+\|\s+Version\s+4.5', out) if not m: raise Exception('Expecting LFTP version 4.5.') else: logger.info("Found an acceptable version of LFTP.") pushByType(run, options, logger, 'DEM') #pushByType(run, options, logger, 'ORTHO') # need to wait for format decision # Wipe at the end cmd = "rm -rf " + run.getFolder() logger.info(cmd) os.system(cmd)
icebridge_common.switchWorkDir() if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_ortho_log') (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) processFolder = os.path.join(options.outputFolder, 'processed') # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processFolder = os.path.join(processFolder, options.processingSubfolder) logger.info('Reading from processing subfolder: ' + options.processingSubfolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath +
if options.bundleAdjust: # TODO: Solve for intrinsics? bundlePrefix = os.path.join(outputFolder, "bundle/out") cmd = "bundle_adjust %s %s %s %s -o %s %s -t nadirpinhole --local-pinhole" % ( imageA, imageB, cameraA, cameraB, bundlePrefix, threadText, ) print (cmd) # Point to the new camera models cameraA = bundlePrefix + "-" + os.path.basename(cameraA) cameraB = bundlePrefix + "-" + os.path.basename(cameraB) asp_system_utils.executeCommand(cmd, cameraA, suppressOutput, redo) # Update the baseArgString baseArgString = "%s %s %s %s %s -t nadirpinhole --alignment-method epipolar" % ( imageA, imageB, cameraA, cameraB, outputPrefix, ) if options.use_sgm: # PPRC cmd = "stereo_pprc %s %s" % (baseArgString, threadText) pprcOutput = outputPrefix + "-L.tif" asp_system_utils.executeCommand(cmd, pprcOutput, suppressOutput, redo)
def main(argsIn): try: usage = '''usage: process_icebridge_run.py <image_folder> <camera_folder> <lidar_folder> <output_folder>''' parser = optparse.OptionParser(usage=usage) # Data selection optios parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=-1, type='int', help='The frame number to finish processing with.') parser.add_option('--south', action='store_true', default=False, dest='isSouth', help='MUST be set if the images are in the southern hemisphere.') # Processing options parser.add_option('--stereo-arguments', dest='stereoArgs', default='', help='Additional argument string to be passed to the stereo command.') parser.add_option('--bundle-length', dest='bundleLength', default=2, type='int', help='Number of images to bundle adjust and process at once.') parser.add_option('--image-stereo-interval', dest='imageStereoInterval', default=None, type='int', help='Advance this many frames to get the stereo pair. Default is auto-calculate') parser.add_option('--solve-intrinsics', action='store_true', default=False, dest='solve_intr', help='If to float the intrinsics params.') #parser.add_option('--dem-resolution', dest='demResolution', default=0.4, # type='float', help='Generate output DEMs at this resolution.') parser.add_option('--max-displacement', dest='maxDisplacement', default=20, type='float', help='Max displacement value passed to pc_align.') # Performance options parser.add_option('--num-processes', dest='numProcesses', default=1, type='int', help='The number of simultaneous processes to run.') parser.add_option('--num-threads', dest='numThreads', default=None, type='int', help='The number threads to use per process.') # Action options parser.add_option('--interactive', action='store_true', default=False, dest='interactive', help='If to wait on user input to terminate the jobs.') parser.add_option('--log-batches', action='store_true', default=False, dest='logBatches', help="Just log the batch commands to a file.") parser.add_option('--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_option('--many-ip', action='store_true', default=False, dest='manyip', help='If to use a lot of IP in bundle adjustment from the beginning.') parser.add_option('--dry-run', action='store_true', default=False, dest='dryRun', help="Print but don't launch the processing jobs.") parser.add_option('--ortho-folder', dest='orthoFolder', default=None, help='Use ortho files to adjust processing to the image spacing.') parser.add_option('--fireball-folder', dest='fireballFolder', default=None, help='Location of fireball DEMs for comparison.') parser.add_option('--reference-dem', dest='referenceDem', default=None, help='Reference DEM used to calculate the expected GSD.') (options, args) = parser.parse_args(argsIn) if len(args) < 4: print(usage) return 0 imageFolder = args[0] cameraFolder = args[1] lidarFolder = args[2] outputFolder = args[3] except optparse.OptionError as msg: raise Usage(msg) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Check the inputs for f in [imageFolder, cameraFolder, lidarFolder]: if not os.path.exists(f): logger.error('Input folder '+ f +' does not exist!') return 0 asp_system_utils.mkdir_p(outputFolder) suppressOutput = False redo = False logger.info('\nStarting processing...') # Get a list of all the input files imageCameraPairs = icebridge_common.getImageCameraPairs(imageFolder, cameraFolder, options.startFrame, options.stopFrame, logger) numFiles = len(list(imageCameraPairs)) if numFiles < 2: raise Exception('Failed to find any image camera pairs!') # Check that the files are properly aligned lastFrame = -1 availableFrames = [] for (image, camera) in imageCameraPairs: frameNumber = icebridge_common.getFrameNumberFromFilename(image) availableFrames.append(frameNumber) if (icebridge_common.getFrameNumberFromFilename(camera) != frameNumber): logger.error('Error: input files do not align!\n' + str((image, camera))) return -1 if frameNumber <= lastFrame: logger.error('Error: input frames not sorted properly!\n') return -1 lastFrame = frameNumber # Do not compute output resolution. Will be overwritten anyway per frame. ## Set the output resolution as the computed mean GSD ## TODO: This should be cashed, and recomputed only when the batch file changes! #NUM_GSD_FRAMES = 20 #logger.info('Computing GSD with ' + str(NUM_GSD_FRAMES) + ' frames.') #gsdFrameSkip = len(imageCameraPairs) / NUM_GSD_FRAMES #if gsdFrameSkip < 1: # gsdFrameSkip = 1 #medianGsd = getRunMedianGsd(imageCameraPairs, options.referenceDem, options.isSouth, # gsdFrameSkip) #outputResolution = icebridge_common.gsdToDemRes(medianGsd) #logger.info('OUTPUT_RESOLUTION: ' + str(outputResolution)) outputResolution = 0.4 # Generate a map of initial camera positions orbitvizBefore = os.path.join(outputFolder, 'cameras_in.kml') vizString = '' for (image, camera) in imageCameraPairs: vizString += camera+' ' cmd = 'orbitviz_pinhole --hide-labels -o '+ orbitvizBefore +' '+ vizString logger.info('Running orbitviz on input files...') # Suppress (potentially long) output asp_system_utils.executeCommand(cmd, orbitvizBefore, True, redo) # Set up options for process_icebridge_batch extraOptions = '' if options.numThreads: extraOptions += ' --num-threads ' + str(options.numThreads) if options.solve_intr: extraOptions += ' --solve-intrinsics ' if options.isSouth: extraOptions += ' --south ' if options.maxDisplacement: extraOptions += ' --max-displacement ' + str(options.maxDisplacement) if options.fireballFolder: extraOptions += ' --fireball-folder ' + str(options.fireballFolder) if options.cleanup: extraOptions += ' --cleanup ' if options.manyip: extraOptions += ' --many-ip ' # We ran this before, as part of fetching, so hopefully all the data is cached forceAllFramesInRange = False (breaks, largeSkips) = getImageSpacing(options.orthoFolder, availableFrames, options.startFrame, options.stopFrame, forceAllFramesInRange) if options.imageStereoInterval: logger.info('Using manually specified image stereo interval: ' + str(options.imageStereoInterval)) largeSkips = [] # Always use a manually specified skip interval else: options.imageStereoInterval = 1 sleepTime = 20 # If all we are doing is logging commands then one process is sufficient. # - Wipe the output file while we are at it. batchLogPath = '' batchNum = 0 if options.logBatches: options.numProcesses = 1 sleepTime = 1 batchLogPath = os.path.join(outputFolder, BATCH_COMMAND_LOG_FILE) os.system('rm -f ' + batchLogPath) logger.info('Just generating batch log file '+batchLogPath+', no processing will occur.') logger.info('Starting processing pool with ' + str(options.numProcesses) +' processes.') pool = NonDaemonPool(options.numProcesses) # Call process_icebridge_batch on each batch of images. # - Batch size should be the largest number of images which can be effectively bundle-adjusted. taskHandles = [] batchImageCameraPairs = [] frameNumbers = [] i = 0 # The frame index that starts the current batch while True: # Loop for adding batches # Bugfix: arrived at the last feasible frame (maybe there are more but # they lack cameras). if i >= len(list(imageCameraPairs)): break firstBundleFrame = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[i][0]) # Determine the frame skip amount for this batch (set by the first frame) thisSkipInterval = options.imageStereoInterval if firstBundleFrame in largeSkips: #print(" ", firstBundleFrame, " in largeskips!" ) thisSkipInterval = largeSkips[firstBundleFrame] thisBatchSize = options.bundleLength + thisSkipInterval - 1 # Keep adding frames until we have enough or run out of frames j = i # The frame index that ends the batch while True: if j >= len(imageCameraPairs): # Bugfix: arrived at the last feasible frame. break frameNumber = icebridge_common.getFrameNumberFromFilename(imageCameraPairs[j][0]) # Useful debugging code #print("Framenumber is ", frameNumber) #if int(frameNumber) > 8531: # # pass # for t in range(8340, 8360): # print("i frame is ", t, imageCameraPairs[t][0]) # print("breaks are ", breaks) # sys.exit(1) # Update conditions hitBreakFrame = (frameNumber in breaks) lastFrame = (frameNumber > options.stopFrame) or (j >= numFiles) endBatch = ( len(frameNumbers) >= thisBatchSize ) if lastFrame or endBatch: break # The new frame is too much, don't add it to the batch # Add frame to the list for the current batch batchImageCameraPairs.append(imageCameraPairs[j]) frameNumbers.append(frameNumber) if hitBreakFrame: logger.info("Hit a break, won't start a batch with frame: " + str(frameNumber)) break # Break after this frame, it is the last one added to the batch. j = j + 1 # Done adding frames to this batch if len(frameNumbers) <= thisSkipInterval: logger.info('Batch from frame: ' + str(firstBundleFrame) + ' is too small to run. Skipping.') else: # Submit the batch if not options.logBatches: logger.info('Processing frame number: ' + str(firstBundleFrame)) # The output folder is named after the first and last frame in the batch. # We count on this convention in blend_dems.py. batchFolderName = icebridge_common.batchFolderName(frameNumbers[0], frameNumbers[-1], options.bundleLength) thisOutputFolder = os.path.join(outputFolder, batchFolderName) if not options.logBatches: logger.info('Running processing batch in output folder: ' + \ thisOutputFolder + '\n' + 'with options: ' + \ extraOptions + ' --stereo-arguments ' + \ options.stereoArgs) if not options.dryRun: # Generate the command call taskHandles.append(pool.apply_async(processBatch, (batchImageCameraPairs, lidarFolder, options.referenceDem, thisSkipInterval, thisOutputFolder, extraOptions, outputResolution, options.stereoArgs, batchNum, batchLogPath))) batchNum += 1 # Reset these lists batchImageCameraPairs = [] frameNumbers = [] # Advance to the frame that starts the next batch if hitBreakFrame: # When we hit a break in the frames we need to start the # next batch after the break frame #print("Hit break frame for i, j, frameNumber", i, j, frameNumber) i = j + 1 else: # Start in the next frame that was not used as a "left" stereo image. i = i + options.bundleLength - 1 if lastFrame: break # Quit the main loop if we hit the end of the frame list. # End of loop through input file pairs logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') # Wait for all the tasks to complete icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, options.interactive, quitKey='q', sleepTime=sleepTime) # Either all the tasks are finished or the user requested a cancel. # Clean up the processing pool icebridge_common.stopTaskPool(pool) logger.info('Finished process_icebridge_run.') # to avoid ending a log with 'waiting ...'
def main(argsIn): try: # Sample usage: # python full_processing_script.py \ # --yyyymmdd 20091016 --site AN --num-processes 1 --num-threads 12 --bundle-length 12 \ # --start-frame 350 --stop-frame 353 --skip-validate \ # --camera-calibration-folder camera_calib \ # --reference-dem-folder ref_dem_folder # An output folder will be crated automatically (with a name like # AN_20091016), or its name can be specified via the --output-folder # option. usage = '''full_processing_script.py <options>''' parser = argparse.ArgumentParser(usage=usage) # Run selection parser.add_argument( "--yyyymmdd", dest="yyyymmdd", required=True, help="Specify the year, month, and day in one YYYYMMDD string.") parser.add_argument( "--site", dest="site", required=True, help="Name of the location of the images (AN, GR, or AL)") parser.add_argument("--output-folder", dest="outputFolder", default=None, help="Name of the output folder. If not specified, " + \ "use something like AN_YYYYMMDD.") parser.add_argument("--camera-lookup-file", dest="cameraLookupFile", default=None, help="The file to use to find which camera was used for which " + \ "flight. By default it is in the same directory as this script " + \ "and named camera_lookup.txt.") # Processing options parser.add_argument('--bundle-length', dest='bundleLength', default=2, type=int, help="The number of images to bundle adjust and process " + \ "in a single batch.") # TODO: Compute this automatically?? parser.add_argument('--overlap-limit', dest='overlapLimit', default=2, type=int, help="The number of images to treat as overlapping for " + \ "bundle adjustment.") parser.add_argument( '--max-overlap-ratio', dest='maxOverlapRatio', default=0.85, type=float, help= 'The maximum ratio of overlap between images to be accepted as part of a stereo pair. When floating intrinsics, this will be set to 1, to not upset some bookkeeping.' ) parser.add_argument( '--stereo-arguments', dest='stereoArgs', # set --min-xcorr-level 0 to do the left-to-right # and right-to-left consistency check at the lowest level. default='--stereo-algorithm 2 --min-xcorr-level 0', help='Extra arguments to pass to stereo.') parser.add_argument('--start-frame', dest='startFrame', type=int, default=icebridge_common.getSmallestFrame(), help="Frame to start with. Leave this and stop-frame blank to " + \ "process all frames.") parser.add_argument('--stop-frame', dest='stopFrame', type=int, default=icebridge_common.getLargestFrame(), help='Frame to stop on.') parser.add_argument( '--frames-file', dest='framesFile', default="", help= 'Specific frames to run ortho2pinhole on within this frame range.') parser.add_argument('--max-num-lidar-to-fetch', dest='maxNumLidarToFetch', default=None, type=int, help="The maximum number of lidar files to fetch. " + \ "This is used in debugging.") parser.add_argument("--camera-calibration-folder", dest="inputCalFolder", default=None, help="The folder containing camera calibration.") parser.add_argument( "--input-calibration-camera", dest="inputCalCamera", default="", help= "Instead of looking up the calibrated camera in the calibration folder, use this one." ) parser.add_argument( "--output-calibration-camera", dest="outputCalCamera", default="", help= "If specified, float the intrinsics and write the optimized model here." ) parser.add_argument( "--output-model-type", dest="outputModelType", default="RPC", help="Generate a distortion model of type RPC, RPC5, or RPC6.") parser.add_argument( "--reference-dem-folder", dest="refDemFolder", default=None, help="The folder containing DEMs that created orthoimages.") parser.add_argument("--processing-subfolder", dest="processingSubfolder", default=None, help="Specify a subfolder name where the processing outputs will go. " + \ "fault is no additional folder") parser.add_argument( "--simple-cameras", action="store_true", dest="simpleCameras", default=False, help="Don't use orthoimages to refine the camera models.") # This option is only needed when generating camera models from the nav files. parser.add_argument( '--camera-mounting', default=0, dest='cameraMounting', type=int, help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) # Performance options parser.add_argument( '--num-processes', dest='numProcesses', default=1, type=int, help='The number of simultaneous processes to run.') parser.add_argument( '--num-ortho-processes', dest='numOrthoProcesses', default=-1, type=int, help='The number of simultaneous ortho processes to run.') parser.add_argument('--num-threads', dest='numThreads', default=8, type=int, help='The number of threads per process.') # Action control parser.add_argument("--skip-fetch", action="store_true", dest="noFetch", default=False, help="Skip data fetching.") parser.add_argument("--skip-convert", action="store_true", dest="noConvert", default=False, help="Skip data conversion.") parser.add_argument("--stop-after-fetch", action="store_true", dest="stopAfterFetch", default=False, help="Stop program after data fetching.") parser.add_argument("--stop-after-convert", action="store_true", dest="stopAfterConvert", default=False, help="Stop program after data conversion.") parser.add_argument("--skip-validate", action="store_true", dest="skipValidate", default=False, help="Skip input data validation.") parser.add_argument("--ignore-missing-lidar", action="store_true", dest="ignoreMissingLidar", default=False, help="Keep going if the lidar is missing.") parser.add_argument( "--log-batches", action="store_true", dest="logBatches", default=False, help="Log the required batch commands without running them.") parser.add_argument( '--cleanup', action='store_true', default=False, dest='cleanup', help='If the final result is produced delete intermediate files.') parser.add_argument( '--many-ip', action='store_true', default=False, dest='manyip', help= 'If to use a lot of IP in bundle adjustment from the beginning.') parser.add_argument( "--dry-run", action="store_true", dest="dryRun", default=False, help= "Set up the input directories but do not fetch/process any imagery." ) parser.add_argument("--refetch", action="store_true", dest="reFetch", default=False, help="Try fetching again if some files turned out invalid " + \ "during conversions.") parser.add_argument("--refetch-index", action="store_true", dest="refetchIndex", default=False, help="Force refetch of the index file.") parser.add_argument("--refetch-nav", action="store_true", dest="refetchNav", default=False, help="Force refetch of the nav file.") parser.add_argument("--stop-after-index-fetch", action="store_true", dest="stopAfterIndexFetch", default=False, help="Stop after fetching the indices.") parser.add_argument("--no-nav", action="store_true", dest="noNavFetch", default=False, help="Don't fetch or convert the nav data.") parser.add_argument("--no-lidar-convert", action="store_true", dest="noLidarConvert", default=False, help="Skip lidar files in the conversion step.") parser.add_argument( "--no-ortho-convert", action="store_true", dest="noOrthoConvert", default=False, help="Skip generating camera models in the conversion step.") parser.add_argument("--skip-fast-conversions", action="store_true", dest="skipFastConvert", default=False, help="Skips all non-ortho conversions.") options = parser.parse_args(argsIn) except argparse.ArgumentError as msg: parser.error(msg) icebridge_common.switchWorkDir() if options.numOrthoProcesses < 0: options.numOrthoProcesses = options.numProcesses isSouth = icebridge_common.checkSite(options.site) # Turned off elevation limits here since they are being set from LIDAR data. ## Add the site based elevation limits to the stereoArgs option #altLimits = icebridge_common.getElevationLimits(options.site) #options.stereoArgs = (' %s --elevation-limit %f %f ' # % (options.stereoArgs, altLimits[0], altLimits[1])) options.stereoArgs = (' %s ' % (options.stereoArgs)) if options.cameraLookupFile is None: options.cameraLookupFile = P.join(basepath, 'camera_lookup.txt') if not os.path.isfile(options.cameraLookupFile): raise Exception("Can't find camera file: " + options.cameraLookupFile) if len(options.yyyymmdd) != 8 and len(options.yyyymmdd) != 9: # Make an exception for 20100422a raise Exception("The --yyyymmdd field must have length 8 or 9.") if options.outputFolder is None: options.outputFolder = icebridge_common.outputFolder( options.site, options.yyyymmdd) if options.stopAfterIndexFetch: options.stopAfterFetch = True os.system('mkdir -p ' + options.outputFolder) logLevel = logging.INFO # Record everything logger = icebridge_common.setUpLogger(options.outputFolder, logLevel, 'icebridge_processing_log_frames_' + \ str(options.startFrame) + "_" + str(options.stopFrame)) # Make sure we later know what we were doing logger.info("full_processing_script.py " + " ".join(argsIn)) (out, err, status) = asp_system_utils.executeCommand(['uname', '-a'], suppressOutput=True) logger.info("Running on machine: " + out) logger.info("Work dir is " + os.getcwd()) os.system("ulimit -c 0") # disable core dumps os.system("umask 022") # enforce files be readable by others # Perform some input checks and initializations # These are not needed unless cameras are initialized if options.inputCalFolder is None or not os.path.exists( options.inputCalFolder): raise Exception("Missing camera calibration folder.") if options.refDemFolder is None or not os.path.exists( options.refDemFolder): raise Exception("Missing reference DEM folder.") refDemName = icebridge_common.getReferenceDemName(options.site) refDemPath = os.path.join(options.refDemFolder, refDemName) if not os.path.exists(refDemPath): raise Exception("Missing reference DEM: " + refDemPath) # TODO: CLEAN UP!!! # Set up the output folders cameraFolder = icebridge_common.getCameraFolder(options.outputFolder) imageFolder = icebridge_common.getImageFolder(options.outputFolder) jpegFolder = icebridge_common.getJpegFolder(options.outputFolder) orthoFolder = icebridge_common.getOrthoFolder(options.outputFolder) fireballFolder = icebridge_common.getFireballFolder(options.outputFolder) corrFireballFolder = icebridge_common.getCorrFireballFolder( options.outputFolder) lidarFolder = icebridge_common.getLidarFolder(options.outputFolder) navFolder = icebridge_common.getNavFolder(options.outputFolder) navCameraFolder = icebridge_common.getNavCameraFolder(options.outputFolder) processedFolder = icebridge_common.getProcessedFolder(options.outputFolder) if options.outputCalCamera != "": if options.maxOverlapRatio < 1: raise Exception ("For optimizing intrinsics, must set --max-overlap-ratio to 1, " + \ "to always use consecutive frames.") # Prepare to solve for intrinsics. Note that this modifies some things along the way. (options, cameraFolder, navCameraFolder, processedFolder) = \ solveIntrinsics_Part1(options, jpegFolder, cameraFolder, navCameraFolder, processedFolder, logger) # Handle subfolder option. This is useful for comparing results with different parameters! if options.processingSubfolder: processedFolder = os.path.join(processedFolder, options.processingSubfolder) logger.info('Will write to processing subfolder: ' + options.processingSubfolder) # If something failed in the first attempt either in fetch or in # convert, we will wipe bad files, and try to refetch/re-convert. numAttempts = 1 if options.reFetch and (not options.noFetch): numAttempts = 2 for attempt in range(numAttempts): if numAttempts > 1: logger.info("Fetch/convert attempt: " + str(attempt + 1)) ans = runFetchConvert(options, isSouth, cameraFolder, imageFolder, jpegFolder, orthoFolder, fireballFolder, corrFireballFolder, lidarFolder, processedFolder, navFolder, navCameraFolder, refDemPath, logger) if ans == 0: break if options.stopAfterFetch or options.dryRun or options.stopAfterConvert: logger.info('Fetch/convert finished!') return 0 # Call the processing routine processTheRun(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, corrFireballFolder, processedFolder, isSouth, refDemPath) if options.outputCalCamera != "": # Finish solving for intrinscs. solveIntrinsics_Part2(options, imageFolder, cameraFolder, lidarFolder, orthoFolder, processedFolder, isSouth, logger)
threadText = '' if options.numThreads: threadText = ' --threads ' + str(options.numThreads) + ' ' # BUNDLE_ADJUST if options.bundleAdjust: # TODO: Solve for intrinsics? bundlePrefix = os.path.join(outputFolder, 'bundle/out') cmd = ( 'bundle_adjust %s %s %s %s -o %s %s -t nadirpinhole --local-pinhole' % (imageA, imageB, cameraA, cameraB, bundlePrefix, threadText)) print(cmd) # Point to the new camera models cameraA = bundlePrefix + '-' + os.path.basename(cameraA) cameraB = bundlePrefix + '-' + os.path.basename(cameraB) asp_system_utils.executeCommand(cmd, cameraA, suppressOutput, redo) # Update the baseArgString baseArgString = ( '%s %s %s %s %s -t nadirpinhole --alignment-method epipolar' % (imageA, imageB, cameraA, cameraB, outputPrefix)) if options.use_sgm: # PPRC cmd = ('stereo_pprc %s %s' % (baseArgString, threadText)) pprcOutput = outputPrefix + '-L.tif' asp_system_utils.executeCommand(cmd, pprcOutput, suppressOutput, redo) # CORR # - This should be single threaded to use the SGM processing. # - TODO: Use blob filtering to reduce outliers?
if baOverlapLimit < MIN_BA_OVERLAP: baOverlapLimit = MIN_BA_OVERLAP cmd = ( 'bundle_adjust %s -o %s %s --camera-weight %d -t nadirpinhole --local-pinhole --overlap-limit %d' % (imageCameraString, bundlePrefix, threadText, CAMERA_WEIGHT, baOverlapLimit)) if options.solve_intr: cmd += ' --solve-intrinsics' # Point to the new camera models for pair in inputPairs: newCamera = bundlePrefix + '-' + os.path.basename(pair[1]) pair[1] = newCamera imageCameraString.replace(pair[1], newCamera) # Run the BA command asp_system_utils.executeCommand(cmd, newCamera, suppressOutput, redo) # Generate a map of initial camera positions orbitvizAfter = os.path.join(outputFolder, 'cameras_out.kml') vizString = '' for (image, camera) in inputPairs: vizString += image + ' ' + camera + ' ' cmd = 'orbitviz --hide-labels -t nadirpinhole -r wgs84 -o ' + \ orbitvizAfter +' '+ vizString asp_system_utils.executeCommand(cmd, orbitvizAfter, suppressOutput, redo) # STEREO # Call stereo seperately on each pair of cameras demFiles = []