def getImageList(self, prependFolder=False): '''Return a list containing all the currently stored image files''' imageFolder = self.getImageFolder() images = icebridge_common.getTifs(imageFolder) if prependFolder: images = [os.path.join(imageFolder, x) for x in images] images.sort() return images
if not fileList: logger.error('Unable to find any camera files in ' + calFolder) return -1 for fileName in fileList: cameraPath = os.path.join(calFolder, fileName) # Check if this path is valid with open(cameraPath, 'r') as f: for line in f: if 'fu' in line: goodFile = True break if goodFile: break # Get the ortho list orthoFiles = icebridge_common.getTifs(orthoFolder) logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.') # Look up the frame numbers for each ortho file infoDict = {} for ortho in orthoFiles: if ('gray' in ortho) or ('sub' in ortho): continue frame = icebridge_common.getFrameNumberFromFilename(ortho) if frame < options.startFrame or frame > options.stopFrame: continue infoDict[frame] = [ortho, ''] # Get the image file list try: imageFiles = icebridge_common.getTifs(imageFolder)
def pushByType(run, options, logger, dataType): # Fetch the ortho index from NSIDC if missing outputFolder = run.getFolder() logger.info("Output folder is " + outputFolder) os.system("mkdir -p " + outputFolder) # Current directory. It is important to go from /u to the real dir which is /nobackup... unpackDir = os.path.realpath(os.getcwd()) logger.info("Unpack directory is " + unpackDir) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): fetchIndices(options, logger) logger.info("Reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) # Fetch unarchived folder if missing if dataType == 'DEM': unarchivedFolder = run.getAssemblyFolder() elif dataType == 'ORTHO': unarchivedFolder = run.getProcessFolder() else: raise Exception("Unknown data type: " + dataType) logger.info("Unarchived data folder is " + unarchivedFolder) # Especially for ortho, force-fetch each time, as there is no good way # of checking if we fetched well before. start_time() if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType): return stop_time("fetching archived data by type: " + dataType, logger) # Make the output directory at NSIDC m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd) if m: outDir = options.site + "_" + m.group(1) + "." + m.group( 2) + "." + m.group(3) else: raise Exception("Could not parse: " + options.yyyymmdd) # Keep the output directory locally here localDirPath = os.path.join(outputFolder, dataType, outDir) os.system("mkdir -p " + localDirPath) logger.info("Storing the renamed " + dataType + " files in " + localDirPath) logger.info("Directory name at NSIDC: " + outDir) # Read the DEMs and orthos, and copy them to outDir according to the final convention if dataType == 'DEM': dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True) else: dataFiles = glob.glob( os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif')) for dataFile in dataFiles: # Here we use the convention from archive_functions.py for DEMs and from how we store orthos. if dataType == 'DEM': m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \ "\.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) else: m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \ "out-ortho.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) if frameNumber < options.startFrame or frameNumber > options.stopFrame: continue # For each data file, copy from the ortho its meta info if not frameNumber in orthoFrameDict.keys(): # Bugfix: Ortho fetching failed, try again fetchIndices(options, logger) logger.info("Re-reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if not frameNumber in orthoFrameDict.keys(): # This time there is nothing we can do raise Exception("Cannot find ortho for frame: " + str(frameNumber)) orthoFile = orthoFrameDict[frameNumber] [dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile) # It is always possible that the ortho file date will be the next day # after the current flight date, if the flight goes after midnight. # So it is not unreasonable that options.yyyymmdd != dateString. if dataType == 'DEM': outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber)) else: # TODO: Need to think more of the naming convention. outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber)) cmd = "/bin/cp -fv " + dataFile + " " + os.path.join( localDirPath, outFile) logger.info(cmd) os.system(cmd) # Push the directory to NSIDC remoteDirPath = os.path.join( os.path.basename(os.path.dirname(localDirPath)), os.path.basename(localDirPath)) remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath) logger.info("Storing at NSIDC in: " + remoteDirPath) cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \ ' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo logger.info(cmd) start_time() (output, err, status) = asp_system_utils.executeCommand(cmd, suppressOutput=True) #status = os.system(cmd) logger.info("LFTP output and error: " + output + ' ' + err) logger.info("LFTP status: " + str(status)) #if status != 0: # raise Exception("Problem pushing") stop_time("push to NSIDC", logger)
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, inputCalCamera, cameraLookupPath, noNav, navCameraFolder, yyyymmdd, site, refDemPath, cameraFolder, simpleCameras, startFrame, stopFrame, framesFile, numProcesses, numThreads, logger): '''Generate camera models from the ortho files. Returns false if any files were not generated.''' logger.info('Generating camera models from ortho images...') imageFiles = icebridge_common.getTifs(imageFolder) orthoFiles = icebridge_common.getTifs(orthoFolder) if navCameraFolder != "": estimateFiles = icebridge_common.getByExtension( navCameraFolder, '.tsai') else: estimateFiles = [] # See if to process frames from file filesSet = set() if framesFile != "": filesSet = icebridge_common.readLinesInSet(framesFile) # Make a dictionary of ortho files by frame # - The orthoFiles list contains _gray.tif as well as the original # images. Prefer the gray versions because it saves a bit of time # in the ortho2pinhole process. orthoFrames = {} for f in orthoFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue # Record this file if it is the first of this frame or # if it is the gray version of this frame. if (frame not in orthoFrames) or ('_gray.tif' in f): orthoFrames[frame] = f # Make a dictionary of estimated camera files by frame estimatedFrames = {} for f in estimateFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue estimatedFrames[frame] = f imageFiles.sort() logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] outputFiles = [] for imageFile in imageFiles: # Skip non-image files (including _sub images made by stereo_gui) # TODO: Use a function here from icebridge_common. Also replace # all similar locations. ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) if not ((frame >= startFrame) and (frame <= stopFrame)): continue if (framesFile != "") and (str(frame) not in filesSet): continue if not frame in orthoFrames.keys(): continue # Find the estimated camera file to use with this ortho frame. orthoFile = orthoFrames[frame] try: estimatedCameraFile = estimatedFrames[frame] estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile) except: # For now treat this as an error, a missing nav file suggests # that something is going wrong with the flight! if not noNav: logger.error('Missing nav estimated camera for frame ' + str(frame)) continue else: estimatedCameraFile = None estimatedCameraPath = None #estimatedCameraFile = None #estimatedCameraPath = None # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join( cameraFolder, icebridge_common.getCameraFileName(imageFile)) outputFiles.append(outputCamFile) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs continue # Determine which input camera file will be used for this frame if inputCalCamera == "": inputCamFile = getCalibrationFileForFrame(cameraLookupPath, inputCalFolder, frame, yyyymmdd, site) else: # This logic will force to use a given camera rather than # looking it up. This is not the usual way of doing things. inputCamFile = inputCalCamera if not os.path.exists(inputCalCamera): raise Exception("Could not find: " + inputCalCamera) orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCameras, numThreads) if numProcesses > 1: # Add ortho2pinhole command to the task pool taskHandles.append( pool.apply_async(cameraFromOrthoWrapper, orthoArgs)) else: # Single process, more logging info this way cameraFromOrthoWrapper(*orthoArgs) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q') # All tasks should be finished icebridge_common.stopTaskPool(pool) logger.info('Finished ortho processing.') # Run a check to see if we got all the output files for f in outputFiles: if not os.path.exists(f): return False return True
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder=True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder=True) if not skipValidate: validFilesList = icebridge_common.validFilesList( os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk( validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder=True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ((frame >= startFrame) and (frame <= stopFrame)): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile( inputPath, orthoFrameDict[frame]) except Exception, e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90' cmd = ('%s %s -colorspace Gray %s %s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE | re.MULTILINE | re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw
def pushByType(run, options, logger, dataType): # Fetch the ortho index from NSIDC if missing outputFolder = run.getFolder() logger.info("Output folder is " + outputFolder) os.system("mkdir -p " + outputFolder) # Current directory. It is important to go from /u to the real dir which is /nobackup... unpackDir = os.path.realpath(os.getcwd()) logger.info("Unpack directory is " + unpackDir) orthoFolder = icebridge_common.getOrthoFolder(outputFolder) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): fetchIndices(options, logger) logger.info("Reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) # Fetch unarchived folder if missing if dataType == 'DEM': unarchivedFolder = run.getAssemblyFolder() elif dataType == 'ORTHO': unarchivedFolder = run.getProcessFolder() else: raise Exception("Unknown data type: " + dataType) logger.info("Unarchived data folder is " + unarchivedFolder) # Especially for ortho, force-fetch each time, as there is no good way # of checking if we fetched well before. start_time() if not archive_functions.fetchProcessedByType(run, unpackDir, logger, dataType): return stop_time("fetching archived data by type: " + dataType, logger) # Make the output directory at NSIDC m = re.match("(\d\d\d\d)(\d\d)(\d\d)", options.yyyymmdd) if m: outDir = options.site + "_" + m.group(1) + "." + m.group(2) + "." + m.group(3) else: raise Exception("Could not parse: " + options.yyyymmdd) # Keep the output directory locally here localDirPath = os.path.join(outputFolder, dataType, outDir) os.system("mkdir -p " + localDirPath) logger.info("Storing the renamed " + dataType + " files in " + localDirPath) logger.info("Directory name at NSIDC: " + outDir) # Read the DEMs and orthos, and copy them to outDir according to the final convention if dataType == 'DEM': dataFiles = icebridge_common.getTifs(unarchivedFolder, prependFolder=True) else: dataFiles = glob.glob(os.path.join(unarchivedFolder, 'batch_*', 'out-ortho.tif')) for dataFile in dataFiles: # Here we use the convention from archive_functions.py for DEMs and from how we store orthos. if dataType == 'DEM': m = re.match("^.*?" + unarchivedFolder + "/F_(\d+)_\d+_" + dataType + \ "\.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) else: m = re.match("^.*?" + unarchivedFolder + "/batch_(\d+)_\d+_\d+/" + \ "out-ortho.tif$", dataFile) if not m: continue frameNumber = int(m.group(1)) if frameNumber < options.startFrame or frameNumber > options.stopFrame: continue # For each data file, copy from the ortho its meta info if not frameNumber in orthoFrameDict.keys(): # Bugfix: Ortho fetching failed, try again fetchIndices(options, logger) logger.info("Re-reading ortho index: " + orthoIndexPath) (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath) if not frameNumber in orthoFrameDict.keys(): # This time there is nothing we can do raise Exception("Cannot find ortho for frame: " + str(frameNumber)) orthoFile = orthoFrameDict[frameNumber] [dateString, timeString] = icebridge_common.parseTimeStamps(orthoFile) # It is always possible that the ortho file date will be the next day # after the current flight date, if the flight goes after midnight. # So it is not unreasonable that options.yyyymmdd != dateString. if dataType == 'DEM': outFile = ('IODEM3_%s_%s_%05d_DEM.tif' % (dateString, timeString, frameNumber)) else: # TODO: Need to think more of the naming convention. outFile = ('IODEM3_%s_%s_%05d_ORTHO.tif' % (dateString, timeString, frameNumber)) cmd = "/bin/cp -fv " + dataFile + " " + os.path.join(localDirPath, outFile) logger.info(cmd) os.system(cmd) # Push the directory to NSIDC remoteDirPath = os.path.join(os.path.basename(os.path.dirname(localDirPath)), os.path.basename(localDirPath)) remoteDirPath = os.path.join('/incoming', 'Ames', remoteDirPath) logger.info("Storing at NSIDC in: " + remoteDirPath) cmd = 'lftp -e "mirror -P 20 -c -R -vvv --delete --delete-first ' + localDirPath + \ ' ' + remoteDirPath + ' -i \'\.(tif)$\'; bye\" -u ' + options.loginInfo logger.info(cmd) start_time() (output, err, status) = asp_system_utils.executeCommand(cmd, suppressOutput = True) #status = os.system(cmd) logger.info("LFTP output and error: " + output + ' ' + err) logger.info("LFTP status: " + str(status)) #if status != 0: # raise Exception("Problem pushing") stop_time("push to NSIDC", logger)
def main(argsIn): # Command line parsing try: usage = "usage: camera_models_from_nav.py <image_folder> <ortho_folder> <cal_folder> <nav_folder> <output_folder> [options]" parser = optparse.OptionParser(usage=usage) parser.add_option('--start-frame', dest='startFrame', default=-1, type='int', help='The frame number to start processing with.') parser.add_option('--stop-frame', dest='stopFrame', default=999999, type='int', help='The frame number to finish processing with.') parser.add_option("--input-calibration-camera", dest="inputCalCamera", default="", help="Use this input calibrated camera.") parser.add_option( '--camera-mounting', dest='cameraMounting', default=0, type='int', help= '0=right-forwards, 1=left-forwards, 2=top-forwards, 3=bottom-forwards.' ) (options, args) = parser.parse_args(argsIn) if len(args) < 5: print('Error: Missing arguments.') print(usage) return -1 imageFolder = os.path.abspath(args[0]) orthoFolder = os.path.abspath(args[1]) calFolder = os.path.abspath(args[2]) navFolder = os.path.abspath(args[3]) outputFolder = os.path.abspath(args[4]) except optparse.OptionError as msg: raise Usage(msg) runDir = os.path.dirname(orthoFolder) os.system("mkdir -p " + runDir) logLevel = logging.INFO # Make this an option?? logger = icebridge_common.setUpLogger(runDir, logLevel, 'camera_models_from_nav_log') if not os.path.exists(orthoFolder): logger.error('Ortho folder ' + orthoFolder + ' does not exist!') return -1 # Find the nav file # - There should only be one or two nav files per flight. fileList = os.listdir(navFolder) fileList = [x for x in fileList if '.out' in x] if len(fileList) == 0: logger.error('No nav files in: ' + navFolder) return -1 navPath = os.path.join(navFolder, fileList[0]) parsedNavPath = navPath.replace('.out', '.txt') if not asp_file_utils.fileIsNonZero(navPath): logger.error('Nav file ' + navPath + ' is invalid!') return -1 # Create the output file only if it is empty or does not exist isNonEmpty = asp_file_utils.fileIsNonZero(parsedNavPath) if not isNonEmpty: # Initialize the output file as being empty logger.info("Create empty file: " + parsedNavPath) open(parsedNavPath, 'w').close() # Append to the output parsed nav file for fileName in fileList: # Convert the nav file from binary to text navPath = os.path.join(navFolder, fileName) with open(navPath, 'r') as f: try: text = f.readline() if 'HTML' in text: # Sometimes the server is down, and instead of the binary nav file # we are given an html file with an error message. logger.info("Have invalid nav file: " + navPath) return -1 # Die in this case! except UnicodeDecodeError as e: # Got a binary file, that means likely we are good pass cmd = asp_system_utils.which( 'sbet2txt.pl') + ' -q ' + navPath + ' >> ' + parsedNavPath logger.info(cmd) if not isNonEmpty: os.system(cmd) cameraPath = options.inputCalCamera if cameraPath == "": # No input camera file provided, look one up. It does not matter much, # as later ortho2pinhole will insert the correct intrinsics. goodFile = False fileList = os.listdir(calFolder) fileList = [x for x in fileList if (('.tsai' in x) and ('~' not in x))] if not fileList: logger.error('Unable to find any camera files in ' + calFolder) return -1 for fileName in fileList: cameraPath = os.path.join(calFolder, fileName) # Check if this path is valid with open(cameraPath, 'r') as f: for line in f: if 'fu' in line: goodFile = True break if goodFile: break # Get the ortho list orthoFiles = icebridge_common.getTifs(orthoFolder) logger.info('Found ' + str(len(orthoFiles)) + ' ortho files.') # Look up the frame numbers for each ortho file infoDict = {} for ortho in orthoFiles: if ('gray' in ortho) or ('sub' in ortho): continue frame = icebridge_common.getFrameNumberFromFilename(ortho) if frame < options.startFrame or frame > options.stopFrame: continue infoDict[frame] = [ortho, ''] # Get the image file list try: imageFiles = icebridge_common.getTifs(imageFolder) except Exception as e: raise Exception( "Cannot continue with nav generation, will resume later when images are created. This is not a fatal error. " + str(e)) logger.info('Found ' + str(len(imageFiles)) + ' image files.') # Update the second part of each dictionary object for image in imageFiles: if ('gray' in image) or ('sub' in image): continue frame = icebridge_common.getFrameNumberFromFilename(image) if frame < options.startFrame or frame > options.stopFrame: continue if frame not in infoDict: logger.info('Image missing ortho file: ' + image) # don't throw here, that will mess the whole batch, we will recover # the missing one later. continue infoDict[frame][1] = image os.system('mkdir -p ' + outputFolder) orthoListFile = os.path.join( outputFolder, 'ortho_file_list_' + str(options.startFrame) + "_" + str(options.stopFrame) + '.csv') # Open the output file for writing logger.info("Writing: " + orthoListFile) with open(orthoListFile, 'w') as outputFile: # Loop through frames in order for key in sorted(infoDict): # Write the ortho name and the output camera name to the file (ortho, image) = infoDict[key] if not image: #raise Exception('Ortho missing image file: ' +ortho) continue camera = image.replace('.tif', '.tsai') outputFile.write(ortho + ', ' + camera + '\n') # Check if we already have all of the output camera files. haveAllFiles = True with open(orthoListFile, 'r') as inputFile: for line in inputFile: parts = line.split(',') camPath = os.path.join(outputFolder, parts[1].strip()) if not asp_file_utils.fileIsNonZero(camPath): logger.info('Missing file -> ' + camPath) haveAllFiles = False break # Call the C++ tool to generate a camera model for each ortho file if not haveAllFiles: cmd = ( 'nav2cam --input-cam %s --nav-file %s --cam-list %s --output-folder %s --camera-mounting %d' % (cameraPath, parsedNavPath, orthoListFile, outputFolder, options.cameraMounting)) logger.info(cmd) os.system(cmd) else: logger.info("All nav files were already generated.") # Generate a kml file for the nav camera files kmlPath = os.path.join(outputFolder, 'nav_cameras.kml') # This is a hack. If we are invoked from a Pleiades node, do not # create this kml file, as nodes will just overwrite each other. # This job may happen anyway earlier or later when on the head node. if not 'PBS_NODEFILE' in os.environ: try: tempPath = os.path.join(outputFolder, 'list.txt') logger.info('Generating nav camera kml file: ' + kmlPath) os.system('ls ' + outputFolder + '/*.tsai > ' + tempPath) orbitviz_pinhole = asp_system_utils.which('orbitviz_pinhole') cmd = orbitviz_pinhole + ' --hide-labels -o ' + kmlPath + ' --input-list ' + tempPath logger.info(cmd) asp_system_utils.executeCommand(cmd, kmlPath, suppressOutput=True, redo=False) os.remove(tempPath) except Exception as e: logger.info("Warning: " + str(e)) logger.info('Finished generating camera models from nav!') return 0
def getCameraModelsFromOrtho(imageFolder, orthoFolder, inputCalFolder, inputCalCamera, cameraLookupFile, noNav, navCameraFolder, yyyymmdd, site, refDemPath, cameraFolder, simpleCameras, startFrame, stopFrame, framesFile, numProcesses, numThreads, logger): '''Generate camera models from the ortho files. Returns false if any files were not generated.''' logger.info('Generating camera models from ortho images...') imageFiles = icebridge_common.getTifs(imageFolder) orthoFiles = icebridge_common.getTifs(orthoFolder) if navCameraFolder != "": estimateFiles = icebridge_common.getByExtension(navCameraFolder, '.tsai') else: estimateFiles = [] # See if to process frames from file filesSet = set() if framesFile != "": filesSet = icebridge_common.readLinesInSet(framesFile) # Make a dictionary of ortho files by frame # - The orthoFiles list contains _gray.tif as well as the original # images. Prefer the gray versions because it saves a bit of time # in the ortho2pinhole process. orthoFrames = {} for f in orthoFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue # Record this file if it is the first of this frame or # if it is the gray version of this frame. if (frame not in orthoFrames) or ('_gray.tif' in f): orthoFrames[frame] = f # Make a dictionary of estimated camera files by frame estimatedFrames = {} for f in estimateFiles: frame = icebridge_common.getFrameNumberFromFilename(f) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue estimatedFrames[frame] = f imageFiles.sort() logger.info('Starting ortho processing pool with ' + str(numProcesses) + ' processes.') pool = multiprocessing.Pool(numProcesses) # Loop through all input images taskHandles = [] outputFiles = [] for imageFile in imageFiles: # Skip non-image files (including _sub images made by stereo_gui) # TODO: Use a function here from icebridge_common. Also replace # all similar locations. ext = os.path.splitext(imageFile)[1] if (ext != '.tif') or ('_sub' in imageFile) or ('pct.tif' in imageFile): continue # Get associated orthofile frame = icebridge_common.getFrameNumberFromFilename(imageFile) if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if (framesFile != "") and (str(frame) not in filesSet): continue if not frame in orthoFrames.keys(): continue # Find the estimated camera file to use with this ortho frame. orthoFile = orthoFrames[frame] try: estimatedCameraFile = estimatedFrames[frame] estimatedCameraPath = os.path.join(navCameraFolder, estimatedCameraFile) except: # For now treat this as an error, a missing nav file suggests # that something is going wrong with the flight! if not noNav: logger.error('Missing nav estimated camera for frame ' + str(frame)) continue else: estimatedCameraFile = None estimatedCameraPath = None #estimatedCameraFile = None #estimatedCameraPath = None # Check output file inputPath = os.path.join(imageFolder, imageFile) orthoPath = os.path.join(orthoFolder, orthoFile) outputCamFile = os.path.join(cameraFolder, icebridge_common.getCameraFileName(imageFile)) outputFiles.append(outputCamFile) if os.path.exists(outputCamFile): logger.info("File exists, skipping: " + outputCamFile) os.system("rm -f " + outputCamFile + "*-log-*") # wipe logs continue # Determine which input camera file will be used for this frame if inputCalCamera == "": inputCamFile = getCalibrationFileForFrame(cameraLookupFile, inputCalFolder, frame, yyyymmdd, site, logger) else: # This logic will force to use a given camera rather than # looking it up. This is not the usual way of doing things. inputCamFile = inputCalCamera if not os.path.exists(inputCalCamera): raise Exception("Could not find: " + inputCalCamera) orthoArgs = (inputPath, orthoPath, inputCamFile, estimatedCameraPath, outputCamFile, refDemPath, simpleCameras, numThreads) if numProcesses > 1: # Add ortho2pinhole command to the task pool taskHandles.append(pool.apply_async(cameraFromOrthoWrapper, orthoArgs)) else: # Single process, more logging info this way cameraFromOrthoWrapper(*orthoArgs) # Wait for all the tasks to complete logger.info('Finished adding ' + str(len(taskHandles)) + ' tasks to the pool.') icebridge_common.waitForTaskCompletionOrKeypress(taskHandles, logger, interactive=False, quitKey='q') # All tasks should be finished icebridge_common.stopTaskPool(pool) logger.info('Finished ortho processing.') # Run a check to see if we got all the output files for f in outputFiles: if not os.path.exists(f): return False return True
def convertJpegs(jpegFolder, imageFolder, startFrame, stopFrame, skipValidate, cameraMounting, logger): '''Convert jpeg images from RGB to single channel. Returns false if any files failed.''' badFiles = False logger.info('Converting input images to grayscale...') os.system('mkdir -p ' + imageFolder) # Loop through all the input images jpegIndexPath = icebridge_common.csvIndexFile(jpegFolder) if not os.path.exists(jpegIndexPath): raise Exception("Error: Missing jpeg index file: " + jpegIndexPath + ".") (jpegFrameDict, jpegUrlDict) = icebridge_common.readIndexFile(jpegIndexPath, prependFolder = True) # Need the orthos to get the timestamp orthoFolder = icebridge_common.getOrthoFolder(os.path.dirname(jpegFolder)) orthoIndexPath = icebridge_common.csvIndexFile(orthoFolder) if not os.path.exists(orthoIndexPath): raise Exception("Error: Missing ortho index file: " + orthoIndexPath + ".") (orthoFrameDict, orthoUrlDict) = icebridge_common.readIndexFile(orthoIndexPath, prependFolder = True) if not skipValidate: validFilesList = icebridge_common.validFilesList(os.path.dirname(jpegFolder), startFrame, stopFrame) validFilesSet = set() validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) numInitialValidFiles = len(validFilesSet) # Fast check for missing images. This is fragile, as maybe it gets # the wrong file with a similar name, but an honest check is very slow. imageFiles = icebridge_common.getTifs(imageFolder, prependFolder = True) imageFrameDict = {} for imageFile in imageFiles: frame = icebridge_common.getFrameNumberFromFilename(imageFile) if frame < startFrame or frame > stopFrame: continue imageFrameDict[frame] = imageFile for frame in sorted(jpegFrameDict.keys()): inputPath = jpegFrameDict[frame] # Only deal with frames in range if not ( (frame >= startFrame) and (frame <= stopFrame) ): continue if frame in imageFrameDict.keys() and skipValidate: # Fast, hackish check continue if frame not in orthoFrameDict: logger.info("Error: Could not find ortho image for jpeg frame: " + str(frame)) # Don't want to throw here. Just ignore the missing ortho continue # Make sure the timestamp and frame number are in the output file name try: outputPath = icebridge_common.jpegToImageFile(inputPath, orthoFrameDict[frame]) except Exception as e: logger.info(str(e)) logger.info("Removing bad file: " + inputPath) os.system('rm -f ' + inputPath) # will not throw badFiles = True continue # Skip existing valid files if skipValidate: if os.path.exists(outputPath): logger.info("File exists, skipping: " + outputPath) continue else: if outputPath in validFilesSet and os.path.exists(outputPath): #logger.info('Previously validated: ' + outputPath) # very verbose validFilesSet.add(inputPath) # Must have this continue if icebridge_common.isValidImage(outputPath): #logger.info("File exists and is valid, skipping: " + outputPath) # verbose if not skipValidate: # Mark both the input and the output as validated validFilesSet.add(inputPath) validFilesSet.add(outputPath) continue # Use ImageMagick tool to convert from RGB to grayscale # - Some image orientations are rotated to make stereo processing easier. rotateString = '' if cameraMounting == 2: # Flight direction towards top of image rotateString = '-rotate 90 ' if cameraMounting == 3: # Flight direction towards bottom of image rotateString = '-rotate -90 ' cmd = ('%s %s -colorspace Gray %s%s') % \ (asp_system_utils.which('convert'), inputPath, rotateString, outputPath) logger.info(cmd) # Run command and fetch its output p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) output, error = p.communicate() if p.returncode != 0: badFiles = True logger.error("Command failed.") logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not os.path.exists(outputPath): badFiles = True logger.error('Failed to convert jpeg file: ' + inputPath) logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw # Check for corrupted files if error is not None: output += error m = re.match("^.*?premature\s+end", output, re.IGNORECASE|re.MULTILINE|re.DOTALL) if m: badFiles = True logger.error("Wiping bad files: " + inputPath + " and " + outputPath + '\n' + output) os.system('rm -f ' + inputPath) # will not throw os.system('rm -f ' + outputPath) # will not throw if not skipValidate: # Write to disk the list of validated files, but only if new # validations happened. First re-read that list, in case a # different process modified it in the meantime, such as if two # managers are running at the same time. numFinalValidFiles = len(validFilesSet) if numInitialValidFiles != numFinalValidFiles: validFilesSet = icebridge_common.updateValidFilesListFromDisk(validFilesList, validFilesSet) icebridge_common.writeValidFilesList(validFilesList, validFilesSet) if badFiles: logger.error("Converstion of JPEGs failed. If any files were corrupted, " + "they were removed, and need to be re-fetched.") return (not badFiles)