def removeCompressedOutputs(dataSetName, dryRun=False): """Removes the large local output files in a completed run""" # Make sure the data set is complete before removing any files ds = DataSet(dataSetName) if not ds.isComplete(): raise Exception('Attempting to clear compressed on incomplete data set: ' + dataSetName) localFolder = ds.getLocalFolder() compressedFilePath = os.path.join(localFolder, 'results/output-CompressedOutputs.tar') if not os.path.exists(compressedFilePath): raise Exception('Attempting to clear compressed but output TAR file is missing!') # Each of these files will be deleted filesToDelete = ['output-Colormap.tif', 'output-Confidence.tif', 'output-DEM.tif', 'output-Hillshade.tif', 'output-MapProjLeft.tif', 'output-MapProjRight.tif', 'output-MapProjLeftUint8.tif', # Files after here are not compressed but can 'output-MapProjRightUint8.tif', # quickly be regenerated from compressed files. 'output-IntersectionErrorX.tif', 'output-IntersectionErrorY.tif', 'output-IntersectionErrorZ.tif'] for f in filesToDelete: # For each file in the list above inputPath = os.path.join(localFolder, 'results/' + f) #print 'Deleting file ' + inputPath if dryRun: # Print output action print '- rm ' + inputPath else: # Actually delete the file IrgFileFunctions.removeIfExists(inputPath)
def flagIncompleteDataSets(dryRun=False): """Flags incomplete data sets for repeat processing""" if not os.path.exists(INCOMPLETE_STATUS_PATH): raise Exception('Incomplete file list not present, run --check-local to generate it.') # Loop through all data sets in the incomplete folder incompleteFile = open(INCOMPLETE_STATUS_PATH, 'r') for line in incompleteFile: entries = line.split(' ') dataSetName = entries[0] # Get the status of this data set ds = DataSet(dataSetName) # Check that it is actually incomplete if ds.status != DataSet.INCOMPLETE: print 'WARNING: non-incomplete set found in incomplete file:' print line continue # For incomplete data sets, remove the downloadLog.txt file # - This will cause the auto-processor code to run the data set again. flagFile = os.path.join(ds.getLocalFolder(), 'downloadLog.txt') if dryRun: print '- Remove file ' + flagFile else: # Actually removet the file IrgFileFunctions.removeIfExists(flagFile) # Need to ru-run the update function to update this file incompleteFile.close()
def prepareCtxImage(inputPath, outputFolder, keep): """Prepare a single CTX image for processing""" # Set up paths cubPath = IrgFileFunctions.replaceExtensionAndFolder( inputPath, outputFolder, '.cub') calPath = IrgFileFunctions.replaceExtensionAndFolder( inputPath, outputFolder, '.cal.cub') # Convert to ISIS format if not os.path.exists(cubPath): cmd = 'mroctx2isis from=' + inputPath + ' to=' + cubPath os.system(cmd) # Init Spice data cmd = 'spiceinit from=' + cubPath os.system(cmd) # Apply image correction if not os.path.exists(calPath): cmd = 'ctxcal from=' + cubPath + ' to=' + calPath os.system(cmd) #you can also optionally run ctxevenodd on the cal.cub files, if needed if not keep: os.remove(cubPath) return calPath
def prepareCtxImage(inputPath, outputFolder, keep): """Prepare a single CTX image for processing""" # Set up paths cubPath = IrgFileFunctions.replaceExtensionAndFolder(inputPath, outputFolder, '.cub') calPath = IrgFileFunctions.replaceExtensionAndFolder(inputPath, outputFolder, '.cal.cub') # Convert to ISIS format if not os.path.exists(cubPath): cmd = 'mroctx2isis from=' + inputPath + ' to=' + cubPath os.system(cmd) # Init Spice data cmd = 'spiceinit from=' + cubPath os.system(cmd) # Apply image correction if not os.path.exists(calPath): cmd = 'ctxcal from='+cubPath+' to='+calPath os.system(cmd) #you can also optionally run ctxevenodd on the cal.cub files, if needed if not keep: os.remove(cubPath) return calPath
def makeSpkSetupFile(leapSecondFilePath, outputPath): """Creates the required mkspk setup file if it does not already exist""" # If the file already exists, delete it and rewrite it. IrgFileFunctions.removeIfExists(outputPath) # print 'Generating LRONAC compatible .pvl file ' + halfResFilePath f = open(outputPath, 'w') f.write("\\begindata\n") f.write("INPUT_DATA_TYPE = 'STATES'\n") f.write("OUTPUT_SPK_TYPE = 13\n") f.write("OBJECT_ID = -85\n") # LRO f.write("CENTER_ID = 301\n") # Moon f.write("REF_FRAME_NAME = 'J2000'\n") f.write("PRODUCER_ID = 'Lronac Pipeline'\n") f.write("DATA_ORDER = 'epoch x y z vx vy vz'\n") f.write("DATA_DELIMITER = ','\n") f.write("LEAPSECONDS_FILE = '" + leapSecondFilePath + "'\n") f.write("LINES_PER_RECORD = 1\n") f.write("TIME_WRAPPER = '# ETSECONDS'\n") #f.write("EPOCH_STR_LENGTH = 16\n") f.write("INPUT_DATA_UNITS = ('ANGLES=DEGREES' 'DISTANCES=km')\n") f.write("POLYNOM_DEGREE = 11\n") f.write("SEGMENT_ID = 'SPK_STATES_13'\n") # f.write("INPUT_DATA_FILE = 'spkDataFile.txt'") # f.write("OUTPUT_SPK_FILE = '/home/smcmich1/testSpkFile.bsp'") f.write("\\begintext\n") f.close()
def robustLronacPcAlignCall(demPointsPath, lolaPath, outputPrefix): # Use pc-align to compare points to LOLA DEM, compute rotation and offset # The max-displacement threshold will be adjusted until we are either using a certain number of LOLA points # or until we are using a certain percentage of the input points. MIN_NUM_LOLA_POINTS = 4000 MIN_LOLA_PERCENTAGE = 0.005 STARTING_DISPLACEMENT = 50 # --max-displacement starts at this value.. DISPLACEMENT_INCREMENT = 50 # Increments by this value... MAX_MAX_DISPLACEMENT = 1800 # And we quit when it hits this value. # Determine the number of points we want numLolaPoints = IrgFileFunctions.getFileLineCount(lolaPath) - 1 currentMaxDisplacement = STARTING_DISPLACEMENT minNumPointsToUse = min(MIN_NUM_LOLA_POINTS, int(MIN_LOLA_PERCENTAGE*float(numLolaPoints))) print 'Using pc_align to compute a transform between intersected points and LOLA points...' print 'Starting pc_align, looking for ' + str(minNumPointsToUse) + ' lola point matches.' transformPath = outputPrefix + '-inverse-transform.txt' endErrorPath = outputPrefix + '-end_errors.csv' while(True): cmd = ('pc_align --highest-accuracy --max-displacement ' + str(currentMaxDisplacement) + ' --datum D_MOON ' + '--save-inv-transformed-reference-points ' + demPointsPath + ' ' + lolaPath + ' -o ' + outputPrefix + ' --compute-translation-only') print cmd os.system(cmd) if not os.path.exists(endErrorPath): numLolaPointsUsed = 0 # Failed to produce any output, maybe raising the error cutoff will help? else: numLolaPointsUsed = IrgFileFunctions.getFileLineCount(endErrorPath) - 1 if (numLolaPointsUsed >= minNumPointsToUse): break # Success! elif (currentMaxDisplacement >= MAX_MAX_DISPLACEMENT): # Hit the maximum max limit! raise Exception('Error! Unable to find a good value for max-displacement in pc_align. Wanted ' + str(minNumPointsToUse) + ' points, only found ' + str(numLolaPointsUsed)) else: # Try again with a higher max limit print ('Trying pc_align again, only got ' + str(numLolaPointsUsed) + ' lola point matches with value ' + str(currentMaxDisplacement)) currentMaxDisplacement = currentMaxDisplacement + DISPLACEMENT_INCREMENT if not os.path.exists(transformPath): raise Exception('pc_align call failed!') return True
def correctAndCropImage(inputPath, labelPath, outputPrefix, areaBB=None): '''Can crop and restore missing metadata.''' # At least one image was too tall and had to be split in half for this to work inputIsJp2 = (inputPath[-4:] == ".JP2") if inputIsJp2: # Call the JP2 function if areaBB: areaString = getChunkAreaString(areaBB) else: areaString = None return jp2ToImgAndBack(inputPath, labelPath, outputPrefix, areaString) else: # Input is a GeoTIFF outputPath = outputPrefix + '.TIF' # This is simpler, we just crop using gdal_translat cmd = 'gdal_translate -of GTiff ' + inputPath +''+ outputPath if areaBB: # Handle crop bbString = ' -srcwin %d %d %d %d ' % areaBB cmd += bbString print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(outputPath): raise Exception('Could not crop TIFF file: ' + inputPath) return outputPath
def runInGnuParallel( numParallelProcesses, commandString, argumentFilePath, parallelArgs=[], nodeListPath=None, verbose=False ): """Use GNU Parallel to spread task across multiple computers and processes""" # Make sure GNU parallel is installed if not IrgFileFunctions.checkIfToolExists("parallel"): raise Exception("Need GNU Parallel to distribute the jobs.") # Use GNU parallel with given number of processes. # - Let output be interspersed, read input series from file cmd = ["parallel", "-u", "-a", argumentFilePath] # Add number of processes if specified (default is one job per CPU core) if numParallelProcesses is not None: cmd += ["-P", str(numParallelProcesses)] # Add list of nodes as argument to the parallel tool if it is available if nodeListPath is not None: cmd += ["--sshloginfile", nodeListPath] # Append any additional arguments to parallel cmd += parallelArgs # Append the actual command we want to call to the GNU Parallel call cmd += [commandString] if verbose: # Echo the command line call we are about to make print(" ".join(cmd)) returnCode = subprocess.call(cmd) return returnCode
def writeColorMapInfo(colormapPath, lutFilePath, demPath, outputPath): """Writes a file containing the color map information""" colormapPercentiles = [] numColorSteps = IrgFileFunctions.getFileLineCount(lutFilePath) for i in range(0,numColorSteps): # This loop generates the percentage values from the color profile we are using colormapPercentiles.append(i / float(numColorSteps-1)) # Get the min and max elevation of the DEM elevationBounds = IrgGeoFunctions.getImageStats(demPath) # Get elevation values for each percentile colormapPercentileValues = IrgMathFunctions.getPercentileValues(elevationBounds[0][0], elevationBounds[0][1], colormapPercentiles) # Now write out a version of the LUT file with values added inputFile = open(lutFilePath, 'r') outputFile = open(outputPath, 'w') # Add a header line to the output file outputFile.write('Percent of range, R, G, B, Elevation (meters above datum)\n') # Write a copy of the input file with the elevation values appended to each line for (line, value) in zip(inputFile, colormapPercentileValues): newLine = line[:-1] + ', ' + str(round(value,2)) + '\n' outputFile.write(newLine) inputFile.close() outputFile.close()
def runInGnuParallel(numParallelProcesses, commandString, argumentFilePath, parallelArgs=[], nodeListPath=None, verbose=False): """Use GNU Parallel to spread task across multiple computers and processes""" # Make sure GNU parallel is installed if not IrgFileFunctions.checkIfToolExists('parallel'): raise Exception('Need GNU Parallel to distribute the jobs.') # Use GNU parallel with given number of processes. # - Let output be interspersed, read input series from file cmd = ['parallel', '-u', '-a', argumentFilePath] # Add number of processes if specified (default is one job per CPU core) if numParallelProcesses is not None: cmd += ['-P', str(numParallelProcesses)] # Add list of nodes as argument to the parallel tool if it is available if nodeListPath is not None: cmd += ['--sshloginfile', nodeListPath] # Append any additional arguments to parallel cmd += parallelArgs # Append the actual command we want to call to the GNU Parallel call cmd += [commandString] if verbose: # Echo the command line call we are about to make print(" ".join(cmd)) returnCode = subprocess.call(cmd) return returnCode
def correctAndCropImage(inputPath, labelPath, outputPrefix, areaBB=None): '''Can crop and restore missing metadata.''' # At least one image was too tall and had to be split in half for this to work inputIsJp2 = (inputPath[-4:] == ".JP2") if inputIsJp2: # Call the JP2 function if areaBB: areaString = getChunkAreaString(areaBB) else: areaString = None return jp2ToImgAndBack(inputPath, labelPath, outputPrefix, areaString) else: # Input is a GeoTIFF outputPath = outputPrefix + '.TIF' # This is simpler, we just crop using gdal_translat cmd = 'gdal_translate -of GTiff ' + inputPath + '' + outputPath if areaBB: # Handle crop bbString = ' -srcwin %d %d %d %d ' % areaBB cmd += bbString print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(outputPath): raise Exception('Could not crop TIFF file: ' + inputPath) return outputPath
def jp2ToImgAndBack(jp2Path, labelPath, outputPrefix, areaString=None): '''Convert a Jp2 to an IMG and back. Can crop and restore missing metadata.''' # Convert from JP2 to IMG, possibly adding in missing metadata and cropping. tempImgPath = outputPrefix + '.IMG' cmd = ( '/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/JP2_to_PDS -Force -Input ' + jp2Path + ' -Output ' + tempImgPath + ' -LAbel ' + labelPath) if areaString: cmd += ' -Area ' + areaString print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(tempImgPath): raise Exception('Failed to convert chunk to IMG file: ' + jp2Path) # Convert back to JP2 format outputPath = outputPrefix + '.JP2' cmd = ( '/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/PDS_to_JP2 -Force -Geotiff -Input ' + tempImgPath + ' -Output ' + outputPath) print cmd os.system(cmd) # If converting back to JP2 failed, try converting to TIF instead if not IrgFileFunctions.fileIsNonZero(outputPath): #raise Exception('Failed to convert chunk to JP2 file: ' + outputPath) print 'WARNING: Failed to convert IMG file to JP2 file! Trying to convert to TIF instead...' outputPath = outputPrefix + '.TIF' cmd = 'gdal_translate -of GTiff ' + tempImgPath + ' ' + outputPath print cmd os.system(cmd) # If we still failed, return an exception if not IrgFileFunctions.fileIsNonZero(outputPath): raise Exception('Could not convert IMG file to anything! ' + tempImgPath) # Successfully created an output file os.remove(tempImgPath) return outputPath
def jp2ToImgAndBack(jp2Path, labelPath, outputPrefix, areaString=None): '''Convert a Jp2 to an IMG and back. Can crop and restore missing metadata.''' # Convert from JP2 to IMG, possibly adding in missing metadata and cropping. tempImgPath = outputPrefix + '.IMG' cmd = ('/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/JP2_to_PDS -Force -Input ' + jp2Path + ' -Output ' + tempImgPath + ' -LAbel ' + labelPath) if areaString: cmd += ' -Area ' + areaString print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(tempImgPath): raise Exception('Failed to convert chunk to IMG file: ' + jp2Path) # Convert back to JP2 format outputPath = outputPrefix + '.JP2' cmd = ('/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/PDS_to_JP2 -Force -Geotiff -Input ' + tempImgPath + ' -Output ' + outputPath) print cmd os.system(cmd) # If converting back to JP2 failed, try converting to TIF instead if not IrgFileFunctions.fileIsNonZero(outputPath): #raise Exception('Failed to convert chunk to JP2 file: ' + outputPath) print 'WARNING: Failed to convert IMG file to JP2 file! Trying to convert to TIF instead...' outputPath = outputPrefix + '.TIF' cmd = 'gdal_translate -of GTiff ' + tempImgPath +' '+ outputPath print cmd os.system(cmd) # If we still failed, return an exception if not IrgFileFunctions.fileIsNonZero(outputPath): raise Exception('Could not convert IMG file to anything! ' + tempImgPath) # Successfully created an output file os.remove(tempImgPath) return outputPath
outputPathStereo, stereoMosaicWorkDir, carry)) print 'Starting mosaic call threads' leftThread.start() rightThread.start() print 'Waiting for mosaic threads to complete...' leftThread.join() rightThread.join() mosaicTime = time.time() logging.info('Mosaics finished in %f seconds', mosaicTime - noprojTime) # Clean up temporary files if not options.keep: print 'Deleting temporary files' IrgFileFunctions.removeIfExists(leftCorrectedPath) IrgFileFunctions.removeIfExists(rightCorrectedPath) IrgFileFunctions.removeIfExists(leftStereoCorrectedPath) IrgFileFunctions.removeIfExists(rightStereoCorrectedPath) IrgFileFunctions.removeIfExists(pvlPath) IrgFileFunctions.removeIfExists(leftNoprojPath) IrgFileFunctions.removeIfExists(rightNoprojPath) IrgFileFunctions.removeIfExists(leftStereoNoprojPath) IrgFileFunctions.removeIfExists(rightStereoNoprojPath) #IrgFileFunctions.removeFolderIfExists(mainMosaicWorkDir) #IrgFileFunctions.removeFolderIfExists(stereoMosaicWorkDir) #if (hadToCreateTempFolder): # IrgFileFunctions.removeFolderIfExists(tempFolder) ## Remove all the .kml files #fileList = [ f for f in os.listdir(tempFolder) if f.endswith(".kml") ]
def archiveDataSet(dataSetName, deleteLocalFiles=False, dryRun=False): """Archives a data set to long term storage on Lou""" # The local files to be archived are already in tar files # Make sure the data set is complete before archiving it ds = DataSet(dataSetName) if not ds.isComplete(): raise Exception('Attempting to archive incomplete data set: ' + dataSetName) localFolder = ds.getLocalFolder() resultsFolder = os.path.join(localFolder, 'results/') # In order to archive the file it just has to be moved over to the Lou filesystem. # - Tape archiving is performed automatically when needed. # - All tar files are stored in the same directory filesToArchive = ['output-CompressedOutputs.tar'] for f in filesToArchive: # Get storage name to use archiveName = 'NAC_DTM_' + dataSetName + '_CompressedOutputs.tar' inputPath = os.path.join(resultsFolder, f) outputPath = os.path.join(LOU_STORAGE_PATH, archiveName) # Move the file over to Lou #cmd = ['shiftc', '--wait', '--verify', inputPath, outputPath] cmd = 'shiftc --wait --verify ' + inputPath + ' ' + outputPath print cmd if not dryRun: # Actually transfer the file os.system(cmd) #p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) #textOutput, err = p.communicate() #print textOutput #print 'ppp' #print err #print textOutput.find('done') #if not ('done' in textOutput): # raise Exception('Error transferring file!') # Verify that the file was actually transferred resp = subprocess.call(['ssh', LOU_SSH_TARGET, 'test -e ' + pipes.quote(LOU_LOCAL_STORAGE_PATH)]) if resp == 0: print 'shiftc file transfer confirmed.' else: raise Exception('Failed to transfer file using shiftc!') if deleteLocalFiles: # Remove almost all files # Keep some result files and move them to the main data set directory filesToKeep = ['output-LOLA_diff_stats.txt', 'output-Log.txt', 'output-PcAlignLog.txt', 'output-CompressedInputs.tar.bz2', 'output-CompressedDiagnostics.tar.bz2'] for f in filesToKeep: inputPath = os.path.join(resultsFolder, f) outputPath = os.path.join(localFolder, f) print 'mv ' + inputPath + ' ' + outputPath if not dryRun: # Actually move the files shutil.move(inputPath, outputPath) # Now that we have saved a few files, delete everything else cmdRmImg = 'rm ' + localFolder + '/*.IMG' rdrPath = os.path.join(localFolder, 'lolaRdrPoints.csv') prtPath = os.path.join(localFolder, 'print.prt') #logPath = os.path.join(localFolder, 'stdOutputLog.txt') workDir = os.path.join(localFolder, 'workDir') resultsDir = os.path.join(localFolder, 'results') print cmdRmImg print 'rm ' + rdrPath print 'rm ' + prtPath #print 'rm ' + logPath print 'rm -rf ' + workDir print 'rm -rf ' + resultsDir if not dryRun: # Actually delete the files os.system(cmdRmImg) IrgFileFunctions.removeIfExists(rdrPath) IrgFileFunctions.removeIfExists(prtPath) #IrgFileFunctions.removeIfExists(logPath) IrgFileFunctions.removeFolderIfExists(workDir) IrgFileFunctions.removeFolderIfExists(resultsDir)
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir): '''Retrieves a remote file and prepares it for upload''' #print 'Uploading file ' + setName if subtype != 'DEM': # Handles RED and COLOR images # The label file URL is the same as the image but with a different extension remoteLabelURL = getLabelPathFromImagePath(remoteURL) localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) localLabelPath = os.path.join(workDir, os.path.basename(remoteLabelURL)) # Retrieve the header file if not os.path.exists(localLabelPath): # Try to get the label locally! pdsStart = remoteLabelURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteLabelURL[pdsStart:]) print localPdsPath if os.path.exists(localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath +' '+ localLabelPath else: # Download the image cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath print cmd os.system(cmd) # Retrieve the image file if not os.path.exists(localFilePath): # Need to get it from somewhere # Try to get the image locally! pdsStart = remoteURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:]) if os.path.exists(localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath +' '+ localFilePath else: # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(localFilePath): if not IrgFileFunctions.fileIsNonZero(localLabelPath): print 'Could not find label or image file, DELETING DATA RECORD!' common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, setName) raise Exception('Unable to download from URL: ' + remoteURL) # Check if there is geo data in the JP2 file jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath) # If there is no label file, try to generate an artificial one. fakeLabelFile = False if not IrgFileFunctions.fileIsNonZero(localLabelPath): #raise Exception('Unable to download from URL: ' + remoteLabelURL) print 'WARNING: Unable to download from URL: ' + remoteLabelURL if not jp2HasGeoData: raise Exception('No geo data in JP2 and no Label file! Cannot handle this!') print 'Generating a fake LBL file to proceed...' localLabelPath = writeFakeLabelFromJp2(localFilePath) fakeLabelFile = True # At this point we always have a label file but it may be fake # Some images are missing geo information but we can add it from the label file localImageIsTiff = False localImagePath = localFilePath if not jp2HasGeoData: print 'Correcting JP2 file with no geo information!!!!' # Correct the local file, then remove the old (bad) file outputPrefix = localFilePath[0:-4] localImagePath = correctAndCropImage(localFilePath, localLabelPath, outputPrefix) print 'Generated ' + localImagePath if localFilePath != localImagePath: print 'Deleting JP2 file without metadata!' os.remove(localFilePath) localImageIsTiff = (localImagePath[-4:] == ".TIF") if not localImageIsTiff: # Call code to fix the header information in the JP2 file! cmd = FIX_JP2_TOOL +' '+ localImagePath print cmd os.system(cmd) # Check the projection type projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath) (width, height) = IrgIsisFunctions.getImageSize(localImagePath) if (projType == 'POLAR STEREOGRAPHIC') and False: #(width < MAX_POLAR_UPLOAD_WIDTH): # Google has trouble digesting these files so handle them differently. #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!') print 'Special handling for POLAR STEROGRAPHIC image!' if fakeLabelFile: print 'Cannot reprocess polar image without a label file!' print 'All we can do is upload the file and hope for the best.' # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # Compute how many chunks are needed for this image numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE) # Determine which chunk this DB entry is for chunkNum = getChunkNum(setName) print 'This is chunk number ' + str(chunkNum) if chunkNum >= numChunks: # Check for chunk number error raise Exception('Illegal chunk number: ' + setName) # If this is the main DB entry, we need to make sure the other DB entries exist! if chunkNum == 0: # Go ahead and try to add each chunk, the call will only go through if it does not already exist. for i in range(1,numChunks): chunkSetName = makeChunkSetName(setName, i) print 'Add chunk set name to DB: ' + chunkSetName common.addDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, chunkSetName, remoteURL) raise Exception('DEBUG') # Now actually generate the desired chunk # - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it. fileBasePath = os.path.splitext(localImagePath)[0] localImgPath = fileBasePath + '.IMG' localChunkPrefix = fileBasePath + '_' + str(chunkNum) chunkBB = getChunkBoundingBox(width, height, chunkNum) localChunkPath = correctAndCropImage(localImagePath, localLabelPath, localChunkPrefix, chunkBB) # Just use the same label file, we don't care if the DB has per-chunk boundaries. return [localChunkPath, localLabelPath] else: # A normal, non-polar file. # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # TODO: Handle POLAR DEMS else: # Handle DEMs # For DEMs there is no label file localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) if not os.path.exists(localFilePath): # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(localFilePath): # Make sure we got the file raise Exception('Unable to download from URL: ' + remoteURL) # Generate a header file from the IMG file localLabelPath = localFilePath[:-4] + '.LBL' cmd = 'head -n 90 ' + localFilePath +' > '+ localLabelPath print cmd os.system(cmd) # Check if this is a polar stereographic image isPolar = False f = open(localLabelPath) for line in f: if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC" in line): isPolar = True print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY' break #os.remove(localFilePath) #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!') f.close() # Convert from IMG to TIF tiffFilePath = localFilePath[:-4] + '.TIF' if not os.path.exists(tiffFilePath): cmd = 'gdal_translate -of GTiff ' + localFilePath +' '+ tiffFilePath print cmd os.system(cmd) if not isPolar: # Only EQC files need to be corrected # Correct projected coordinates problems cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath print cmd os.system(cmd) os.remove(localFilePath) # Clean up source image return [tiffFilePath, localLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir): '''Retrieves a remote file and prepares it for upload''' #print 'Uploading file ' + setName # Images with a center over this latitude will use polar stereographic projection # instead of simple cylindrical projection. HIGH_LATITUDE_CUTOFF = 65 # Degrees asuImagePath = os.path.join(workDir, setName + '_noGeo.jp2') # Map projected image from ASU asuLabelPath = os.path.join(workDir, setName + '_noGeo.lbl') # Label from ASU edrPath = os.path.join(workDir, setName + '.IMG') # Raw image from PDS timePath = os.path.join( workDir, setName + '.time') # Contains only the file capture time string #cubPath = os.path.join(workDir, setName + '.cub') # Output of mroctx2isis #calPath = os.path.join(workDir, setName + '.cal.cub') # Output of ctxcal mapPath = os.path.join(workDir, setName + '.map.cub') # Output of cam2map mapLabelPath = os.path.join(workDir, setName + '.map.pvl') # Specify projection to cam2map # Generate the remote URLs from the data prefix and volume stored in these parameters asuImageUrl, asuLabelUrl, edrUrl = generatePdsPath(setName, subtype) if True: # Map project the EDR ourselves <-- This takes way too long! print 'Projecting the EDR image using ISIS...' localFilePath = os.path.join(workDir, setName + '.tif') # The output file we will upload # Check if this is a "flat" calibration image badImage = checkForBadFile(edrUrl) if badImage: raise Exception('TODO: Remove bad images from the DB!') if not os.path.exists(edrPath): # Download the EDR file cmd = 'wget ' + edrUrl + ' -O ' + edrPath print cmd os.system(cmd) # Extract the image capture time from the .IMG file if not os.path.exists(timePath): timeString = getCreationTimeHelper(edrPath) f = open(timePath, 'w') f.write(timeString) f.close() # Convert and apply calibration to the CTX file calPath = IrgIsisFunctions.prepareCtxImage(edrPath, workDir, False) ## Find out the center latitude of the file and determine if it is high latitude centerLat = IrgIsisFunctions.getCubeCenterLatitude(calPath, workDir) print centerLat highLat = abs(float(centerLat)) > HIGH_LATITUDE_CUTOFF if True: #not os.path.exists(mapLabelPath): # Generate the map label file generateDefaultMappingPvl(mapLabelPath, highLat) if True: #not os.path.exists(mapPath): # Generate the map projected file cmd = [ 'timeout', '6h', 'cam2map', 'matchmap=', 'False', 'from=', calPath, 'to=', mapPath, 'map=', mapLabelPath ] print cmd #os.system(cmd) p = subprocess.Popen(cmd) p.communicate() if (p.returncode != 0): raise Exception( 'Error or timeout running cam2map, returnCode = ' + str(p.returncode)) if True: #not os.path.exists(localFilePath): # Generate the final image to upload cmd = 'gdal_translate -of GTiff ' + mapPath + ' ' + localFilePath print cmd os.system(cmd) # Clean up intermediate files os.remove(mapLabelPath) os.remove(edrPath) os.remove(calPath) os.remove(mapPath) # Two local files are left around, the first should be uploaded. return [localFilePath, timePath] else: # Use the map projected image from the ASU web site print 'Using ASU projected image...' localFilePath = os.path.join(workDir, setName + '.jp2') # The output file we will upload # Note: ASU seems to be missing some files! # We are using the label path in both projection cases if not os.path.exists(asuLabelPath): # Download the label file cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero( asuLabelPath): # Try the alternate label path os.remove(asuLabelPath) asuLabelUrl = asuLabelUrl.replace('.scyl.', '.ps.') asuLabelPath = asuLabelPath.replace('.scyl.', '.ps.') print 'Trying alternate label path: ' + asuLabelUrl # Download the label file cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(asuLabelPath): raise Exception('Failed to download file label at URL: ' + asuLabelUrl) # Check the projection type projType = IrgGeoFunctions.getProjectionFromIsisLabel(asuLabelPath) if projType != 'SimpleCylindrical': print 'WARNING: projType = ' + projType print 'Maps Engine may fail to ingest this file!' #os.remove(asuLabelPath) #raise Exception(projType + ' images on hold until Google fixes a bug!') if not os.path.exists(asuImagePath): # Download the image file cmd = 'wget "' + asuImageUrl + '" -O ' + asuImagePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(asuImagePath): raise Exception('Failed to download image file at URL: ' + asuImageUrl) ## Correct the ISIS header if needed #fixedAsuHeaderPath = putIsisHeaderIn180(asuLabelPath) #if (fixedAsuHeaderPath != asuLabelPath): # os.remove(asuLabelPath) # Delete replaced header if True: # This is fast now, so do it every time # Correct the file - The JP2 file from ASU needs the geo data from the label file! #cmd = 'addGeoToAsuCtxJp2.py --keep --label '+ asuLabelPath +' '+ asuImagePath +' '+ localFilePath #print cmd #os.system(cmd) # TODO: Remove unnecessary image copy here (correctedPath, sidecarPath) = addGeoDataToAsuJp2File(asuImagePath, asuLabelPath, localFilePath, keep=False) if not IrgFileFunctions.fileIsNonZero(sidecarPath): raise Exception('Script to add geo data to JP2 file failed!') # Clean up os.remove(asuImagePath) # Three local files are left around, the first should be uploaded. return [correctedPath, sidecarPath, asuLabelPath]
def functionStartupCheck(): # These calls will raise an exception if the tool is not found IrgFileFunctions.checkIfToolExists('calibrationReport.py') IrgFileFunctions.checkIfToolExists('noproj') IrgFileFunctions.checkIfToolExists('lronacjitreg') IrgFileFunctions.checkIfToolExists('handmos') IrgFileFunctions.checkIfToolExists('cubenorm') IrgFileFunctions.checkIfToolExists('stereoDoubleCalibrationProcess.py') return True
def functionStartupCheck(): # These calls will raise an exception if the tool is not found IrgFileFunctions.checkIfToolExists('lronac2refinedMosaics.py') IrgFileFunctions.checkIfToolExists('makeDemAndCompare.py') return True
fullWidth = int(projectionInfo[widthStart+7 : heightStart-1]) fullHeight = int(projectionInfo[heightStart+8 : heightEnd]) print 'Output image size is ' + str(fullWidth) + ' by ' + str(fullHeight) + ' pixels.' # For now we just break up the image into a user-specified tile size (default 1000x1000) numTilesX, numTilesY, tileList = generateTileList(fullWidth, fullHeight, options.tileSize) numTiles = numTilesX * numTilesY print 'Splitting into ' + str(numTilesX) + ' by ' + str(numTilesY) + ' tiles.' # Set up output folder outputFolder = os.path.dirname(options.outputPath) if outputFolder == '': outputFolder = './' # Handle calls in same directory outputName = os.path.basename(options.outputPath) IrgFileFunctions.createFolder(outputFolder) # Make a temporary directory to store the tiles if options.workDir: tempFolder = options.workDir else: # No folder provided, create a default one tempFolder = os.path.join(outputFolder, outputName.replace('.', '_') + '_tiles/') IrgFileFunctions.createFolder(tempFolder) # Generate a text file that contains the boundaries for each tile argumentFilePath = os.path.join(tempFolder, 'argumentList.txt') argumentFile = file(argumentFilePath, 'w') for tile in tileList: argumentFile.write(str(tile[0]) + '\t' + str(tile[1]) + '\t' + str(tile[2]) + '\t' + str(tile[3]) + '\n') argumentFile.close()
def main(argsIn): print '#################################################################################' print "Running makeDemAndCompare.py" try: try: usage = "usage: makeDemAndCompare.py [--output <path>][--manual]\n " parser = optparse.OptionParser(usage=usage) inputGroup = optparse.OptionGroup(parser, 'Input Paths') inputGroup.add_option("--left", dest="leftPath", help="Path to left cube file") inputGroup.add_option("--right", dest="rightPath", help="Path to right cube file") inputGroup.add_option("--lola", dest="lolaPath", help="Path to LOLA DEM") inputGroup.add_option("--asu", dest="asuPath", help="Path to ASU DEM") inputGroup.add_option("--node-file", dest="nodeFilePath", help="Path to file containing list of available nodes") parser.add_option_group(inputGroup) # The default working directory path is kind of ugly... parser.add_option("--workDir", dest="workDir", help="Folder to store temporary files in") parser.add_option("--prefix", dest="prefix", help="Output prefix.") parser.add_option("--log-path", dest="logPath", help="Where to write the output log file.") parser.add_option("--crop", dest="cropAmount", help="Crops the output image to reduce processing time.") parser.add_option("--manual", action="callback", callback=man, help="Read the manual.") parser.add_option("--keep", action="store_true", dest="keep", help="Do not delete the temporary files.") (options, args) = parser.parse_args(argsIn) if not options.leftPath: parser.error("Need left input path") if not options.rightPath: parser.error("Need right input path") if not options.prefix: parser.error("Need output prefix") except optparse.OptionError, msg: raise Usage(msg) print "Beginning processing....." startTime = time.time() # Make sure we have all the functions we need functionStartupCheck() # Set this to true to force steps after it to activate carry = False # Set up the output folders outputFolder = os.path.dirname(options.prefix) inputBaseName = os.path.basename(options.leftPath) tempFolder = outputFolder + '/' + inputBaseName + '_stereoCalibrationTemp/' if (options.workDir): tempFolder = options.workDir if not os.path.exists(outputFolder): os.mkdir(outputFolder) hadToCreateTempFolder = not os.path.exists(tempFolder) if not os.path.exists(tempFolder): os.mkdir(tempFolder) # Set up logging if not options.logPath: options.logPath = options.prefix + '-Log.txt' logging.basicConfig(filename=options.logPath,level=logging.INFO) # Go ahead and set up all the output paths # -- Deliverables demPath = options.prefix + '-DEM.tif' intersectionErrorPath = options.prefix + '-IntersectionErr.tif' hillshadePath = options.prefix + '-Hillshade.tif' colormapPath = options.prefix + '-Colormap.tif' colormapLegendPath = options.prefix + '-ColormapLegend.csv' mapProjectLeftPath = options.prefix + '-MapProjLeft.tif' mapProjectRightPath = options.prefix + '-MapProjRight.tif' confidenceLevelPath = options.prefix + '-Confidence.tif' confidenceLegendPath = options.prefix + '-ConfidenceLegend.csv' # -- Diagnostic intersectionViewPathX = options.prefix + '-IntersectionErrorX.tif' intersectionViewPathY = options.prefix + '-IntersectionErrorY.tif' intersectionViewPathZ = options.prefix + '-IntersectionErrorZ.tif' lolaDiffStatsPath = options.prefix + '-LOLA_diff_stats.txt' lolaDiffPointsPath = options.prefix + '-LOLA_diff_points.csv' lolaAsuDiffStatsPath = options.prefix + '-ASU_LOLA_diff_stats.txt' lolaAsuDiffPointsPath = options.prefix + '-ASU_LOLA_diff_points.csv' mapProjectLeftUint8Path = options.prefix + '-MapProjLeftUint8.tif' mapProjectRightUint8Path = options.prefix + '-MapProjRightUint8.tif' # If specified, crop the inputs that will be passed into the stereo function to reduce processing time mainMosaicCroppedPath = os.path.join(tempFolder, 'mainMosaicCropped.cub') stereoMosaicCroppedPath = os.path.join(tempFolder, 'stereoMosaicCropped.cub') if options.cropAmount and (options.cropAmount > 0): # Verify input files are present if not os.path.exists(options.leftPath): raise Exception('Input file ' + options.leftPath + ' not found!') if not os.path.exists(options.rightPath): raise Exception('Input file ' + options.rightPath + ' not found!') if (not os.path.exists(mainMosaicCroppedPath)) or carry: cmd = ('crop from= ' + options.leftPath + ' to= ' + mainMosaicCroppedPath + ' nlines= ' + str(options.cropAmount))# + ' line=24200') print cmd os.system(cmd) if (not os.path.exists(stereoMosaicCroppedPath) or carry): cmd = ('crop from= ' + options.rightPath + ' to= ' + stereoMosaicCroppedPath + ' nlines= ' + str(options.cropAmount))# + ' line=24200') print cmd os.system(cmd) options.leftPath = mainMosaicCroppedPath options.rightPath = stereoMosaicCroppedPath print '\n-------------------------------------------------------------------------\n' # Call stereo to generate a point cloud from the two images # - This step takes a really long time. stereoOutputPrefix = os.path.join(tempFolder, 'stereoWorkDir/stereo') stereoOutputFolder = os.path.join(tempFolder, 'stereoWorkDir') pointCloudPath = stereoOutputPrefix + '-PC.tif' stereoOptionString = ('--corr-timeout 400 --alignment-method AffineEpipolar --subpixel-mode ' + str(SUBPIXEL_MODE) + ' ' + options.leftPath + ' ' + options.rightPath + ' --job-size-w 4096 --job-size-h 4096 ' + # Reduce number of tile files created ' ' + stereoOutputPrefix + ' --processes 10 --threads-multiprocess 4' + ' --threads-singleprocess 32 --compute-error-vector' + ' --filter-mode 1' + ' --erode-max-size 5000 --subpixel-kernel 35 35 --subpixel-max-levels 0') if (not os.path.exists(pointCloudPath) and not os.path.exists(demPath)) or carry: # Verify input files are present if not os.path.exists(options.leftPath): raise Exception('Input file ' + options.leftPath + ' not found!') if not os.path.exists(options.rightPath): raise Exception('Input file ' + options.rightPath + ' not found!') cmd = ('parallel_stereo ' + stereoOptionString) print cmd os.system(cmd) # Compute percentage of good pixels percentGood = IrgAspFunctions.getStereoGoodPixelPercentage(stereoOutputPrefix) print 'Stereo completed with good pixel percentage: ' + str(percentGood) logging.info('Final stereo completed with good pixel percentage: %s', str(percentGood)) else: print 'Stereo file ' + pointCloudPath + ' already exists, skipping stereo step.' stereoTime = time.time() logging.info('Stereo finished in %f seconds', stereoTime - startTime) # Find out the center latitude of the mosaic if os.path.exists(options.leftPath): centerLat = IrgIsisFunctions.getCubeCenterLatitude(options.leftPath, tempFolder) elif os.path.exists(demPath): # Input file has been deleted but we still have the info demInfo = IrgGeoFunctions.getImageGeoInfo(demPath, False) centerLat = demInfo['standard_parallel_1'] else: raise Exception("Can't delete the input files before creating the DEM!") # Generate a DEM if (not os.path.exists(demPath)) or carry: # Equirectangular style projection # - Latitude of true scale = center latitude = lat_ts # - Latitude of origin = 0 = lat+0 # - Longitude of projection center = Central meridian = lon+0 cmd = ('point2dem --dem-hole-fill-len 15 --remove-outliers --errorimage -o ' + options.prefix + ' ' + pointCloudPath + ' -r moon --tr ' + str(DEM_METERS_PER_PIXEL) + ' --t_srs "+proj=eqc +lat_ts=' + str(centerLat) + ' +lat_0=0 +a='+str(MOON_RADIUS)+' +b='+str(MOON_RADIUS)+' +units=m" --nodata ' + str(DEM_NODATA)) os.system(cmd) else: print 'DEM file ' + demPath + ' already exists, skipping point2dem step.' # Create a hillshade image to visualize the output if (not os.path.exists(hillshadePath)) or carry: cmd = 'hillshade ' + demPath + ' -o ' + hillshadePath print cmd os.system(cmd) else: print 'Output file ' + hillshadePath + ' already exists, skipping hillshade step.' # Create a colorized version of the hillshade # - Uses a blue-red color map from here: http://www.sandia.gov/~kmorel/documents/ColorMaps/ if (not os.path.exists(colormapPath)) or (not os.path.exists(colormapLegendPath)) or carry: # The color LUT is kept with the source code lutFilePath = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'colorProfileBlueRed.csv') # Generate the initial version of colormap colormapTempPath = options.prefix + '-ColormapTemp.tif' cmd = 'colormap ' + demPath + ' -o ' + colormapTempPath + ' -s ' + hillshadePath + ' --lut-file ' + lutFilePath print cmd os.system(cmd) # Now convert to the final output version (remove transparency layer) and remove the temp file IrgFileFunctions.stripRgbImageAlphaChannel(colormapTempPath, colormapPath) os.remove(colormapTempPath) # Generate another file storing the colormap info writeColorMapInfo(colormapPath, lutFilePath, demPath, colormapLegendPath) else: print 'Output file ' + colormapPath + ' already exists, skipping colormap step.' ## Create a 3d mesh of the point cloud #meshPath = os.path.join(outputFolder, 'mesh.ive') #meshPrefix = os.path.join(outputFolder, 'mesh') #cmd = 'point2mesh ' + pointCloudPath + ' ' + options.leftPath + ' -o ' + meshPrefix #if not os.path.exists(meshPath): # print cmd # os.system(cmd) if not options.keep: # Remove stereo folder here to cut down on file count before mapproject calls IrgFileFunctions.removeFolderIfExists(stereoOutputFolder) # Convert the intersection error to a viewable format cmdX = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 1 ' + intersectionErrorPath + ' ' + intersectionViewPathX cmdY = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 2 ' + intersectionErrorPath + ' ' + intersectionViewPathY cmdZ = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 3 ' + intersectionErrorPath + ' ' + intersectionViewPathZ if not os.path.exists(intersectionViewPathX) or carry: print cmdX os.system(cmdX) if not os.path.exists(intersectionViewPathY) or carry: print cmdY os.system(cmdY) if not os.path.exists(intersectionViewPathZ) or carry: print cmdZ os.system(cmdZ) # Generate a confidence plot from the intersection error if not os.path.exists(confidenceLevelPath): thresholdString = str(PIXEL_ACCURACY_THRESHOLDS)[1:-1].replace(',', '') cmd = ('maskFromIntersectError ' + intersectionErrorPath + ' ' + confidenceLevelPath + ' --legend ' + confidenceLegendPath + ' --scaleOutput --thresholds ' + thresholdString) print cmd os.system(cmd) hillshadeTime = time.time() logging.info('DEM and hillshade finished in %f seconds', hillshadeTime - stereoTime) # Call script to compare LOLA data with the DEM if options.lolaPath: compareDemToLola(options.lolaPath, demPath, lolaDiffStatsPath, lolaDiffPointsPath, carry) # Call script to compare LOLA data with the ASU DEM if options.asuPath: compareDemToLola(options.lolaPath, options.asuPath, lolaAsuDiffStatsPath, lolaAsuDiffPointsPath, carry) # Generate a map projected version of the left and right images # - This step is done last since it is so slow! mapProjectImage(options.leftPath, demPath, mapProjectLeftPath, MAP_PROJECT_METERS_PER_PIXEL, centerLat, options.nodeFilePath, carry) mapProjectImage(options.rightPath, demPath, mapProjectRightPath, MAP_PROJECT_METERS_PER_PIXEL, centerLat, options.nodeFilePath, carry) # Generate 8 bit versions of the mapproject files for debugging cmdLeft = 'gdal_translate -scale -ot byte ' + mapProjectLeftPath + ' ' + mapProjectLeftUint8Path cmdRight = 'gdal_translate -scale -ot byte ' + mapProjectRightPath + ' ' + mapProjectRightUint8Path if not os.path.exists(mapProjectLeftUint8Path) or carry: print cmdLeft os.system(cmdLeft) if not os.path.exists(mapProjectRightUint8Path) or carry: print cmdRight os.system(cmdRight) mapProjectTime = time.time() logging.info('Map project finished in %f seconds', mapProjectTime - hillshadeTime) # Clean up temporary files if not options.keep: print 'Removing temporary files' IrgFileFunctions.removeIfExists(mainMosaicCroppedPath) IrgFileFunctions.removeIfExists(stereoMosaicCroppedPath) #IrgFileFunctions.removeIntermediateStereoFiles(stereoOutputPrefix) # Limited clear IrgFileFunctions.removeFolderIfExists(stereoOutputFolder) # Larger clear #if (hadToCreateTempFolder): Not done since stereo output needs to be retained # IrgFileFunctions.removeFolderIfExists(tempFolder) endTime = time.time() logging.info('makeDemAndCompare.py finished in %f seconds', endTime - startTime) print "Finished in " + str(endTime - startTime) + " seconds." print '#################################################################################' return 0
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir): '''Retrieves a remote file and prepares it for upload''' #print 'Uploading file ' + setName # Images with a center over this latitude will use polar stereographic projection # instead of simple cylindrical projection. HIGH_LATITUDE_CUTOFF = 65 # Degrees asuImagePath = os.path.join(workDir, setName + '_noGeo.jp2') # Map projected image from ASU asuLabelPath = os.path.join(workDir, setName + '_noGeo.lbl') # Label from ASU edrPath = os.path.join(workDir, setName + '.IMG') # Raw image from PDS timePath = os.path.join(workDir, setName + '.time') # Contains only the file capture time string #cubPath = os.path.join(workDir, setName + '.cub') # Output of mroctx2isis #calPath = os.path.join(workDir, setName + '.cal.cub') # Output of ctxcal mapPath = os.path.join(workDir, setName + '.map.cub') # Output of cam2map mapLabelPath = os.path.join(workDir, setName + '.map.pvl') # Specify projection to cam2map # Generate the remote URLs from the data prefix and volume stored in these parameters asuImageUrl, asuLabelUrl, edrUrl = generatePdsPath(setName, subtype) if True: # Map project the EDR ourselves <-- This takes way too long! print 'Projecting the EDR image using ISIS...' localFilePath = os.path.join(workDir, setName + '.tif') # The output file we will upload # Check if this is a "flat" calibration image badImage = checkForBadFile(edrUrl) if badImage: raise Exception('TODO: Remove bad images from the DB!') if not os.path.exists(edrPath): # Download the EDR file cmd = 'wget ' + edrUrl + ' -O ' + edrPath print cmd os.system(cmd) # Extract the image capture time from the .IMG file if not os.path.exists(timePath): timeString = getCreationTimeHelper(edrPath) f = open(timePath, 'w') f.write(timeString) f.close() # Convert and apply calibration to the CTX file calPath = IrgIsisFunctions.prepareCtxImage(edrPath, workDir, False) ## Find out the center latitude of the file and determine if it is high latitude centerLat = IrgIsisFunctions.getCubeCenterLatitude(calPath, workDir) print centerLat highLat = abs(float(centerLat)) > HIGH_LATITUDE_CUTOFF if True:#not os.path.exists(mapLabelPath): # Generate the map label file generateDefaultMappingPvl(mapLabelPath, highLat) if True:#not os.path.exists(mapPath): # Generate the map projected file cmd = ['timeout', '6h', 'cam2map', 'matchmap=','False', 'from=', calPath, 'to=', mapPath, 'map=', mapLabelPath] print cmd #os.system(cmd) p = subprocess.Popen(cmd) p.communicate() if (p.returncode != 0): raise Exception('Error or timeout running cam2map, returnCode = ' + str(p.returncode)) if True: #not os.path.exists(localFilePath): # Generate the final image to upload cmd = 'gdal_translate -of GTiff ' + mapPath + ' ' + localFilePath print cmd os.system(cmd) # Clean up intermediate files os.remove(mapLabelPath) os.remove(edrPath) os.remove(calPath) os.remove(mapPath) # Two local files are left around, the first should be uploaded. return [localFilePath, timePath] else: # Use the map projected image from the ASU web site print 'Using ASU projected image...' localFilePath = os.path.join(workDir, setName + '.jp2') # The output file we will upload # Note: ASU seems to be missing some files! # We are using the label path in both projection cases if not os.path.exists(asuLabelPath): # Download the label file cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(asuLabelPath): # Try the alternate label path os.remove(asuLabelPath) asuLabelUrl = asuLabelUrl.replace( '.scyl.', '.ps.') asuLabelPath = asuLabelPath.replace('.scyl.', '.ps.') print 'Trying alternate label path: ' + asuLabelUrl # Download the label file cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(asuLabelPath): raise Exception('Failed to download file label at URL: ' + asuLabelUrl) # Check the projection type projType = IrgGeoFunctions.getProjectionFromIsisLabel(asuLabelPath) if projType != 'SimpleCylindrical': print 'WARNING: projType = ' + projType print 'Maps Engine may fail to ingest this file!' #os.remove(asuLabelPath) #raise Exception(projType + ' images on hold until Google fixes a bug!') if not os.path.exists(asuImagePath): # Download the image file cmd = 'wget "' + asuImageUrl + '" -O ' + asuImagePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(asuImagePath): raise Exception('Failed to download image file at URL: ' + asuImageUrl) ## Correct the ISIS header if needed #fixedAsuHeaderPath = putIsisHeaderIn180(asuLabelPath) #if (fixedAsuHeaderPath != asuLabelPath): # os.remove(asuLabelPath) # Delete replaced header if True: # This is fast now, so do it every time # Correct the file - The JP2 file from ASU needs the geo data from the label file! #cmd = 'addGeoToAsuCtxJp2.py --keep --label '+ asuLabelPath +' '+ asuImagePath +' '+ localFilePath #print cmd #os.system(cmd) # TODO: Remove unnecessary image copy here (correctedPath, sidecarPath) = addGeoDataToAsuJp2File(asuImagePath, asuLabelPath, localFilePath, keep=False) if not IrgFileFunctions.fileIsNonZero(sidecarPath): raise Exception('Script to add geo data to JP2 file failed!') # Clean up os.remove(asuImagePath) # Three local files are left around, the first should be uploaded. return [correctedPath, sidecarPath, asuLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir): '''Retrieves a remote file and prepares it for upload''' #print 'Uploading file ' + setName if subtype != 'DEM': # Handles RED and COLOR images # The label file URL is the same as the image but with a different extension remoteLabelURL = getLabelPathFromImagePath(remoteURL) localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) localLabelPath = os.path.join(workDir, os.path.basename(remoteLabelURL)) # Retrieve the header file if not os.path.exists(localLabelPath): # Try to get the label locally! pdsStart = remoteLabelURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteLabelURL[pdsStart:]) print localPdsPath if os.path.exists( localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath + ' ' + localLabelPath else: # Download the image cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath print cmd os.system(cmd) # Retrieve the image file if not os.path.exists(localFilePath): # Need to get it from somewhere # Try to get the image locally! pdsStart = remoteURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:]) if os.path.exists( localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath + ' ' + localFilePath else: # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(localFilePath): if not IrgFileFunctions.fileIsNonZero(localLabelPath): print 'Could not find label or image file, DELETING DATA RECORD!' common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, setName) raise Exception('Unable to download from URL: ' + remoteURL) # Check if there is geo data in the JP2 file jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath) # If there is no label file, try to generate an artificial one. fakeLabelFile = False if not IrgFileFunctions.fileIsNonZero(localLabelPath): #raise Exception('Unable to download from URL: ' + remoteLabelURL) print 'WARNING: Unable to download from URL: ' + remoteLabelURL if not jp2HasGeoData: raise Exception( 'No geo data in JP2 and no Label file! Cannot handle this!' ) print 'Generating a fake LBL file to proceed...' localLabelPath = writeFakeLabelFromJp2(localFilePath) fakeLabelFile = True # At this point we always have a label file but it may be fake # Some images are missing geo information but we can add it from the label file localImageIsTiff = False localImagePath = localFilePath if not jp2HasGeoData: print 'Correcting JP2 file with no geo information!!!!' # Correct the local file, then remove the old (bad) file outputPrefix = localFilePath[0:-4] localImagePath = correctAndCropImage(localFilePath, localLabelPath, outputPrefix) print 'Generated ' + localImagePath if localFilePath != localImagePath: print 'Deleting JP2 file without metadata!' os.remove(localFilePath) localImageIsTiff = (localImagePath[-4:] == ".TIF") if not localImageIsTiff: # Call code to fix the header information in the JP2 file! cmd = FIX_JP2_TOOL + ' ' + localImagePath print cmd os.system(cmd) # Check the projection type projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath) (width, height) = IrgIsisFunctions.getImageSize(localImagePath) if (projType == 'POLAR STEREOGRAPHIC' ) and False: #(width < MAX_POLAR_UPLOAD_WIDTH): # Google has trouble digesting these files so handle them differently. #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!') print 'Special handling for POLAR STEROGRAPHIC image!' if fakeLabelFile: print 'Cannot reprocess polar image without a label file!' print 'All we can do is upload the file and hope for the best.' # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # Compute how many chunks are needed for this image numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE) # Determine which chunk this DB entry is for chunkNum = getChunkNum(setName) print 'This is chunk number ' + str(chunkNum) if chunkNum >= numChunks: # Check for chunk number error raise Exception('Illegal chunk number: ' + setName) # If this is the main DB entry, we need to make sure the other DB entries exist! if chunkNum == 0: # Go ahead and try to add each chunk, the call will only go through if it does not already exist. for i in range(1, numChunks): chunkSetName = makeChunkSetName(setName, i) print 'Add chunk set name to DB: ' + chunkSetName common.addDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, chunkSetName, remoteURL) raise Exception('DEBUG') # Now actually generate the desired chunk # - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it. fileBasePath = os.path.splitext(localImagePath)[0] localImgPath = fileBasePath + '.IMG' localChunkPrefix = fileBasePath + '_' + str(chunkNum) chunkBB = getChunkBoundingBox(width, height, chunkNum) localChunkPath = correctAndCropImage(localImagePath, localLabelPath, localChunkPrefix, chunkBB) # Just use the same label file, we don't care if the DB has per-chunk boundaries. return [localChunkPath, localLabelPath] else: # A normal, non-polar file. # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # TODO: Handle POLAR DEMS else: # Handle DEMs # For DEMs there is no label file localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) if not os.path.exists(localFilePath): # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero( localFilePath): # Make sure we got the file raise Exception('Unable to download from URL: ' + remoteURL) # Generate a header file from the IMG file localLabelPath = localFilePath[:-4] + '.LBL' cmd = 'head -n 90 ' + localFilePath + ' > ' + localLabelPath print cmd os.system(cmd) # Check if this is a polar stereographic image isPolar = False f = open(localLabelPath) for line in f: if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC" in line): isPolar = True print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY' break #os.remove(localFilePath) #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!') f.close() # Convert from IMG to TIF tiffFilePath = localFilePath[:-4] + '.TIF' if not os.path.exists(tiffFilePath): cmd = 'gdal_translate -of GTiff ' + localFilePath + ' ' + tiffFilePath print cmd os.system(cmd) if not isPolar: # Only EQC files need to be corrected # Correct projected coordinates problems cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath print cmd os.system(cmd) os.remove(localFilePath) # Clean up source image return [tiffFilePath, localLabelPath]
def main(): print '#################################################################################' print "Running lronac2dem.py" try: try: usage = "usage: lronac2dem.py [--output <path>][--manual]\n " parser = optparse.OptionParser(usage=usage) parser.set_defaults(keep=False) inputGroup = optparse.OptionGroup(parser, 'Input Paths') inputGroup.add_option("--left", dest="leftPath", help="Path to LE .IMG file") inputGroup.add_option("--right", dest="rightPath", help="Path to RE .IMG file") inputGroup.add_option("--stereo-left", dest="stereoLeft", help="Path to LE .IMG file with overlapping view of --left file") inputGroup.add_option("--stereo-right", dest="stereoRight", help="Path to RE .IMG file with overlapping view of --right file") inputGroup.add_option("--lola", dest="lolaPath", help="Path to LOLA DEM") inputGroup.add_option("--asu", dest="asuPath", help="Path to ASU DEM") inputGroup.add_option("--node-file", dest="nodeFilePath", help="Path to list of available computing nodes") parser.add_option_group(inputGroup) # The default working directory path is kind of ugly... parser.add_option("--workDir", dest="workDir", help="Folder to store temporary files in") parser.add_option("--outputPrefix", dest="outputPrefix", help="Output prefix.") parser.add_option("--crop", dest="cropAmount", help="Crops the output image to reduce processing time.") parser.add_option("--manual", action="callback", callback=man, help="Read the manual.") parser.add_option("--keep", action="store_true", dest="keep", help="Do not delete the temporary files.") (options, args) = parser.parse_args() if not options.leftPath: parser.error("Need left input path") if not options.rightPath: parser.error("Need right input path") if not options.stereoLeft: parser.error("Need stereo left input path") if not options.stereoRight: parser.error("Need stereo right input path") if not options.lolaPath: parser.error("Need LOLA data path") if not options.outputPrefix: parser.error("Need output prefix") except optparse.OptionError, msg: raise Usage(msg) startTime = time.time() # Set up the output folders outputFolder = os.path.dirname(options.outputPrefix) inputBaseName = os.path.basename(options.leftPath) tempFolder = outputFolder + '/' + inputBaseName + '_stereoCalibrationTemp/' if (options.workDir): tempFolder = options.workDir IrgFileFunctions.createFolder(outputFolder) hadToCreateTempFolder = not os.path.exists(tempFolder) IrgFileFunctions.createFolder(tempFolder) # Set up logging logPath = options.outputPrefix + '-Log.txt' logging.basicConfig(filename=logPath,level=logging.INFO) # These are the output mosaic paths filename = os.path.splitext(options.leftPath)[0] + '.correctedMosaic.cub' mainMosaicPath = os.path.join(tempFolder, os.path.basename(filename)) filename = os.path.splitext(options.stereoLeft)[0] + '.correctedMosaic.cub' stereoMosaicPath = os.path.join(tempFolder, os.path.basename(filename)) # List of all the output files that will be created by makeDemAndCompare.py demPath = options.outputPrefix + '-DEM.tif' intersectionErrorPath = options.outputPrefix + '-IntersectionErr.tif' hillshadePath = options.outputPrefix + '-Hillshade.tif' colormapPath = options.outputPrefix + '-Colormap.tif' colormapLegendPath = options.outputPrefix + '-ColormapLegend.csv' confidencePath = options.outputPrefix + '-Confidence.tif' confidenceLegendPath = options.outputPrefix + '-ConfidenceLegend.csv' mapProjectLeftPath = options.outputPrefix + '-MapProjLeft.tif' mapProjectRightPath = options.outputPrefix + '-MapProjRight.tif' compressedInputPath = options.outputPrefix + '-CompressedInputs.tar.bz2' compressedOutputPath = options.outputPrefix + '-CompressedOutputs.tar' compressedDebugPath = options.outputPrefix + '-CompressedDiagnostics.tar.bz2' # List of .LBL text files to be created demLabelPath = options.outputPrefix + '-DEM.LBL' intersectionErrorLabelPath = options.outputPrefix + '-IntersectionErr.LBL' hillshadeLabelPath = options.outputPrefix + '-Hillshade.LBL' colormapLabelPath = options.outputPrefix + '-Colormap.LBL' confidenceLabelPath = options.outputPrefix + '-Confidence.LBL' mapProjectLeftLabelPath = options.outputPrefix + '-MapProjLeft.LBL' mapProjectRightLabelPath = options.outputPrefix + '-MapProjRight.LBL' # List of diagnostic files we want to archive # Currently specified below. Should be expanded. # Call lronac2refinedMosaics.py if outputs do not already exist if ( (not os.path.exists(mainMosaicPath) or not os.path.exists(stereoMosaicPath)) and not os.path.exists(demPath) ): refineTemp = os.path.join(tempFolder, 'refinement') cmdArgs = ['--left', options.leftPath, '--right', options.rightPath, '--stereo-left', options.stereoLeft, '--stereo-right', options.stereoRight, '--lola', options.lolaPath, '--workDir', refineTemp, '--output-folder', tempFolder, '--log-path', logPath] if options.keep: cmdArgs.append('--keep') print cmdArgs lronac2refinedMosaics.main(cmdArgs) # Copy the pc_align log file to the output folder pcAlignLogPath = os.path.join(tempFolder, 'pcAlignLog.txt') shutil.copyfile(pcAlignLogPath, options.outputPrefix + '-PcAlignLog.txt') # Check that we successfully created the output files if (not os.path.exists(mainMosaicPath) or not os.path.exists(stereoMosaicPath)): raise Exception('lronac2refinedMosaics failed to produce mosaics!') # Call makeDemAndCompare.py (This won't do much if outputs already exist) cmdArgs = ['--left', mainMosaicPath, '--right', stereoMosaicPath, '--lola', options.lolaPath, '--workDir', tempFolder, '--prefix', options.outputPrefix, '--log-path', logPath] if options.keep: cmdArgs.append('--keep') if options.asuPath: cmdArgs.append('--asu') cmdArgs.append(options.asuPath) if options.cropAmount: cmdArgs.append('--crop') cmdArgs.append(str(options.cropAmount)) if options.nodeFilePath: cmdArgs.append('--node-file') cmdArgs.append(options.nodeFilePath) print cmdArgs makeDemAndCompare.main(cmdArgs) print 'Finished image processing, setting up final output files...' logging.info('Started label file generation') # Get the data set name if (options.asuPath): # If ASU has a name for this set, use it startPos = options.asuPath.rfind('_') + 1 stopPos = options.asuPath.rfind('.') - 1 dataSetName = options.asuPath[startPos:stopPos] else: # Generate a data set in format NAC_DTM_MXXXXXX_MXXXXXXX dataSetName = IsisTools.makeDataSetName(options.leftPath, options.stereoLeft) # Get some data for the label file thisFilePath = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'lronac2dem.py') versionId = IrgFileFunctions.getLastGitTag(thisFilePath) demDescription = 'High-resolution NAC digital terrain models in GeoTIFF format. DEM is IEEE floating point TIFF, 32 bits/sample, 1 samples/pixel in single image plane configuration. The NoDATA value is -3.40282266e+038.' hillshadeDescription = 'Shaded-relief derived from NAC digital terrain models. Image is a TIFF image, 8 bits/sample, 1 samples/pixel in single image plane configuration.' colormapDescription = 'Color shaded-relief derived from NAC digital terrain models. Image is 3-channel RGB TIFF, 8 bits/sample, 3 samples/pixel in single image plane configuration.' confidenceDescription = 'Discretized visualization of intersection error in NAC digital terrain model. Image is a TIFF image, 8 bits/sample, 1 samples/pixel in single image plane configuration.' mapProjectDescription = 'High-resolution NAC orthoprojected image in GeoTIFF format. Image is IEEE floating point TIFF, 32 bits/sample, 1 samples/pixel in single image plane configuration. The NoDATA value is -3.40282266e+038.' # Copy the colormap legend file in to the colormap label file! colormapLegendText = ('\n/* Colormap legend information: */\n') colormapLegendFile = open(colormapLegendPath, 'r') for line in colormapLegendFile: colormapLegendText = colormapLegendText + '/* ' + line.strip() + ' */\n' # Copy the confidence legend file in to the confidence label file! confidenceLegendText = ('\n/* Confidence legend information: */\n') confidenceLegendFile = open(confidenceLegendPath, 'r') for line in confidenceLegendFile: confidenceLegendText = confidenceLegendText + '/* ' + line.strip() + ' */\n' # Generate label files for each of the output files writeLabelFile(demPath, demLabelPath, dataSetName, versionId, demDescription) writeLabelFile(hillshadePath, hillshadeLabelPath, dataSetName, versionId, hillshadeDescription) writeLabelFile(colormapPath, colormapLabelPath, dataSetName, versionId, colormapDescription, colormapLegendText) writeLabelFile(confidencePath, confidenceLabelPath, dataSetName, versionId, confidenceDescription, confidenceLegendText) writeLabelFile(mapProjectLeftPath, mapProjectLeftLabelPath, dataSetName, versionId, mapProjectDescription) writeLabelFile(mapProjectRightPath, mapProjectRightLabelPath, dataSetName, versionId, mapProjectDescription) logging.info('Done with label files, compressing results.') # Compress the input files to save disk space if not os.path.exists(compressedInputPath): listOfInputFiles = [options.leftPath, options.rightPath, options.stereoLeft, options.stereoRight, options.lolaPath] if options.asuPath: # Handle optional input listOfInputFiles.append(options.asuPath) IrgFileFunctions.tarFileList(listOfInputFiles, compressedInputPath, True) # Tar the output files to save file count (they are already compressed) if not os.path.exists(compressedOutputPath): listOfDeliveryFiles = [demPath, hillshadePath, colormapPath, confidencePath, mapProjectLeftPath, mapProjectRightPath, demLabelPath, hillshadeLabelPath, colormapLabelPath, confidenceLabelPath, mapProjectLeftLabelPath, mapProjectRightLabelPath] # The file names as they should appear in the tar file baseTarPath = os.path.join(outputFolder, dataSetName + '_') demTarBase = baseTarPath + 'DEM' hillTarBase = baseTarPath + 'Hillshade' colorTarBase = baseTarPath + 'Colormap' confTarBase = baseTarPath + 'Confidence' leftTarBase = baseTarPath + 'MapProjLeft' rightTarBase = baseTarPath + 'MapProjRight' listOfTarDeliveryFiles = [demTarBase+'.TIF', hillTarBase+'.TIF', colorTarBase+'.TIF', confTarBase+'.TIF', leftTarBase+'.TIF', rightTarBase+'.TIF', demTarBase+'.LBL', hillTarBase+'.LBL', colorTarBase+'.LBL', confTarBase+'.LBL', leftTarBase+'.LBL', rightTarBase+'.LBL'] # This function can handle renaming the files as they get put in the tar. IrgFileFunctions.tarFileList(listOfDeliveryFiles, compressedOutputPath, False, listOfTarDeliveryFiles) # Compress the diagnostic files to save disk space if not os.path.exists(compressedDebugPath): listOfDebugFiles = [options.outputPrefix + '-Log.txt', options.outputPrefix + '-PcAlignLog.txt', options.outputPrefix + '-LOLA_diff_points.csv', options.outputPrefix + '-LOLA_diff_points.kml',] IrgFileFunctions.tarFileList(listOfDebugFiles, compressedDebugPath, True) if not options.keep: pass #print 'Deleting temporary files' #IrgFileFunctions.removeIfExists(mainMosaicPath) #IrgFileFunctions.removeIfExists(stereoMosaicPath) endTime = time.time() logging.info('lronac2dem finished in %f seconds', endTime - startTime) print "Finished in " + str(endTime - startTime) + " seconds." print '#################################################################################' return 0
def functionStartupCheck(): # These calls will raise an exception if the tool is not found IrgFileFunctions.checkIfToolExists('lola_compare') IrgFileFunctions.checkIfToolExists('parallel_stereo') IrgFileFunctions.checkIfToolExists('parallel_mapproject.py') IrgFileFunctions.checkIfToolExists('point2dem') IrgFileFunctions.checkIfToolExists('hillshade') IrgFileFunctions.checkIfToolExists('colormap') IrgFileFunctions.checkIfToolExists('crop') IrgFileFunctions.checkIfToolExists('tar') IrgFileFunctions.checkIfToolExists('gdal_translate') IrgFileFunctions.checkIfToolExists('maskFromIntersectError') return True