def removeCompressedOutputs(dataSetName, dryRun=False):
"""Removes the large local output files in a completed run"""
# Make sure the data set is complete before removing any files
ds = DataSet(dataSetName)
if not ds.isComplete():
raise Exception('Attempting to clear compressed on incomplete data set: ' + dataSetName)
localFolder = ds.getLocalFolder()
compressedFilePath = os.path.join(localFolder, 'results/output-CompressedOutputs.tar')
if not os.path.exists(compressedFilePath):
raise Exception('Attempting to clear compressed but output TAR file is missing!')
# Each of these files will be deleted
filesToDelete = ['output-Colormap.tif',
'output-Confidence.tif',
'output-DEM.tif',
'output-Hillshade.tif',
'output-MapProjLeft.tif',
'output-MapProjRight.tif',
'output-MapProjLeftUint8.tif', # Files after here are not compressed but can
'output-MapProjRightUint8.tif', # quickly be regenerated from compressed files.
'output-IntersectionErrorX.tif',
'output-IntersectionErrorY.tif',
'output-IntersectionErrorZ.tif']
for f in filesToDelete: # For each file in the list above
inputPath = os.path.join(localFolder, 'results/' + f)
#print 'Deleting file ' + inputPath
if dryRun: # Print output action
print '- rm ' + inputPath
else: # Actually delete the file
IrgFileFunctions.removeIfExists(inputPath)
def flagIncompleteDataSets(dryRun=False):
"""Flags incomplete data sets for repeat processing"""
if not os.path.exists(INCOMPLETE_STATUS_PATH):
raise Exception('Incomplete file list not present, run --check-local to generate it.')
# Loop through all data sets in the incomplete folder
incompleteFile = open(INCOMPLETE_STATUS_PATH, 'r')
for line in incompleteFile:
entries = line.split(' ')
dataSetName = entries[0]
# Get the status of this data set
ds = DataSet(dataSetName)
# Check that it is actually incomplete
if ds.status != DataSet.INCOMPLETE:
print 'WARNING: non-incomplete set found in incomplete file:'
print line
continue
# For incomplete data sets, remove the downloadLog.txt file
# - This will cause the auto-processor code to run the data set again.
flagFile = os.path.join(ds.getLocalFolder(), 'downloadLog.txt')
if dryRun:
print '- Remove file ' + flagFile
else: # Actually removet the file
IrgFileFunctions.removeIfExists(flagFile)
# Need to ru-run the update function to update this file
incompleteFile.close()
def prepareCtxImage(inputPath, outputFolder, keep):
"""Prepare a single CTX image for processing"""
# Set up paths
cubPath = IrgFileFunctions.replaceExtensionAndFolder(
inputPath, outputFolder, '.cub')
calPath = IrgFileFunctions.replaceExtensionAndFolder(
inputPath, outputFolder, '.cal.cub')
# Convert to ISIS format
if not os.path.exists(cubPath):
cmd = 'mroctx2isis from=' + inputPath + ' to=' + cubPath
os.system(cmd)
# Init Spice data
cmd = 'spiceinit from=' + cubPath
os.system(cmd)
# Apply image correction
if not os.path.exists(calPath):
cmd = 'ctxcal from=' + cubPath + ' to=' + calPath
os.system(cmd)
#you can also optionally run ctxevenodd on the cal.cub files, if needed
if not keep:
os.remove(cubPath)
return calPath
def prepareCtxImage(inputPath, outputFolder, keep):
"""Prepare a single CTX image for processing"""
# Set up paths
cubPath = IrgFileFunctions.replaceExtensionAndFolder(inputPath, outputFolder, '.cub')
calPath = IrgFileFunctions.replaceExtensionAndFolder(inputPath, outputFolder, '.cal.cub')
# Convert to ISIS format
if not os.path.exists(cubPath):
cmd = 'mroctx2isis from=' + inputPath + ' to=' + cubPath
os.system(cmd)
# Init Spice data
cmd = 'spiceinit from=' + cubPath
os.system(cmd)
# Apply image correction
if not os.path.exists(calPath):
cmd = 'ctxcal from='+cubPath+' to='+calPath
os.system(cmd)
#you can also optionally run ctxevenodd on the cal.cub files, if needed
if not keep:
os.remove(cubPath)
return calPath
def makeSpkSetupFile(leapSecondFilePath, outputPath):
"""Creates the required mkspk setup file if it does not already exist"""
# If the file already exists, delete it and rewrite it.
IrgFileFunctions.removeIfExists(outputPath)
# print 'Generating LRONAC compatible .pvl file ' + halfResFilePath
f = open(outputPath, 'w')
f.write("\\begindata\n")
f.write("INPUT_DATA_TYPE = 'STATES'\n")
f.write("OUTPUT_SPK_TYPE = 13\n")
f.write("OBJECT_ID = -85\n") # LRO
f.write("CENTER_ID = 301\n") # Moon
f.write("REF_FRAME_NAME = 'J2000'\n")
f.write("PRODUCER_ID = 'Lronac Pipeline'\n")
f.write("DATA_ORDER = 'epoch x y z vx vy vz'\n")
f.write("DATA_DELIMITER = ','\n")
f.write("LEAPSECONDS_FILE = '" + leapSecondFilePath + "'\n")
f.write("LINES_PER_RECORD = 1\n")
f.write("TIME_WRAPPER = '# ETSECONDS'\n")
#f.write("EPOCH_STR_LENGTH = 16\n")
f.write("INPUT_DATA_UNITS = ('ANGLES=DEGREES' 'DISTANCES=km')\n")
f.write("POLYNOM_DEGREE = 11\n")
f.write("SEGMENT_ID = 'SPK_STATES_13'\n")
# f.write("INPUT_DATA_FILE = 'spkDataFile.txt'")
# f.write("OUTPUT_SPK_FILE = '/home/smcmich1/testSpkFile.bsp'")
f.write("\\begintext\n")
f.close()
def robustLronacPcAlignCall(demPointsPath, lolaPath, outputPrefix):
# Use pc-align to compare points to LOLA DEM, compute rotation and offset
# The max-displacement threshold will be adjusted until we are either using a certain number of LOLA points
# or until we are using a certain percentage of the input points.
MIN_NUM_LOLA_POINTS = 4000
MIN_LOLA_PERCENTAGE = 0.005
STARTING_DISPLACEMENT = 50 # --max-displacement starts at this value..
DISPLACEMENT_INCREMENT = 50 # Increments by this value...
MAX_MAX_DISPLACEMENT = 1800 # And we quit when it hits this value.
# Determine the number of points we want
numLolaPoints = IrgFileFunctions.getFileLineCount(lolaPath) - 1
currentMaxDisplacement = STARTING_DISPLACEMENT
minNumPointsToUse = min(MIN_NUM_LOLA_POINTS, int(MIN_LOLA_PERCENTAGE*float(numLolaPoints)))
print 'Using pc_align to compute a transform between intersected points and LOLA points...'
print 'Starting pc_align, looking for ' + str(minNumPointsToUse) + ' lola point matches.'
transformPath = outputPrefix + '-inverse-transform.txt'
endErrorPath = outputPrefix + '-end_errors.csv'
while(True):
cmd = ('pc_align --highest-accuracy --max-displacement ' + str(currentMaxDisplacement) + ' --datum D_MOON ' +
'--save-inv-transformed-reference-points ' + demPointsPath +
' ' + lolaPath + ' -o ' + outputPrefix + ' --compute-translation-only')
print cmd
os.system(cmd)
if not os.path.exists(endErrorPath):
numLolaPointsUsed = 0 # Failed to produce any output, maybe raising the error cutoff will help?
else:
numLolaPointsUsed = IrgFileFunctions.getFileLineCount(endErrorPath) - 1
if (numLolaPointsUsed >= minNumPointsToUse):
break # Success!
elif (currentMaxDisplacement >= MAX_MAX_DISPLACEMENT): # Hit the maximum max limit!
raise Exception('Error! Unable to find a good value for max-displacement in pc_align. Wanted '
+ str(minNumPointsToUse) + ' points, only found ' + str(numLolaPointsUsed))
else: # Try again with a higher max limit
print ('Trying pc_align again, only got ' + str(numLolaPointsUsed)
+ ' lola point matches with value ' + str(currentMaxDisplacement))
currentMaxDisplacement = currentMaxDisplacement + DISPLACEMENT_INCREMENT
if not os.path.exists(transformPath):
raise Exception('pc_align call failed!')
return True
def correctAndCropImage(inputPath, labelPath, outputPrefix, areaBB=None):
'''Can crop and restore missing metadata.'''
# At least one image was too tall and had to be split in half for this to work
inputIsJp2 = (inputPath[-4:] == ".JP2")
if inputIsJp2: # Call the JP2 function
if areaBB:
areaString = getChunkAreaString(areaBB)
else:
areaString = None
return jp2ToImgAndBack(inputPath, labelPath, outputPrefix, areaString)
else: # Input is a GeoTIFF
outputPath = outputPrefix + '.TIF'
# This is simpler, we just crop using gdal_translat
cmd = 'gdal_translate -of GTiff ' + inputPath +''+ outputPath
if areaBB: # Handle crop
bbString = ' -srcwin %d %d %d %d ' % areaBB
cmd += bbString
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(outputPath):
raise Exception('Could not crop TIFF file: ' + inputPath)
return outputPath
def runInGnuParallel(
numParallelProcesses, commandString, argumentFilePath, parallelArgs=[], nodeListPath=None, verbose=False
):
"""Use GNU Parallel to spread task across multiple computers and processes"""
# Make sure GNU parallel is installed
if not IrgFileFunctions.checkIfToolExists("parallel"):
raise Exception("Need GNU Parallel to distribute the jobs.")
# Use GNU parallel with given number of processes.
# - Let output be interspersed, read input series from file
cmd = ["parallel", "-u", "-a", argumentFilePath]
# Add number of processes if specified (default is one job per CPU core)
if numParallelProcesses is not None:
cmd += ["-P", str(numParallelProcesses)]
# Add list of nodes as argument to the parallel tool if it is available
if nodeListPath is not None:
cmd += ["--sshloginfile", nodeListPath]
# Append any additional arguments to parallel
cmd += parallelArgs
# Append the actual command we want to call to the GNU Parallel call
cmd += [commandString]
if verbose: # Echo the command line call we are about to make
print(" ".join(cmd))
returnCode = subprocess.call(cmd)
return returnCode
def writeColorMapInfo(colormapPath, lutFilePath, demPath, outputPath):
"""Writes a file containing the color map information"""
colormapPercentiles = []
numColorSteps = IrgFileFunctions.getFileLineCount(lutFilePath)
for i in range(0,numColorSteps): # This loop generates the percentage values from the color profile we are using
colormapPercentiles.append(i / float(numColorSteps-1))
# Get the min and max elevation of the DEM
elevationBounds = IrgGeoFunctions.getImageStats(demPath)
# Get elevation values for each percentile
colormapPercentileValues = IrgMathFunctions.getPercentileValues(elevationBounds[0][0], elevationBounds[0][1], colormapPercentiles)
# Now write out a version of the LUT file with values added
inputFile = open(lutFilePath, 'r')
outputFile = open(outputPath, 'w')
# Add a header line to the output file
outputFile.write('Percent of range, R, G, B, Elevation (meters above datum)\n')
# Write a copy of the input file with the elevation values appended to each line
for (line, value) in zip(inputFile, colormapPercentileValues):
newLine = line[:-1] + ', ' + str(round(value,2)) + '\n'
outputFile.write(newLine)
inputFile.close()
outputFile.close()
def runInGnuParallel(numParallelProcesses,
commandString,
argumentFilePath,
parallelArgs=[],
nodeListPath=None,
verbose=False):
"""Use GNU Parallel to spread task across multiple computers and processes"""
# Make sure GNU parallel is installed
if not IrgFileFunctions.checkIfToolExists('parallel'):
raise Exception('Need GNU Parallel to distribute the jobs.')
# Use GNU parallel with given number of processes.
# - Let output be interspersed, read input series from file
cmd = ['parallel', '-u', '-a', argumentFilePath]
# Add number of processes if specified (default is one job per CPU core)
if numParallelProcesses is not None:
cmd += ['-P', str(numParallelProcesses)]
# Add list of nodes as argument to the parallel tool if it is available
if nodeListPath is not None:
cmd += ['--sshloginfile', nodeListPath]
# Append any additional arguments to parallel
cmd += parallelArgs
# Append the actual command we want to call to the GNU Parallel call
cmd += [commandString]
if verbose: # Echo the command line call we are about to make
print(" ".join(cmd))
returnCode = subprocess.call(cmd)
return returnCode
def correctAndCropImage(inputPath, labelPath, outputPrefix, areaBB=None):
'''Can crop and restore missing metadata.'''
# At least one image was too tall and had to be split in half for this to work
inputIsJp2 = (inputPath[-4:] == ".JP2")
if inputIsJp2: # Call the JP2 function
if areaBB:
areaString = getChunkAreaString(areaBB)
else:
areaString = None
return jp2ToImgAndBack(inputPath, labelPath, outputPrefix, areaString)
else: # Input is a GeoTIFF
outputPath = outputPrefix + '.TIF'
# This is simpler, we just crop using gdal_translat
cmd = 'gdal_translate -of GTiff ' + inputPath + '' + outputPath
if areaBB: # Handle crop
bbString = ' -srcwin %d %d %d %d ' % areaBB
cmd += bbString
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(outputPath):
raise Exception('Could not crop TIFF file: ' + inputPath)
return outputPath
def jp2ToImgAndBack(jp2Path, labelPath, outputPrefix, areaString=None):
'''Convert a Jp2 to an IMG and back. Can crop and restore missing metadata.'''
# Convert from JP2 to IMG, possibly adding in missing metadata and cropping.
tempImgPath = outputPrefix + '.IMG'
cmd = (
'/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/JP2_to_PDS -Force -Input '
+ jp2Path + ' -Output ' + tempImgPath + ' -LAbel ' + labelPath)
if areaString:
cmd += ' -Area ' + areaString
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(tempImgPath):
raise Exception('Failed to convert chunk to IMG file: ' + jp2Path)
# Convert back to JP2 format
outputPath = outputPrefix + '.JP2'
cmd = (
'/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/PDS_to_JP2 -Force -Geotiff -Input '
+ tempImgPath + ' -Output ' + outputPath)
print cmd
os.system(cmd)
# If converting back to JP2 failed, try converting to TIF instead
if not IrgFileFunctions.fileIsNonZero(outputPath):
#raise Exception('Failed to convert chunk to JP2 file: ' + outputPath)
print 'WARNING: Failed to convert IMG file to JP2 file! Trying to convert to TIF instead...'
outputPath = outputPrefix + '.TIF'
cmd = 'gdal_translate -of GTiff ' + tempImgPath + ' ' + outputPath
print cmd
os.system(cmd)
# If we still failed, return an exception
if not IrgFileFunctions.fileIsNonZero(outputPath):
raise Exception('Could not convert IMG file to anything! ' +
tempImgPath)
# Successfully created an output file
os.remove(tempImgPath)
return outputPath
def jp2ToImgAndBack(jp2Path, labelPath, outputPrefix, areaString=None):
'''Convert a Jp2 to an IMG and back. Can crop and restore missing metadata.'''
# Convert from JP2 to IMG, possibly adding in missing metadata and cropping.
tempImgPath = outputPrefix + '.IMG'
cmd = ('/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/JP2_to_PDS -Force -Input ' + jp2Path +
' -Output ' + tempImgPath +
' -LAbel ' + labelPath)
if areaString:
cmd += ' -Area ' + areaString
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(tempImgPath):
raise Exception('Failed to convert chunk to IMG file: ' + jp2Path)
# Convert back to JP2 format
outputPath = outputPrefix + '.JP2'
cmd = ('/home/pirl/smcmich1/programs/PDS_JP2-3.17_Linux-x86_64/bin/PDS_to_JP2 -Force -Geotiff -Input ' + tempImgPath +
' -Output ' + outputPath)
print cmd
os.system(cmd)
# If converting back to JP2 failed, try converting to TIF instead
if not IrgFileFunctions.fileIsNonZero(outputPath):
#raise Exception('Failed to convert chunk to JP2 file: ' + outputPath)
print 'WARNING: Failed to convert IMG file to JP2 file! Trying to convert to TIF instead...'
outputPath = outputPrefix + '.TIF'
cmd = 'gdal_translate -of GTiff ' + tempImgPath +' '+ outputPath
print cmd
os.system(cmd)
# If we still failed, return an exception
if not IrgFileFunctions.fileIsNonZero(outputPath):
raise Exception('Could not convert IMG file to anything! ' + tempImgPath)
# Successfully created an output file
os.remove(tempImgPath)
return outputPath
outputPathStereo, stereoMosaicWorkDir, carry))
print 'Starting mosaic call threads'
leftThread.start()
rightThread.start()
print 'Waiting for mosaic threads to complete...'
leftThread.join()
rightThread.join()
mosaicTime = time.time()
logging.info('Mosaics finished in %f seconds', mosaicTime - noprojTime)
# Clean up temporary files
if not options.keep:
print 'Deleting temporary files'
IrgFileFunctions.removeIfExists(leftCorrectedPath)
IrgFileFunctions.removeIfExists(rightCorrectedPath)
IrgFileFunctions.removeIfExists(leftStereoCorrectedPath)
IrgFileFunctions.removeIfExists(rightStereoCorrectedPath)
IrgFileFunctions.removeIfExists(pvlPath)
IrgFileFunctions.removeIfExists(leftNoprojPath)
IrgFileFunctions.removeIfExists(rightNoprojPath)
IrgFileFunctions.removeIfExists(leftStereoNoprojPath)
IrgFileFunctions.removeIfExists(rightStereoNoprojPath)
#IrgFileFunctions.removeFolderIfExists(mainMosaicWorkDir)
#IrgFileFunctions.removeFolderIfExists(stereoMosaicWorkDir)
#if (hadToCreateTempFolder):
# IrgFileFunctions.removeFolderIfExists(tempFolder)
## Remove all the .kml files
#fileList = [ f for f in os.listdir(tempFolder) if f.endswith(".kml") ]
def archiveDataSet(dataSetName, deleteLocalFiles=False, dryRun=False):
"""Archives a data set to long term storage on Lou"""
# The local files to be archived are already in tar files
# Make sure the data set is complete before archiving it
ds = DataSet(dataSetName)
if not ds.isComplete():
raise Exception('Attempting to archive incomplete data set: ' + dataSetName)
localFolder = ds.getLocalFolder()
resultsFolder = os.path.join(localFolder, 'results/')
# In order to archive the file it just has to be moved over to the Lou filesystem.
# - Tape archiving is performed automatically when needed.
# - All tar files are stored in the same directory
filesToArchive = ['output-CompressedOutputs.tar']
for f in filesToArchive:
# Get storage name to use
archiveName = 'NAC_DTM_' + dataSetName + '_CompressedOutputs.tar'
inputPath = os.path.join(resultsFolder, f)
outputPath = os.path.join(LOU_STORAGE_PATH, archiveName)
# Move the file over to Lou
#cmd = ['shiftc', '--wait', '--verify', inputPath, outputPath]
cmd = 'shiftc --wait --verify ' + inputPath + ' ' + outputPath
print cmd
if not dryRun: # Actually transfer the file
os.system(cmd)
#p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
#textOutput, err = p.communicate()
#print textOutput
#print 'ppp'
#print err
#print textOutput.find('done')
#if not ('done' in textOutput):
# raise Exception('Error transferring file!')
# Verify that the file was actually transferred
resp = subprocess.call(['ssh', LOU_SSH_TARGET, 'test -e ' + pipes.quote(LOU_LOCAL_STORAGE_PATH)])
if resp == 0:
print 'shiftc file transfer confirmed.'
else:
raise Exception('Failed to transfer file using shiftc!')
if deleteLocalFiles: # Remove almost all files
# Keep some result files and move them to the main data set directory
filesToKeep = ['output-LOLA_diff_stats.txt',
'output-Log.txt',
'output-PcAlignLog.txt',
'output-CompressedInputs.tar.bz2',
'output-CompressedDiagnostics.tar.bz2']
for f in filesToKeep:
inputPath = os.path.join(resultsFolder, f)
outputPath = os.path.join(localFolder, f)
print 'mv ' + inputPath + ' ' + outputPath
if not dryRun: # Actually move the files
shutil.move(inputPath, outputPath)
# Now that we have saved a few files, delete everything else
cmdRmImg = 'rm ' + localFolder + '/*.IMG'
rdrPath = os.path.join(localFolder, 'lolaRdrPoints.csv')
prtPath = os.path.join(localFolder, 'print.prt')
#logPath = os.path.join(localFolder, 'stdOutputLog.txt')
workDir = os.path.join(localFolder, 'workDir')
resultsDir = os.path.join(localFolder, 'results')
print cmdRmImg
print 'rm ' + rdrPath
print 'rm ' + prtPath
#print 'rm ' + logPath
print 'rm -rf ' + workDir
print 'rm -rf ' + resultsDir
if not dryRun: # Actually delete the files
os.system(cmdRmImg)
IrgFileFunctions.removeIfExists(rdrPath)
IrgFileFunctions.removeIfExists(prtPath)
#IrgFileFunctions.removeIfExists(logPath)
IrgFileFunctions.removeFolderIfExists(workDir)
IrgFileFunctions.removeFolderIfExists(resultsDir)
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
if subtype != 'DEM': # Handles RED and COLOR images
# The label file URL is the same as the image but with a different extension
remoteLabelURL = getLabelPathFromImagePath(remoteURL)
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
localLabelPath = os.path.join(workDir, os.path.basename(remoteLabelURL))
# Retrieve the header file
if not os.path.exists(localLabelPath):
# Try to get the label locally!
pdsStart = remoteLabelURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/', remoteLabelURL[pdsStart:])
print localPdsPath
if os.path.exists(localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath +' '+ localLabelPath
else: # Download the image
cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath
print cmd
os.system(cmd)
# Retrieve the image file
if not os.path.exists(localFilePath): # Need to get it from somewhere
# Try to get the image locally!
pdsStart = remoteURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:])
if os.path.exists(localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath +' '+ localFilePath
else: # Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(localFilePath):
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
print 'Could not find label or image file, DELETING DATA RECORD!'
common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, setName)
raise Exception('Unable to download from URL: ' + remoteURL)
# Check if there is geo data in the JP2 file
jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath)
# If there is no label file, try to generate an artificial one.
fakeLabelFile = False
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
#raise Exception('Unable to download from URL: ' + remoteLabelURL)
print 'WARNING: Unable to download from URL: ' + remoteLabelURL
if not jp2HasGeoData:
raise Exception('No geo data in JP2 and no Label file! Cannot handle this!')
print 'Generating a fake LBL file to proceed...'
localLabelPath = writeFakeLabelFromJp2(localFilePath)
fakeLabelFile = True
# At this point we always have a label file but it may be fake
# Some images are missing geo information but we can add it from the label file
localImageIsTiff = False
localImagePath = localFilePath
if not jp2HasGeoData:
print 'Correcting JP2 file with no geo information!!!!'
# Correct the local file, then remove the old (bad) file
outputPrefix = localFilePath[0:-4]
localImagePath = correctAndCropImage(localFilePath, localLabelPath, outputPrefix)
print 'Generated ' + localImagePath
if localFilePath != localImagePath:
print 'Deleting JP2 file without metadata!'
os.remove(localFilePath)
localImageIsTiff = (localImagePath[-4:] == ".TIF")
if not localImageIsTiff:
# Call code to fix the header information in the JP2 file!
cmd = FIX_JP2_TOOL +' '+ localImagePath
print cmd
os.system(cmd)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath)
(width, height) = IrgIsisFunctions.getImageSize(localImagePath)
if (projType == 'POLAR STEREOGRAPHIC') and False: #(width < MAX_POLAR_UPLOAD_WIDTH):
# Google has trouble digesting these files so handle them differently.
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!')
print 'Special handling for POLAR STEROGRAPHIC image!'
if fakeLabelFile:
print 'Cannot reprocess polar image without a label file!'
print 'All we can do is upload the file and hope for the best.'
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# Compute how many chunks are needed for this image
numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE)
# Determine which chunk this DB entry is for
chunkNum = getChunkNum(setName)
print 'This is chunk number ' + str(chunkNum)
if chunkNum >= numChunks: # Check for chunk number error
raise Exception('Illegal chunk number: ' + setName)
# If this is the main DB entry, we need to make sure the other DB entries exist!
if chunkNum == 0:
# Go ahead and try to add each chunk, the call will only go through if it does not already exist.
for i in range(1,numChunks):
chunkSetName = makeChunkSetName(setName, i)
print 'Add chunk set name to DB: ' + chunkSetName
common.addDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, chunkSetName, remoteURL)
raise Exception('DEBUG')
# Now actually generate the desired chunk
# - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it.
fileBasePath = os.path.splitext(localImagePath)[0]
localImgPath = fileBasePath + '.IMG'
localChunkPrefix = fileBasePath + '_' + str(chunkNum)
chunkBB = getChunkBoundingBox(width, height, chunkNum)
localChunkPath = correctAndCropImage(localImagePath, localLabelPath, localChunkPrefix, chunkBB)
# Just use the same label file, we don't care if the DB has per-chunk boundaries.
return [localChunkPath, localLabelPath]
else: # A normal, non-polar file.
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# TODO: Handle POLAR DEMS
else: # Handle DEMs
# For DEMs there is no label file
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
if not os.path.exists(localFilePath):
# Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(localFilePath): # Make sure we got the file
raise Exception('Unable to download from URL: ' + remoteURL)
# Generate a header file from the IMG file
localLabelPath = localFilePath[:-4] + '.LBL'
cmd = 'head -n 90 ' + localFilePath +' > '+ localLabelPath
print cmd
os.system(cmd)
# Check if this is a polar stereographic image
isPolar = False
f = open(localLabelPath)
for line in f:
if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC" in line):
isPolar = True
print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY'
break
#os.remove(localFilePath)
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!')
f.close()
# Convert from IMG to TIF
tiffFilePath = localFilePath[:-4] + '.TIF'
if not os.path.exists(tiffFilePath):
cmd = 'gdal_translate -of GTiff ' + localFilePath +' '+ tiffFilePath
print cmd
os.system(cmd)
if not isPolar: # Only EQC files need to be corrected
# Correct projected coordinates problems
cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath
print cmd
os.system(cmd)
os.remove(localFilePath) # Clean up source image
return [tiffFilePath, localLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
# Images with a center over this latitude will use polar stereographic projection
# instead of simple cylindrical projection.
HIGH_LATITUDE_CUTOFF = 65 # Degrees
asuImagePath = os.path.join(workDir, setName +
'_noGeo.jp2') # Map projected image from ASU
asuLabelPath = os.path.join(workDir,
setName + '_noGeo.lbl') # Label from ASU
edrPath = os.path.join(workDir, setName + '.IMG') # Raw image from PDS
timePath = os.path.join(
workDir,
setName + '.time') # Contains only the file capture time string
#cubPath = os.path.join(workDir, setName + '.cub') # Output of mroctx2isis
#calPath = os.path.join(workDir, setName + '.cal.cub') # Output of ctxcal
mapPath = os.path.join(workDir, setName + '.map.cub') # Output of cam2map
mapLabelPath = os.path.join(workDir, setName +
'.map.pvl') # Specify projection to cam2map
# Generate the remote URLs from the data prefix and volume stored in these parameters
asuImageUrl, asuLabelUrl, edrUrl = generatePdsPath(setName, subtype)
if True: # Map project the EDR ourselves <-- This takes way too long!
print 'Projecting the EDR image using ISIS...'
localFilePath = os.path.join(workDir, setName +
'.tif') # The output file we will upload
# Check if this is a "flat" calibration image
badImage = checkForBadFile(edrUrl)
if badImage:
raise Exception('TODO: Remove bad images from the DB!')
if not os.path.exists(edrPath):
# Download the EDR file
cmd = 'wget ' + edrUrl + ' -O ' + edrPath
print cmd
os.system(cmd)
# Extract the image capture time from the .IMG file
if not os.path.exists(timePath):
timeString = getCreationTimeHelper(edrPath)
f = open(timePath, 'w')
f.write(timeString)
f.close()
# Convert and apply calibration to the CTX file
calPath = IrgIsisFunctions.prepareCtxImage(edrPath, workDir, False)
## Find out the center latitude of the file and determine if it is high latitude
centerLat = IrgIsisFunctions.getCubeCenterLatitude(calPath, workDir)
print centerLat
highLat = abs(float(centerLat)) > HIGH_LATITUDE_CUTOFF
if True: #not os.path.exists(mapLabelPath):
# Generate the map label file
generateDefaultMappingPvl(mapLabelPath, highLat)
if True: #not os.path.exists(mapPath):
# Generate the map projected file
cmd = [
'timeout', '6h', 'cam2map', 'matchmap=', 'False', 'from=',
calPath, 'to=', mapPath, 'map=', mapLabelPath
]
print cmd
#os.system(cmd)
p = subprocess.Popen(cmd)
p.communicate()
if (p.returncode != 0):
raise Exception(
'Error or timeout running cam2map, returnCode = ' +
str(p.returncode))
if True: #not os.path.exists(localFilePath):
# Generate the final image to upload
cmd = 'gdal_translate -of GTiff ' + mapPath + ' ' + localFilePath
print cmd
os.system(cmd)
# Clean up intermediate files
os.remove(mapLabelPath)
os.remove(edrPath)
os.remove(calPath)
os.remove(mapPath)
# Two local files are left around, the first should be uploaded.
return [localFilePath, timePath]
else: # Use the map projected image from the ASU web site
print 'Using ASU projected image...'
localFilePath = os.path.join(workDir, setName +
'.jp2') # The output file we will upload
# Note: ASU seems to be missing some files!
# We are using the label path in both projection cases
if not os.path.exists(asuLabelPath):
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(
asuLabelPath): # Try the alternate label path
os.remove(asuLabelPath)
asuLabelUrl = asuLabelUrl.replace('.scyl.', '.ps.')
asuLabelPath = asuLabelPath.replace('.scyl.', '.ps.')
print 'Trying alternate label path: ' + asuLabelUrl
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuLabelPath):
raise Exception('Failed to download file label at URL: ' +
asuLabelUrl)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(asuLabelPath)
if projType != 'SimpleCylindrical':
print 'WARNING: projType = ' + projType
print 'Maps Engine may fail to ingest this file!'
#os.remove(asuLabelPath)
#raise Exception(projType + ' images on hold until Google fixes a bug!')
if not os.path.exists(asuImagePath):
# Download the image file
cmd = 'wget "' + asuImageUrl + '" -O ' + asuImagePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuImagePath):
raise Exception('Failed to download image file at URL: ' +
asuImageUrl)
## Correct the ISIS header if needed
#fixedAsuHeaderPath = putIsisHeaderIn180(asuLabelPath)
#if (fixedAsuHeaderPath != asuLabelPath):
# os.remove(asuLabelPath) # Delete replaced header
if True: # This is fast now, so do it every time
# Correct the file - The JP2 file from ASU needs the geo data from the label file!
#cmd = 'addGeoToAsuCtxJp2.py --keep --label '+ asuLabelPath +' '+ asuImagePath +' '+ localFilePath
#print cmd
#os.system(cmd)
# TODO: Remove unnecessary image copy here
(correctedPath,
sidecarPath) = addGeoDataToAsuJp2File(asuImagePath,
asuLabelPath,
localFilePath,
keep=False)
if not IrgFileFunctions.fileIsNonZero(sidecarPath):
raise Exception('Script to add geo data to JP2 file failed!')
# Clean up
os.remove(asuImagePath)
# Three local files are left around, the first should be uploaded.
return [correctedPath, sidecarPath, asuLabelPath]
def functionStartupCheck():
# These calls will raise an exception if the tool is not found
IrgFileFunctions.checkIfToolExists('calibrationReport.py')
IrgFileFunctions.checkIfToolExists('noproj')
IrgFileFunctions.checkIfToolExists('lronacjitreg')
IrgFileFunctions.checkIfToolExists('handmos')
IrgFileFunctions.checkIfToolExists('cubenorm')
IrgFileFunctions.checkIfToolExists('stereoDoubleCalibrationProcess.py')
return True
def functionStartupCheck():
# These calls will raise an exception if the tool is not found
IrgFileFunctions.checkIfToolExists('lronac2refinedMosaics.py')
IrgFileFunctions.checkIfToolExists('makeDemAndCompare.py')
return True
fullWidth = int(projectionInfo[widthStart+7 : heightStart-1])
fullHeight = int(projectionInfo[heightStart+8 : heightEnd])
print 'Output image size is ' + str(fullWidth) + ' by ' + str(fullHeight) + ' pixels.'
# For now we just break up the image into a user-specified tile size (default 1000x1000)
numTilesX, numTilesY, tileList = generateTileList(fullWidth, fullHeight, options.tileSize)
numTiles = numTilesX * numTilesY
print 'Splitting into ' + str(numTilesX) + ' by ' + str(numTilesY) + ' tiles.'
# Set up output folder
outputFolder = os.path.dirname(options.outputPath)
if outputFolder == '':
outputFolder = './' # Handle calls in same directory
outputName = os.path.basename(options.outputPath)
IrgFileFunctions.createFolder(outputFolder)
# Make a temporary directory to store the tiles
if options.workDir:
tempFolder = options.workDir
else: # No folder provided, create a default one
tempFolder = os.path.join(outputFolder, outputName.replace('.', '_') + '_tiles/')
IrgFileFunctions.createFolder(tempFolder)
# Generate a text file that contains the boundaries for each tile
argumentFilePath = os.path.join(tempFolder, 'argumentList.txt')
argumentFile = file(argumentFilePath, 'w')
for tile in tileList:
argumentFile.write(str(tile[0]) + '\t' + str(tile[1]) + '\t' + str(tile[2]) + '\t' + str(tile[3]) + '\n')
argumentFile.close()
def main(argsIn):
print '#################################################################################'
print "Running makeDemAndCompare.py"
try:
try:
usage = "usage: makeDemAndCompare.py [--output <path>][--manual]\n "
parser = optparse.OptionParser(usage=usage)
inputGroup = optparse.OptionGroup(parser, 'Input Paths')
inputGroup.add_option("--left", dest="leftPath", help="Path to left cube file")
inputGroup.add_option("--right", dest="rightPath", help="Path to right cube file")
inputGroup.add_option("--lola", dest="lolaPath", help="Path to LOLA DEM")
inputGroup.add_option("--asu", dest="asuPath", help="Path to ASU DEM")
inputGroup.add_option("--node-file", dest="nodeFilePath",
help="Path to file containing list of available nodes")
parser.add_option_group(inputGroup)
# The default working directory path is kind of ugly...
parser.add_option("--workDir", dest="workDir", help="Folder to store temporary files in")
parser.add_option("--prefix", dest="prefix", help="Output prefix.")
parser.add_option("--log-path", dest="logPath",
help="Where to write the output log file.")
parser.add_option("--crop", dest="cropAmount",
help="Crops the output image to reduce processing time.")
parser.add_option("--manual", action="callback", callback=man,
help="Read the manual.")
parser.add_option("--keep", action="store_true", dest="keep",
help="Do not delete the temporary files.")
(options, args) = parser.parse_args(argsIn)
if not options.leftPath:
parser.error("Need left input path")
if not options.rightPath:
parser.error("Need right input path")
if not options.prefix:
parser.error("Need output prefix")
except optparse.OptionError, msg:
raise Usage(msg)
print "Beginning processing....."
startTime = time.time()
# Make sure we have all the functions we need
functionStartupCheck()
# Set this to true to force steps after it to activate
carry = False
# Set up the output folders
outputFolder = os.path.dirname(options.prefix)
inputBaseName = os.path.basename(options.leftPath)
tempFolder = outputFolder + '/' + inputBaseName + '_stereoCalibrationTemp/'
if (options.workDir):
tempFolder = options.workDir
if not os.path.exists(outputFolder):
os.mkdir(outputFolder)
hadToCreateTempFolder = not os.path.exists(tempFolder)
if not os.path.exists(tempFolder):
os.mkdir(tempFolder)
# Set up logging
if not options.logPath:
options.logPath = options.prefix + '-Log.txt'
logging.basicConfig(filename=options.logPath,level=logging.INFO)
# Go ahead and set up all the output paths
# -- Deliverables
demPath = options.prefix + '-DEM.tif'
intersectionErrorPath = options.prefix + '-IntersectionErr.tif'
hillshadePath = options.prefix + '-Hillshade.tif'
colormapPath = options.prefix + '-Colormap.tif'
colormapLegendPath = options.prefix + '-ColormapLegend.csv'
mapProjectLeftPath = options.prefix + '-MapProjLeft.tif'
mapProjectRightPath = options.prefix + '-MapProjRight.tif'
confidenceLevelPath = options.prefix + '-Confidence.tif'
confidenceLegendPath = options.prefix + '-ConfidenceLegend.csv'
# -- Diagnostic
intersectionViewPathX = options.prefix + '-IntersectionErrorX.tif'
intersectionViewPathY = options.prefix + '-IntersectionErrorY.tif'
intersectionViewPathZ = options.prefix + '-IntersectionErrorZ.tif'
lolaDiffStatsPath = options.prefix + '-LOLA_diff_stats.txt'
lolaDiffPointsPath = options.prefix + '-LOLA_diff_points.csv'
lolaAsuDiffStatsPath = options.prefix + '-ASU_LOLA_diff_stats.txt'
lolaAsuDiffPointsPath = options.prefix + '-ASU_LOLA_diff_points.csv'
mapProjectLeftUint8Path = options.prefix + '-MapProjLeftUint8.tif'
mapProjectRightUint8Path = options.prefix + '-MapProjRightUint8.tif'
# If specified, crop the inputs that will be passed into the stereo function to reduce processing time
mainMosaicCroppedPath = os.path.join(tempFolder, 'mainMosaicCropped.cub')
stereoMosaicCroppedPath = os.path.join(tempFolder, 'stereoMosaicCropped.cub')
if options.cropAmount and (options.cropAmount > 0):
# Verify input files are present
if not os.path.exists(options.leftPath):
raise Exception('Input file ' + options.leftPath + ' not found!')
if not os.path.exists(options.rightPath):
raise Exception('Input file ' + options.rightPath + ' not found!')
if (not os.path.exists(mainMosaicCroppedPath)) or carry:
cmd = ('crop from= ' + options.leftPath + ' to= ' + mainMosaicCroppedPath +
' nlines= ' + str(options.cropAmount))# + ' line=24200')
print cmd
os.system(cmd)
if (not os.path.exists(stereoMosaicCroppedPath) or carry):
cmd = ('crop from= ' + options.rightPath + ' to= ' + stereoMosaicCroppedPath +
' nlines= ' + str(options.cropAmount))# + ' line=24200')
print cmd
os.system(cmd)
options.leftPath = mainMosaicCroppedPath
options.rightPath = stereoMosaicCroppedPath
print '\n-------------------------------------------------------------------------\n'
# Call stereo to generate a point cloud from the two images
# - This step takes a really long time.
stereoOutputPrefix = os.path.join(tempFolder, 'stereoWorkDir/stereo')
stereoOutputFolder = os.path.join(tempFolder, 'stereoWorkDir')
pointCloudPath = stereoOutputPrefix + '-PC.tif'
stereoOptionString = ('--corr-timeout 400 --alignment-method AffineEpipolar --subpixel-mode ' + str(SUBPIXEL_MODE) +
' ' + options.leftPath + ' ' + options.rightPath +
' --job-size-w 4096 --job-size-h 4096 ' + # Reduce number of tile files created
' ' + stereoOutputPrefix + ' --processes 10 --threads-multiprocess 4' +
' --threads-singleprocess 32 --compute-error-vector' + ' --filter-mode 1' +
' --erode-max-size 5000 --subpixel-kernel 35 35 --subpixel-max-levels 0')
if (not os.path.exists(pointCloudPath) and not os.path.exists(demPath)) or carry:
# Verify input files are present
if not os.path.exists(options.leftPath):
raise Exception('Input file ' + options.leftPath + ' not found!')
if not os.path.exists(options.rightPath):
raise Exception('Input file ' + options.rightPath + ' not found!')
cmd = ('parallel_stereo ' + stereoOptionString)
print cmd
os.system(cmd)
# Compute percentage of good pixels
percentGood = IrgAspFunctions.getStereoGoodPixelPercentage(stereoOutputPrefix)
print 'Stereo completed with good pixel percentage: ' + str(percentGood)
logging.info('Final stereo completed with good pixel percentage: %s', str(percentGood))
else:
print 'Stereo file ' + pointCloudPath + ' already exists, skipping stereo step.'
stereoTime = time.time()
logging.info('Stereo finished in %f seconds', stereoTime - startTime)
# Find out the center latitude of the mosaic
if os.path.exists(options.leftPath):
centerLat = IrgIsisFunctions.getCubeCenterLatitude(options.leftPath, tempFolder)
elif os.path.exists(demPath): # Input file has been deleted but we still have the info
demInfo = IrgGeoFunctions.getImageGeoInfo(demPath, False)
centerLat = demInfo['standard_parallel_1']
else:
raise Exception("Can't delete the input files before creating the DEM!")
# Generate a DEM
if (not os.path.exists(demPath)) or carry:
# Equirectangular style projection
# - Latitude of true scale = center latitude = lat_ts
# - Latitude of origin = 0 = lat+0
# - Longitude of projection center = Central meridian = lon+0
cmd = ('point2dem --dem-hole-fill-len 15 --remove-outliers --errorimage -o ' + options.prefix + ' ' + pointCloudPath +
' -r moon --tr ' + str(DEM_METERS_PER_PIXEL) + ' --t_srs "+proj=eqc +lat_ts=' + str(centerLat) +
' +lat_0=0 +a='+str(MOON_RADIUS)+' +b='+str(MOON_RADIUS)+' +units=m" --nodata ' + str(DEM_NODATA))
os.system(cmd)
else:
print 'DEM file ' + demPath + ' already exists, skipping point2dem step.'
# Create a hillshade image to visualize the output
if (not os.path.exists(hillshadePath)) or carry:
cmd = 'hillshade ' + demPath + ' -o ' + hillshadePath
print cmd
os.system(cmd)
else:
print 'Output file ' + hillshadePath + ' already exists, skipping hillshade step.'
# Create a colorized version of the hillshade
# - Uses a blue-red color map from here: http://www.sandia.gov/~kmorel/documents/ColorMaps/
if (not os.path.exists(colormapPath)) or (not os.path.exists(colormapLegendPath)) or carry:
# The color LUT is kept with the source code
lutFilePath = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'colorProfileBlueRed.csv')
# Generate the initial version of colormap
colormapTempPath = options.prefix + '-ColormapTemp.tif'
cmd = 'colormap ' + demPath + ' -o ' + colormapTempPath + ' -s ' + hillshadePath + ' --lut-file ' + lutFilePath
print cmd
os.system(cmd)
# Now convert to the final output version (remove transparency layer) and remove the temp file
IrgFileFunctions.stripRgbImageAlphaChannel(colormapTempPath, colormapPath)
os.remove(colormapTempPath)
# Generate another file storing the colormap info
writeColorMapInfo(colormapPath, lutFilePath, demPath, colormapLegendPath)
else:
print 'Output file ' + colormapPath + ' already exists, skipping colormap step.'
## Create a 3d mesh of the point cloud
#meshPath = os.path.join(outputFolder, 'mesh.ive')
#meshPrefix = os.path.join(outputFolder, 'mesh')
#cmd = 'point2mesh ' + pointCloudPath + ' ' + options.leftPath + ' -o ' + meshPrefix
#if not os.path.exists(meshPath):
# print cmd
# os.system(cmd)
if not options.keep: # Remove stereo folder here to cut down on file count before mapproject calls
IrgFileFunctions.removeFolderIfExists(stereoOutputFolder)
# Convert the intersection error to a viewable format
cmdX = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 1 ' + intersectionErrorPath + ' ' + intersectionViewPathX
cmdY = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 2 ' + intersectionErrorPath + ' ' + intersectionViewPathY
cmdZ = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 3 ' + intersectionErrorPath + ' ' + intersectionViewPathZ
if not os.path.exists(intersectionViewPathX) or carry:
print cmdX
os.system(cmdX)
if not os.path.exists(intersectionViewPathY) or carry:
print cmdY
os.system(cmdY)
if not os.path.exists(intersectionViewPathZ) or carry:
print cmdZ
os.system(cmdZ)
# Generate a confidence plot from the intersection error
if not os.path.exists(confidenceLevelPath):
thresholdString = str(PIXEL_ACCURACY_THRESHOLDS)[1:-1].replace(',', '')
cmd = ('maskFromIntersectError ' + intersectionErrorPath + ' ' + confidenceLevelPath
+ ' --legend ' + confidenceLegendPath +
' --scaleOutput --thresholds ' + thresholdString)
print cmd
os.system(cmd)
hillshadeTime = time.time()
logging.info('DEM and hillshade finished in %f seconds', hillshadeTime - stereoTime)
# Call script to compare LOLA data with the DEM
if options.lolaPath:
compareDemToLola(options.lolaPath, demPath, lolaDiffStatsPath, lolaDiffPointsPath, carry)
# Call script to compare LOLA data with the ASU DEM
if options.asuPath:
compareDemToLola(options.lolaPath, options.asuPath, lolaAsuDiffStatsPath, lolaAsuDiffPointsPath, carry)
# Generate a map projected version of the left and right images
# - This step is done last since it is so slow!
mapProjectImage(options.leftPath, demPath, mapProjectLeftPath, MAP_PROJECT_METERS_PER_PIXEL, centerLat, options.nodeFilePath, carry)
mapProjectImage(options.rightPath, demPath, mapProjectRightPath, MAP_PROJECT_METERS_PER_PIXEL, centerLat, options.nodeFilePath, carry)
# Generate 8 bit versions of the mapproject files for debugging
cmdLeft = 'gdal_translate -scale -ot byte ' + mapProjectLeftPath + ' ' + mapProjectLeftUint8Path
cmdRight = 'gdal_translate -scale -ot byte ' + mapProjectRightPath + ' ' + mapProjectRightUint8Path
if not os.path.exists(mapProjectLeftUint8Path) or carry:
print cmdLeft
os.system(cmdLeft)
if not os.path.exists(mapProjectRightUint8Path) or carry:
print cmdRight
os.system(cmdRight)
mapProjectTime = time.time()
logging.info('Map project finished in %f seconds', mapProjectTime - hillshadeTime)
# Clean up temporary files
if not options.keep:
print 'Removing temporary files'
IrgFileFunctions.removeIfExists(mainMosaicCroppedPath)
IrgFileFunctions.removeIfExists(stereoMosaicCroppedPath)
#IrgFileFunctions.removeIntermediateStereoFiles(stereoOutputPrefix) # Limited clear
IrgFileFunctions.removeFolderIfExists(stereoOutputFolder) # Larger clear
#if (hadToCreateTempFolder): Not done since stereo output needs to be retained
# IrgFileFunctions.removeFolderIfExists(tempFolder)
endTime = time.time()
logging.info('makeDemAndCompare.py finished in %f seconds', endTime - startTime)
print "Finished in " + str(endTime - startTime) + " seconds."
print '#################################################################################'
return 0
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
# Images with a center over this latitude will use polar stereographic projection
# instead of simple cylindrical projection.
HIGH_LATITUDE_CUTOFF = 65 # Degrees
asuImagePath = os.path.join(workDir, setName + '_noGeo.jp2') # Map projected image from ASU
asuLabelPath = os.path.join(workDir, setName + '_noGeo.lbl') # Label from ASU
edrPath = os.path.join(workDir, setName + '.IMG') # Raw image from PDS
timePath = os.path.join(workDir, setName + '.time') # Contains only the file capture time string
#cubPath = os.path.join(workDir, setName + '.cub') # Output of mroctx2isis
#calPath = os.path.join(workDir, setName + '.cal.cub') # Output of ctxcal
mapPath = os.path.join(workDir, setName + '.map.cub') # Output of cam2map
mapLabelPath = os.path.join(workDir, setName + '.map.pvl') # Specify projection to cam2map
# Generate the remote URLs from the data prefix and volume stored in these parameters
asuImageUrl, asuLabelUrl, edrUrl = generatePdsPath(setName, subtype)
if True: # Map project the EDR ourselves <-- This takes way too long!
print 'Projecting the EDR image using ISIS...'
localFilePath = os.path.join(workDir, setName + '.tif') # The output file we will upload
# Check if this is a "flat" calibration image
badImage = checkForBadFile(edrUrl)
if badImage:
raise Exception('TODO: Remove bad images from the DB!')
if not os.path.exists(edrPath):
# Download the EDR file
cmd = 'wget ' + edrUrl + ' -O ' + edrPath
print cmd
os.system(cmd)
# Extract the image capture time from the .IMG file
if not os.path.exists(timePath):
timeString = getCreationTimeHelper(edrPath)
f = open(timePath, 'w')
f.write(timeString)
f.close()
# Convert and apply calibration to the CTX file
calPath = IrgIsisFunctions.prepareCtxImage(edrPath, workDir, False)
## Find out the center latitude of the file and determine if it is high latitude
centerLat = IrgIsisFunctions.getCubeCenterLatitude(calPath, workDir)
print centerLat
highLat = abs(float(centerLat)) > HIGH_LATITUDE_CUTOFF
if True:#not os.path.exists(mapLabelPath):
# Generate the map label file
generateDefaultMappingPvl(mapLabelPath, highLat)
if True:#not os.path.exists(mapPath):
# Generate the map projected file
cmd = ['timeout', '6h', 'cam2map', 'matchmap=','False', 'from=', calPath, 'to=', mapPath, 'map=', mapLabelPath]
print cmd
#os.system(cmd)
p = subprocess.Popen(cmd)
p.communicate()
if (p.returncode != 0):
raise Exception('Error or timeout running cam2map, returnCode = ' + str(p.returncode))
if True: #not os.path.exists(localFilePath):
# Generate the final image to upload
cmd = 'gdal_translate -of GTiff ' + mapPath + ' ' + localFilePath
print cmd
os.system(cmd)
# Clean up intermediate files
os.remove(mapLabelPath)
os.remove(edrPath)
os.remove(calPath)
os.remove(mapPath)
# Two local files are left around, the first should be uploaded.
return [localFilePath, timePath]
else: # Use the map projected image from the ASU web site
print 'Using ASU projected image...'
localFilePath = os.path.join(workDir, setName + '.jp2') # The output file we will upload
# Note: ASU seems to be missing some files!
# We are using the label path in both projection cases
if not os.path.exists(asuLabelPath):
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuLabelPath): # Try the alternate label path
os.remove(asuLabelPath)
asuLabelUrl = asuLabelUrl.replace( '.scyl.', '.ps.')
asuLabelPath = asuLabelPath.replace('.scyl.', '.ps.')
print 'Trying alternate label path: ' + asuLabelUrl
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuLabelPath):
raise Exception('Failed to download file label at URL: ' + asuLabelUrl)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(asuLabelPath)
if projType != 'SimpleCylindrical':
print 'WARNING: projType = ' + projType
print 'Maps Engine may fail to ingest this file!'
#os.remove(asuLabelPath)
#raise Exception(projType + ' images on hold until Google fixes a bug!')
if not os.path.exists(asuImagePath):
# Download the image file
cmd = 'wget "' + asuImageUrl + '" -O ' + asuImagePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuImagePath):
raise Exception('Failed to download image file at URL: ' + asuImageUrl)
## Correct the ISIS header if needed
#fixedAsuHeaderPath = putIsisHeaderIn180(asuLabelPath)
#if (fixedAsuHeaderPath != asuLabelPath):
# os.remove(asuLabelPath) # Delete replaced header
if True: # This is fast now, so do it every time
# Correct the file - The JP2 file from ASU needs the geo data from the label file!
#cmd = 'addGeoToAsuCtxJp2.py --keep --label '+ asuLabelPath +' '+ asuImagePath +' '+ localFilePath
#print cmd
#os.system(cmd)
# TODO: Remove unnecessary image copy here
(correctedPath, sidecarPath) = addGeoDataToAsuJp2File(asuImagePath, asuLabelPath, localFilePath, keep=False)
if not IrgFileFunctions.fileIsNonZero(sidecarPath):
raise Exception('Script to add geo data to JP2 file failed!')
# Clean up
os.remove(asuImagePath)
# Three local files are left around, the first should be uploaded.
return [correctedPath, sidecarPath, asuLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
if subtype != 'DEM': # Handles RED and COLOR images
# The label file URL is the same as the image but with a different extension
remoteLabelURL = getLabelPathFromImagePath(remoteURL)
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
localLabelPath = os.path.join(workDir,
os.path.basename(remoteLabelURL))
# Retrieve the header file
if not os.path.exists(localLabelPath):
# Try to get the label locally!
pdsStart = remoteLabelURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/',
remoteLabelURL[pdsStart:])
print localPdsPath
if os.path.exists(
localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath + ' ' + localLabelPath
else: # Download the image
cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath
print cmd
os.system(cmd)
# Retrieve the image file
if not os.path.exists(localFilePath): # Need to get it from somewhere
# Try to get the image locally!
pdsStart = remoteURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:])
if os.path.exists(
localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath + ' ' + localFilePath
else: # Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(localFilePath):
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
print 'Could not find label or image file, DELETING DATA RECORD!'
common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype,
setName)
raise Exception('Unable to download from URL: ' + remoteURL)
# Check if there is geo data in the JP2 file
jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath)
# If there is no label file, try to generate an artificial one.
fakeLabelFile = False
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
#raise Exception('Unable to download from URL: ' + remoteLabelURL)
print 'WARNING: Unable to download from URL: ' + remoteLabelURL
if not jp2HasGeoData:
raise Exception(
'No geo data in JP2 and no Label file! Cannot handle this!'
)
print 'Generating a fake LBL file to proceed...'
localLabelPath = writeFakeLabelFromJp2(localFilePath)
fakeLabelFile = True
# At this point we always have a label file but it may be fake
# Some images are missing geo information but we can add it from the label file
localImageIsTiff = False
localImagePath = localFilePath
if not jp2HasGeoData:
print 'Correcting JP2 file with no geo information!!!!'
# Correct the local file, then remove the old (bad) file
outputPrefix = localFilePath[0:-4]
localImagePath = correctAndCropImage(localFilePath, localLabelPath,
outputPrefix)
print 'Generated ' + localImagePath
if localFilePath != localImagePath:
print 'Deleting JP2 file without metadata!'
os.remove(localFilePath)
localImageIsTiff = (localImagePath[-4:] == ".TIF")
if not localImageIsTiff:
# Call code to fix the header information in the JP2 file!
cmd = FIX_JP2_TOOL + ' ' + localImagePath
print cmd
os.system(cmd)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath)
(width, height) = IrgIsisFunctions.getImageSize(localImagePath)
if (projType == 'POLAR STEREOGRAPHIC'
) and False: #(width < MAX_POLAR_UPLOAD_WIDTH):
# Google has trouble digesting these files so handle them differently.
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!')
print 'Special handling for POLAR STEROGRAPHIC image!'
if fakeLabelFile:
print 'Cannot reprocess polar image without a label file!'
print 'All we can do is upload the file and hope for the best.'
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# Compute how many chunks are needed for this image
numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE)
# Determine which chunk this DB entry is for
chunkNum = getChunkNum(setName)
print 'This is chunk number ' + str(chunkNum)
if chunkNum >= numChunks: # Check for chunk number error
raise Exception('Illegal chunk number: ' + setName)
# If this is the main DB entry, we need to make sure the other DB entries exist!
if chunkNum == 0:
# Go ahead and try to add each chunk, the call will only go through if it does not already exist.
for i in range(1, numChunks):
chunkSetName = makeChunkSetName(setName, i)
print 'Add chunk set name to DB: ' + chunkSetName
common.addDataRecord(db, common.SENSOR_TYPE_HiRISE,
subtype, chunkSetName, remoteURL)
raise Exception('DEBUG')
# Now actually generate the desired chunk
# - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it.
fileBasePath = os.path.splitext(localImagePath)[0]
localImgPath = fileBasePath + '.IMG'
localChunkPrefix = fileBasePath + '_' + str(chunkNum)
chunkBB = getChunkBoundingBox(width, height, chunkNum)
localChunkPath = correctAndCropImage(localImagePath,
localLabelPath,
localChunkPrefix, chunkBB)
# Just use the same label file, we don't care if the DB has per-chunk boundaries.
return [localChunkPath, localLabelPath]
else: # A normal, non-polar file.
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# TODO: Handle POLAR DEMS
else: # Handle DEMs
# For DEMs there is no label file
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
if not os.path.exists(localFilePath):
# Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(
localFilePath): # Make sure we got the file
raise Exception('Unable to download from URL: ' + remoteURL)
# Generate a header file from the IMG file
localLabelPath = localFilePath[:-4] + '.LBL'
cmd = 'head -n 90 ' + localFilePath + ' > ' + localLabelPath
print cmd
os.system(cmd)
# Check if this is a polar stereographic image
isPolar = False
f = open(localLabelPath)
for line in f:
if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC"
in line):
isPolar = True
print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY'
break
#os.remove(localFilePath)
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!')
f.close()
# Convert from IMG to TIF
tiffFilePath = localFilePath[:-4] + '.TIF'
if not os.path.exists(tiffFilePath):
cmd = 'gdal_translate -of GTiff ' + localFilePath + ' ' + tiffFilePath
print cmd
os.system(cmd)
if not isPolar: # Only EQC files need to be corrected
# Correct projected coordinates problems
cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath
print cmd
os.system(cmd)
os.remove(localFilePath) # Clean up source image
return [tiffFilePath, localLabelPath]
def main():
print '#################################################################################'
print "Running lronac2dem.py"
try:
try:
usage = "usage: lronac2dem.py [--output <path>][--manual]\n "
parser = optparse.OptionParser(usage=usage)
parser.set_defaults(keep=False)
inputGroup = optparse.OptionGroup(parser, 'Input Paths')
inputGroup.add_option("--left", dest="leftPath", help="Path to LE .IMG file")
inputGroup.add_option("--right", dest="rightPath", help="Path to RE .IMG file")
inputGroup.add_option("--stereo-left", dest="stereoLeft",
help="Path to LE .IMG file with overlapping view of --left file")
inputGroup.add_option("--stereo-right", dest="stereoRight",
help="Path to RE .IMG file with overlapping view of --right file")
inputGroup.add_option("--lola", dest="lolaPath", help="Path to LOLA DEM")
inputGroup.add_option("--asu", dest="asuPath", help="Path to ASU DEM")
inputGroup.add_option("--node-file", dest="nodeFilePath",
help="Path to list of available computing nodes")
parser.add_option_group(inputGroup)
# The default working directory path is kind of ugly...
parser.add_option("--workDir", dest="workDir", help="Folder to store temporary files in")
parser.add_option("--outputPrefix", dest="outputPrefix", help="Output prefix.")
parser.add_option("--crop", dest="cropAmount",
help="Crops the output image to reduce processing time.")
parser.add_option("--manual", action="callback", callback=man,
help="Read the manual.")
parser.add_option("--keep", action="store_true", dest="keep",
help="Do not delete the temporary files.")
(options, args) = parser.parse_args()
if not options.leftPath:
parser.error("Need left input path")
if not options.rightPath:
parser.error("Need right input path")
if not options.stereoLeft:
parser.error("Need stereo left input path")
if not options.stereoRight:
parser.error("Need stereo right input path")
if not options.lolaPath:
parser.error("Need LOLA data path")
if not options.outputPrefix:
parser.error("Need output prefix")
except optparse.OptionError, msg:
raise Usage(msg)
startTime = time.time()
# Set up the output folders
outputFolder = os.path.dirname(options.outputPrefix)
inputBaseName = os.path.basename(options.leftPath)
tempFolder = outputFolder + '/' + inputBaseName + '_stereoCalibrationTemp/'
if (options.workDir):
tempFolder = options.workDir
IrgFileFunctions.createFolder(outputFolder)
hadToCreateTempFolder = not os.path.exists(tempFolder)
IrgFileFunctions.createFolder(tempFolder)
# Set up logging
logPath = options.outputPrefix + '-Log.txt'
logging.basicConfig(filename=logPath,level=logging.INFO)
# These are the output mosaic paths
filename = os.path.splitext(options.leftPath)[0] + '.correctedMosaic.cub'
mainMosaicPath = os.path.join(tempFolder, os.path.basename(filename))
filename = os.path.splitext(options.stereoLeft)[0] + '.correctedMosaic.cub'
stereoMosaicPath = os.path.join(tempFolder, os.path.basename(filename))
# List of all the output files that will be created by makeDemAndCompare.py
demPath = options.outputPrefix + '-DEM.tif'
intersectionErrorPath = options.outputPrefix + '-IntersectionErr.tif'
hillshadePath = options.outputPrefix + '-Hillshade.tif'
colormapPath = options.outputPrefix + '-Colormap.tif'
colormapLegendPath = options.outputPrefix + '-ColormapLegend.csv'
confidencePath = options.outputPrefix + '-Confidence.tif'
confidenceLegendPath = options.outputPrefix + '-ConfidenceLegend.csv'
mapProjectLeftPath = options.outputPrefix + '-MapProjLeft.tif'
mapProjectRightPath = options.outputPrefix + '-MapProjRight.tif'
compressedInputPath = options.outputPrefix + '-CompressedInputs.tar.bz2'
compressedOutputPath = options.outputPrefix + '-CompressedOutputs.tar'
compressedDebugPath = options.outputPrefix + '-CompressedDiagnostics.tar.bz2'
# List of .LBL text files to be created
demLabelPath = options.outputPrefix + '-DEM.LBL'
intersectionErrorLabelPath = options.outputPrefix + '-IntersectionErr.LBL'
hillshadeLabelPath = options.outputPrefix + '-Hillshade.LBL'
colormapLabelPath = options.outputPrefix + '-Colormap.LBL'
confidenceLabelPath = options.outputPrefix + '-Confidence.LBL'
mapProjectLeftLabelPath = options.outputPrefix + '-MapProjLeft.LBL'
mapProjectRightLabelPath = options.outputPrefix + '-MapProjRight.LBL'
# List of diagnostic files we want to archive
# Currently specified below. Should be expanded.
# Call lronac2refinedMosaics.py if outputs do not already exist
if ( (not os.path.exists(mainMosaicPath) or not os.path.exists(stereoMosaicPath))
and not os.path.exists(demPath) ):
refineTemp = os.path.join(tempFolder, 'refinement')
cmdArgs = ['--left', options.leftPath,
'--right', options.rightPath,
'--stereo-left', options.stereoLeft,
'--stereo-right', options.stereoRight,
'--lola', options.lolaPath,
'--workDir', refineTemp,
'--output-folder', tempFolder,
'--log-path', logPath]
if options.keep:
cmdArgs.append('--keep')
print cmdArgs
lronac2refinedMosaics.main(cmdArgs)
# Copy the pc_align log file to the output folder
pcAlignLogPath = os.path.join(tempFolder, 'pcAlignLog.txt')
shutil.copyfile(pcAlignLogPath, options.outputPrefix + '-PcAlignLog.txt')
# Check that we successfully created the output files
if (not os.path.exists(mainMosaicPath) or not os.path.exists(stereoMosaicPath)):
raise Exception('lronac2refinedMosaics failed to produce mosaics!')
# Call makeDemAndCompare.py (This won't do much if outputs already exist)
cmdArgs = ['--left', mainMosaicPath,
'--right', stereoMosaicPath,
'--lola', options.lolaPath,
'--workDir', tempFolder,
'--prefix', options.outputPrefix,
'--log-path', logPath]
if options.keep:
cmdArgs.append('--keep')
if options.asuPath:
cmdArgs.append('--asu')
cmdArgs.append(options.asuPath)
if options.cropAmount:
cmdArgs.append('--crop')
cmdArgs.append(str(options.cropAmount))
if options.nodeFilePath:
cmdArgs.append('--node-file')
cmdArgs.append(options.nodeFilePath)
print cmdArgs
makeDemAndCompare.main(cmdArgs)
print 'Finished image processing, setting up final output files...'
logging.info('Started label file generation')
# Get the data set name
if (options.asuPath): # If ASU has a name for this set, use it
startPos = options.asuPath.rfind('_') + 1
stopPos = options.asuPath.rfind('.') - 1
dataSetName = options.asuPath[startPos:stopPos]
else: # Generate a data set in format NAC_DTM_MXXXXXX_MXXXXXXX
dataSetName = IsisTools.makeDataSetName(options.leftPath, options.stereoLeft)
# Get some data for the label file
thisFilePath = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'lronac2dem.py')
versionId = IrgFileFunctions.getLastGitTag(thisFilePath)
demDescription = 'High-resolution NAC digital terrain models in GeoTIFF format. DEM is IEEE floating point TIFF, 32 bits/sample, 1 samples/pixel in single image plane configuration. The NoDATA value is -3.40282266e+038.'
hillshadeDescription = 'Shaded-relief derived from NAC digital terrain models. Image is a TIFF image, 8 bits/sample, 1 samples/pixel in single image plane configuration.'
colormapDescription = 'Color shaded-relief derived from NAC digital terrain models. Image is 3-channel RGB TIFF, 8 bits/sample, 3 samples/pixel in single image plane configuration.'
confidenceDescription = 'Discretized visualization of intersection error in NAC digital terrain model. Image is a TIFF image, 8 bits/sample, 1 samples/pixel in single image plane configuration.'
mapProjectDescription = 'High-resolution NAC orthoprojected image in GeoTIFF format. Image is IEEE floating point TIFF, 32 bits/sample, 1 samples/pixel in single image plane configuration. The NoDATA value is -3.40282266e+038.'
# Copy the colormap legend file in to the colormap label file!
colormapLegendText = ('\n/* Colormap legend information: */\n')
colormapLegendFile = open(colormapLegendPath, 'r')
for line in colormapLegendFile:
colormapLegendText = colormapLegendText + '/* ' + line.strip() + ' */\n'
# Copy the confidence legend file in to the confidence label file!
confidenceLegendText = ('\n/* Confidence legend information: */\n')
confidenceLegendFile = open(confidenceLegendPath, 'r')
for line in confidenceLegendFile:
confidenceLegendText = confidenceLegendText + '/* ' + line.strip() + ' */\n'
# Generate label files for each of the output files
writeLabelFile(demPath, demLabelPath, dataSetName, versionId, demDescription)
writeLabelFile(hillshadePath, hillshadeLabelPath, dataSetName, versionId, hillshadeDescription)
writeLabelFile(colormapPath, colormapLabelPath, dataSetName, versionId, colormapDescription, colormapLegendText)
writeLabelFile(confidencePath, confidenceLabelPath, dataSetName, versionId, confidenceDescription, confidenceLegendText)
writeLabelFile(mapProjectLeftPath, mapProjectLeftLabelPath, dataSetName, versionId, mapProjectDescription)
writeLabelFile(mapProjectRightPath, mapProjectRightLabelPath, dataSetName, versionId, mapProjectDescription)
logging.info('Done with label files, compressing results.')
# Compress the input files to save disk space
if not os.path.exists(compressedInputPath):
listOfInputFiles = [options.leftPath, options.rightPath, options.stereoLeft, options.stereoRight, options.lolaPath]
if options.asuPath: # Handle optional input
listOfInputFiles.append(options.asuPath)
IrgFileFunctions.tarFileList(listOfInputFiles, compressedInputPath, True)
# Tar the output files to save file count (they are already compressed)
if not os.path.exists(compressedOutputPath):
listOfDeliveryFiles = [demPath, hillshadePath, colormapPath, confidencePath, mapProjectLeftPath, mapProjectRightPath,
demLabelPath, hillshadeLabelPath, colormapLabelPath, confidenceLabelPath, mapProjectLeftLabelPath, mapProjectRightLabelPath]
# The file names as they should appear in the tar file
baseTarPath = os.path.join(outputFolder, dataSetName + '_')
demTarBase = baseTarPath + 'DEM'
hillTarBase = baseTarPath + 'Hillshade'
colorTarBase = baseTarPath + 'Colormap'
confTarBase = baseTarPath + 'Confidence'
leftTarBase = baseTarPath + 'MapProjLeft'
rightTarBase = baseTarPath + 'MapProjRight'
listOfTarDeliveryFiles = [demTarBase+'.TIF', hillTarBase+'.TIF', colorTarBase+'.TIF', confTarBase+'.TIF', leftTarBase+'.TIF', rightTarBase+'.TIF',
demTarBase+'.LBL', hillTarBase+'.LBL', colorTarBase+'.LBL', confTarBase+'.LBL', leftTarBase+'.LBL', rightTarBase+'.LBL']
# This function can handle renaming the files as they get put in the tar.
IrgFileFunctions.tarFileList(listOfDeliveryFiles, compressedOutputPath, False, listOfTarDeliveryFiles)
# Compress the diagnostic files to save disk space
if not os.path.exists(compressedDebugPath):
listOfDebugFiles = [options.outputPrefix + '-Log.txt',
options.outputPrefix + '-PcAlignLog.txt',
options.outputPrefix + '-LOLA_diff_points.csv',
options.outputPrefix + '-LOLA_diff_points.kml',]
IrgFileFunctions.tarFileList(listOfDebugFiles, compressedDebugPath, True)
if not options.keep:
pass
#print 'Deleting temporary files'
#IrgFileFunctions.removeIfExists(mainMosaicPath)
#IrgFileFunctions.removeIfExists(stereoMosaicPath)
endTime = time.time()
logging.info('lronac2dem finished in %f seconds', endTime - startTime)
print "Finished in " + str(endTime - startTime) + " seconds."
print '#################################################################################'
return 0
def functionStartupCheck():
# These calls will raise an exception if the tool is not found
IrgFileFunctions.checkIfToolExists('lola_compare')
IrgFileFunctions.checkIfToolExists('parallel_stereo')
IrgFileFunctions.checkIfToolExists('parallel_mapproject.py')
IrgFileFunctions.checkIfToolExists('point2dem')
IrgFileFunctions.checkIfToolExists('hillshade')
IrgFileFunctions.checkIfToolExists('colormap')
IrgFileFunctions.checkIfToolExists('crop')
IrgFileFunctions.checkIfToolExists('tar')
IrgFileFunctions.checkIfToolExists('gdal_translate')
IrgFileFunctions.checkIfToolExists('maskFromIntersectError')
return True
