def removeCompressedOutputs(dataSetName, dryRun=False):
"""Removes the large local output files in a completed run"""
# Make sure the data set is complete before removing any files
ds = DataSet(dataSetName)
if not ds.isComplete():
raise Exception('Attempting to clear compressed on incomplete data set: ' + dataSetName)
localFolder = ds.getLocalFolder()
compressedFilePath = os.path.join(localFolder, 'results/output-CompressedOutputs.tar')
if not os.path.exists(compressedFilePath):
raise Exception('Attempting to clear compressed but output TAR file is missing!')
# Each of these files will be deleted
filesToDelete = ['output-Colormap.tif',
'output-Confidence.tif',
'output-DEM.tif',
'output-Hillshade.tif',
'output-MapProjLeft.tif',
'output-MapProjRight.tif',
'output-MapProjLeftUint8.tif', # Files after here are not compressed but can
'output-MapProjRightUint8.tif', # quickly be regenerated from compressed files.
'output-IntersectionErrorX.tif',
'output-IntersectionErrorY.tif',
'output-IntersectionErrorZ.tif']
for f in filesToDelete: # For each file in the list above
inputPath = os.path.join(localFolder, 'results/' + f)
#print 'Deleting file ' + inputPath
if dryRun: # Print output action
print '- rm ' + inputPath
else: # Actually delete the file
IrgFileFunctions.removeIfExists(inputPath)
def flagIncompleteDataSets(dryRun=False):
"""Flags incomplete data sets for repeat processing"""
if not os.path.exists(INCOMPLETE_STATUS_PATH):
raise Exception('Incomplete file list not present, run --check-local to generate it.')
# Loop through all data sets in the incomplete folder
incompleteFile = open(INCOMPLETE_STATUS_PATH, 'r')
for line in incompleteFile:
entries = line.split(' ')
dataSetName = entries[0]
# Get the status of this data set
ds = DataSet(dataSetName)
# Check that it is actually incomplete
if ds.status != DataSet.INCOMPLETE:
print 'WARNING: non-incomplete set found in incomplete file:'
print line
continue
# For incomplete data sets, remove the downloadLog.txt file
# - This will cause the auto-processor code to run the data set again.
flagFile = os.path.join(ds.getLocalFolder(), 'downloadLog.txt')
if dryRun:
print '- Remove file ' + flagFile
else: # Actually removet the file
IrgFileFunctions.removeIfExists(flagFile)
# Need to ru-run the update function to update this file
incompleteFile.close()
def makeSpkSetupFile(leapSecondFilePath, outputPath):
"""Creates the required mkspk setup file if it does not already exist"""
# If the file already exists, delete it and rewrite it.
IrgFileFunctions.removeIfExists(outputPath)
# print 'Generating LRONAC compatible .pvl file ' + halfResFilePath
f = open(outputPath, 'w')
f.write("\\begindata\n")
f.write("INPUT_DATA_TYPE = 'STATES'\n")
f.write("OUTPUT_SPK_TYPE = 13\n")
f.write("OBJECT_ID = -85\n") # LRO
f.write("CENTER_ID = 301\n") # Moon
f.write("REF_FRAME_NAME = 'J2000'\n")
f.write("PRODUCER_ID = 'Lronac Pipeline'\n")
f.write("DATA_ORDER = 'epoch x y z vx vy vz'\n")
f.write("DATA_DELIMITER = ','\n")
f.write("LEAPSECONDS_FILE = '" + leapSecondFilePath + "'\n")
f.write("LINES_PER_RECORD = 1\n")
f.write("TIME_WRAPPER = '# ETSECONDS'\n")
#f.write("EPOCH_STR_LENGTH = 16\n")
f.write("INPUT_DATA_UNITS = ('ANGLES=DEGREES' 'DISTANCES=km')\n")
f.write("POLYNOM_DEGREE = 11\n")
f.write("SEGMENT_ID = 'SPK_STATES_13'\n")
# f.write("INPUT_DATA_FILE = 'spkDataFile.txt'")
# f.write("OUTPUT_SPK_FILE = '/home/smcmich1/testSpkFile.bsp'")
f.write("\\begintext\n")
f.close()
outputPathStereo, stereoMosaicWorkDir, carry))
print 'Starting mosaic call threads'
leftThread.start()
rightThread.start()
print 'Waiting for mosaic threads to complete...'
leftThread.join()
rightThread.join()
mosaicTime = time.time()
logging.info('Mosaics finished in %f seconds', mosaicTime - noprojTime)
# Clean up temporary files
if not options.keep:
print 'Deleting temporary files'
IrgFileFunctions.removeIfExists(leftCorrectedPath)
IrgFileFunctions.removeIfExists(rightCorrectedPath)
IrgFileFunctions.removeIfExists(leftStereoCorrectedPath)
IrgFileFunctions.removeIfExists(rightStereoCorrectedPath)
IrgFileFunctions.removeIfExists(pvlPath)
IrgFileFunctions.removeIfExists(leftNoprojPath)
IrgFileFunctions.removeIfExists(rightNoprojPath)
IrgFileFunctions.removeIfExists(leftStereoNoprojPath)
IrgFileFunctions.removeIfExists(rightStereoNoprojPath)
#IrgFileFunctions.removeFolderIfExists(mainMosaicWorkDir)
#IrgFileFunctions.removeFolderIfExists(stereoMosaicWorkDir)
#if (hadToCreateTempFolder):
# IrgFileFunctions.removeFolderIfExists(tempFolder)
## Remove all the .kml files
#fileList = [ f for f in os.listdir(tempFolder) if f.endswith(".kml") ]
def archiveDataSet(dataSetName, deleteLocalFiles=False, dryRun=False):
"""Archives a data set to long term storage on Lou"""
# The local files to be archived are already in tar files
# Make sure the data set is complete before archiving it
ds = DataSet(dataSetName)
if not ds.isComplete():
raise Exception('Attempting to archive incomplete data set: ' + dataSetName)
localFolder = ds.getLocalFolder()
resultsFolder = os.path.join(localFolder, 'results/')
# In order to archive the file it just has to be moved over to the Lou filesystem.
# - Tape archiving is performed automatically when needed.
# - All tar files are stored in the same directory
filesToArchive = ['output-CompressedOutputs.tar']
for f in filesToArchive:
# Get storage name to use
archiveName = 'NAC_DTM_' + dataSetName + '_CompressedOutputs.tar'
inputPath = os.path.join(resultsFolder, f)
outputPath = os.path.join(LOU_STORAGE_PATH, archiveName)
# Move the file over to Lou
#cmd = ['shiftc', '--wait', '--verify', inputPath, outputPath]
cmd = 'shiftc --wait --verify ' + inputPath + ' ' + outputPath
print cmd
if not dryRun: # Actually transfer the file
os.system(cmd)
#p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
#textOutput, err = p.communicate()
#print textOutput
#print 'ppp'
#print err
#print textOutput.find('done')
#if not ('done' in textOutput):
# raise Exception('Error transferring file!')
# Verify that the file was actually transferred
resp = subprocess.call(['ssh', LOU_SSH_TARGET, 'test -e ' + pipes.quote(LOU_LOCAL_STORAGE_PATH)])
if resp == 0:
print 'shiftc file transfer confirmed.'
else:
raise Exception('Failed to transfer file using shiftc!')
if deleteLocalFiles: # Remove almost all files
# Keep some result files and move them to the main data set directory
filesToKeep = ['output-LOLA_diff_stats.txt',
'output-Log.txt',
'output-PcAlignLog.txt',
'output-CompressedInputs.tar.bz2',
'output-CompressedDiagnostics.tar.bz2']
for f in filesToKeep:
inputPath = os.path.join(resultsFolder, f)
outputPath = os.path.join(localFolder, f)
print 'mv ' + inputPath + ' ' + outputPath
if not dryRun: # Actually move the files
shutil.move(inputPath, outputPath)
# Now that we have saved a few files, delete everything else
cmdRmImg = 'rm ' + localFolder + '/*.IMG'
rdrPath = os.path.join(localFolder, 'lolaRdrPoints.csv')
prtPath = os.path.join(localFolder, 'print.prt')
#logPath = os.path.join(localFolder, 'stdOutputLog.txt')
workDir = os.path.join(localFolder, 'workDir')
resultsDir = os.path.join(localFolder, 'results')
print cmdRmImg
print 'rm ' + rdrPath
print 'rm ' + prtPath
#print 'rm ' + logPath
print 'rm -rf ' + workDir
print 'rm -rf ' + resultsDir
if not dryRun: # Actually delete the files
os.system(cmdRmImg)
IrgFileFunctions.removeIfExists(rdrPath)
IrgFileFunctions.removeIfExists(prtPath)
#IrgFileFunctions.removeIfExists(logPath)
IrgFileFunctions.removeFolderIfExists(workDir)
IrgFileFunctions.removeFolderIfExists(resultsDir)
def main(argsIn):
print '#################################################################################'
print "Running makeDemAndCompare.py"
try:
try:
usage = "usage: makeDemAndCompare.py [--output <path>][--manual]\n "
parser = optparse.OptionParser(usage=usage)
inputGroup = optparse.OptionGroup(parser, 'Input Paths')
inputGroup.add_option("--left", dest="leftPath", help="Path to left cube file")
inputGroup.add_option("--right", dest="rightPath", help="Path to right cube file")
inputGroup.add_option("--lola", dest="lolaPath", help="Path to LOLA DEM")
inputGroup.add_option("--asu", dest="asuPath", help="Path to ASU DEM")
inputGroup.add_option("--node-file", dest="nodeFilePath",
help="Path to file containing list of available nodes")
parser.add_option_group(inputGroup)
# The default working directory path is kind of ugly...
parser.add_option("--workDir", dest="workDir", help="Folder to store temporary files in")
parser.add_option("--prefix", dest="prefix", help="Output prefix.")
parser.add_option("--log-path", dest="logPath",
help="Where to write the output log file.")
parser.add_option("--crop", dest="cropAmount",
help="Crops the output image to reduce processing time.")
parser.add_option("--manual", action="callback", callback=man,
help="Read the manual.")
parser.add_option("--keep", action="store_true", dest="keep",
help="Do not delete the temporary files.")
(options, args) = parser.parse_args(argsIn)
if not options.leftPath:
parser.error("Need left input path")
if not options.rightPath:
parser.error("Need right input path")
if not options.prefix:
parser.error("Need output prefix")
except optparse.OptionError, msg:
raise Usage(msg)
print "Beginning processing....."
startTime = time.time()
# Make sure we have all the functions we need
functionStartupCheck()
# Set this to true to force steps after it to activate
carry = False
# Set up the output folders
outputFolder = os.path.dirname(options.prefix)
inputBaseName = os.path.basename(options.leftPath)
tempFolder = outputFolder + '/' + inputBaseName + '_stereoCalibrationTemp/'
if (options.workDir):
tempFolder = options.workDir
if not os.path.exists(outputFolder):
os.mkdir(outputFolder)
hadToCreateTempFolder = not os.path.exists(tempFolder)
if not os.path.exists(tempFolder):
os.mkdir(tempFolder)
# Set up logging
if not options.logPath:
options.logPath = options.prefix + '-Log.txt'
logging.basicConfig(filename=options.logPath,level=logging.INFO)
# Go ahead and set up all the output paths
# -- Deliverables
demPath = options.prefix + '-DEM.tif'
intersectionErrorPath = options.prefix + '-IntersectionErr.tif'
hillshadePath = options.prefix + '-Hillshade.tif'
colormapPath = options.prefix + '-Colormap.tif'
colormapLegendPath = options.prefix + '-ColormapLegend.csv'
mapProjectLeftPath = options.prefix + '-MapProjLeft.tif'
mapProjectRightPath = options.prefix + '-MapProjRight.tif'
confidenceLevelPath = options.prefix + '-Confidence.tif'
confidenceLegendPath = options.prefix + '-ConfidenceLegend.csv'
# -- Diagnostic
intersectionViewPathX = options.prefix + '-IntersectionErrorX.tif'
intersectionViewPathY = options.prefix + '-IntersectionErrorY.tif'
intersectionViewPathZ = options.prefix + '-IntersectionErrorZ.tif'
lolaDiffStatsPath = options.prefix + '-LOLA_diff_stats.txt'
lolaDiffPointsPath = options.prefix + '-LOLA_diff_points.csv'
lolaAsuDiffStatsPath = options.prefix + '-ASU_LOLA_diff_stats.txt'
lolaAsuDiffPointsPath = options.prefix + '-ASU_LOLA_diff_points.csv'
mapProjectLeftUint8Path = options.prefix + '-MapProjLeftUint8.tif'
mapProjectRightUint8Path = options.prefix + '-MapProjRightUint8.tif'
# If specified, crop the inputs that will be passed into the stereo function to reduce processing time
mainMosaicCroppedPath = os.path.join(tempFolder, 'mainMosaicCropped.cub')
stereoMosaicCroppedPath = os.path.join(tempFolder, 'stereoMosaicCropped.cub')
if options.cropAmount and (options.cropAmount > 0):
# Verify input files are present
if not os.path.exists(options.leftPath):
raise Exception('Input file ' + options.leftPath + ' not found!')
if not os.path.exists(options.rightPath):
raise Exception('Input file ' + options.rightPath + ' not found!')
if (not os.path.exists(mainMosaicCroppedPath)) or carry:
cmd = ('crop from= ' + options.leftPath + ' to= ' + mainMosaicCroppedPath +
' nlines= ' + str(options.cropAmount))# + ' line=24200')
print cmd
os.system(cmd)
if (not os.path.exists(stereoMosaicCroppedPath) or carry):
cmd = ('crop from= ' + options.rightPath + ' to= ' + stereoMosaicCroppedPath +
' nlines= ' + str(options.cropAmount))# + ' line=24200')
print cmd
os.system(cmd)
options.leftPath = mainMosaicCroppedPath
options.rightPath = stereoMosaicCroppedPath
print '\n-------------------------------------------------------------------------\n'
# Call stereo to generate a point cloud from the two images
# - This step takes a really long time.
stereoOutputPrefix = os.path.join(tempFolder, 'stereoWorkDir/stereo')
stereoOutputFolder = os.path.join(tempFolder, 'stereoWorkDir')
pointCloudPath = stereoOutputPrefix + '-PC.tif'
stereoOptionString = ('--corr-timeout 400 --alignment-method AffineEpipolar --subpixel-mode ' + str(SUBPIXEL_MODE) +
' ' + options.leftPath + ' ' + options.rightPath +
' --job-size-w 4096 --job-size-h 4096 ' + # Reduce number of tile files created
' ' + stereoOutputPrefix + ' --processes 10 --threads-multiprocess 4' +
' --threads-singleprocess 32 --compute-error-vector' + ' --filter-mode 1' +
' --erode-max-size 5000 --subpixel-kernel 35 35 --subpixel-max-levels 0')
if (not os.path.exists(pointCloudPath) and not os.path.exists(demPath)) or carry:
# Verify input files are present
if not os.path.exists(options.leftPath):
raise Exception('Input file ' + options.leftPath + ' not found!')
if not os.path.exists(options.rightPath):
raise Exception('Input file ' + options.rightPath + ' not found!')
cmd = ('parallel_stereo ' + stereoOptionString)
print cmd
os.system(cmd)
# Compute percentage of good pixels
percentGood = IrgAspFunctions.getStereoGoodPixelPercentage(stereoOutputPrefix)
print 'Stereo completed with good pixel percentage: ' + str(percentGood)
logging.info('Final stereo completed with good pixel percentage: %s', str(percentGood))
else:
print 'Stereo file ' + pointCloudPath + ' already exists, skipping stereo step.'
stereoTime = time.time()
logging.info('Stereo finished in %f seconds', stereoTime - startTime)
# Find out the center latitude of the mosaic
if os.path.exists(options.leftPath):
centerLat = IrgIsisFunctions.getCubeCenterLatitude(options.leftPath, tempFolder)
elif os.path.exists(demPath): # Input file has been deleted but we still have the info
demInfo = IrgGeoFunctions.getImageGeoInfo(demPath, False)
centerLat = demInfo['standard_parallel_1']
else:
raise Exception("Can't delete the input files before creating the DEM!")
# Generate a DEM
if (not os.path.exists(demPath)) or carry:
# Equirectangular style projection
# - Latitude of true scale = center latitude = lat_ts
# - Latitude of origin = 0 = lat+0
# - Longitude of projection center = Central meridian = lon+0
cmd = ('point2dem --dem-hole-fill-len 15 --remove-outliers --errorimage -o ' + options.prefix + ' ' + pointCloudPath +
' -r moon --tr ' + str(DEM_METERS_PER_PIXEL) + ' --t_srs "+proj=eqc +lat_ts=' + str(centerLat) +
' +lat_0=0 +a='+str(MOON_RADIUS)+' +b='+str(MOON_RADIUS)+' +units=m" --nodata ' + str(DEM_NODATA))
os.system(cmd)
else:
print 'DEM file ' + demPath + ' already exists, skipping point2dem step.'
# Create a hillshade image to visualize the output
if (not os.path.exists(hillshadePath)) or carry:
cmd = 'hillshade ' + demPath + ' -o ' + hillshadePath
print cmd
os.system(cmd)
else:
print 'Output file ' + hillshadePath + ' already exists, skipping hillshade step.'
# Create a colorized version of the hillshade
# - Uses a blue-red color map from here: http://www.sandia.gov/~kmorel/documents/ColorMaps/
if (not os.path.exists(colormapPath)) or (not os.path.exists(colormapLegendPath)) or carry:
# The color LUT is kept with the source code
lutFilePath = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'colorProfileBlueRed.csv')
# Generate the initial version of colormap
colormapTempPath = options.prefix + '-ColormapTemp.tif'
cmd = 'colormap ' + demPath + ' -o ' + colormapTempPath + ' -s ' + hillshadePath + ' --lut-file ' + lutFilePath
print cmd
os.system(cmd)
# Now convert to the final output version (remove transparency layer) and remove the temp file
IrgFileFunctions.stripRgbImageAlphaChannel(colormapTempPath, colormapPath)
os.remove(colormapTempPath)
# Generate another file storing the colormap info
writeColorMapInfo(colormapPath, lutFilePath, demPath, colormapLegendPath)
else:
print 'Output file ' + colormapPath + ' already exists, skipping colormap step.'
## Create a 3d mesh of the point cloud
#meshPath = os.path.join(outputFolder, 'mesh.ive')
#meshPrefix = os.path.join(outputFolder, 'mesh')
#cmd = 'point2mesh ' + pointCloudPath + ' ' + options.leftPath + ' -o ' + meshPrefix
#if not os.path.exists(meshPath):
# print cmd
# os.system(cmd)
if not options.keep: # Remove stereo folder here to cut down on file count before mapproject calls
IrgFileFunctions.removeFolderIfExists(stereoOutputFolder)
# Convert the intersection error to a viewable format
cmdX = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 1 ' + intersectionErrorPath + ' ' + intersectionViewPathX
cmdY = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 2 ' + intersectionErrorPath + ' ' + intersectionViewPathY
cmdZ = 'gdal_translate -ot byte -scale 0 10 0 255 -outsize 50% 50% -b 3 ' + intersectionErrorPath + ' ' + intersectionViewPathZ
if not os.path.exists(intersectionViewPathX) or carry:
print cmdX
os.system(cmdX)
if not os.path.exists(intersectionViewPathY) or carry:
print cmdY
os.system(cmdY)
if not os.path.exists(intersectionViewPathZ) or carry:
print cmdZ
os.system(cmdZ)
# Generate a confidence plot from the intersection error
if not os.path.exists(confidenceLevelPath):
thresholdString = str(PIXEL_ACCURACY_THRESHOLDS)[1:-1].replace(',', '')
cmd = ('maskFromIntersectError ' + intersectionErrorPath + ' ' + confidenceLevelPath
+ ' --legend ' + confidenceLegendPath +
' --scaleOutput --thresholds ' + thresholdString)
print cmd
os.system(cmd)
hillshadeTime = time.time()
logging.info('DEM and hillshade finished in %f seconds', hillshadeTime - stereoTime)
# Call script to compare LOLA data with the DEM
if options.lolaPath:
compareDemToLola(options.lolaPath, demPath, lolaDiffStatsPath, lolaDiffPointsPath, carry)
# Call script to compare LOLA data with the ASU DEM
if options.asuPath:
compareDemToLola(options.lolaPath, options.asuPath, lolaAsuDiffStatsPath, lolaAsuDiffPointsPath, carry)
# Generate a map projected version of the left and right images
# - This step is done last since it is so slow!
mapProjectImage(options.leftPath, demPath, mapProjectLeftPath, MAP_PROJECT_METERS_PER_PIXEL, centerLat, options.nodeFilePath, carry)
mapProjectImage(options.rightPath, demPath, mapProjectRightPath, MAP_PROJECT_METERS_PER_PIXEL, centerLat, options.nodeFilePath, carry)
# Generate 8 bit versions of the mapproject files for debugging
cmdLeft = 'gdal_translate -scale -ot byte ' + mapProjectLeftPath + ' ' + mapProjectLeftUint8Path
cmdRight = 'gdal_translate -scale -ot byte ' + mapProjectRightPath + ' ' + mapProjectRightUint8Path
if not os.path.exists(mapProjectLeftUint8Path) or carry:
print cmdLeft
os.system(cmdLeft)
if not os.path.exists(mapProjectRightUint8Path) or carry:
print cmdRight
os.system(cmdRight)
mapProjectTime = time.time()
logging.info('Map project finished in %f seconds', mapProjectTime - hillshadeTime)
# Clean up temporary files
if not options.keep:
print 'Removing temporary files'
IrgFileFunctions.removeIfExists(mainMosaicCroppedPath)
IrgFileFunctions.removeIfExists(stereoMosaicCroppedPath)
#IrgFileFunctions.removeIntermediateStereoFiles(stereoOutputPrefix) # Limited clear
IrgFileFunctions.removeFolderIfExists(stereoOutputFolder) # Larger clear
#if (hadToCreateTempFolder): Not done since stereo output needs to be retained
# IrgFileFunctions.removeFolderIfExists(tempFolder)
endTime = time.time()
logging.info('makeDemAndCompare.py finished in %f seconds', endTime - startTime)
print "Finished in " + str(endTime - startTime) + " seconds."
print '#################################################################################'
return 0
