def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
# Images with a center over this latitude will use polar stereographic projection
# instead of simple cylindrical projection.
HIGH_LATITUDE_CUTOFF = 65 # Degrees
asuImagePath = os.path.join(workDir, setName + '_noGeo.jp2') # Map projected image from ASU
asuLabelPath = os.path.join(workDir, setName + '_noGeo.lbl') # Label from ASU
edrPath = os.path.join(workDir, setName + '.IMG') # Raw image from PDS
timePath = os.path.join(workDir, setName + '.time') # Contains only the file capture time string
#cubPath = os.path.join(workDir, setName + '.cub') # Output of mroctx2isis
#calPath = os.path.join(workDir, setName + '.cal.cub') # Output of ctxcal
mapPath = os.path.join(workDir, setName + '.map.cub') # Output of cam2map
mapLabelPath = os.path.join(workDir, setName + '.map.pvl') # Specify projection to cam2map
# Generate the remote URLs from the data prefix and volume stored in these parameters
asuImageUrl, asuLabelUrl, edrUrl = generatePdsPath(setName, subtype)
if True: # Map project the EDR ourselves <-- This takes way too long!
print 'Projecting the EDR image using ISIS...'
localFilePath = os.path.join(workDir, setName + '.tif') # The output file we will upload
# Check if this is a "flat" calibration image
badImage = checkForBadFile(edrUrl)
if badImage:
raise Exception('TODO: Remove bad images from the DB!')
if not os.path.exists(edrPath):
# Download the EDR file
cmd = 'wget ' + edrUrl + ' -O ' + edrPath
print cmd
os.system(cmd)
# Extract the image capture time from the .IMG file
if not os.path.exists(timePath):
timeString = getCreationTimeHelper(edrPath)
f = open(timePath, 'w')
f.write(timeString)
f.close()
# Convert and apply calibration to the CTX file
calPath = IrgIsisFunctions.prepareCtxImage(edrPath, workDir, False)
## Find out the center latitude of the file and determine if it is high latitude
centerLat = IrgIsisFunctions.getCubeCenterLatitude(calPath, workDir)
print centerLat
highLat = abs(float(centerLat)) > HIGH_LATITUDE_CUTOFF
if True:#not os.path.exists(mapLabelPath):
# Generate the map label file
generateDefaultMappingPvl(mapLabelPath, highLat)
if True:#not os.path.exists(mapPath):
# Generate the map projected file
cmd = ['timeout', '6h', 'cam2map', 'matchmap=','False', 'from=', calPath, 'to=', mapPath, 'map=', mapLabelPath]
print cmd
#os.system(cmd)
p = subprocess.Popen(cmd)
p.communicate()
if (p.returncode != 0):
raise Exception('Error or timeout running cam2map, returnCode = ' + str(p.returncode))
if True: #not os.path.exists(localFilePath):
# Generate the final image to upload
cmd = 'gdal_translate -of GTiff ' + mapPath + ' ' + localFilePath
print cmd
os.system(cmd)
# Clean up intermediate files
os.remove(mapLabelPath)
os.remove(edrPath)
os.remove(calPath)
os.remove(mapPath)
# Two local files are left around, the first should be uploaded.
return [localFilePath, timePath]
else: # Use the map projected image from the ASU web site
print 'Using ASU projected image...'
localFilePath = os.path.join(workDir, setName + '.jp2') # The output file we will upload
# Note: ASU seems to be missing some files!
# We are using the label path in both projection cases
if not os.path.exists(asuLabelPath):
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuLabelPath): # Try the alternate label path
os.remove(asuLabelPath)
asuLabelUrl = asuLabelUrl.replace( '.scyl.', '.ps.')
asuLabelPath = asuLabelPath.replace('.scyl.', '.ps.')
print 'Trying alternate label path: ' + asuLabelUrl
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuLabelPath):
raise Exception('Failed to download file label at URL: ' + asuLabelUrl)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(asuLabelPath)
if projType != 'SimpleCylindrical':
print 'WARNING: projType = ' + projType
print 'Maps Engine may fail to ingest this file!'
#os.remove(asuLabelPath)
#raise Exception(projType + ' images on hold until Google fixes a bug!')
if not os.path.exists(asuImagePath):
# Download the image file
cmd = 'wget "' + asuImageUrl + '" -O ' + asuImagePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuImagePath):
raise Exception('Failed to download image file at URL: ' + asuImageUrl)
## Correct the ISIS header if needed
#fixedAsuHeaderPath = putIsisHeaderIn180(asuLabelPath)
#if (fixedAsuHeaderPath != asuLabelPath):
# os.remove(asuLabelPath) # Delete replaced header
if True: # This is fast now, so do it every time
# Correct the file - The JP2 file from ASU needs the geo data from the label file!
#cmd = 'addGeoToAsuCtxJp2.py --keep --label '+ asuLabelPath +' '+ asuImagePath +' '+ localFilePath
#print cmd
#os.system(cmd)
# TODO: Remove unnecessary image copy here
(correctedPath, sidecarPath) = addGeoDataToAsuJp2File(asuImagePath, asuLabelPath, localFilePath, keep=False)
if not IrgFileFunctions.fileIsNonZero(sidecarPath):
raise Exception('Script to add geo data to JP2 file failed!')
# Clean up
os.remove(asuImagePath)
# Three local files are left around, the first should be uploaded.
return [correctedPath, sidecarPath, asuLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
# Images with a center over this latitude will use polar stereographic projection
# instead of simple cylindrical projection.
HIGH_LATITUDE_CUTOFF = 65 # Degrees
asuImagePath = os.path.join(workDir, setName +
'_noGeo.jp2') # Map projected image from ASU
asuLabelPath = os.path.join(workDir,
setName + '_noGeo.lbl') # Label from ASU
edrPath = os.path.join(workDir, setName + '.IMG') # Raw image from PDS
timePath = os.path.join(
workDir,
setName + '.time') # Contains only the file capture time string
#cubPath = os.path.join(workDir, setName + '.cub') # Output of mroctx2isis
#calPath = os.path.join(workDir, setName + '.cal.cub') # Output of ctxcal
mapPath = os.path.join(workDir, setName + '.map.cub') # Output of cam2map
mapLabelPath = os.path.join(workDir, setName +
'.map.pvl') # Specify projection to cam2map
# Generate the remote URLs from the data prefix and volume stored in these parameters
asuImageUrl, asuLabelUrl, edrUrl = generatePdsPath(setName, subtype)
if True: # Map project the EDR ourselves <-- This takes way too long!
print 'Projecting the EDR image using ISIS...'
localFilePath = os.path.join(workDir, setName +
'.tif') # The output file we will upload
# Check if this is a "flat" calibration image
badImage = checkForBadFile(edrUrl)
if badImage:
raise Exception('TODO: Remove bad images from the DB!')
if not os.path.exists(edrPath):
# Download the EDR file
cmd = 'wget ' + edrUrl + ' -O ' + edrPath
print cmd
os.system(cmd)
# Extract the image capture time from the .IMG file
if not os.path.exists(timePath):
timeString = getCreationTimeHelper(edrPath)
f = open(timePath, 'w')
f.write(timeString)
f.close()
# Convert and apply calibration to the CTX file
calPath = IrgIsisFunctions.prepareCtxImage(edrPath, workDir, False)
## Find out the center latitude of the file and determine if it is high latitude
centerLat = IrgIsisFunctions.getCubeCenterLatitude(calPath, workDir)
print centerLat
highLat = abs(float(centerLat)) > HIGH_LATITUDE_CUTOFF
if True: #not os.path.exists(mapLabelPath):
# Generate the map label file
generateDefaultMappingPvl(mapLabelPath, highLat)
if True: #not os.path.exists(mapPath):
# Generate the map projected file
cmd = [
'timeout', '6h', 'cam2map', 'matchmap=', 'False', 'from=',
calPath, 'to=', mapPath, 'map=', mapLabelPath
]
print cmd
#os.system(cmd)
p = subprocess.Popen(cmd)
p.communicate()
if (p.returncode != 0):
raise Exception(
'Error or timeout running cam2map, returnCode = ' +
str(p.returncode))
if True: #not os.path.exists(localFilePath):
# Generate the final image to upload
cmd = 'gdal_translate -of GTiff ' + mapPath + ' ' + localFilePath
print cmd
os.system(cmd)
# Clean up intermediate files
os.remove(mapLabelPath)
os.remove(edrPath)
os.remove(calPath)
os.remove(mapPath)
# Two local files are left around, the first should be uploaded.
return [localFilePath, timePath]
else: # Use the map projected image from the ASU web site
print 'Using ASU projected image...'
localFilePath = os.path.join(workDir, setName +
'.jp2') # The output file we will upload
# Note: ASU seems to be missing some files!
# We are using the label path in both projection cases
if not os.path.exists(asuLabelPath):
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(
asuLabelPath): # Try the alternate label path
os.remove(asuLabelPath)
asuLabelUrl = asuLabelUrl.replace('.scyl.', '.ps.')
asuLabelPath = asuLabelPath.replace('.scyl.', '.ps.')
print 'Trying alternate label path: ' + asuLabelUrl
# Download the label file
cmd = 'wget "' + asuLabelUrl + '" -O ' + asuLabelPath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuLabelPath):
raise Exception('Failed to download file label at URL: ' +
asuLabelUrl)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(asuLabelPath)
if projType != 'SimpleCylindrical':
print 'WARNING: projType = ' + projType
print 'Maps Engine may fail to ingest this file!'
#os.remove(asuLabelPath)
#raise Exception(projType + ' images on hold until Google fixes a bug!')
if not os.path.exists(asuImagePath):
# Download the image file
cmd = 'wget "' + asuImageUrl + '" -O ' + asuImagePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(asuImagePath):
raise Exception('Failed to download image file at URL: ' +
asuImageUrl)
## Correct the ISIS header if needed
#fixedAsuHeaderPath = putIsisHeaderIn180(asuLabelPath)
#if (fixedAsuHeaderPath != asuLabelPath):
# os.remove(asuLabelPath) # Delete replaced header
if True: # This is fast now, so do it every time
# Correct the file - The JP2 file from ASU needs the geo data from the label file!
#cmd = 'addGeoToAsuCtxJp2.py --keep --label '+ asuLabelPath +' '+ asuImagePath +' '+ localFilePath
#print cmd
#os.system(cmd)
# TODO: Remove unnecessary image copy here
(correctedPath,
sidecarPath) = addGeoDataToAsuJp2File(asuImagePath,
asuLabelPath,
localFilePath,
keep=False)
if not IrgFileFunctions.fileIsNonZero(sidecarPath):
raise Exception('Script to add geo data to JP2 file failed!')
# Clean up
os.remove(asuImagePath)
# Three local files are left around, the first should be uploaded.
return [correctedPath, sidecarPath, asuLabelPath]
