def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
if subtype != 'DEM': # Handles RED and COLOR images
# The label file URL is the same as the image but with a different extension
remoteLabelURL = getLabelPathFromImagePath(remoteURL)
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
localLabelPath = os.path.join(workDir,
os.path.basename(remoteLabelURL))
# Retrieve the header file
if not os.path.exists(localLabelPath):
# Try to get the label locally!
pdsStart = remoteLabelURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/',
remoteLabelURL[pdsStart:])
print localPdsPath
if os.path.exists(
localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath + ' ' + localLabelPath
else: # Download the image
cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath
print cmd
os.system(cmd)
# Retrieve the image file
if not os.path.exists(localFilePath): # Need to get it from somewhere
# Try to get the image locally!
pdsStart = remoteURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:])
if os.path.exists(
localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath + ' ' + localFilePath
else: # Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(localFilePath):
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
print 'Could not find label or image file, DELETING DATA RECORD!'
common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype,
setName)
raise Exception('Unable to download from URL: ' + remoteURL)
# Check if there is geo data in the JP2 file
jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath)
# If there is no label file, try to generate an artificial one.
fakeLabelFile = False
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
#raise Exception('Unable to download from URL: ' + remoteLabelURL)
print 'WARNING: Unable to download from URL: ' + remoteLabelURL
if not jp2HasGeoData:
raise Exception(
'No geo data in JP2 and no Label file! Cannot handle this!'
)
print 'Generating a fake LBL file to proceed...'
localLabelPath = writeFakeLabelFromJp2(localFilePath)
fakeLabelFile = True
# At this point we always have a label file but it may be fake
# Some images are missing geo information but we can add it from the label file
localImageIsTiff = False
localImagePath = localFilePath
if not jp2HasGeoData:
print 'Correcting JP2 file with no geo information!!!!'
# Correct the local file, then remove the old (bad) file
outputPrefix = localFilePath[0:-4]
localImagePath = correctAndCropImage(localFilePath, localLabelPath,
outputPrefix)
print 'Generated ' + localImagePath
if localFilePath != localImagePath:
print 'Deleting JP2 file without metadata!'
os.remove(localFilePath)
localImageIsTiff = (localImagePath[-4:] == ".TIF")
if not localImageIsTiff:
# Call code to fix the header information in the JP2 file!
cmd = FIX_JP2_TOOL + ' ' + localImagePath
print cmd
os.system(cmd)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath)
(width, height) = IrgIsisFunctions.getImageSize(localImagePath)
if (projType == 'POLAR STEREOGRAPHIC'
) and False: #(width < MAX_POLAR_UPLOAD_WIDTH):
# Google has trouble digesting these files so handle them differently.
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!')
print 'Special handling for POLAR STEROGRAPHIC image!'
if fakeLabelFile:
print 'Cannot reprocess polar image without a label file!'
print 'All we can do is upload the file and hope for the best.'
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# Compute how many chunks are needed for this image
numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE)
# Determine which chunk this DB entry is for
chunkNum = getChunkNum(setName)
print 'This is chunk number ' + str(chunkNum)
if chunkNum >= numChunks: # Check for chunk number error
raise Exception('Illegal chunk number: ' + setName)
# If this is the main DB entry, we need to make sure the other DB entries exist!
if chunkNum == 0:
# Go ahead and try to add each chunk, the call will only go through if it does not already exist.
for i in range(1, numChunks):
chunkSetName = makeChunkSetName(setName, i)
print 'Add chunk set name to DB: ' + chunkSetName
common.addDataRecord(db, common.SENSOR_TYPE_HiRISE,
subtype, chunkSetName, remoteURL)
raise Exception('DEBUG')
# Now actually generate the desired chunk
# - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it.
fileBasePath = os.path.splitext(localImagePath)[0]
localImgPath = fileBasePath + '.IMG'
localChunkPrefix = fileBasePath + '_' + str(chunkNum)
chunkBB = getChunkBoundingBox(width, height, chunkNum)
localChunkPath = correctAndCropImage(localImagePath,
localLabelPath,
localChunkPrefix, chunkBB)
# Just use the same label file, we don't care if the DB has per-chunk boundaries.
return [localChunkPath, localLabelPath]
else: # A normal, non-polar file.
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# TODO: Handle POLAR DEMS
else: # Handle DEMs
# For DEMs there is no label file
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
if not os.path.exists(localFilePath):
# Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(
localFilePath): # Make sure we got the file
raise Exception('Unable to download from URL: ' + remoteURL)
# Generate a header file from the IMG file
localLabelPath = localFilePath[:-4] + '.LBL'
cmd = 'head -n 90 ' + localFilePath + ' > ' + localLabelPath
print cmd
os.system(cmd)
# Check if this is a polar stereographic image
isPolar = False
f = open(localLabelPath)
for line in f:
if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC"
in line):
isPolar = True
print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY'
break
#os.remove(localFilePath)
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!')
f.close()
# Convert from IMG to TIF
tiffFilePath = localFilePath[:-4] + '.TIF'
if not os.path.exists(tiffFilePath):
cmd = 'gdal_translate -of GTiff ' + localFilePath + ' ' + tiffFilePath
print cmd
os.system(cmd)
if not isPolar: # Only EQC files need to be corrected
# Correct projected coordinates problems
cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath
print cmd
os.system(cmd)
os.remove(localFilePath) # Clean up source image
return [tiffFilePath, localLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir):
'''Retrieves a remote file and prepares it for upload'''
#print 'Uploading file ' + setName
if subtype != 'DEM': # Handles RED and COLOR images
# The label file URL is the same as the image but with a different extension
remoteLabelURL = getLabelPathFromImagePath(remoteURL)
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
localLabelPath = os.path.join(workDir, os.path.basename(remoteLabelURL))
# Retrieve the header file
if not os.path.exists(localLabelPath):
# Try to get the label locally!
pdsStart = remoteLabelURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/', remoteLabelURL[pdsStart:])
print localPdsPath
if os.path.exists(localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath +' '+ localLabelPath
else: # Download the image
cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath
print cmd
os.system(cmd)
# Retrieve the image file
if not os.path.exists(localFilePath): # Need to get it from somewhere
# Try to get the image locally!
pdsStart = remoteURL.find('PDS')
localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:])
if os.path.exists(localPdsPath): # File available locally, just copy it!
cmd = 'cp ' + localPdsPath +' '+ localFilePath
else: # Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(localFilePath):
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
print 'Could not find label or image file, DELETING DATA RECORD!'
common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, setName)
raise Exception('Unable to download from URL: ' + remoteURL)
# Check if there is geo data in the JP2 file
jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath)
# If there is no label file, try to generate an artificial one.
fakeLabelFile = False
if not IrgFileFunctions.fileIsNonZero(localLabelPath):
#raise Exception('Unable to download from URL: ' + remoteLabelURL)
print 'WARNING: Unable to download from URL: ' + remoteLabelURL
if not jp2HasGeoData:
raise Exception('No geo data in JP2 and no Label file! Cannot handle this!')
print 'Generating a fake LBL file to proceed...'
localLabelPath = writeFakeLabelFromJp2(localFilePath)
fakeLabelFile = True
# At this point we always have a label file but it may be fake
# Some images are missing geo information but we can add it from the label file
localImageIsTiff = False
localImagePath = localFilePath
if not jp2HasGeoData:
print 'Correcting JP2 file with no geo information!!!!'
# Correct the local file, then remove the old (bad) file
outputPrefix = localFilePath[0:-4]
localImagePath = correctAndCropImage(localFilePath, localLabelPath, outputPrefix)
print 'Generated ' + localImagePath
if localFilePath != localImagePath:
print 'Deleting JP2 file without metadata!'
os.remove(localFilePath)
localImageIsTiff = (localImagePath[-4:] == ".TIF")
if not localImageIsTiff:
# Call code to fix the header information in the JP2 file!
cmd = FIX_JP2_TOOL +' '+ localImagePath
print cmd
os.system(cmd)
# Check the projection type
projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath)
(width, height) = IrgIsisFunctions.getImageSize(localImagePath)
if (projType == 'POLAR STEREOGRAPHIC') and False: #(width < MAX_POLAR_UPLOAD_WIDTH):
# Google has trouble digesting these files so handle them differently.
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!')
print 'Special handling for POLAR STEROGRAPHIC image!'
if fakeLabelFile:
print 'Cannot reprocess polar image without a label file!'
print 'All we can do is upload the file and hope for the best.'
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# Compute how many chunks are needed for this image
numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE)
# Determine which chunk this DB entry is for
chunkNum = getChunkNum(setName)
print 'This is chunk number ' + str(chunkNum)
if chunkNum >= numChunks: # Check for chunk number error
raise Exception('Illegal chunk number: ' + setName)
# If this is the main DB entry, we need to make sure the other DB entries exist!
if chunkNum == 0:
# Go ahead and try to add each chunk, the call will only go through if it does not already exist.
for i in range(1,numChunks):
chunkSetName = makeChunkSetName(setName, i)
print 'Add chunk set name to DB: ' + chunkSetName
common.addDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, chunkSetName, remoteURL)
raise Exception('DEBUG')
# Now actually generate the desired chunk
# - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it.
fileBasePath = os.path.splitext(localImagePath)[0]
localImgPath = fileBasePath + '.IMG'
localChunkPrefix = fileBasePath + '_' + str(chunkNum)
chunkBB = getChunkBoundingBox(width, height, chunkNum)
localChunkPath = correctAndCropImage(localImagePath, localLabelPath, localChunkPrefix, chunkBB)
# Just use the same label file, we don't care if the DB has per-chunk boundaries.
return [localChunkPath, localLabelPath]
else: # A normal, non-polar file.
# First file is for upload, second contains the timestamp.
return [localImagePath, localLabelPath]
# TODO: Handle POLAR DEMS
else: # Handle DEMs
# For DEMs there is no label file
localFilePath = os.path.join(workDir, os.path.basename(remoteURL))
if not os.path.exists(localFilePath):
# Download the image
cmd = 'wget ' + remoteURL + ' -O ' + localFilePath
print cmd
os.system(cmd)
if not IrgFileFunctions.fileIsNonZero(localFilePath): # Make sure we got the file
raise Exception('Unable to download from URL: ' + remoteURL)
# Generate a header file from the IMG file
localLabelPath = localFilePath[:-4] + '.LBL'
cmd = 'head -n 90 ' + localFilePath +' > '+ localLabelPath
print cmd
os.system(cmd)
# Check if this is a polar stereographic image
isPolar = False
f = open(localLabelPath)
for line in f:
if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC" in line):
isPolar = True
print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY'
break
#os.remove(localFilePath)
#os.remove(localLabelPath)
#raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!')
f.close()
# Convert from IMG to TIF
tiffFilePath = localFilePath[:-4] + '.TIF'
if not os.path.exists(tiffFilePath):
cmd = 'gdal_translate -of GTiff ' + localFilePath +' '+ tiffFilePath
print cmd
os.system(cmd)
if not isPolar: # Only EQC files need to be corrected
# Correct projected coordinates problems
cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath
print cmd
os.system(cmd)
os.remove(localFilePath) # Clean up source image
return [tiffFilePath, localLabelPath]
