def getImageSize(imagePath): return IrgIsisFunctions.getImageSize(imagePath)
#generateKmlFromGdcPoints(os.path.join(doubleCalWorkFolder, 'initialGdcCheck'), tempFolder, 'dem-trans_source.csv', 'transformedPoints' 1000, 'blue', 'normal', , False) generateKmlFromGdcPoints(os.path.join(doubleCalWorkFolder, 'pcAlignOutput'), tempFolder, 'dem-trans_reference.csv', 'transformedGdcPoints', 160, 'red', 'tiny', carry) generateKmlFromGdcPoints(os.path.join(doubleCalWorkFolder, 'pcAlignOutput'), tempFolder, 'dem-beg_errors.csv', 'beg-errors', 2, 'yellow', 'tiny', carry) generateKmlFromGdcPoints(os.path.join(doubleCalWorkFolder, 'pcAlignOutput'), tempFolder, 'dem-end_errors.csv', 'end-errors', 2, 'green', 'tiny', carry) print 'Finished generating KML plots' # Delay check for left path to allow debug KML files to be generated if caughtException or not os.path.exists(leftCorrectedPath) or not os.path.exists(rightStereoCorrectedPath): raise Exception('Failed to run stereo calibration process - ' + exceptionText) print '\n-------------------------------------------------------------------------\n' # Generate a PVL file that we need for noproj pvlPath = os.path.join(tempFolder, 'noprojInstruments.pvl') imageSize = IrgIsisFunctions.getImageSize(leftCorrectedPath) isHalfRes = imageSize[0] < 5000 if not os.path.exists(pvlPath): print 'Writing PVL' IsisTools.writeLronacPvlFile(pvlPath, isHalfRes) correctTime = time.time() logging.info('Nav correction finished in %f seconds', correctTime - startTime) # Noproj the corrected data leftNoprojPath = os.path.join(tempFolder, 'leftFinalCorrected.noproj.cub') rightNoprojPath = os.path.join(tempFolder, 'rightFinalCorrected.noproj.cub') leftStereoNoprojPath = os.path.join(tempFolder, 'leftStereoFinalCorrected.noproj.cub') rightStereoNoprojPath = os.path.join(tempFolder, 'rightStereoFinalCorrected.noproj.cub') # Set up thread objects
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir): '''Retrieves a remote file and prepares it for upload''' #print 'Uploading file ' + setName if subtype != 'DEM': # Handles RED and COLOR images # The label file URL is the same as the image but with a different extension remoteLabelURL = getLabelPathFromImagePath(remoteURL) localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) localLabelPath = os.path.join(workDir, os.path.basename(remoteLabelURL)) # Retrieve the header file if not os.path.exists(localLabelPath): # Try to get the label locally! pdsStart = remoteLabelURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteLabelURL[pdsStart:]) print localPdsPath if os.path.exists( localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath + ' ' + localLabelPath else: # Download the image cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath print cmd os.system(cmd) # Retrieve the image file if not os.path.exists(localFilePath): # Need to get it from somewhere # Try to get the image locally! pdsStart = remoteURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:]) if os.path.exists( localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath + ' ' + localFilePath else: # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(localFilePath): if not IrgFileFunctions.fileIsNonZero(localLabelPath): print 'Could not find label or image file, DELETING DATA RECORD!' common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, setName) raise Exception('Unable to download from URL: ' + remoteURL) # Check if there is geo data in the JP2 file jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath) # If there is no label file, try to generate an artificial one. fakeLabelFile = False if not IrgFileFunctions.fileIsNonZero(localLabelPath): #raise Exception('Unable to download from URL: ' + remoteLabelURL) print 'WARNING: Unable to download from URL: ' + remoteLabelURL if not jp2HasGeoData: raise Exception( 'No geo data in JP2 and no Label file! Cannot handle this!' ) print 'Generating a fake LBL file to proceed...' localLabelPath = writeFakeLabelFromJp2(localFilePath) fakeLabelFile = True # At this point we always have a label file but it may be fake # Some images are missing geo information but we can add it from the label file localImageIsTiff = False localImagePath = localFilePath if not jp2HasGeoData: print 'Correcting JP2 file with no geo information!!!!' # Correct the local file, then remove the old (bad) file outputPrefix = localFilePath[0:-4] localImagePath = correctAndCropImage(localFilePath, localLabelPath, outputPrefix) print 'Generated ' + localImagePath if localFilePath != localImagePath: print 'Deleting JP2 file without metadata!' os.remove(localFilePath) localImageIsTiff = (localImagePath[-4:] == ".TIF") if not localImageIsTiff: # Call code to fix the header information in the JP2 file! cmd = FIX_JP2_TOOL + ' ' + localImagePath print cmd os.system(cmd) # Check the projection type projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath) (width, height) = IrgIsisFunctions.getImageSize(localImagePath) if (projType == 'POLAR STEREOGRAPHIC' ) and False: #(width < MAX_POLAR_UPLOAD_WIDTH): # Google has trouble digesting these files so handle them differently. #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!') print 'Special handling for POLAR STEROGRAPHIC image!' if fakeLabelFile: print 'Cannot reprocess polar image without a label file!' print 'All we can do is upload the file and hope for the best.' # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # Compute how many chunks are needed for this image numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE) # Determine which chunk this DB entry is for chunkNum = getChunkNum(setName) print 'This is chunk number ' + str(chunkNum) if chunkNum >= numChunks: # Check for chunk number error raise Exception('Illegal chunk number: ' + setName) # If this is the main DB entry, we need to make sure the other DB entries exist! if chunkNum == 0: # Go ahead and try to add each chunk, the call will only go through if it does not already exist. for i in range(1, numChunks): chunkSetName = makeChunkSetName(setName, i) print 'Add chunk set name to DB: ' + chunkSetName common.addDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, chunkSetName, remoteURL) raise Exception('DEBUG') # Now actually generate the desired chunk # - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it. fileBasePath = os.path.splitext(localImagePath)[0] localImgPath = fileBasePath + '.IMG' localChunkPrefix = fileBasePath + '_' + str(chunkNum) chunkBB = getChunkBoundingBox(width, height, chunkNum) localChunkPath = correctAndCropImage(localImagePath, localLabelPath, localChunkPrefix, chunkBB) # Just use the same label file, we don't care if the DB has per-chunk boundaries. return [localChunkPath, localLabelPath] else: # A normal, non-polar file. # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # TODO: Handle POLAR DEMS else: # Handle DEMs # For DEMs there is no label file localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) if not os.path.exists(localFilePath): # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero( localFilePath): # Make sure we got the file raise Exception('Unable to download from URL: ' + remoteURL) # Generate a header file from the IMG file localLabelPath = localFilePath[:-4] + '.LBL' cmd = 'head -n 90 ' + localFilePath + ' > ' + localLabelPath print cmd os.system(cmd) # Check if this is a polar stereographic image isPolar = False f = open(localLabelPath) for line in f: if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC" in line): isPolar = True print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY' break #os.remove(localFilePath) #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!') f.close() # Convert from IMG to TIF tiffFilePath = localFilePath[:-4] + '.TIF' if not os.path.exists(tiffFilePath): cmd = 'gdal_translate -of GTiff ' + localFilePath + ' ' + tiffFilePath print cmd os.system(cmd) if not isPolar: # Only EQC files need to be corrected # Correct projected coordinates problems cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath print cmd os.system(cmd) os.remove(localFilePath) # Clean up source image return [tiffFilePath, localLabelPath]
def fetchAndPrepFile(db, setName, subtype, remoteURL, workDir): '''Retrieves a remote file and prepares it for upload''' #print 'Uploading file ' + setName if subtype != 'DEM': # Handles RED and COLOR images # The label file URL is the same as the image but with a different extension remoteLabelURL = getLabelPathFromImagePath(remoteURL) localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) localLabelPath = os.path.join(workDir, os.path.basename(remoteLabelURL)) # Retrieve the header file if not os.path.exists(localLabelPath): # Try to get the label locally! pdsStart = remoteLabelURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteLabelURL[pdsStart:]) print localPdsPath if os.path.exists(localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath +' '+ localLabelPath else: # Download the image cmd = 'wget ' + remoteLabelURL + ' -O ' + localLabelPath print cmd os.system(cmd) # Retrieve the image file if not os.path.exists(localFilePath): # Need to get it from somewhere # Try to get the image locally! pdsStart = remoteURL.find('PDS') localPdsPath = os.path.join('/HiRISE/Data/', remoteURL[pdsStart:]) if os.path.exists(localPdsPath): # File available locally, just copy it! cmd = 'cp ' + localPdsPath +' '+ localFilePath else: # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(localFilePath): if not IrgFileFunctions.fileIsNonZero(localLabelPath): print 'Could not find label or image file, DELETING DATA RECORD!' common.removeDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, setName) raise Exception('Unable to download from URL: ' + remoteURL) # Check if there is geo data in the JP2 file jp2HasGeoData = IrgGeoFunctions.doesImageHaveGeoData(localFilePath) # If there is no label file, try to generate an artificial one. fakeLabelFile = False if not IrgFileFunctions.fileIsNonZero(localLabelPath): #raise Exception('Unable to download from URL: ' + remoteLabelURL) print 'WARNING: Unable to download from URL: ' + remoteLabelURL if not jp2HasGeoData: raise Exception('No geo data in JP2 and no Label file! Cannot handle this!') print 'Generating a fake LBL file to proceed...' localLabelPath = writeFakeLabelFromJp2(localFilePath) fakeLabelFile = True # At this point we always have a label file but it may be fake # Some images are missing geo information but we can add it from the label file localImageIsTiff = False localImagePath = localFilePath if not jp2HasGeoData: print 'Correcting JP2 file with no geo information!!!!' # Correct the local file, then remove the old (bad) file outputPrefix = localFilePath[0:-4] localImagePath = correctAndCropImage(localFilePath, localLabelPath, outputPrefix) print 'Generated ' + localImagePath if localFilePath != localImagePath: print 'Deleting JP2 file without metadata!' os.remove(localFilePath) localImageIsTiff = (localImagePath[-4:] == ".TIF") if not localImageIsTiff: # Call code to fix the header information in the JP2 file! cmd = FIX_JP2_TOOL +' '+ localImagePath print cmd os.system(cmd) # Check the projection type projType = IrgGeoFunctions.getProjectionFromIsisLabel(localLabelPath) (width, height) = IrgIsisFunctions.getImageSize(localImagePath) if (projType == 'POLAR STEREOGRAPHIC') and False: #(width < MAX_POLAR_UPLOAD_WIDTH): # Google has trouble digesting these files so handle them differently. #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC images on hold until Google fixes a bug!') print 'Special handling for POLAR STEROGRAPHIC image!' if fakeLabelFile: print 'Cannot reprocess polar image without a label file!' print 'All we can do is upload the file and hope for the best.' # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # Compute how many chunks are needed for this image numChunks = ceil(width / POLAR_WIDTH_CHUNK_SIZE) # Determine which chunk this DB entry is for chunkNum = getChunkNum(setName) print 'This is chunk number ' + str(chunkNum) if chunkNum >= numChunks: # Check for chunk number error raise Exception('Illegal chunk number: ' + setName) # If this is the main DB entry, we need to make sure the other DB entries exist! if chunkNum == 0: # Go ahead and try to add each chunk, the call will only go through if it does not already exist. for i in range(1,numChunks): chunkSetName = makeChunkSetName(setName, i) print 'Add chunk set name to DB: ' + chunkSetName common.addDataRecord(db, common.SENSOR_TYPE_HiRISE, subtype, chunkSetName, remoteURL) raise Exception('DEBUG') # Now actually generate the desired chunk # - Need to use PIRL tools to extract a chunk to an IMG format, then convert that back to JP2 so that Google can read it. fileBasePath = os.path.splitext(localImagePath)[0] localImgPath = fileBasePath + '.IMG' localChunkPrefix = fileBasePath + '_' + str(chunkNum) chunkBB = getChunkBoundingBox(width, height, chunkNum) localChunkPath = correctAndCropImage(localImagePath, localLabelPath, localChunkPrefix, chunkBB) # Just use the same label file, we don't care if the DB has per-chunk boundaries. return [localChunkPath, localLabelPath] else: # A normal, non-polar file. # First file is for upload, second contains the timestamp. return [localImagePath, localLabelPath] # TODO: Handle POLAR DEMS else: # Handle DEMs # For DEMs there is no label file localFilePath = os.path.join(workDir, os.path.basename(remoteURL)) if not os.path.exists(localFilePath): # Download the image cmd = 'wget ' + remoteURL + ' -O ' + localFilePath print cmd os.system(cmd) if not IrgFileFunctions.fileIsNonZero(localFilePath): # Make sure we got the file raise Exception('Unable to download from URL: ' + remoteURL) # Generate a header file from the IMG file localLabelPath = localFilePath[:-4] + '.LBL' cmd = 'head -n 90 ' + localFilePath +' > '+ localLabelPath print cmd os.system(cmd) # Check if this is a polar stereographic image isPolar = False f = open(localLabelPath) for line in f: if ("MAP_PROJECTION_TYPE" in line) and ("POLAR STEREOGRAPHIC" in line): isPolar = True print 'WARNING: POLAR STEREOGRAPHIC DEM MAY NOT UPLOAD PROPERLY' break #os.remove(localFilePath) #os.remove(localLabelPath) #raise Exception('POLAR STEREOGRAPHIC DEMs on hold until Google fixes a bug!') f.close() # Convert from IMG to TIF tiffFilePath = localFilePath[:-4] + '.TIF' if not os.path.exists(tiffFilePath): cmd = 'gdal_translate -of GTiff ' + localFilePath +' '+ tiffFilePath print cmd os.system(cmd) if not isPolar: # Only EQC files need to be corrected # Correct projected coordinates problems cmd = 'python /home/pirl/smcmich1/repo/Tools/geoTiffTool.py --normalize-eqc-lon ' + tiffFilePath print cmd os.system(cmd) os.remove(localFilePath) # Clean up source image return [tiffFilePath, localLabelPath]