def extractProductForCompare(diff_tar,tarStorage,tiffsStorage,fmaskShellCall,quadsFolder,outRasterFolder): print "call to extractProductForCompare with: "+ diff_tar try: # call download_landsat_data_by_sceneid.php using the downloadScene function # in the event of a DownloadError, retry downloadScene up to 5 times, with a 5 second delay btw calls retry(1, 5, DownloadError,downloadScene, diff_tar) # now extract the downloaded file accordingly extractedPath = checkExisting(os.path.join(LSF.tarStorage, diff_tar+".tar.gz"), tiffsStorage) #do the other pre-processing stuff rasterAnalysis_GDAL.runFmask(extractedPath,LSF.fmaskShellCall) # get DN min number from each band in the scene and write to database dnminExists = checkForDNminExist(extractedPath) # May not be needed in final design, used during testing if dnminExists == False: dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath) # fix bug introduced by me in commit de5df07c4ca3ff71ae4d7da27b6018fe1bc2df04 writeDNminToDB(dnMinDict,extractedPath) # create quads from the input scene quadPaths = rasterAnalysis_GDAL.cropToQuad(extractedPath,outRasterFolder,quadsFolder) writeQuadToDB(quadPaths) # get cloud cover percentage for each quad quadCCDict = getQuadCCpercent(quadPaths) # ========================================================================= # write input scene quads cloud cover percentage to the landsat_metadata table in the database writeQuadsCCtoDB(quadCCDict,extractedPath) print os.getcwd() except Exception as e: print "Error in extractProductForCompare" # make sure we've returned to the original directory before raising this exception raise
def cloudMask(date1, date2): # dateFns have completed therefore assume that, at least, the # quadscenes have been created in projectStorage # (i.e., sceneID1U*, sceneID1L*, sceneID2U*, and sceneID2L* directories are populuated) # eros_data may or may not be present # cloudMask dependencies are on files in tiffStorage, /lsfdata/eros_data/extractedTars/ outBasename = date1.sceneID + "_" + date2.sceneID + '_Fmask.tif' wrs2Name=date1.sceneID[3:9] if not os.path.exists(date1.folder+"/"+date1.sceneID+"_MTLFmask.TIF"): rasterAnalysis_GDAL.runFmask(os.path.join(LSF.tiffsStorage, date1.sceneID[:-2]), LSF.fmaskShellCall) # create quads from the input scene quadPaths = rasterAnalysis_GDAL.cropToQuad(os.path.join(LSF.tiffsStorage, date1.sceneID[:-2]), LSF.projectStorage, LSF.quadsFolder) landsatFactTools_GDAL.writeQuadToDB(quadPaths) # get cloud cover percentage for each quad quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths) # ========================================================================= # write input scene quads cloud cover percentage to the landsat_metadata table in the database landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,date1.folder.replace('UR','').replace('UL','').replace('LR','').replace('LL','')) if not os.path.exists(date2.folder+"/"+date2.sceneID+"_MTLFmask.TIF"): rasterAnalysis_GDAL.runFmask(os.path.join(LSF.tiffsStorage, date2.sceneID[:-2]), LSF.fmaskShellCall) # create quads from the input scene quadPaths = rasterAnalysis_GDAL.cropToQuad(os.path.join(LSF.tiffsStorage, date2.sceneID[:-2]), LSF.projectStorage, LSF.quadsFolder) landsatFactTools_GDAL.writeQuadToDB(quadPaths) # get cloud cover percentage for each quad quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths) # ========================================================================= # write input scene quads cloud cover percentage to the landsat_metadata table in the database landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,date1.folder.replace('UR','').replace('UL','').replace('LR','').replace('LL','')) outputTiffName=os.path.join(LSF.outFMASKfolder,outBasename) if not os.path.exists(outputTiffName): if os.path.exists(date1.folder+"/"+date1.sceneID+"_MTLFmask.TIF") and os.path.exists(date2.folder+"/"+date2.sceneID+"_MTLFmask.TIF"): FmaskReclassedArray1 = date1.reclassFmask() FmaskReclassedArray2 = date2.reclassFmask() FmaskReclassedArray = FmaskReclassedArray1 * FmaskReclassedArray2 shpName=os.path.join(LSF.quadsFolder, 'wrs2_'+ wrs2Name + date1.folder[-2:]+'.shp') LSFGeoTIFF.Unsigned8BitLSFGeoTIFF.fromArray(FmaskReclassedArray, date1.geoTiffAtts).write(outputTiffName, shpName) landsatFactTools_GDAL.writeProductToDB(os.path.basename(outputTiffName),date1.sceneID,date2.sceneID,'CLOUD',date2.sceneID[9:16], 'CR') # if the product's been created, should be a row in products # note if the row is missing else: resultRowExists = rowExists(outBasename, 'products', 'product_id') if resultRowExists == False: print "No row for {} in products".format(outBasename)
def extractedTar(quadsceneID): sceneID=quadsceneID[:-2] # guarantee that level 1 product tar file for sceneID now exists in /lsfdata/eros_data/extractedTars tar(sceneID) # if /lsfdata/extractedTars/sceneID doesn't contain the level 1 product band files, extract them from the tar file # if not(re.search(sceneID, ' '.join(glob.glob(os.path.join(LSF.tiffsStorage, '*'))))): # make sure that the existing tar is valid existingTar=os.path.join(LSF.tarStorage, sceneID+".tar.gz") err=localLib.validTar(existingTar) if err: raise Exception(err) # for now use checkExisting. change to use tarHandling class when completed extractedPath = landsatFactTools_GDAL.checkExisting(existingTar, LSF.tiffsStorage) rasterAnalysis_GDAL.runFmask(extractedPath,LSF.fmaskShellCall) # get DN min number from each band in the scene and write to database dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing if dnminExists == False: dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath) # fix bug introduced by me in commit de5df07c4ca3ff71ae4d7da27b6018fe1bc2df04 landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath)
if err: os.remove(tar) landsatFactTools_GDAL.retry(1, 4, landsatFactTools_GDAL.DownloadError,landsatFactTools_GDAL.downloadScene, tar[:-7]) # all paths are now imported from LSF.py DM - 5/10/2016 # ========================================================================= # sets full path for the tarfile to be analyzed inNewSceneTar = os.path.join(tarStorage, tar) # check to see if the tar file has been extracted, if not extract the files print "Checking for: "+tar extractedPath = landsatFactTools_GDAL.checkExisting(inNewSceneTar, tiffsStorage) # run Fmask # jdm 4/22/15: after spending a couple of days trying to get FMASK installed on cloud4 # I have not been able to get it to work. Therefore, for now I am commenting this out # rasterAnalysis_GDAL.runFmask(extractedPath,Fmaskexe) #BM's original print extractedPath runFmaskBool = rasterAnalysis_GDAL.runFmask(extractedPath,fmaskShellCall) #print "Fmask Boolean: "+runFmaskBool if (runFmaskBool == True): # get DN min number from each band in the scene and write to database wrs2Name=tar[3:9] dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing if dnminExists == False: dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath) landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath) # create quads from the input scene quadPaths = rasterAnalysis_GDAL.cropToQuad(extractedPath,projectStorage,quadsFolder) landsatFactTools_GDAL.writeQuadToDB(quadPaths) # get cloud cover percentage for each quad # write input scene quads cloud cover percentage to the landsat_metadata table in the database quadCCDict=landsatFactTools_GDAL.readAndWriteQuadCC(quadPaths, extractedPath) # for each quad this finds the closest scene that passes the cloud cover threshold for processing
outb7folder = os.path.join(productStorage,'b7Diff_SR') outGAPfolder = os.path.join(productStorage,'gapMask') # set cloud cover threshold level, quad scene's with a higher percentage of # cloud cover will not be processed. cloudCoverThreshold = .50 # ========================================================================= # sets full path for the tarfile to be analyzed inNewSceneTar = os.path.join(tarStorage, tar) # check to see if the tar file has been extracted, if not extract the files extractedPath = landsatFactTools_GDAL.checkExisting(inNewSceneTar, tiffsStorage) # run Fmask # jdm 4/22/15: after spending a couple of days trying to get FMASK installed on cloud4 # I have not been able to get it to work. Therefore, for now I am commenting this out # rasterAnalysis_GDAL.runFmask(extractedPath,Fmaskexe) #BM's original print extractedPath rasterAnalysis_GDAL.runFmask(extractedPath,fmaskShellCall) # get DN min number from each band in the scene and write to database dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing if dnminExists == False: dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath) landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath) # create quads from the input scene quadPaths = rasterAnalysis_GDAL.cropToQuad(extractedPath,quadsFolder) landsatFactTools_GDAL.writeQuadToDB(quadPaths) # get cloud cover percentage for each quad quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths) # ========================================================================= # write input scene quads cloud cover percentage to the landsat_metadata table in the database landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,extractedPath) # for each quad this finds the closest scene that passes the cloud cover threshold for processing quadTiffList2Process = landsatFactTools_GDAL.getNextBestQuad(quadCCDict,cloudCoverThreshold)