Exemplo n.º 1
0
def cloudMask(date1, date2):
    # dateFns have completed therefore assume that, at least, the
    # quadscenes have been created in projectStorage
    # (i.e., sceneID1U*, sceneID1L*, sceneID2U*, and sceneID2L* directories are populuated)
    # eros_data may or may not be present
	# cloudMask dependencies are on files in tiffStorage, /lsfdata/eros_data/extractedTars/

    outBasename = date1.sceneID + "_" + date2.sceneID + '_Fmask.tif'
    wrs2Name=date1.sceneID[3:9]

    if not os.path.exists(date1.folder+"/"+date1.sceneID+"_MTLFmask.TIF"):
	   rasterAnalysis_GDAL.cloudMask(os.path.join(LSF.tiffsStorage, date1.sceneID[:-2]))
	   # create quads from the input scene
	   quadPaths = rasterAnalysis_GDAL.cropToQuad(os.path.join(LSF.tiffsStorage, date1.sceneID[:-2]), LSF.projectStorage, LSF.quadsFolder)
	   landsatFactTools_GDAL.writeQuadToDB(quadPaths)
	   # get cloud cover percentage for each quad
	   quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths)
	   # =========================================================================
	   # write input scene quads cloud cover percentage to the landsat_metadata table in the database
	   landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,date1.folder.replace('UR','').replace('UL','').replace('LR','').replace('LL',''))
    if not os.path.exists(date2.folder+"/"+date2.sceneID+"_MTLFmask.TIF"):
	   rasterAnalysis_GDAL.cloudMask(os.path.join(LSF.tiffsStorage, date2.sceneID[:-2]))
	   # create quads from the input scene
	   quadPaths = rasterAnalysis_GDAL.cropToQuad(os.path.join(LSF.tiffsStorage, date2.sceneID[:-2]), LSF.projectStorage, LSF.quadsFolder)
	   landsatFactTools_GDAL.writeQuadToDB(quadPaths)
	   # get cloud cover percentage for each quad
	   quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths)
	   # =========================================================================
	   # write input scene quads cloud cover percentage to the landsat_metadata table in the database
	   landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,date1.folder.replace('UR','').replace('UL','').replace('LR','').replace('LL',''))

    outputTiffName=os.path.join(LSF.outFMASKfolder,outBasename)
    if not os.path.exists(outputTiffName):
	   if os.path.exists(date1.folder+"/"+date1.sceneID+"_MTLFmask.TIF") and os.path.exists(date2.folder+"/"+date2.sceneID+"_MTLFmask.TIF"):
             qaTiffName=os.path.join(LSF.tiffsStorage, date1.sceneID[:-2], date1.sceneID[:-2]) + "_BQA.TIF"
             if os.path.exists(qaTiffName):
               cloud_mask_type='BQA'
             else:
               cloud_mask_type='FMASK'
             FmaskReclassedArray1 = date1.cloudMaskArray()
             FmaskReclassedArray2 = date2.cloudMaskArray()
             FmaskReclassedArray = FmaskReclassedArray1 * FmaskReclassedArray2
             FmaskReclassedArrayPlus1 = FmaskReclassedArray + 1
             shpName=os.path.join(LSF.quadsFolder, 'wrs2_'+ wrs2Name + date1.folder[-2:]+'.shp')
             LSFGeoTIFF.Unsigned8BitLSFGeoTIFF.fromArray(FmaskReclassedArrayPlus1, date1.geoTiffAtts).write(outputTiffName, shpName)
             landsatFactTools_GDAL.writeProductToDB(os.path.basename(outputTiffName),date1.sceneID,date2.sceneID,'CLOUD',date2.sceneID[9:16], 'CR',cloud_mask_type)
    # if the product's been created, should  be a row in products
    # note if the row is missing
    else:
	   resultRowExists = rowExists(outBasename, 'products', 'product_id')
	   if resultRowExists == False:
             print "No row for {} in products".format(outBasename)
def extractProductForCompare(diff_tar,tarStorage,tiffsStorage,fmaskShellCall,quadsFolder,outRasterFolder):
    print "call to extractProductForCompare with: "+ diff_tar
    try:
        # call download_landsat_data_by_sceneid.php using the downloadScene function
        # in the event of a DownloadError, retry downloadScene up to 5 times, with a 5 second delay btw calls
        retry(1, 5, DownloadError,downloadScene, diff_tar)
        # now extract the downloaded file accordingly
        extractedPath = checkExisting(os.path.join(LSF.tarStorage, diff_tar+".tar.gz"), tiffsStorage)
        #do the other pre-processing stuff
        rasterAnalysis_GDAL.runFmask(extractedPath,LSF.fmaskShellCall)
        # get DN min number from each band in the scene and write to database
        dnminExists = checkForDNminExist(extractedPath) # May not be needed in final design, used during testing
        if dnminExists == False:
            dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath)
            # fix bug introduced by me in commit de5df07c4ca3ff71ae4d7da27b6018fe1bc2df04
            writeDNminToDB(dnMinDict,extractedPath)
        # create quads from the input scene
        quadPaths = rasterAnalysis_GDAL.cropToQuad(extractedPath,outRasterFolder,quadsFolder)
        writeQuadToDB(quadPaths)
        # get cloud cover percentage for each quad
        quadCCDict = getQuadCCpercent(quadPaths)
        # =========================================================================
        # write input scene quads cloud cover percentage to the landsat_metadata table in the database
        writeQuadsCCtoDB(quadCCDict,extractedPath)
        print os.getcwd()
    except Exception as e:
        print "Error in extractProductForCompare"
        # make sure we've returned to the original directory before raising this exception
        raise
def quadScene(quadsceneID):
    # this function is a little more complicated because it tries to take one of two possible shortcuts.
    # if there are clipped TIFFs, the previous work artifacts can safely no longer be there.
    # if the clipped TIFFs aren't there, look for unclipped TIFFs in tiffsStorage first.
    # if the unclipped TIFFs aren't there, the extractedTar "target" function will cause the tar to be downloaded

    sceneID=quadsceneID[:-2]
    if not(re.search(quadsceneID, ' '.join(glob.glob(os.path.join(LSF.projectStorage, '*'))))):
        # if the band files for quadsceneID in are not in projectStorage, get them from tiffsStorage, if possible, and clip them
        if not (re.search(sceneID, ' '.join(glob.glob(os.path.join(LSF.tiffsStorage, '*'))))):
            extractedTar(quadsceneID)
        quadPaths = rasterAnalysis_GDAL.cropToQuad(os.path.join(LSF.tiffsStorage, sceneID), LSF.projectStorage, LSF.quadsFolder)
        landsatFactTools_GDAL.writeQuadToDB(quadPaths)
        # get cloud cover percentage for each quad
        quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths)
        # =========================================================================
        # write input scene quads cloud cover percentage to the landsat_metadata table in the database
        landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,os.path.join(LSF.tiffsStorage, sceneID))

    # if the quad's been processed, should at least be a row in extracted_imagery
    # note if the row is missing
    else:
        resultRowExists = rowExists(quadsceneID, 'extracted_imagery', 'quad_scene')
        if resultRowExists == False:
            print "No row for {} in extracted_imagery".format(quadsceneID)
 # run Fmask
 # jdm 4/22/15: after spending a couple of days trying to get FMASK installed on cloud4
 # I have not been able to get it to work.  Therefore, for now I am commenting this out
 # rasterAnalysis_GDAL.runFmask(extractedPath,Fmaskexe) #BM's original
 print extractedPath
 runFmaskBool = rasterAnalysis_GDAL.runFmask(extractedPath,fmaskShellCall)
 #print "Fmask Boolean: "+runFmaskBool
 if (runFmaskBool == True):
     # get DN min number from each band in the scene and write to database
     wrs2Name=tar[3:9]
     dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing
     if dnminExists == False:
         dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath)
         landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath)
     # create quads from the input scene
     quadPaths = rasterAnalysis_GDAL.cropToQuad(extractedPath,projectStorage,quadsFolder)
     landsatFactTools_GDAL.writeQuadToDB(quadPaths)
     # get cloud cover percentage for each quad
     # write input scene quads cloud cover percentage to the landsat_metadata table in the database
     quadCCDict=landsatFactTools_GDAL.readAndWriteQuadCC(quadPaths, extractedPath)
     # for each quad this finds the closest scene that passes the cloud cover threshold for processing
     quadTiffList2Process = landsatFactTools_GDAL.getNextBestQuad(quadCCDict,cloudCoverThreshold)
     # =========================================================================
     #jdm: Need to make sure the next best quads we are going to be comparing to are actually extracted
     #and pre-processed to a level appropriate for differencing.
     print "quadTiffList2Process: ", quadTiffList2Process
     #TO-DO: create/call a function in landsatFactTools_GDAL called extractProductForCompare()
     #loop through quadTiffList2Process and get unique list of data to download
     for quad_pair in quadTiffList2Process:
         base_quad = quad_pair[1]
         diff_quad = quad_pair[0]
Exemplo n.º 5
0
	inNewSceneTar = os.path.join(tarStorage, tar)
	# check to see if the tar file has been extracted, if not extract the files
	extractedPath = landsatFactTools_GDAL.checkExisting(inNewSceneTar, tiffsStorage)
	# run Fmask
	# jdm 4/22/15: after spending a couple of days trying to get FMASK installed on cloud4
	# I have not been able to get it to work.  Therefore, for now I am commenting this out
	# rasterAnalysis_GDAL.runFmask(extractedPath,Fmaskexe) #BM's original
	print extractedPath
	rasterAnalysis_GDAL.runFmask(extractedPath,fmaskShellCall)
	# get DN min number from each band in the scene and write to database
	dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing
	if dnminExists == False:
		dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath)
		landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath)
	# create quads from the input scene
	quadPaths = rasterAnalysis_GDAL.cropToQuad(extractedPath,quadsFolder)
	landsatFactTools_GDAL.writeQuadToDB(quadPaths)
	# get cloud cover percentage for each quad
	quadCCDict = landsatFactTools_GDAL.getQuadCCpercent(quadPaths)
	# =========================================================================
	# write input scene quads cloud cover percentage to the landsat_metadata table in the database
	landsatFactTools_GDAL.writeQuadsCCtoDB(quadCCDict,extractedPath)
	# for each quad this finds the closest scene that passes the cloud cover threshold for processing
	quadTiffList2Process = landsatFactTools_GDAL.getNextBestQuad(quadCCDict,cloudCoverThreshold)
	# =========================================================================
	#print "quadTiffList2Process: ", quadTiffList2Process
	# checks the list of quads, if 0 then there were no quads under the cloud cover threshold
	# so there is nothing to process for that scene
	if len(quadTiffList2Process) > 0:
		#print "quadTiffList2Process: ", quadTiffList2Process
		# for each quad pair perform the change analysis