def getCustomRequestFromDB(): quadPairList=[] cr_zip_name=aoi_id=None resultsTup = landsatFactTools_GDAL.postgresCommand('select F.aoi_id from ( SELECT * FROM get_pendingcustomrequests() order by aoi_id limit 1) as F;') if resultsTup: aoi_id = resultsTup[0][0] resultsTup = landsatFactTools_GDAL.postgresCommand('select F.request_id from (SELECT * FROM get_customrequestsquads({}) limit 1) AS F;'.format(aoi_id)) cr_zip_name = resultsTup[0][0] resultsTup = landsatFactTools_GDAL.postgresCommand('select concat(F.scene_id, quad_location) from (SELECT * FROM get_customrequestsquads({}) order by quad_id) AS F;'.format(aoi_id)) for a, b in pairwise(resultsTup): quadPairList.append([a[0], b[0]]) return [quadPairList, cr_zip_name, aoi_id]
def __init__(self, sceneID, mtlFile): keys=['sun_elevation', 'earth_sun_distance', 'radiance_mult_band_3', 'radiance_mult_band_4', 'radiance_mult_band_5', 'radiance_mult_band_6', 'radiance_mult_band_7', 'radiance_mult_band_10', 'radiance_add_band_3', 'radiance_add_band_4', 'radiance_add_band_5', 'radiance_add_band_6', 'radiance_add_band_6_VCID_2','radiance_add_band_7', 'radiance_add_band_10','reflectance_mult_band_3', 'reflectance_mult_band_4', 'reflectance_mult_band_5', 'reflectance_mult_band_6','radiance_mult_band_6_VCID_2','reflectance_mult_band_7', 'reflectance_add_band_3', 'reflectance_add_band_4','reflectance_add_band_5', 'reflectance_add_band_6', 'reflectance_add_band_7', 'radiance_maximum_band_3', 'radiance_maximum_band_4', 'radiance_maximum_band_5', 'radiance_maximum_band_6', 'radiance_maximum_band_7','radiance_maximum_band_10', 'reflectance_maximum_band_3', 'reflectance_maximum_band_4', 'reflectance_maximum_band_5', 'reflectance_maximum_band_6','reflectance_maximum_band_7'] self.sceneID = sceneID if not os.path.exists(mtlFile): statement = "select {0} FROM level1_metadata INNER JOIN landsat_metadata ON level1_metadata.level1_id = landsat_metadata.l1_key WHERE (scene_id= '{1}');".format( ','.join(keys),sceneID) res = landsatFactTools_GDAL.postgresCommand(statement) if not res: raise RuntimeError('Expecting level1_metadata but no results returned for {0} from DB query'.format(sceneID)) else: values=res[0] self.mtl = dict(zip(keys,values)) else: self.mtl=dict() mtlDoc = open(mtlFile,'r') mtl = mtlDoc.readlines() for key in keys: self.mtl[key]=self.getMTLobject(key.upper(), mtl) mtlDoc.close() self.writeToDB() self.pltType = sceneID[0:3]
def getQuadListFromDB(): sqlStatement = """select scene_id from vw_scenes_less_five where rank=3 OR rank=4 ORDER BY wrs2_code, rank;""" #debugList=[['LE70140352013096EDC00', 'LC80140352013312LGN00']] # ['LE70150402015173EDC00', 'LE70150402015189EDC00'], # ['LC80150412014290LGN00', 'LC80150412015021LGN00'], # ['LE70150422015141EDC00', 'LE70150422015285EDC00'], # ['LC80230402014314LGN00', 'LC80230402014330LGN00'], # ['LE70270402015049EDC00', 'LC80270402015057LGN00'], # ['LC80270412014358LGN00', 'LE70270412015049EDC00']] #scenePairList=debugList scenePairList=[] resultsTup = landsatFactTools_GDAL.postgresCommand(sqlStatement) if resultsTup: for a, b in customRequest.pairwise(resultsTup): scenePairList.append([a[0], b[0]]) quadPairList=[] for scenePair in scenePairList: quadPairList.append([scenePair[0]+'LL', scenePair[1]+'LL']) quadPairList.append([scenePair[0]+'UL', scenePair[1]+'UL']) quadPairList.append([scenePair[0]+'UR', scenePair[1]+'UR']) quadPairList.append([scenePair[0]+'LR', scenePair[1]+'LR']) return quadPairList
def callZonalStats(aoiID, swirs, cloudMasks, gapMasks, statsOutPath): print aoiID, swirs, cloudMasks, gapMasks in_dir = os.getcwd() os.chdir('/tmp') print os.getcwd() sqlString = 'select st_asgeojson(geom) from aoi_alerts where aoi_id={};'.format(aoiID) resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString) if resultsTup: geo = resultsTup[0][0] writeListToFile([geo],'tmpAOIFile') writeListToFile(map(lambda x: os.path.join(LSF.productStorage, 'swir', x),swirs),'tmpSwirsFile') writeListToFile(map(lambda x: os.path.join(LSF.productStorage, 'cloud_mask', x),cloudMasks),'tmpCloudMasksFile') writeListToFile(map(lambda x: os.path.join(LSF.productStorage, 'gap_mask', x),gapMasks),'tmpGapMasksFile') writeRemapCSV('remap.csv') codeIn = [LSF.path_projects + '/geoprocessing/LSF_zonalstats.py', 'tmpAOIFile', 'tmpSwirsFile', 'remap.csv', statsOutPath, '-c', 'tmpCloudMasksFile', '-g', 'tmpGapMasksFile'] process = subprocess.Popen(codeIn,stdout=subprocess.PIPE, stderr=subprocess.PIPE) out,err = process.communicate() errcode = process.returncode os.chdir(in_dir) print out, err
def writeToDB(self): noNullsDict = {k: v for k, v in self.mtl.items() if v is not None} lid=landsatFactTools_GDAL.postgresCommand('select * from keys_and_values(\'('+','.join(noNullsDict.keys()) + ')\', \'(' + ','.join(map(str,noNullsDict.values())) + ')\',\'' + self.sceneID+ '\');')
import numpy as np import psycopg2 from LSF import * import localLib import pdb runList=[] extractedList = [] exceptList = [] DNminDict = {} tableName = 'vw_failed_l1metadata' tableColumnList = ['scene_id'] statement = "SELECT {0} FROM {1};".format(tableColumnList[0],tableName) resultsTup = landsatFactTools_GDAL.postgresCommand(statement) print resultsTup print '' runList=[] extractedList = [] exceptList = [] for scene in resultsTup: runList.append(scene[0]) for scene in runList: try: # sets full path for the tarfile to be analyzed
def rowExists(rowID, tableName, tableColumn): # checks to see if there is a row with rowID in table tableName statement = "SELECT exists (SELECT true FROM {1} WHERE {0} = '{2}');".format(tableColumn,tableName,rowID) resultsTup = landsatFactTools_GDAL.postgresCommand(statement)[0][0] return resultsTup
def compare(quadsceneID1, quadsceneID2): if int(quadsceneID1[9:16]) > int(quadsceneID2[9:16]): comparisonProduct(quadsceneID2, quadsceneID1) else: comparisonProduct(quadsceneID1, quadsceneID2) # process one change request customRequestInfo=getCustomRequestFromDB() lol=customRequestInfo[0] request_id=customRequestInfo[1] aoi_id=customRequestInfo[2] if lol: # status = "Process Start" landsatFactTools_GDAL.postgresCommand('insert into custom_request_dates (aoi_id, custom_request_date, custom_request_status_id) VALUES(\'{}\', now(), 2);'.format(aoi_id)) # keep track of the number of times we process comparisons error free so that we can send partial zips if need be successfulComparisons=0 if not os.path.exists(os.path.join(outCustomRequestFolder, request_id)): # build change products as necessary for quadScenePairList in lol: try: compare(quadScenePairList[0], quadScenePairList[1]) successfulComparisons+=1 except: tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \ str(sys.exc_type)+ ": " + str(sys.exc_value) + "\n" print pymsg
def statsForAOINewProducts(): """ # Function to retrieve areas of interest with new products, call a function to determine the amount of # change, and record the statistics for those areas of change in the DB # @param None # @return None """ aDict = defaultdict(list) resultsTup = landsatFactTools_GDAL.postgresCommand('SELECT * FROM vw_aoi_alerts;') if resultsTup: for k, v in resultsTup: aDict[k].append(v) for k, v in aDict.iteritems(): aoi = k swirs = list(filter(lambda x: 'SWIR' in x, aDict[k])) eventDateString=sorted(map(input2Date, swirs))[-1] sqlString='select (date \'{0}-01-01\' + integer \'{1}\' - 1);'.format(eventDateString[:4], eventDateString[-3:]) resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString) dateString = resultsTup[0][0] cloudMasks = list(filter(lambda x: 'Fmask' in x, aDict[k])) gapMasks = list(filter(lambda x: 'GapMask' in x, aDict[k])) sqlString = "SELECT st_area(geom::geography) from aoi_alerts where aoi_id={0}".format(aoi) resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString) aoiAcres=float(resultsTup[0][0])*0.0002471044 if aoiAcres > 50000: print 'Skipped aoi_id {0} because it is too large, {1} acres'.format(aoi, aoiAcres) continue sqlString = "SELECT exists (SELECT true FROM aoi_events WHERE aoi_id = {0} AND event_date = \'{1}\');".format(aoi,dateString) existingRow = landsatFactTools_GDAL.postgresCommand(sqlString)[0][0] if existingRow: print 'aoi_event for {0} on {1} has already been run'.format(aoi, dateString) continue # status = "Process Start" sqlString='insert into aoi_events(aoi_event_id, aoi_id, alert_status_id) VALUES(DEFAULT,\'{}\', 2) RETURNING aoi_event_id;'.format(aoi) resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString) event_id = resultsTup[0][0] callZonalStats(aoi, swirs, cloudMasks, gapMasks, 'tmpStatsFile.csv') stats=[] with open('/tmp/tmpStatsFile.csv', 'rb') as csvfile: reader = csv.DictReader(csvfile) for row in reader: s=(row['Sum']) if s: h=int(float(s)) else: h=0 stats.append((int(row['Unique region ID']), float(row['Area in square meters']), int(row['Raster value']), h, int(row['Count']), int(row['Count*maxVal']))) #sorted_by_region_ID = sorted(stats, key=lambda tup: tup[0]) def patchVal(v):return filter(lambda tup: tup[2]==v, stats) sumArea=reduce(lambda x,y: x+y, map(lambda x: x[1], stats), 0)*0.0002471044 maxArea=reduce(lambda x,y: max(x,y), map(lambda x: x[1], stats), 0)*0.0002471044 acresChange=reduce(lambda x,y: x+y, map(lambda x: x[1], patchVal(1)), 0)*0.0002471044 acresAnalyzed=acresChange + reduce(lambda x,y: x+y, map(lambda x: x[1], patchVal(0)), 0)*0.0002471044 percentAnalyzed=float(acresChange)/float(aoiAcres)*100 percentChange=(float(acresChange)/float(acresAnalyzed))*100 largestPatch=reduce(lambda x,y: max(x,y), map(lambda x: x[1], patchVal(1)), 0)*0.0002471044 patchCount=len(patchVal(1)) smallestPatch=reduce(lambda x,y: min(x,y), map(lambda x: x[1], patchVal(1)), 0)*0.0002471044 sumOfSum=reduce(lambda x,y: x+y, map(lambda x: x[3], patchVal(1)), 0) sumOfMax=reduce(lambda x,y: x+y, map(lambda x: x[5], patchVal(1)), 0) if sumOfMax: maxPatchSeverity=(float(sumOfSum)/float(sumOfMax))*100 else: maxPatchSeverity=0 resultsTup = landsatFactTools_GDAL.postgresCommand('select * from write_aoi_events({0},{1},{2},{3},{4},{5},{6},{7},{8},\'{9}\''.format( aoi, acresChange, percentChange, acresAnalyzed, percentAnalyzed, smallestPatch, largestPatch, patchCount, event_id, dateString, maxPatchSeverity)+ ',\''+ ','.join(swirs) + '\');') if resultsTup[0][0] == False: print 'Table update failed' else: # status = "Process Complete" sqlString='UPDATE aoi_events set ("alert_status_id") = (3) where aoi_event_id={0};'.format(event_id) resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString)