def getCustomRequestFromDB():
quadPairList=[]
cr_zip_name=aoi_id=None
resultsTup = landsatFactTools_GDAL.postgresCommand('select F.aoi_id from ( SELECT * FROM get_pendingcustomrequests() order by aoi_id limit 1) as F;')
if resultsTup:
aoi_id = resultsTup[0][0]
resultsTup = landsatFactTools_GDAL.postgresCommand('select F.request_id from (SELECT * FROM get_customrequestsquads({}) limit 1) AS F;'.format(aoi_id))
cr_zip_name = resultsTup[0][0]
resultsTup = landsatFactTools_GDAL.postgresCommand('select concat(F.scene_id, quad_location) from (SELECT * FROM get_customrequestsquads({}) order by quad_id) AS F;'.format(aoi_id))
for a, b in pairwise(resultsTup):
quadPairList.append([a[0], b[0]])
return [quadPairList, cr_zip_name, aoi_id]
def __init__(self, sceneID, mtlFile):
keys=['sun_elevation', 'earth_sun_distance', 'radiance_mult_band_3', 'radiance_mult_band_4', 'radiance_mult_band_5',
'radiance_mult_band_6', 'radiance_mult_band_7', 'radiance_mult_band_10', 'radiance_add_band_3',
'radiance_add_band_4', 'radiance_add_band_5', 'radiance_add_band_6', 'radiance_add_band_6_VCID_2','radiance_add_band_7',
'radiance_add_band_10','reflectance_mult_band_3', 'reflectance_mult_band_4', 'reflectance_mult_band_5',
'reflectance_mult_band_6','radiance_mult_band_6_VCID_2','reflectance_mult_band_7', 'reflectance_add_band_3',
'reflectance_add_band_4','reflectance_add_band_5', 'reflectance_add_band_6', 'reflectance_add_band_7',
'radiance_maximum_band_3', 'radiance_maximum_band_4', 'radiance_maximum_band_5', 'radiance_maximum_band_6',
'radiance_maximum_band_7','radiance_maximum_band_10', 'reflectance_maximum_band_3', 'reflectance_maximum_band_4',
'reflectance_maximum_band_5', 'reflectance_maximum_band_6','reflectance_maximum_band_7']
self.sceneID = sceneID
if not os.path.exists(mtlFile):
statement = "select {0} FROM level1_metadata INNER JOIN landsat_metadata ON level1_metadata.level1_id = landsat_metadata.l1_key WHERE (scene_id= '{1}');".format( ','.join(keys),sceneID)
res = landsatFactTools_GDAL.postgresCommand(statement)
if not res:
raise RuntimeError('Expecting level1_metadata but no results returned for {0} from DB query'.format(sceneID))
else:
values=res[0]
self.mtl = dict(zip(keys,values))
else:
self.mtl=dict()
mtlDoc = open(mtlFile,'r')
mtl = mtlDoc.readlines()
for key in keys:
self.mtl[key]=self.getMTLobject(key.upper(), mtl)
mtlDoc.close()
self.writeToDB()
self.pltType = sceneID[0:3]
def getQuadListFromDB():
sqlStatement = """select scene_id from vw_scenes_less_five where rank=3 OR rank=4 ORDER BY wrs2_code, rank;"""
#debugList=[['LE70140352013096EDC00', 'LC80140352013312LGN00']]
# ['LE70150402015173EDC00', 'LE70150402015189EDC00'],
# ['LC80150412014290LGN00', 'LC80150412015021LGN00'],
# ['LE70150422015141EDC00', 'LE70150422015285EDC00'],
# ['LC80230402014314LGN00', 'LC80230402014330LGN00'],
# ['LE70270402015049EDC00', 'LC80270402015057LGN00'],
# ['LC80270412014358LGN00', 'LE70270412015049EDC00']]
#scenePairList=debugList
scenePairList=[]
resultsTup = landsatFactTools_GDAL.postgresCommand(sqlStatement)
if resultsTup:
for a, b in customRequest.pairwise(resultsTup):
scenePairList.append([a[0], b[0]])
quadPairList=[]
for scenePair in scenePairList:
quadPairList.append([scenePair[0]+'LL', scenePair[1]+'LL'])
quadPairList.append([scenePair[0]+'UL', scenePair[1]+'UL'])
quadPairList.append([scenePair[0]+'UR', scenePair[1]+'UR'])
quadPairList.append([scenePair[0]+'LR', scenePair[1]+'LR'])
return quadPairList
def callZonalStats(aoiID, swirs, cloudMasks, gapMasks, statsOutPath):
print aoiID, swirs, cloudMasks, gapMasks
in_dir = os.getcwd()
os.chdir('/tmp')
print os.getcwd()
sqlString = 'select st_asgeojson(geom) from aoi_alerts where aoi_id={};'.format(aoiID)
resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString)
if resultsTup:
geo = resultsTup[0][0]
writeListToFile([geo],'tmpAOIFile')
writeListToFile(map(lambda x: os.path.join(LSF.productStorage, 'swir', x),swirs),'tmpSwirsFile')
writeListToFile(map(lambda x: os.path.join(LSF.productStorage, 'cloud_mask', x),cloudMasks),'tmpCloudMasksFile')
writeListToFile(map(lambda x: os.path.join(LSF.productStorage, 'gap_mask', x),gapMasks),'tmpGapMasksFile')
writeRemapCSV('remap.csv')
codeIn = [LSF.path_projects + '/geoprocessing/LSF_zonalstats.py', 'tmpAOIFile',
'tmpSwirsFile', 'remap.csv', statsOutPath, '-c', 'tmpCloudMasksFile', '-g', 'tmpGapMasksFile']
process = subprocess.Popen(codeIn,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out,err = process.communicate()
errcode = process.returncode
os.chdir(in_dir)
print out, err
def writeToDB(self):
noNullsDict = {k: v for k, v in self.mtl.items() if v is not None}
lid=landsatFactTools_GDAL.postgresCommand('select * from keys_and_values(\'('+','.join(noNullsDict.keys()) + ')\', \'(' + ','.join(map(str,noNullsDict.values())) + ')\',\'' + self.sceneID+ '\');')
import numpy as np
import psycopg2
from LSF import *
import localLib
import pdb
runList=[]
extractedList = []
exceptList = []
DNminDict = {}
tableName = 'vw_failed_l1metadata'
tableColumnList = ['scene_id']
statement = "SELECT {0} FROM {1};".format(tableColumnList[0],tableName)
resultsTup = landsatFactTools_GDAL.postgresCommand(statement)
print resultsTup
print ''
runList=[]
extractedList = []
exceptList = []
for scene in resultsTup:
runList.append(scene[0])
for scene in runList:
try:
# sets full path for the tarfile to be analyzed
def rowExists(rowID, tableName, tableColumn):
# checks to see if there is a row with rowID in table tableName
statement = "SELECT exists (SELECT true FROM {1} WHERE {0} = '{2}');".format(tableColumn,tableName,rowID)
resultsTup = landsatFactTools_GDAL.postgresCommand(statement)[0][0]
return resultsTup
def compare(quadsceneID1, quadsceneID2):
if int(quadsceneID1[9:16]) > int(quadsceneID2[9:16]):
comparisonProduct(quadsceneID2, quadsceneID1)
else:
comparisonProduct(quadsceneID1, quadsceneID2)
# process one change request
customRequestInfo=getCustomRequestFromDB()
lol=customRequestInfo[0]
request_id=customRequestInfo[1]
aoi_id=customRequestInfo[2]
if lol:
# status = "Process Start"
landsatFactTools_GDAL.postgresCommand('insert into custom_request_dates (aoi_id, custom_request_date, custom_request_status_id) VALUES(\'{}\', now(), 2);'.format(aoi_id))
# keep track of the number of times we process comparisons error free so that we can send partial zips if need be
successfulComparisons=0
if not os.path.exists(os.path.join(outCustomRequestFolder, request_id)):
# build change products as necessary
for quadScenePairList in lol:
try:
compare(quadScenePairList[0], quadScenePairList[1])
successfulComparisons+=1
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type)+ ": " + str(sys.exc_value) + "\n"
print pymsg
def statsForAOINewProducts():
"""
# Function to retrieve areas of interest with new products, call a function to determine the amount of
# change, and record the statistics for those areas of change in the DB
# @param None
# @return None
"""
aDict = defaultdict(list)
resultsTup = landsatFactTools_GDAL.postgresCommand('SELECT * FROM vw_aoi_alerts;')
if resultsTup:
for k, v in resultsTup:
aDict[k].append(v)
for k, v in aDict.iteritems():
aoi = k
swirs = list(filter(lambda x: 'SWIR' in x, aDict[k]))
eventDateString=sorted(map(input2Date, swirs))[-1]
sqlString='select (date \'{0}-01-01\' + integer \'{1}\' - 1);'.format(eventDateString[:4], eventDateString[-3:])
resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString)
dateString = resultsTup[0][0]
cloudMasks = list(filter(lambda x: 'Fmask' in x, aDict[k]))
gapMasks = list(filter(lambda x: 'GapMask' in x, aDict[k]))
sqlString = "SELECT st_area(geom::geography) from aoi_alerts where aoi_id={0}".format(aoi)
resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString)
aoiAcres=float(resultsTup[0][0])*0.0002471044
if aoiAcres > 50000:
print 'Skipped aoi_id {0} because it is too large, {1} acres'.format(aoi, aoiAcres)
continue
sqlString = "SELECT exists (SELECT true FROM aoi_events WHERE aoi_id = {0} AND event_date = \'{1}\');".format(aoi,dateString)
existingRow = landsatFactTools_GDAL.postgresCommand(sqlString)[0][0]
if existingRow:
print 'aoi_event for {0} on {1} has already been run'.format(aoi, dateString)
continue
# status = "Process Start"
sqlString='insert into aoi_events(aoi_event_id, aoi_id, alert_status_id) VALUES(DEFAULT,\'{}\', 2) RETURNING aoi_event_id;'.format(aoi)
resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString)
event_id = resultsTup[0][0]
callZonalStats(aoi, swirs, cloudMasks, gapMasks, 'tmpStatsFile.csv')
stats=[]
with open('/tmp/tmpStatsFile.csv', 'rb') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
s=(row['Sum'])
if s:
h=int(float(s))
else:
h=0
stats.append((int(row['Unique region ID']), float(row['Area in square meters']),
int(row['Raster value']), h, int(row['Count']), int(row['Count*maxVal'])))
#sorted_by_region_ID = sorted(stats, key=lambda tup: tup[0])
def patchVal(v):return filter(lambda tup: tup[2]==v, stats)
sumArea=reduce(lambda x,y: x+y, map(lambda x: x[1], stats), 0)*0.0002471044
maxArea=reduce(lambda x,y: max(x,y), map(lambda x: x[1], stats), 0)*0.0002471044
acresChange=reduce(lambda x,y: x+y, map(lambda x: x[1], patchVal(1)), 0)*0.0002471044
acresAnalyzed=acresChange + reduce(lambda x,y: x+y, map(lambda x: x[1], patchVal(0)), 0)*0.0002471044
percentAnalyzed=float(acresChange)/float(aoiAcres)*100
percentChange=(float(acresChange)/float(acresAnalyzed))*100
largestPatch=reduce(lambda x,y: max(x,y), map(lambda x: x[1], patchVal(1)), 0)*0.0002471044
patchCount=len(patchVal(1))
smallestPatch=reduce(lambda x,y: min(x,y), map(lambda x: x[1], patchVal(1)), 0)*0.0002471044
sumOfSum=reduce(lambda x,y: x+y, map(lambda x: x[3], patchVal(1)), 0)
sumOfMax=reduce(lambda x,y: x+y, map(lambda x: x[5], patchVal(1)), 0)
if sumOfMax:
maxPatchSeverity=(float(sumOfSum)/float(sumOfMax))*100
else:
maxPatchSeverity=0
resultsTup = landsatFactTools_GDAL.postgresCommand('select * from write_aoi_events({0},{1},{2},{3},{4},{5},{6},{7},{8},\'{9}\''.format(
aoi, acresChange, percentChange, acresAnalyzed, percentAnalyzed,
smallestPatch, largestPatch, patchCount, event_id, dateString, maxPatchSeverity)+ ',\''+ ','.join(swirs) + '\');')
if resultsTup[0][0] == False:
print 'Table update failed'
else:
# status = "Process Complete"
sqlString='UPDATE aoi_events set ("alert_status_id") = (3) where aoi_event_id={0};'.format(event_id)
resultsTup = landsatFactTools_GDAL.postgresCommand(sqlString)