def __preProcessIncomingRequest__(self, request):
try:
if (params.DEBUG_LIVE == True):
self.logger.info(
"(" + self.name +
"):__preProcessIncomingRequest__: params.DEBUG_LIVE is set to True. There will be a lot of textual output for this run."
)
uniqueid = request['uniqueid']
self.__insertProgressDb__(uniqueid)
self.__write_JobStarted_To_DB__(
uniqueid, str(request)) # Log when Job has started.
#self.logger.info("Processing Request: "+uniqueid)
self.logger.info("(" + self.name +
"):__preProcessIncomingRequest__: uniqueid: " +
str(uniqueid))
datatype = request['datatype']
begintime = request['begintime']
endtime = request['endtime']
intervaltype = request['intervaltype']
# KS Refactor 2015 // Dirty override for download operations type.
operationtype = request[
'operationtype'] # Original line (just get the operation param)
# KS Refactor 2015 // Dirty override for download operations type.
#self.mathop = pMath.mathOperations(operationtype,1,params.dataTypes[datatype]['fillValue'],None)
#self.logger.info("("+self.name+"):__preProcessIncomingRequest__: DEBUG: About to do the DIRTY OVERRIDE! operationtype value: "+ str(operationtype))
if (params.parameters[operationtype][1] == 'download'):
# If this is a download dataset request, set the self.mathop prop to 0 (or 'max' operator.. this is just so I don't have to refactor a ton of code just to get this feature working at this time... note: Refactor of this IS needed!)
self.mathop = pMath.mathOperations(
0, 1, params.dataTypes[datatype]['fillValue'], None)
# Additional customized code for download jobs
self.isDownloadJob = True
self.dj_OperationName = "download"
else:
# This is pass through for all normal requests..
self.mathop = pMath.mathOperations(
operationtype, 1, params.dataTypes[datatype]['fillValue'],
None)
self.isDownloadJob = False
self.dj_OperationName = "NotDLoad"
#self.logger.info("("+self.name+"):__preProcessIncomingRequest__: DEBUG: MADE IT PASSED THE DIRTY OVERRIDE! requestID: "+uniqueid)
size = params.getGridDimension(int(datatype))
dates = dproc.getListOfTimes(begintime, endtime, intervaltype)
if (intervaltype == 0):
dates = params.dataTypes[datatype]['indexer'].cullDateList(
dates)
# KS Developer Note: The issue here is that I need to only cut simple rectangle shaped images out of the data.
# All I really need is the largest bounding box that encompases all points (regardless of how complex the original polygon was)
# Seems simple right? :)
# The other part of this issue is that this only needs to happen on download data requests. If I change the code for all requests, it becomes less efficient for stats type jobs.
#self.logger.info("("+self.name+"):__preProcessIncomingRequest__: DEBUG ALERT: Right now, only user drawn polygons are supported for download requests. Need to write a function that gets geometry values from features as well.. VERY IMPORTANT TODO BEFORE RELEASE!!")
#geometry_ToPass = None
polygon_Str_ToPass = None
dataTypeCategory = params.dataTypes[datatype][
'data_category'] # == 'ClimateModel'
geotransform, wkt = rp.getSpatialReference(int(datatype))
self.logger.info(
"*****************************code*****************************"
)
# User Drawn Polygon
if ('geometry' in request):
if (params.DEBUG_LIVE == True):
self.logger.info(
"(" + self.name +
"):__preProcessIncomingRequest__: DEBUG: GEOMETRY FOUND (POLYGON DRAWN BY USER)"
)
# Get the polygon string
polygonstring = request['geometry']
# Process input polygon string
geometry = geoutils.decodeGeoJSON(polygonstring)
#geometry = geoutils.decodeGeoJSON(polygon_Str_ToPass)
if (params.DEBUG_LIVE == True):
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : polygonstring (request['geometry']) value: "
+ str(polygonstring))
# Needed for download types
#polygon_Str_ToPass = polygonstring
# IMPORTANT BEFORE RELEASING ALL DATA DOWNLOADS
# running the below if statement part breaks the mask generation...
# Latest test shows that CHIRPS dataset actually produces a working image
# and that seasonal forecasts do as well...
# Lets see if there is a way to keep the mask on file downloads..
# BillyZ Fixed this
self.logger.info('*****dataTypeCategory:*****' +
dataTypeCategory)
if ((self.dj_OperationName == "download") |
(dataTypeCategory == 'ClimateModel') |
(dataTypeCategory == 'CHIRPS')):
#if(self.dj_OperationName == "download"):
#if((self.dj_OperationName == "download") | (dataTypeCategory == 'ClimateModel')| (dataTypeCategory == 'NDVI')):
self.logger.info(
"*****************************BYPASSED*****************************"
)
polygon_Str_ToPass = extractTif.get_ClimateDataFiltered_PolygonString_FromSingleGeometry(
geometry)
if (params.DEBUG_LIVE == True):
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : polygon_Str_ToPass (request['geometry']) value: "
+ str(polygon_Str_ToPass))
geometry = geoutils.decodeGeoJSON(polygon_Str_ToPass)
bounds, mask = mg.rasterizePolygon(geotransform, size[0],
size[1], geometry)
else:
self.logger.info(
"*****************************Not bypassed*****************************"
)
polygon_Str_ToPass = polygonstring
bounds, mask = mg.rasterizePolygon(geotransform, size[0],
size[1], geometry)
#polygon_Str_ToPass = extractTif.get_ClimateDataFiltered_PolygonString_FromSingleGeometry(geometry)
if (params.DEBUG_LIVE == True):
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : polygon_Str_ToPass (request['geometry']) value: "
+ str(polygon_Str_ToPass))
#geometry = geoutils.decodeGeoJSON(polygon_Str_ToPass)
#bounds, mask = mg.rasterizePolygon(geotransform, size[0], size[1], geometry)
# ks refactor // Getting geometry and bounds info.
#geometry_ToPass = geometry
if (params.DEBUG_LIVE == True):
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : polygonstring (request['geometry']) value: "
+ str(polygonstring))
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : (user defined polygon) geometry value: "
+ str(geometry))
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : bounds value: " +
str(bounds))
# User Selected a Feature
elif ('layerid' in request):
if (params.DEBUG_LIVE == True):
self.logger.info(
"(" + self.name +
"):__preProcessIncomingRequest__: DEBUG: LAYERID FOUND (FEATURE SELECTED BY USER)"
)
layerid = request['layerid']
featureids = request['featureids']
geometries = sf.getPolygons(layerid, featureids)
if (params.DEBUG_LIVE == True):
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : (FeatureSelection) geometries value: "
+ str(geometries))
# For Download data types, convert all of the geometries into a bounding box that covers the whole map.
# RIGHT HERE!!
#if(self.dj_OperationName == "download"):
if ((self.dj_OperationName == "download") |
(dataTypeCategory == 'ClimateModel')):
# Convert all the geometries to the rounded polygon string, and then pass that through the system
polygonstring = extractTif.get_ClimateDataFiltered_PolygonString_FromMultipleGeometries(
geometries)
polygon_Str_ToPass = polygonstring
geometry = geoutils.decodeGeoJSON(polygonstring)
bounds, mask = mg.rasterizePolygon(geotransform, size[0],
size[1], geometry)
else:
bounds, mask = mg.rasterizePolygons(
geotransform, size[0], size[1], geometries)
#Break up date
#Check for cached polygon
#if no cached polygon exists rasterize polygon
clippedmask = mask[bounds[2]:bounds[3], bounds[0]:bounds[1]]
#self.logger.debug("("+self.name+"):__preProcessIncomingRequest__ : debug : Value of 'mask': " + str(mask))
#self.logger.debug("("+self.name+"):__preProcessIncomingRequest__ : debug : Value of 'clippedmask': " + str(clippedmask))
self.__writeMask__(uniqueid, clippedmask, bounds)
del mask
del clippedmask
worklist = []
for date in dates:
workid = uu.getUUID()
#workdict = {'uid':uniqueid,'workid':workid,'bounds':bounds,'datatype':datatype,'operationtype':operationtype, 'intervaltype':intervaltype}
workdict = {
'uid': uniqueid,
'workid': workid,
'bounds': bounds,
'datatype': datatype,
'operationtype': operationtype,
'intervaltype': intervaltype,
'polygon_Str_ToPass': polygon_Str_ToPass
} #'geometryToClip':geometry_ToPass}
if (intervaltype == 0):
workdict['year'] = date[2]
workdict['month'] = date[1]
workdict['day'] = date[0]
dateObject = dateutils.createDateFromYearMonthDay(
date[2], date[1], date[0])
workdict['isodate'] = dateObject.strftime(
params.intervals[0]['pattern'])
workdict['epochTime'] = dateObject.strftime("%s")
worklist.extend([workdict])
elif (intervaltype == 1):
workdict['year'] = date[1]
workdict['month'] = date[0]
dateObject = dateutils.createDateFromYearMonth(
date[1], date[0])
workdict['isodate'] = dateObject.strftime(
params.intervals[0]['pattern'])
workdict['epochTime'] = dateObject.strftime("%s")
worklist.extend([workdict])
elif (intervaltype == 2):
workdict['year'] = date
dateObject = dateutils.createDateFromYear(date)
workdict['isodate'] = dateObject.strftime(
params.intervals[0]['pattern'])
workdict['epochTime'] = dateObject.strftime("%s")
worklist.extend([workdict])
# ks Refactor // Understanding how the work is distributed among worker threads.
if (params.DEBUG_LIVE == True):
self.logger.debug(
"(" + self.name +
"):__preProcessIncomingRequest__ : worklist array value: "
+ str(worklist))
return None, worklist
except Exception as e:
self.logger.warn(
"(" + self.name +
"):Error processing Request in HeadProcessor: uniqueid: " +
str(uniqueid) + " Exception Error Message: " + str(e))
return e, None
def submitDataRequest(request):
'''
Submit a data request for processing
:param request: actual request that contains the data needed to put together the request for
processing
'''
logger.debug("Submitting Data Request")
error = []
#Get datatype
try:
logger.debug("looking at getting datatype" + str(request))
datatype = int(request.GET["datatype"])
print datatype
except KeyError:
logger.warn("issue with datatype" + str(request))
#error.append("Datatype not supported")
#Get begin and end times
try:
begintime = request.GET["begintime"]
print begintime
except KeyError:
logger.warn("issue with begintime" + str(request))
#error.append("Error with begintime")
try:
endtime = request.GET["endtime"]
except KeyError:
logger.warn("issue with endtime" + str(request))
#error.append("Error with endtime")
#Get interval type
try:
intervaltype = int(request.GET["intervaltype"])
except KeyError:
logger.warn("issue with intervaltype" + str(request))
#error.append("Error with intervaltype")
#Get geometry from parameter
#Or extract from shapefile
geometry = None
featureList = False
if "layerid" in request.GET:
try:
layerid = str(request.GET["layerid"])
fids = str(request.GET["featureids"]).split(',')
featureids = []
for fid in fids:
value, isInt = intTryParse(fid)
if (isInt == True):
featureids.append(value)
featureList = True
##Go get geometry
logger.debug(
"submitDataRequest: Loaded feature ids, featureids: " +
str(featureids))
except KeyError:
logger.warn("issue with finding geometry")
#error.append("Error with finding geometry: layerid:"+str(layerid)+" featureid: "+str(featureids))
else:
try:
polygonstring = request.GET["geometry"]
geometry = decodeGeoJSON(polygonstring)
#create geometry
except KeyError:
logger.warn("Problem with geometry")
#error.append("problem decoding geometry "+str(polygonstring))
if geometry is None:
logger.warn("Problem in that the geometry is a problem")
else:
logger.warn(geometry)
try:
operationtype = int(request.GET["operationtype"])
except KeyError:
logger.warn("issue with operationtype" + str(request.GET))
#error.append("Error with operationtype")
uniqueid = uutools.getUUID()
logger.info("Submitting " + uniqueid)
#Submit requests to the ipcluster service to get data
if (len(error) == 0):
dictionary = {
'uniqueid': uniqueid,
'datatype': datatype,
'begintime': begintime,
'endtime': endtime,
'intervaltype': intervaltype,
'operationtype': operationtype
}
if (featureList == True):
dictionary['layerid'] = layerid
dictionary['featureids'] = featureids
else:
dictionary['geometry'] = polygonstring
##logger.info("submitting ",dictionary)
context = zmq.Context()
sender = context.socket(zmq.PUSH)
sender.connect("ipc:///tmp/servir/Q1/input")
sender.send_string(json.dumps(dictionary))
return processCallBack(request, json.dumps([uniqueid]),
"application/json")
else:
return processCallBack(request, json.dumps(error), "application/json")
def _MonthlyRainfallAnalysis__make_CHIRPS_workList(
uniqueid, request, datatype_uuid_for_CHIRPS,
datatype_uuid_for_SeasonalForecast):
worklist = []
sub_type_name = 'CHIRPS_REQUEST' # Choices for now are: 'CHIRPS_REQUEST' and 'SEASONAL_FORECAST'
datatype = chirps_dataType # Much of the copy/paste code already references this as 'datatype'
begintime = chirps_dateRange_earlyMonth + "/" + chirps_dateRange_earlyDay + "/" + chirps_dateRange_earlyYear
endtime = chirps_dateRange_lateMonth + "/" + chirps_dateRange_lateDay + "/" + chirps_dateRange_lateYear
intervaltype = 0 # Daily
operationtype = 5 # 5 == average, 0 == max, 1 == min
size = params.getGridDimension(int(datatype))
dates = dproc.getListOfTimes(begintime, endtime, intervaltype)
if (intervaltype == 0):
dates = params.dataTypes[datatype]['indexer'].cullDateList(dates)
# PROCESS GEOMETRY STUFF NOW
geotransform, wkt = rp.getSpatialReference(int(datatype))
# User Drawn Polygon
#bounds = None
#mask = None
polygon_Str_ToPass = None
if ('geometry' in request):
# Get the polygon string
polygonstring = request['geometry']
# Process input polygon string
geometry = geoutils.decodeGeoJSON(polygonstring)
# # this is not a download type or a climate model type --START
polygon_Str_ToPass = polygonstring
bounds, mask = mg.rasterizePolygon(geotransform, size[0], size[1],
geometry)
# # this is not a download type or a climate model type --END
# User Selected a Feature
elif ('layerid' in request):
layerid = request['layerid']
featureids = request['featureids']
geometries = sf.getPolygons(layerid, featureids)
# If we MUST have a polygon_Str_ToPass, uncomment the next two lines.
#polygonstring = extractTif.get_ClimateDataFiltered_PolygonString_FromMultipleGeometries(geometries)
#polygon_Str_ToPass = polygonstring
# # this is not a download type or a climate model type --START
bounds, mask = mg.rasterizePolygons(geotransform, size[0], size[1],
geometries)
# # this is not a download type or a climate model type --END
# if no cached polygon exists rasterize polygon
clippedmask = mask[bounds[2]:bounds[3], bounds[0]:bounds[1]]
# TODO, Create System of multiple masks for the Monthly Analysis process.
# self.__writeMask__(uniqueid, clippedmask, bounds) # mst.writeHMaskToTempStorage(uid,array,bounds)
#mst.writeHMaskToTempStorage(uniqueid,clippedmask,bounds) # NEED TO FIND OUT HOW AND WHERE THIS IS USED IN THE DEEPER PROCESSING CODE, AND MAKE A SYSTEM THAT WILL ALLOW MORE THAN JUST ONE MASK..
mst.writeHMaskToTempStorage(datatype_uuid_for_CHIRPS, clippedmask, bounds)
del mask
del clippedmask
# Build the worklist for each date in the dates
for date in dates:
workid = uu.getUUID()
workdict = {
'uid': uniqueid,
'workid': workid,
'datatype': datatype,
'operationtype': operationtype,
'intervaltype': intervaltype,
'bounds': bounds,
'polygon_Str_ToPass': polygon_Str_ToPass,
'datatype_uuid_for_CHIRPS': datatype_uuid_for_CHIRPS,
'datatype_uuid_for_SeasonalForecast':
datatype_uuid_for_SeasonalForecast,
'current_mask_and_storage_uuid':
datatype_uuid_for_CHIRPS, # Only one chirps type request needed so using same uuid
'sub_type_name': sub_type_name,
'derived_product': True,
'special_type': 'MonthlyRainfallAnalysis'
}
# Daily dates processing # if (intervaltype == 0): # It is in this case, daily.
workdict['year'] = date[2]
workdict['month'] = date[1]
workdict['day'] = date[0]
dateObject = dateutils.createDateFromYearMonthDay(
date[2], date[1], date[0])
workdict['isodate'] = dateObject.strftime(
params.intervals[0]['pattern'])
workdict['epochTime'] = dateObject.strftime("%s")
worklist.extend(
[workdict]
) # Basically adds the entire workdict object to the worklist (could also be written as, worklist.append(workdict)
return worklist