def processInputCrossSection( self, request: TaskRequest, train_node: OpNode, inputDset: EDASDataset ) -> EDASDataset:
self.reseed()
nIterations = train_node.getParm( "iterations", 1 )
self.logger.info( "Executing fit-model {} times".format(nIterations) )
val_loss_values = []
for idx in range( nIterations ):
performanceTracker = PerformanceTracker( self.stop_condition )
master_node, model = self.getModel( train_node )
self.buildLearningModel( master_node, model )
self.fitModel( master_node, train_node, model, inputDset, performanceTracker )
val_loss_values.append( performanceTracker.minValLoss )
self.logger.info( "Worker training results: val losses = " + str(val_loss_values) )
return self.buildResultDataset(inputDset, train_node)
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
data = variable.persist()
axisIndex = variable.getAxisIndex(node.axes, 0, 0)
dim = data.dims[axisIndex]
window_size = node.getParm("wsize", data.shape[axisIndex] // 8)
detrend_args = {
dim: int(window_size),
"center": True,
"min_periods": 1
}
trend = data.rolling(**detrend_args).mean()
detrend: EDASArray = variable - variable.updateXa(trend, "trend")
return detrend
def loadModel(cls, modelPath: str ) -> Model:
modelData: EDASDataset = EDASDataset.open_dataset( modelPath )
layersSpec = modelData["layers"]
assert layersSpec, "Missing levels spec in model data"
layerNodes = [ OpNode.deserialize(spec) for spec in layersSpec.split(";") ]
layers = KerasModel.instantiateLayers( layerNodes )
model = KerasModel.getModel( layers )
initial_weights = model.get_weights()
model.compile(loss="mse", optimizer=SGD(), metrics=['accuracy'])
weights = KerasModel.packWeights( "finalWts", modelData )
print( "INITIAL WEIGHTS: " + str( initial_weights ) )
print( "INPUT WEIGHTS: " + str( weights ) )
model.set_weights( weights )
return model
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
modelId = node.getParm( "model", "model" )
model: Model = ModelOps.loadModel( self.archivePath( modelId, {} ) )
out_diff = K.mean((model.layers[-1].output - 1) ** 2)
grad = K.gradients(out_diff, [model.input])[0]
grad /= K.maximum(K.sqrt(K.mean(grad ** 2)), K.epsilon())
iterate = K.function( [model.input, K.learning_phase()], [out_diff, grad] )
input_img_data = np.zeros( shape=variable.xrArray.shape )
self.logger.info("Back Projection Map, Iterations:")
for i in range(20):
out_loss, out_grad = iterate([input_img_data, 0])
input_img_data -= out_grad * 0.1
self.logger.info( str(i) + ": loss = " + str(out_loss) )
return EDASArray( "Back Projection Map", variable.domId, xa.DataArray(input_img_data) )
def preprocessInputs( self, request: TaskRequest, op: OpNode, inputDataset: EDASDataset ) -> EDASDataset:
# interp_na = bool(op.getParm("interp_na", False))
# if interp_na: inputs: Dict[str,EDASArray] = { id: input.updateXa( input.xr.interpolate_na( dim="t", method='linear' ),"interp_na" ) for (id, input) in inputDset.arrayMap.items() }
# else: inputs: Dict[str,EDASArray] = { id: input for (id, input) in inputDset.arrayMap.items() }
if op.isSimple and not self.requiresAlignment:
result = inputDataset
else:
resultArrays: OrderedDict[str,EDASArray] = OrderedDict()
arrayList = list(inputDataset.arrayMap.values())
for aid,array in inputDataset.arrayMap.items():
unapplied_domains: Set[str] = array.unapplied_domains(arrayList, op.domain)
if len( unapplied_domains ) > 0:
merged_domain: str = request.intersectDomains(unapplied_domains, False)
processed_domain: Domain = request.cropDomain(merged_domain, arrayList )
sub_array = array.subset( processed_domain, unapplied_domains )
resultArrays[aid] = sub_array
else:
resultArrays[aid] = array
resultDataset = EDASDataset( resultArrays, inputDataset.attrs )
alignmentTarget = resultDataset.getAlignmentVariable( op.getParm("align","lowest") )
preprop_result = resultDataset.align( alignmentTarget )
result: EDASDataset = preprop_result.groupby( op.grouping ).resample( op.resampling )
print( " $$$$ processInputCrossSection: " + op.name + " -> " + str( result.ids ) )
return result.purge()
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
cacheId = node.getParm("result")
EDASKCacheMgr.cache(cacheId, variable)
return variable
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
gridSpec = node.getParam(Param("gridder", True))
return variable.regrid(gridSpec)
def processInputCrossSection(self, request: TaskRequest, node: OpNode,
inputs: EDASDataset) -> EDASDataset:
self.logger.info(
f"Computing WorldClim fields for domains: { [str(d) for d in request.operationManager.domains.domains.values()] }"
)
version = node.getParm("version", "mean")
self.results: Dict[str, EDASArray] = {}
moistID = node.getParm("moist", "moist")
moistVar = inputs.findArray(moistID)
assert moistVar is not None, f"Can't locate moisture variable {moistID} in inputs: {inputs.ids}"
tempID = node.getParm("temp", "temp")
tempVar: EDASArray = inputs.findArray(tempID)
pscale = float(node.getParm("pscale", 1.0))
if tempVar is None:
taxis = "t"
tempMaxID = node.getParm("maxTemp", "maxTemp")
tempMinID = node.getParm("minTemp", "minTemp")
Tmax: EDASArray = inputs.findArray(tempMaxID)
Tmin: EDASArray = inputs.findArray(tempMinID)
monthlyPrecip: EDASArray = moistVar
assert Tmax is not None and Tmax is not None, f"Must specify the temperature input variables using either the '{tempID}' parameter (hourly) or the '{tempMaxID}','{tempMinID}' parameters (monthly)"
else:
taxis = "m"
tempVar: EDASArray = inputs.findArray(tempID)
assert tempVar is not None, f"Can't locate temperature variable {tempID} in inputs: {inputs.ids}"
tempVar = self.toCelcius(tempVar)
dailyTmaxmin = tempVar.timeResample("1D", "max,min")
Tmaxmin = dailyTmaxmin[0].timeAgg("month", "max,min")
Tmax: EDASArray = Tmaxmin[0]
Tmin: EDASArray = Tmaxmin[1]
monthlyPrecip = moistVar.timeAgg("month", version)[0]
self.logger.info(f" --> version = {version}, pscale = {pscale}")
if pscale != float(1.0): monthlyPrecip = monthlyPrecip * pscale
Tave = (Tmax + Tmin) / 2.0
TKave = Tave + 273.15
Trange = (Tmax - Tmin) / 2.0
self.start_time = time.time()
Tave.persist()
monthlyPrecip.persist()
# self.logger.info( f"Tmax sample: {Tmax.xr.to_masked_array()[2,10:12,10:12]}")
# self.logger.info( f"Tmin sample: {Tmin.xr.to_masked_array()[2,10:12,10:12]}")
# self.logger.info( f"Trange sample: {Trange.xr.to_masked_array()[2,10:12,10:12]}")
self.setResult('1', Tave.ave([taxis], name="bio1"))
self.setResult('2', Trange.ave([taxis], name="bio2"))
self.setResult('4', Tave.std([taxis], name="bio4"))
self.setResult('4a', ((TKave.std([taxis], keep_attrs=True) * 100) /
(self.results['1'] + 273.15)).rename("bio4a"))
self.setResult('5', Tmax.max([taxis], name="bio5"))
self.setResult('6', Tmin.min([taxis], name="bio6"))
self.setResult('7',
(self.results['5'] - self.results['6']).rename("bio7"))
self.setResult(
'8',
self.getValueForSelectedQuarter(taxis, Tave, monthlyPrecip, "max",
"bio8"))
self.setResult(
'9',
self.getValueForSelectedQuarter(taxis, Tave, monthlyPrecip, "min",
"bio9"))
self.setResult('3', ((self.results['2'] * 100) /
self.results['7']).rename("bio3"))
self.setResult(
'10',
self.getValueForSelectedQuarter(taxis, Tave, Tave, "max", "bio10"))
self.setResult(
'11',
self.getValueForSelectedQuarter(taxis, Tave, Tave, "min", "bio11"))
self.setResult('12', monthlyPrecip.sum([taxis], name="bio12"))
self.setResult('13', monthlyPrecip.max([taxis], name="bio13"))
self.setResult('14', monthlyPrecip.min([taxis], name="bio14"))
self.setResult('15', ((monthlyPrecip.std([taxis]) * 100) /
((self.results['12'] / 12) + 1)).rename("bio15"))
self.setResult(
'16',
self.getValueForSelectedQuarter(taxis, None, monthlyPrecip, "max",
"bio16"))
self.setResult(
'17',
self.getValueForSelectedQuarter(taxis, None, monthlyPrecip, "min",
"bio17"))
self.setResult(
'18',
self.getValueForSelectedQuarter(taxis, monthlyPrecip, Tave, "max",
"bio18"))
self.setResult(
'19',
self.getValueForSelectedQuarter(taxis, monthlyPrecip, Tave, "min",
"bio19"))
results: List[EDASArray] = [
moistVar.updateXa(result.xr, "bio-" + index)
for index, result in self.results.items()
]
self.logger.info(
f"Completed WorldClim computation, elapsed = {(time.time()-self.start_time)/60.0} m"
)
return self.buildProduct(inputs.id, request, node, results,
inputs.attrs)
def processVariables(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> List[EDASArray]:
variable.persist()
freq = node.getParm("freq", 'month')
operation = str(node.getParm("op", 'mean')).lower()
return variable.timeResample(freq, operation)
def processVariables(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> List[EDASArray]:
variable.persist()
period = node.getParm("period", 'month')
operation = str(node.getParm("op", 'mean')).lower()
return variable.timeAgg(period, operation)
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
selection = node.findParm("sel.*")
assert ( len(node.axes) == 0 ) or ( ( len(node.axes) == 1 ) and (node.axes[0] == 't') ), "Filter currently can only operate on the time axis"
result = variable.filter( Axis.T, selection )
return result
def decompose(self, node: OpNode ) -> Tuple[ List[str], List[ List[str] ] ]:
if self._decomposable and node.hasAxis( Axis.T ) and len( node.axes ) > 1:
axis_groups = [ [ax.lower() for ax in node.axes if not ax.lower().startswith("t")], ['t'] ]
return node.axes, axis_groups
else:
return node.axes, [ node.axes ]
def renameResults(self, results: List[EDASArray], node: OpNode ) -> "OrderedDict[str,EDASArray]":
resultMap: OrderedDict[str,EDASArray] = {}
for result in results:
result["rid"] = node.getResultId(result.name)
resultMap[result.name] = result
return resultMap
def buildProduct(self, dsid: str, request: TaskRequest, node: OpNode, result_arrays: List[EDASArray], attrs: Dict[str,str] ):
result_dset = EDASDataset.init( self.renameResults(result_arrays,node), attrs )
for parm in [ "product", "archive" ]: result_dset[parm] = node.getParm( parm, "" )
result_dset.name = node.getResultId( dsid )
return self.signResult( result_dset, request, node )
