def processInputCrossSection( self, request: TaskRequest, node: OpNode, inputs: EDASDataset ) -> EDASDataset:
self.logger.info( f"Computing WorldClim fields for domains: { [str(d) for d in request.operationManager.domains.domains.values()] }" )
version = node.getParm("version", "mean")
self.results: Dict[str,EDASArray] = {}
moistID = node.getParm("moist", "moist")
moistVar = inputs.findArray(moistID)
assert moistVar is not None, f"Can't locate moisture variable {moistID} in inputs: {inputs.ids}"
tempID = node.getParm("temp","temp")
tempVar: EDASArray = inputs.findArray(tempID)
hscale = float( node.getParm( "hscale", 1.0 ) )
tscale = float( node.getParm( "tscale", 1.0 ) )
if tempVar is None:
taxis = "t"
tempMaxID = node.getParm("maxTemp", "maxTemp")
tempMinID = node.getParm("minTemp", "minTemp")
Tmax: EDASArray = inputs.findArray(tempMaxID)
Tmin: EDASArray = inputs.findArray(tempMinID)
humid: EDASArray = moistVar
assert Tmax is not None and Tmax is not None, f"Must specify the temperature input variables using either the '{tempID}' parameter (hourly) or the '{tempMaxID}','{tempMinID}' parameters (monthly)"
else:
taxis = "m"
tempVar: EDASArray = inputs.findArray( tempID )
assert tempVar is not None, f"Can't locate temperature variable {tempID} in inputs: {inputs.ids}"
tempVar = self.toCelcius( tempVar )
dailyTmaxmin = tempVar.timeResample("1D","max,min")
Tmaxmin = dailyTmaxmin[0].timeAgg("month", "max,min")
Tmax: EDASArray = Tmaxmin[0]
Tmin: EDASArray = Tmaxmin[1]
humid = moistVar.timeAgg("month", version)[0]
results = self.computeIndices( Tmax.compute(), Tmin.compute(), humid.compute(), version = version, hscale=hscale, tscale=tscale, taxis=taxis )
return self.buildProduct(inputs.id, request, node, results, inputs.attrs)
def processInputCrossSection(self, request: TaskRequest, node: OpNode,
inputDset: EDASDataset) -> EDASDataset:
nModes = int(node.getParm("modes", 16))
center = bool(node.getParm("center", "false"))
merged_input_data, info = self.get_input_array(inputDset)
shapes = info['shapes']
slicers = info['slicers']
solver = Eof(merged_input_data, center=center)
results = []
for iMode, eofs_result in enumerate(solver.eofs(neofs=nModes)):
for iVar, eofs_data in enumerate(
self.getResults(eofs_result, slicers, shapes)):
input = inputDset.inputs[iVar]
results.append(
EDASArray("-".join(["eof-", str(iMode), input.name]),
input.domId, eofs_data))
pcs_result = solver.pcs(npcs=nModes)
pcs = EDASArray(
"pcs[" + inputDset.id + "]", inputDset.inputs[0].domId,
EDASArray.cleanupCoords(pcs_result, {
"mode": "m",
"pc": "m"
}).transpose())
results.append(pcs)
fracs = solver.varianceFraction(neigs=nModes)
pves = [str(round(float(frac * 100.), 1)) + '%' for frac in fracs]
for result in results:
result["pves"] = str(pves)
return EDASDataset.init(self.renameResults(results, node),
inputDset.attrs)
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
data = variable.persist()
norm = bool(node.getParm("norm", False))
grouping = node.getParm("groupby", 't.month')
climatology = data.groupby(grouping).mean('t')
anomalies = data.groupby(grouping) - climatology
if norm:
anomalies = anomalies.groupby(grouping) / data.groupby(grouping).std('t')
return variable.updateXa( anomalies, "decycle" )
def preprocessInputs(self, request: TaskRequest, op: OpNode,
inputDataset: EDASDataset) -> EDASDataset:
# interp_na = bool(op.getParm("interp_na", False))
# if interp_na: inputs: Dict[str,EDASArray] = { id: input.updateXa( input.xr.interpolate_na( dim="t", method='linear' ),"interp_na" ) for (id, input) in inputDset.arrayMap.items() }
# else: inputs: Dict[str,EDASArray] = { id: input for (id, input) in inputDset.arrayMap.items() }
if op.isSimple and not self.requiresAlignment:
result = inputDataset
else:
resultArrays: OrderedDict[str, EDASArray] = OrderedDict()
arrayList = list(inputDataset.arrayMap.values())
for aid, array in inputDataset.arrayMap.items():
unapplied_domains: Set[str] = array.unapplied_domains(
arrayList, op.domain)
if len(unapplied_domains) > 0:
merged_domain: str = request.intersectDomains(
unapplied_domains, False)
processed_domain: Domain = request.cropDomain(
merged_domain, arrayList)
sub_array = array.subset(processed_domain,
unapplied_domains)
resultArrays[aid] = sub_array
else:
resultArrays[aid] = array
resultDataset = EDASDataset(resultArrays, inputDataset.attrs)
alignmentTarget = resultDataset.getAlignmentVariable(
op.getParm("align", "lowest"))
preprop_result = resultDataset.align(alignmentTarget)
result: EDASDataset = preprop_result.groupby(op.grouping).resample(
op.resampling)
print(" $$$$ processInputCrossSection: " + op.name + " -> " +
str(result.ids))
return result.purge()
def buildProduct(self, dsid: str, request: TaskRequest, node: OpNode,
result_arrays: List[EDASArray], attrs: Dict[str, str]):
result_dset = EDASDataset.init(self.renameResults(result_arrays, node),
attrs)
for parm in ["product", "archive"]:
result_dset[parm] = node.getParm(parm, "")
result_dset.name = node.getResultId(dsid)
return self.signResult(result_dset, request, node)
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
variable.persist()
axisIndex = variable.getAxisIndex( node.axes, 0, 0 )
dim = variable.xr.dims[axisIndex]
window_size = node.getParm("wsize", variable.xr.shape[axisIndex]//8 )
lowpass_args = { dim:int(window_size), "center":True, "min_periods": 1 }
lowpass = variable.xr.rolling(**lowpass_args).mean()
return EDASArray( variable.name, variable.domId, lowpass )
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
modelId = node.getParm( "model", "model" )
model = ModelOps.loadModel( self.archivePath( modelId, {} ) )
weights = model.get_weights()
print( "MODEL WEIGHTS: " + str(weights))
input_size = weights[0].shape[0]
input = KerasModel.getNetworkInput( node, variable, input_size )
return KerasModel.map( "predict", model, input )
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
data = variable.persist()
axisIndex = variable.getAxisIndex( node.axes, 0, 0 )
dim = data.dims[axisIndex]
window_size = node.getParm("wsize", data.shape[axisIndex] // 8)
detrend_args = {dim: int(window_size), "center": True, "min_periods": 1}
trend = data.rolling(**detrend_args).mean()
detrend: EDASArray = variable - variable.updateXa(trend, "trend")
return detrend
def processInputCrossSection( self, request: TaskRequest, train_node: OpNode, inputDset: EDASDataset ) -> EDASDataset:
self.reseed()
nIterations = train_node.getParm( "iterations", 1 )
self.logger.info( "Executing fit-model {} times".format(nIterations) )
val_loss_values = []
for idx in range( nIterations ):
performanceTracker = PerformanceTracker( self.stop_condition )
master_node, model = self.getModel( train_node )
self.buildLearningModel( master_node, model )
self.fitModel( master_node, train_node, model, inputDset, performanceTracker )
val_loss_values.append( performanceTracker.minValLoss )
self.logger.info( "Worker training results: val losses = " + str(val_loss_values) )
return self.buildResultDataset(inputDset, train_node)
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
modelId = node.getParm( "model", "model" )
model: Model = ModelOps.loadModel( self.archivePath( modelId, {} ) )
out_diff = K.mean((model.layers[-1].output - 1) ** 2)
grad = K.gradients(out_diff, [model.input])[0]
grad /= K.maximum(K.sqrt(K.mean(grad ** 2)), K.epsilon())
iterate = K.function( [model.input, K.learning_phase()], [out_diff, grad] )
input_img_data = np.zeros( shape=variable.xrArray.shape )
self.logger.info("Back Projection Map, Iterations:")
for i in range(20):
out_loss, out_grad = iterate([input_img_data, 0])
input_img_data -= out_grad * 0.1
self.logger.info( str(i) + ": loss = " + str(out_loss) )
return EDASArray( "Back Projection Map", variable.domId, xa.DataArray(input_img_data) )
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
cacheId = node.getParm("result")
EDASKCacheMgr.cache(cacheId, variable)
return variable
def processInputCrossSection(self, request: TaskRequest, node: OpNode,
inputs: EDASDataset) -> EDASDataset:
self.logger.info(
f"Computing WorldClim fields for domains: { [str(d) for d in request.operationManager.domains.domains.values()] }"
)
version = node.getParm("version", "mean")
self.results: Dict[str, EDASArray] = {}
moistID = node.getParm("moist", "moist")
moistVar = inputs.findArray(moistID)
assert moistVar is not None, f"Can't locate moisture variable {moistID} in inputs: {inputs.ids}"
tempID = node.getParm("temp", "temp")
tempVar: EDASArray = inputs.findArray(tempID)
pscale = float(node.getParm("pscale", 1.0))
if tempVar is None:
taxis = "t"
tempMaxID = node.getParm("maxTemp", "maxTemp")
tempMinID = node.getParm("minTemp", "minTemp")
Tmax: EDASArray = inputs.findArray(tempMaxID)
Tmin: EDASArray = inputs.findArray(tempMinID)
monthlyPrecip: EDASArray = moistVar
assert Tmax is not None and Tmax is not None, f"Must specify the temperature input variables using either the '{tempID}' parameter (hourly) or the '{tempMaxID}','{tempMinID}' parameters (monthly)"
else:
taxis = "m"
tempVar: EDASArray = inputs.findArray(tempID)
assert tempVar is not None, f"Can't locate temperature variable {tempID} in inputs: {inputs.ids}"
tempVar = self.toCelcius(tempVar)
dailyTmaxmin = tempVar.timeResample("1D", "max,min")
Tmaxmin = dailyTmaxmin[0].timeAgg("month", "max,min")
Tmax: EDASArray = Tmaxmin[0]
Tmin: EDASArray = Tmaxmin[1]
monthlyPrecip = moistVar.timeAgg("month", version)[0]
self.logger.info(f" --> version = {version}, pscale = {pscale}")
if pscale != float(1.0): monthlyPrecip = monthlyPrecip * pscale
Tave = (Tmax + Tmin) / 2.0
TKave = Tave + 273.15
Trange = (Tmax - Tmin) / 2.0
self.start_time = time.time()
Tave.persist()
monthlyPrecip.persist()
# self.logger.info( f"Tmax sample: {Tmax.xr.to_masked_array()[2,10:12,10:12]}")
# self.logger.info( f"Tmin sample: {Tmin.xr.to_masked_array()[2,10:12,10:12]}")
# self.logger.info( f"Trange sample: {Trange.xr.to_masked_array()[2,10:12,10:12]}")
self.setResult('1', Tave.ave([taxis], name="bio1"))
self.setResult('2', Trange.ave([taxis], name="bio2"))
self.setResult('4', Tave.std([taxis], name="bio4"))
self.setResult('4a', ((TKave.std([taxis], keep_attrs=True) * 100) /
(self.results['1'] + 273.15)).rename("bio4a"))
self.setResult('5', Tmax.max([taxis], name="bio5"))
self.setResult('6', Tmin.min([taxis], name="bio6"))
self.setResult('7',
(self.results['5'] - self.results['6']).rename("bio7"))
self.setResult(
'8',
self.getValueForSelectedQuarter(taxis, Tave, monthlyPrecip, "max",
"bio8"))
self.setResult(
'9',
self.getValueForSelectedQuarter(taxis, Tave, monthlyPrecip, "min",
"bio9"))
self.setResult('3', ((self.results['2'] * 100) /
self.results['7']).rename("bio3"))
self.setResult(
'10',
self.getValueForSelectedQuarter(taxis, Tave, Tave, "max", "bio10"))
self.setResult(
'11',
self.getValueForSelectedQuarter(taxis, Tave, Tave, "min", "bio11"))
self.setResult('12', monthlyPrecip.sum([taxis], name="bio12"))
self.setResult('13', monthlyPrecip.max([taxis], name="bio13"))
self.setResult('14', monthlyPrecip.min([taxis], name="bio14"))
self.setResult('15', ((monthlyPrecip.std([taxis]) * 100) /
((self.results['12'] / 12) + 1)).rename("bio15"))
self.setResult(
'16',
self.getValueForSelectedQuarter(taxis, None, monthlyPrecip, "max",
"bio16"))
self.setResult(
'17',
self.getValueForSelectedQuarter(taxis, None, monthlyPrecip, "min",
"bio17"))
self.setResult(
'18',
self.getValueForSelectedQuarter(taxis, monthlyPrecip, Tave, "max",
"bio18"))
self.setResult(
'19',
self.getValueForSelectedQuarter(taxis, monthlyPrecip, Tave, "min",
"bio19"))
results: List[EDASArray] = [
moistVar.updateXa(result.xr, "bio-" + index)
for index, result in self.results.items()
]
self.logger.info(
f"Completed WorldClim computation, elapsed = {(time.time()-self.start_time)/60.0} m"
)
return self.buildProduct(inputs.id, request, node, results,
inputs.attrs)
def processVariables(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> List[EDASArray]:
variable.persist()
freq = node.getParm("freq", 'month')
operation = str(node.getParm("op", 'mean')).lower()
return variable.timeResample(freq, operation)
def processVariables(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> List[EDASArray]:
variable.persist()
period = node.getParm("period", 'month')
operation = str(node.getParm("op", 'mean')).lower()
return variable.timeAgg(period, operation)