def processInputCrossSection(self, request: TaskRequest, node: OpNode,
inputDset: EDASDataset) -> EDASDataset:
nModes = int(node.getParm("modes", 16))
center = bool(node.getParm("center", "false"))
merged_input_data, info = self.get_input_array(inputDset)
shapes = info['shapes']
slicers = info['slicers']
solver = Eof(merged_input_data, center=center)
results = []
for iMode, eofs_result in enumerate(solver.eofs(neofs=nModes)):
for iVar, eofs_data in enumerate(
self.getResults(eofs_result, slicers, shapes)):
input = inputDset.inputs[iVar]
results.append(
EDASArray("-".join(["eof-", str(iMode), input.name]),
input.domId, eofs_data))
pcs_result = solver.pcs(npcs=nModes)
pcs = EDASArray(
"pcs[" + inputDset.id + "]", inputDset.inputs[0].domId,
EDASArray.cleanupCoords(pcs_result, {
"mode": "m",
"pc": "m"
}).transpose())
results.append(pcs)
fracs = solver.varianceFraction(neigs=nModes)
pves = [str(round(float(frac * 100.), 1)) + '%' for frac in fracs]
for result in results:
result["pves"] = str(pves)
return EDASDataset.init(self.renameResults(results, node),
inputDset.attrs)
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
variable.persist()
axisIndex = variable.getAxisIndex( node.axes, 0, 0 )
dim = variable.xr.dims[axisIndex]
window_size = node.getParm("wsize", variable.xr.shape[axisIndex]//8 )
lowpass_args = { dim:int(window_size), "center":True, "min_periods": 1 }
lowpass = variable.xr.rolling(**lowpass_args).mean()
return EDASArray( variable.name, variable.domId, lowpass )
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
data = variable.persist()
axisIndex = variable.getAxisIndex( node.axes, 0, 0 )
dim = data.dims[axisIndex]
window_size = node.getParm("wsize", data.shape[axisIndex] // 8)
detrend_args = {dim: int(window_size), "center": True, "min_periods": 1}
trend = data.rolling(**detrend_args).mean()
detrend: EDASArray = variable - variable.updateXa(trend, "trend")
return detrend
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
data = variable.persist()
norm = bool(node.getParm("norm", False))
grouping = node.getParm("groupby", 't.month')
climatology = data.groupby(grouping).mean('t')
anomalies = data.groupby(grouping) - climatology
if norm:
anomalies = anomalies.groupby(grouping) / data.groupby(grouping).std('t')
return variable.updateXa( anomalies, "decycle" )
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
variable.persist()
parms = self.getParameters( node, [ Param("lat"), Param("lon")])
aIndex = variable.xr.get_axis_num('t')
center: xa.DataArray = variable.selectPoint( float(parms["lon"]), float(parms["lat"]) ).xr
cmean, data_mean = center.mean(axis=aIndex), variable.xr.mean(axis=aIndex)
cstd, data_std = center.std(axis=aIndex), variable.xr.std(axis=aIndex)
cov = np.sum((variable.xr - data_mean) * (center - cmean), axis=aIndex) / variable.xr.shape[aIndex]
cor = cov / (cstd * data_std)
return EDASArray( variable.name, variable.domId, cor )
def mergeEnsembles(self, op: OpNode, dset: EDASDataset) -> EDASDataset:
self.logger.info(" ---> Merge Ensembles: ")
for xarray in dset.xarrays:
self.logger.info(
f" Variable {xarray.name}: dims: {xarray.dims}, coords: {xarray.coords.keys()} "
)
sarray: xr.DataArray = xr.concat(dset.xarrays, dim=op.ensDim)
result = EDASArray(dset.id, list(dset.domains)[0], sarray)
return EDASDataset.init(OrderedDict([(dset.id, result)]), dset.attrs)
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
selection = node.findParm("sel.*")
assert (len(node.axes) == 0) or (
(len(node.axes) == 1) and
(node.axes[0]
== 't')), "Filter currently can only operate on the time axis"
result = variable.filter(Axis.T, selection)
return result
def buildResultDataset(self, inputDset: EDASDataset, train_node: OpNode )-> EDASDataset:
result = self.bestFitResult
master_node, model = self.getModel(train_node)
arrays = OrderedDict()
loss_coord = ( "steps", range(result.train_loss_history.shape[0]) )
arrays["loss"] = EDASArray( "loss" , None, xa.DataArray( result.train_loss_history, coords=( loss_coord, ) ), [] )
arrays["val_loss"] = EDASArray( "val_loss" , None, xa.DataArray( result.val_loss_history, coords=( loss_coord, ) ), [] )
attrs = copy.deepcopy(inputDset.attrs)
attrs["loss"] = result.train_loss
attrs["val_loss"] = result.val_loss
attrs["nEpocs"] = result.nEpocs
attrs["nInstances"] = result.nInstances
attrs["merge"] = "min:val_loss"
attrs["layers"] = ";".join( [ op.serialize() for op in master_node["layerNodes"] ] )
arrays.update( KerasModel.unpackWeights( "initWts", result.initial_weights ) )
arrays.update( KerasModel.unpackWeights( "finalWts", result.final_weights ) )
attrs["nlayers"] = int( len(result.final_weights)/2 )
model.set_weights(result.final_weights)
inputData = KerasModel.getTrainingData(master_node, inputDset, 1)
targetData = KerasModel.getTargetData(train_node, inputDset, 1)
arrays["prediction"] = KerasModel.map( "prediction", model, inputData[0] )
arrays["target"] = targetData[0]
rv = EDASDataset( arrays, attrs )
return rv
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray:
modelId = node.getParm( "model", "model" )
model: Model = ModelOps.loadModel( self.archivePath( modelId, {} ) )
out_diff = K.mean((model.layers[-1].output - 1) ** 2)
grad = K.gradients(out_diff, [model.input])[0]
grad /= K.maximum(K.sqrt(K.mean(grad ** 2)), K.epsilon())
iterate = K.function( [model.input, K.learning_phase()], [out_diff, grad] )
input_img_data = np.zeros( shape=variable.xrArray.shape )
self.logger.info("Back Projection Map, Iterations:")
for i in range(20):
out_loss, out_grad = iterate([input_img_data, 0])
input_img_data -= out_grad * 0.1
self.logger.info( str(i) + ": loss = " + str(out_loss) )
return EDASArray( "Back Projection Map", variable.domId, xa.DataArray(input_img_data) )
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
gridSpec = node.getParam(Param("gridder", True))
return variable.regrid(gridSpec)
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
variable.persist()
return variable - variable.ave(node.axes)
def getHistoryDataArray(self, history: History, id: str, nEpochs: int )-> EDASArray:
data = xa.DataArray(history.history[id], coords=[ range( nEpochs ) ], dims=["epochs"])
return EDASArray( id, None, data )
def setResult(self, key: str, value: EDASArray):
self.logger.info(f"Computed value for WorldClim field bio-{key}")
value.persist()
self.results[key] = value
def processVariables(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> List[EDASArray]:
variable.persist()
freq = node.getParm("freq", 'month')
operation = str(node.getParm("op", 'mean')).lower()
return variable.timeResample(freq, operation)
def processVariables(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> List[EDASArray]:
variable.persist()
period = node.getParm("period", 'month')
operation = str(node.getParm("op", 'mean')).lower()
return variable.timeAgg(period, operation)
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
return variable.mean(node.axes)
def getWtsArray(self, array: np.ndarray, id: str )-> EDASArray:
coords = [ ('inputs',range(array.shape[0]) ), ('nodes',range(array.shape[1]) ) ] if array.ndim == 2 else [ ('nodes',range(array.shape[0]) ) ]
return EDASArray( id, None, xa.DataArray( array, coords=coords ) )
def processVariable(self, request: TaskRequest, node: OpNode,
variable: EDASArray) -> EDASArray:
variable.persist()
centered_result = variable - variable.ave(node.axes)
return centered_result / centered_result.std(node.axes)
import matplotlib.pyplot as plt
from edas.workflow.data import EDASArray
from edas.process.test import LocalTestManager
from edas.process.manager import ProcessManager, ExecHandler
from edas.config import EdasEnv
appConf = {"sources.allowed": "collection,https", "log.metrics": "true"}
EdasEnv.update(appConf)
processManager = ProcessManager.initManager(EdasEnv.parms)
kernel = WorldClimKernel()
ds_tmin = xa.open_dataset(
"https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/reanalysis/MERRA2/mon/atmos/tasmin.ncml"
)
tasmin = ds_tmin["tasmin"]
tasmin = EDASArray("tasmin", "d0", tasmin[0:12, :, :].compute())
ds_tmax = xa.open_dataset(
"https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/reanalysis/MERRA2/mon/atmos/tasmax.ncml"
)
tasmax = ds_tmax["tasmax"]
tasmax = EDASArray("tasmax", "d0", tasmax[0:12, :, :].compute())
ds_pr = xa.open_dataset(
"https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/reanalysis/MERRA2/mon/atmos/pr.ncml"
)
pr = ds_pr["pr"]
pr = EDASArray("pr", "d0", pr[0:12, :, :].compute())
results = kernel.computeIndices(tasmin, tasmax, pr, tscale=10.0, hscale=1.0e5)