def processInputCrossSection(self, request: TaskRequest, node: OpNode, inputDset: EDASDataset) -> EDASDataset: nModes = int(node.getParm("modes", 16)) center = bool(node.getParm("center", "false")) merged_input_data, info = self.get_input_array(inputDset) shapes = info['shapes'] slicers = info['slicers'] solver = Eof(merged_input_data, center=center) results = [] for iMode, eofs_result in enumerate(solver.eofs(neofs=nModes)): for iVar, eofs_data in enumerate( self.getResults(eofs_result, slicers, shapes)): input = inputDset.inputs[iVar] results.append( EDASArray("-".join(["eof-", str(iMode), input.name]), input.domId, eofs_data)) pcs_result = solver.pcs(npcs=nModes) pcs = EDASArray( "pcs[" + inputDset.id + "]", inputDset.inputs[0].domId, EDASArray.cleanupCoords(pcs_result, { "mode": "m", "pc": "m" }).transpose()) results.append(pcs) fracs = solver.varianceFraction(neigs=nModes) pves = [str(round(float(frac * 100.), 1)) + '%' for frac in fracs] for result in results: result["pves"] = str(pves) return EDASDataset.init(self.renameResults(results, node), inputDset.attrs)
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray: variable.persist() axisIndex = variable.getAxisIndex( node.axes, 0, 0 ) dim = variable.xr.dims[axisIndex] window_size = node.getParm("wsize", variable.xr.shape[axisIndex]//8 ) lowpass_args = { dim:int(window_size), "center":True, "min_periods": 1 } lowpass = variable.xr.rolling(**lowpass_args).mean() return EDASArray( variable.name, variable.domId, lowpass )
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray: data = variable.persist() axisIndex = variable.getAxisIndex( node.axes, 0, 0 ) dim = data.dims[axisIndex] window_size = node.getParm("wsize", data.shape[axisIndex] // 8) detrend_args = {dim: int(window_size), "center": True, "min_periods": 1} trend = data.rolling(**detrend_args).mean() detrend: EDASArray = variable - variable.updateXa(trend, "trend") return detrend
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray: data = variable.persist() norm = bool(node.getParm("norm", False)) grouping = node.getParm("groupby", 't.month') climatology = data.groupby(grouping).mean('t') anomalies = data.groupby(grouping) - climatology if norm: anomalies = anomalies.groupby(grouping) / data.groupby(grouping).std('t') return variable.updateXa( anomalies, "decycle" )
def processVariable( self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray: variable.persist() parms = self.getParameters( node, [ Param("lat"), Param("lon")]) aIndex = variable.xr.get_axis_num('t') center: xa.DataArray = variable.selectPoint( float(parms["lon"]), float(parms["lat"]) ).xr cmean, data_mean = center.mean(axis=aIndex), variable.xr.mean(axis=aIndex) cstd, data_std = center.std(axis=aIndex), variable.xr.std(axis=aIndex) cov = np.sum((variable.xr - data_mean) * (center - cmean), axis=aIndex) / variable.xr.shape[aIndex] cor = cov / (cstd * data_std) return EDASArray( variable.name, variable.domId, cor )
def mergeEnsembles(self, op: OpNode, dset: EDASDataset) -> EDASDataset: self.logger.info(" ---> Merge Ensembles: ") for xarray in dset.xarrays: self.logger.info( f" Variable {xarray.name}: dims: {xarray.dims}, coords: {xarray.coords.keys()} " ) sarray: xr.DataArray = xr.concat(dset.xarrays, dim=op.ensDim) result = EDASArray(dset.id, list(dset.domains)[0], sarray) return EDASDataset.init(OrderedDict([(dset.id, result)]), dset.attrs)
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> EDASArray: selection = node.findParm("sel.*") assert (len(node.axes) == 0) or ( (len(node.axes) == 1) and (node.axes[0] == 't')), "Filter currently can only operate on the time axis" result = variable.filter(Axis.T, selection) return result
def buildResultDataset(self, inputDset: EDASDataset, train_node: OpNode )-> EDASDataset: result = self.bestFitResult master_node, model = self.getModel(train_node) arrays = OrderedDict() loss_coord = ( "steps", range(result.train_loss_history.shape[0]) ) arrays["loss"] = EDASArray( "loss" , None, xa.DataArray( result.train_loss_history, coords=( loss_coord, ) ), [] ) arrays["val_loss"] = EDASArray( "val_loss" , None, xa.DataArray( result.val_loss_history, coords=( loss_coord, ) ), [] ) attrs = copy.deepcopy(inputDset.attrs) attrs["loss"] = result.train_loss attrs["val_loss"] = result.val_loss attrs["nEpocs"] = result.nEpocs attrs["nInstances"] = result.nInstances attrs["merge"] = "min:val_loss" attrs["layers"] = ";".join( [ op.serialize() for op in master_node["layerNodes"] ] ) arrays.update( KerasModel.unpackWeights( "initWts", result.initial_weights ) ) arrays.update( KerasModel.unpackWeights( "finalWts", result.final_weights ) ) attrs["nlayers"] = int( len(result.final_weights)/2 ) model.set_weights(result.final_weights) inputData = KerasModel.getTrainingData(master_node, inputDset, 1) targetData = KerasModel.getTargetData(train_node, inputDset, 1) arrays["prediction"] = KerasModel.map( "prediction", model, inputData[0] ) arrays["target"] = targetData[0] rv = EDASDataset( arrays, attrs ) return rv
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray ) -> EDASArray: modelId = node.getParm( "model", "model" ) model: Model = ModelOps.loadModel( self.archivePath( modelId, {} ) ) out_diff = K.mean((model.layers[-1].output - 1) ** 2) grad = K.gradients(out_diff, [model.input])[0] grad /= K.maximum(K.sqrt(K.mean(grad ** 2)), K.epsilon()) iterate = K.function( [model.input, K.learning_phase()], [out_diff, grad] ) input_img_data = np.zeros( shape=variable.xrArray.shape ) self.logger.info("Back Projection Map, Iterations:") for i in range(20): out_loss, out_grad = iterate([input_img_data, 0]) input_img_data -= out_grad * 0.1 self.logger.info( str(i) + ": loss = " + str(out_loss) ) return EDASArray( "Back Projection Map", variable.domId, xa.DataArray(input_img_data) )
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> EDASArray: gridSpec = node.getParam(Param("gridder", True)) return variable.regrid(gridSpec)
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> EDASArray: variable.persist() return variable - variable.ave(node.axes)
def getHistoryDataArray(self, history: History, id: str, nEpochs: int )-> EDASArray: data = xa.DataArray(history.history[id], coords=[ range( nEpochs ) ], dims=["epochs"]) return EDASArray( id, None, data )
def setResult(self, key: str, value: EDASArray): self.logger.info(f"Computed value for WorldClim field bio-{key}") value.persist() self.results[key] = value
def processVariables(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> List[EDASArray]: variable.persist() freq = node.getParm("freq", 'month') operation = str(node.getParm("op", 'mean')).lower() return variable.timeResample(freq, operation)
def processVariables(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> List[EDASArray]: variable.persist() period = node.getParm("period", 'month') operation = str(node.getParm("op", 'mean')).lower() return variable.timeAgg(period, operation)
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> EDASArray: return variable.mean(node.axes)
def getWtsArray(self, array: np.ndarray, id: str )-> EDASArray: coords = [ ('inputs',range(array.shape[0]) ), ('nodes',range(array.shape[1]) ) ] if array.ndim == 2 else [ ('nodes',range(array.shape[0]) ) ] return EDASArray( id, None, xa.DataArray( array, coords=coords ) )
def processVariable(self, request: TaskRequest, node: OpNode, variable: EDASArray) -> EDASArray: variable.persist() centered_result = variable - variable.ave(node.axes) return centered_result / centered_result.std(node.axes)
import matplotlib.pyplot as plt from edas.workflow.data import EDASArray from edas.process.test import LocalTestManager from edas.process.manager import ProcessManager, ExecHandler from edas.config import EdasEnv appConf = {"sources.allowed": "collection,https", "log.metrics": "true"} EdasEnv.update(appConf) processManager = ProcessManager.initManager(EdasEnv.parms) kernel = WorldClimKernel() ds_tmin = xa.open_dataset( "https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/reanalysis/MERRA2/mon/atmos/tasmin.ncml" ) tasmin = ds_tmin["tasmin"] tasmin = EDASArray("tasmin", "d0", tasmin[0:12, :, :].compute()) ds_tmax = xa.open_dataset( "https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/reanalysis/MERRA2/mon/atmos/tasmax.ncml" ) tasmax = ds_tmax["tasmax"] tasmax = EDASArray("tasmax", "d0", tasmax[0:12, :, :].compute()) ds_pr = xa.open_dataset( "https://dataserver.nccs.nasa.gov/thredds/dodsC/bypass/CREATE-IP/reanalysis/MERRA2/mon/atmos/pr.ncml" ) pr = ds_pr["pr"] pr = EDASArray("pr", "d0", pr[0:12, :, :].compute()) results = kernel.computeIndices(tasmin, tasmax, pr, tscale=10.0, hscale=1.0e5)