def preprocessInputs(self, request: TaskRequest, op: OpNode, inputDataset: EDASDataset) -> EDASDataset: # interp_na = bool(op.getParm("interp_na", False)) # if interp_na: inputs: Dict[str,EDASArray] = { id: input.updateXa( input.xr.interpolate_na( dim="t", method='linear' ),"interp_na" ) for (id, input) in inputDset.arrayMap.items() } # else: inputs: Dict[str,EDASArray] = { id: input for (id, input) in inputDset.arrayMap.items() } if op.isSimple and not self.requiresAlignment: result = inputDataset else: resultArrays: OrderedDict[str, EDASArray] = OrderedDict() arrayList = list(inputDataset.arrayMap.values()) for aid, array in inputDataset.arrayMap.items(): unapplied_domains: Set[str] = array.unapplied_domains( arrayList, op.domain) if len(unapplied_domains) > 0: merged_domain: str = request.intersectDomains( unapplied_domains, False) processed_domain: Domain = request.cropDomain( merged_domain, arrayList) sub_array = array.subset(processed_domain, unapplied_domains) resultArrays[aid] = sub_array else: resultArrays[aid] = array resultDataset = EDASDataset(resultArrays, inputDataset.attrs) alignmentTarget = resultDataset.getAlignmentVariable( op.getParm("align", "lowest")) preprop_result = resultDataset.align(alignmentTarget) result: EDASDataset = preprop_result.groupby(op.grouping).resample( op.resampling) print(" $$$$ processInputCrossSection: " + op.name + " -> " + str(result.ids)) return result.purge()
def plotPrediction(self, results: EDASDataset, title, **kwargs): plt.title(title) print("Plotting: " + ",".join(list(results.ids))) prediction: np.ndarray = results.getArray("prediction").xr.values target: np.ndarray = results.getArray("target").xr.values x = range(prediction.shape[0]) plt.plot(x, prediction, "r-", label="prediction") plt.plot(x, target, "b--", label="target") plt.legend() plt.show()
def plotPerformance(self, results: EDASDataset, title, **kwargs): plt.title(title) print("Plotting: " + ",".join(list(results.ids))) valLoss: np.ndarray = results.getArray("val_loss").xr.values trainlLoss: np.ndarray = results.getArray("loss").xr.values x = range(valLoss.shape[0]) plt.plot(x, valLoss, "r-", label="Validation Loss") plt.plot(x, trainlLoss, "b--", label="Training Loss") plt.legend() plt.show()
def signResult(self, result: EDASDataset, request: TaskRequest, node: WorkflowNode, **kwargs ) -> EDASDataset: result["proj"] = request.project result["exp"] = request.experiment result["uid"] = str(request.uid) for key,value in kwargs.items(): result[key] = value archive = node.getParm("archive") if archive: result["archive"] = archive if node.isBranch: result.persist() return result
def getInputCrossSections(self, inputs: EDASDatasetCollection ) -> Dict[str,EDASDataset]: inputCrossSections = {} for dsKey, dset in inputs.items(): for index, (akey, array) in enumerate(dset.arrayMap.items()): merge_set: EDASDataset = inputCrossSections.setdefault( index, EDASDataset(OrderedDict(), inputs.attrs ) ) merge_set[dsKey + "-" + akey] = array return inputCrossSections
def buildProduct(self, dsid: str, request: TaskRequest, node: OpNode, result_arrays: List[EDASArray], attrs: Dict[str, str]): result_dset = EDASDataset.init(self.renameResults(result_arrays, node), attrs) for parm in ["product", "archive"]: result_dset[parm] = node.getParm(parm, "") result_dset.name = node.getResultId(dsid) return self.signResult(result_dset, request, node)
def mergeResults(self) -> List[EDASDataset]: if self.results[0].getResultClass() == "METADATA": return self.results mergeMethod: str = self.results[0]["merge"] if mergeMethod is None: return EDASDataset.merge( self.results ) mergeToks = mergeMethod.split(":") return self.getBestResult( mergeToks[0].strip().lower(), mergeToks[1].strip().lower() )
def mergeEnsembles(self, op: OpNode, dset: EDASDataset) -> EDASDataset: self.logger.info(" ---> Merge Ensembles: ") for xarray in dset.xarrays: self.logger.info( f" Variable {xarray.name}: dims: {xarray.dims}, coords: {xarray.coords.keys()} " ) sarray: xr.DataArray = xr.concat(dset.xarrays, dim=op.ensDim) result = EDASArray(dset.id, list(dset.domains)[0], sarray) return EDASDataset.init(OrderedDict([(dset.id, result)]), dset.attrs)
def getCachedDataset(self, snode: SourceNode) -> Optional[EDASDataset]: cache_status = self.getCacheStatus(snode) if cache_status != CacheStatus.Ignore: cid = snode.varSource.getId() variable = EDASKCacheMgr[cid] if variable is None: assert cache_status == CacheStatus.Option, "Missing cached input: " + cid else: return EDASDataset.init(OrderedDict([(cid, variable)]), {}) return None
def processDataset(self, request: TaskRequest, dset: xr.Dataset, snode: SourceNode) -> EDASDataset: coordMap = Axis.getDatasetCoordMap(dset) filteredCoordMap = snode.varSource.name2id(coordMap) edset: EDASDataset = EDASDataset.new( dset, {id: snode.domain for id in snode.varSource.ids}, filteredCoordMap) processed_domain: Domain = request.cropDomain(snode.domain, edset.inputs, snode.offset) result = edset.subset(processed_domain) if snode.domain else edset self.logger.info( f"###### ProcessDataset, coordMap = {filteredCoordMap}, dset coords = {list(edset.xr[0].coords.keys())}" ) return self.signResult(result, request, snode, sources=snode.varSource.getId())
def eof_plot(self, mtype: int, dset: EDASDataset): dset.plotMaps(view="mol", mtype=mtype)
def subset(self, domId: str, dset: EDASDataset) -> EDASDataset: return dset.subset( self.domain(domId)) if dset.requiresSubset(domId) else dset
def processDataset(self, request: TaskRequest, dset: xr.Dataset, snode: SourceNode) -> EDASDataset: coordMap = Axis.getDatasetCoordMap( dset ) edset: EDASDataset = EDASDataset.new( dset, { id:snode.domain for id in snode.varSource.ids}, snode.varSource.name2id(coordMap) ) processed_domain: Domain = request.cropDomain( snode.domain, edset.inputs, snode.offset ) result = edset.subset( processed_domain ) if snode.domain else edset return self.signResult(result, request, snode, sources=snode.varSource.getId())