def refactorRegistry(self, data, reg, **kwargs): """Check for any @ symbols in registry after keys 'input', 'output', denoting that input data should be filled in here The placeholder @ can be used in two ways: 1. @varname. @varname will be replaced by the value of variable 'varname'. This variable should be given in argument 'data'. 2. @name1.varname. Idem, but with varname in kwargs[name1]. Parameters: data (DataContainer) - Input data from input file reg (DataContainer) - entry from registry kwargs (dict of DataContainers, optional) - additional source to search for replacing @ symbols using method 2. above """ for tag in ['input', 'output', 'inputInit']: # tags for whole module inputList = toList(reg.v(tag)) inputList = self.__refactorUtil(inputList, data, **kwargs) reg.addData(tag, inputList) # tags for submodule submodules = reg.v('submodules') if submodules: for mod in submodules: inputList = toList(reg.v(mod, tag)) inputList = self.__refactorUtil(inputList, data, **kwargs) reg.merge({mod:{tag: inputList}}) return
def run(self): cwdpath = cfm.CWD folder = os.path.join(cwdpath, self.input.v('folder')) file = self.input.v('file') # If files is 'all' then get all files that have a .p extension if file == 'all': file = [ f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f)) and f.endswith('.p') ] # check if only one file was requested if len(ny.toList(file)) > 1: raise KnownError( 'Found multiple files to load in module %s. This module can only load one file' % (self.__module__)) # take the single file (in case files has a list type) file = ny.toList(file)[0] if self.input.v('variables') == 'all': varlist = None else: varlist = ny.toList(self.input.v('variables')) + ['grid'] d = ny.pickleload(os.path.join(folder, file), varlist) return d
def __returnSubmoduleRequirement(self, property, submoduleList, toRun=True): """List property 'property' of submodules as given in the registry. Searches in (1) the general module properties or (2) the submodule properties of the submodules in submodulelist. Parameters: property - (string) property to search for submoduleList - (list) list of submodules toRun - (bool, optional) only take properties of submodules that are on the run list (or of the whole module if it is on the run list). Default: True Returns: list of properties """ reqList = [] # 1. search in module registry data addData = self.__register.v(property) if addData is not None: reqList = list(set(reqList + toList(addData))) # 2. search for the property per submodule in the submodule list for mod in submoduleList: if not toRun or mod in self.__submodulesToRunInit: addData = self.__register.v(mod, property) if addData is not None: reqList = list( set(reqList + toList(addData))) # add but prevent double entries return reqList
def amp_phase_input(amp, phase, shape): """ Compile complex quantity u from the amplitude and phase. Reshape this to 'shape', where the information is put in the last dimension of 'shape' Args: amp (scalar, list, 1D array): amplitude of first components of u. number of elements may be smaller or the same as the size of the last dimension of u phase (scalar, list, 1D array): phase (deg) of first components of u. number of elements may be smaller or the same as the size of the last dimension of u shape (tuple): shape of u. Returns: u - complex array of shape 'shape """ nodim = len(shape) amp = ny.toList(amp) phase = ny.toList(phase) phase = np.asarray(ny.toList(phase[:min(len(amp), shape[-1])])) phase = phase.reshape((1, ) * (nodim - 1) + (len(phase), )) amp = np.asarray(ny.toList(amp[:min(len(amp), shape[-1])])) amp = amp.reshape((1, ) * (nodim - 1) + (len(amp), )) u = np.zeros(shape, dtype=complex) u[[slice(None)] * (nodim - 1) + [slice(None, amp.shape[-1])]] = amp u[[slice(None)] * (nodim - 1) + [slice(None, phase.shape[-1])]] = u[ [slice(None)] * (nodim - 1) + [slice(None, phase.shape[-1])]] * np.exp( -1j * phase / 180. * np.pi) return u
def updateToRun(self, inputList): """Update if this module should be run and which submodules should be run. The update is based on whether this module provides output variables that are present in argument inputList Note: All submodules will be run if the the general module output (i.e. not for submodules) is required. However, if the submodules to run are specified on input, this will not be overruled. Parameters: inputList - (list) list of variables """ # check if general output by module is required for other modules varlist = [ i for i in toList(self.__register.v('output')) if i in inputList ] # if there is any variable in the output that is also given in inputList, run the module if len(varlist) > 0: self.runModule = True self.__submodulesToRunInit = self.__submoduleList() # additionally, check if individual submodules contribute to required output sublist = self.__submoduleList() for i in sublist: varlist = [ j for j in toList(self.__register.v(i, 'output')) if j in inputList ] if len(varlist) > 0: self.runModule = True self.__submodulesToRunInit = list( set(self.__submodulesToRunInit + toList(i))) self.submodulesToRun = self.__submodulesToRunInit return
def updateToRunIter(self, inputList): """Update submodules to run in iterations after the initialisation. The update is based on the input variables updated in the iteration and the dependencies of the submodules Parameters: inputList - (list) list of variables """ self.__submodulesToRunIter = [] # check if general input by module is changed in the loop varlist = [ i for i in toList(self.__register.v('input')) if i in inputList ] if len(varlist) > 0: self.__submodulesToRunIter = self.__submoduleList() # additionally, check if individual submodules contribute to required output sublist = self.__submoduleList() for i in sublist: varlist = [ j for j in toList(self.__register.v(i, 'input')) if j in inputList ] if len(varlist) > 0: self.__submodulesToRunIter = list( set(self.__submodulesToRunIter + toList(i))) return
def __submoduleList(self): ''' Make a list of the submodules based on the optional keywords 'submodules' and 'excludeSubmodules' Use all submodules, unless the input tells something else ''' if self.__register.v('submodules'): submodReqList = toList(self.__input.v('submodules')) submodExcludeList = toList(self.__input.v('excludeSubmodules')) if submodReqList != [] and submodReqList != ['all']: sublist = [ i for i in submodReqList if i in self.__register.v('submodules') ] elif submodExcludeList != [] and submodExcludeList != [ 'none' ] and submodExcludeList != ['None']: sublist = [ i for i in self.__register.v('submodules') if i not in submodExcludeList ] else: sublist = toList(self.__register.v('submodules')) else: sublist = [] return sublist
def __getSubmoduleRequirements(self, outputVariables): # refactor output requirements to key tuples to check against dataContainer keys. Load this into reqKeys reqKeys = [] for var in toList(outputVariables): submoduleRequirements = self.input.v('submodules', var) if submoduleRequirements == None or submoduleRequirements == 'all': reqKeys.append((var, )) else: for submod in toList(submoduleRequirements): reqKeys.append((var, submod)) return reqKeys
def run_init(self): """ """ ## Setup loop based on self.loopstyle self.loopstyle = toList(self.input.v('loopstyle')) # change self.loopstyle if 'permutations' or 'simultaneous' + check if self.loopstyle has the correct format if len(self.loopstyle) == 1 and self.loopstyle[ 0] == 'permutations': # all possible combinations self.loopstyle = range(0, len(self.variables)) elif len(self.loopstyle) == 1 and self.loopstyle[ 0] == 'simultaneous': # everything simultaneous self.loopstyle = [0] * len(self.variables) elif len(self.loopstyle) == len(self.variables): pass else: raise KnownError( 'self.loopstyle "%s" in input of module %s unknown ' % (str(self.loopstyle), self.__module__)) # make list of unique elements of self.loopstyle self.ls_list = [i for i in sorted(list(set(toList(self.loopstyle))))] # verify that number of values is the same in covarying variables num_items = [] for ls in self.ls_list: num_items.append( len(self.values[self.variables[self.loopstyle.index(ls)]])) for i, var in enumerate(self.variables): if ls == self.loopstyle[i] and len( self.values[var]) != num_items[-1]: if self.input.v('loopstyle') == 'simultaneous': raise KnownError( 'Problem in input of module %s for loopstyle "simultaneous". Number of values in "%s" is unequal to number of values in "%s" ' % (self.__module__, self.variables[i], self.variables[0])) else: raise KnownError( 'Problem in input of module %s for loopstyle "%s". Number of values in "%s" is unequal to number of values in "%s" ' % (self.__module__, ls, self.variables[i], self.variables[self.loopstyle.index(ls)])) # determine number of loops per element in permutation self.numLoops = num_items # run first iteration self.iteration = 0 d = self.run() return d
def run(self): self.timer.tic() self.logger.info('Running module Sediment Capacity') ## Init d = {} self.submodulesToRun = ny.toList(self.input.v('submodules')) self.erosion_method = self.input.v('erosion_formulation') self.frictionpar = self.input.v( 'friction' ) # friction parameter used for the erosion, by default the total roughness if self.frictionpar == None: self.frictionpar = 'Roughness' jmax = self.input.v('grid', 'maxIndex', 'x') kmax = self.input.v('grid', 'maxIndex', 'z') fmax = self.input.v('grid', 'maxIndex', 'f') self.Kv = self.input.v('Kv', range(0, jmax + 1), range(0, kmax + 1), range(0, fmax + 1)) self.ws = self.input.v('ws0', range(0, jmax + 1), range(0, kmax + 1), range(0, fmax + 1)) ## Run orders d.update(self.leading_order(d)) d.update(self.first_order(d)) d.update(self.second_order_river(d)) self.timer.toc() self.timer.disp('time sediment capacity') self.timer.reset() # d['hatc0']['a']['erosion'][:, :, 0] += 1e-4 return d
def callDataOnGrid(dataContainer, keys, grid, gridname, reshape): """Call data corresponding to keys from dataContainer on a certain grid Parameters: dataContainer (DataContainer) - contains all data keys (tuple of str or str) - single (!) key plus optional subkeys grid (DataContainer) - a data container containing a grid satisfying grid conventions. This function currently only supports 'Regular' grid types gridname (str) - key under which the grid can be found in the grid DataContainer reshape (bool) - reshape the requested data to grid size or keep in original format? Returns: Data requested on grid in varying format (str, ndarray, list, etxc). flag: -1: error: no suitable grid 0: warning: could not convert, returned original data 1: conversion succesful """ axes = {} if grid.v(gridname, 'gridtype') == 'Regular': axes = {} for dim in ny.toList(grid.v(gridname, 'dimensions')): axes[dim] = grid.v(gridname, 'axis', dim) axes[dim] = axes[dim].reshape(reduce(lambda x, y: x*y, axes[dim].shape)) try: data = dataContainer.v(*keys, reshape=reshape, **axes) flag = 1 except: data = dataContainer.v(*keys) flag = 0 return data, flag
def interpretValues(self, values): """inpterpret values on input as space-separated list or as pure python input Parameters values - values to evaluate Returns: values - evaluated values """ values = toList(values) # case 1: pure python: check for (, [, range, np.arange # merge list to a string valString = ' '.join([str(f) for f in values]) # try to interpret as python string if any([i in valString for i in ['(', '[', ',', '/', '*', '+', '-']]): try: valuespy = None exec('valuespy =' + valString) return valuespy except Exception as e: try: errorString = ': ' + e.message except: errorString = '' raise KnownError( 'Failed to interpret input as python command %s in input: %s' % (errorString, valString), e) # case 2: else interpret as space-separated list else: return values
def compute_source(self): ################################################################################################################ ## Sourceterm G ################################################################################################################ jmax = self.input.v('grid', 'maxIndex', 'x') B = self.input.v('B', range(0, jmax + 1), 0, 0) x = ny.dimensionalAxis(self.input, 'x')[:, 0, 0] G = np.zeros(jmax+1) # Upstream source if self.input.v('Qsed') is not None: G[:] = self.input.v('Qsed') / B # Other sources if self.input.v('sedsource') is not None: Qsedsource = ny.toList(self.input.v('sedsource')) for i, s in enumerate(Qsedsource): if s[0] == 'point': xs = s[1] source = s[2] G[np.where(x < xs)] += source / B[np.where(x < xs)] elif s[0] == 'line': xmin = s[1] xmax = s[2] xi = np.zeros(jmax+1) xi[1:] = 0.5 * x[1:] + 0.5 * x[:-1] source = s[3] G += source / B * np.minimum(np.maximum(xmax - xi, 0), xmax - xmin) return G
def run(self): Q = self.input.v('Q1') Qsource = ny.toList(self.input.v('Qsource')) Q = Q + sum(Qsource) d = {} d['xc'] = 41000. * Q**-0.24 return d
def value(self, x, f, **kwargs): x = x*self.L f = [1-bool(j) for j in toList(f)] f = np.asarray(f).reshape(1,len(f)) s = self.ssea/2.*(1-np.tanh((x-self.xc)/self.xl)) s = s.reshape(len(s), 1)*f return s
def __init__(self, input): self.input = input # also instantiate dependent modules modules = ny.toList(self.input.v('modules')) for module in modules: module.instantiateModule() return
def value(self, x, f, **kwargs): x = x*self.L f = [1-bool(j) for j in toList(f)] f = np.asarray(f).reshape(1, len(f)) s = self.ssea*np.exp(-x/self.Ls) s = s.reshape(len(s), 1)*f return s
def updateIteratorRegistry(self, moduleList): """Update the register of the special iterator module (IteratorModule in register). This has an input variable modules that it controls and it takes its registry data""" if self.isIteratorModule(): modules = toList(self.__input.v('modules')) modules = [ i for j in modules for i in moduleList if (i.getName().split('.')[0] + '.' + i.getName().split('.')[-1] == j) ] inputInit = toList(self.__register.v('inputInit')) input = toList(self.__register.v('input')) output = toList(self.__register.v('output')) run = [] for module in modules: # get registry data of the dependent modules inputInit += toList(module.getInputRequirements(True)) input += toList(module.getInputRequirements()) output += toList(module.getOutputVariables()) # remove the dependent module from the module list moduleList.remove(module) # compile final list of register data self.__register.addData('inputInit', list(set(inputInit))) self.__register.addData('input', list(set(input))) self.__register.addData('output', list(set(output))) self.__toRun() # append input data with the module object references, instead of the module names self.addInputData({'modules': modules}) return moduleList
def read(self, iteration): file = self.files[iteration] if self.input.v('variables') == 'all': varlist = None else: varlist = ny.toList(self.input.v('variables')) + ['grid'] d = ny.pickleload(os.path.join(self.folder, file), varlist) d['experimentnumber'] = iteration return d
def dictExpand(self, d, subindex, subsubindices): if not subindex in d: d[subindex] = {} elif not isinstance(d[subindex], dict): d[subindex] = {} for ssi in ny.toList(subsubindices): if not ssi in d[subindex]: d[subindex][ssi] = 0 return d
def run_init(self): d = {} modules = ny.toList(self.input.v('modules')) for module in modules: module.addInputData(self.input.data) d1 = module.run(init=True) d.update(d1.data) return d
def __toRun(self, alwaysRun=False): """Check if this module should be run based on the output requirements. If so, also set the submodules that should be run (if there are any submodules) Notes: This is not the final verdict on whether this module will run and what list of submodules will be run. An update will be done from the module list Sets the class variables runModule - (bool) should this module run _submodulesToRun - (list) list of submodules to run or an empty list """ self.__submodulesToRunInit = [] self.runModule = alwaysRun # default setting: False, except when it is forced to be run from outside # check if the module has submodules if self.__register.v('submodules'): sublist = self.__submoduleList() # loop over variables in the output requirements for var in toList(self.__outputReq.v('requirements')): submoduleList = [] # gather the submodules that have 'var' as output for submod in sublist: outputList = toList(self.__register.v(submod, 'output')) if var in outputList: submoduleList.append(submod) self.__submodulesToRunInit = list( set(self.__submodulesToRunInit + submoduleList)) if self.__submodulesToRunInit: self.runModule = True # if any submodules should be run, then so does the module # check general module output # loop over variables in the output requirements for var in toList(self.__outputReq.v('requirements')): outputList = toList(self.__register.v('output')) if var in outputList: self.runModule = True # set submodules to run self.submodulesToRun = self.__submodulesToRunInit return
def run_init(self): d = {} modules = ny.toList(self.input.v('modules')) for module in modules: module.addInputData(self.input.data) d1 = module.run(init=True) d.update(d1) self.input.merge(d1) # make output of last run module available to the next ones return d
def __init__(self, input): self.input = input self.variables = toList(self.input.v('variables')) # check if values of variables are provided for var in self.variables: if self.input.v(var) is None: message = ("Not all required variables are given. Missing '%s' in module '%s'" % (var, self.__module__)) raise KnownError(message) # load values to loop over from input self.values = {} for var in self.variables: # check if values are given directly or in a separate substructure varkeys = self.input.getKeysOf(var) # case 1: values supplied directly if not varkeys: values = self.input.v(var) values = self.interpretValues(values) self.values[var] = toList(values) # case 2: values in sub-dict else: # load values per key values = {} for key in varkeys: temp = self.input.v(var, key) values[key] = self.interpretValues(temp) # check that all keys have the same number of arguments for key in varkeys: if len(values[key]) != len(values[varkeys[0]]): raise KnownError('Problem with values of "%s" in input of module %s. Number of values in "%s" is unequal to number of values in "%s" ' % (var, self.__module__, key, varkeys[0])) # rewrite dict with lists to list with dicts self.values[var] = [] for i in range(0,len(values[varkeys[0]])): self.values[var].append({}) for key in varkeys: self.values[var][i][key]=values[key][i] return
def __refactorUtil(self, inputList, data, **kwargs): """Replace tags @{}, +{} and if{}. Do this recursively by removing an item from the inputList, replacing one occurrence of @{} and +{} and putting the result at the back of inputList. """ i = 0 while i<len(inputList): item = inputList[i] if item.find('@')>=0 or item.find('+{')>=0 or item.find('if{')>=0: inputList.pop(i) if item.find('@')>=0 and item.find('@{')<0: if '.' in item: dict = item[1:].split('.')[0] key = item[1:].split('.')[-1] item = toList(kwargs[dict].v(key)) else: item = toList(data.v(item[1:])) inputList = inputList + item elif item.find('@{')>=0: start = item.find('@{') end = item.find('}') if '.' in item[start+2:end]: dict = item[start+2:end].split('.')[0] key = item[start+2:end].split('.')[-1] item = [item[:start]+j+item[end+1:] for j in toList(kwargs[dict].v(key))] else: item = [item[:start]+str(j)+item[end+1:] for j in toList(data.v(item[start+2:end]), forceNone=True)] #20-01-2017 added forceNone=True inputList = inputList + item elif item.find('+{')>=0: start = item.find('+{') end = item.find('}') setrange = range(*eval(item[start+2:end])) item = item[:start]+'%s'+item[end+1:] for k in setrange: inputList = inputList + toList(item % k) elif item.find('if{')>=0: # added YMD 02-11-2016 start = item.find('if{') end = item.find('}') item = item[start+3:end] item = item.split(',') if len(item)!=2: raise KnownError('error in registry in if{}. Does not have two comma-separated arguments. Make sure there are no spaces in the if-statement') try: if eval(item[1]): inputList = inputList + toList(item[0]) except NameError: item1 = item[1].split('==')[0] item2 = item[1].split('==')[1] if item1 == eval(item2): inputList = inputList + toList(item[0]) else: i += 1 return list(set(inputList))
def run(self): self.logger.info('Module calcQ.py is running') d = {} QTrib = nty.toList(self.input.v('QTributary')) Q1 = self.input.v('Q') * QTrib[0] / 100. Qsource = [self.input.v('Q')*QTrib[1]/100., self.input.v('Q')*QTrib[2]/100.] d['Q1'] = Q1 d['Qsource'] = Qsource return d
def derivative(self, x, f, **kwargs): if kwargs['dim'] == 'x': x = np.asarray(x*self.L) f = [1-bool(j) for j in toList(f)] f = np.asarray(f).reshape(1, len(f)) sx = -self.ssea/self.Ls*np.exp(-x/self.Ls) sx = sx.reshape(len(sx), 1)*f else: sx = None FunctionBase.derivative(self) return sx
def derivative(self, x, f, **kwargs): if kwargs['dim'] == 'x': x = np.asarray(x*self.L) f = [1-bool(j) for j in toList(f)] f = np.asarray(f).reshape(1,len(f)) sx = self.ssea/2.*(np.tanh((x-self.xc)/self.xl)**2-1)/self.xl sx = sx.reshape(len(sx),1)*f else: sx = None FunctionBase.derivative(self) return sx
def run(self): d = {} modules = ny.toList(self.input.v('modules')) for module in modules: module.addInputData(self.input.data) d1 = module.run() d.update(d1.data) if self.iteration >= self.MAXITERS: # output variable indicating if iterotor has converged or not. d['converged'] = False else: d['converged'] = True return d
def run(self): d = {} modules = ny.toList(self.input.v('modules')) for module in modules: module.addInputData(self.input.data) d1 = module.run() d.update(d1) self.input.merge(d1) # make output of last run module available to the next ones if self.iteration >= self.MAXITERS: # output variable indicating if iterotor has converged or not. d['converged'] = False else: d['converged'] = True return d