def refactorRegistry(self, data, reg, **kwargs):
"""Check for any @ symbols in registry after keys 'input', 'output', denoting that input data should be filled in here
The placeholder @ can be used in two ways:
1. @varname. @varname will be replaced by the value of variable 'varname'. This variable should be given in argument 'data'.
2. @name1.varname. Idem, but with varname in kwargs[name1].
Parameters:
data (DataContainer) - Input data from input file
reg (DataContainer) - entry from registry
kwargs (dict of DataContainers, optional) - additional source to search for replacing @ symbols using method 2. above
"""
for tag in ['input', 'output', 'inputInit']:
# tags for whole module
inputList = toList(reg.v(tag))
inputList = self.__refactorUtil(inputList, data, **kwargs)
reg.addData(tag, inputList)
# tags for submodule
submodules = reg.v('submodules')
if submodules:
for mod in submodules:
inputList = toList(reg.v(mod, tag))
inputList = self.__refactorUtil(inputList, data, **kwargs)
reg.merge({mod:{tag: inputList}})
return
def run(self):
cwdpath = cfm.CWD
folder = os.path.join(cwdpath, self.input.v('folder'))
file = self.input.v('file')
# If files is 'all' then get all files that have a .p extension
if file == 'all':
file = [
f for f in os.listdir(folder)
if os.path.isfile(os.path.join(folder, f)) and f.endswith('.p')
]
# check if only one file was requested
if len(ny.toList(file)) > 1:
raise KnownError(
'Found multiple files to load in module %s. This module can only load one file'
% (self.__module__))
# take the single file (in case files has a list type)
file = ny.toList(file)[0]
if self.input.v('variables') == 'all':
varlist = None
else:
varlist = ny.toList(self.input.v('variables')) + ['grid']
d = ny.pickleload(os.path.join(folder, file), varlist)
return d
def __returnSubmoduleRequirement(self,
property,
submoduleList,
toRun=True):
"""List property 'property' of submodules as given in the registry.
Searches in (1) the general module properties
or (2) the submodule properties of the submodules in submodulelist.
Parameters:
property - (string) property to search for
submoduleList - (list) list of submodules
toRun - (bool, optional) only take properties of submodules that are on the run list
(or of the whole module if it is on the run list). Default: True
Returns:
list of properties
"""
reqList = []
# 1. search in module registry data
addData = self.__register.v(property)
if addData is not None:
reqList = list(set(reqList + toList(addData)))
# 2. search for the property per submodule in the submodule list
for mod in submoduleList:
if not toRun or mod in self.__submodulesToRunInit:
addData = self.__register.v(mod, property)
if addData is not None:
reqList = list(
set(reqList +
toList(addData))) # add but prevent double entries
return reqList
def amp_phase_input(amp, phase, shape):
""" Compile complex quantity u from the amplitude and phase.
Reshape this to 'shape', where the information is put in the last dimension of 'shape'
Args:
amp (scalar, list, 1D array): amplitude of first components of u. number of elements may be smaller or the same as the size of the last dimension of u
phase (scalar, list, 1D array): phase (deg) of first components of u. number of elements may be smaller or the same as the size of the last dimension of u
shape (tuple): shape of u.
Returns:
u - complex array of shape 'shape
"""
nodim = len(shape)
amp = ny.toList(amp)
phase = ny.toList(phase)
phase = np.asarray(ny.toList(phase[:min(len(amp), shape[-1])]))
phase = phase.reshape((1, ) * (nodim - 1) + (len(phase), ))
amp = np.asarray(ny.toList(amp[:min(len(amp), shape[-1])]))
amp = amp.reshape((1, ) * (nodim - 1) + (len(amp), ))
u = np.zeros(shape, dtype=complex)
u[[slice(None)] * (nodim - 1) + [slice(None, amp.shape[-1])]] = amp
u[[slice(None)] * (nodim - 1) + [slice(None, phase.shape[-1])]] = u[
[slice(None)] * (nodim - 1) + [slice(None, phase.shape[-1])]] * np.exp(
-1j * phase / 180. * np.pi)
return u
def updateToRun(self, inputList):
"""Update if this module should be run and which submodules should be run.
The update is based on whether this module provides output variables that are present in argument inputList
Note:
All submodules will be run if the the general module output (i.e. not for submodules) is required.
However, if the submodules to run are specified on input, this will not be overruled.
Parameters:
inputList - (list) list of variables
"""
# check if general output by module is required for other modules
varlist = [
i for i in toList(self.__register.v('output')) if i in inputList
] # if there is any variable in the output that is also given in inputList, run the module
if len(varlist) > 0:
self.runModule = True
self.__submodulesToRunInit = self.__submoduleList()
# additionally, check if individual submodules contribute to required output
sublist = self.__submoduleList()
for i in sublist:
varlist = [
j for j in toList(self.__register.v(i, 'output'))
if j in inputList
]
if len(varlist) > 0:
self.runModule = True
self.__submodulesToRunInit = list(
set(self.__submodulesToRunInit + toList(i)))
self.submodulesToRun = self.__submodulesToRunInit
return
def updateToRunIter(self, inputList):
"""Update submodules to run in iterations after the initialisation.
The update is based on the input variables updated in the iteration and the dependencies of the submodules
Parameters:
inputList - (list) list of variables
"""
self.__submodulesToRunIter = []
# check if general input by module is changed in the loop
varlist = [
i for i in toList(self.__register.v('input')) if i in inputList
]
if len(varlist) > 0:
self.__submodulesToRunIter = self.__submoduleList()
# additionally, check if individual submodules contribute to required output
sublist = self.__submoduleList()
for i in sublist:
varlist = [
j for j in toList(self.__register.v(i, 'input'))
if j in inputList
]
if len(varlist) > 0:
self.__submodulesToRunIter = list(
set(self.__submodulesToRunIter + toList(i)))
return
def __submoduleList(self):
''' Make a list of the submodules based on the optional keywords 'submodules' and 'excludeSubmodules'
Use all submodules, unless the input tells something else
'''
if self.__register.v('submodules'):
submodReqList = toList(self.__input.v('submodules'))
submodExcludeList = toList(self.__input.v('excludeSubmodules'))
if submodReqList != [] and submodReqList != ['all']:
sublist = [
i for i in submodReqList
if i in self.__register.v('submodules')
]
elif submodExcludeList != [] and submodExcludeList != [
'none'
] and submodExcludeList != ['None']:
sublist = [
i for i in self.__register.v('submodules')
if i not in submodExcludeList
]
else:
sublist = toList(self.__register.v('submodules'))
else:
sublist = []
return sublist
def __getSubmoduleRequirements(self, outputVariables):
# refactor output requirements to key tuples to check against dataContainer keys. Load this into reqKeys
reqKeys = []
for var in toList(outputVariables):
submoduleRequirements = self.input.v('submodules', var)
if submoduleRequirements == None or submoduleRequirements == 'all':
reqKeys.append((var, ))
else:
for submod in toList(submoduleRequirements):
reqKeys.append((var, submod))
return reqKeys
def run_init(self):
"""
"""
## Setup loop based on self.loopstyle
self.loopstyle = toList(self.input.v('loopstyle'))
# change self.loopstyle if 'permutations' or 'simultaneous' + check if self.loopstyle has the correct format
if len(self.loopstyle) == 1 and self.loopstyle[
0] == 'permutations': # all possible combinations
self.loopstyle = range(0, len(self.variables))
elif len(self.loopstyle) == 1 and self.loopstyle[
0] == 'simultaneous': # everything simultaneous
self.loopstyle = [0] * len(self.variables)
elif len(self.loopstyle) == len(self.variables):
pass
else:
raise KnownError(
'self.loopstyle "%s" in input of module %s unknown ' %
(str(self.loopstyle), self.__module__))
# make list of unique elements of self.loopstyle
self.ls_list = [i for i in sorted(list(set(toList(self.loopstyle))))]
# verify that number of values is the same in covarying variables
num_items = []
for ls in self.ls_list:
num_items.append(
len(self.values[self.variables[self.loopstyle.index(ls)]]))
for i, var in enumerate(self.variables):
if ls == self.loopstyle[i] and len(
self.values[var]) != num_items[-1]:
if self.input.v('loopstyle') == 'simultaneous':
raise KnownError(
'Problem in input of module %s for loopstyle "simultaneous". Number of values in "%s" is unequal to number of values in "%s" '
% (self.__module__, self.variables[i],
self.variables[0]))
else:
raise KnownError(
'Problem in input of module %s for loopstyle "%s". Number of values in "%s" is unequal to number of values in "%s" '
% (self.__module__, ls, self.variables[i],
self.variables[self.loopstyle.index(ls)]))
# determine number of loops per element in permutation
self.numLoops = num_items
# run first iteration
self.iteration = 0
d = self.run()
return d
def run(self):
self.timer.tic()
self.logger.info('Running module Sediment Capacity')
## Init
d = {}
self.submodulesToRun = ny.toList(self.input.v('submodules'))
self.erosion_method = self.input.v('erosion_formulation')
self.frictionpar = self.input.v(
'friction'
) # friction parameter used for the erosion, by default the total roughness
if self.frictionpar == None:
self.frictionpar = 'Roughness'
jmax = self.input.v('grid', 'maxIndex', 'x')
kmax = self.input.v('grid', 'maxIndex', 'z')
fmax = self.input.v('grid', 'maxIndex', 'f')
self.Kv = self.input.v('Kv', range(0, jmax + 1), range(0, kmax + 1),
range(0, fmax + 1))
self.ws = self.input.v('ws0', range(0, jmax + 1), range(0, kmax + 1),
range(0, fmax + 1))
## Run orders
d.update(self.leading_order(d))
d.update(self.first_order(d))
d.update(self.second_order_river(d))
self.timer.toc()
self.timer.disp('time sediment capacity')
self.timer.reset()
# d['hatc0']['a']['erosion'][:, :, 0] += 1e-4
return d
def callDataOnGrid(dataContainer, keys, grid, gridname, reshape):
"""Call data corresponding to keys from dataContainer on a certain grid
Parameters:
dataContainer (DataContainer) - contains all data
keys (tuple of str or str) - single (!) key plus optional subkeys
grid (DataContainer) - a data container containing a grid satisfying grid conventions.
This function currently only supports 'Regular' grid types
gridname (str) - key under which the grid can be found in the grid DataContainer
reshape (bool) - reshape the requested data to grid size or keep in original format?
Returns:
Data requested on grid in varying format (str, ndarray, list, etxc).
flag: -1: error: no suitable grid
0: warning: could not convert, returned original data
1: conversion succesful
"""
axes = {}
if grid.v(gridname, 'gridtype') == 'Regular':
axes = {}
for dim in ny.toList(grid.v(gridname, 'dimensions')):
axes[dim] = grid.v(gridname, 'axis', dim)
axes[dim] = axes[dim].reshape(reduce(lambda x, y: x*y, axes[dim].shape))
try:
data = dataContainer.v(*keys, reshape=reshape, **axes)
flag = 1
except:
data = dataContainer.v(*keys)
flag = 0
return data, flag
def interpretValues(self, values):
"""inpterpret values on input as space-separated list or as pure python input
Parameters
values - values to evaluate
Returns:
values - evaluated values
"""
values = toList(values)
# case 1: pure python: check for (, [, range, np.arange
# merge list to a string
valString = ' '.join([str(f) for f in values])
# try to interpret as python string
if any([i in valString for i in ['(', '[', ',', '/', '*', '+', '-']]):
try:
valuespy = None
exec('valuespy =' + valString)
return valuespy
except Exception as e:
try:
errorString = ': ' + e.message
except:
errorString = ''
raise KnownError(
'Failed to interpret input as python command %s in input: %s'
% (errorString, valString), e)
# case 2: else interpret as space-separated list
else:
return values
def compute_source(self):
################################################################################################################
## Sourceterm G
################################################################################################################
jmax = self.input.v('grid', 'maxIndex', 'x')
B = self.input.v('B', range(0, jmax + 1), 0, 0)
x = ny.dimensionalAxis(self.input, 'x')[:, 0, 0]
G = np.zeros(jmax+1)
# Upstream source
if self.input.v('Qsed') is not None:
G[:] = self.input.v('Qsed') / B
# Other sources
if self.input.v('sedsource') is not None:
Qsedsource = ny.toList(self.input.v('sedsource'))
for i, s in enumerate(Qsedsource):
if s[0] == 'point':
xs = s[1]
source = s[2]
G[np.where(x < xs)] += source / B[np.where(x < xs)]
elif s[0] == 'line':
xmin = s[1]
xmax = s[2]
xi = np.zeros(jmax+1)
xi[1:] = 0.5 * x[1:] + 0.5 * x[:-1]
source = s[3]
G += source / B * np.minimum(np.maximum(xmax - xi, 0), xmax - xmin)
return G
def run(self):
Q = self.input.v('Q1')
Qsource = ny.toList(self.input.v('Qsource'))
Q = Q + sum(Qsource)
d = {}
d['xc'] = 41000. * Q**-0.24
return d
def value(self, x, f, **kwargs):
x = x*self.L
f = [1-bool(j) for j in toList(f)]
f = np.asarray(f).reshape(1,len(f))
s = self.ssea/2.*(1-np.tanh((x-self.xc)/self.xl))
s = s.reshape(len(s), 1)*f
return s
def __init__(self, input):
self.input = input
# also instantiate dependent modules
modules = ny.toList(self.input.v('modules'))
for module in modules:
module.instantiateModule()
return
def value(self, x, f, **kwargs):
x = x*self.L
f = [1-bool(j) for j in toList(f)]
f = np.asarray(f).reshape(1, len(f))
s = self.ssea*np.exp(-x/self.Ls)
s = s.reshape(len(s), 1)*f
return s
def updateIteratorRegistry(self, moduleList):
"""Update the register of the special iterator module (IteratorModule in register).
This has an input variable modules that it controls and it takes its registry data"""
if self.isIteratorModule():
modules = toList(self.__input.v('modules'))
modules = [
i for j in modules for i in moduleList
if (i.getName().split('.')[0] + '.' +
i.getName().split('.')[-1] == j)
]
inputInit = toList(self.__register.v('inputInit'))
input = toList(self.__register.v('input'))
output = toList(self.__register.v('output'))
run = []
for module in modules:
# get registry data of the dependent modules
inputInit += toList(module.getInputRequirements(True))
input += toList(module.getInputRequirements())
output += toList(module.getOutputVariables())
# remove the dependent module from the module list
moduleList.remove(module)
# compile final list of register data
self.__register.addData('inputInit', list(set(inputInit)))
self.__register.addData('input', list(set(input)))
self.__register.addData('output', list(set(output)))
self.__toRun()
# append input data with the module object references, instead of the module names
self.addInputData({'modules': modules})
return moduleList
def read(self, iteration):
file = self.files[iteration]
if self.input.v('variables') == 'all':
varlist = None
else:
varlist = ny.toList(self.input.v('variables')) + ['grid']
d = ny.pickleload(os.path.join(self.folder, file), varlist)
d['experimentnumber'] = iteration
return d
def dictExpand(self, d, subindex, subsubindices):
if not subindex in d:
d[subindex] = {}
elif not isinstance(d[subindex], dict):
d[subindex] = {}
for ssi in ny.toList(subsubindices):
if not ssi in d[subindex]:
d[subindex][ssi] = 0
return d
def run_init(self):
d = {}
modules = ny.toList(self.input.v('modules'))
for module in modules:
module.addInputData(self.input.data)
d1 = module.run(init=True)
d.update(d1.data)
return d
def __toRun(self, alwaysRun=False):
"""Check if this module should be run based on the output requirements.
If so, also set the submodules that should be run (if there are any submodules)
Notes:
This is not the final verdict on whether this module will run and what list of submodules will be run.
An update will be done from the module list
Sets the class variables
runModule - (bool) should this module run
_submodulesToRun - (list) list of submodules to run or an empty list
"""
self.__submodulesToRunInit = []
self.runModule = alwaysRun # default setting: False, except when it is forced to be run from outside
# check if the module has submodules
if self.__register.v('submodules'):
sublist = self.__submoduleList()
# loop over variables in the output requirements
for var in toList(self.__outputReq.v('requirements')):
submoduleList = []
# gather the submodules that have 'var' as output
for submod in sublist:
outputList = toList(self.__register.v(submod, 'output'))
if var in outputList:
submoduleList.append(submod)
self.__submodulesToRunInit = list(
set(self.__submodulesToRunInit + submoduleList))
if self.__submodulesToRunInit:
self.runModule = True # if any submodules should be run, then so does the module
# check general module output
# loop over variables in the output requirements
for var in toList(self.__outputReq.v('requirements')):
outputList = toList(self.__register.v('output'))
if var in outputList:
self.runModule = True
# set submodules to run
self.submodulesToRun = self.__submodulesToRunInit
return
def run_init(self):
d = {}
modules = ny.toList(self.input.v('modules'))
for module in modules:
module.addInputData(self.input.data)
d1 = module.run(init=True)
d.update(d1)
self.input.merge(d1) # make output of last run module available to the next ones
return d
def __init__(self, input):
self.input = input
self.variables = toList(self.input.v('variables'))
# check if values of variables are provided
for var in self.variables:
if self.input.v(var) is None:
message = ("Not all required variables are given. Missing '%s' in module '%s'" % (var, self.__module__))
raise KnownError(message)
# load values to loop over from input
self.values = {}
for var in self.variables:
# check if values are given directly or in a separate substructure
varkeys = self.input.getKeysOf(var)
# case 1: values supplied directly
if not varkeys:
values = self.input.v(var)
values = self.interpretValues(values)
self.values[var] = toList(values)
# case 2: values in sub-dict
else:
# load values per key
values = {}
for key in varkeys:
temp = self.input.v(var, key)
values[key] = self.interpretValues(temp)
# check that all keys have the same number of arguments
for key in varkeys:
if len(values[key]) != len(values[varkeys[0]]):
raise KnownError('Problem with values of "%s" in input of module %s. Number of values in "%s" is unequal to number of values in "%s" ' % (var, self.__module__, key, varkeys[0]))
# rewrite dict with lists to list with dicts
self.values[var] = []
for i in range(0,len(values[varkeys[0]])):
self.values[var].append({})
for key in varkeys:
self.values[var][i][key]=values[key][i]
return
def __refactorUtil(self, inputList, data, **kwargs):
"""Replace tags @{}, +{} and if{}. Do this recursively by removing an item from the inputList, replacing one occurrence
of @{} and +{} and putting the result at the back of inputList.
"""
i = 0
while i<len(inputList):
item = inputList[i]
if item.find('@')>=0 or item.find('+{')>=0 or item.find('if{')>=0:
inputList.pop(i)
if item.find('@')>=0 and item.find('@{')<0:
if '.' in item:
dict = item[1:].split('.')[0]
key = item[1:].split('.')[-1]
item = toList(kwargs[dict].v(key))
else:
item = toList(data.v(item[1:]))
inputList = inputList + item
elif item.find('@{')>=0:
start = item.find('@{')
end = item.find('}')
if '.' in item[start+2:end]:
dict = item[start+2:end].split('.')[0]
key = item[start+2:end].split('.')[-1]
item = [item[:start]+j+item[end+1:] for j in toList(kwargs[dict].v(key))]
else:
item = [item[:start]+str(j)+item[end+1:] for j in toList(data.v(item[start+2:end]), forceNone=True)] #20-01-2017 added forceNone=True
inputList = inputList + item
elif item.find('+{')>=0:
start = item.find('+{')
end = item.find('}')
setrange = range(*eval(item[start+2:end]))
item = item[:start]+'%s'+item[end+1:]
for k in setrange:
inputList = inputList + toList(item % k)
elif item.find('if{')>=0: # added YMD 02-11-2016
start = item.find('if{')
end = item.find('}')
item = item[start+3:end]
item = item.split(',')
if len(item)!=2:
raise KnownError('error in registry in if{}. Does not have two comma-separated arguments. Make sure there are no spaces in the if-statement')
try:
if eval(item[1]):
inputList = inputList + toList(item[0])
except NameError:
item1 = item[1].split('==')[0]
item2 = item[1].split('==')[1]
if item1 == eval(item2):
inputList = inputList + toList(item[0])
else:
i += 1
return list(set(inputList))
def run(self):
self.logger.info('Module calcQ.py is running')
d = {}
QTrib = nty.toList(self.input.v('QTributary'))
Q1 = self.input.v('Q') * QTrib[0] / 100.
Qsource = [self.input.v('Q')*QTrib[1]/100., self.input.v('Q')*QTrib[2]/100.]
d['Q1'] = Q1
d['Qsource'] = Qsource
return d
def derivative(self, x, f, **kwargs):
if kwargs['dim'] == 'x':
x = np.asarray(x*self.L)
f = [1-bool(j) for j in toList(f)]
f = np.asarray(f).reshape(1, len(f))
sx = -self.ssea/self.Ls*np.exp(-x/self.Ls)
sx = sx.reshape(len(sx), 1)*f
else:
sx = None
FunctionBase.derivative(self)
return sx
def derivative(self, x, f, **kwargs):
if kwargs['dim'] == 'x':
x = np.asarray(x*self.L)
f = [1-bool(j) for j in toList(f)]
f = np.asarray(f).reshape(1,len(f))
sx = self.ssea/2.*(np.tanh((x-self.xc)/self.xl)**2-1)/self.xl
sx = sx.reshape(len(sx),1)*f
else:
sx = None
FunctionBase.derivative(self)
return sx
def run(self):
d = {}
modules = ny.toList(self.input.v('modules'))
for module in modules:
module.addInputData(self.input.data)
d1 = module.run()
d.update(d1.data)
if self.iteration >= self.MAXITERS: # output variable indicating if iterotor has converged or not.
d['converged'] = False
else:
d['converged'] = True
return d
def run(self):
d = {}
modules = ny.toList(self.input.v('modules'))
for module in modules:
module.addInputData(self.input.data)
d1 = module.run()
d.update(d1)
self.input.merge(d1) # make output of last run module available to the next ones
if self.iteration >= self.MAXITERS: # output variable indicating if iterotor has converged or not.
d['converged'] = False
else:
d['converged'] = True
return d
