def dump_sg_results(sg_results,
variables,
config_data,
user_core_models={},
save=False):
'''
Adds sg_results for each core_model to the csv file indexed under core_model['ia']
where the system_type of each core_model is specified in config_data.
If the core_model is not in the database, you must provide your own core_model
in user_core_models and index it under its system_type.
'''
new_rows = {}
for key in sg_results:
system_type = config_data[key].get('system_type')
core_model = user_core_models.get(system_type)
if not core_model:
core_model = mh.quick_search(system_type)
fixed_parameters = list(config_data[key]['parameter_values'].keys())
new_row = update_core_model(sg_result=sg_results[key],
model_variables=variables[key],
core_model=core_model,
save=save,
fixed_parameters=fixed_parameters,
**config_data[key])
new_rows[key] = new_row
return new_rows
def export_sg_results(sg_results,
variables,
config_data,
user_core_models={},
filename='sg_results.yaml'):
'''
Exports the results as a human-readable .yaml file
'''
yaml_dict = {}
for key in sg_results:
system_type = config_data[key].get('system_type')
core_model = user_core_models.get(system_type)
if not core_model:
core_model = mh.quick_search(system_type)
fixed_parameters = list(config_data[key]['parameter_values'].keys())
nested_dict = make_new_nested_dict(sg_result=sg_results[key],
model_variables=variables[key],
core_model=core_model,
fixed_parameters=fixed_parameters,
**config_data[key])
nested_dict = {**{'system_type': system_type}, **nested_dict}
yaml_dict[key] = nested_dict
outfile = filename if filename[-5:] == '.yaml' else filename + '.yaml'
try:
with open(outfile, 'w') as file:
ya.dump(yaml_dict, file, sort_keys=False)
except FileNotFoundError: #user wants to use relative path
cwd = Path(os.getcwd())
rel_path = Path(outfile)
folder = rel_path.parent
stem = rel_path.stem
ext = rel_path.suffix
outfile = stem + ext if ext == '.yaml' else stem + ext + '.yaml'
os.mkdir(folder)
full_path = cwd / folder / outfile
with open(full_path, 'w') as file:
ya.dump(yaml_dict, file, sort_keys=False)
except Exception as e:
raise e
return yaml_dict
def get_sampler_args_bh(filename, user_core_models={}):
'''Shortcut for setting up differential evolution for one model
:meta: private
'''
config_data = from_config(filename, 'bh') if type(filename) == str else filename
core_models = [mh.quick_search(config_data[key]['system_type']) for key in config_data]
models, params = compile_models(core_models, config_data)
guess = {}
bounds = {}
priors = {}
fixed_parameters = []
step_size = {}
scipy_args = {}
for key in config_data:
models[key]['init'] = config_data[key]['init']
models[key]['tspan'] = config_data[key]['tspan']
models[key]['int_args']['solver_args'] = config_data[key]['solver_args']
temp = {param + '_' + str(key): float(config_data[key]['guess'][param]) for param in config_data[key]['guess']}
guess = {**guess, **temp}
temp = {param + '_' + str(key): config_data[key]['parameter_bounds'][param] for param in config_data[key]['parameter_bounds']}
bounds = {**bounds, **temp}
temp = {param + '_' + str(key): config_data[key]['priors'][param] for param in config_data[key]['priors']}
priors = {**priors, **temp}
temp = {param + '_' + str(key): config_data[key]['step_size'][param] for param in config_data[key]['step_size']}
step_size = {**step_size, **temp}
fixed_parameters += [param + '_' + str(key) for param in config_data[key]['fixed_parameters']]
fixed_parameters += [i + '_' + str(key) for i in core_models[key-1]['inputs']]
scipy_args = config_data[key]['bh_args'] if 'bh_args' in config_data[key] else scipy_args
sampler_args = {'data' : {},
'guess' : guess,
'models' : models,
'fixed_parameters' : fixed_parameters,
'priors' : priors,
'bounds' : bounds,
'step_size' : step_size,
'scipy_args' : scipy_args,
'step_size_is_ratio' : False
}
return sampler_args, config_data
def backend_add_to_database(settings, database, dialog=False):
'''Supporting function for add_to_database. Do not run.
:meta private:
'''
#Check core model exists
core_model = mh.quick_search(settings['system_type'])
#Prevent duplicates between MBase and UBase
system_type_ensemble_name = (settings['system_type'],
settings['settings_name'])
if database == mh.MBase and system_type_ensemble_name in list_settings(
database=mh.UBase):
raise Exception(
str(system_type_ensemble_name) +
' cannot be added to MBase as it is already in UBase.')
if database == mh.UBase and system_type_ensemble_name in list_settings(
database=mh.MBase):
raise Exception(
str(system_type_ensemble_name) +
' cannot be added to UBase as it is already in MBase.')
params_dict = settings['parameters'].to_dict('list')
params_dict = {
key: params_dict[key]
for key in settings['parameters'].to_dict('list')
}
row = {
'system_type': settings['system_type'],
'settings_name': settings['settings_name'],
'units': str(settings['units']),
'parameters': str(params_dict),
}
for key1 in ['init', 'priors', 'parameter_bounds']:
row[key1] = str(
{key2: list(settings[key1][key2])
for key2 in settings[key1]})
row['tspan'] = str([list(span) for span in settings['tspan']])
row['fixed_parameters'] = str(settings['fixed_parameters'])
row['solver_args'] = str(settings['solver_args'])
mh.add_row('settings', row, database)
print('Added settings ' + row['settings_name'])
return row['system_type'], row['settings_name']
def setup_helper(filename, reader, user_core_models={}):
'''
The first return value is the config_data which will be generated based on
three possible scenarios:
1. filename is a dictionary
The filename is already config_data. No processing needed.
2. filename is a string
filename is name of the settings file to be opened and parsed by reader.
3. filename is neither of the above
filename is an iterable containing names of settings file. It will be
iteratively parsed by reader and subsequently reindexed to give config_data.
The second return value is a list of core_model data structures associated with
config_data. The list is arranged in the order given by config_data and the
core_models are retrieved based on two possible scenarios:
1. The system_type in config_data[model_num]['system_type'] is in core_models
The core_model indexed under core_models[system_type] will be used.
2. The system_type in config_data[model_num]['system_type'] is not in core_models
The function searches the BMSS database for the appropriate core_model
using quick_search
:meta: private
'''
if type(filename) == str:
config_data = reader(filename)
elif type(filename) == dict: #Assume user has already imported the data
config_data = filename
else: #Assume data is spread across multiple files
config_data = [value for f in filename for value in reader(f).values()]
config_data = dict(enumerate(config_data, start=1))
core_models = [
user_core_models[config_data[key]['system_type']]
if config_data[key]['system_type'] in user_core_models else
mh.quick_search(config_data[key]['system_type']) for key in config_data
]
core_models = [
mh.copy_core_model(core_model) for core_model in core_models
]
return config_data, core_models
def make_settings_template(system_types_settings_names, filename='', user_core_models={}):
'''Writes settings to a config file. If you are using a core_model that is
not in the database, you must provide the core_model using the core_models
argument where the key is the system_type.Returns the code as a string.
:param system_types_settings_names: Pairs of tuples containing (system_type, settings_name)
:type system_types_settings_names: list or tuple
:param filename: The name of the file to write to
:type filename: str, optional
:param user_core_model: A dictionary of core_models indexed by their system_type.
core_models already in the database do not need to be specified here.
:type user_core_model: dict, optional
'''
result = ''
system_types_settings_names1 = [system_types_settings_names] if type(system_types_settings_names) == str else system_types_settings_names
for pair in system_types_settings_names1:
try:
system_type, settings_name = pair
except:
system_type, settings_name = pair, '__default__'
core_model = user_core_models[system_type] if system_type in user_core_models else mh.quick_search(system_type)
parameters = core_model['parameters']
inputs = core_model['inputs']
states = core_model['states']
section_header = '[' + core_model['system_type'] + ']'
settings = {'system_type' : system_type,
'settings_name' : settings_name,
'parameters' : {},
'units' : {},
'init' : {},
'priors' : {},
'parameter_bounds': {},
'tspan' : [],
'fixed_parameters' : [],
'solver_args' : {'rtol' : 1.49012e-8,
'atol' : 1.49012e-8,
'tcrit' : [],
'h0' : 0.0,
'hmax' : 0.0,
'hmin' : 0.0,
'mxstep' : 0
},
}
if settings_name:
settings = sh.quick_search(system_type, settings_name, skip_constructor=True)
longest = len(max(parameters, key=len))
longest_ = len(max(inputs, key=len)) if inputs else None
init_values = dict_template('init', states, longest, settings['init'], default = 0)
param_values = dict_template('parameter_values', parameters, longest, settings['parameters'], default = 1)
input_conditions = dict_template('input_conditions', inputs, longest_, dict(zip(inputs, [0]*len(inputs)))) if inputs else ''
fixed_parameters = list_template('fixed_parameters', settings['fixed_parameters'])
measured_states = list_template('measured_states', states)
model_id = '#id = ' + str(core_model['id'])
model_equations = '#equations = \n' + '\n'.join(['#\t' + line for line in core_model['equations'] ])
section_header = '\n'.join([section_header, model_id, model_equations])
result += '\n\n'.join([section_header, param_values, init_values, fixed_parameters, measured_states, input_conditions])
if filename:
with open(filename, 'w') as file:
file.write(result)
return result
def make_settings_template(system_types_settings_names, filename=''):
'''Writes settings to a config file. Returns the code as a string.
:param system_types_settings_names: Pairs of tuples containing (system_type, settings_name)
:type system_types_settings_names: list or tuple
:param filename: The name of the file to write to
:type filename: str, optional
'''
result = ''
system_types_settings_names1 = [system_types_settings_names] if type(system_types_settings_names) == str else system_types_settings_names
for pair in system_types_settings_names1:
try:
system_type, settings_name = pair
except:
system_type, settings_name = pair, '__default__'
if type(system_type) == str:
core_model = mh.quick_search(system_type)
else:
core_model = system_type
system_type = core_model['system_type']
parameters = core_model['parameters'] + core_model['inputs']
states = core_model['states']
section_header = '[]\nsystem_type = ' + core_model['system_type']
if settings_name:
settings = quick_search(system_type, settings_name, skip_constructor=True)
else:
settings = {'system_type' : system_type,
'settings_name' : settings_name,
'parameters' : {},
'units' : {},
'init' : {},
'priors' : {},
'parameter_bounds' : {},
'tspan' : [],
'fixed_parameters' : [],
'solver_args' : {'rtol' : 1.49012e-8,
'atol' : 1.49012e-8,
'tcrit' : [],
'h0' : 0.0,
'hmax' : 0.0,
'hmin' : 0.0,
'mxstep' : 0
},
}
settings['init'] = DataFrame.from_dict(settings['init'], orient='index', columns=states).to_dict('list')
longest = len(max(parameters, key=len))
longest_ = len(max(settings['solver_args'], key=len))
solver_args = settings['solver_args'].keys()
init_values = dict_template('init', states, longest, settings['init'])
param_values = dict_template('parameter_values', parameters, longest, settings['parameters'])
prior_values = dict_template('priors', parameters, longest, settings['priors'])
bounds_values = dict_template('parameter_bounds', parameters, longest, settings['parameter_bounds'])
units_values = dict_template('units', parameters, longest, settings['units'], default='')
tspan = list_template('tspan', [list(segment) for segment in settings['tspan']])
fixed_parameters = list_template('fixed_parameters', settings['fixed_parameters'])
solver_args = dict_template('solver_args', solver_args, longest_, settings['solver_args'])
model_id = '#id = ' + str(core_model['id'])
model_equations = '#equations = \n' + '\n'.join(['#\t' + line for line in core_model['equations'] ])
section_header = '\n'.join([section_header, model_id, model_equations])
result += '\n\n'.join([section_header, init_values, param_values, prior_values, bounds_values, units_values, tspan, fixed_parameters, solver_args])
if filename:
with open(filename, 'w') as file:
file.write(result)
return result
def make_settings(system_type, settings_name, units,
parameters, init=None, tspan=None,
priors={}, parameter_bounds={}, fixed_parameters=[],
solver_args={}, init_orient='scenario',
user_core_model=None,
**ignored):
'''Checks and standardizes formatting of data types. Ignores irrelavant keyword arguments.
Returns a settings data structure which is a dict.
:param system_type: A string of keywords serving as a unique identifier for
the core_model separated by commas, will be formatted so there is one space
after each comma, keywords should be in CamelCase
:type system_type: str
:param settings_name: A string, forms a unique identifier when paired with system_type
:type settings_name: str
:param units: A dict where the keys are parameter names and the values are the units in strings
:type units: dict
:param parameters: A pandas DataFrame of parameter values OR a dictionary,
column names/keys must match the parameter names of the corresponding core_model
:type parameters: pandas DataFrame or dict
:param init: A dictionary where the keys correspond to a state in the core_model and
the values are the initial values for numerical integration OR where the keys
correspond to different scenarios
:type init: dict
:param tspan: A list of segments for piecewise numerical integration where each segments
is a list of time points, defaults to values generated using numpy.linspace
:type tspan: list, optional
:param priors: A dictionary of Gaussian priors for parameter estimation where each key
is a parameter name and each value a tuple in the form (mean, standard_deviation)
:type priors: dict, optional
:param parameter_bounds: A dictionary of parameter bounds for parameter estimation
where each key is a parameter name and each value a tuple in the form (lower, upper)
:type parameter_bounds: dict, optional
:param fixed_parameters: A list of parameter names that will be fixed during parameter estimation.
:type fixed_parameters: list, optional
:param solver_args: Additional keyword arguments for scipy.integrate.odeint, defaults to None
:type solver_args: dict, optional
:param init_orient: 'scenario' if the keys in init are scenario numbers, 'state'
if the keys are the state names, defaults to 'scenario'
:type init_orient: str
:param user_core_model: A core_model data structure for cross-checking. If None,
this function will search the database for the core_model.
:type user_core_model: dict
'''
system_type1 = system_type if type(system_type) == str else ', '.join(system_type)
if user_core_model:
core_model = user_core_model
if core_model['system_type'] != system_type1:
raise Exception('system_type does not match that of user_core_model.')
print('Making settings using user_core_model')
else:
core_model = mh.quick_search(system_type1)
param_values = sc.check_and_assign_param_values(core_model, parameters)
init1 = sc.check_and_assign_init(core_model['states'], init, init_orient)
tspan1 = sc.check_and_assign_tspan(tspan)
priors1 = sc.check_and_assign_priors(param_values, priors)
bounds1 = sc.check_and_assign_parameter_bounds(param_values, parameter_bounds)
solver_args1 = sc.check_and_assign_solver_args(solver_args)
if fixed_parameters:
fixable_parameters = core_model['parameters'] + core_model['inputs']
if not all([p in fixable_parameters for p in fixed_parameters]):
raise Exception('Error in fixed parameters. Unexpected parameters found.')
settings_name1 = '__default__' if settings_name == '_' else settings_name
result = {'system_type' : core_model['system_type'],
'settings_name' : settings_name1,
'units' : units,
'parameters' : param_values,
'init' : init1,
'priors' : priors1,
'parameter_bounds' : bounds1,
'tspan' : tspan1,
'fixed_parameters' : fixed_parameters,
'solver_args' : solver_args1,
}
return result
def from_config(filename, user_core_models={}):
'''Returns a settings data structure.
:param filename: Name of file to read.
:type filename: str
:param user_core_models: A dictionary of core_models required for calling make_settings,
to be ignored if the core_model is already in the database.
:type user_core_models: dict, optional
'''
config = configparser.RawConfigParser()
config.optionxform = lambda option: option
all_settings = []
with open(filename, 'r') as file:
config.read_file(file)
for section in config.sections():
if section in ['id', 'system_type', 'states', 'parameters', 'inputs', 'equations', 'ia']:
continue
if 'system_type' not in config[section]:
continue
system_type = config[section]['system_type']
params = config[section]['parameter_values']
units = config[section]['units']
init = config[section].get('init', {})
priors = config[section].get('priors', {})
bounds = config[section].get('parameter_bounds', {})
tspan = config[section].get('tspan', '[[0, 600, 31]]')
fixed_parameters = config[section].get('fixed_parameters', '[]')
settings_name = str(section)
core_model = mh.quick_search(system_type)
system_type = core_model['system_type']
states = core_model['states']
if not settings_name:
raise Exception('Ensemble name not valid!')
settings_name = '__default__' if settings_name == '_' else settings_name
if init:
if '=' in init:
init = string_to_dict(init)
try:
init = DataFrame(init)
init.index += 1
init = init.T.to_dict('list')
except:
init = {1: list(init.values())}
for key in init:
if len(init[key]) != len(states):
raise Exception('Length of init must match number of states. system_type given: ' + str(system_type))
else:
init = eval(init)
if len(init) != len(states):
raise Exception('Length of init must match number of states. system_type given: ' + str(system_type))
init = {1: init}
params = string_to_dict(params)
units = string_to_dict(units)
priors = string_to_dict(priors) if priors else {}
bounds = string_to_dict(bounds) if bounds else {}
fixed_parameters = string_to_list_string(fixed_parameters)
tspan = eval_tspan_string(tspan)
settings = {'system_type' : system_type,
'settings_name' : settings_name,
'parameters' : params,
'units' : units,
'init' : init,
'priors' : priors,
'parameter_bounds' : bounds,
'tspan' : tspan,
'fixed_parameters' : fixed_parameters
}
user_core_model = user_core_models.get(settings['system_type'], {})
all_settings.append(make_settings(**settings, user_core_model=user_core_model))
return all_settings
def search_database(system_type='', settings_name='', database=None, skip_constructor=False, active_only=True):
'''Searches database for settings data structure based on system_type and settings
name. Returns a list of settings dictionaries.
:param system_type: A string that will be matched against entries in the database
:type system_type: str
:param settings_name: A string corresponding to any key in the core_model, will
be used for matching
:type settings_name: str
:param database: Can be either MBase or UBase, if None, this function will search
both databases, defaults to None
:type database: SQL Connection, optional
:param skip_constructor: For backend use, defaults to False
:type skip_constructor: bool, optional
:param active_only: A boolean for backend use, if True, this limits the search
to rows where the value of active is True, defaults to True
:type active_only: bool, optional
'''
#Can only add a settings for active models
core_model = mh.quick_search(system_type, active_only=True)
result = []
if settings_name and system_type:
comm = 'SELECT * FROM settings WHERE system_type LIKE "%' + system_type + '%" AND settings_name LIKE "%' + settings_name
elif settings_name:
comm = 'SELECT * FROM settings WHERE settings_name LIKE "%' + settings_name
else:
comm = 'SELECT * FROM settings WHERE system_type LIKE "%' + system_type
if active_only:
comm += '%" AND active = 1;'
else:
comm += '%";'
databases = [database] if database else [mh.MBase, mh.UBase]
for database in databases:
with database as db:
cursor = db.execute(comm)
all_settings = cursor.fetchall()
columns = database.execute('PRAGMA table_info(settings);')
columns = columns.fetchall()
columns = [column[1] for column in columns if column[1]!='active']
all_settings = [dict(zip(columns, s[:-1])) for s in all_settings]
for s in all_settings:
s['units'] = eval(s['units'])
s['parameters'] = eval(s['parameters'])
s['init'] = eval(s['init'])
s['priors'] = eval(s['priors'])
s['parameter_bounds'] = eval(s['parameter_bounds'])
s['tspan'] = eval(s['tspan'])
s['fixed_parameters'] = eval(s['fixed_parameters'])
s['solver_args'] = eval(s['solver_args'])
if skip_constructor:
result += all_settings
else:
result += [make_settings(**s) for s in all_settings]
return result
