def test_hdf5_input_output():
"""Checks the data inside a .hdf5 input file."""
config_file = './study/profit_hdf5.yaml'
config = Config.from_file(config_file)
try:
run(f"profit run {config_file}", shell=True, timeout=TIMEOUT)
data_in = load(config['files'].get('input'))
assert data_in.shape == (2, 1)
assert data_in.dtype.names == ('u', 'v', 'w')
finally:
clean(config)
def multi_test_1d(study, config_file, output_file):
""" test 1D with different config files """
config_file = path.join(study, config_file)
output_file = path.join(study, output_file)
config = Config.from_file(config_file)
try:
run(f"profit run {config_file}", shell=True, timeout=TIMEOUT)
output = load(output_file)
assert output.shape == (7, 1)
assert all(output['f'] - array([0.7836, -0.5511, 1.0966, 0.4403, 1.6244, -0.4455, 0.0941]).reshape((7, 1))
< 1e-4)
finally:
clean(config)
def load_model(cls, path):
"""Loads a saved model from a .hdf5 file and updates its attributes. In case of a multi-output model, the .pkl
file is loaded, since .hdf5 is not supported yet.
Parameters:
path (str): Path including the file name, from where the model should be loaded.
Returns:
GPy.models: Instantiated surrogate model.
"""
from profit.util import load
from .encoders import Encoder
from GPy import models
self = cls()
try:
sur_dict = load(path, as_type='dict')
self.model = models.GPRegression.from_dict(sur_dict['model'])
self.Xtrain = sur_dict['Xtrain']
self.ytrain = sur_dict['ytrain']
self.encoder = [
Encoder(func, cols, out)
for func, cols, out in eval(sur_dict['encoder'])
]
except (OSError, FileNotFoundError):
from pickle import load as pload
from os.path import splitext
# Load multi-output model from pickle file
print(
"File {} not found. Trying to find a .pkl file with multi-output instead."
.format(path))
self.model, self.Xtrain, self.ytrain, encoder_str = pload(
open(splitext(path)[0] + '.pkl', 'rb'))
self.encoder = [
Encoder(func, cols, out)
for func, cols, out in eval(encoder_str)
]
self.output_ndim = int(max(self.model.X[:, -1])) + 1
self.multi_output = True
# Initialize the encoder by encoding and decoding the training data once.
self.encode_training_data()
self.decode_training_data()
self.kernel = self.model.kern
self._set_hyperparameters_from_model()
self.ndim = self.Xtrain.shape[-1]
self.trained = True
self.print_hyperparameters("Loaded")
return self
def load_model(cls, filename):
""" Load a saved GPySurrogate object with hdf5 format. """
from profit.util import load
sur_dict = eval(load(filename, as_type='dict').get('data'))
self = cls()
for attr, value in sur_dict.items():
setattr(
self, attr, value if attr != 'm' else
cls.GPy.models.GPRegression.from_dict(value))
if self.m:
self.xtrain = self.m.X
self.ytrain = self.m.Y
self.kern = self.m.kern
return self
def load_model(cls, path):
"""Load a saved SklGPSurrogate model from a pickle file and update its attributes.
Parameters:
path (str): Path including the file name, from where the model should be loaded.
Returns:
profit.sur.gaussian_process.SklearnGPSurrogate: Instantiated surrogate model.
"""
from pickle import load
self = cls()
self.model = load(open(path, 'rb'))
self.Xtrain = self.model.X_train_
self.ytrain = self.model.y_train_
self.kernel = self.model.kernel_
self.ndim = self.Xtrain.shape[-1]
self.fixed_sigma_n = self.model.alpha != 1e-5
self.trained = True
self._set_hyperparameters_from_model()
self.print_hyperparameters("Loaded")
return self
def load_model(cls, path):
"""Loads a saved model from a .hdf5 file and updates its attributes. In case of a multi-output model, the .pkl
file is loaded, since .hdf5 is not supported yet.
Parameters:
path (str): Path including the file name, from where the model should be loaded.
Returns:
GPy.models: Instantiated surrogate model.
"""
from profit.util import load
from GPy import models
self = cls()
try:
model_dict = load(path, as_type='dict')
self.model = models.GPRegression.from_dict(model_dict)
self.Xtrain = self.model.X
self.ytrain = self.model.Y
except (OSError, FileNotFoundError):
from pickle import load as pload
from os.path import splitext
# Load multi-output model from pickle file
self.model = pload(open(splitext(path)[0] + '.pkl', 'rb'))
self.output_ndim = int(max(self.model.X[:, -1])) + 1
self.Xtrain = self.model.X[:len(self.model.X) // self.output_ndim, :-1]
self.ytrain = self.model.Y.reshape(-1, self.output_ndim, order='F')
self.multi_output = True
self.kernel = self.model.kern
self._set_hyperparameters_from_model()
self.ndim = self.Xtrain.shape[-1]
self.trained = True
self.print_hyperparameters("Loaded")
return self
def init_app(config):
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
server = app.server
app.config.suppress_callback_exceptions = False
indata = load(config['files']['input']).flatten()
outdata = load(config['files']['output']).flatten()
invars = indata.dtype.names
outvars = outdata.dtype.names
dd_opts_in = [{'label': invar, 'value': invar} for invar in invars]
dd_opts_out = [{'label': outvar, 'value': outvar} for outvar in outvars]
col_width = 400
txt_width = 100
dd_width = 250
log_width = 50
graph_height = 620
txt_check_width = 50
check_txt_width = txt_width-txt_check_width
ax_opt_tit_sty = {'width': col_width}
ax_opt_txt_sty = {'width': txt_width}
ax_opt_log_sty = {'width': log_width}
dd_sty = {'width': dd_width}
axis_options_div_style = {'display': 'flex', 'align-items': 'center', 'height':36, 'padding': 1}
fit_opt_txt_sty = {'width': txt_width}
headline_sty = {'text-align': 'center', 'display': 'block', 'width': col_width-25}
input_div_sty = {'height': 40}
input_sty = {'width': 125}
col_sty = {'padding-left':5, 'padding-right':5}
col_sty_th = {**col_sty, 'text-align':'center'}
button_sty = {'padding-left':15, 'padding-right':15}
# try to load model with 'save' and 'fit' config option
path = config['fit']['save'] or config['fit']['load']
try:
sur = Surrogate.load_model(path)
except (TypeError, FileNotFoundError):
print('Model could not be loaded')
app.layout = html.Div(children=[
html.Table(children=[html.Tr(children=[
html.Td(id='axis-options', style={'width': '20%'}, children=[
html.Div(dcc.RadioItems(
id='graph-type',
options=[{'label': i, 'value': i} for i in ['1D', '2D', '2D contour', '3D']],
value='1D',
labelStyle={'display': 'inline-block'})),
html.Div(id='header-opt', children=[html.B("Axis options:", style=headline_sty)], style=ax_opt_tit_sty),
html.Div(id='invar-1-div', style=axis_options_div_style, children=[
html.B('x: ', style=ax_opt_txt_sty),
dcc.Dropdown(id='invar', options=dd_opts_in, value=invars[0], style=dd_sty),
dcc.Checklist(id='invar-1-log', options=[{'label': 'log', 'value': 'log'}], style=ax_opt_log_sty),
]),
html.Div(id='invar-2-div', style=axis_options_div_style, children=[
html.B('y: ', style=ax_opt_txt_sty),
dcc.Dropdown(
id='invar_2',
options=dd_opts_in,
value=invars[1] if len(invars) > 1 else invars[0],
style=dd_sty, ),
dcc.Checklist(
id='invar-2-log',
options=[{'label': 'log', 'value': 'log'}],
style=ax_opt_log_sty, ),
]),
html.Div(id='invar-3-div', style=axis_options_div_style, children=[
html.B('z: ', style=ax_opt_txt_sty),
dcc.Dropdown(
id='invar_3',
options=dd_opts_in,
value=invars[2] if len(invars) > 2 else invars[0],
style=dd_sty,
),
dcc.Checklist(
id='invar-3-log',
options=[{'label': 'log', 'value': 'log'}],
style=ax_opt_log_sty,
),
]),
html.Div(id='outvar-div', style=axis_options_div_style, children=[
html.B('output: ', style=ax_opt_txt_sty),
dcc.Dropdown(
id='outvar',
options=dd_opts_out,
value=outvars[0],
style=dd_sty, ),
dcc.Checklist(
id='outvar-log',
options=[{'label': 'log', 'value': 'log'}],
style=ax_opt_log_sty, ),
]),
html.Div(id='color-div', style=axis_options_div_style, children=[
html.B("color: ", style={'width': txt_check_width}),
dcc.Checklist(
id='color-use',
options=[{'label': '', 'value': 'true'}],
style={'width': check_txt_width},
value=['true'], ),
dcc.Dropdown(
id='color-dropdown',
options=[{'label': 'OUTPUT', 'value': 'OUTPUT'}] + dd_opts_in + dd_opts_out,
value='OUTPUT',
style=dd_sty, ),
]),
html.Div(id='error-div', style=axis_options_div_style, children=[
html.B("error: ", style={'width': txt_check_width}),
dcc.Checklist(
id='error-use',
options=[{'label': '', 'value': 'true'}],
style={'width': check_txt_width}, ),
dcc.Dropdown(
id='error-dropdown',
options=dd_opts_out,
value=outvars[-1],
style=dd_sty, ),
]),
html.Div(id='fit-opt', children=html.B("Fit options:", style=headline_sty), style=ax_opt_tit_sty),
html.Div(id='fit-use-div', style=axis_options_div_style, children=[
html.B("display fit:", style=fit_opt_txt_sty),
dcc.Checklist(
id='fit-use',
options=[{'label': '', 'value': 'show'}],
labelStyle={'display': 'inline-block'}, ),
]),
html.Div(id='fit-multiinput-div', style=axis_options_div_style, children=[
html.B("multi-fit:", style=fit_opt_txt_sty),
dcc.Dropdown(
id='fit-multiinput-dropdown',
options=dd_opts_in,
value=invars[-1],
style=dd_sty, ),
]),
html.Div(id='fit-number-div', style=axis_options_div_style, children=[
html.B("#fits:", style=fit_opt_txt_sty),
dcc.Input(id='fit-number', type='number', value=1, min=1),
]),
html.Div(id='fit-conf-div', style=axis_options_div_style, children=[
html.B("\u03c3-confidence:", style=fit_opt_txt_sty),
dcc.Input(id='fit-conf', type='number', value=2, min=0),
]),
html.Div(id='fit-noise-div', style=axis_options_div_style, children=[
dcc.Checklist(
id='fit-var',
options=[{'label': 'add noise covariance', 'value': 'add'}],
style={'margin-left': txt_width},
)
]),
html.Div(id='fit-color-div', style=axis_options_div_style, children=[
html.B("fit-color:", style=fit_opt_txt_sty),
dcc.RadioItems(
id='fit-color',
options=[{'label': 'output', 'value': 'output'},
{'label': 'multi-fit', 'value': 'multi-fit'},
{'label': 'marker-color', 'value': 'marker-color'}],
value='output',
labelStyle={'display': 'inline-block'}, ),
]),
html.Div(id='fit-opacity-div', style=axis_options_div_style, children=[
html.B("fit-opacity:", style=fit_opt_txt_sty),
html.Div(style={'width': col_width-txt_width}, children=[
dcc.Slider(
id='fit-opacity',
min=0,
max=1,
step=0.1,
value=0.5,
marks={i: {'label': f'{100 * i:.0f}%'} for i in [0, 0.2, 0.4, 0.6, 0.8, 1]},
),
]),
]),
html.Div(id='fit-sampling-div', style=axis_options_div_style, children=[
html.B("#points:", style=fit_opt_txt_sty),
dcc.Input(id='fit-sampling', type='number', value=50, min=1, debounce=True, style={'appearance': 'textfield'}),
]),
]),
html.Td(id='graph', style={'width': '80%'}, children=[html.Div(dcc.Graph(id='graph1'))]),
])]),
html.Div(html.Table(id='filters', children=[html.Tr([
html.Td(html.Div([
dcc.Dropdown(
id='filter-dropdown',
options=dd_opts_in,
value=invars[0],
style={'width': 200, 'margin-right': 10},
),
html.Button("Add Filter", id='add-filter', n_clicks=0, style=button_sty),
], style={'display': 'flex'}), style={**col_sty, 'border-bottom-width':0}),
html.Td(html.Button("Clear all Filter", id='clear-all-filter', n_clicks=0, style=button_sty), style={**col_sty, 'border-bottom-width':0}),
html.Td(dcc.Slider(id='scale-slider',
min=-0.5, max=0.5,
value=0, step=0.01,
marks={i: f'{100*i:.0f}%' for i in [-0.5, -0.25, 0, 0.25, 0.5]},
),
style={**col_sty, 'width': 500, 'border-bottom-width':0}
),
html.Td(html.Button("Scale Filter span", id='scale', n_clicks=0, style=button_sty), style={**col_sty, 'border-bottom-width':0}),
])])),
html.Div(html.Table(id='param-table', children=[
html.Thead(id='param-table-head', children=[
html.Tr(children=[
html.Th("Parameter", style={**col_sty, **input_sty}),
html.Th("log", style=col_sty_th),
html.Th("Slider", style={**col_sty_th, 'width': 300}),
html.Th("Range (min/max)", style=col_sty_th),
html.Th("center/span", style=col_sty_th),
html.Th("filter active", style=col_sty_th),
html.Th("#digits", style=col_sty_th),
html.Th("reset", style=col_sty_th),
html.Th("", style=col_sty_th),
]),
]),
html.Tbody(id='param-table-body', children=[
html.Tr(children=[
html.Td(html.Div(id='param-text-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-log-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-slider-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-range-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-center-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-active-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-digits-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-reset-div', children=[]), style=col_sty),
html.Td(html.Div(id='param-clear-div', children=[]), style=col_sty),
]),
]),
])),
])
@app.callback(
[Output('param-text-div', 'children'),
Output('param-log-div', 'children'),
Output('param-slider-div', 'children'),
Output('param-range-div', 'children'),
Output('param-center-div', 'children'),
Output('param-active-div', 'children'),
Output('param-digits-div', 'children'),
Output('param-reset-div', 'children'),
Output('param-clear-div', 'children'), ],
[Input('add-filter', 'n_clicks'),
Input('clear-all-filter', 'n_clicks'),
Input({'type': 'param-clear', 'index': ALL}, 'n_clicks')],
[State('filter-dropdown', 'value'),
State('param-text-div', 'children'),
State('param-log-div', 'children'),
State('param-slider-div', 'children'),
State('param-range-div', 'children'),
State('param-center-div', 'children'),
State('param-active-div', 'children'),
State('param-digits-div', 'children'),
State('param-reset-div', 'children'),
State('param-clear-div', 'children')],
)
def add_filterrow(n_clicks, clear_all, clear_clicks, filter_dd, text, log, slider, range_div, center_div, active_div, dig_div, reset_div, clear_div):
ctx = dash.callback_context
trigger_id = ctx.triggered[0]["prop_id"].split(".")[0]
if trigger_id == 'clear-all-filter':
return [], [], [], [], [], [], [], [], []
elif trigger_id == 'add-filter':
for i in range(len(text)):
if text[i]['props']['children'][0] == filter_dd:
#return text, log, slider, range_div, center_div, active_div, dig_div, reset_div, clear_div
raise PreventUpdate
ind = invars.index(filter_dd) #index of the variable
new_text = html.Div(id={'type': 'dyn-text', 'index': ind}, children=[filter_dd], style={**input_div_sty, **input_sty})
new_log = html.Div(id={'type': 'dyn-log', 'index': ind}, style={**input_div_sty, 'text-align':'center'}, children=[
dcc.Checklist(id={'type': 'param-log', 'index': ind}, options=[{'label': '', 'value': 'log'}])], )
new_slider = html.Div(id={'type': 'dyn-slider', 'index': ind}, style=input_div_sty, children=[
create_slider(filter_dd)], )
new_range = html.Div(id={'type': 'dyn-range', 'index': ind}, style=input_div_sty, children=[
dcc.Input(id={'type': 'param-range-min', 'index': ind}, type='number', debounce=True, style={**input_sty, 'appearance': 'textfield'}),
dcc.Input(id={'type': 'param-range-max', 'index': ind}, type='number', debounce=True, style={**input_sty, 'appearance': 'textfield'}),
], )
new_center = html.Div(id={'type': 'dyn-center', 'index': ind}, style=input_div_sty, children=[
dcc.Input(id={'type': 'param-center', 'index': ind}, type='number', debounce=True, style={**input_sty, 'appearance': 'textfield'}),
dcc.Input(id={'type': 'param-span', 'index': ind}, type='number', debounce=True, style={**input_sty, 'appearance': 'textfield'}),
], )
new_active = html.Div(id={'type': 'dyn-active', 'index': ind},
style={**input_div_sty, 'text-align': 'center'},
children=[
dcc.Checklist(id={'type': 'param-active', 'index': ind},
options=[{'label': '', 'value': 'act'}],
value=['act'], )
])
new_dig = html.Div(id={'type': 'dyn-dig', 'index': ind}, children=[
dcc.Input(id={'type': 'param-dig', 'index': ind}, type='number', value=5, min=0, style={'width':100})
], style={'height':40})
new_reset = html.Div(id={'type': 'dyn-reset', 'index': ind}, children=[
html.Button("reset", id={'type': 'param-reset', 'index': ind}, n_clicks=0,
style={'padding-left':15, 'padding-right':15})
])
new_clear = html.Div(id={'type': 'dyn-clear', 'index': ind}, children=[
html.Button("x", id={'type': 'param-clear', 'index': ind}, n_clicks=0,
style={'border':'none', 'padding-left':5, 'padding-right':5})
])
text.append(new_text)
log.append(new_log)
slider.append(new_slider)
range_div.append(new_range)
center_div.append(new_center)
active_div.append(new_active)
dig_div.append(new_dig)
reset_div.append(new_reset)
clear_div.append(new_clear)
elif len(trigger_id) >=1 and trigger_id[0] == "{":
for i in range(len(text)):
# search table row to delete
if int(text[i]['props']['id']['index']) == int(trigger_id.split(',')[0].split(':')[1]):
text.pop(i)
log.pop(i)
slider.pop(i)
range_div.pop(i)
center_div.pop(i)
active_div.pop(i)
dig_div.pop(i)
reset_div.pop(i)
clear_div.pop(i)
break
return text, log, slider, range_div, center_div, active_div, dig_div, reset_div, clear_div
@app.callback(
[Output({'type': 'param-range-min', 'index': MATCH}, 'step'),
Output({'type': 'param-range-max', 'index': MATCH}, 'step'),
Output({'type': 'param-center', 'index': MATCH}, 'step'),
Output({'type': 'param-span', 'index': MATCH}, 'step'),
Output({'type': 'param-slider', 'index': MATCH}, 'step'), ],
Input({'type': 'param-dig', 'index': MATCH}, 'value')
)
def update_step(dig):
""" Function to update an synchronise step-sizes throughout the filter-table.
Args:
dig (int): Number of digits to be used. Selected by the user via a 'dcc.Input'-layout-element.
Returns:
step: Step-size for the 4 'dcc.Input'-Elements and the slider step.
"""
step = 10**(-dig)
return step, step, step, step, step
@app.callback(
[Output({'type': 'param-range-min', 'index': MATCH}, 'value'),
Output({'type': 'param-range-max', 'index': MATCH}, 'value'),
Output({'type': 'param-slider', 'index': MATCH}, 'value'),
Output({'type': 'param-slider', 'index': MATCH}, 'min'),
Output({'type': 'param-slider', 'index': MATCH}, 'max'),
Output({'type': 'param-center', 'index': MATCH}, 'value'),
Output({'type': 'param-span', 'index': MATCH}, 'value'),
Output({'type': 'param-slider', 'index': MATCH}, 'marks'), ],
[Input('param-text-div', 'children'),
Input({'type': 'param-log', 'index': MATCH}, 'value'),
Input({'type': 'param-range-min', 'index': MATCH}, 'value'),
Input({'type': 'param-range-max', 'index': MATCH}, 'value'),
Input({'type': 'param-slider', 'index': MATCH}, 'value'),
Input({'type': 'param-center', 'index': MATCH}, 'value'),
Input({'type': 'param-span', 'index': MATCH}, 'value'),
Input('scale', 'n_clicks'),
Input({'type': 'param-dig', 'index': MATCH}, 'value'),
Input({'type': 'param-reset', 'index': MATCH}, 'n_clicks'), ],
[State({'type': 'param-slider', 'index': MATCH}, 'id'),
State('scale-slider', 'value'),
State({'type': 'param-slider', 'index': MATCH}, 'marks'), ]
)
def update_dyn_slider_range(text_div, log_act, dyn_min, dyn_max, slider_val, center, span, scale, dig, reset, id, scale_slider, marks):
ctx = dash.callback_context
try:
trigger_id = ctx.triggered[0]["prop_id"].split('}')[0].split(',')[1].split(':')[1]
except IndexError:
trigger_id = ctx.triggered[0]["prop_id"]
mark_lim = [float(i) for i in list(marks.keys())]
data_in = indata[invars[id['index']]]
data_min_0 = min(data_in[data_in > 0])
if trigger_id == '"param-log"' and log_act != ['log']:
dyn_min = 10**dyn_min
dyn_max = 10**dyn_max
slider_val = [10**val for val in slider_val]
mark_lim = [10 ** lim for lim in mark_lim]
if min(data_in) < 0 and slider_val[0] > 0:
mark_lim[0] = min(data_in)
span = (slider_val[1] - slider_val[0])/2
center = (slider_val[0] + slider_val[1])/2
if trigger_id != '"param-log"' and log_act == ['log']:
dyn_min = 10 ** dyn_min
dyn_max = 10 ** dyn_max
slider_val = [10 ** val for val in slider_val]
mark_lim = [10 ** lim for lim in mark_lim]
if min(data_in) < 0 and slider_val[0] > 0:
mark_lim[0] = min(data_in)
if trigger_id == '"param-reset"':
slider_val = [min(data_in), max(data_in)]
if ctx.triggered[0]["prop_id"] == "scale.n_clicks":
# print('scale')
span = span * (1 + scale_slider)
if log_act == ['log']:
trigger_id = '"param-span"'
else:
dyn_min = center - span
dyn_max = center + span
slider_val = [dyn_min, dyn_max]
if trigger_id == '"param-center"' or trigger_id == '"param-span"' and (center and span):
# print('center')
if log_act == ['log']:
dyn_min = 10**(center-span)
dyn_max = 10**(center+span)
else:
dyn_min = center - span
dyn_max = center + span
slider_val = [dyn_min, dyn_max]
elif (trigger_id == '"param-range-min"' or trigger_id == '"param-range-max"') and (
dyn_min is not None and dyn_max is not None):
# print('range')
slider_val = [dyn_min, dyn_max]
span = (slider_val[1] - slider_val[0]) / 2
center = (slider_val[0] + slider_val[1]) / 2
elif slider_val:
# print('slider')
dyn_min = slider_val[0]
dyn_max = slider_val[1]
span = (slider_val[1] - slider_val[0]) / 2
center = (slider_val[0] + slider_val[1]) / 2
if log_act == ['log']:
# log values
try:
log_dyn_min = log10(dyn_min)
except ValueError:
log_dyn_min = log10(data_min_0)
log_dyn_max = log10(dyn_max)
log_slider_val = [log_dyn_min, log_dyn_max]
try:
log_mark_lim = [log10(mark) for mark in mark_lim]
except ValueError:
log_mark_lim = [log10(data_min_0), log10(mark_lim[1])]
log_span = (log_slider_val[1] - log_slider_val[0])/2
log_center = log_slider_val[0] + log_span
log_marks = {log_mark_lim[0]: str(round(log_mark_lim[0], dig)),
log_mark_lim[1]: str(round(log_mark_lim[1], dig))}
return round(log_dyn_min, dig), round(log_dyn_max, dig), log_slider_val, log_mark_lim[0], log_mark_lim[1],\
round(log_center, dig), round(log_span, dig), log_marks
else:
marks = {mark_lim[0]: str(round(mark_lim[0], dig)), mark_lim[1]: str(round(mark_lim[1], dig))}
return round(dyn_min, dig), round(dyn_max, dig), slider_val, mark_lim[0], mark_lim[1], \
round(center, dig), round(span, dig), marks
@app.callback(
[Output('invar-2-div', 'style'),
Output('invar-3-div', 'style'),
Output('color-div', 'style'),
Output('error-div', 'style'),
Output('fit-use-div', 'style'),
Output('fit-multiinput-div', 'style'),
Output('fit-number-div', 'style'),
Output('fit-number', 'value'),
Output('fit-conf-div', 'style'),
Output('fit-noise-div', 'style'),
Output('fit-color-div', 'style'),
Output('fit-opacity-div', 'style'), ],
[Input('graph-type', 'value'), ],
[State('fit-number', 'value'), ]
)
def div_visibility(graph_type, fits):
hide = axis_options_div_style.copy()
hide['visibility'] = 'hidden'
show = axis_options_div_style.copy()
show['visibility'] = 'visible'
if graph_type == '1D':
return hide, hide, show, show, show, show, show, fits, show, show, hide, show
if graph_type == '2D':
if len(invars) <= 2:
return show, hide, show, show, show, hide, hide, 1, show, show, show, show
else:
return show, hide, show, show, show, show, show, fits, show, show, show, show
if graph_type == '2D contour':
return show, hide, show, hide, hide, hide, hide, fits, hide, hide, hide, hide
if graph_type == '3D':
return show, show, hide, hide, show, hide, show, fits, hide, hide, hide, show
else:
return show, show, show, show, show, show, show, fits, show, show, show, show
@app.callback(
Output('graph1', 'figure'),
[Input('invar', 'value'),
Input('invar_2', 'value'),
Input('invar_3', 'value'),
Input('outvar', 'value'),
Input('invar-1-log', 'value'),
Input('invar-2-log', 'value'),
Input('invar-3-log', 'value'),
Input('outvar-log', 'value'),
Input({'type': 'param-slider', 'index': ALL}, 'value'),
Input('graph-type', 'value'),
Input('color-use', 'value'),
Input('color-dropdown', 'value'),
Input('error-use', 'value'),
Input('error-dropdown', 'value'),
Input({'type': 'param-active', 'index': ALL}, 'value'),
Input('fit-use', 'value'),
Input('fit-multiinput-dropdown', 'value'),
Input('fit-number', 'value'),
Input('fit-conf', 'value'),
Input('fit-var', 'value'),
Input('fit-color', 'value'),
Input('fit-opacity', 'value'),
Input('fit-sampling', 'value'), ],
[State({'type': 'param-slider', 'index': ALL}, 'id'),
State({'type': 'param-center', 'index': ALL}, 'value'),
State({'type': 'param-log', 'index': ALL}, 'value')],
)
def update_figure(invar, invar_2, invar_3, outvar, invar1_log, invar2_log, invar3_log, outvar_log, param_slider,
graph_type, color_use, color_dd, error_use, error_dd, filter_active, fit_use, fit_dd, fit_num, fit_conf, add_noise_var, fit_color,
fit_opacity, fit_sampling, id_type, param_center, param_log):
for i in range(len(param_slider)):
if param_log[i] == ['log']:
param_slider[i] = [10**val for val in param_slider[i]]
param_center[i] = 10**param_center[i]
if invar is None:
return go.Figure()
sel_y = np.full((len(outdata),), True)
dds_value = []
for iteration, values in enumerate(param_slider):
dds_value.append(invars[id_type[iteration]['index']])
# filter for minimum
sel_y_min = np.array(indata[dds_value[iteration]] >= param_slider[iteration][0])
# filter for maximum
sel_y_max = np.array(indata[dds_value[iteration]] <= param_slider[iteration][1])
# print('iter ', iteration, 'filer', filter_active[iteration][0])
if filter_active != [[]]:
if filter_active[iteration] == ['act']:
sel_y = sel_y_min & sel_y_max & sel_y
if graph_type == '1D':
fig = go.Figure(
data=[go.Scatter(
x=indata[invar][sel_y],
y=outdata[outvar][sel_y],
mode='markers',
name='data',
error_y=dict(type='data', array=outdata[error_dd][sel_y], visible= error_use == ['true']),
# text=[(invar, outvar) for i in range(len(indata[invar][sel_y]))],
# hovertemplate=" %{text} <br> %{x} <br> %{y}",
)],
layout=go.Layout(xaxis=dict(title=invar, rangeslider=dict(visible=True)), yaxis=dict(title=outvar))
)
if fit_use == ['show']:
mesh_in, mesh_out, mesh_out_std, fit_dd_values = mesh_fit(param_slider, id_type, fit_dd, fit_num,
param_center, [invar], [invar1_log],
outvar, fit_sampling, add_noise_var)
for i in range(len(fit_dd_values)):
fig.add_trace(go.Scatter(
x=mesh_in[i][invars.index(invar)],
y=mesh_out[i],
mode='lines',
name=f'fit: {fit_dd}={fit_dd_values[i]:.1e}',
line_color=colormap(indata[fit_dd].min(), indata[fit_dd].max(), fit_dd_values[i]),
marker_line=dict(coloraxis="coloraxis2"),
))
fig.add_trace(go.Scatter(
x=np.hstack((mesh_in[i][invars.index(invar)], mesh_in[i][invars.index(invar)][::-1])),
y=np.hstack((mesh_out[i] + fit_conf * mesh_out_std[i], mesh_out[i][::-1] - fit_conf * mesh_out_std[i][::-1])),
showlegend=False,
fill='toself',
line_color=colormap(indata[fit_dd].min(), indata[fit_dd].max(), fit_dd_values[i]),
marker_line=dict(coloraxis="coloraxis2"),
opacity=fit_opacity,
))
elif graph_type == '2D':
fig = go.Figure(
data=[go.Scatter3d(
x=indata[invar][sel_y],
y=indata[invar_2][sel_y],
z=outdata[outvar][sel_y],
mode='markers',
name='Data',
error_z=dict(type='data', array=outdata[error_dd][sel_y], visible=error_use == ['true'], width= 10)
)],
layout=go.Layout(scene=dict(xaxis_title=invar, yaxis_title=invar_2, zaxis_title=outvar))
)
if fit_use == ['show'] and invar != invar_2:
mesh_in, mesh_out, mesh_out_std, fit_dd_values = mesh_fit(param_slider, id_type, fit_dd, fit_num,
param_center, [invar, invar_2],
[invar1_log, invar2_log], outvar,
fit_sampling, add_noise_var)
for i in range(len(fit_dd_values)):
fig.add_trace(go.Surface(
x=mesh_in[i][invars.index(invar)].reshape((fit_sampling, fit_sampling)),
y=mesh_in[i][invars.index(invar_2)].reshape((fit_sampling, fit_sampling)),
z=mesh_out[i].reshape((fit_sampling, fit_sampling)),
name=f'fit: {fit_dd}={fit_dd_values[i]:.2f}',
surfacecolor=fit_dd_values[i] * np.ones([fit_sampling, fit_sampling])
if fit_color == 'multi-fit' else
(mesh_in[i][invars.index(color_dd)].reshape((fit_sampling, fit_sampling))
if (fit_color == 'marker-color' and color_dd in invars) else
mesh_out[i].reshape((fit_sampling, fit_sampling))),
opacity=fit_opacity,
coloraxis="coloraxis2" if (fit_color == 'multi-fit' or
(fit_color == 'output' and (color_dd != outvar and color_dd != 'OUTPUT'))) else "coloraxis",
showlegend=True if len(invars) > 2 else False,
))
if fit_conf > 0:
fig.add_trace(go.Surface(
x=mesh_in[i][invars.index(invar)].reshape((fit_sampling, fit_sampling)),
y=mesh_in[i][invars.index(invar_2)].reshape((fit_sampling, fit_sampling)),
z=mesh_out[i].reshape((fit_sampling, fit_sampling)) + fit_conf * mesh_out_std[i].reshape((fit_sampling, fit_sampling)),
showlegend=False,
name=f'fit+v: {fit_dd}={fit_dd_values[i]:.2f}',
surfacecolor=fit_dd_values[i] * np.ones([fit_sampling, fit_sampling])
if fit_color == 'multi-fit' else
(mesh_in[i][invars.index(color_dd)].reshape((fit_sampling, fit_sampling))
if (fit_color == 'marker-color' and color_dd in invars) else
mesh_out[i].reshape((fit_sampling, fit_sampling))),
opacity=fit_opacity,
coloraxis="coloraxis2" if (fit_color == 'multi-fit' or
(fit_color == 'output' and (color_dd != outvar and color_dd != 'OUTPUT')))
else "coloraxis",
))
fig.add_trace(go.Surface(
x=mesh_in[i][invars.index(invar)].reshape((fit_sampling, fit_sampling)),
y=mesh_in[i][invars.index(invar_2)].reshape((fit_sampling, fit_sampling)),
z=mesh_out[i].reshape((fit_sampling, fit_sampling)) - fit_conf * mesh_out_std[i].reshape((fit_sampling, fit_sampling)),
showlegend=False,
name=f'fit-v: {fit_dd}={fit_dd_values[i]:.2f}',
surfacecolor=fit_dd_values[i] * np.ones([fit_sampling, fit_sampling])
if fit_color == 'multi-fit' else
(mesh_in[i][invars.index(color_dd)].reshape((fit_sampling, fit_sampling))
if (fit_color == 'marker-color' and color_dd in invars) else
mesh_out[i].reshape((fit_sampling, fit_sampling))),
opacity=fit_opacity,
coloraxis="coloraxis2" if (fit_color == 'multi-fit' or
(fit_color == 'output' and (color_dd != outvar and color_dd != 'OUTPUT')))
else "coloraxis",
))
fig.update_layout(coloraxis2=dict(
colorbar=dict(title=outvar if fit_color == 'output' else fit_dd),
cmin=min(fit_dd_values) if fit_color == 'multi-fit' else None,
cmax=max(fit_dd_values) if fit_color == 'multi-fit' else None,
))
elif graph_type == '2D contour':
mesh_in, mesh_out, mesh_out_std, fit_dd_values = mesh_fit(param_slider, id_type, fit_dd, fit_num,
param_center, [invar, invar_2],
[invar1_log, invar2_log], outvar,
fit_sampling, add_noise_var)
data_x = mesh_in[0][invars.index(invar)]
data_y = mesh_in[0][invars.index(invar_2)]
fig = go.Figure()
if min(data_x) != max(data_x):
if min(data_y) != max(data_y):
fig.add_trace(go.Scatter(
x=indata[invar][sel_y],
y=indata[invar_2][sel_y],
mode='markers',
name='Data',
))
fig.add_trace(go.Contour(
x=mesh_in[0][invars.index(invar)],
y=mesh_in[0][invars.index(invar_2)],
z=mesh_out[0],
contours_coloring='heatmap',
contours_showlabels=True,
coloraxis='coloraxis2',
name='fit',
))
fig.update_xaxes(
range=[log10(min(fig.data[1]['x'])), log10(max(fig.data[1]['x']))] if invar1_log == ['log']
else [min(fig.data[1]['x']), max(fig.data[1]['x'])])
fig.update_yaxes(
range=[log10(min(fig.data[1]['y'])), log10(max(fig.data[1]['y']))] if invar2_log == ['log']
else [min(fig.data[1]['y']), max(fig.data[1]['y'])])
fig.update_layout(xaxis_title=invar,
yaxis_title=invar_2,
coloraxis2=dict(colorbar=dict(title=outvar),
colorscale='solar',
cmin=min(fig.data[1]['z']),
cmax=max(fig.data[1]['z'])))
else:
fig.update_layout(title="y-data is constant, no contour-plot possible")
else:
fig.update_layout(title="x-data is constant, no contour-plot possible")
elif graph_type == '3D':
fig = go.Figure(
data=go.Scatter3d(
x=indata[invar][sel_y],
y=indata[invar_2][sel_y],
z=indata[invar_3][sel_y],
mode='markers',
marker=dict(
color=outdata[outvar][sel_y],
coloraxis="coloraxis2",
),
name='Data',
),
layout=go.Layout(scene=dict(xaxis_title=invar, yaxis_title=invar_2, zaxis_title=invar_3)),
)
fig.update_layout(coloraxis2=dict(
colorbar=dict(title=outvar),
))
if fit_use == ['show'] and len({invar, invar_2, invar_3}) == 3:
mesh_in, mesh_out, mesh_out_std, fit_dd_values = mesh_fit(param_slider, id_type, fit_dd, fit_num,
param_center, [invar, invar_2, invar_3],
[invar1_log, invar2_log, invar3_log], outvar,
fit_sampling, add_noise_var)
for i in range(len(fit_dd_values)):
fig.add_trace(
go.Isosurface(
x=mesh_in[i][invars.index(invar)],
y=mesh_in[i][invars.index(invar_2)],
z=mesh_in[i][invars.index(invar_3)],
value=mesh_out[i],
surface_count=fit_num,
coloraxis="coloraxis2",
isomin=mesh_out[i].min() * 1.1,
isomax=mesh_out[i].max() * 0.9,
caps=dict(x_show=False, y_show=False, z_show=False),
opacity=fit_opacity,
),
)
else:
fig = go.Figure()
fig.update_layout(legend=dict(xanchor="left", x=0.01))
# log scale
log_dict = {'1D': (invar1_log, outvar_log),
'2D': (invar1_log, invar2_log, outvar_log),
'2D contour': (invar1_log, invar2_log),
'3D': (invar1_log, invar2_log, invar3_log),}
log_list = ['linear' if log is None or len(log) == 0 else log[0] for log in log_dict[graph_type]]
log_key = ['xaxis', 'yaxis', 'zaxis']
comb_dict = dict(zip(log_key, [{'type': log} for log in log_list]))
if len(log_list) < 3 :
fig.update_layout(**comb_dict)
else:
fig.update_scenes(**comb_dict)
# color
if color_use == ['true']:
if fit_use == ['show'] and (graph_type=='2D' and (fit_color=='multi-fit' and color_dd==fit_dd)):
fig.update_traces(
marker=dict(
coloraxis="coloraxis2",
color=indata[color_dd][sel_y] if color_dd in indata.dtype.names else outdata[color_dd][sel_y],
),
selector=dict(mode='markers'),
)
elif graph_type == '3D':
fig.update_traces(
marker=dict(
coloraxis="coloraxis2",
color=outdata[outvar][sel_y],
),
selector=dict(mode='markers'),
)
elif graph_type=='1D':
fig.update_traces(
marker=dict(
coloraxis="coloraxis2",
color=outdata[outvar][sel_y] if color_dd == 'OUTPUT' else
(indata[color_dd][sel_y] if color_dd in indata.dtype.names else outdata[color_dd][sel_y]),
),
selector=dict(mode='markers'),
)
if color_dd==fit_dd:
fig.update_layout(coloraxis2=dict(colorscale='cividis', colorbar=dict(title=fit_dd)))
elif color_dd == 'OUTPUT':
fig.update_layout(coloraxis2=dict(colorscale='plasma', colorbar=dict(title=outvar)))
else:
fig.update_layout(coloraxis2=dict(colorscale='plasma', colorbar=dict(title=color_dd)))
elif graph_type =='2D contour':
fig.update_traces(
marker=dict(
coloraxis="coloraxis",
color=outdata[outvar][sel_y] if color_dd == 'OUTPUT' else
(indata[color_dd][sel_y] if color_dd in indata.dtype.names else outdata[color_dd][sel_y]),
),
selector=dict(mode='markers'),
)
if color_dd == outvar or color_dd == 'OUTPUT':
fig.update_traces(marker_coloraxis="coloraxis2", selector=dict(mode='markers'))
else:
fig.update_layout(coloraxis=dict(colorbar=dict(title=color_dd, x=1.1),
colorscale='ice'))
else:
fig.update_traces(
marker=dict(
coloraxis="coloraxis",
color=outdata[outvar][sel_y] if color_dd == 'OUTPUT' else
(indata[color_dd][sel_y] if color_dd in indata.dtype.names else outdata[color_dd][sel_y]),
),
selector=dict(mode='markers'),
)
fig.update_layout(coloraxis=dict(
colorbar=dict(title=outvar if color_dd == 'OUTPUT' else color_dd, x=1.1),
colorscale='viridis',
))
fig.update_layout(height=graph_height)
return fig
def mesh_fit(param_slider, id_type, fit_dd, fit_num, param_center, invar_list, invar_log_list, outvar, num_samples, add_noise_var):
try: # collecting min/max of slider for variable of multifit
fit_dd_min, fit_dd_max = param_slider[[i['index'] for i in id_type].index(invars.index(fit_dd))]
except ValueError:
fit_dd_min = min(indata[fit_dd])
fit_dd_max = max(indata[fit_dd])
if fit_num == 1: # generate list of value of variable of multifit
fit_dd_values = np.array([(fit_dd_max + fit_dd_min) / 2])
else:
fit_dd_values = np.linspace(fit_dd_min, fit_dd_max, fit_num)
for iteration, fit_dd_value in enumerate(fit_dd_values): # iteration for each fit
# set fit parameter for all invars as center of range
fit_params = [(max(indata[var_invar]) + min(indata[var_invar])) / 2 for var_invar in invars]
# for all invars with filter change fit_param to center defined by filter
flt_ind_list = [] # list of filter indices
for i, center_values in enumerate(param_center):
flt_ind_list.append(id_type[i]['index'])
fit_params[flt_ind_list[i]] = center_values
# change param of fit-variable
fit_params[invars.index(fit_dd)] = fit_dd_value
# change param for axis invars
for i, ax_in in enumerate(invar_list):
if invars.index(ax_in) in flt_ind_list:
ax_min, ax_max = param_slider[flt_ind_list.index(invars.index(ax_in))]
else:
ax_min = min(indata[ax_in])
ax_max = max(indata[ax_in])
if invar_log_list[i] == ['log']:
fit_params[invars.index(ax_in)] = np.logspace(log10(ax_min), log10(ax_max), num_samples)
else:
fit_params[invars.index(ax_in)] = np.linspace(ax_min, ax_max, num_samples)
grid = np.meshgrid(*fit_params) # generate grid
x_pred = np.vstack([g.flatten() for g in grid]).T # extract vector for predict
fit_data, fit_var = sur.predict(x_pred, add_noise_var == ['add']) # generate fit data and variance
# generated data
new_mesh_in = np.array([[grid[invars.index(invar)].flatten() for invar in invars]])
new_mesh_out = np.array([fit_data[:, outvars.index(outvar)]])
new_mesh_out_std = np.array([np.sqrt(fit_var[:, 0])])
# stack data together
if iteration == 0:
mesh_in = new_mesh_in
mesh_out = new_mesh_out
mesh_out_std = new_mesh_out_std
else:
mesh_in = np.vstack((mesh_in, new_mesh_in))
mesh_out = np.vstack((mesh_out, new_mesh_out))
mesh_out_std = np.vstack((mesh_out_std, new_mesh_out_std))
return mesh_in, mesh_out, mesh_out_std, fit_dd_values
def create_slider(dd_value):
ind = invars.index(dd_value)
slider_min = indata[dd_value].min()
slider_max = indata[dd_value].max()
step_exponent = -3
new_slider = dcc.RangeSlider(
id={'type': 'param-slider', 'index': ind},
step=10 ** step_exponent,
min=slider_min,
max=slider_max,
value=[slider_min, slider_max],
marks={slider_min: str(round(slider_min, -step_exponent)),
slider_max: str(round(slider_max, -step_exponent))},
)
return new_slider
def colormap(cmin, cmax, c):
if cmin == cmax:
c_scal = 0.5
else:
c_scal = (c-cmin)/(cmax-cmin)
return color2hex(colormaps.cividis(c_scal))
return app
def main():
"""
Main command line interface
sys.argv is an array whose values are the entered series of command
(e.g.: sys.argv=['profit','run', '--active-learning', '/home/user/example'])
"""
""" Get parameters from argv """
parser = ArgumentParser(
usage='profit <mode> (base-dir)',
description=
"Probabilistic Response Model Fitting with Interactive Tools",
formatter_class=RawTextHelpFormatter)
parser.add_argument(
'mode', # ToDo: subparsers?
metavar='mode',
choices=['run', 'fit', 'ui', 'clean'],
help='run ... start simulation runs \n'
'fit ... fit data with Gaussian Process \n'
'ui ... visualise results \n'
'clean ... remove run directories and input/output files')
parser.add_argument(
'base_dir',
metavar='base-dir',
help='path to config file (default: current working directory)',
default=getcwd(),
nargs='?')
args = parser.parse_args()
print(args)
""" Instantiate Config class from the given file """
config_file = safe_path_to_file(args.base_dir, default='profit.yaml')
config = Config.from_file(config_file)
sys.path.append(config['base_dir'])
if args.mode == 'run':
from tqdm import tqdm
from profit.pre import get_eval_points, write_input
from profit.util import save
runner = Runner.from_config(config['run'], config)
eval_points = get_eval_points(config)
write_input(config['files']['input'], eval_points)
if 'activelearning' in (safe_str(v['kind'])
for v in config['input'].values()):
from profit.fit import ActiveLearning
from profit.sur.sur import Surrogate
runner.fill(eval_points)
if 'active_learning' not in config:
config['active_learning'] = {}
ActiveLearning.handle_config(config['active_learning'], config)
al = ActiveLearning.from_config(runner, config['active_learning'],
config)
al.run_first()
al.learn()
if config['active_learning'].get('save'):
al.save(config['active_learning']['save'])
else:
params_array = [row[0] for row in eval_points]
runner.spawn_array(tqdm(params_array), blocking=True)
if config['run']['clean']:
runner.clean()
if config['files']['output'].endswith('.txt'):
data = runner.structured_output_data
save(config['files']['output'], data.reshape(data.size, 1))
else:
save(config['files']['output'], runner.output_data)
elif args.mode == 'fit':
from numpy import arange, hstack, meshgrid
from profit.util import load
from profit.sur.sur import Surrogate
sur = Surrogate.from_config(config['fit'], config)
if not sur.trained:
x = load(config['files']['input'])
y = load(config['files']['output'])
x = hstack([x[key] for key in x.dtype.names])
y = hstack([y[key] for key in y.dtype.names])
sur.train(x, y)
if config['fit'].get('save'):
sur.save_model(config['fit']['save'])
if config['fit'].get('plot'):
try:
xpred = [
arange(minv, maxv, step)
for minv, maxv, step in config['fit']['plot'].get('xpred')
]
xpred = hstack(
[xi.flatten().reshape(-1, 1) for xi in meshgrid(*xpred)])
except AttributeError:
xpred = None
sur.plot(xpred, independent=config['independent'], show=True)
elif args.mode == 'ui':
from profit.ui import init_app
app = init_app(config)
app.run_server(debug=True)
elif args.mode == 'clean':
from shutil import rmtree
from os import path, remove
run_dir = config['run_dir']
question = "Are you sure you want to remove the run directories in {} " \
"and input/output files? (y/N) ".format(config['run_dir'])
if yes:
print(question + 'y')
else:
answer = input(question)
if not answer.lower().startswith('y'):
print('exit...')
sys.exit()
for krun in range(config['ntrain']):
single_run_dir = path.join(run_dir, f'run_{krun:03d}')
if path.exists(single_run_dir):
rmtree(single_run_dir)
if path.exists(config['files']['input']):
remove(config['files']['input'])
if path.exists(config['files']['output']):
remove(config['files']['output'])
runner = Runner.from_config(config['run'], config)
runner.clean()
try:
rmtree(config['run']['log_path'])
except FileNotFoundError:
pass
