def _extract_summary_from_model_scores(report, extracted):
n = ('data', 'calibration', 'model_scores', 'model_selections')
if not dsp_utl.are_in_nested_dicts(report, *n):
return False
sel = dsp_utl.get_nested_dicts(report, *n)
for k, v in dsp_utl.stack_nested_keys(extracted, depth=3):
n = k[1::-1]
if k[-1] == 'output' and dsp_utl.are_in_nested_dicts(sel, *n):
gen = dsp_utl.get_nested_dicts(sel, *n)
gen = ((d['model_id'], d['status']) for d in gen if 'status' in d)
o = _format_dict(gen, 'status %s')
v.update(o)
return True
def _scores2df(data):
n = ('data', 'calibration', 'model_scores')
if not dsp_utl.are_in_nested_dicts(data, *n):
return {}
scores = dsp_utl.get_nested_dicts(data, *n)
it = (('model_selections', ['model_id'], 2, ('stage', 'cycle'),
()), ('score_by_model', ['model_id'], 1, ('cycle', ), ()),
('scores', ['model_id', 'param_id'], 2, ('cycle', 'cycle'),
()), ('param_selections', ['param_id'], 2, ('stage', 'cycle'),
()), ('models_uuid', ['cycle'], 0, (), ('cycle', )))
dfs = []
for k, idx, depth, col_keys, row_keys in it:
if k not in scores:
continue
df = _dd2df(scores[k],
idx,
depth=depth,
col_key=functools.partial(_sort_key, p_keys=col_keys),
row_key=functools.partial(_sort_key, p_keys=row_keys))
setattr(df, 'name', k)
dfs.append(df)
if dfs:
return {'.'.join(n): dfs}
else:
return {}
def _process_folder_files(*args, result_listener=None, **kwargs):
"""
Process all xls-files in a folder with CO2MPAS-model.
:param list input_files:
A list of input xl-files.
:param output_folder:
Output folder.
:type output_folder: str
:param plot_workflow:
If to show the CO2MPAS model workflow.
:type plot_workflow: bool, optional
:param output_template:
The xlsx-file to use as template and import existing sheets from.
- If file already exists, a clone gets updated with new sheets.
- If it is None, it copies and uses the input-file as template.
- if it is `False`, it does not use any template and a fresh output
xlsx-file is created.
:type output_folder: None,False,str
"""
start_time = datetime.datetime.today()
summary, n = {}, ('solution', 'summary')
for res in _yield_folder_files_results(start_time, *args, **kwargs):
if dsp_utl.are_in_nested_dicts(res, *n):
_add2summary(summary, dsp_utl.get_nested_dicts(res, *n))
notify_result_listener(result_listener, res)
return summary, start_time
def compare_outputs_vs_targets(data):
"""
Compares model outputs vs targets.
:param data:
Model data.
:type data: dict
:return:
Comparison results.
:rtype: dict
"""
res = {}
metrics = _get_metrics()
for k, t in dsp_utl.stack_nested_keys(data.get('target', {}), depth=3):
if not dsp_utl.are_in_nested_dicts(data, 'output', *k):
continue
o = dsp_utl.get_nested_dicts(data, 'output', *k)
v = _compare(t, o, metrics=metrics)
if v:
dsp_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def _add_times_base(data, scope='base', usage='input', **match):
if scope != 'base':
return
sh_type = _get_sheet_type(scope=scope, usage=usage, **match)
n = (scope, 'target')
if sh_type == 'ts' and dsp_utl.are_in_nested_dicts(data, *n):
t = dsp_utl.get_nested_dicts(data, *n)
for k, v in dsp_utl.stack_nested_keys(t, key=n, depth=2):
if 'times' not in v:
n = list(k + ('times', ))
n[1] = usage
if dsp_utl.are_in_nested_dicts(data, *n):
v['times'] = dsp_utl.get_nested_dicts(data, *n)
else:
for i, j in dsp_utl.stack_nested_keys(data, depth=4):
if 'times' in j:
v['times'] = j['times']
break
def _add_special_data2report(data, report, to_keys, *from_keys):
if from_keys[-1] != 'times' and \
dsp_utl.are_in_nested_dicts(data, *from_keys):
v = dsp_utl.get_nested_dicts(data, *from_keys)
n = to_keys + ('{}.{}'.format(from_keys[0], from_keys[-1]),)
dsp_utl.get_nested_dicts(report, *n[:-1],
default=collections.OrderedDict)[n[-1]] = v
return True, v
return False, None
def _extract_summary_from_summary(report, extracted):
n = ('summary', 'results')
if dsp_utl.are_in_nested_dicts(report, *n):
for j, w in dsp_utl.get_nested_dicts(report, *n).items():
if j in ('declared_co2_emission', 'co2_emission',
'fuel_consumption'):
for k, v in dsp_utl.stack_nested_keys(w, depth=3):
if v:
dsp_utl.get_nested_dicts(extracted, *k).update(v)
def format_report_scores(data):
res = {}
scores = 'data', 'calibration', 'model_scores'
if dsp_utl.are_in_nested_dicts(data, *scores):
n = scores + ('param_selections',)
v = _format_selection(dsp_utl.get_nested_dicts(data, *n), 2, 'param_id')
if v:
dsp_utl.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
n = scores + ('model_selections',)
v = _format_selection(dsp_utl.get_nested_dicts(data, *n), 3)
if v:
dsp_utl.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
n = scores + ('score_by_model',)
v = _format_selection(dsp_utl.get_nested_dicts(data, *n), 2)
if v:
dsp_utl.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
n = scores + ('scores',)
v = _format_scores(dsp_utl.get_nested_dicts(data, *n))
if v:
dsp_utl.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
v = []
for k in ('nedc_h', 'nedc_l', 'wltp_h', 'wltp_l'):
n = 'data', 'prediction', 'models_%s' % k
if dsp_utl.are_in_nested_dicts(data, *n):
v.append({
'cycle': k,
'uuid': base64.encodebytes(
dill.dumps(dsp_utl.get_nested_dicts(data, *n))
)
})
if v:
n = scores + ('models_uuid',)
dsp_utl.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
return res
def select_declaration_data(data, diff=None):
res = {}
for k, v in dsp_utl.stack_nested_keys(constants.con_vals.DECLARATION_DATA):
if v and dsp_utl.are_in_nested_dicts(data, *k):
v = dsp_utl.get_nested_dicts(data, *k)
dsp_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
if diff is not None:
diff.clear()
diff.update(v[0] for v in dsp_utl.stack_nested_keys(data, depth=4))
it = (v[0] for v in dsp_utl.stack_nested_keys(res, depth=4))
diff.difference_update(it)
return res
def re_sample_targets(data):
res = {}
for k, v in dsp_utl.stack_nested_keys(data.get('target', {}), depth=2):
if dsp_utl.are_in_nested_dicts(data, 'output', *k):
o = dsp_utl.get_nested_dicts(data, 'output', *k)
o = _split_by_data_format(o)
t = dsp_utl.selector(o, _split_by_data_format(v), allow_miss=True)
if 'times' not in t.get('ts', {}) or 'times' not in o['ts']:
t.pop('ts', None)
else:
time_series = t['ts']
x, xp = o['ts']['times'], time_series.pop('times')
if not _is_equal(x, xp):
for i, fp in time_series.items():
time_series[i] = np.interp(x, xp, fp)
v = dsp_utl.combine_dicts(*t.values())
dsp_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def get_selection(data):
n = ('data', 'calibration', 'model_scores', 'model_selections')
if dsp_utl.are_in_nested_dicts(data, *n):
return _format_selection(dsp_utl.get_nested_dicts(data, *n), 3)
return {}
