def _extract_summary_from_model_scores(report, extracted):
n = ('data', 'calibration', 'model_scores', 'model_selections')
if not sh.are_in_nested_dicts(report, *n):
return False
sel = sh.get_nested_dicts(report, *n)
s = ('data', 'calibration', 'model_scores', 'score_by_model')
score = sh.get_nested_dicts(report, *s)
s = ('data', 'calibration', 'model_scores', 'scores')
scores = sh.get_nested_dicts(report, *s)
for k, v in sh.stack_nested_keys(extracted, depth=3):
n = k[1::-1]
if k[-1] == 'output' and sh.are_in_nested_dicts(sel, *n):
gen = sh.get_nested_dicts(sel, *n)
gen = ((d['model_id'], d['status']) for d in gen if 'status' in d)
o = _format_dict(gen, 'status %s')
v.update(o)
if k[1] == 'calibration' and k[0] in score:
gen = score[k[0]]
gen = ((d['model_id'], d['score']) for d in gen
if 'score' in d)
o = _format_dict(gen, 'score %s')
v.update(o)
for i, j in scores[k[0]].items():
gen = (('/'.join(
(d['model_id'], d['param_id'])), d['score']) for d in j
if 'score' in d)
o = _format_dict(gen, 'score {}/%s'.format(i))
v.update(o)
return True
def extract_dice_report(encrypt_inputs, vehicle_family_id, start_time, report):
from co2mpas import version
res = {
'info': {
'encrypt_inputs': encrypt_inputs,
'vehicle_family_id': vehicle_family_id,
'CO2MPAS_version': version,
'datetime': start_time.strftime('%Y/%m/%d-%H:%M:%S')
}
}
# deviation
keys = 'summary', 'comparison', 'prediction'
if sh.are_in_nested_dicts(report, *keys):
deviation = 'declared_co2_emission_value', 'prediction_target_ratio'
for cycle, d in sh.get_nested_dicts(report, *keys).items():
if sh.are_in_nested_dicts(d, *deviation):
v = (sh.get_nested_dicts(d, *deviation) - 1) * 100
sh.get_nested_dicts(res, 'deviation')[cycle] = v
# vehicle
keys = [('summary', 'results', 'vehicle'), ('prediction', 'output')]
vehicle = 'fuel_type', 'engine_capacity', 'gear_box_type', 'engine_is_turbo'
if sh.are_in_nested_dicts(report, *keys[0]):
for cycle, d in sh.get_nested_dicts(report, *keys[0]).items():
if sh.are_in_nested_dicts(d, *keys[1]):
v = sh.selector(vehicle,
sh.get_nested_dicts(d, *keys[1]),
allow_miss=True)
if v:
sh.get_nested_dicts(res, 'vehicle', cycle).update(v)
# model scores
keys = 'data', 'calibration', 'model_scores'
model_scores = 'model_selections', 'param_selections', 'score_by_model', \
'scores'
if sh.are_in_nested_dicts(report, *keys):
sh.get_nested_dicts(res, 'model_scores').update(
sh.selector(model_scores,
sh.get_nested_dicts(report, *keys),
allow_miss=True))
res = copy.deepcopy(res)
for k, v in list(stack(res)):
if isinstance(v, np.generic):
sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v.item()
return res
def _format_report_scores(data):
res = {}
scores = 'data', 'calibration', 'model_scores'
if sh.are_in_nested_dicts(data, *scores):
n = scores + ('param_selections', )
v = _format_selection(sh.get_nested_dicts(data, *n), 2, 'param_id')
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
n = scores + ('model_selections', )
v = _format_selection(sh.get_nested_dicts(data, *n), 3)
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
n = scores + ('score_by_model', )
v = _format_selection(sh.get_nested_dicts(data, *n), 2)
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
n = scores + ('scores', )
v = _format_scores(sh.get_nested_dicts(data, *n))
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
return res
def _ta_mode(data):
data = data.copy()
base, errors = _validate_base_with_schema(data.get('base', {}))
if _log_errors_msg(errors):
return False
data['base'], _, diff = _extract_declaration_data(base, {})
for k in ('plan', 'flag'):
diff.update((k, ) + tuple(j.split('.')) for j in data.get(k, {}))
diff -= {('flag', 'input_version'), ('flag', 'vehicle_family_id'),
('flag', 'encrypt_inputs'), ('flag', 'encryption_keys')}
if diff:
diff = ['.'.join(k) for k in sorted(diff)]
log.info(
'Since CO2MPAS is launched in type approval mode the '
'following data cannot be used:\n %s\n'
'If you want to include these data use the cmd batch.',
',\n'.join(diff))
return False
if not sh.are_in_nested_dicts(data, 'flag', 'vehicle_family_id') \
or not data['flag']['vehicle_family_id']:
log.info('Since CO2MPAS is launched in type approval mode the '
'`vehicle_family_id` is required!\n'
'If you want to run without it use the cmd batch.')
return False
return True
def _scores2df(data):
n = ('data', 'calibration', 'model_scores')
if not sh.are_in_nested_dicts(data, *n):
return {}
scores = sh.get_nested_dicts(data, *n)
it = (('model_selections', ['model_id'], 2, ('stage', 'cycle'), ()),
('score_by_model', ['model_id'], 1, ('cycle',), ()),
('scores', ['model_id', 'param_id'], 2, ('cycle', 'cycle'), ()),
('param_selections', ['param_id'], 2, ('stage', 'cycle'), ()),
('models_uuid', ['cycle'], 0, (), ('cycle',)))
dfs = []
for k, idx, depth, col_keys, row_keys in it:
if k not in scores:
continue
df = _dd2df(
scores[k], idx, depth=depth,
col_key=functools.partial(_sort_key, p_keys=col_keys),
row_key=functools.partial(_sort_key, p_keys=row_keys)
)
setattr(df, 'name', k)
dfs.append(df)
if dfs:
return {'.'.join(n): dfs}
else:
return {}
def compare_outputs_vs_targets(data):
"""
Compares model outputs vs targets.
:param data:
Model data.
:type data: dict
:return:
Comparison results.
:rtype: dict
"""
res = {}
metrics = _get_metrics()
for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3):
if not sh.are_in_nested_dicts(data, 'output', *k):
continue
o = sh.get_nested_dicts(data, 'output', *k)
v = _compare(t, o, metrics=metrics)
if v:
sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def notify_result_listener(result_listener, res, out_fpath=None):
"""Utility func to send to the listener the output-file discovered from the results."""
are_in = sh.are_in_nested_dicts
if result_listener:
if not out_fpath:
it = []
for k in ('output_file_name', 'output_ta_file'):
if sh.are_in_nested_dicts(res, 'solution', k):
it.append(res['solution'][k])
else:
it = sh.stlp(out_fpath)
try:
for fpath in it:
result_listener((fpath, res))
except Exception as ex:
try:
keys = list(res)
except Exception:
keys = '<no keys>'
log.warning(
"Failed notifying result-listener due to: %s\n result-keys: %s",
ex,
keys,
exc_info=1)
def _process_folder_files(*args, result_listener=None, **kwargs):
"""
Process all xls-files in a folder with CO2MPAS-model.
:param list input_files:
A list of input xl-files.
:param output_folder:
Output folder.
:type output_folder: str
:param plot_workflow:
If to show the CO2MPAS model workflow.
:type plot_workflow: bool, optional
:param output_template:
The xlsx-file to use as template and import existing sheets from.
- If file already exists, a clone gets updated with new sheets.
- If it is None, it copies and uses the input-file as template.
- if it is `False`, it does not use any template and a fresh output
xlsx-file is created.
:type output_folder: None,False,str
"""
start_time = datetime.datetime.today()
summary, n = {}, ('solution', 'summary')
for res in _yield_folder_files_results(start_time, *args, **kwargs):
if sh.are_in_nested_dicts(res, *n):
_add2summary(summary, sh.get_nested_dicts(res, *n))
notify_result_listener(result_listener, res)
return summary, start_time
def _add_special_data2report(data, report, to_keys, *from_keys):
if from_keys[-1] != 'times' and \
sh.are_in_nested_dicts(data, *from_keys):
v = sh.get_nested_dicts(data, *from_keys)
n = to_keys + ('{}.{}'.format(from_keys[0], from_keys[-1]), )
sh.get_nested_dicts(report, *n[:-1],
default=collections.OrderedDict)[n[-1]] = v
return True, v
return False, None
def _extract_summary_from_summary(report, extracted):
n = ('summary', 'results')
if sh.are_in_nested_dicts(report, *n):
for j, w in sh.get_nested_dicts(report, *n).items():
if j in ('declared_co2_emission', 'co2_emission',
'fuel_consumption'):
for k, v in sh.stack_nested_keys(w, depth=3):
if v:
sh.get_nested_dicts(extracted, *k).update(v)
def _add_times_base(data, scope='base', usage='input', **match):
if scope != 'base':
return
sh_type = _get_sheet_type(scope=scope, usage=usage, **match)
n = (scope, 'target')
if sh_type == 'ts' and sh.are_in_nested_dicts(data, *n):
t = sh.get_nested_dicts(data, *n)
for k, v in sh.stack_nested_keys(t, key=n, depth=2):
if 'times' not in v:
n = list(k + ('times',))
n[1] = usage
if sh.are_in_nested_dicts(data, *n):
v['times'] = sh.get_nested_dicts(data, *n)
else:
for i, j in sh.stack_nested_keys(data, depth=4):
if 'times' in j:
v['times'] = j['times']
break
def format_report_scores(data):
res = {}
scores = 'data', 'calibration', 'model_scores'
if sh.are_in_nested_dicts(data, *scores):
n = scores + ('param_selections', )
v = _format_selection(sh.get_nested_dicts(data, *n), 2, 'param_id')
if v:
sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
n = scores + ('model_selections', )
v = _format_selection(sh.get_nested_dicts(data, *n), 3)
if v:
sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
n = scores + ('score_by_model', )
v = _format_selection(sh.get_nested_dicts(data, *n), 2)
if v:
sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
n = scores + ('scores', )
v = _format_scores(sh.get_nested_dicts(data, *n))
if v:
sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
v = []
for k in ('nedc_h', 'nedc_l', 'wltp_h', 'wltp_l'):
n = 'data', 'prediction', 'models_%s' % k
if sh.are_in_nested_dicts(data, *n):
v.append({
'cycle':
k,
'uuid':
base64.encodebytes(
dill.dumps(sh.get_nested_dicts(data, *n)))
})
if v:
n = scores + ('models_uuid', )
sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v))
return res
def select_declaration_data(data, diff=None):
res = {}
for k, v in sh.stack_nested_keys(constants.con_vals.DECLARATION_DATA):
if v and sh.are_in_nested_dicts(data, *k):
v = sh.get_nested_dicts(data, *k)
sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
if diff is not None:
diff.clear()
diff.update(v[0] for v in sh.stack_nested_keys(data, depth=4))
it = (v[0] for v in sh.stack_nested_keys(res, depth=4))
diff.difference_update(it)
return res
def _compare_outputs_vs_targets(data):
res = {}
metrics = _get_metrics()
for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3):
if not sh.are_in_nested_dicts(data, 'output', *k):
continue
o = sh.get_nested_dicts(data, 'output', *k)
v = _compare(t, o, metrics=metrics)
if v:
sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v
return res
def _extract_summary_from_summary(report, extracted, augmented_summary=False):
n = ('summary', 'results')
keys = (
'corrected_sustaining_co2_emission',
'declared_sustaining_co2_emission',
'declared_co2_emission',
'co2_emission',
'corrected_co2_emission',
)
if augmented_summary:
keys += 'fuel_consumption',
if sh.are_in_nested_dicts(report, *n):
for j, w in sh.get_nested_dicts(report, *n).items():
if j in keys:
for k, v in sh.stack_nested_keys(w, depth=3):
if v:
sh.get_nested_dicts(extracted, *k).update(v)
def re_sample_targets(data):
res = {}
for k, v in sh.stack_nested_keys(data.get('target', {}), depth=2):
if sh.are_in_nested_dicts(data, 'output', *k):
o = sh.get_nested_dicts(data, 'output', *k)
o = _split_by_data_format(o)
t = sh.selector(o, _split_by_data_format(v), allow_miss=True)
if 'times' not in t.get('ts', {}) or 'times' not in o['ts']:
t.pop('ts', None)
else:
time_series = t['ts']
x, xp = o['ts']['times'], time_series.pop('times')
if not _is_equal(x, xp):
for i, fp in time_series.items():
time_series[i] = np.interp(x, xp, fp)
v = sh.combine_dicts(*t.values())
sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def get_selection(data):
n = ('data', 'calibration', 'model_scores', 'model_selections')
if sh.are_in_nested_dicts(data, *n):
return _format_selection(sh.get_nested_dicts(data, *n), 3)
return {}
def test_are_in_nested_dicts(self):
d = {'a': {'b': {'c': ('d',)}}, 'A': {'B': {'C': ('D',)}}}
self.assertTrue(sh.are_in_nested_dicts(d, 'a', 'b', 'c'))
self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'C'))
self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'c', 'd'))
def test_are_in_nested_dicts(self):
d = {'a': {'b': {'c': ('d', )}}, 'A': {'B': {'C': ('D', )}}}
self.assertTrue(sh.are_in_nested_dicts(d, 'a', 'b', 'c'))
self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'C'))
self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'c', 'd'))
