def parse_dsp_solution(solution):
"""
Parses the co2mpas model results.
:param solution:
Co2mpas model after dispatching.
:type solution: schedula.Solution
:return:
Mapped outputs.
:rtype: dict[dict]
"""
res = {}
for k, v in solution.items():
sh.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v))
for k, v in list(sh.stack_nested_keys(res, depth=3)):
n, k = k[:-1], k[-1]
if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'):
v = sh.selector(('co2_emission_value', ), v, allow_miss=True)
if v:
d = sh.get_nested_dicts(res, 'target', 'prediction')
d[k] = sh.combine_dicts(v, d.get(k, {}))
res['pipe'] = solution.pipe
return res
def _extract_summary_from_output(report, extracted, augmented_summary=False):
for k, v in sh.stack_nested_keys(report.get('output', {}), depth=2):
k = k[::-1]
for u, i, j in _param_names_values(v.get('pa', {})):
o = {}
if i in ('has_sufficient_power', ):
o = {i: j}
elif augmented_summary:
if i == 'co2_params_calibrated':
o = _format_dict(j.valuesdict().items(), 'co2_params %s')
elif i == 'calibration_status':
o = _format_dict(enumerate(j), 'status co2_params step %d',
lambda x: x[0])
elif i == 'willans_factors':
o = j
elif i == 'phases_willans_factors':
for n, m in enumerate(j):
o.update(
_format_dict(m.items(), '%s phase {}'.format(n)))
elif i == 'co2_rescaling_scores':
o = sh.map_list(
['rescaling_mean', 'rescaling_std', 'rescaling_n'], *j)
if o:
sh.get_nested_dicts(extracted, *(k + (u, ))).update(o)
def test_get_nested_dicts(self):
d = {'a': {'b': {'c': ('d', )}}, 'A': {'B': {'C': ('D', )}}}
output = sh.get_nested_dicts(d, 'a', 'b', 'c')
result = ('d', )
self.assertEqual(output, result)
output = sh.get_nested_dicts(d, 0, default=list)
self.assertIsInstance(output, list)
self.assertTrue(0 in d)
import collections
output = sh.get_nested_dicts(d,
0,
init_nesting=collections.OrderedDict)
self.assertIsInstance(output, list)
output = sh.get_nested_dicts(d,
1,
init_nesting=collections.OrderedDict)
self.assertIsInstance(output, collections.OrderedDict)
self.assertTrue(1 in d)
output = sh.get_nested_dicts(d,
2,
3,
default=list,
init_nesting=collections.OrderedDict)
self.assertIsInstance(output, list)
self.assertTrue(2 in d)
self.assertIsInstance(d[2], collections.OrderedDict)
def _run_variations(plan, bases, core_model, timestamp):
for r in _ProgressBar(plan, _format_meter=_format_meter):
sol, data = bases[r['base']], r['data']
if 'solution' in sol:
s = sol['solution']
base = _define_inputs(s, sh.combine_nested_dicts(sh.selector(
data, s, allow_miss=True
), data))
elif 'base' in sol:
base = sh.combine_nested_dicts(sol['base'], data, depth=2)
else:
continue
for i, d in base.items():
if hasattr(d, 'items'):
base[i] = {k: v for k, v in d.items() if v is not sh.EMPTY}
sol = core_model(_define_inputs(sol, dict(
base=base,
vehicle_name='-'.join((str(r['id']), sol['vehicle_name'])),
timestamp=timestamp
)))
summary, keys = {}, {
tuple(k.split('.')[:0:-1]) for k in base if k.startswith('output.')
}
for k, v in data.items():
k = ('plan %s' % k).split('.')[::-1]
sh.get_nested_dicts(summary, *k).update(v)
for k, v in sh.stack_nested_keys(sol['summary'], depth=3):
if k[:-1] not in keys:
sh.get_nested_dicts(summary, *k).update(v)
sol['summary'] = summary
yield sol
def _extract_summary_from_model_scores(report, extracted):
n = ('data', 'calibration', 'model_scores', 'model_selections')
if not sh.are_in_nested_dicts(report, *n):
return False
sel = sh.get_nested_dicts(report, *n)
s = ('data', 'calibration', 'model_scores', 'score_by_model')
score = sh.get_nested_dicts(report, *s)
s = ('data', 'calibration', 'model_scores', 'scores')
scores = sh.get_nested_dicts(report, *s)
for k, v in sh.stack_nested_keys(extracted, depth=3):
n = k[1::-1]
if k[-1] == 'output' and sh.are_in_nested_dicts(sel, *n):
gen = sh.get_nested_dicts(sel, *n)
gen = ((d['model_id'], d['status']) for d in gen if 'status' in d)
o = _format_dict(gen, 'status %s')
v.update(o)
if k[1] == 'calibration' and k[0] in score:
gen = score[k[0]]
gen = ((d['model_id'], d['score']) for d in gen
if 'score' in d)
o = _format_dict(gen, 'score %s')
v.update(o)
for i, j in scores[k[0]].items():
gen = (('/'.join(
(d['model_id'], d['param_id'])), d['score']) for d in j
if 'score' in d)
o = _format_dict(gen, 'score {}/%s'.format(i))
v.update(o)
return True
def get_chart_reference(report):
r, _map = {}, _map_cycle_report_graphs()
out = report.get('output', {})
it = sh.stack_nested_keys(out, key=('output', ), depth=3)
for k, v in sorted(it):
if k[-1] == 'ts' and 'times' in v:
label = '{}/%s'.format(co2_exl._sheet_name(k))
for i, j in sorted(v.items()):
param_id = co2_exl._re_params_name.match(i)['param']
m = _map.get(param_id, None)
if m:
d = {
'x': k + ('times', ),
'y': k + (i, ),
'label': label % i
}
n = k[2], param_id, 'series'
sh.get_nested_dicts(r, *n, default=list).append(d)
for k, v in sh.stack_nested_keys(r, depth=2):
m = _map[k[1]]
m.pop('label', None)
v.update(m)
return r
def parse_data(raw_data, sets_mapping=None):
"""
Extract and rename the data-sets to _process.
:param raw_data:
Raw Data.
:type raw_data: dict[str, dict[str, numpy.array]]
:param sets_mapping:
Mapping of data-sets to _process.
It is like `{"<set-name>": {"<new-name>": "<old-name>", ...}, ...}`.
:type sets_mapping: dict[str, dict[str, str]]
:return:
Model data.
:rtype: dict
"""
if sets_mapping is None:
data = raw_data
else:
data = {}
for (i, j), k in sh.stack_nested_keys(sets_mapping):
sh.get_nested_dicts(data, i)[j] = raw_data[i][k]
parsed_data = {}
for (i, j), v in sh.stack_nested_keys(data):
if not np.isnan(v).all():
sh.get_nested_dicts(parsed_data, i)[j] = v
return parsed_data
def _format_selection(score_by_model, depth=-1, index='model_id'):
res = {}
for k, v in sorted(sh.stack_nested_keys(score_by_model, depth=depth)):
v = v.copy()
v[index] = k[0]
sh.get_nested_dicts(res, *k[1:], default=list).append(v)
return res
def compare_outputs_vs_targets(data):
"""
Compares model outputs vs targets.
:param data:
Model data.
:type data: dict
:return:
Comparison results.
:rtype: dict
"""
res = {}
metrics = _get_metrics()
for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3):
if not sh.are_in_nested_dicts(data, 'output', *k):
continue
o = sh.get_nested_dicts(data, 'output', *k)
v = _compare(t, o, metrics=metrics)
if v:
sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def _calibrate_gsm(
velocity_speed_ratios, on_engine, anomalies, gear, velocities,
stop_velocity, idle_engine_speed):
# noinspection PyProtectedMember
from .at_gear.cmv import CMV, _filter_gear_shifting_velocity as filter_gs
idle = idle_engine_speed[0] - idle_engine_speed[1]
_vsr = sh.combine_dicts(velocity_speed_ratios, base={0: 0})
limits = {
0: {False: [0]},
1: {True: [stop_velocity]},
max(_vsr): {True: [dfl.INF]}
}
shifts = np.unique(sum(map(_shift, (on_engine, anomalies)), []))
for i, j in sh.pairwise(shifts):
if on_engine[i:j].all() and not anomalies[i:j].any():
for v in np.array(list(sh.pairwise(_shift(gear[i:j])))) + i:
if j != v[1]:
v, (g, ng) = velocities[slice(*v)], gear[[v[1] - 1, v[1]]]
up = g < ng
sh.get_nested_dicts(limits, g, up, default=list).append(
v.max() if up else v.min()
)
for k, v in list(limits.items()):
limits[k] = v.get(False, [_vsr[k] * idle] * 2), v.get(True, [])
d = {j: i for i, j in enumerate(sorted(limits))}
gsm = CMV(filter_gs(sh.map_dict(d, limits), stop_velocity))
gsm.velocity_speed_ratios = sh.selector(gsm, sh.map_dict(d, _vsr))
gsm.convert(_vsr)
return gsm
def parse_solution(solution):
"""
Parse the CO2MPAS model solution.
:param solution:
CO2MPAS model solution.
:type solution: schedula.Solution
:return:
CO2MPAS outputs.
:rtype: dict[dict]
"""
res = {}
for k, v in solution.items():
k = k.split('.')
sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v
for k, v in list(sh.stack_nested_keys(res, depth=3)):
n, k = k[:-1], k[-1]
if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'):
v = sh.selector(('co2_emission_value', ), v, allow_miss=True)
if v:
d = sh.get_nested_dicts(res, 'target', 'prediction')
d[k] = sh.combine_dicts(v, d.get(k, {}))
res['pipe'] = solution.pipe
return res
def split_prediction_models(scores, models, default_models):
"""
Split prediction models.
:param scores:
Models score.
:type scores: dict
:param models:
Calibrated models.
:type models: dict
:param default_models:
Default calibrated models.
:type default_models: dict
:return:
Scores and prediction models.
:rtype: tuple
"""
sbm, model_sel, par = {}, {}, {}
for (k, c), v in sh.stack_nested_keys(scores, depth=2):
r = sh.selector(['models'], v, allow_miss=True)
for m in r.get('models', ()):
sh.get_nested_dicts(par, m, 'calibration')[c] = c
r.update(v.get('score', {}))
sh.get_nested_dicts(sbm, k)[c] = r
r = sh.selector(['success'], r, allow_miss=True)
r = sh.map_dict({'success': 'status'}, r, {'from': c})
sh.get_nested_dicts(model_sel, k, 'calibration')[c] = r
p = {i: dict.fromkeys(default_models, 'input') for i in prediction_cycles}
mdls = {i: default_models.copy() for i in prediction_cycles}
for k, n in sorted(models.items()):
d = n.get(sh.NONE, (None, True, {}))
for i in prediction_cycles:
c, s, m = n.get(i, d)
if m:
s = {'from': c, 'status': s}
sh.get_nested_dicts(model_sel, k, 'prediction')[i] = s
mdls[i].update(m)
p[i].update(dict.fromkeys(m, c))
for k, v in sh.stack_nested_keys(p, ('prediction', ), depth=2):
sh.get_nested_dicts(par, k[-1], *k[:-2])[k[-2]] = v
s = {
'param_selections': par,
'model_selections': model_sel,
'score_by_model': sbm,
'scores': scores
}
return (s, ) + tuple(mdls.get(k, {}) for k in prediction_cycles)
def _extract_summary_from_summary(report, extracted):
n = ('summary', 'results')
if sh.are_in_nested_dicts(report, *n):
for j, w in sh.get_nested_dicts(report, *n).items():
if j in ('declared_co2_emission', 'co2_emission',
'fuel_consumption'):
for k, v in sh.stack_nested_keys(w, depth=3):
if v:
sh.get_nested_dicts(extracted, *k).update(v)
def _add_special_data2report(data, report, to_keys, *from_keys):
if from_keys[-1] != 'times' and \
sh.are_in_nested_dicts(data, *from_keys):
v = sh.get_nested_dicts(data, *from_keys)
n = to_keys + ('{}.{}'.format(from_keys[0], from_keys[-1]), )
sh.get_nested_dicts(report, *n[:-1],
default=collections.OrderedDict)[n[-1]] = v
return True, v
return False, None
def filter_summary(changes, new_outputs, summary):
l, variations = {tuple(k.split('.')[:0:-1]) for k in new_outputs}, {}
for k, v in changes.items():
n = k[-2:1:-1]
l.add(n)
k = n + ('plan.%s' % '.'.join(i for i in k[:-1] if k not in n), k[-1])
sh.get_nested_dicts(variations, *k, default=co2_utl.ret_v(v))
for k, v in sh.stack_nested_keys(summary, depth=3):
if k[:-1] in l:
sh.get_nested_dicts(variations, *k, default=co2_utl.ret_v(v))
return variations
def overwrite_declaration_config_data(data):
config = constants.con_vals.DECLARATION_SELECTOR_CONFIG
res = sh.combine_nested_dicts(data, depth=3)
key = ('config', 'selector', 'all')
d = copy.deepcopy(sh.get_nested_dicts(res, *key))
for k, v in sh.stack_nested_keys(config):
sh.get_nested_dicts(d, *k, default=co2_utl.ret_v(v))
sh.get_nested_dicts(res, *key[:-1])[key[-1]] = d
return res
def select_declaration_data(data, diff=None):
res = {}
for k, v in sh.stack_nested_keys(constants.con_vals.DECLARATION_DATA):
if v and sh.are_in_nested_dicts(data, *k):
v = sh.get_nested_dicts(data, *k)
sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
if diff is not None:
diff.clear()
diff.update(v[0] for v in sh.stack_nested_keys(data, depth=4))
it = (v[0] for v in sh.stack_nested_keys(res, depth=4))
diff.difference_update(it)
return res
def update_inputs(self, references=None):
if not self.builder:
return
self.inputs = inp = collections.OrderedDict()
for k, rng in self.func.inputs.items():
try:
rng = rng or Ranges().push((references or {})[k])
except KeyError:
sh.get_nested_dicts(inp, Error.errors['#REF!'],
default=list).append(k)
continue
for r in rng.ranges:
sh.get_nested_dicts(inp, r['name'], default=list).append(k)
def _compare_outputs_vs_targets(data):
res = {}
metrics = _get_metrics()
for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3):
if not sh.are_in_nested_dicts(data, 'output', *k):
continue
o = sh.get_nested_dicts(data, 'output', *k)
v = _compare(t, o, metrics=metrics)
if v:
sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v
return res
def _format_report_scores(data):
res = {}
scores = 'data', 'calibration', 'model_scores'
if sh.are_in_nested_dicts(data, *scores):
n = scores + ('param_selections', )
v = _format_selection(sh.get_nested_dicts(data, *n), 2, 'param_id')
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
n = scores + ('model_selections', )
v = _format_selection(sh.get_nested_dicts(data, *n), 3)
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
n = scores + ('score_by_model', )
v = _format_selection(sh.get_nested_dicts(data, *n), 2)
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
n = scores + ('scores', )
v = _format_scores(sh.get_nested_dicts(data, *n))
if v:
sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v
return res
def _summary2df(data):
res = []
summary = data.get('summary', {})
if 'results' in summary:
r = {}
index = ['cycle', 'stage', 'usage']
for k, v in sh.stack_nested_keys(summary['results'], depth=4):
l = sh.get_nested_dicts(r, k[0], default=list)
l.append(sh.combine_dicts(sh.map_list(index, *k[1:]), v))
if r:
df = _dd2df(r,
index=index,
depth=2,
col_key=functools.partial(_sort_key,
p_keys=('param', ) * 2),
row_key=functools.partial(_sort_key, p_keys=index))
import pandas as pd
df.columns = pd.MultiIndex.from_tuples(_add_units(df.columns))
setattr(df, 'name', 'results')
res.append(df)
if 'selection' in summary:
df = _dd2df(summary['selection'], ['model_id'],
depth=2,
col_key=functools.partial(_sort_key,
p_keys=('stage', 'cycle')),
row_key=functools.partial(_sort_key, p_keys=()))
setattr(df, 'name', 'selection')
res.append(df)
if 'comparison' in summary:
r = {}
for k, v in sh.stack_nested_keys(summary['comparison'], depth=3):
v = sh.combine_dicts(v, base={'param_id': k[-1]})
sh.get_nested_dicts(r, *k[:-1], default=list).append(v)
if r:
df = _dd2df(r, ['param_id'],
depth=2,
col_key=functools.partial(_sort_key,
p_keys=('stage', 'cycle')),
row_key=functools.partial(_sort_key, p_keys=()))
setattr(df, 'name', 'comparison')
res.append(df)
if res:
return {'summary': res}
return {}
def setUp(self):
self.filename = osp.join(mydir, _filename)
self.link_filename = osp.join(mydir, _link_filename)
self.filename_compile = osp.join(mydir, _filename_compile)
self.filename_circular = osp.join(mydir, _filename_circular)
self.results = _file2books(self.filename, self.link_filename)
sh.get_nested_dicts(self.results, 'EXTRA.XLSX', 'EXTRA').update({
'A1': 1,
'B1': 1
})
self.results_compile = _book2dict(
load_workbook(self.filename_compile, data_only=True))['DATA']
self.results_circular = _file2books(self.filename_circular)
self.maxDiff = None
def get_values(data, keys, tag=(), update=lambda k, v: v, base=None):
k = ('input', 'target', 'output')
data = sh.selector(k, data, allow_miss=True)
base = {} if base is None else base
for k, v in sh.stack_nested_keys(data, depth=3):
k = k[::-1]
v = sh.selector(keys, v, allow_miss=True)
v = update(k, v)
if v:
k = tag + k
sh.get_nested_dicts(base, *k, default=co2_utl.ret_v(v))
return base
def read_json(input_fpath, data_names=None):
"""
Reads the json file.
:param input_fpath:
Input file path.
:type input_fpath: str
:param data_names:
Data names to filter out the data sets to synchronize.
:type data_names: list
:return:
Raw data-sets.
:rtype: dict[str, dict[str, numpy.array]]
"""
import json
import numpy as np
data = {}
with open(input_fpath) as file:
for k, v in sh.stack_nested_keys(json.load(file)):
if not data_names or k[0] in data_names:
sh.get_nested_dicts(data,
k[0])[sh.bypass(*k[1:])] = np.array(v)
return data
def _scores2df(data):
n = ('data', 'calibration', 'model_scores')
if not sh.are_in_nested_dicts(data, *n):
return {}
scores = sh.get_nested_dicts(data, *n)
it = (('model_selections', ['model_id'], 2, ('stage', 'cycle'), ()),
('score_by_model', ['model_id'], 1, ('cycle',), ()),
('scores', ['model_id', 'param_id'], 2, ('cycle', 'cycle'), ()),
('param_selections', ['param_id'], 2, ('stage', 'cycle'), ()),
('models_uuid', ['cycle'], 0, (), ('cycle',)))
dfs = []
for k, idx, depth, col_keys, row_keys in it:
if k not in scores:
continue
df = _dd2df(
scores[k], idx, depth=depth,
col_key=functools.partial(_sort_key, p_keys=col_keys),
row_key=functools.partial(_sort_key, p_keys=row_keys)
)
setattr(df, 'name', k)
dfs.append(df)
if dfs:
return {'.'.join(n): dfs}
else:
return {}
def _process_folder_files(*args, result_listener=None, **kwargs):
"""
Process all xls-files in a folder with CO2MPAS-model.
:param list input_files:
A list of input xl-files.
:param output_folder:
Output folder.
:type output_folder: str
:param plot_workflow:
If to show the CO2MPAS model workflow.
:type plot_workflow: bool, optional
:param output_template:
The xlsx-file to use as template and import existing sheets from.
- If file already exists, a clone gets updated with new sheets.
- If it is None, it copies and uses the input-file as template.
- if it is `False`, it does not use any template and a fresh output
xlsx-file is created.
:type output_folder: None,False,str
"""
start_time = datetime.datetime.today()
summary, n = {}, ('solution', 'summary')
for res in _yield_folder_files_results(start_time, *args, **kwargs):
if sh.are_in_nested_dicts(res, *n):
_add2summary(summary, sh.get_nested_dicts(res, *n))
notify_result_listener(result_listener, res)
return summary, start_time
def validate_meta(meta=None, hard_validation=False):
"""
Validate meta data.
:param meta:
Meta data.
:type meta: dict
:param hard_validation:
Add extra data validations.
:type hard_validation: bool
:return:
Validated meta data.
:rtype: dict
"""
i, e = _validate_base_with_schema(meta or {}, depth=2)
if hard_validation:
from schema import SchemaError
from .hard import _hard_validation
for k, v in sorted(sh.stack_nested_keys(i, depth=1)):
for c, msg in _hard_validation(v, 'meta'):
sh.get_nested_dicts(e, *k)[c] = SchemaError([], [msg])
if _log_errors_msg(e):
return sh.NONE
return i
def format_report_output(data):
res = {}
func = functools.partial(sh.get_nested_dicts,
default=collections.OrderedDict)
for k, v in sh.stack_nested_keys(data.get('output', {}), depth=3):
_add_special_data2report(data, res, k[:-1], 'target', *k)
s, iv = _add_special_data2report(data, res, k[:-1], 'input', *k)
if not s or (s and not _is_equal(iv, v)):
func(res, *k[:-1])[k[-1]] = v
output = {}
for k, v in sh.stack_nested_keys(res, depth=2):
v = _split_by_data_format(v)
sh.get_nested_dicts(output, *k, default=co2_utl.ret_v(v))
return output
def _validate_base_with_schema(data, depth=4):
from ..schema import define_data_schema
inputs, errors, validate = {}, {}, define_data_schema().validate
for k, v in sorted(sh.stack_nested_keys(data, depth=depth)):
d = sh.get_nested_dicts(inputs, *k[:-1])
_add_validated_input(d, validate, k, v, errors)
return inputs, errors
def _compare(self, books, results):
for k, other in sh.stack_nested_keys(results, depth=3):
value = sh.get_nested_dicts(books, *k)
msg = '[{}]{}!{}'.format(*k)
if is_number(value) and is_number(other):
self.assertAlmostEqual(float(value), float(other), msg=msg)
else:
self.assertEqual(value, other, msg=msg)
def test_get_nested_dicts(self):
d = {'a': {'b': {'c': ('d',)}}, 'A': {'B': {'C': ('D',)}}}
output = sh.get_nested_dicts(d, 'a', 'b', 'c')
result = ('d',)
self.assertEqual(output, result)
output = sh.get_nested_dicts(d, 0, default=list)
self.assertIsInstance(output, list)
self.assertTrue(0 in d)
import collections
output = sh.get_nested_dicts(d, 0, init_nesting=collections.OrderedDict)
self.assertIsInstance(output, list)
output = sh.get_nested_dicts(d, 1, init_nesting=collections.OrderedDict)
self.assertIsInstance(output, collections.OrderedDict)
self.assertTrue(1 in d)
output = sh.get_nested_dicts(d, 2, 3, default=list,
init_nesting=collections.OrderedDict)
self.assertIsInstance(output, list)
self.assertTrue(2 in d)
self.assertIsInstance(d[2], collections.OrderedDict)
