def parse_dsp_solution(solution): """ Parses the co2mpas model results. :param solution: Co2mpas model after dispatching. :type solution: schedula.Solution :return: Mapped outputs. :rtype: dict[dict] """ res = {} for k, v in solution.items(): sh.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v)) for k, v in list(sh.stack_nested_keys(res, depth=3)): n, k = k[:-1], k[-1] if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'): v = sh.selector(('co2_emission_value', ), v, allow_miss=True) if v: d = sh.get_nested_dicts(res, 'target', 'prediction') d[k] = sh.combine_dicts(v, d.get(k, {})) res['pipe'] = solution.pipe return res
def _extract_summary_from_output(report, extracted, augmented_summary=False): for k, v in sh.stack_nested_keys(report.get('output', {}), depth=2): k = k[::-1] for u, i, j in _param_names_values(v.get('pa', {})): o = {} if i in ('has_sufficient_power', ): o = {i: j} elif augmented_summary: if i == 'co2_params_calibrated': o = _format_dict(j.valuesdict().items(), 'co2_params %s') elif i == 'calibration_status': o = _format_dict(enumerate(j), 'status co2_params step %d', lambda x: x[0]) elif i == 'willans_factors': o = j elif i == 'phases_willans_factors': for n, m in enumerate(j): o.update( _format_dict(m.items(), '%s phase {}'.format(n))) elif i == 'co2_rescaling_scores': o = sh.map_list( ['rescaling_mean', 'rescaling_std', 'rescaling_n'], *j) if o: sh.get_nested_dicts(extracted, *(k + (u, ))).update(o)
def test_get_nested_dicts(self): d = {'a': {'b': {'c': ('d', )}}, 'A': {'B': {'C': ('D', )}}} output = sh.get_nested_dicts(d, 'a', 'b', 'c') result = ('d', ) self.assertEqual(output, result) output = sh.get_nested_dicts(d, 0, default=list) self.assertIsInstance(output, list) self.assertTrue(0 in d) import collections output = sh.get_nested_dicts(d, 0, init_nesting=collections.OrderedDict) self.assertIsInstance(output, list) output = sh.get_nested_dicts(d, 1, init_nesting=collections.OrderedDict) self.assertIsInstance(output, collections.OrderedDict) self.assertTrue(1 in d) output = sh.get_nested_dicts(d, 2, 3, default=list, init_nesting=collections.OrderedDict) self.assertIsInstance(output, list) self.assertTrue(2 in d) self.assertIsInstance(d[2], collections.OrderedDict)
def _run_variations(plan, bases, core_model, timestamp): for r in _ProgressBar(plan, _format_meter=_format_meter): sol, data = bases[r['base']], r['data'] if 'solution' in sol: s = sol['solution'] base = _define_inputs(s, sh.combine_nested_dicts(sh.selector( data, s, allow_miss=True ), data)) elif 'base' in sol: base = sh.combine_nested_dicts(sol['base'], data, depth=2) else: continue for i, d in base.items(): if hasattr(d, 'items'): base[i] = {k: v for k, v in d.items() if v is not sh.EMPTY} sol = core_model(_define_inputs(sol, dict( base=base, vehicle_name='-'.join((str(r['id']), sol['vehicle_name'])), timestamp=timestamp ))) summary, keys = {}, { tuple(k.split('.')[:0:-1]) for k in base if k.startswith('output.') } for k, v in data.items(): k = ('plan %s' % k).split('.')[::-1] sh.get_nested_dicts(summary, *k).update(v) for k, v in sh.stack_nested_keys(sol['summary'], depth=3): if k[:-1] not in keys: sh.get_nested_dicts(summary, *k).update(v) sol['summary'] = summary yield sol
def _extract_summary_from_model_scores(report, extracted): n = ('data', 'calibration', 'model_scores', 'model_selections') if not sh.are_in_nested_dicts(report, *n): return False sel = sh.get_nested_dicts(report, *n) s = ('data', 'calibration', 'model_scores', 'score_by_model') score = sh.get_nested_dicts(report, *s) s = ('data', 'calibration', 'model_scores', 'scores') scores = sh.get_nested_dicts(report, *s) for k, v in sh.stack_nested_keys(extracted, depth=3): n = k[1::-1] if k[-1] == 'output' and sh.are_in_nested_dicts(sel, *n): gen = sh.get_nested_dicts(sel, *n) gen = ((d['model_id'], d['status']) for d in gen if 'status' in d) o = _format_dict(gen, 'status %s') v.update(o) if k[1] == 'calibration' and k[0] in score: gen = score[k[0]] gen = ((d['model_id'], d['score']) for d in gen if 'score' in d) o = _format_dict(gen, 'score %s') v.update(o) for i, j in scores[k[0]].items(): gen = (('/'.join( (d['model_id'], d['param_id'])), d['score']) for d in j if 'score' in d) o = _format_dict(gen, 'score {}/%s'.format(i)) v.update(o) return True
def get_chart_reference(report): r, _map = {}, _map_cycle_report_graphs() out = report.get('output', {}) it = sh.stack_nested_keys(out, key=('output', ), depth=3) for k, v in sorted(it): if k[-1] == 'ts' and 'times' in v: label = '{}/%s'.format(co2_exl._sheet_name(k)) for i, j in sorted(v.items()): param_id = co2_exl._re_params_name.match(i)['param'] m = _map.get(param_id, None) if m: d = { 'x': k + ('times', ), 'y': k + (i, ), 'label': label % i } n = k[2], param_id, 'series' sh.get_nested_dicts(r, *n, default=list).append(d) for k, v in sh.stack_nested_keys(r, depth=2): m = _map[k[1]] m.pop('label', None) v.update(m) return r
def parse_data(raw_data, sets_mapping=None): """ Extract and rename the data-sets to _process. :param raw_data: Raw Data. :type raw_data: dict[str, dict[str, numpy.array]] :param sets_mapping: Mapping of data-sets to _process. It is like `{"<set-name>": {"<new-name>": "<old-name>", ...}, ...}`. :type sets_mapping: dict[str, dict[str, str]] :return: Model data. :rtype: dict """ if sets_mapping is None: data = raw_data else: data = {} for (i, j), k in sh.stack_nested_keys(sets_mapping): sh.get_nested_dicts(data, i)[j] = raw_data[i][k] parsed_data = {} for (i, j), v in sh.stack_nested_keys(data): if not np.isnan(v).all(): sh.get_nested_dicts(parsed_data, i)[j] = v return parsed_data
def _format_selection(score_by_model, depth=-1, index='model_id'): res = {} for k, v in sorted(sh.stack_nested_keys(score_by_model, depth=depth)): v = v.copy() v[index] = k[0] sh.get_nested_dicts(res, *k[1:], default=list).append(v) return res
def compare_outputs_vs_targets(data): """ Compares model outputs vs targets. :param data: Model data. :type data: dict :return: Comparison results. :rtype: dict """ res = {} metrics = _get_metrics() for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3): if not sh.are_in_nested_dicts(data, 'output', *k): continue o = sh.get_nested_dicts(data, 'output', *k) v = _compare(t, o, metrics=metrics) if v: sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) return res
def _calibrate_gsm( velocity_speed_ratios, on_engine, anomalies, gear, velocities, stop_velocity, idle_engine_speed): # noinspection PyProtectedMember from .at_gear.cmv import CMV, _filter_gear_shifting_velocity as filter_gs idle = idle_engine_speed[0] - idle_engine_speed[1] _vsr = sh.combine_dicts(velocity_speed_ratios, base={0: 0}) limits = { 0: {False: [0]}, 1: {True: [stop_velocity]}, max(_vsr): {True: [dfl.INF]} } shifts = np.unique(sum(map(_shift, (on_engine, anomalies)), [])) for i, j in sh.pairwise(shifts): if on_engine[i:j].all() and not anomalies[i:j].any(): for v in np.array(list(sh.pairwise(_shift(gear[i:j])))) + i: if j != v[1]: v, (g, ng) = velocities[slice(*v)], gear[[v[1] - 1, v[1]]] up = g < ng sh.get_nested_dicts(limits, g, up, default=list).append( v.max() if up else v.min() ) for k, v in list(limits.items()): limits[k] = v.get(False, [_vsr[k] * idle] * 2), v.get(True, []) d = {j: i for i, j in enumerate(sorted(limits))} gsm = CMV(filter_gs(sh.map_dict(d, limits), stop_velocity)) gsm.velocity_speed_ratios = sh.selector(gsm, sh.map_dict(d, _vsr)) gsm.convert(_vsr) return gsm
def parse_solution(solution): """ Parse the CO2MPAS model solution. :param solution: CO2MPAS model solution. :type solution: schedula.Solution :return: CO2MPAS outputs. :rtype: dict[dict] """ res = {} for k, v in solution.items(): k = k.split('.') sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v for k, v in list(sh.stack_nested_keys(res, depth=3)): n, k = k[:-1], k[-1] if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'): v = sh.selector(('co2_emission_value', ), v, allow_miss=True) if v: d = sh.get_nested_dicts(res, 'target', 'prediction') d[k] = sh.combine_dicts(v, d.get(k, {})) res['pipe'] = solution.pipe return res
def split_prediction_models(scores, models, default_models): """ Split prediction models. :param scores: Models score. :type scores: dict :param models: Calibrated models. :type models: dict :param default_models: Default calibrated models. :type default_models: dict :return: Scores and prediction models. :rtype: tuple """ sbm, model_sel, par = {}, {}, {} for (k, c), v in sh.stack_nested_keys(scores, depth=2): r = sh.selector(['models'], v, allow_miss=True) for m in r.get('models', ()): sh.get_nested_dicts(par, m, 'calibration')[c] = c r.update(v.get('score', {})) sh.get_nested_dicts(sbm, k)[c] = r r = sh.selector(['success'], r, allow_miss=True) r = sh.map_dict({'success': 'status'}, r, {'from': c}) sh.get_nested_dicts(model_sel, k, 'calibration')[c] = r p = {i: dict.fromkeys(default_models, 'input') for i in prediction_cycles} mdls = {i: default_models.copy() for i in prediction_cycles} for k, n in sorted(models.items()): d = n.get(sh.NONE, (None, True, {})) for i in prediction_cycles: c, s, m = n.get(i, d) if m: s = {'from': c, 'status': s} sh.get_nested_dicts(model_sel, k, 'prediction')[i] = s mdls[i].update(m) p[i].update(dict.fromkeys(m, c)) for k, v in sh.stack_nested_keys(p, ('prediction', ), depth=2): sh.get_nested_dicts(par, k[-1], *k[:-2])[k[-2]] = v s = { 'param_selections': par, 'model_selections': model_sel, 'score_by_model': sbm, 'scores': scores } return (s, ) + tuple(mdls.get(k, {}) for k in prediction_cycles)
def _extract_summary_from_summary(report, extracted): n = ('summary', 'results') if sh.are_in_nested_dicts(report, *n): for j, w in sh.get_nested_dicts(report, *n).items(): if j in ('declared_co2_emission', 'co2_emission', 'fuel_consumption'): for k, v in sh.stack_nested_keys(w, depth=3): if v: sh.get_nested_dicts(extracted, *k).update(v)
def _add_special_data2report(data, report, to_keys, *from_keys): if from_keys[-1] != 'times' and \ sh.are_in_nested_dicts(data, *from_keys): v = sh.get_nested_dicts(data, *from_keys) n = to_keys + ('{}.{}'.format(from_keys[0], from_keys[-1]), ) sh.get_nested_dicts(report, *n[:-1], default=collections.OrderedDict)[n[-1]] = v return True, v return False, None
def filter_summary(changes, new_outputs, summary): l, variations = {tuple(k.split('.')[:0:-1]) for k in new_outputs}, {} for k, v in changes.items(): n = k[-2:1:-1] l.add(n) k = n + ('plan.%s' % '.'.join(i for i in k[:-1] if k not in n), k[-1]) sh.get_nested_dicts(variations, *k, default=co2_utl.ret_v(v)) for k, v in sh.stack_nested_keys(summary, depth=3): if k[:-1] in l: sh.get_nested_dicts(variations, *k, default=co2_utl.ret_v(v)) return variations
def overwrite_declaration_config_data(data): config = constants.con_vals.DECLARATION_SELECTOR_CONFIG res = sh.combine_nested_dicts(data, depth=3) key = ('config', 'selector', 'all') d = copy.deepcopy(sh.get_nested_dicts(res, *key)) for k, v in sh.stack_nested_keys(config): sh.get_nested_dicts(d, *k, default=co2_utl.ret_v(v)) sh.get_nested_dicts(res, *key[:-1])[key[-1]] = d return res
def select_declaration_data(data, diff=None): res = {} for k, v in sh.stack_nested_keys(constants.con_vals.DECLARATION_DATA): if v and sh.are_in_nested_dicts(data, *k): v = sh.get_nested_dicts(data, *k) sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) if diff is not None: diff.clear() diff.update(v[0] for v in sh.stack_nested_keys(data, depth=4)) it = (v[0] for v in sh.stack_nested_keys(res, depth=4)) diff.difference_update(it) return res
def update_inputs(self, references=None): if not self.builder: return self.inputs = inp = collections.OrderedDict() for k, rng in self.func.inputs.items(): try: rng = rng or Ranges().push((references or {})[k]) except KeyError: sh.get_nested_dicts(inp, Error.errors['#REF!'], default=list).append(k) continue for r in rng.ranges: sh.get_nested_dicts(inp, r['name'], default=list).append(k)
def _compare_outputs_vs_targets(data): res = {} metrics = _get_metrics() for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3): if not sh.are_in_nested_dicts(data, 'output', *k): continue o = sh.get_nested_dicts(data, 'output', *k) v = _compare(t, o, metrics=metrics) if v: sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v return res
def _format_report_scores(data): res = {} scores = 'data', 'calibration', 'model_scores' if sh.are_in_nested_dicts(data, *scores): n = scores + ('param_selections', ) v = _format_selection(sh.get_nested_dicts(data, *n), 2, 'param_id') if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v n = scores + ('model_selections', ) v = _format_selection(sh.get_nested_dicts(data, *n), 3) if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v n = scores + ('score_by_model', ) v = _format_selection(sh.get_nested_dicts(data, *n), 2) if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v n = scores + ('scores', ) v = _format_scores(sh.get_nested_dicts(data, *n)) if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v return res
def _summary2df(data): res = [] summary = data.get('summary', {}) if 'results' in summary: r = {} index = ['cycle', 'stage', 'usage'] for k, v in sh.stack_nested_keys(summary['results'], depth=4): l = sh.get_nested_dicts(r, k[0], default=list) l.append(sh.combine_dicts(sh.map_list(index, *k[1:]), v)) if r: df = _dd2df(r, index=index, depth=2, col_key=functools.partial(_sort_key, p_keys=('param', ) * 2), row_key=functools.partial(_sort_key, p_keys=index)) import pandas as pd df.columns = pd.MultiIndex.from_tuples(_add_units(df.columns)) setattr(df, 'name', 'results') res.append(df) if 'selection' in summary: df = _dd2df(summary['selection'], ['model_id'], depth=2, col_key=functools.partial(_sort_key, p_keys=('stage', 'cycle')), row_key=functools.partial(_sort_key, p_keys=())) setattr(df, 'name', 'selection') res.append(df) if 'comparison' in summary: r = {} for k, v in sh.stack_nested_keys(summary['comparison'], depth=3): v = sh.combine_dicts(v, base={'param_id': k[-1]}) sh.get_nested_dicts(r, *k[:-1], default=list).append(v) if r: df = _dd2df(r, ['param_id'], depth=2, col_key=functools.partial(_sort_key, p_keys=('stage', 'cycle')), row_key=functools.partial(_sort_key, p_keys=())) setattr(df, 'name', 'comparison') res.append(df) if res: return {'summary': res} return {}
def setUp(self): self.filename = osp.join(mydir, _filename) self.link_filename = osp.join(mydir, _link_filename) self.filename_compile = osp.join(mydir, _filename_compile) self.filename_circular = osp.join(mydir, _filename_circular) self.results = _file2books(self.filename, self.link_filename) sh.get_nested_dicts(self.results, 'EXTRA.XLSX', 'EXTRA').update({ 'A1': 1, 'B1': 1 }) self.results_compile = _book2dict( load_workbook(self.filename_compile, data_only=True))['DATA'] self.results_circular = _file2books(self.filename_circular) self.maxDiff = None
def get_values(data, keys, tag=(), update=lambda k, v: v, base=None): k = ('input', 'target', 'output') data = sh.selector(k, data, allow_miss=True) base = {} if base is None else base for k, v in sh.stack_nested_keys(data, depth=3): k = k[::-1] v = sh.selector(keys, v, allow_miss=True) v = update(k, v) if v: k = tag + k sh.get_nested_dicts(base, *k, default=co2_utl.ret_v(v)) return base
def read_json(input_fpath, data_names=None): """ Reads the json file. :param input_fpath: Input file path. :type input_fpath: str :param data_names: Data names to filter out the data sets to synchronize. :type data_names: list :return: Raw data-sets. :rtype: dict[str, dict[str, numpy.array]] """ import json import numpy as np data = {} with open(input_fpath) as file: for k, v in sh.stack_nested_keys(json.load(file)): if not data_names or k[0] in data_names: sh.get_nested_dicts(data, k[0])[sh.bypass(*k[1:])] = np.array(v) return data
def _scores2df(data): n = ('data', 'calibration', 'model_scores') if not sh.are_in_nested_dicts(data, *n): return {} scores = sh.get_nested_dicts(data, *n) it = (('model_selections', ['model_id'], 2, ('stage', 'cycle'), ()), ('score_by_model', ['model_id'], 1, ('cycle',), ()), ('scores', ['model_id', 'param_id'], 2, ('cycle', 'cycle'), ()), ('param_selections', ['param_id'], 2, ('stage', 'cycle'), ()), ('models_uuid', ['cycle'], 0, (), ('cycle',))) dfs = [] for k, idx, depth, col_keys, row_keys in it: if k not in scores: continue df = _dd2df( scores[k], idx, depth=depth, col_key=functools.partial(_sort_key, p_keys=col_keys), row_key=functools.partial(_sort_key, p_keys=row_keys) ) setattr(df, 'name', k) dfs.append(df) if dfs: return {'.'.join(n): dfs} else: return {}
def _process_folder_files(*args, result_listener=None, **kwargs): """ Process all xls-files in a folder with CO2MPAS-model. :param list input_files: A list of input xl-files. :param output_folder: Output folder. :type output_folder: str :param plot_workflow: If to show the CO2MPAS model workflow. :type plot_workflow: bool, optional :param output_template: The xlsx-file to use as template and import existing sheets from. - If file already exists, a clone gets updated with new sheets. - If it is None, it copies and uses the input-file as template. - if it is `False`, it does not use any template and a fresh output xlsx-file is created. :type output_folder: None,False,str """ start_time = datetime.datetime.today() summary, n = {}, ('solution', 'summary') for res in _yield_folder_files_results(start_time, *args, **kwargs): if sh.are_in_nested_dicts(res, *n): _add2summary(summary, sh.get_nested_dicts(res, *n)) notify_result_listener(result_listener, res) return summary, start_time
def validate_meta(meta=None, hard_validation=False): """ Validate meta data. :param meta: Meta data. :type meta: dict :param hard_validation: Add extra data validations. :type hard_validation: bool :return: Validated meta data. :rtype: dict """ i, e = _validate_base_with_schema(meta or {}, depth=2) if hard_validation: from schema import SchemaError from .hard import _hard_validation for k, v in sorted(sh.stack_nested_keys(i, depth=1)): for c, msg in _hard_validation(v, 'meta'): sh.get_nested_dicts(e, *k)[c] = SchemaError([], [msg]) if _log_errors_msg(e): return sh.NONE return i
def format_report_output(data): res = {} func = functools.partial(sh.get_nested_dicts, default=collections.OrderedDict) for k, v in sh.stack_nested_keys(data.get('output', {}), depth=3): _add_special_data2report(data, res, k[:-1], 'target', *k) s, iv = _add_special_data2report(data, res, k[:-1], 'input', *k) if not s or (s and not _is_equal(iv, v)): func(res, *k[:-1])[k[-1]] = v output = {} for k, v in sh.stack_nested_keys(res, depth=2): v = _split_by_data_format(v) sh.get_nested_dicts(output, *k, default=co2_utl.ret_v(v)) return output
def _validate_base_with_schema(data, depth=4): from ..schema import define_data_schema inputs, errors, validate = {}, {}, define_data_schema().validate for k, v in sorted(sh.stack_nested_keys(data, depth=depth)): d = sh.get_nested_dicts(inputs, *k[:-1]) _add_validated_input(d, validate, k, v, errors) return inputs, errors
def _compare(self, books, results): for k, other in sh.stack_nested_keys(results, depth=3): value = sh.get_nested_dicts(books, *k) msg = '[{}]{}!{}'.format(*k) if is_number(value) and is_number(other): self.assertAlmostEqual(float(value), float(other), msg=msg) else: self.assertEqual(value, other, msg=msg)
def test_get_nested_dicts(self): d = {'a': {'b': {'c': ('d',)}}, 'A': {'B': {'C': ('D',)}}} output = sh.get_nested_dicts(d, 'a', 'b', 'c') result = ('d',) self.assertEqual(output, result) output = sh.get_nested_dicts(d, 0, default=list) self.assertIsInstance(output, list) self.assertTrue(0 in d) import collections output = sh.get_nested_dicts(d, 0, init_nesting=collections.OrderedDict) self.assertIsInstance(output, list) output = sh.get_nested_dicts(d, 1, init_nesting=collections.OrderedDict) self.assertIsInstance(output, collections.OrderedDict) self.assertTrue(1 in d) output = sh.get_nested_dicts(d, 2, 3, default=list, init_nesting=collections.OrderedDict) self.assertIsInstance(output, list) self.assertTrue(2 in d) self.assertIsInstance(d[2], collections.OrderedDict)