def _extract_summary_from_model_scores(report, extracted): n = ('data', 'calibration', 'model_scores', 'model_selections') if not sh.are_in_nested_dicts(report, *n): return False sel = sh.get_nested_dicts(report, *n) s = ('data', 'calibration', 'model_scores', 'score_by_model') score = sh.get_nested_dicts(report, *s) s = ('data', 'calibration', 'model_scores', 'scores') scores = sh.get_nested_dicts(report, *s) for k, v in sh.stack_nested_keys(extracted, depth=3): n = k[1::-1] if k[-1] == 'output' and sh.are_in_nested_dicts(sel, *n): gen = sh.get_nested_dicts(sel, *n) gen = ((d['model_id'], d['status']) for d in gen if 'status' in d) o = _format_dict(gen, 'status %s') v.update(o) if k[1] == 'calibration' and k[0] in score: gen = score[k[0]] gen = ((d['model_id'], d['score']) for d in gen if 'score' in d) o = _format_dict(gen, 'score %s') v.update(o) for i, j in scores[k[0]].items(): gen = (('/'.join( (d['model_id'], d['param_id'])), d['score']) for d in j if 'score' in d) o = _format_dict(gen, 'score {}/%s'.format(i)) v.update(o) return True
def extract_dice_report(encrypt_inputs, vehicle_family_id, start_time, report): from co2mpas import version res = { 'info': { 'encrypt_inputs': encrypt_inputs, 'vehicle_family_id': vehicle_family_id, 'CO2MPAS_version': version, 'datetime': start_time.strftime('%Y/%m/%d-%H:%M:%S') } } # deviation keys = 'summary', 'comparison', 'prediction' if sh.are_in_nested_dicts(report, *keys): deviation = 'declared_co2_emission_value', 'prediction_target_ratio' for cycle, d in sh.get_nested_dicts(report, *keys).items(): if sh.are_in_nested_dicts(d, *deviation): v = (sh.get_nested_dicts(d, *deviation) - 1) * 100 sh.get_nested_dicts(res, 'deviation')[cycle] = v # vehicle keys = [('summary', 'results', 'vehicle'), ('prediction', 'output')] vehicle = 'fuel_type', 'engine_capacity', 'gear_box_type', 'engine_is_turbo' if sh.are_in_nested_dicts(report, *keys[0]): for cycle, d in sh.get_nested_dicts(report, *keys[0]).items(): if sh.are_in_nested_dicts(d, *keys[1]): v = sh.selector(vehicle, sh.get_nested_dicts(d, *keys[1]), allow_miss=True) if v: sh.get_nested_dicts(res, 'vehicle', cycle).update(v) # model scores keys = 'data', 'calibration', 'model_scores' model_scores = 'model_selections', 'param_selections', 'score_by_model', \ 'scores' if sh.are_in_nested_dicts(report, *keys): sh.get_nested_dicts(res, 'model_scores').update( sh.selector(model_scores, sh.get_nested_dicts(report, *keys), allow_miss=True)) res = copy.deepcopy(res) for k, v in list(stack(res)): if isinstance(v, np.generic): sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v.item() return res
def _format_report_scores(data): res = {} scores = 'data', 'calibration', 'model_scores' if sh.are_in_nested_dicts(data, *scores): n = scores + ('param_selections', ) v = _format_selection(sh.get_nested_dicts(data, *n), 2, 'param_id') if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v n = scores + ('model_selections', ) v = _format_selection(sh.get_nested_dicts(data, *n), 3) if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v n = scores + ('score_by_model', ) v = _format_selection(sh.get_nested_dicts(data, *n), 2) if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v n = scores + ('scores', ) v = _format_scores(sh.get_nested_dicts(data, *n)) if v: sh.get_nested_dicts(res, *n[:-1])[n[-1]] = v return res
def _ta_mode(data): data = data.copy() base, errors = _validate_base_with_schema(data.get('base', {})) if _log_errors_msg(errors): return False data['base'], _, diff = _extract_declaration_data(base, {}) for k in ('plan', 'flag'): diff.update((k, ) + tuple(j.split('.')) for j in data.get(k, {})) diff -= {('flag', 'input_version'), ('flag', 'vehicle_family_id'), ('flag', 'encrypt_inputs'), ('flag', 'encryption_keys')} if diff: diff = ['.'.join(k) for k in sorted(diff)] log.info( 'Since CO2MPAS is launched in type approval mode the ' 'following data cannot be used:\n %s\n' 'If you want to include these data use the cmd batch.', ',\n'.join(diff)) return False if not sh.are_in_nested_dicts(data, 'flag', 'vehicle_family_id') \ or not data['flag']['vehicle_family_id']: log.info('Since CO2MPAS is launched in type approval mode the ' '`vehicle_family_id` is required!\n' 'If you want to run without it use the cmd batch.') return False return True
def _scores2df(data): n = ('data', 'calibration', 'model_scores') if not sh.are_in_nested_dicts(data, *n): return {} scores = sh.get_nested_dicts(data, *n) it = (('model_selections', ['model_id'], 2, ('stage', 'cycle'), ()), ('score_by_model', ['model_id'], 1, ('cycle',), ()), ('scores', ['model_id', 'param_id'], 2, ('cycle', 'cycle'), ()), ('param_selections', ['param_id'], 2, ('stage', 'cycle'), ()), ('models_uuid', ['cycle'], 0, (), ('cycle',))) dfs = [] for k, idx, depth, col_keys, row_keys in it: if k not in scores: continue df = _dd2df( scores[k], idx, depth=depth, col_key=functools.partial(_sort_key, p_keys=col_keys), row_key=functools.partial(_sort_key, p_keys=row_keys) ) setattr(df, 'name', k) dfs.append(df) if dfs: return {'.'.join(n): dfs} else: return {}
def compare_outputs_vs_targets(data): """ Compares model outputs vs targets. :param data: Model data. :type data: dict :return: Comparison results. :rtype: dict """ res = {} metrics = _get_metrics() for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3): if not sh.are_in_nested_dicts(data, 'output', *k): continue o = sh.get_nested_dicts(data, 'output', *k) v = _compare(t, o, metrics=metrics) if v: sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) return res
def notify_result_listener(result_listener, res, out_fpath=None): """Utility func to send to the listener the output-file discovered from the results.""" are_in = sh.are_in_nested_dicts if result_listener: if not out_fpath: it = [] for k in ('output_file_name', 'output_ta_file'): if sh.are_in_nested_dicts(res, 'solution', k): it.append(res['solution'][k]) else: it = sh.stlp(out_fpath) try: for fpath in it: result_listener((fpath, res)) except Exception as ex: try: keys = list(res) except Exception: keys = '<no keys>' log.warning( "Failed notifying result-listener due to: %s\n result-keys: %s", ex, keys, exc_info=1)
def _process_folder_files(*args, result_listener=None, **kwargs): """ Process all xls-files in a folder with CO2MPAS-model. :param list input_files: A list of input xl-files. :param output_folder: Output folder. :type output_folder: str :param plot_workflow: If to show the CO2MPAS model workflow. :type plot_workflow: bool, optional :param output_template: The xlsx-file to use as template and import existing sheets from. - If file already exists, a clone gets updated with new sheets. - If it is None, it copies and uses the input-file as template. - if it is `False`, it does not use any template and a fresh output xlsx-file is created. :type output_folder: None,False,str """ start_time = datetime.datetime.today() summary, n = {}, ('solution', 'summary') for res in _yield_folder_files_results(start_time, *args, **kwargs): if sh.are_in_nested_dicts(res, *n): _add2summary(summary, sh.get_nested_dicts(res, *n)) notify_result_listener(result_listener, res) return summary, start_time
def _add_special_data2report(data, report, to_keys, *from_keys): if from_keys[-1] != 'times' and \ sh.are_in_nested_dicts(data, *from_keys): v = sh.get_nested_dicts(data, *from_keys) n = to_keys + ('{}.{}'.format(from_keys[0], from_keys[-1]), ) sh.get_nested_dicts(report, *n[:-1], default=collections.OrderedDict)[n[-1]] = v return True, v return False, None
def _extract_summary_from_summary(report, extracted): n = ('summary', 'results') if sh.are_in_nested_dicts(report, *n): for j, w in sh.get_nested_dicts(report, *n).items(): if j in ('declared_co2_emission', 'co2_emission', 'fuel_consumption'): for k, v in sh.stack_nested_keys(w, depth=3): if v: sh.get_nested_dicts(extracted, *k).update(v)
def _add_times_base(data, scope='base', usage='input', **match): if scope != 'base': return sh_type = _get_sheet_type(scope=scope, usage=usage, **match) n = (scope, 'target') if sh_type == 'ts' and sh.are_in_nested_dicts(data, *n): t = sh.get_nested_dicts(data, *n) for k, v in sh.stack_nested_keys(t, key=n, depth=2): if 'times' not in v: n = list(k + ('times',)) n[1] = usage if sh.are_in_nested_dicts(data, *n): v['times'] = sh.get_nested_dicts(data, *n) else: for i, j in sh.stack_nested_keys(data, depth=4): if 'times' in j: v['times'] = j['times'] break
def format_report_scores(data): res = {} scores = 'data', 'calibration', 'model_scores' if sh.are_in_nested_dicts(data, *scores): n = scores + ('param_selections', ) v = _format_selection(sh.get_nested_dicts(data, *n), 2, 'param_id') if v: sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v)) n = scores + ('model_selections', ) v = _format_selection(sh.get_nested_dicts(data, *n), 3) if v: sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v)) n = scores + ('score_by_model', ) v = _format_selection(sh.get_nested_dicts(data, *n), 2) if v: sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v)) n = scores + ('scores', ) v = _format_scores(sh.get_nested_dicts(data, *n)) if v: sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v)) v = [] for k in ('nedc_h', 'nedc_l', 'wltp_h', 'wltp_l'): n = 'data', 'prediction', 'models_%s' % k if sh.are_in_nested_dicts(data, *n): v.append({ 'cycle': k, 'uuid': base64.encodebytes( dill.dumps(sh.get_nested_dicts(data, *n))) }) if v: n = scores + ('models_uuid', ) sh.get_nested_dicts(res, *n, default=co2_utl.ret_v(v)) return res
def select_declaration_data(data, diff=None): res = {} for k, v in sh.stack_nested_keys(constants.con_vals.DECLARATION_DATA): if v and sh.are_in_nested_dicts(data, *k): v = sh.get_nested_dicts(data, *k) sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) if diff is not None: diff.clear() diff.update(v[0] for v in sh.stack_nested_keys(data, depth=4)) it = (v[0] for v in sh.stack_nested_keys(res, depth=4)) diff.difference_update(it) return res
def _compare_outputs_vs_targets(data): res = {} metrics = _get_metrics() for k, t in sh.stack_nested_keys(data.get('target', {}), depth=3): if not sh.are_in_nested_dicts(data, 'output', *k): continue o = sh.get_nested_dicts(data, 'output', *k) v = _compare(t, o, metrics=metrics) if v: sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v return res
def _extract_summary_from_summary(report, extracted, augmented_summary=False): n = ('summary', 'results') keys = ( 'corrected_sustaining_co2_emission', 'declared_sustaining_co2_emission', 'declared_co2_emission', 'co2_emission', 'corrected_co2_emission', ) if augmented_summary: keys += 'fuel_consumption', if sh.are_in_nested_dicts(report, *n): for j, w in sh.get_nested_dicts(report, *n).items(): if j in keys: for k, v in sh.stack_nested_keys(w, depth=3): if v: sh.get_nested_dicts(extracted, *k).update(v)
def re_sample_targets(data): res = {} for k, v in sh.stack_nested_keys(data.get('target', {}), depth=2): if sh.are_in_nested_dicts(data, 'output', *k): o = sh.get_nested_dicts(data, 'output', *k) o = _split_by_data_format(o) t = sh.selector(o, _split_by_data_format(v), allow_miss=True) if 'times' not in t.get('ts', {}) or 'times' not in o['ts']: t.pop('ts', None) else: time_series = t['ts'] x, xp = o['ts']['times'], time_series.pop('times') if not _is_equal(x, xp): for i, fp in time_series.items(): time_series[i] = np.interp(x, xp, fp) v = sh.combine_dicts(*t.values()) sh.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) return res
def get_selection(data): n = ('data', 'calibration', 'model_scores', 'model_selections') if sh.are_in_nested_dicts(data, *n): return _format_selection(sh.get_nested_dicts(data, *n), 3) return {}
def test_are_in_nested_dicts(self): d = {'a': {'b': {'c': ('d',)}}, 'A': {'B': {'C': ('D',)}}} self.assertTrue(sh.are_in_nested_dicts(d, 'a', 'b', 'c')) self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'C')) self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'c', 'd'))
def test_are_in_nested_dicts(self): d = {'a': {'b': {'c': ('d', )}}, 'A': {'B': {'C': ('D', )}}} self.assertTrue(sh.are_in_nested_dicts(d, 'a', 'b', 'c')) self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'C')) self.assertFalse(sh.are_in_nested_dicts(d, 'a', 'b', 'c', 'd'))