def get_values(data, keys, tag=(), update=lambda k, v: v, base=None): k = ('input', 'target', 'output') data = dsp_utl.selector(k, data, allow_miss=True) base = {} if base is None else base for k, v in co2_utl.stack_nested_keys(data, depth=3): k = k[::-1] v = dsp_utl.selector(keys, v, allow_miss=True) v = update(k, v) if v: k = tag + k co2_utl.get_nested_dicts(base, *k, default=co2_utl.ret_v(v)) return base
def parse_dsp_solution(solution): """ Parses the co2mpas model results. :param solution: Co2mpas model after dispatching. :type solution: co2mpas.dispatcher.Solution :return: Mapped outputs. :rtype: dict[dict] """ res = {} for k, v in solution.items(): dsp_utl.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v)) for k, v in list(dsp_utl.stack_nested_keys(res, depth=3)): n, k = k[:-1], k[-1] if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'): v = dsp_utl.selector(('co2_emission_value', ), v, allow_miss=True) if v: d = dsp_utl.get_nested_dicts(res, 'target', 'prediction') d[k] = dsp_utl.combine_dicts(v, d.get(k, {})) res['pipe'] = solution.pipe return res
def select_prediction_data(data, new_data=(), theoretical=True): """ Selects the data required to predict the CO2 emissions with CO2MPAS model. :param data: Output data. :type data: dict :param new_data: New data. :type new_data: dict :param theoretical: If false :type theoretical: bool :return: Data required to predict the CO2 emissions with CO2MPAS model. :rtype: dict """ ids = [ 'angle_slope', 'alternator_nominal_voltage', 'alternator_efficiency', 'battery_capacity', 'cycle_type', 'cycle_name', 'engine_capacity', 'engine_stroke', 'engine_thermostat_temperature', 'final_drive_efficiency', 'frontal_area', 'aerodynamic_drag_coefficient', 'fuel_type', 'ignition_type', 'gear_box_type', 'engine_max_power', 'engine_max_speed_at_max_power', 'rolling_resistance_coeff', 'time_cold_hot_transition', 'engine_idle_fuel_consumption', 'engine_type', 'engine_is_turbo', 'engine_fuel_lower_heating_value', 'has_start_stop', 'has_energy_recuperation', 'fuel_carbon_content_percentage', 'f0', 'f1', 'f2', 'vehicle_mass', 'full_load_speeds', 'plateau_acceleration', 'full_load_powers', 'fuel_saving_at_strategy', 'stand_still_torque_ratio', 'lockup_speed_ratio', 'change_gear_window_width', 'alternator_start_window_width', 'stop_velocity', 'min_time_engine_on_after_start', 'min_engine_on_speed', 'max_velocity_full_load_correction', 'is_hybrid', 'tyre_code', 'engine_has_cylinder_deactivation', 'active_cylinder_ratios', 'engine_has_variable_valve_actuation', 'has_torque_converter', 'has_gear_box_thermal_management', 'has_lean_burn', 'ki_factor', 'n_wheel_drive', 'has_periodically_regenerating_systems', 'has_selective_catalytic_reduction', 'has_exhausted_gas_recirculation' ] if not theoretical: ids += ['times', 'velocities', 'gears'] data = dsp_utl.selector(ids, data, allow_miss=True) if new_data: data = dsp_utl.combine_dicts(data, new_data) if 'gears' in data and 'gears' not in new_data: if data.get('gear_box_type', 0) == 'automatic' or \ len(data.get('velocities', ())) != len(data['gears']): data.pop('gears') return data
def get_selection(data): res = [] n = ('data', 'calibration', 'model_scores', 'selections') if co2_utl.are_in_nested_dicts(data, *n): for k, v in sorted(co2_utl.get_nested_dicts(data, *n).items()): d = dsp_utl.selector(('from', 'status'), v['best']) d['model_id'] = k res.append(d) return res
def test_files(self): mydir = osp.dirname(__file__) if SEATBELT_FILE and osp.isfile(SEATBELT_FILE): res_file = SEATBELT_FILE else: tmpdir = tempfile.gettempdir() res_file = osp.join(tmpdir, 'co2mpas_seatbelt_demos.dill') log.info( "\n OVERWRITE_SEATBELT: %s \n" " RUN_INPUT_FOLDER: %s \n" " RUN_ALL_FILES: %s \n" " SEATBELT_FILE: %s", OVERWRITE_SEATBELT, RUN_INPUT_FOLDER, RUN_ALL_FILES, res_file) if not OVERWRITE_SEATBELT and osp.isfile(res_file): old_results = dsp_utl.load_dispatcher(res_file) log.info("Old results loaded!") else: old_results = None path = RUN_INPUT_FOLDER or osp.join(mydir, '..', 'co2mpas', 'demos') file = (path if (RUN_ALL_FILES or RUN_INPUT_FOLDER) else osp.join( path, 'co2mpas_demo-0.xlsx')) model = vehicle_processing_model() results = [] inp_files = file_finder([file]) if not inp_files: raise AssertionError("DataCheck found no input-files in %r!" % file) for fpath in inp_files: fname = osp.splitext(osp.basename(fpath))[0] log.info('Processing: %s', fname) inputs = { 'input_file_name': fpath, 'variation': { 'flag.only_summary': True } } r = model.dispatch(inputs=inputs) r = dsp_utl.selector(['report', 'summary'], r['solution']) r.get('report', {}).pop('pipe', None) results.append(sorted(dsp_utl.stack_nested_keys(r))) if not OVERWRITE_SEATBELT and osp.isfile(res_file): log.info('Comparing...') self._check_results(results, old_results) else: os.environ["OVERWRITE_SEATBELT"] = '0' dsp_utl.save_dispatcher(results, res_file) log.info('Overwritten seat belt %r.', res_file)
def _check_sign_currents(data, *args): c = ('battery_currents', 'alternator_currents') try: a = dsp_utl.selector(c, data, output_type='list') s = check_sign_currents(*a) if not all(s): s = ' and '.join([k for k, v in zip(c, s) if not v]) msg = "Probably '{}' have the wrong sign!".format(s) return c, msg except KeyError: # `c` is not in `data`. pass
def _check_initial_temperature(data, *args): t = ('initial_temperature', 'engine_coolant_temperatures', 'engine_speeds_out', 'idle_engine_speed_median') try: a = dsp_utl.selector(t, data, output_type='list') if not check_initial_temperature(*a): msg = "Initial engine temperature outside permissible limits " \ "according to GTR!" return t, msg except KeyError: # `t` is not in `data`. pass
def split_prediction_models( scores, calibrated_models, input_models, cycle_ids=()): sbm, model_sel, par = {}, {}, {} for (k, c), v in dsp_utl.stack_nested_keys(scores, depth=2): r = dsp_utl.selector(['models'], v, allow_miss=True) for m in r.get('models', ()): dsp_utl.get_nested_dicts(par, m, 'calibration')[c] = c r.update(v.get('score', {})) dsp_utl.get_nested_dicts(sbm, k, c, default=co2_utl.ret_v(r)) r = dsp_utl.selector(['success'], r, allow_miss=True) r = dsp_utl.map_dict({'success': 'status'}, r, {'from': c}) dsp_utl.get_nested_dicts(model_sel, k, 'calibration')[c] = r p = {i: dict.fromkeys(input_models, 'input') for i in cycle_ids} models = {i: input_models.copy() for i in cycle_ids} for k, n in sorted(calibrated_models.items()): d = n.get(dsp_utl.NONE, (None, True, {})) for i in cycle_ids: c, s, m = n.get(i, d) if m: s = {'from': c, 'status': s} dsp_utl.get_nested_dicts(model_sel, k, 'prediction')[i] = s models[i].update(m) p[i].update(dict.fromkeys(m, c)) for k, v in dsp_utl.stack_nested_keys(p, ('prediction',), depth=2): dsp_utl.get_nested_dicts(par, k[-1], *k[:-1], default=co2_utl.ret_v(v)) s = { 'param_selections': par, 'model_selections': model_sel, 'score_by_model': sbm, 'scores': scores } return (s,) + tuple(models.get(k, {}) for k in cycle_ids)
def hard_validation(data): c = ('battery_currents', 'alternator_currents') try: a = dsp_utl.selector(c, data, output_type='list') s = check_sign_currents(*a) if not all(s): s = ' and '.join([k for k, v in zip(c, s) if not v]) msg = "Probably '{}' have the wrong sign!".format(s) yield c, msg except KeyError: # `c` is not in `data`. pass t = ('initial_temperature', 'engine_coolant_temperatures', 'engine_speeds_out', 'idle_engine_speed_median') try: a = dsp_utl.selector(t, data, output_type='list') if not check_initial_temperature(*a): msg = "Initial engine temperature outside permissible limits " \ "according to GTR!" yield t, msg except KeyError: # `t` is not in `data`. pass
def _make_summarydf( nested_dict, index=None, depth=0, add_units=True, parts=()): df = _dd2df(nested_dict, index=index, depth=depth) p = _param_orders() p = dsp_utl.selector(parts + ('param',), p, output_type='list') gen = partial(zip_longest, p[:-1], fillvalue=p[-1]) c = sorted(df.columns, key=lambda x: [_match_part(m, v) for m, v in gen(x)]) df = df.reindex_axis(c, axis=1, copy=False) if add_units: c = _add_units(c) df.columns = pd.MultiIndex.from_tuples(c) return df
def _get_theoretical(profile): defaults = { 'cycle_type': 'WLTP', 'gear_box_type': 'manual', 'wltp_class': 'class3b', 'downscale_factor': 0 } profile = {k: v for k, v in profile.items() if v} profile = dsp_utl.combine_dicts(defaults, profile) profile['cycle_type'] = profile['cycle_type'].upper() profile['wltp_class'] = profile['wltp_class'].lower() profile['gear_box_type'] = profile['gear_box_type'].lower() from co2mpas.model.physical.cycle import cycle res = cycle().dispatch(inputs=profile, outputs=['times', 'velocities']) data = dsp_utl.selector(['times', 'velocities'], res, output_type='list') return pd.DataFrame(data).T
def get_dfl(wltp_base_model): """ Gets default values from wltp base model. :param wltp_base_model: WLTP base model params. :type wltp_base_model: dict :return: Default values from wltp base model. :rtype: list """ params = wltp_base_model['params'] keys = 'driver_mass', 'resistance_coeffs_regression_curves', 'wltc_data' return dsp_utl.selector(keys, params, output_type='list')
def define_new_inputs(data, base, dsp_model): remove = [] for k, v in co2_utl.stack_nested_keys(data, depth=2): if v is dsp_utl.EMPTY: remove.append(k) dsp = dsp_model.get_sub_dsp_from_workflow(data, check_inputs=False) n = set(base) - set(dsp.data_nodes) n.update(data) inp = dsp_utl.selector(n, base, allow_miss=True) d = co2_utl.combine_nested_dicts(inp, data, depth=2) for n, k in remove: co2_utl.get_nested_dicts(d, n).pop(k) return d
def select_initial_friction_params(co2_params_initial_guess): """ Selects initial guess of friction params l & l2 for the calculation of the motoring curve. :param co2_params_initial_guess: Initial guess of CO2 emission model params. :type co2_params_initial_guess: lmfit.Parameters :return: Initial guess of friction params l & l2. :rtype: float, float """ params = co2_params_initial_guess.valuesdict() return dsp_utl.selector(('l', 'l2'), params, output_type='list')
def combine_scores(scores): scores = {k[:-9]: v for k, v in scores.items() if v} if not scores: return {} s = {} for (k, c), v in co2_utl.stack_nested_keys(scores, depth=2): r = {'models': v['models']} if 'models' in v else {} r.update(v.get('score', {})) co2_utl.get_nested_dicts(s, k, c, default=co2_utl.ret_v(r)) if not co2_utl.are_in_nested_dicts(s, k, 'best'): keys = {'models': 'selected_models', 'success': 'status'} best = dsp_utl.map_dict(keys, dsp_utl.selector(keys, r)) best['from'] = c co2_utl.get_nested_dicts(s, k, 'best', default=co2_utl.ret_v(best)) return {'selections': s, 'scores': scores}
def make_simulation_plan(plan, timestamp, variation, flag, model=None): model, summary = model or batch.vehicle_processing_model(), {} run_base = model.get_node('run_base')[0].dsp run_modes = tuple( run_base.get_sub_dsp_from_workflow( ('data', 'vehicle_name'), check_inputs=False, graph=run_base.dmap).data_nodes) + ('start_time', 'vehicle_name') var = json.dumps(variation, sort_keys=True) o_cache, o_folder = flag['overwrite_cache'], flag['output_folder'] modelconf = flag.get('modelconf', None) kw, bases = dsp_utl.combine_dicts(flag, {'run_base': True}), set() for (i, base_fpath, run), p in tqdm.tqdm(plan, disable=False): try: base = get_results(model, o_cache, base_fpath, timestamp, run, var, o_folder, modelconf) except KeyError: log.warn('Base model "%s" of variation "%s" cannot be parsed!', base_fpath, i) continue name = base['vehicle_name'] if 'summary' in base and name not in bases: batch._add2summary(summary, base['summary']) bases.add(name) name = '{}-{}'.format(name, i) new_base, o = define_new_inputs(p, base) inputs = batch.prepare_data(new_base, {}, base_fpath, o_cache, o_folder, timestamp, False, modelconf)[0] inputs.update(dsp_utl.selector(set(base).difference(run_modes), base)) inputs['vehicle_name'] = name inputs.update(kw) res = run_base.dispatch(inputs) batch.notify_result_listener(plan_listener, res) s = filter_summary(p, o, res.get('summary', {})) base_keys = { 'vehicle_name': (base_fpath, name, run), } batch._add2summary(summary, s, base_keys) return summary
def make_simulation_plan(plan, timestamp, output_folder, main_flags): model, summary = vehicle_processing_model(), {} run_modes = tuple(model.get_sub_dsp_from_workflow( ('validated_data', 'vehicle_name'), check_inputs=False, graph=model.dmap ).data_nodes) + ('start_time', 'vehicle_name') kw = { 'output_folder': output_folder, 'plan': False, 'timestamp': timestamp, } kw, bases = dsp_utl.combine_dicts(main_flags, kw), set() for (i, base_fpath, defaults_fpats), p in tqdm(plan, disable=False): base = get_results(model, base_fpath, **kw) name = base['vehicle_name'] if name not in bases: _add2summary(summary, base.get('summary', {})) bases.add(name) name = '{}-{}'.format(name, i) inputs = dsp_utl.selector(set(base).difference(run_modes), base) inputs['vehicle_name'] = name dsp_model = base['dsp_model'] outputs = dsp_model.data_output dfl = build_default_models(model, defaults_fpats, **kw) if dfl: dfl = {'data.prediction.models': dfl} outputs = co2_utl.combine_nested_dicts(dfl, outputs, depth=2) inputs['validated_data'] = define_new_inputs(p, outputs, dsp_model) inputs.update(kw) res = _process_vehicle(model, **inputs) s = filter_summary(p, res.get('summary', {})) base_keys = { 'vehicle_name': (defaults_fpats, base_fpath, name), } _add2summary(summary, s, base_keys) return summary
def re_sample_targets(data): res = {} for k, v in co2_utl.stack_nested_keys(data.get('target', {}), depth=2): if co2_utl.are_in_nested_dicts(data, 'output', *k): o = co2_utl.get_nested_dicts(data, 'output', *k) o = _split_by_data_format(o) t = dsp_utl.selector(o, _split_by_data_format(v), allow_miss=True) if 'times' not in t.get('ts', {}) or 'times' not in o['ts']: t.pop('ts', None) else: time_series = t['ts'] x, xp = o['ts']['times'], time_series.pop('times') if not _is_equal(x, xp): for i, fp in time_series.items(): time_series[i] = np.interp(x, xp, fp) v = dsp_utl.combine_dicts(*t.values()) co2_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) return res
def _error(name, data_id, data_out, setting): d = dsp.Dispatcher( name='%s-%s error vs %s' % (name, data_id, data_out), description='Calculates the error of calibrated model of a reference.', ) default_settings = { 'inputs_map': {}, 'targets': [], 'metrics_inputs': {}, 'up_limit': None, 'dn_limit': None } default_settings.update(setting) it = dsp_utl.selector(['up_limit', 'dn_limit'], default_settings).items() for k, v in it: if v is not None: default_settings[k] = dsp_utl.map_list(setting['targets'], *v) d.add_function( function_id='select_inputs', function=dsp_utl.map_dict, inputs=['inputs_map', 'data'], outputs=['inputs<0>'] ) d.add_function( function_id='select_inputs', function=functools.partial(dsp_utl.selector, allow_miss=True), inputs=['inputs', 'inputs<0>'], outputs=['inputs<1>'] ) d.add_function( function=dsp_utl.combine_dicts, inputs=['calibrated_models', 'inputs<1>'], outputs=['prediction_inputs'] ) d.add_function( function_id='select_targets', function=functools.partial(dsp_utl.selector, allow_miss=True), inputs=['targets', 'data'], outputs=['references'] ) d.add_function( function=functools.partial( default_settings.pop('dsp', lambda x: x), {} ), inputs=['prediction_inputs', 'calibrated_models'], outputs=['results'] ) d.add_function( function_id='select_outputs', function=select_outputs, inputs=['outputs', 'targets', 'results'], outputs=['predictions'] ) d.add_function( function_id='select_metrics_inputs', function=functools.partial(dsp_utl.selector, allow_miss=True), inputs=['metrics_inputs', 'data'], outputs=['metrics_args'] ) d.add_function( function=make_metrics, inputs=['metrics', 'references', 'predictions', 'metrics_args'], outputs=['errors'] ) d.add_function( function=check_limits, inputs=['errors', 'up_limit', 'dn_limit'], outputs=['status'] ) for k, v in default_settings.items(): d.add_data(k, v) func = dsp_utl.SubDispatch( dsp=d, outputs=['errors', 'status'], output_type='list' ) return func
def tyre_models_selector(models_ids, data): models = dsp_utl.selector(models_ids, data, allow_miss=True) if 'tyre_dynamic_rolling_coefficient' in models: models.pop('r_dynamic', None) return models
def select_outputs(outputs, targets, results): results = dsp_utl.selector(outputs, results, allow_miss=True) results = dsp_utl.map_dict(dict(zip(outputs, targets)), results) it = ((k, results[k]) for k in targets if k in results) return collections.OrderedDict(it)
def get_dfl(wltp_base_model): params = wltp_base_model['params'] keys = 'driver_mass', 'resistance_coeffs_regression_curves', 'wltc_data' return dsp_utl.selector(keys, params, output_type='list')