def _add2summary(total_summary, summary, base_keys=None): base_keys = base_keys or {} for k, v in dsp_utl.stack_nested_keys(summary, depth=3): d = dsp_utl.get_nested_dicts(total_summary, *k, default=list) if isinstance(v, list): for j in v: d.append(dsp_utl.combine_dicts(j, base_keys)) else: d.append(dsp_utl.combine_dicts(v, base_keys))
def _summary2df(data): res = [] summary = data.get('summary', {}) if 'results' in summary: r = {} index = ['cycle', 'stage', 'usage'] for k, v in dsp_utl.stack_nested_keys(summary['results'], depth=4): l = dsp_utl.get_nested_dicts(r, k[0], default=list) l.append(dsp_utl.combine_dicts(dsp_utl.map_list(index, *k[1:]), v)) if r: df = _dd2df(r, index=index, depth=2, col_key=functools.partial(_sort_key, p_keys=('param', ) * 2), row_key=functools.partial(_sort_key, p_keys=index)) df.columns = pd.MultiIndex.from_tuples(_add_units(df.columns)) setattr(df, 'name', 'results') res.append(df) if 'selection' in summary: df = _dd2df(summary['selection'], ['model_id'], depth=2, col_key=functools.partial(_sort_key, p_keys=('stage', 'cycle')), row_key=functools.partial(_sort_key, p_keys=())) setattr(df, 'name', 'selection') res.append(df) if 'comparison' in summary: r = {} for k, v in dsp_utl.stack_nested_keys(summary['comparison'], depth=3): v = dsp_utl.combine_dicts(v, base={'param_id': k[-1]}) dsp_utl.get_nested_dicts(r, *k[:-1], default=list).append(v) if r: df = _dd2df(r, ['param_id'], depth=2, col_key=functools.partial(_sort_key, p_keys=('stage', 'cycle')), row_key=functools.partial(_sort_key, p_keys=())) setattr(df, 'name', 'comparison') res.append(df) if res: return {'summary': res} return {}
def write_to_excel(data, output_file_name, template_file_name): if template_file_name: log.debug('Writing into xl-file(%s) based on template(%s)...', output_file_name, template_file_name) shutil.copy(template_file_name, output_file_name) writer = clone_excel(template_file_name, output_file_name) else: log.debug('Writing into xl-file(%s)...', output_file_name) writer = pd.ExcelWriter(output_file_name, engine='xlsxwriter') xlref = [] for k, v in sorted(data.items(), key=_sort_sheets): if not k.startswith('graphs.'): if k.endswith('pa'): kw = {'named_ranges': ('rows', ), 'index': True, 'k0': 1} elif k.endswith('ts'): kw = {'named_ranges': ('columns', ), 'index': False, 'k0': 1} else: kw = {} down = not k.endswith('proc_info') xlref.extend(_write_sheets(writer, k, v, down=down, **kw)) else: _chart2excel(writer, k, v) if xlref: xlref = sorted(dsp_utl.combine_dicts(*[x[1] for x in xlref]).items()) xlref = pd.DataFrame(xlref) xlref.set_index([0], inplace=True) _df2excel(writer, 'xlref', xlref, 0, (), index=True, header=False) writer.save() log.info('Written into xl-file(%s)...', output_file_name)
def select_prediction_data(data, new_data=(), theoretical=True): """ Selects the data required to predict the CO2 emissions with CO2MPAS model. :param data: Output data. :type data: dict :param new_data: New data. :type new_data: dict :param theoretical: If false :type theoretical: bool :return: Data required to predict the CO2 emissions with CO2MPAS model. :rtype: dict """ ids = [ 'angle_slope', 'alternator_nominal_voltage', 'alternator_efficiency', 'battery_capacity', 'cycle_type', 'cycle_name', 'engine_capacity', 'engine_stroke', 'engine_thermostat_temperature', 'final_drive_efficiency', 'frontal_area', 'aerodynamic_drag_coefficient', 'fuel_type', 'ignition_type', 'gear_box_type', 'engine_max_power', 'engine_max_speed_at_max_power', 'rolling_resistance_coeff', 'time_cold_hot_transition', 'engine_idle_fuel_consumption', 'engine_type', 'engine_is_turbo', 'engine_fuel_lower_heating_value', 'has_start_stop', 'has_energy_recuperation', 'fuel_carbon_content_percentage', 'f0', 'f1', 'f2', 'vehicle_mass', 'full_load_speeds', 'plateau_acceleration', 'full_load_powers', 'fuel_saving_at_strategy', 'stand_still_torque_ratio', 'lockup_speed_ratio', 'change_gear_window_width', 'alternator_start_window_width', 'stop_velocity', 'min_time_engine_on_after_start', 'min_engine_on_speed', 'max_velocity_full_load_correction', 'is_hybrid', 'tyre_code', 'engine_has_cylinder_deactivation', 'active_cylinder_ratios', 'engine_has_variable_valve_actuation', 'has_torque_converter', 'has_gear_box_thermal_management', 'has_lean_burn', 'ki_factor', 'n_wheel_drive', 'has_periodically_regenerating_systems', 'has_selective_catalytic_reduction', 'has_exhausted_gas_recirculation' ] if not theoretical: ids += ['times', 'velocities', 'gears'] data = dsp_utl.selector(ids, data, allow_miss=True) if new_data: data = dsp_utl.combine_dicts(data, new_data) if 'gears' in data and 'gears' not in new_data: if data.get('gear_box_type', 0) == 'automatic' or \ len(data.get('velocities', ())) != len(data['gears']): data.pop('gears') return data
def _parse_plan_data(plans, match, sheet, sheet_name, re_params_name=_re_params_name): # noinspection PyBroadException xl_ref = '#%s!A1(R):._:R:"recurse"' data = lasso(xl_ref % sheet_name, sheet=sheet) try: data = pd.DataFrame(data[1:], columns=data[0]) except IndexError: return None if 'id' not in data: data['id'] = data.index + 1 data.set_index(['id'], inplace=True) data.dropna(how='all', inplace=True) data.dropna(axis=1, how='all', inplace=True) plan = pd.DataFrame() defaults = {'usage': 'input', 'stage': 'calibration'} match = dsp_utl.combine_dicts(defaults, match) for k, v in parse_values(data, match, re_params_name): k = k[-1] if k[-1] in ('base', 'defaults') else '.'.join(k[1:]) plan[k] = v plans.append(plan)
def parse_dsp_solution(solution): """ Parses the co2mpas model results. :param solution: Co2mpas model after dispatching. :type solution: co2mpas.dispatcher.Solution :return: Mapped outputs. :rtype: dict[dict] """ res = {} for k, v in solution.items(): dsp_utl.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v)) for k, v in list(dsp_utl.stack_nested_keys(res, depth=3)): n, k = k[:-1], k[-1] if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'): v = dsp_utl.selector(('co2_emission_value', ), v, allow_miss=True) if v: d = dsp_utl.get_nested_dicts(res, 'target', 'prediction') d[k] = dsp_utl.combine_dicts(v, d.get(k, {})) res['pipe'] = solution.pipe return res
def _process_vehicle(model, plot_workflow=False, **kw): inputs = {'plot_workflow': plot_workflow} res = model.dispatch(inputs=dsp_utl.combine_dicts(inputs, kw)) plot_model_workflow(model, **res) return res
def _comparison2df(comparison): res = {} it = co2_utl.stack_nested_keys(comparison, depth=3) keys = ['usage', 'cycle', 'param'] gen = [(dsp_utl.map_list(keys, *k), k, v) for k, v in it] for s, k, v in _yield_sorted_params(gen, keys=keys): l = co2_utl.get_nested_dicts(res, *k[:-1], default=list) l.append(dsp_utl.combine_dicts({'param_id': k[-1]}, v)) if res: return _dd2df(res, 'param_id', depth=2)
def _parse_base_data(res, match, sheet, sheet_name, re_params_name=_re_params_name): r = {} defaults = {'usage': 'input', 'stage': 'calibration'} if 'type' not in match: match['type'] = 'pa' if 'cycle' not in match else 'ts' match = dsp_utl.combine_dicts(defaults, match) if match['type'] == 'pa': xl_ref = '#%s!B2:C_:["pipe", ["dict", "recurse"]]' % sheet_name data = lasso(xl_ref, sheet=sheet) else: # noinspection PyBroadException try: xl_ref = '#%s!A2(R):.3:RD:["df", {"header": 0}]' % sheet_name data = lasso(xl_ref, sheet=sheet) except: return {} data.dropna(how='all', inplace=True) data.dropna(axis=1, how='all', inplace=True) mask = data.count(0) == len(data._get_axis(0)) # noinspection PyUnresolvedReferences drop = [k for k, v in mask.items() if not v] if drop: msg = 'Columns {} in {} sheet contains nan.\n ' \ 'Please correct the inputs!' raise ValueError(msg.format(drop, sheet_name)) for k, v in parse_values(data, match, re_params_name): co2_utl.get_nested_dicts(r, *k[:-1])[k[-1]] = v n = (match['scope'], 'target') if match['type'] == 'ts' and co2_utl.are_in_nested_dicts(r, *n): t = co2_utl.get_nested_dicts(r, *n) for k, v in co2_utl.stack_nested_keys(t, key=n, depth=2): if 'times' not in v: n = list(k + ('times', )) n[1] = match['usage'] if co2_utl.are_in_nested_dicts(r, *n): v['times'] = co2_utl.get_nested_dicts(r, *n) else: for i, j in co2_utl.stack_nested_keys(r, depth=4): if 'times' in j: v['times'] = j['times'] break co2_utl.combine_nested_dicts(r, depth=5, base=res)
def parse_excel_file(file_path, re_sheet_name=_re_input_sheet_name, re_params_name=_re_params_name): """ Reads cycle's data and simulation plans. :param file_path: Excel file path. :type file_path: str :param re_sheet_name: Regular expression to parse sheet names. :type re_sheet_name: regex.Regex :param re_params_name: Regular expression to parse param names. :type re_params_name: regex.Regex :return: A pandas DataFrame with cycle's time series. :rtype: dict, pandas.DataFrame """ excel_file = pd.ExcelFile(file_path) res, plans = {}, [] defaults = {'scope': 'base'} book = excel_file.book for sheet_name in excel_file.sheet_names: match = re_sheet_name.match(sheet_name) if not match: continue match = {k: v.lower() for k, v in match.groupdict().items() if v} match = dsp_utl.combine_dicts(defaults, match) sheet = _open_sheet_by_name_or_index(book, 'book', sheet_name) if match['scope'] == 'base': _parse_base_data(res, match, sheet, sheet_name, re_params_name) elif match['scope'] == 'plan': _parse_plan_data(plans, match, sheet, sheet_name, re_params_name) for k, v in co2_utl.stack_nested_keys(res.get('base', {}), depth=3): if k[0] != 'target': v['cycle_type'] = v.get('cycle_type', k[-1].split('_')[0]).upper() v['cycle_name'] = v.get('cycle_name', k[-1]).upper() res['plan'] = _finalize_plan(res, plans, file_path) return res
def _parse_values(data, default=None, where=''): default = default or {} for k, v in data.items(): match = _re_params_name.match(k) if k is not None else None if not match: log.warning("Parameter '%s' %s cannot be parsed!", k, where) continue elif _isempty(v): continue match = {i: j.lower() for i, j in match.groupdict().items() if j} for key in _parse_key(**dsp_utl.combine_dicts(default, match)): yield key, v
def _get_theoretical(profile): defaults = { 'cycle_type': 'WLTP', 'gear_box_type': 'manual', 'wltp_class': 'class3b', 'downscale_factor': 0 } profile = {k: v for k, v in profile.items() if v} profile = dsp_utl.combine_dicts(defaults, profile) profile['cycle_type'] = profile['cycle_type'].upper() profile['wltp_class'] = profile['wltp_class'].lower() profile['gear_box_type'] = profile['gear_box_type'].lower() from co2mpas.model.physical.cycle import cycle res = cycle().dispatch(inputs=profile, outputs=['times', 'velocities']) data = dsp_utl.selector(['times', 'velocities'], res, output_type='list') return pd.DataFrame(data).T
def parse_values(data, default=None, re_params_name=_re_params_name): default = default or {'scope': 'base'} if 'usage' not in default: default['usage'] = 'input' if 'cycle' not in default or default['cycle'] == 'all': default['cycle'] = ('nedc_h', 'nedc_l', 'wltp_p', 'wltp_h', 'wltp_l') elif default['cycle'] == 'wltp': default['cycle'] = ('wltp_h', 'wltp_l') elif default['cycle'] == 'nedc': default['cycle'] = ('nedc_h', 'nedc_l') else: default['cycle'] = default['cycle'].replace('-', '_') for k, v in data.items(): match = re_params_name.match(k) if k is not None else None if not match or _isempty(v): continue match = {i: j.lower() for i, j in match.groupdict().items() if j} if 'stage' not in match and match.get('usage', None) == 'target': match['stage'] = 'prediction' match = dsp_utl.combine_dicts(default, match) match['stage'] = match['stage'].replace(' ', '') if match['stage'] == 'input': match['stage'] = 'calibration' i = match['param'] if match['cycle'] == 'wltp': match['cycle'] = ('wltp_h', 'wltp_l') elif match['cycle'] == 'nedc': match['cycle'] = ('nedc_h', 'nedc_l') elif match['cycle'] == 'all': match['cycle'] = ('nedc_h', 'nedc_l', 'wltp_p', 'wltp_h', 'wltp_l') for c in stlp(match['cycle']): c = c.replace('-', '_') if c == 'wltp_p': stage = 'precondition' elif 'nedc' in c: stage = 'prediction' else: stage = match['stage'] yield (match['scope'], match['usage'], stage, c, i), v
def make_simulation_plan(plan, timestamp, variation, flag, model=None): model, summary = model or batch.vehicle_processing_model(), {} run_base = model.get_node('run_base')[0].dsp run_modes = tuple( run_base.get_sub_dsp_from_workflow( ('data', 'vehicle_name'), check_inputs=False, graph=run_base.dmap).data_nodes) + ('start_time', 'vehicle_name') var = json.dumps(variation, sort_keys=True) o_cache, o_folder = flag['overwrite_cache'], flag['output_folder'] modelconf = flag.get('modelconf', None) kw, bases = dsp_utl.combine_dicts(flag, {'run_base': True}), set() for (i, base_fpath, run), p in tqdm.tqdm(plan, disable=False): try: base = get_results(model, o_cache, base_fpath, timestamp, run, var, o_folder, modelconf) except KeyError: log.warn('Base model "%s" of variation "%s" cannot be parsed!', base_fpath, i) continue name = base['vehicle_name'] if 'summary' in base and name not in bases: batch._add2summary(summary, base['summary']) bases.add(name) name = '{}-{}'.format(name, i) new_base, o = define_new_inputs(p, base) inputs = batch.prepare_data(new_base, {}, base_fpath, o_cache, o_folder, timestamp, False, modelconf)[0] inputs.update(dsp_utl.selector(set(base).difference(run_modes), base)) inputs['vehicle_name'] = name inputs.update(kw) res = run_base.dispatch(inputs) batch.notify_result_listener(plan_listener, res) s = filter_summary(p, o, res.get('summary', {})) base_keys = { 'vehicle_name': (base_fpath, name, run), } batch._add2summary(summary, s, base_keys) return summary
def make_simulation_plan(plan, timestamp, output_folder, main_flags): model, summary = vehicle_processing_model(), {} run_modes = tuple(model.get_sub_dsp_from_workflow( ('validated_data', 'vehicle_name'), check_inputs=False, graph=model.dmap ).data_nodes) + ('start_time', 'vehicle_name') kw = { 'output_folder': output_folder, 'plan': False, 'timestamp': timestamp, } kw, bases = dsp_utl.combine_dicts(main_flags, kw), set() for (i, base_fpath, defaults_fpats), p in tqdm(plan, disable=False): base = get_results(model, base_fpath, **kw) name = base['vehicle_name'] if name not in bases: _add2summary(summary, base.get('summary', {})) bases.add(name) name = '{}-{}'.format(name, i) inputs = dsp_utl.selector(set(base).difference(run_modes), base) inputs['vehicle_name'] = name dsp_model = base['dsp_model'] outputs = dsp_model.data_output dfl = build_default_models(model, defaults_fpats, **kw) if dfl: dfl = {'data.prediction.models': dfl} outputs = co2_utl.combine_nested_dicts(dfl, outputs, depth=2) inputs['validated_data'] = define_new_inputs(p, outputs, dsp_model) inputs.update(kw) res = _process_vehicle(model, **inputs) s = filter_summary(p, res.get('summary', {})) base_keys = { 'vehicle_name': (defaults_fpats, base_fpath, name), } _add2summary(summary, s, base_keys) return summary
def re_sample_targets(data): res = {} for k, v in co2_utl.stack_nested_keys(data.get('target', {}), depth=2): if co2_utl.are_in_nested_dicts(data, 'output', *k): o = co2_utl.get_nested_dicts(data, 'output', *k) o = _split_by_data_format(o) t = dsp_utl.selector(o, _split_by_data_format(v), allow_miss=True) if 'times' not in t.get('ts', {}) or 'times' not in o['ts']: t.pop('ts', None) else: time_series = t['ts'] x, xp = o['ts']['times'], time_series.pop('times') if not _is_equal(x, xp): for i, fp in time_series.items(): time_series[i] = np.interp(x, xp, fp) v = dsp_utl.combine_dicts(*t.values()) co2_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v)) return res
def define_wltp_base_model(base_model): return dsp_utl.combine_dicts(_get_model_base(), base_model)
def prepare_data(raw_data, variation, input_file_name, overwrite_cache, output_folder, timestamp, type_approval_mode, modelconf): """ Prepare the data to be processed. :param raw_data: Raw data from the input file. :type raw_data: dict :param variation: Variations to be applied. :type variation: dict :param input_file_name: Input file name. :type input_file_name: str :param overwrite_cache: Overwrite saved cache? :type overwrite_cache: bool :param output_folder: Output folder. :type output_folder: str :param timestamp: Run timestamp. :type timestamp: str :param type_approval_mode: Is launched for TA? :type type_approval_mode: bool :param modelconf: Path of modelconf that has modified the defaults. :type modelconf: str :return: Prepared data. :rtype: dict """ has_plan = 'plan' in raw_data and (not raw_data['plan'].empty) match = { 'scope': 'plan' if has_plan else 'base', } r = {} sheets_factory = xleash.SheetsFactory() from co2mpas.io import check_xlasso for k, v in excel._parse_values(variation, match, "in variations"): if isinstance(v, str) and check_xlasso(v): v = xleash.lasso(v, sheets_factory, url_file=input_file_name) dsp_utl.get_nested_dicts(r, *k[:-1])[k[-1]] = v if 'plan' in r: if has_plan: plan = raw_data['plan'].copy() for k, v in dsp_utl.stack_nested_keys(r['plan'], 4): plan['.'.join(k)] = v else: gen = dsp_utl.stack_nested_keys(r['plan'], 4) plan = pd.DataFrame([{'.'.join(k): v for k, v in gen}]) excel._add_index_plan(plan, input_file_name) r['plan'] = plan has_plan = True if 'base' in r: r['base'] = dsp_utl.combine_nested_dicts(raw_data.get('base', {}), r['base'], depth=4) if 'flag' in r: r['flag'] = dsp_utl.combine_nested_dicts(raw_data.get('flag', {}), r['flag'], depth=1) data = dsp_utl.combine_dicts(raw_data, r) if type_approval_mode: variation, has_plan = {}, False if not schema._ta_mode(data): return {}, pd.DataFrame([]) flag = data.get('flag', {}).copy() if 'run_base' not in flag: flag['run_base'] = not has_plan if 'run_plan' not in flag: flag['run_plan'] = has_plan flag['type_approval_mode'] = type_approval_mode flag['output_folder'] = output_folder flag['overwrite_cache'] = overwrite_cache if modelconf: flag['modelconf'] = modelconf if timestamp is not None: flag['timestamp'] = timestamp flag = schema.validate_flags(flag) if flag is dsp_utl.NONE: return {}, pd.DataFrame([]) schema.check_data_version(flag) res = { 'flag': flag, 'variation': variation, 'input_file_name': input_file_name, } res = dsp_utl.combine_dicts(flag, res) base = dsp_utl.combine_dicts(res, {'data': data.get('base', {})}) plan = dsp_utl.combine_dicts(res, {'data': data.get('plan', pd.DataFrame([]))}) return base, plan
def combine_outputs(models): return dsp_utl.combine_dicts(*models.values())