def get_values(data, keys, tag=(), update=lambda k, v: v, base=None):
k = ('input', 'target', 'output')
data = dsp_utl.selector(k, data, allow_miss=True)
base = {} if base is None else base
for k, v in co2_utl.stack_nested_keys(data, depth=3):
k = k[::-1]
v = dsp_utl.selector(keys, v, allow_miss=True)
v = update(k, v)
if v:
k = tag + k
co2_utl.get_nested_dicts(base, *k, default=co2_utl.ret_v(v))
return base
def parse_dsp_solution(solution):
"""
Parses the co2mpas model results.
:param solution:
Co2mpas model after dispatching.
:type solution: co2mpas.dispatcher.Solution
:return:
Mapped outputs.
:rtype: dict[dict]
"""
res = {}
for k, v in solution.items():
dsp_utl.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v))
for k, v in list(dsp_utl.stack_nested_keys(res, depth=3)):
n, k = k[:-1], k[-1]
if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'):
v = dsp_utl.selector(('co2_emission_value', ), v, allow_miss=True)
if v:
d = dsp_utl.get_nested_dicts(res, 'target', 'prediction')
d[k] = dsp_utl.combine_dicts(v, d.get(k, {}))
res['pipe'] = solution.pipe
return res
def select_prediction_data(data, new_data=(), theoretical=True):
"""
Selects the data required to predict the CO2 emissions with CO2MPAS model.
:param data:
Output data.
:type data: dict
:param new_data:
New data.
:type new_data: dict
:param theoretical:
If false
:type theoretical: bool
:return:
Data required to predict the CO2 emissions with CO2MPAS model.
:rtype: dict
"""
ids = [
'angle_slope', 'alternator_nominal_voltage', 'alternator_efficiency',
'battery_capacity', 'cycle_type', 'cycle_name', 'engine_capacity',
'engine_stroke', 'engine_thermostat_temperature',
'final_drive_efficiency', 'frontal_area',
'aerodynamic_drag_coefficient', 'fuel_type', 'ignition_type',
'gear_box_type', 'engine_max_power', 'engine_max_speed_at_max_power',
'rolling_resistance_coeff', 'time_cold_hot_transition',
'engine_idle_fuel_consumption', 'engine_type', 'engine_is_turbo',
'engine_fuel_lower_heating_value', 'has_start_stop',
'has_energy_recuperation', 'fuel_carbon_content_percentage', 'f0',
'f1', 'f2', 'vehicle_mass', 'full_load_speeds', 'plateau_acceleration',
'full_load_powers', 'fuel_saving_at_strategy',
'stand_still_torque_ratio', 'lockup_speed_ratio',
'change_gear_window_width', 'alternator_start_window_width',
'stop_velocity', 'min_time_engine_on_after_start',
'min_engine_on_speed', 'max_velocity_full_load_correction',
'is_hybrid', 'tyre_code', 'engine_has_cylinder_deactivation',
'active_cylinder_ratios', 'engine_has_variable_valve_actuation',
'has_torque_converter', 'has_gear_box_thermal_management',
'has_lean_burn', 'ki_factor', 'n_wheel_drive',
'has_periodically_regenerating_systems',
'has_selective_catalytic_reduction', 'has_exhausted_gas_recirculation'
]
if not theoretical:
ids += ['times', 'velocities', 'gears']
data = dsp_utl.selector(ids, data, allow_miss=True)
if new_data:
data = dsp_utl.combine_dicts(data, new_data)
if 'gears' in data and 'gears' not in new_data:
if data.get('gear_box_type', 0) == 'automatic' or \
len(data.get('velocities', ())) != len(data['gears']):
data.pop('gears')
return data
def get_selection(data):
res = []
n = ('data', 'calibration', 'model_scores', 'selections')
if co2_utl.are_in_nested_dicts(data, *n):
for k, v in sorted(co2_utl.get_nested_dicts(data, *n).items()):
d = dsp_utl.selector(('from', 'status'), v['best'])
d['model_id'] = k
res.append(d)
return res
def test_files(self):
mydir = osp.dirname(__file__)
if SEATBELT_FILE and osp.isfile(SEATBELT_FILE):
res_file = SEATBELT_FILE
else:
tmpdir = tempfile.gettempdir()
res_file = osp.join(tmpdir, 'co2mpas_seatbelt_demos.dill')
log.info(
"\n OVERWRITE_SEATBELT: %s \n"
" RUN_INPUT_FOLDER: %s \n"
" RUN_ALL_FILES: %s \n"
" SEATBELT_FILE: %s", OVERWRITE_SEATBELT, RUN_INPUT_FOLDER,
RUN_ALL_FILES, res_file)
if not OVERWRITE_SEATBELT and osp.isfile(res_file):
old_results = dsp_utl.load_dispatcher(res_file)
log.info("Old results loaded!")
else:
old_results = None
path = RUN_INPUT_FOLDER or osp.join(mydir, '..', 'co2mpas', 'demos')
file = (path if (RUN_ALL_FILES or RUN_INPUT_FOLDER) else osp.join(
path, 'co2mpas_demo-0.xlsx'))
model = vehicle_processing_model()
results = []
inp_files = file_finder([file])
if not inp_files:
raise AssertionError("DataCheck found no input-files in %r!" %
file)
for fpath in inp_files:
fname = osp.splitext(osp.basename(fpath))[0]
log.info('Processing: %s', fname)
inputs = {
'input_file_name': fpath,
'variation': {
'flag.only_summary': True
}
}
r = model.dispatch(inputs=inputs)
r = dsp_utl.selector(['report', 'summary'], r['solution'])
r.get('report', {}).pop('pipe', None)
results.append(sorted(dsp_utl.stack_nested_keys(r)))
if not OVERWRITE_SEATBELT and osp.isfile(res_file):
log.info('Comparing...')
self._check_results(results, old_results)
else:
os.environ["OVERWRITE_SEATBELT"] = '0'
dsp_utl.save_dispatcher(results, res_file)
log.info('Overwritten seat belt %r.', res_file)
def _check_sign_currents(data, *args):
c = ('battery_currents', 'alternator_currents')
try:
a = dsp_utl.selector(c, data, output_type='list')
s = check_sign_currents(*a)
if not all(s):
s = ' and '.join([k for k, v in zip(c, s) if not v])
msg = "Probably '{}' have the wrong sign!".format(s)
return c, msg
except KeyError: # `c` is not in `data`.
pass
def _check_initial_temperature(data, *args):
t = ('initial_temperature', 'engine_coolant_temperatures',
'engine_speeds_out', 'idle_engine_speed_median')
try:
a = dsp_utl.selector(t, data, output_type='list')
if not check_initial_temperature(*a):
msg = "Initial engine temperature outside permissible limits " \
"according to GTR!"
return t, msg
except KeyError: # `t` is not in `data`.
pass
def split_prediction_models(
scores, calibrated_models, input_models, cycle_ids=()):
sbm, model_sel, par = {}, {}, {}
for (k, c), v in dsp_utl.stack_nested_keys(scores, depth=2):
r = dsp_utl.selector(['models'], v, allow_miss=True)
for m in r.get('models', ()):
dsp_utl.get_nested_dicts(par, m, 'calibration')[c] = c
r.update(v.get('score', {}))
dsp_utl.get_nested_dicts(sbm, k, c, default=co2_utl.ret_v(r))
r = dsp_utl.selector(['success'], r, allow_miss=True)
r = dsp_utl.map_dict({'success': 'status'}, r, {'from': c})
dsp_utl.get_nested_dicts(model_sel, k, 'calibration')[c] = r
p = {i: dict.fromkeys(input_models, 'input') for i in cycle_ids}
models = {i: input_models.copy() for i in cycle_ids}
for k, n in sorted(calibrated_models.items()):
d = n.get(dsp_utl.NONE, (None, True, {}))
for i in cycle_ids:
c, s, m = n.get(i, d)
if m:
s = {'from': c, 'status': s}
dsp_utl.get_nested_dicts(model_sel, k, 'prediction')[i] = s
models[i].update(m)
p[i].update(dict.fromkeys(m, c))
for k, v in dsp_utl.stack_nested_keys(p, ('prediction',), depth=2):
dsp_utl.get_nested_dicts(par, k[-1], *k[:-1], default=co2_utl.ret_v(v))
s = {
'param_selections': par,
'model_selections': model_sel,
'score_by_model': sbm,
'scores': scores
}
return (s,) + tuple(models.get(k, {}) for k in cycle_ids)
def hard_validation(data):
c = ('battery_currents', 'alternator_currents')
try:
a = dsp_utl.selector(c, data, output_type='list')
s = check_sign_currents(*a)
if not all(s):
s = ' and '.join([k for k, v in zip(c, s) if not v])
msg = "Probably '{}' have the wrong sign!".format(s)
yield c, msg
except KeyError: # `c` is not in `data`.
pass
t = ('initial_temperature', 'engine_coolant_temperatures',
'engine_speeds_out', 'idle_engine_speed_median')
try:
a = dsp_utl.selector(t, data, output_type='list')
if not check_initial_temperature(*a):
msg = "Initial engine temperature outside permissible limits " \
"according to GTR!"
yield t, msg
except KeyError: # `t` is not in `data`.
pass
def _make_summarydf(
nested_dict, index=None, depth=0, add_units=True,
parts=()):
df = _dd2df(nested_dict, index=index, depth=depth)
p = _param_orders()
p = dsp_utl.selector(parts + ('param',), p, output_type='list')
gen = partial(zip_longest, p[:-1], fillvalue=p[-1])
c = sorted(df.columns, key=lambda x: [_match_part(m, v) for m, v in gen(x)])
df = df.reindex_axis(c, axis=1, copy=False)
if add_units:
c = _add_units(c)
df.columns = pd.MultiIndex.from_tuples(c)
return df
def _get_theoretical(profile):
defaults = {
'cycle_type': 'WLTP',
'gear_box_type': 'manual',
'wltp_class': 'class3b',
'downscale_factor': 0
}
profile = {k: v for k, v in profile.items() if v}
profile = dsp_utl.combine_dicts(defaults, profile)
profile['cycle_type'] = profile['cycle_type'].upper()
profile['wltp_class'] = profile['wltp_class'].lower()
profile['gear_box_type'] = profile['gear_box_type'].lower()
from co2mpas.model.physical.cycle import cycle
res = cycle().dispatch(inputs=profile, outputs=['times', 'velocities'])
data = dsp_utl.selector(['times', 'velocities'], res, output_type='list')
return pd.DataFrame(data).T
def get_dfl(wltp_base_model):
"""
Gets default values from wltp base model.
:param wltp_base_model:
WLTP base model params.
:type wltp_base_model: dict
:return:
Default values from wltp base model.
:rtype: list
"""
params = wltp_base_model['params']
keys = 'driver_mass', 'resistance_coeffs_regression_curves', 'wltc_data'
return dsp_utl.selector(keys, params, output_type='list')
def define_new_inputs(data, base, dsp_model):
remove = []
for k, v in co2_utl.stack_nested_keys(data, depth=2):
if v is dsp_utl.EMPTY:
remove.append(k)
dsp = dsp_model.get_sub_dsp_from_workflow(data, check_inputs=False)
n = set(base) - set(dsp.data_nodes)
n.update(data)
inp = dsp_utl.selector(n, base, allow_miss=True)
d = co2_utl.combine_nested_dicts(inp, data, depth=2)
for n, k in remove:
co2_utl.get_nested_dicts(d, n).pop(k)
return d
def select_initial_friction_params(co2_params_initial_guess):
"""
Selects initial guess of friction params l & l2 for the calculation of
the motoring curve.
:param co2_params_initial_guess:
Initial guess of CO2 emission model params.
:type co2_params_initial_guess: lmfit.Parameters
:return:
Initial guess of friction params l & l2.
:rtype: float, float
"""
params = co2_params_initial_guess.valuesdict()
return dsp_utl.selector(('l', 'l2'), params, output_type='list')
def combine_scores(scores):
scores = {k[:-9]: v for k, v in scores.items() if v}
if not scores:
return {}
s = {}
for (k, c), v in co2_utl.stack_nested_keys(scores, depth=2):
r = {'models': v['models']} if 'models' in v else {}
r.update(v.get('score', {}))
co2_utl.get_nested_dicts(s, k, c, default=co2_utl.ret_v(r))
if not co2_utl.are_in_nested_dicts(s, k, 'best'):
keys = {'models': 'selected_models', 'success': 'status'}
best = dsp_utl.map_dict(keys, dsp_utl.selector(keys, r))
best['from'] = c
co2_utl.get_nested_dicts(s, k, 'best', default=co2_utl.ret_v(best))
return {'selections': s, 'scores': scores}
def make_simulation_plan(plan, timestamp, variation, flag, model=None):
model, summary = model or batch.vehicle_processing_model(), {}
run_base = model.get_node('run_base')[0].dsp
run_modes = tuple(
run_base.get_sub_dsp_from_workflow(
('data', 'vehicle_name'), check_inputs=False,
graph=run_base.dmap).data_nodes) + ('start_time', 'vehicle_name')
var = json.dumps(variation, sort_keys=True)
o_cache, o_folder = flag['overwrite_cache'], flag['output_folder']
modelconf = flag.get('modelconf', None)
kw, bases = dsp_utl.combine_dicts(flag, {'run_base': True}), set()
for (i, base_fpath, run), p in tqdm.tqdm(plan, disable=False):
try:
base = get_results(model, o_cache, base_fpath, timestamp, run, var,
o_folder, modelconf)
except KeyError:
log.warn('Base model "%s" of variation "%s" cannot be parsed!',
base_fpath, i)
continue
name = base['vehicle_name']
if 'summary' in base and name not in bases:
batch._add2summary(summary, base['summary'])
bases.add(name)
name = '{}-{}'.format(name, i)
new_base, o = define_new_inputs(p, base)
inputs = batch.prepare_data(new_base, {}, base_fpath, o_cache,
o_folder, timestamp, False, modelconf)[0]
inputs.update(dsp_utl.selector(set(base).difference(run_modes), base))
inputs['vehicle_name'] = name
inputs.update(kw)
res = run_base.dispatch(inputs)
batch.notify_result_listener(plan_listener, res)
s = filter_summary(p, o, res.get('summary', {}))
base_keys = {
'vehicle_name': (base_fpath, name, run),
}
batch._add2summary(summary, s, base_keys)
return summary
def make_simulation_plan(plan, timestamp, output_folder, main_flags):
model, summary = vehicle_processing_model(), {}
run_modes = tuple(model.get_sub_dsp_from_workflow(
('validated_data', 'vehicle_name'), check_inputs=False, graph=model.dmap
).data_nodes) + ('start_time', 'vehicle_name')
kw = {
'output_folder': output_folder,
'plan': False,
'timestamp': timestamp,
}
kw, bases = dsp_utl.combine_dicts(main_flags, kw), set()
for (i, base_fpath, defaults_fpats), p in tqdm(plan, disable=False):
base = get_results(model, base_fpath, **kw)
name = base['vehicle_name']
if name not in bases:
_add2summary(summary, base.get('summary', {}))
bases.add(name)
name = '{}-{}'.format(name, i)
inputs = dsp_utl.selector(set(base).difference(run_modes), base)
inputs['vehicle_name'] = name
dsp_model = base['dsp_model']
outputs = dsp_model.data_output
dfl = build_default_models(model, defaults_fpats, **kw)
if dfl:
dfl = {'data.prediction.models': dfl}
outputs = co2_utl.combine_nested_dicts(dfl, outputs, depth=2)
inputs['validated_data'] = define_new_inputs(p, outputs, dsp_model)
inputs.update(kw)
res = _process_vehicle(model, **inputs)
s = filter_summary(p, res.get('summary', {}))
base_keys = {
'vehicle_name': (defaults_fpats, base_fpath, name),
}
_add2summary(summary, s, base_keys)
return summary
def re_sample_targets(data):
res = {}
for k, v in co2_utl.stack_nested_keys(data.get('target', {}), depth=2):
if co2_utl.are_in_nested_dicts(data, 'output', *k):
o = co2_utl.get_nested_dicts(data, 'output', *k)
o = _split_by_data_format(o)
t = dsp_utl.selector(o, _split_by_data_format(v), allow_miss=True)
if 'times' not in t.get('ts', {}) or 'times' not in o['ts']:
t.pop('ts', None)
else:
time_series = t['ts']
x, xp = o['ts']['times'], time_series.pop('times')
if not _is_equal(x, xp):
for i, fp in time_series.items():
time_series[i] = np.interp(x, xp, fp)
v = dsp_utl.combine_dicts(*t.values())
co2_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def _error(name, data_id, data_out, setting):
d = dsp.Dispatcher(
name='%s-%s error vs %s' % (name, data_id, data_out),
description='Calculates the error of calibrated model of a reference.',
)
default_settings = {
'inputs_map': {},
'targets': [],
'metrics_inputs': {},
'up_limit': None,
'dn_limit': None
}
default_settings.update(setting)
it = dsp_utl.selector(['up_limit', 'dn_limit'], default_settings).items()
for k, v in it:
if v is not None:
default_settings[k] = dsp_utl.map_list(setting['targets'], *v)
d.add_function(
function_id='select_inputs',
function=dsp_utl.map_dict,
inputs=['inputs_map', 'data'],
outputs=['inputs<0>']
)
d.add_function(
function_id='select_inputs',
function=functools.partial(dsp_utl.selector, allow_miss=True),
inputs=['inputs', 'inputs<0>'],
outputs=['inputs<1>']
)
d.add_function(
function=dsp_utl.combine_dicts,
inputs=['calibrated_models', 'inputs<1>'],
outputs=['prediction_inputs']
)
d.add_function(
function_id='select_targets',
function=functools.partial(dsp_utl.selector, allow_miss=True),
inputs=['targets', 'data'],
outputs=['references']
)
d.add_function(
function=functools.partial(
default_settings.pop('dsp', lambda x: x), {}
),
inputs=['prediction_inputs', 'calibrated_models'],
outputs=['results']
)
d.add_function(
function_id='select_outputs',
function=select_outputs,
inputs=['outputs', 'targets', 'results'],
outputs=['predictions']
)
d.add_function(
function_id='select_metrics_inputs',
function=functools.partial(dsp_utl.selector, allow_miss=True),
inputs=['metrics_inputs', 'data'],
outputs=['metrics_args']
)
d.add_function(
function=make_metrics,
inputs=['metrics', 'references', 'predictions', 'metrics_args'],
outputs=['errors']
)
d.add_function(
function=check_limits,
inputs=['errors', 'up_limit', 'dn_limit'],
outputs=['status']
)
for k, v in default_settings.items():
d.add_data(k, v)
func = dsp_utl.SubDispatch(
dsp=d,
outputs=['errors', 'status'],
output_type='list'
)
return func
def tyre_models_selector(models_ids, data):
models = dsp_utl.selector(models_ids, data, allow_miss=True)
if 'tyre_dynamic_rolling_coefficient' in models:
models.pop('r_dynamic', None)
return models
def select_outputs(outputs, targets, results):
results = dsp_utl.selector(outputs, results, allow_miss=True)
results = dsp_utl.map_dict(dict(zip(outputs, targets)), results)
it = ((k, results[k]) for k in targets if k in results)
return collections.OrderedDict(it)
def get_dfl(wltp_base_model):
params = wltp_base_model['params']
keys = 'driver_mass', 'resistance_coeffs_regression_curves', 'wltc_data'
return dsp_utl.selector(keys, params, output_type='list')
