def define_sub_model(d, inputs, outputs, models, **kwargs):
missing = set(outputs).difference(d.nodes)
if missing:
outputs = set(outputs).difference(missing)
if inputs is not None:
inputs = set(inputs).union(models)
return dsp_utl.SubDispatch(d.shrink_dsp(inputs, outputs))
def _selector(name, data_in, data_out, setting):
d = dsp.Dispatcher(
name='%s selector' % name,
description='Select the calibrated %s.' % name,
)
errors, setting = [], setting or {}
_sort_models = setting.pop('sort_models', sort_models)
if 'weights' in setting:
_weights = dsp_utl.map_list(setting['targets'], *setting.pop('weights'))
else:
_weights = None
_get_best_model = functools.partial(
setting.pop('get_best_model', get_best_model),
models_wo_err=setting.pop('models_wo_err', None),
selector_id=d.name
)
d.add_data(
data_id='selector_settings',
default_value={})
node_ids = ['error_settings', 'best_model_settings']
d.add_function(
function=functools.partial(define_selector_settings, node_ids=node_ids),
inputs=['selector_settings'],
outputs=node_ids
)
for i in data_in:
e = 'error/%s' % i
errors.append(e)
d.add_function(
function=_errors(name, i, data_out, setting),
inputs=['error_settings', i] + [k for k in data_out if k != i],
outputs=[e]
)
d.add_function(
function_id='sort_models',
function=functools.partial(_sort_models, weights=_weights),
inputs=errors,
outputs=['rank']
)
d.add_function(
function_id='get_best_model',
function=_get_best_model,
inputs=['rank', 'best_model_settings'],
outputs=['model', 'errors']
)
return dsp_utl.SubDispatch(d, outputs=['model', 'errors'],
output_type='list')
def run_plan():
"""
Defines the plan model.
.. dispatcher:: d
>>> d = run_plan()
:return:
The plan model.
:rtype: Dispatcher
"""
d = dsp.Dispatcher(name='run_plan',
description='Processes a vehicle plan.')
d.add_data(data_id='engineering_mode', default_value=False)
d.add_data(data_id='use_selector', default_value=False)
d.add_data(data_id='soft_validation', default_value=False)
d.add_function(function=dsp_utl.add_args(schema.validate_plan),
inputs=[
'run_plan', 'data', 'engineering_mode',
'soft_validation', 'use_selector'
],
outputs=['validated_plan'],
input_domain=check_first_arg)
d.add_function(function=default_start_time, outputs=['start_time'])
d.add_function(function=default_timestamp,
inputs=['start_time'],
outputs=['timestamp'])
from .plan import make_simulation_plan
d.add_function(function=make_simulation_plan,
inputs=['validated_plan', 'timestamp', 'variation', 'flag'],
outputs=['summary'])
return dsp_utl.SubDispatch(d)
def co2_params_selector(
name='co2_params', data_in=('wltp_h', 'wltp_l'),
data_out=('wltp_h', 'wltp_l'), setting=None):
"""
Defines the co2_params model selector.
.. dispatcher:: d
>>> d = co2_params_selector()
:return:
The co2_params model selector.
:rtype: SubDispatch
"""
from . import _selector
d = _selector(name, data_in + ('ALL',), data_out, setting).dsp
n = d.get_node('sort_models', node_attr=None)[0]
errors, sort_models = n['inputs'], n['function']
d.dmap.remove_node('sort_models')
d.add_function(
function=sort_models,
inputs=errors[:-1],
outputs=['rank<0>']
)
d.add_function(
function=functools.partial(calibrate_co2_params_all, data_id=data_in),
inputs=['rank<0>'] + errors[:-1],
outputs=['ALL']
)
d.add_function(
function=functools.partial(co2_sort_models, **sort_models.keywords),
inputs=['rank<0>'] + [errors[-1]],
outputs=['rank']
)
return dsp_utl.SubDispatch(d, outputs=['model', 'errors'],
output_type='list')
def _yield_folder_files_results(start_time,
input_files,
output_folder,
overwrite_cache=False,
model=None,
variation=None,
type_approval_mode=False,
modelconf=None):
model = model or vehicle_processing_model()
kw = {
'output_folder': output_folder,
'overwrite_cache': overwrite_cache,
'modelconf': modelconf,
'timestamp': start_time.strftime('%Y%m%d_%H%M%S'),
'variation': variation or {},
'type_approval_mode': type_approval_mode
}
_process_vehicle = dsp_utl.SubDispatch(model)
for fpath in _custom_tqdm(input_files, bar_format='{l_bar}{bar}{r_bar}'):
yield _process_vehicle({'input_file_name': fpath}, kw)
def _error(name, data_id, data_out, setting):
d = dsp.Dispatcher(
name='%s-%s error vs %s' % (name, data_id, data_out),
description='Calculates the error of calibrated model of a reference.',
)
default_settings = {
'inputs_map': {},
'targets': [],
'metrics_inputs': {},
'up_limit': None,
'dn_limit': None
}
default_settings.update(setting)
it = dsp_utl.selector(['up_limit', 'dn_limit'], default_settings).items()
for k, v in it:
if v is not None:
default_settings[k] = dsp_utl.map_list(setting['targets'], *v)
d.add_function(
function_id='select_inputs',
function=dsp_utl.map_dict,
inputs=['inputs_map', 'data'],
outputs=['inputs<0>']
)
d.add_function(
function_id='select_inputs',
function=functools.partial(dsp_utl.selector, allow_miss=True),
inputs=['inputs', 'inputs<0>'],
outputs=['inputs<1>']
)
d.add_function(
function=dsp_utl.combine_dicts,
inputs=['calibrated_models', 'inputs<1>'],
outputs=['prediction_inputs']
)
d.add_function(
function_id='select_targets',
function=functools.partial(dsp_utl.selector, allow_miss=True),
inputs=['targets', 'data'],
outputs=['references']
)
d.add_function(
function=functools.partial(
default_settings.pop('dsp', lambda x: x), {}
),
inputs=['prediction_inputs', 'calibrated_models'],
outputs=['results']
)
d.add_function(
function_id='select_outputs',
function=select_outputs,
inputs=['outputs', 'targets', 'results'],
outputs=['predictions']
)
d.add_function(
function_id='select_metrics_inputs',
function=functools.partial(dsp_utl.selector, allow_miss=True),
inputs=['metrics_inputs', 'data'],
outputs=['metrics_args']
)
d.add_function(
function=make_metrics,
inputs=['metrics', 'references', 'predictions', 'metrics_args'],
outputs=['errors']
)
d.add_function(
function=check_limits,
inputs=['errors', 'up_limit', 'dn_limit'],
outputs=['status']
)
for k, v in default_settings.items():
d.add_data(k, v)
func = dsp_utl.SubDispatch(
dsp=d,
outputs=['errors', 'status'],
output_type='list'
)
return func
def sub_models():
models = {}
from ..physical.engine.thermal import thermal
models['engine_coolant_temperature_model'] = {
'd': thermal(),
'models': ['engine_temperature_regression_model',
'max_engine_coolant_temperature'],
'inputs': ['times', 'accelerations', 'final_drive_powers_in',
'engine_speeds_out_hot', 'initial_engine_temperature'],
'outputs': ['engine_coolant_temperatures'],
'targets': ['engine_coolant_temperatures'],
'metrics': [sk_met.mean_absolute_error],
'up_limit': [3],
}
from ..physical.engine.start_stop import start_stop
models['start_stop_model'] = {
'd': start_stop(),
'models': ['start_stop_model', 'use_basic_start_stop'],
'inputs': ['times', 'velocities', 'accelerations',
'engine_coolant_temperatures', 'state_of_charges',
'gears', 'correct_start_stop_with_gears',
'start_stop_activation_time',
'min_time_engine_on_after_start', 'has_start_stop'],
'outputs': ['on_engine', 'engine_starts'],
'targets': ['on_engine', 'engine_starts'],
'metrics': [sk_met.accuracy_score] * 2,
'weights': [-1, -1],
'dn_limit': [0.7] * 2,
}
from ..physical import physical
models['engine_speed_model'] = {
'd': physical(),
'select_models': tyre_models_selector,
'models': ['final_drive_ratio', 'gear_box_ratios',
'idle_engine_speed_median', 'idle_engine_speed_std',
'CVT', 'max_speed_velocity_ratio',
'tyre_dynamic_rolling_coefficient'],
'inputs': ['velocities', 'gears', 'times', 'on_engine', 'gear_box_type',
'accelerations', 'final_drive_powers_in',
'engine_thermostat_temperature', 'tyre_code'],
'outputs': ['engine_speeds_out_hot'],
'targets': ['engine_speeds_out'],
'metrics_inputs': ['times', 'velocities', 'gear_shifts', 'on_engine',
'stop_velocity'],
'metrics': [metric_engine_speed_model],
'up_limit': [40],
}
from ..physical.engine import calculate_engine_speeds_out
from ..physical.engine.cold_start import cold_start
d = cold_start()
d.add_function(
function=calculate_engine_speeds_out,
inputs=['on_engine', 'idle_engine_speed', 'engine_speeds_out_hot',
'cold_start_speeds_delta'],
outputs=['engine_speeds_out']
)
models['engine_cold_start_speed_model'] = {
'd': d,
'models': ['cold_start_speed_model'],
'inputs': ['engine_speeds_out_hot', 'engine_coolant_temperatures',
'on_engine', 'idle_engine_speed'],
'outputs': ['engine_speeds_out'],
'targets': ['engine_speeds_out'],
'metrics_inputs': ['cold_start_speeds_phases'],
'metrics': [metric_engine_cold_start_speed_model],
'up_limit': [100],
}
from ..physical.clutch_tc import clutch_torque_converter
d = clutch_torque_converter()
d.add_function(
function=calculate_engine_speeds_out,
inputs=['on_engine', 'idle_engine_speed', 'engine_speeds_out_hot',
'clutch_tc_speeds_delta'],
outputs=['engine_speeds_out']
)
models['clutch_torque_converter_model'] = {
'd': d,
'models': ['clutch_window', 'clutch_model', 'torque_converter_model'],
'inputs': ['gear_box_speeds_in', 'on_engine', 'idle_engine_speed',
'gear_box_type', 'gears', 'accelerations', 'times',
'gear_shifts', 'engine_speeds_out_hot', 'velocities',
'lock_up_tc_limits', 'has_torque_converter'],
'define_sub_model': lambda d, **kwargs: dsp_utl.SubDispatch(d),
'outputs': ['engine_speeds_out'],
'targets': ['engine_speeds_out'],
'metrics_inputs': ['on_engine'],
'metrics': [metric_clutch_torque_converter_model],
'up_limit': [100],
}
from ..physical.engine.co2_emission import co2_emission
from .co2_params import co2_params_selector
models['co2_params'] = {
'd': co2_emission(),
'model_selector': co2_params_selector,
'models': ['co2_params_calibrated', 'calibration_status',
'initial_friction_params', 'engine_idle_fuel_consumption'],
'inputs': ['co2_emissions_model'],
'outputs': ['co2_emissions', 'calibration_status'],
'targets': ['identified_co2_emissions', 'calibration_status'],
'metrics': [sk_met.mean_absolute_error, metric_calibration_status],
'up_limit': [0.5, None],
'weights': [1, None]
}
from ..physical.electrics import electrics
models['alternator_model'] = {
'd': electrics(),
'models': ['alternator_status_model', 'alternator_nominal_voltage',
'alternator_current_model', 'max_battery_charging_current',
'start_demand', 'electric_load', 'alternator_nominal_power',
'alternator_efficiency', 'alternator_initialization_time'],
'inputs': [
'battery_capacity', 'alternator_nominal_voltage',
'initial_state_of_charge', 'times', 'gear_box_powers_in',
'on_engine', 'engine_starts', 'accelerations'],
'outputs': ['alternator_currents', 'battery_currents',
'state_of_charges', 'alternator_statuses'],
'targets': ['alternator_currents', 'battery_currents',
'state_of_charges', 'alternator_statuses'],
'metrics': [sk_met.mean_absolute_error] * 3 + [sk_met.accuracy_score],
'up_limit': [60, 60, None, None],
'weights': [1, 1, 0, 0]
}
from ..physical.gear_box.at_gear import at_gear
at_pred_inputs = [
'engine_max_power', 'engine_max_speed_at_max_power',
'idle_engine_speed', 'full_load_curve', 'road_loads', 'vehicle_mass',
'accelerations', 'motive_powers', 'engine_speeds_out',
'engine_coolant_temperatures', 'time_cold_hot_transition', 'times',
'use_dt_gear_shifting', 'specific_gear_shifting',
'velocity_speed_ratios', 'velocities', 'MVL', 'fuel_saving_at_strategy',
'change_gear_window_width', 'stop_velocity', 'plateau_acceleration',
'max_velocity_full_load_correction', 'cycle_type'
]
models['at_model'] = {
'd': at_gear(),
'select_models': functools.partial(
at_models_selector, at_gear(), at_pred_inputs
),
'models': ['MVL', 'CMV', 'CMV_Cold_Hot', 'DT_VA', 'DT_VAT', 'DT_VAP',
'DT_VATP', 'GSPV', 'GSPV_Cold_Hot',
'specific_gear_shifting', 'change_gear_window_width',
'max_velocity_full_load_correction', 'plateau_acceleration'],
'inputs': at_pred_inputs,
'define_sub_model': lambda d, **kwargs: dsp_utl.SubDispatch(d),
'outputs': ['gears', 'max_gear'],
'targets': ['gears', 'max_gear'],
'metrics': [sk_met.accuracy_score, None],
'weights': [-1, 0]
}
return models
def run_base():
"""
Defines the vehicle-processing model.
.. dispatcher:: d
>>> d = run_base()
:return:
The vehicle-processing model.
:rtype: Dispatcher
"""
d = dsp.Dispatcher(
name='run_base',
description='Processes a vehicle from the file path to the write of its'
' outputs.')
d.add_data(data_id='engineering_mode', default_value=False)
d.add_data(data_id='output_folder', default_value='.')
d.add_data(data_id='use_selector', default_value=False)
d.add_data(data_id='soft_validation', default_value=False)
d.add_function(function=dsp_utl.add_args(schema.validate_base),
inputs=[
'run_base', 'data', 'engineering_mode',
'soft_validation', 'use_selector'
],
outputs=['validated_base'],
input_domain=check_first_arg,
weight=10)
d.add_data(data_id='only_summary', default_value=False)
d.add_function(function=default_vehicle_name,
inputs=['input_file_name'],
outputs=['vehicle_name'])
d.add_function(function=default_start_time, outputs=['start_time'])
d.add_function(function=default_timestamp,
inputs=['start_time'],
outputs=['timestamp'])
d.add_function(function=default_output_file_name,
inputs=['output_folder', 'vehicle_name', 'timestamp'],
outputs=['output_file_name'])
from .model import model
d.add_function(
function=dsp_utl.SubDispatch(model()),
inputs=['validated_base'],
outputs=['dsp_solution'],
)
d.add_function(function=parse_dsp_solution,
inputs=['dsp_solution'],
outputs=['output_data'])
from .report import report
d.add_function(
function=report(),
inputs=['output_data', 'vehicle_name'],
outputs=['report', 'summary'],
)
d.add_function(function=get_template_file_name,
inputs=['output_template', 'input_file_name'],
outputs=['template_file_name'])
d.add_data(data_id='output_template',
default_value=_get_co2mpas_output_template_fpath(),
initial_dist=10)
from .io import write_outputs
d.add_function(function=dsp_utl.add_args(write_outputs()),
inputs=[
'only_summary', 'output_file_name',
'template_file_name', 'report', 'start_time', 'flag'
],
outputs=[dsp_utl.SINK],
input_domain=lambda *args: not args[0])
d.add_function(
function=dsp_utl.add_args(plot_model_workflow),
inputs=['plot_workflow', 'output_file_name', 'vehicle_name'],
outputs=[dsp_utl.PLOT],
weight=30,
input_domain=check_first_arg)
return dsp_utl.SubDispatch(d)
def model():
"""
Defines the CO2MPAS model.
.. dispatcher:: d
>>> d = model()
:return:
The CO2MPAS model.
:rtype: co2mpas.dispatcher.Dispatcher
"""
from .physical import physical
d = dsp.Dispatcher(
name='CO2MPAS model',
description='Calibrates the models with WLTP data and predicts NEDC '
'cycle.')
############################################################################
# PRECONDITIONING CYCLE
############################################################################
d.add_data(
data_id='input.precondition.wltp_p',
description='Dictionary that has all inputs of the calibration cycle.',
default_value={})
d.add_function(
function_id='calculate_precondition_output',
function=dsp_utl.SubDispatch(physical()),
inputs=['input.precondition.wltp_p'],
outputs=['output.precondition.wltp_p'],
description='Wraps all functions needed to calculate the precondition '
'outputs.')
############################################################################
# WLTP - HIGH CYCLE
############################################################################
d.add_data(data_id='input.calibration.wltp_h', default_value={})
d.add_function(
function=select_calibration_data,
inputs=['input.calibration.wltp_h', 'output.precondition.wltp_p'],
outputs=['data.calibration.wltp_h'],
)
d.add_function(
function_id='calibrate_with_wltp_h',
function=dsp_utl.SubDispatch(physical()),
inputs=['data.calibration.wltp_h'],
outputs=['output.calibration.wltp_h'],
description='Wraps all functions needed to calibrate the models to '
'predict light-vehicles\' CO2 emissions.')
d.add_data(data_id='input.prediction.wltp_h', default_value={})
d.add_function(
function=select_prediction_data,
inputs=['output.calibration.wltp_h', 'input.prediction.wltp_h'],
outputs=['data.prediction.wltp_h'])
d.add_function(
function_id='predict_wltp_h',
function=dsp_utl.SubDispatch(physical()),
inputs=['data.prediction.models_wltp_h', 'data.prediction.wltp_h'],
outputs=['output.prediction.wltp_h'],
description='Wraps all functions needed to predict CO2 emissions.')
############################################################################
# WLTP - LOW CYCLE
############################################################################
d.add_data(data_id='input.calibration.wltp_l', default_value={})
d.add_function(
function=select_calibration_data,
inputs=['input.calibration.wltp_l', 'output.precondition.wltp_p'],
outputs=['data.calibration.wltp_l'],
)
d.add_function(
function_id='calibrate_with_wltp_l',
function=dsp_utl.SubDispatch(physical()),
inputs=['data.calibration.wltp_l'],
outputs=['output.calibration.wltp_l'],
description='Wraps all functions needed to calibrate the models to '
'predict light-vehicles\' CO2 emissions.')
d.add_data(data_id='input.prediction.wltp_l', default_value={})
d.add_function(
function=select_prediction_data,
inputs=['output.calibration.wltp_l', 'input.prediction.wltp_l'],
outputs=['data.prediction.wltp_l'])
d.add_function(
function_id='predict_wltp_l',
function=dsp_utl.SubDispatch(physical()),
inputs=['data.prediction.models_wltp_l', 'data.prediction.wltp_l'],
outputs=['output.prediction.wltp_l'],
description='Wraps all functions needed to predict CO2 emissions.')
############################################################################
# MODEL SELECTOR
############################################################################
from .selector import selector
pred_cyl_ids = ('nedc_h', 'nedc_l', 'wltp_h', 'wltp_l')
sel = selector('wltp_h', 'wltp_l', pred_cyl_ids=pred_cyl_ids)
d.add_data(data_id='config.selector.all', default_value={})
d.add_data(data_id='input.prediction.models', default_value={})
d.add_function(function_id='extract_calibrated_models',
function=sel,
inputs=[
'config.selector.all', 'input.prediction.models',
'output.calibration.wltp_h', 'output.calibration.wltp_l'
],
outputs=['data.calibration.model_scores'] +
['data.prediction.models_%s' % k for k in pred_cyl_ids])
############################################################################
# NEDC - HIGH CYCLE
############################################################################
d.add_function(
function_id='predict_nedc_h',
function=dsp_utl.SubDispatch(physical()),
inputs=['data.prediction.models_nedc_h', 'input.prediction.nedc_h'],
outputs=['output.prediction.nedc_h'],
)
############################################################################
# NEDC - LOW CYCLE
############################################################################
d.add_function(
function_id='predict_nedc_l',
function=dsp_utl.SubDispatch(physical()),
inputs=['data.prediction.models_nedc_l', 'input.prediction.nedc_l'],
outputs=['output.prediction.nedc_l'],
)
return d
def vehicle_processing_model():
"""
Defines the vehicle-processing model.
.. dispatcher:: dsp
>>> dsp = vehicle_processing_model()
:return:
The vehicle-processing model.
:rtype: Dispatcher
"""
dsp = Dispatcher(
name='CO2MPAS vehicle_processing_model',
description='Processes a vehicle from the file path to the write of its'
' outputs.')
dsp.add_data(data_id='overwrite_cache', default_value=False)
dsp.add_data(data_id='soft_validation', default_value=False)
dsp.add_data(data_id='with_output_file', default_value=False)
dsp.add_function(function=default_vehicle_name,
inputs=['input_file_name'],
outputs=['vehicle_name'])
dsp.add_function(function=default_start_time, outputs=['start_time'])
dsp.add_function(function=default_timestamp,
inputs=['start_time'],
outputs=['timestamp'])
dsp.add_function(function=dsp_utl.add_args(default_output_file_name),
inputs=[
'with_output_file', 'output_folder', 'vehicle_name',
'timestamp'
],
outputs=['output_file_name'],
input_domain=lambda *args: args[0])
from .io import load_inputs, write_outputs
dsp.add_dispatcher(dsp=load_inputs(),
inputs={
'input_file_name': 'input_file_name',
'overwrite_cache': 'overwrite_cache',
'soft_validation': 'soft_validation'
},
outputs={
'validated_data': 'validated_data',
'validated_plan': 'validated_plan'
})
from .model import model
dsp.add_function(function=dsp_utl.add_args(
dsp_utl.SubDispatch(model(), output_type='dsp')),
inputs=['plan', 'validated_data'],
outputs=['dsp_model'],
input_domain=lambda *args: not args[0])
dsp.add_function(function=parse_dsp_model,
inputs=['dsp_model'],
outputs=['output_data'])
from .report import report
dsp.add_function(
function=report(),
inputs=['output_data', 'vehicle_name'],
outputs=['report', 'summary'],
)
dsp.add_function(function=dsp_utl.bypass,
inputs=['output_data'],
outputs=['report'],
weight=1)
dsp.add_function(function=get_template_file_name,
inputs=['output_template', 'input_file_name'],
outputs=['template_file_name'])
dsp.add_data(data_id='output_template', default_value='', initial_dist=10)
main_flags = ('template_file_name', 'overwrite_cache', 'soft_validation',
'with_output_file', 'plot_workflow')
dsp.add_function(function=partial(dsp_utl.map_list, main_flags),
inputs=main_flags,
outputs=['main_flags'])
dsp.add_function(function=write_outputs(),
inputs=[
'output_file_name', 'template_file_name', 'report',
'start_time', 'main_flags'
],
outputs=[dsp_utl.SINK],
input_domain=check_first_arg)
dsp.add_function(function_id='has_plan',
function=check_first_arg,
inputs=['validated_plan'],
outputs=['plan'])
from .plan import make_simulation_plan
dsp.add_function(function=dsp_utl.add_args(make_simulation_plan),
inputs=[
'plan', 'validated_plan', 'timestamp',
'output_folder', 'main_flags'
],
outputs=['summary'],
input_domain=check_first_arg)
return dsp