def define_sub_model(d, inputs, outputs, models, **kwargs):
missing = set(outputs).difference(d.nodes)
if missing:
outputs = set(outputs).difference(missing)
if inputs is not None:
inputs = set(inputs).union(models)
return sh.SubDispatch(d.shrink_dsp(inputs, outputs))
def setUp(self):
ss_dsp = sh.Dispatcher()
fun = lambda a: (a + 1, 5, a - 1)
dom = lambda kw: True
c = '|!"£$%&/()=?^*+éè[]#¶ù§çò@:;-_.,<>'
ss_dsp.add_function(function=fun,
inputs=['a'],
outputs=['b', sh.SINK, c],
input_domain=dom,
weight=1)
def raise_fun(a):
raise ValueError('Error')
ss_dsp.add_function(function=raise_fun, inputs=['a'], outputs=['b'])
sdspfunc = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction', ['a'],
['b', c])
sdsppipe = sh.SubDispatchPipe(ss_dsp, 'SubDispatchPipe', ['a'],
['b', c])
spipe = sh.DispatchPipe(ss_dsp, 'DispatchPipe', ['a'], [c])
sdsp = sh.SubDispatch(ss_dsp, ['b', c], output_type='list')
s_dsp = sh.Dispatcher()
s_dsp.add_function(None, sdspfunc, ['a'], ['b', 'c'], weight=2)
s_dsp.add_function(None,
sdsppipe, ['a'], ['b', 'c'],
out_weight={'c': 5})
s_dsp.add_function(None, spipe, ['a'], ['b'], weight=2)
s_dsp.add_function('SubDispatch', sdsp, ['d'], ['e', 'f'])
dsp = sh.Dispatcher()
dsp.add_data('A', default_value=[0] * 1000)
dsp.add_data('D', default_value={'a': 3})
dsp.add_dispatcher(dsp=s_dsp,
inputs={
'A': 'a',
'D': 'd'
},
outputs={
'b': 'B',
'c': 'C',
'e': 'E',
'f': 'F'
},
inp_weight={'A': 3})
self.sol = dsp.dispatch()
self.dsp = dsp
dsp = sh.Dispatcher()
dsp.add_function(function=sh.bypass, inputs=['a'], outputs=[sh.PLOT])
self.dsp_plot = dsp
def setUp(self):
sub_dsp = sh.Dispatcher()
def fun(a):
return a + 1, a - 1
sub_dsp.add_function('fun', fun, ['a'], ['b', 'c'])
dispatch = sh.SubDispatch(sub_dsp, ['a', 'b', 'c'])
dispatch_dict = sh.SubDispatch(sub_dsp, ['c'], output_type='dict')
dispatch_list = sh.SubDispatch(sub_dsp, ['a', 'c'], output_type='list')
dispatch_val = sh.SubDispatch(sub_dsp, ['c'], output_type='list')
dsp = sh.Dispatcher()
dsp.add_function('dispatch', dispatch, ['d'], ['e'])
dsp.add_function('dispatch_dict', dispatch_dict, ['d'], ['f'])
dsp.add_function('dispatch_list', dispatch_list, ['d'], ['g'])
dsp.add_function('dispatch_val', dispatch_val, ['d'], ['h'])
self.dsp = dsp
def run_synchronisation():
# Dedicated logging for this run
fileh = logging.FileHandler(co2wui_fpath("sync", "logfile.txt"), "w")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
frmt = "%(asctime)-15s:%(levelname)5.5s:%(name)s:%(message)s"
logging.basicConfig(level=logging.INFO, format=frmt)
logger.addHandler(fileh)
# Input file
inputs = listdir_inputs("sync", "input")
input_file = str(inputs[0])
# Output file
output_name = os.path.splitext(os.path.basename(input_file))[0]
output_file = co2wui_fpath("sync", "output", output_name + ".sync.xlsx")
# Remove old output files
previous = listdir_outputs("sync", "output")
for f in previous:
os.remove(str(f))
# Arguments
kwargs = {
"x_label": request.form.get("x_label")
if request.form.get("x_label")
else "times",
"y_label": request.form.get("y_label")
if request.form.get("y_label")
else "velocities",
"interpolation_method": request.form.get("interpolation_method"),
"header": request.form.get("header"),
"reference_name": request.form.get("reference_name")
if request.form.get("reference_name")
else "theoretical",
}
kwargs = {k: v for k, v in kwargs.items() if v}
try:
# Dispatcher
_process = sh.SubDispatch(syncing.dsp, ["written"], output_type="value")
ret = _process(
dict(input_fpath=input_file, output_fpath=str(output_file), **kwargs)
)
fileh.close()
logger.removeHandler(fileh)
return "OK"
except Exception as e:
logger.error(_("Synchronisation failed: ") + str(e))
fileh.close()
logger.removeHandler(fileh)
return "KO"
def run_synchronisation():
# Dedicated logging for this run
fileh = logging.FileHandler("sync/logfile.txt", "w")
logger = logging.getLogger()
logger.setLevel(logging.INFO)
frmt = "%(asctime)-15s:%(levelname)5.5s:%(name)s:%(message)s"
logging.basicConfig(level=logging.INFO, format=frmt)
logger.addHandler(fileh)
# Input and output files
inputs = [
f for f in listdir_inputs("sync/input")
if osp.isfile(osp.join("sync/input", f))
]
input_file = osp.join("sync", "input", inputs[0])
output_file = osp.join("sync", "output", "datasync.sync.xlsx")
# Arguments
kwargs = {
"x_label":
request.form.get("x_label")
if request.form.get("x_label") else "times",
"y_label":
request.form.get("y_label")
if request.form.get("y_label") else "velocities",
"interpolation_method":
request.form.get("interpolation_method"),
"header":
request.form.get("header"),
"reference_name":
request.form.get("reference_name")
if request.form.get("reference_name") else "theoretical",
}
kwargs = {k: v for k, v in kwargs.items() if v}
try:
# Dispatcher
_process = sh.SubDispatch(syncing.dsp, ["written"],
output_type="value")
ret = _process(
dict(input_fpath=input_file,
output_fpath=output_file,
**kwargs))
fileh.close()
logger.removeHandler(fileh)
return "OK"
except Exception as e:
logger.error(_("Synchronisation failed: ") + str(e))
fileh.close()
logger.removeHandler(fileh)
return "KO"
def _mdl_error(mdl):
dsp = sh.BlueDispatcher(
name=mdl.name,
description='Calculates the error of calibrated model of a reference.',
)
dsp.add_data('inputs_map', getattr(mdl, 'inputs_map', {}))
dsp.add_function(function_id='select_inputs',
function=sh.map_dict,
inputs=['inputs_map', 'data'],
outputs=['inputs<0>'])
dsp.add_data('inputs', getattr(mdl, 'inputs', []))
dsp.add_function(function_id='select_inputs',
function=_select_data,
inputs=['inputs', 'inputs<0>'],
outputs=['inputs<1>'])
dsp.add_function(function=sh.combine_dicts,
inputs=['calibrated_models', 'inputs<1>'],
outputs=['prediction_inputs'])
dsp.add_data('targets', getattr(mdl, 'targets', []))
dsp.add_function(function_id='select_targets',
function=_select_data,
inputs=['targets', 'data'],
outputs=['references'])
dsp.add_function(function=sh.SubDispatch(mdl.dsp),
inputs=['prediction_inputs', 'calibrated_models'],
outputs=['results'])
dsp.add_data('outputs', getattr(mdl, 'outputs', []))
dsp.add_func(select_predictions, outputs=['predictions'])
dsp.add_data('metrics_inputs', getattr(mdl, 'metrics_inputs', {}))
dsp.add_function(function_id='select_metrics_inputs',
function=_select_data,
inputs=['metrics_inputs', 'data'],
outputs=['metrics_kwargs'])
dsp.add_data('metrics', getattr(mdl, 'metrics', {}))
dsp.add_func(calculate_errors, outputs=['errors'])
dsp.add_data('up_limit', getattr(mdl, 'up_limit', None))
dsp.add_data('dn_limit', getattr(mdl, 'dn_limit', None))
dsp.add_func(calculate_calibration_status,
inputs_kwargs=True,
outputs=['status'])
return dsp
def _selector(name, data_in, data_out, setting):
d = sh.Dispatcher(name='%s selector' % name,
description='Select the calibrated %s.' % name)
errors, setting = [], setting or {}
_sort_models = setting.pop('sort_models', sort_models)
if 'weights' in setting:
_weights = sh.map_list(setting.get('names', setting['targets']),
*setting.pop('weights'))
else:
_weights = None
_get_best_model = functools.partial(setting.pop('get_best_model',
get_best_model),
models_wo_err=setting.pop(
'models_wo_err', None),
selector_id=d.name)
d.add_data(data_id='selector_settings', default_value={})
node_ids = ['error_settings', 'best_model_settings']
d.add_function(function=functools.partial(define_selector_settings,
node_ids=node_ids),
inputs=['selector_settings'],
outputs=node_ids)
for i in data_in:
e = 'error/%s' % i
errors.append(e)
d.add_function(function=_errors(name, i, data_out, setting),
inputs=['error_settings', i] +
[k for k in data_out if k != i],
outputs=[e])
d.add_function(function_id='sort_models',
function=functools.partial(_sort_models, weights=_weights),
inputs=errors,
outputs=['rank'])
d.add_function(function_id='get_best_model',
function=_get_best_model,
inputs=['rank', 'best_model_settings'],
outputs=['model', 'errors'])
return sh.SubDispatch(d, outputs=['model', 'errors'], output_type='list')
def setUp(self):
import functools
ss_dsp = sh.Dispatcher()
def fun(a, c):
"""
:param a:
Nice a.
:type a: float
:param c:
Nice c.
:type c: float
:return:
Something.
:rtype: tuple
"""
return a + 1, c, a - 1
ss_dsp.add_function('fun', fun, ['a', 'e'], ['b', 'c', 'd'])
ss_dsp_func = sh.DispatchPipe(ss_dsp, 'func', ['e', 'a'],
['c', 'd', 'b'])
sub_disfun = sh.add_args(functools.partial(ss_dsp_func, 5))
s_dsp = sh.Dispatcher()
s_dsp.add_data('a', 1)
s_dsp.add_data('d', 4)
s_dsp.add_function('sub_dispatch', sub_disfun, ['d', 'a'],
['b', 'c', sh.SINK])
dispatch = sh.SubDispatch(s_dsp, ['b', 'c', 'a'], output_type='list')
self.dsp = dsp = sh.Dispatcher()
dsp.add_data('input', default_value={'a': 3})
dsp.add_function('dispatch',
dispatch, ['input'], [sh.SINK, 'h', 'i'],
inp_weight={'input': 4},
out_weight={
'h': 3,
'i': 6
})
dsp.add_function('fun', lambda: None, None, ['j'])
dsp.add_dispatcher(s_dsp,
inputs=('a', ),
outputs=('b', 'c'),
include_defaults=True)
def run_synchronisation():
# Dedicated logging for this run
fileh = logging.FileHandler("sync/logfile.txt", "w")
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
fileh.setFormatter(formatter)
log = logging.getLogger()
log.setLevel(logging.DEBUG)
for hdlr in log.handlers[:]:
log.removeHandler(hdlr)
log.addHandler(fileh)
# Input and output files
input_file = "sync/input/datasync.xlsx"
output_file = "sync/output/datasync.sync.xlsx"
# Arguments
kwargs = {
"x_label":
request.form.get("x_label")
if request.form.get("x_label") else 'times',
"y_label":
request.form.get("y_label")
if request.form.get("y_label") else 'velocities',
"interpolation_method":
request.form.get("interpolation_method"),
"header":
request.form.get("header"),
"reference_name":
request.form.get("reference_name")
if request.form.get("reference_name") else 'theoretical',
}
kwargs = {k: v for k, v in kwargs.items() if v}
try:
# Dispatcher
_process = sh.SubDispatch(syncing.dsp, ['written'],
output_type='value')
ret = _process(
dict(input_fpath=input_file,
output_fpath=output_file,
**kwargs))
return 'OK'
except Exception as e:
return 'KO'
def mdl_selector(name, package=None):
"""
Defines a model selector for a specific model.
:param name:
Model name.
:type name: str
:param package:
Package name.
:type package: str
:return:
Model selector.
:rtype: schedula.utils.blue.BlueDispatcher
"""
import importlib
mdl = importlib.import_module(name, package)
dsp = sh.BlueDispatcher(name='%s selector' % mdl.name,
description='Select the calibrated %s.' % mdl.name)
err_func = sh.SubDispatch(_mdl_error(mdl),
outputs=['errors', 'status'],
output_type='list')
for data_id in calibration_cycles:
mdl_err, mdl_err_name = _mdl_errors(mdl, data_id, err_func)
dsp.add_func(sh.SubDispatchFunction(
mdl_err,
function_id=mdl_err_name,
inputs=[data_id] + [k
for k in calibration_cycles if k != data_id]),
outputs=['error/%s' % data_id])
# noinspection PyTypeChecker
dsp.add_function(function=functools.partial(sort_models,
weights=getattr(
mdl, 'weights', None)),
inputs=list(map('error/{}'.format, calibration_cycles)),
outputs=['rank'])
dsp.add_func(format_score, outputs=['score'])
dsp.add_func(functools.partial(select_best_model,
selector_id='%s selector' % mdl.name),
outputs=['model'])
return dsp
def setUp(self):
ss_dsp = sh.Dispatcher()
def fun(a, c):
"""
:param a:
Nice a.
:type a: float
:param c:
Nice c.
:type c: float
:return:
Something.
:rtype: tuple
"""
return a + 1, c, a - 1
ss_dsp.add_function('fun', fun, ['a', 'e'], ['b', 'c', 'd'])
ss_dsp_func = sh.SubDispatchFunction(
ss_dsp, 'func', ['e', 'a'], ['c', 'd', 'b'])
sub_disfun = sh.partial(ss_dsp_func, 5)
s_dsp = sh.Dispatcher()
s_dsp.add_function(
'sub_dispatch', sub_disfun, ['a'], ['b', 'c', sh.SINK]
)
dispatch = sh.SubDispatch(s_dsp, ['b', 'c', 'a'], output_type='list')
dsp = sh.Dispatcher()
dsp.add_data('input', default_value={'a': 3})
dsp.add_function('dispatch', dispatch, ['input'], [sh.SINK, 'h', 'i'])
self.sol = dsp.dispatch(inputs={'f': 'new'})
self.dsp = dsp
self.fun = fun
self.sub_dispatch = sub_disfun
self.s_dsp = s_dsp
self.ss_dsp = ss_dsp
self.ss_dsp_func = ss_dsp_func
def run_plan():
"""
Defines the plan model.
.. dispatcher:: d
>>> d = run_plan()
:return:
The plan model.
:rtype: Dispatcher
"""
d = sh.Dispatcher(name='run_plan', description='Processes a vehicle plan.')
d.add_data(data_id='engineering_mode', default_value=False)
d.add_data(data_id='use_selector', default_value=False)
d.add_data(data_id='soft_validation', default_value=False)
d.add_function(function=sh.add_args(schema.validate_plan),
inputs=[
'run_plan', 'data', 'engineering_mode',
'soft_validation', 'use_selector'
],
outputs=['validated_plan'],
input_domain=check_first_arg)
d.add_function(function=default_start_time, outputs=['start_time'])
d.add_function(function=default_timestamp,
inputs=['start_time'],
outputs=['timestamp'])
from .plan import make_simulation_plan
d.add_function(function=make_simulation_plan,
inputs=['validated_plan', 'timestamp', 'variation', 'flag'],
outputs=['summary'])
return sh.SubDispatch(d)
def _yield_folder_files_results(start_time,
input_files,
output_folder,
overwrite_cache=False,
model=None,
variation=None,
type_approval_mode=False,
modelconf=None):
model = model or vehicle_processing_model()
kw = {
'output_folder': output_folder,
'overwrite_cache': overwrite_cache,
'modelconf': modelconf,
'timestamp': start_time.strftime('%Y%m%d_%H%M%S'),
'variation': variation or {},
'type_approval_mode': type_approval_mode
}
_process_vehicle = sh.SubDispatch(model)
for fpath in _custom_tqdm(input_files, bar_format='{l_bar}{bar}{r_bar}'):
yield _process_vehicle({'input_file_name': fpath}, kw)
def co2_params_selector(name='co2_params',
data_in=('wltp_h', 'wltp_l'),
data_out=('wltp_h', 'wltp_l'),
setting=None):
"""
Defines the co2_params model selector.
.. dispatcher:: d
>>> d = co2_params_selector()
:return:
The co2_params model selector.
:rtype: SubDispatch
"""
from . import _selector
d = _selector(name, data_in + ('ALL', ), data_out, setting).dsp
n = d.get_node('sort_models', node_attr=None)[0]
errors, sort_models = n['inputs'], n['function']
d.dmap.remove_node('sort_models')
d.add_function(function=sort_models,
inputs=errors[:-1],
outputs=['rank<0>'])
d.add_data('enable_all', True)
d.add_function(function=functools.partial(calibrate_co2_params_all,
data_id=data_in),
inputs=['enable_all', 'rank<0>'] + errors[:-1],
outputs=['ALL'])
d.add_function(function=functools.partial(co2_sort_models,
**sort_models.keywords),
inputs=['rank<0>'] + [errors[-1]],
outputs=['rank'])
return sh.SubDispatch(d, outputs=['model', 'errors'], output_type='list')
def setUp(self):
ss_dsp = sh.Dispatcher(name='Ciao.')
def fun(b, c, d=0):
"""
Fun description.
:param b:
Second param.
:param int c:
Third param.
:return:
Out param.
:rtype: float
"""
return b + c + d
def dom(a, *args):
"""
:param a:
First param.
"""
return a
ss_dsp.add_function(function=sh.add_args(fun),
inputs=['a', 'b', 'c', 'd'],
outputs=['e'],
input_domain=dom)
ss_dsp.add_function(function=sh.bypass, inputs=['e'], outputs=['f'])
ss_dsp.add_function(function=sh.replicate_value,
inputs=['f'],
outputs=['g', 'h'])
sdspfunc = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction',
['a', 'b', 'c', 'd'], ['g', 'h'])
sdsp = sh.SubDispatch(ss_dsp, ['e', 'f'], output_type='list')
def fun1():
""""""
return
sdspfunci = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction',
['a', 'b', 'c', 'd'])
s_dsp = sh.Dispatcher(name='Sub-Dispatcher')
s_dsp.add_function('3', sdspfunc, ['a', 'b', 'c', 'd'], ['g', 'h'])
s_dsp.add_function('8', sdspfunci, ['a', 'b', 'c', 'd'], ['o'])
s_dsp.add_function('2', sdsp, ['I'], ['e', 'f'])
s_dsp.add_function('4', max, ['e', 'f', 'l'], ['i'])
s_dsp.add_function('5', inputs=['i'], outputs=['n'])
s_dsp.add_function('6', fun1)
s_dsp.add_function('7',
sh.bypass,
inputs=['p'],
input_domain=sh.add_args(lambda p: None))
s_dsp.add_data('i', description='max.')
self.dsp = dsp = sh.Dispatcher()
dsp.add_dispatcher(
dsp_id='1',
dsp=s_dsp,
inputs=('n', 'd', 'I', 'l', {
'm': ('b', 'c'),
('a', 'a1'): 'a'
}),
outputs=['g', 'i', {
'f': ('h', 'f')
}, {
('e', 'h'): ('e', 'e1')
}])
class _Logger(logging.Logger):
def setLevel(self, level):
super(_Logger, self).setLevel(level)
frmt = "%(asctime)-15s:%(levelname)5.5s:%(name)s:%(message)s"
logging.basicConfig(level=level, format=frmt)
rlog = logging.getLogger()
# because `basicConfig()` does not reconfig root-logger when re-invoked.
rlog.level = level
logging.captureWarnings(True)
logger = _Logger('cli')
click_log.basic_config(logger)
_process = sh.SubDispatch(syncing.dsp, ['written'], output_type='value')
@click.group(
'syncing', context_settings=dict(help_option_names=['-h', '--help'])
)
@click.version_option(__version__)
def cli():
"""
SYNCING command line tool.
"""
@cli.command('template', short_help='Generates sample template file.')
@click.argument(
'output-file', default='template.xlsx', required=False,
from files.VBSsetup import VBSsetup
return VBSsetup
process.add_data(data_id='key_mapping',
default_value={},
description='Dictionary renaming keys.')
process.add_function(function_id='parse_data',
function=sh.map_dict,
inputs=['key_mapping', 'raw_data'],
outputs=['data'],
description='Rename the raw data names.')
process.add_function(function_id='execute_VBS',
function=sh.SubDispatch(model()),
inputs=['data'],
outputs=['outputs'],
description='Executes the computational model.')
@sh.add_function(process)
def save_outputs(outputs, output_fpath):
"""
Save model outputs in an Excel file.
:param outputs:
Model outputs.
:type outputs: dict
:param output_fpath:
def run_base():
"""
Defines the vehicle-processing model.
.. dispatcher:: d
>>> d = run_base()
:return:
The vehicle-processing model.
:rtype: Dispatcher
"""
d = sh.Dispatcher(
name='run_base',
description='Processes a vehicle from the file path to the write of its'
' outputs.')
d.add_data(data_id='engineering_mode', default_value=False)
d.add_data(data_id='output_folder', default_value='.')
d.add_data(data_id='use_selector', default_value=False)
d.add_data(data_id='soft_validation', default_value=False)
d.add_function(function=schema.validate_meta,
inputs=['meta', 'soft_validation'],
outputs=['validated_meta'])
d.add_function(function=sh.add_args(schema.validate_base),
inputs=[
'run_base', 'data', 'engineering_mode',
'soft_validation', 'use_selector'
],
outputs=['validated_base'],
input_domain=check_first_arg,
weight=10)
d.add_data(data_id='only_summary', default_value=False)
d.add_function(function=default_vehicle_name,
inputs=['input_file_name'],
outputs=['vehicle_name'])
d.add_function(function=default_start_time, outputs=['start_time'])
d.add_function(function=default_timestamp,
inputs=['start_time'],
outputs=['timestamp'])
d.add_function(function=default_output_file_name,
inputs=['output_folder', 'vehicle_name', 'timestamp'],
outputs=['output_file_name'])
from .model import model
d.add_function(function=sh.add_args(sh.SubDispatch(model())),
inputs=['validated_meta', 'validated_base'],
outputs=['dsp_solution'])
d.add_function(function=parse_dsp_solution,
inputs=['dsp_solution'],
outputs=['output_data'])
from .report import report
d.add_function(
function=report(),
inputs=['output_data', 'vehicle_name'],
outputs=['report', 'summary'],
)
from .io.ta import write_ta_output
from .conf import defaults
dfl = defaults.io_constants_dfl
d.add_data('encrypt_inputs', dfl.ENCRYPT_INPUTS)
d.add_data('encryption_keys', dfl.ENCRYPTION_KEYS_PATH)
d.add_function(function=sh.add_args(write_ta_output()),
inputs=[
'type_approval_mode', 'encrypt_inputs',
'encryption_keys', 'vehicle_family_id', 'start_time',
'timestamp', 'data', 'meta', 'report', 'output_folder'
],
outputs=['output_ta_file'],
input_domain=check_first_arg)
d.add_function(function=get_template_file_name,
inputs=['output_template', 'input_file_name'],
outputs=['template_file_name'])
d.add_data(data_id='output_template',
default_value=_get_co2mpas_output_template_fpath(),
initial_dist=10)
from .io import write_outputs
d.add_function(function=sh.add_args(write_outputs()),
inputs=[
'only_summary', 'output_file_name',
'template_file_name', 'report', 'start_time', 'flag'
],
outputs=[sh.SINK],
input_domain=lambda *args: not args[0])
d.add_function(
function=sh.add_args(plot_model_workflow),
inputs=['plot_workflow', 'output_file_name', 'vehicle_name'],
outputs=[sh.PLOT],
weight=30,
input_domain=check_first_arg)
return sh.SubDispatch(d)
def setUp(self):
ss_dsp = sh.Dispatcher(name='ss_dsp')
fun = lambda a: (a + 1, 5, a - 1)
dom = lambda kw: True
ss_dsp.add_function(function=fun,
inputs=['a'],
outputs=['b', 'd', 'c'],
input_domain=dom,
weight=1)
sdspfunc = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction', ['a'],
['b', 'c'])
sdsppipe = sh.SubDispatchPipe(ss_dsp, 'SubDispatchPipe', ['a'],
['b', 'c'])
sdsp = sh.SubDispatch(ss_dsp, ['b', 'c'], output_type='list')
s_dsp = sh.Dispatcher(name='s_dsp')
s_dsp.add_function(None, sdspfunc, ['a'], ['b', 'c'])
s_dsp.add_function(None, sdsppipe, ['a'], ['g'])
s_dsp.add_function('SubDispatch', sdsp, ['d'], ['e', 'f'])
dsp = sh.Dispatcher(name='model')
dsp.add_data('A', default_value=0)
dsp.add_data('D', default_value={'a': 3})
dsp.add_dispatcher(dsp=s_dsp,
inputs={
'A': 'a',
'D': 'd'
},
outputs={
'b': 'B',
'c': 'C',
'e': 'E',
'f': 'F',
'g': 'G'
},
inp_weight={'A': 3})
self.dsp = dsp
self.sol = sol = dsp.dispatch()
sites = set()
webmap = dsp.web(node_data=('+set_value', ), run=True, sites=sites)
self.site = sites.pop()
self.url = '%s/' % self.site.url
rules = webmap.rules()
self.io = io = []
for rule in rules.values():
n = rule.split('/')[1:]
if not n:
continue
s, k = sol.get_node(*n, node_attr='sol')
k = k[-1]
try:
v = s.workflow.node[k]
except KeyError:
continue
if 'results' not in v:
continue
inputs = s._wf_pred[k] # List of the function's arguments.
inputs = sh.bypass(
*[inputs[k]['value'] for k in s.nodes[k]['inputs']])
io.append((rule, inputs, v['results']))
self.sol1 = sol = dsp.dispatch({'A': 1})
self.io1 = io = []
for rule in rules.values():
n = rule.split('/')[1:]
if not n:
continue
s, k = sol.get_node(*n, node_attr='sol')
k = k[-1]
try:
v = s.workflow.node[k]
except KeyError:
continue
if 'results' not in v:
continue
inputs = s._wf_pred[k] # List of the function's arguments.
inputs = sh.bypass(
*[inputs[k]['value'] for k in s.nodes[k]['inputs']])
io.append((rule, inputs, v['results']))
MODELS = [
v.name
for v in pkgutil.iter_modules([osp.join(osp.dirname(__file__), 'models')])
if not v.name.startswith('_')
]
dsp.add_data('enable_selector',
False,
filters=[lambda x: {
'enable_selector': x
}])
for name in MODELS:
dsp.add_function(function=sh.SubDispatch(mdl_selector(
'.models.%s' % name, __name__),
outputs=['model', 'score'],
output_type='list'),
function_id='%s selector' % name,
inputs=['CO2MPAS_results', 'enable_selector'],
outputs=['models', 'scores'])
def merge_model_selection(adict):
"""
Merge models or scores selection.
:param adict:
Models or scores selection.
:type adict: dict
:return:
def _error(name, setting):
d = sh.Dispatcher(
name=name,
description='Calculates the error of calibrated model of a reference.',
)
default_settings = {
'inputs_map': {},
'targets': [],
'metrics_inputs': {},
'up_limit': None,
'dn_limit': None
}
default_settings.update(setting)
default_settings['names'] = default_settings.get(
'names', default_settings['targets'])
it = sh.selector(['up_limit', 'dn_limit'], default_settings).items()
for k, v in it:
if v is not None:
default_settings[k] = sh.map_list(setting['names'], *v)
d.add_function(function_id='select_inputs',
function=sh.map_dict,
inputs=['inputs_map', 'data'],
outputs=['inputs<0>'])
d.add_function(function_id='select_inputs',
function=functools.partial(sh.selector, allow_miss=True),
inputs=['inputs', 'inputs<0>'],
outputs=['inputs<1>'])
d.add_function(function=sh.combine_dicts,
inputs=['calibrated_models', 'inputs<1>'],
outputs=['prediction_inputs'])
d.add_function(function=select_targets,
inputs=['names', 'targets', 'data'],
outputs=['references'])
d.add_function(function=functools.partial(
default_settings.pop('dsp', lambda x: x), {}),
inputs=['prediction_inputs', 'calibrated_models'],
outputs=['results'])
d.add_function(function=select_outputs,
inputs=['names', 'outputs', 'targets', 'results'],
outputs=['predictions'])
d.add_function(function_id='select_metrics_inputs',
function=functools.partial(sh.selector, allow_miss=True),
inputs=['metrics_inputs', 'data'],
outputs=['metrics_args'])
d.add_function(
function=make_metrics,
inputs=['metrics', 'references', 'predictions', 'metrics_args'],
outputs=['errors'])
d.add_function(function=check_limits,
inputs=['errors', 'up_limit', 'dn_limit'],
outputs=['status'])
for k, v in default_settings.items():
d.add_data(k, v)
func = sh.SubDispatch(dsp=d,
outputs=['errors', 'status'],
output_type='list')
return func
"""
import matplotlib.pyplot as plt
import schedula as sh
from co2mpas_driver.load import dsp as _load
from co2mpas_driver.model import dsp as _model
from co2mpas_driver.plot import dsp as _plot
dsp = sh.Dispatcher()
dsp.add_dispatcher(
dsp=_load,
inputs=['inputs', 'vehicle_id', 'db_path', 'input_path'],
outputs=['data']
)
dsp.add_function(
function=sh.SubDispatch(_model),
inputs=['data'],
outputs=['outputs']
)
dsp.add_function(
function_id='write',
inputs=['output_path', 'outputs']
)
dsp.add_function(
function=sh.SubDispatch(_plot),
inputs=['output_plot_folder', 'outputs']
)
if __name__ == '__main__':
def model():
"""
Defines the CO2MPAS model.
.. dispatcher:: d
>>> d = model()
:return:
The CO2MPAS model.
:rtype: schedula.Dispatcher
"""
from .physical import physical
ph = sh.SubDispatch(physical())
d = sh.Dispatcher(
name='CO2MPAS model',
description='Calibrates the models with WLTP data and predicts NEDC '
'cycle.')
############################################################################
# PRECONDITIONING CYCLE
############################################################################
d.add_data(
data_id='input.precondition.wltp_p',
description='Dictionary that has all inputs of the calibration cycle.',
default_value={})
d.add_function(
function_id='calculate_precondition_output',
function=ph,
inputs=['input.precondition.wltp_p'],
outputs=['output.precondition.wltp_p'],
description='Wraps all functions needed to calculate the precondition '
'outputs.')
############################################################################
# WLTP - HIGH CYCLE
############################################################################
d.add_data(data_id='input.calibration.wltp_h', default_value={})
d.add_function(
function=select_calibration_data,
inputs=['input.calibration.wltp_h', 'output.precondition.wltp_p'],
outputs=['data.calibration.wltp_h'],
)
d.add_function(
function_id='calibrate_with_wltp_h',
function=ph,
inputs=['data.calibration.wltp_h'],
outputs=['output.calibration.wltp_h'],
description='Wraps all functions needed to calibrate the models to '
'predict light-vehicles\' CO2 emissions.')
d.add_data(data_id='input.prediction.wltp_h', default_value={})
d.add_function(function=select_prediction_data,
inputs=[
'output.calibration.wltp_h',
'data.prediction.models_wltp_h',
'input.prediction.wltp_h'
],
outputs=['data.prediction.wltp_h'])
d.add_function(
function_id='predict_wltp_h',
function=ph,
inputs=['data.prediction.wltp_h'],
outputs=['output.prediction.wltp_h'],
description='Wraps all functions needed to predict CO2 emissions.')
############################################################################
# WLTP - LOW CYCLE
############################################################################
d.add_data(data_id='input.calibration.wltp_l', default_value={})
d.add_function(
function=select_calibration_data,
inputs=['input.calibration.wltp_l', 'output.precondition.wltp_p'],
outputs=['data.calibration.wltp_l'],
)
d.add_function(
function_id='calibrate_with_wltp_l',
function=ph,
inputs=['data.calibration.wltp_l'],
outputs=['output.calibration.wltp_l'],
description='Wraps all functions needed to calibrate the models to '
'predict light-vehicles\' CO2 emissions.')
d.add_data(data_id='input.prediction.wltp_l', default_value={})
d.add_function(function=select_prediction_data,
inputs=[
'output.calibration.wltp_l',
'data.prediction.models_wltp_l',
'input.prediction.wltp_l'
],
outputs=['data.prediction.wltp_l'])
d.add_function(
function_id='predict_wltp_l',
function=ph,
inputs=['data.prediction.wltp_l'],
outputs=['output.prediction.wltp_l'],
description='Wraps all functions needed to predict CO2 emissions.')
############################################################################
# MODEL SELECTOR
############################################################################
from .selector import selector
pred_cyl_ids = ('nedc_h', 'nedc_l', 'wltp_h', 'wltp_l')
sel = selector('wltp_h', 'wltp_l', pred_cyl_ids=pred_cyl_ids)
d.add_data(data_id='config.selector.all', default_value={})
d.add_data(data_id='input.prediction.models', default_value={})
d.add_function(function_id='extract_calibrated_models',
function=sel,
inputs=[
'config.selector.all', 'input.prediction.models',
'output.calibration.wltp_h', 'output.calibration.wltp_l'
],
outputs=['data.calibration.model_scores'] +
['data.prediction.models_%s' % k for k in pred_cyl_ids])
############################################################################
# NEDC - HIGH CYCLE
############################################################################
d.add_function(
function_id='predict_nedc_h',
function=ph,
inputs=['data.prediction.models_nedc_h', 'input.prediction.nedc_h'],
outputs=['output.prediction.nedc_h'],
)
############################################################################
# NEDC - LOW CYCLE
############################################################################
d.add_function(
function_id='predict_nedc_l',
function=ph,
inputs=['data.prediction.models_nedc_l', 'input.prediction.nedc_l'],
outputs=['output.prediction.nedc_l'],
)
return d
dsp.add_data('x_label', 'x')
dsp.add_data('y_label', 'y')
dsp.add_data('labels', None, sh.inf(1, 0))
dsp.add_data('methods', None, sh.inf(1, 0))
dsp.add_data('sets_mapping', None, sh.inf(1, 0))
input_keys = [
'methods', 'data', 'reference_name', 'labels', 'x_label', 'y_label',
'interpolation_method'
]
dsp.add_function(
function_id='map_inputs',
function=functools.partial(sh.map_list, input_keys),
filters=[lambda x: {k: v
for k, v in x.items() if v is not None}],
inputs=input_keys,
outputs=['inputs'])
dsp.add_function(function_id='compute_outputs',
function=sh.SubDispatch(model.dsp),
inputs=['inputs'],
outputs=['outputs'],
description='Executes the computational model.')
dsp.add_dispatcher(write.dsp,
inputs=['outputs', 'output_fpath', 'template_fpath'],
outputs=['written'])
if __name__ == '__main__':
dsp.register().plot()
enable_selector=enable_selector).items()
if v is not None
}
data = sh.combine_dicts(raw_data, flag)
data['flag'] = sh.combine_dicts(data.get('flag', {}), cmd_flags or {},
flag)
return data
def check_validation(sol):
"""
Check if the data are verified.
:param sol:
Validation solution.
:type sol: schedula.Solution
:return:
Validated data
:rtype: list[dict]
"""
sol = sol.get('verified') and sol or {}
keys = 'plan', 'flag', 'dice', 'meta', 'base'
return [sol.get('validated_%s' % k, sh.NONE) for k in keys]
dsp.add_function(function=sh.SubDispatch(_validate),
inputs=['data'],
outputs=['plan', 'flag', 'dice', 'meta', 'base'],
filters=[check_validation])
plot
"""
import matplotlib.pyplot as plt
import schedula as sh
from co2mpas_driver.load import dsp as _load
from co2mpas_driver.model import dsp as _model
from co2mpas_driver.plot import dsp as _plot
dsp = sh.Dispatcher()
dsp.add_dispatcher(
dsp=_load,
inputs=["inputs", "vehicle_id", "db_path", "input_path"],
outputs=["data"],
)
dsp.add_function(function=sh.SubDispatch(_model),
inputs=["data"],
outputs=["outputs"])
dsp.add_function(function_id="write", inputs=["output_path", "outputs"])
dsp.add_function(function=sh.SubDispatch(_plot),
inputs=["output_plot_folder", "outputs"])
if __name__ == "__main__":
sol = dsp(
dict(
vehicle_id=39393,
inputs=dict(inputs=dict(
gear_shifting_style=0.9,
desired_velocity=124 / 3.6,
from .scaleTrace import dsp as _scaleTrace
from .calculateShiftpointsNdvFullPC import dsp as _calculateShiftpointsNdvFullPC
dsp = sh.BlueDispatcher(
name="GEARSHIFT model",
description=
"Calculates the speed trace with scaleTrace and predict the gearshift "
"with calculateShiftpointsNdvFullPC",
)
dsp.add_data(data_id="execution_case",
description="User input data of PYCSIS calibration stage.")
dsp.add_function(
function_id="speedTrace",
function=sh.SubDispatch(_scaleTrace),
inputs=["execution_case"],
outputs=["speed_trace"],
description=
"This function calibrates the speed trance, following the Sub-Annex 1",
)
dsp.add_function(
function_id="calculateShiftpointsNdvFullPC",
function=sh.SubDispatch(_calculateShiftpointsNdvFullPC),
inputs=["execution_case", "speed_trace"],
outputs=["shift_points"],
description=
"This function calibrates the speed trance, following the Sub-Annex 1",
)
def sub_models():
models = {}
from ..physical.engine.thermal import thermal
models['engine_coolant_temperature_model'] = {
'd':
thermal(),
'models': [
'engine_temperature_regression_model',
'max_engine_coolant_temperature'
],
'inputs': [
'times', 'accelerations', 'final_drive_powers_in',
'engine_speeds_out_hot', 'initial_engine_temperature'
],
'outputs': ['engine_coolant_temperatures'],
'targets': ['engine_coolant_temperatures'],
'metrics': [sk_met.mean_absolute_error],
'up_limit': [3],
}
from ..physical.engine.start_stop import start_stop
models['start_stop_model'] = {
'd':
start_stop(),
'models': ['start_stop_model', 'use_basic_start_stop'],
'inputs': [
'times', 'velocities', 'accelerations',
'engine_coolant_temperatures', 'state_of_charges', 'gears',
'correct_start_stop_with_gears', 'start_stop_activation_time',
'min_time_engine_on_after_start', 'has_start_stop'
],
'outputs': ['on_engine', 'engine_starts'],
'targets': ['on_engine', 'engine_starts'],
'metrics': [sk_met.accuracy_score] * 2,
'weights': [-1, -1],
'dn_limit': [0.7] * 2,
}
from ..physical import physical
models['engine_speed_model'] = {
'd':
physical(),
'select_models':
tyre_models_selector,
'models': [
'final_drive_ratios', 'gear_box_ratios',
'idle_engine_speed_median', 'idle_engine_speed_std', 'CVT',
'max_speed_velocity_ratio', 'tyre_dynamic_rolling_coefficient'
],
'inputs': [
'velocities', 'gears', 'times', 'on_engine', 'gear_box_type',
'accelerations', 'final_drive_powers_in',
'engine_thermostat_temperature', 'tyre_code'
],
'outputs': ['engine_speeds_out_hot'],
'targets': ['engine_speeds_out'],
'metrics_inputs':
['times', 'velocities', 'gear_shifts', 'on_engine', 'stop_velocity'],
'metrics': [metric_engine_speed_model],
'up_limit': [40],
}
from ..physical.engine import calculate_engine_speeds_out
from ..physical.engine.cold_start import cold_start
d = cold_start()
d.add_function(function=calculate_engine_speeds_out,
inputs=[
'on_engine', 'idle_engine_speed',
'engine_speeds_out_hot', 'cold_start_speeds_delta'
],
outputs=['engine_speeds_out'])
models['engine_cold_start_speed_model'] = {
'd':
d,
'models': ['cold_start_speed_model'],
'inputs': [
'engine_speeds_out_hot', 'engine_coolant_temperatures',
'on_engine', 'idle_engine_speed'
],
'outputs': ['engine_speeds_out'],
'targets': ['engine_speeds_out'],
'metrics_inputs':
['cold_start_speeds_phases', 'engine_coolant_temperatures'],
'metrics': [metric_engine_cold_start_speed_model],
'up_limit': [160],
}
from ..physical.clutch_tc import clutch_torque_converter
d = clutch_torque_converter()
d.add_function(function=calculate_engine_speeds_out,
inputs=[
'on_engine', 'idle_engine_speed',
'engine_speeds_out_hot', 'clutch_tc_speeds_delta'
],
outputs=['engine_speeds_out'])
models['clutch_torque_converter_model'] = {
'd':
d,
'models': ['clutch_window', 'clutch_model', 'torque_converter_model'],
'inputs': [
'gear_box_speeds_in', 'on_engine', 'idle_engine_speed',
'gear_box_type', 'gears', 'accelerations', 'times', 'gear_shifts',
'engine_speeds_out_hot', 'velocities', 'lock_up_tc_limits',
'has_torque_converter'
],
'define_sub_model':
lambda d, **kwargs: sh.SubDispatch(d),
'outputs': ['engine_speeds_out'],
'targets': ['engine_speeds_out'],
'metrics_inputs': ['on_engine'],
'metrics': [metric_clutch_torque_converter_model],
'up_limit': [100],
}
from ..physical.engine.co2_emission import co2_emission
from .co2_params import co2_params_selector
models['co2_params'] = {
'd':
co2_emission(),
'model_selector':
co2_params_selector,
'models': [
'co2_params_calibrated', 'calibration_status',
'initial_friction_params', 'engine_idle_fuel_consumption'
],
'inputs': ['co2_emissions_model'],
'outputs': ['co2_emissions', 'calibration_status'],
'targets': ['identified_co2_emissions', 'calibration_status'],
'metrics': [sk_met.mean_absolute_error, metric_calibration_status],
'up_limit': [0.5, None],
'weights': [1, None]
}
from ..physical.electrics import electrics
models['alternator_model'] = {
'd':
electrics(),
'models': [
'alternator_status_model', 'alternator_nominal_voltage',
'alternator_current_model', 'max_battery_charging_current',
'start_demand', 'electric_load', 'alternator_nominal_power',
'alternator_efficiency', 'alternator_initialization_time'
],
'inputs': [
'battery_capacity', 'alternator_nominal_voltage',
'initial_state_of_charge', 'times', 'gear_box_powers_in',
'on_engine', 'engine_starts', 'accelerations'
],
'outputs': [
'alternator_currents', 'battery_currents', 'state_of_charges',
'alternator_statuses'
],
'targets': [
'alternator_currents', 'battery_currents', 'state_of_charges',
'alternator_statuses'
],
'metrics': [sk_met.mean_absolute_error] * 3 + [sk_met.accuracy_score],
'up_limit': [60, 60, None, None],
'weights': [1, 1, 0, 0]
}
from ..physical.gear_box.at_gear import at_gear
at_pred_inputs = [
'idle_engine_speed', 'full_load_curve', 'road_loads', 'vehicle_mass',
'accelerations', 'motive_powers', 'engine_speeds_out',
'engine_coolant_temperatures', 'time_cold_hot_transition', 'times',
'use_dt_gear_shifting', 'specific_gear_shifting',
'velocity_speed_ratios', 'velocities', 'MVL',
'fuel_saving_at_strategy', 'change_gear_window_width', 'stop_velocity',
'plateau_acceleration', 'max_velocity_full_load_correction',
'cycle_type'
]
models['at_model'] = {
'd':
at_gear(),
'select_models':
functools.partial(at_models_selector, at_gear(), at_pred_inputs),
'models': [
'MVL', 'CMV', 'CMV_Cold_Hot', 'DTGS', 'GSPV', 'GSPV_Cold_Hot',
'specific_gear_shifting', 'change_gear_window_width',
'max_velocity_full_load_correction', 'plateau_acceleration'
],
'inputs':
at_pred_inputs,
'define_sub_model':
lambda d, **kwargs: sh.SubDispatch(d),
'outputs': ['gears', 'max_gear'],
'targets': ['gears', 'max_gear'],
'metrics': [sk_met.accuracy_score, None],
'weights': [-1, 0]
}
return models
