def define_sub_model(d, inputs, outputs, models, **kwargs): missing = set(outputs).difference(d.nodes) if missing: outputs = set(outputs).difference(missing) if inputs is not None: inputs = set(inputs).union(models) return sh.SubDispatch(d.shrink_dsp(inputs, outputs))
def setUp(self): ss_dsp = sh.Dispatcher() fun = lambda a: (a + 1, 5, a - 1) dom = lambda kw: True c = '|!"£$%&/()=?^*+éè[]#¶ù§çò@:;-_.,<>' ss_dsp.add_function(function=fun, inputs=['a'], outputs=['b', sh.SINK, c], input_domain=dom, weight=1) def raise_fun(a): raise ValueError('Error') ss_dsp.add_function(function=raise_fun, inputs=['a'], outputs=['b']) sdspfunc = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction', ['a'], ['b', c]) sdsppipe = sh.SubDispatchPipe(ss_dsp, 'SubDispatchPipe', ['a'], ['b', c]) spipe = sh.DispatchPipe(ss_dsp, 'DispatchPipe', ['a'], [c]) sdsp = sh.SubDispatch(ss_dsp, ['b', c], output_type='list') s_dsp = sh.Dispatcher() s_dsp.add_function(None, sdspfunc, ['a'], ['b', 'c'], weight=2) s_dsp.add_function(None, sdsppipe, ['a'], ['b', 'c'], out_weight={'c': 5}) s_dsp.add_function(None, spipe, ['a'], ['b'], weight=2) s_dsp.add_function('SubDispatch', sdsp, ['d'], ['e', 'f']) dsp = sh.Dispatcher() dsp.add_data('A', default_value=[0] * 1000) dsp.add_data('D', default_value={'a': 3}) dsp.add_dispatcher(dsp=s_dsp, inputs={ 'A': 'a', 'D': 'd' }, outputs={ 'b': 'B', 'c': 'C', 'e': 'E', 'f': 'F' }, inp_weight={'A': 3}) self.sol = dsp.dispatch() self.dsp = dsp dsp = sh.Dispatcher() dsp.add_function(function=sh.bypass, inputs=['a'], outputs=[sh.PLOT]) self.dsp_plot = dsp
def setUp(self): sub_dsp = sh.Dispatcher() def fun(a): return a + 1, a - 1 sub_dsp.add_function('fun', fun, ['a'], ['b', 'c']) dispatch = sh.SubDispatch(sub_dsp, ['a', 'b', 'c']) dispatch_dict = sh.SubDispatch(sub_dsp, ['c'], output_type='dict') dispatch_list = sh.SubDispatch(sub_dsp, ['a', 'c'], output_type='list') dispatch_val = sh.SubDispatch(sub_dsp, ['c'], output_type='list') dsp = sh.Dispatcher() dsp.add_function('dispatch', dispatch, ['d'], ['e']) dsp.add_function('dispatch_dict', dispatch_dict, ['d'], ['f']) dsp.add_function('dispatch_list', dispatch_list, ['d'], ['g']) dsp.add_function('dispatch_val', dispatch_val, ['d'], ['h']) self.dsp = dsp
def run_synchronisation(): # Dedicated logging for this run fileh = logging.FileHandler(co2wui_fpath("sync", "logfile.txt"), "w") logger = logging.getLogger() logger.setLevel(logging.INFO) frmt = "%(asctime)-15s:%(levelname)5.5s:%(name)s:%(message)s" logging.basicConfig(level=logging.INFO, format=frmt) logger.addHandler(fileh) # Input file inputs = listdir_inputs("sync", "input") input_file = str(inputs[0]) # Output file output_name = os.path.splitext(os.path.basename(input_file))[0] output_file = co2wui_fpath("sync", "output", output_name + ".sync.xlsx") # Remove old output files previous = listdir_outputs("sync", "output") for f in previous: os.remove(str(f)) # Arguments kwargs = { "x_label": request.form.get("x_label") if request.form.get("x_label") else "times", "y_label": request.form.get("y_label") if request.form.get("y_label") else "velocities", "interpolation_method": request.form.get("interpolation_method"), "header": request.form.get("header"), "reference_name": request.form.get("reference_name") if request.form.get("reference_name") else "theoretical", } kwargs = {k: v for k, v in kwargs.items() if v} try: # Dispatcher _process = sh.SubDispatch(syncing.dsp, ["written"], output_type="value") ret = _process( dict(input_fpath=input_file, output_fpath=str(output_file), **kwargs) ) fileh.close() logger.removeHandler(fileh) return "OK" except Exception as e: logger.error(_("Synchronisation failed: ") + str(e)) fileh.close() logger.removeHandler(fileh) return "KO"
def run_synchronisation(): # Dedicated logging for this run fileh = logging.FileHandler("sync/logfile.txt", "w") logger = logging.getLogger() logger.setLevel(logging.INFO) frmt = "%(asctime)-15s:%(levelname)5.5s:%(name)s:%(message)s" logging.basicConfig(level=logging.INFO, format=frmt) logger.addHandler(fileh) # Input and output files inputs = [ f for f in listdir_inputs("sync/input") if osp.isfile(osp.join("sync/input", f)) ] input_file = osp.join("sync", "input", inputs[0]) output_file = osp.join("sync", "output", "datasync.sync.xlsx") # Arguments kwargs = { "x_label": request.form.get("x_label") if request.form.get("x_label") else "times", "y_label": request.form.get("y_label") if request.form.get("y_label") else "velocities", "interpolation_method": request.form.get("interpolation_method"), "header": request.form.get("header"), "reference_name": request.form.get("reference_name") if request.form.get("reference_name") else "theoretical", } kwargs = {k: v for k, v in kwargs.items() if v} try: # Dispatcher _process = sh.SubDispatch(syncing.dsp, ["written"], output_type="value") ret = _process( dict(input_fpath=input_file, output_fpath=output_file, **kwargs)) fileh.close() logger.removeHandler(fileh) return "OK" except Exception as e: logger.error(_("Synchronisation failed: ") + str(e)) fileh.close() logger.removeHandler(fileh) return "KO"
def _mdl_error(mdl): dsp = sh.BlueDispatcher( name=mdl.name, description='Calculates the error of calibrated model of a reference.', ) dsp.add_data('inputs_map', getattr(mdl, 'inputs_map', {})) dsp.add_function(function_id='select_inputs', function=sh.map_dict, inputs=['inputs_map', 'data'], outputs=['inputs<0>']) dsp.add_data('inputs', getattr(mdl, 'inputs', [])) dsp.add_function(function_id='select_inputs', function=_select_data, inputs=['inputs', 'inputs<0>'], outputs=['inputs<1>']) dsp.add_function(function=sh.combine_dicts, inputs=['calibrated_models', 'inputs<1>'], outputs=['prediction_inputs']) dsp.add_data('targets', getattr(mdl, 'targets', [])) dsp.add_function(function_id='select_targets', function=_select_data, inputs=['targets', 'data'], outputs=['references']) dsp.add_function(function=sh.SubDispatch(mdl.dsp), inputs=['prediction_inputs', 'calibrated_models'], outputs=['results']) dsp.add_data('outputs', getattr(mdl, 'outputs', [])) dsp.add_func(select_predictions, outputs=['predictions']) dsp.add_data('metrics_inputs', getattr(mdl, 'metrics_inputs', {})) dsp.add_function(function_id='select_metrics_inputs', function=_select_data, inputs=['metrics_inputs', 'data'], outputs=['metrics_kwargs']) dsp.add_data('metrics', getattr(mdl, 'metrics', {})) dsp.add_func(calculate_errors, outputs=['errors']) dsp.add_data('up_limit', getattr(mdl, 'up_limit', None)) dsp.add_data('dn_limit', getattr(mdl, 'dn_limit', None)) dsp.add_func(calculate_calibration_status, inputs_kwargs=True, outputs=['status']) return dsp
def _selector(name, data_in, data_out, setting): d = sh.Dispatcher(name='%s selector' % name, description='Select the calibrated %s.' % name) errors, setting = [], setting or {} _sort_models = setting.pop('sort_models', sort_models) if 'weights' in setting: _weights = sh.map_list(setting.get('names', setting['targets']), *setting.pop('weights')) else: _weights = None _get_best_model = functools.partial(setting.pop('get_best_model', get_best_model), models_wo_err=setting.pop( 'models_wo_err', None), selector_id=d.name) d.add_data(data_id='selector_settings', default_value={}) node_ids = ['error_settings', 'best_model_settings'] d.add_function(function=functools.partial(define_selector_settings, node_ids=node_ids), inputs=['selector_settings'], outputs=node_ids) for i in data_in: e = 'error/%s' % i errors.append(e) d.add_function(function=_errors(name, i, data_out, setting), inputs=['error_settings', i] + [k for k in data_out if k != i], outputs=[e]) d.add_function(function_id='sort_models', function=functools.partial(_sort_models, weights=_weights), inputs=errors, outputs=['rank']) d.add_function(function_id='get_best_model', function=_get_best_model, inputs=['rank', 'best_model_settings'], outputs=['model', 'errors']) return sh.SubDispatch(d, outputs=['model', 'errors'], output_type='list')
def setUp(self): import functools ss_dsp = sh.Dispatcher() def fun(a, c): """ :param a: Nice a. :type a: float :param c: Nice c. :type c: float :return: Something. :rtype: tuple """ return a + 1, c, a - 1 ss_dsp.add_function('fun', fun, ['a', 'e'], ['b', 'c', 'd']) ss_dsp_func = sh.DispatchPipe(ss_dsp, 'func', ['e', 'a'], ['c', 'd', 'b']) sub_disfun = sh.add_args(functools.partial(ss_dsp_func, 5)) s_dsp = sh.Dispatcher() s_dsp.add_data('a', 1) s_dsp.add_data('d', 4) s_dsp.add_function('sub_dispatch', sub_disfun, ['d', 'a'], ['b', 'c', sh.SINK]) dispatch = sh.SubDispatch(s_dsp, ['b', 'c', 'a'], output_type='list') self.dsp = dsp = sh.Dispatcher() dsp.add_data('input', default_value={'a': 3}) dsp.add_function('dispatch', dispatch, ['input'], [sh.SINK, 'h', 'i'], inp_weight={'input': 4}, out_weight={ 'h': 3, 'i': 6 }) dsp.add_function('fun', lambda: None, None, ['j']) dsp.add_dispatcher(s_dsp, inputs=('a', ), outputs=('b', 'c'), include_defaults=True)
def run_synchronisation(): # Dedicated logging for this run fileh = logging.FileHandler("sync/logfile.txt", "w") formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") fileh.setFormatter(formatter) log = logging.getLogger() log.setLevel(logging.DEBUG) for hdlr in log.handlers[:]: log.removeHandler(hdlr) log.addHandler(fileh) # Input and output files input_file = "sync/input/datasync.xlsx" output_file = "sync/output/datasync.sync.xlsx" # Arguments kwargs = { "x_label": request.form.get("x_label") if request.form.get("x_label") else 'times', "y_label": request.form.get("y_label") if request.form.get("y_label") else 'velocities', "interpolation_method": request.form.get("interpolation_method"), "header": request.form.get("header"), "reference_name": request.form.get("reference_name") if request.form.get("reference_name") else 'theoretical', } kwargs = {k: v for k, v in kwargs.items() if v} try: # Dispatcher _process = sh.SubDispatch(syncing.dsp, ['written'], output_type='value') ret = _process( dict(input_fpath=input_file, output_fpath=output_file, **kwargs)) return 'OK' except Exception as e: return 'KO'
def mdl_selector(name, package=None): """ Defines a model selector for a specific model. :param name: Model name. :type name: str :param package: Package name. :type package: str :return: Model selector. :rtype: schedula.utils.blue.BlueDispatcher """ import importlib mdl = importlib.import_module(name, package) dsp = sh.BlueDispatcher(name='%s selector' % mdl.name, description='Select the calibrated %s.' % mdl.name) err_func = sh.SubDispatch(_mdl_error(mdl), outputs=['errors', 'status'], output_type='list') for data_id in calibration_cycles: mdl_err, mdl_err_name = _mdl_errors(mdl, data_id, err_func) dsp.add_func(sh.SubDispatchFunction( mdl_err, function_id=mdl_err_name, inputs=[data_id] + [k for k in calibration_cycles if k != data_id]), outputs=['error/%s' % data_id]) # noinspection PyTypeChecker dsp.add_function(function=functools.partial(sort_models, weights=getattr( mdl, 'weights', None)), inputs=list(map('error/{}'.format, calibration_cycles)), outputs=['rank']) dsp.add_func(format_score, outputs=['score']) dsp.add_func(functools.partial(select_best_model, selector_id='%s selector' % mdl.name), outputs=['model']) return dsp
def setUp(self): ss_dsp = sh.Dispatcher() def fun(a, c): """ :param a: Nice a. :type a: float :param c: Nice c. :type c: float :return: Something. :rtype: tuple """ return a + 1, c, a - 1 ss_dsp.add_function('fun', fun, ['a', 'e'], ['b', 'c', 'd']) ss_dsp_func = sh.SubDispatchFunction( ss_dsp, 'func', ['e', 'a'], ['c', 'd', 'b']) sub_disfun = sh.partial(ss_dsp_func, 5) s_dsp = sh.Dispatcher() s_dsp.add_function( 'sub_dispatch', sub_disfun, ['a'], ['b', 'c', sh.SINK] ) dispatch = sh.SubDispatch(s_dsp, ['b', 'c', 'a'], output_type='list') dsp = sh.Dispatcher() dsp.add_data('input', default_value={'a': 3}) dsp.add_function('dispatch', dispatch, ['input'], [sh.SINK, 'h', 'i']) self.sol = dsp.dispatch(inputs={'f': 'new'}) self.dsp = dsp self.fun = fun self.sub_dispatch = sub_disfun self.s_dsp = s_dsp self.ss_dsp = ss_dsp self.ss_dsp_func = ss_dsp_func
def run_plan(): """ Defines the plan model. .. dispatcher:: d >>> d = run_plan() :return: The plan model. :rtype: Dispatcher """ d = sh.Dispatcher(name='run_plan', description='Processes a vehicle plan.') d.add_data(data_id='engineering_mode', default_value=False) d.add_data(data_id='use_selector', default_value=False) d.add_data(data_id='soft_validation', default_value=False) d.add_function(function=sh.add_args(schema.validate_plan), inputs=[ 'run_plan', 'data', 'engineering_mode', 'soft_validation', 'use_selector' ], outputs=['validated_plan'], input_domain=check_first_arg) d.add_function(function=default_start_time, outputs=['start_time']) d.add_function(function=default_timestamp, inputs=['start_time'], outputs=['timestamp']) from .plan import make_simulation_plan d.add_function(function=make_simulation_plan, inputs=['validated_plan', 'timestamp', 'variation', 'flag'], outputs=['summary']) return sh.SubDispatch(d)
def _yield_folder_files_results(start_time, input_files, output_folder, overwrite_cache=False, model=None, variation=None, type_approval_mode=False, modelconf=None): model = model or vehicle_processing_model() kw = { 'output_folder': output_folder, 'overwrite_cache': overwrite_cache, 'modelconf': modelconf, 'timestamp': start_time.strftime('%Y%m%d_%H%M%S'), 'variation': variation or {}, 'type_approval_mode': type_approval_mode } _process_vehicle = sh.SubDispatch(model) for fpath in _custom_tqdm(input_files, bar_format='{l_bar}{bar}{r_bar}'): yield _process_vehicle({'input_file_name': fpath}, kw)
def co2_params_selector(name='co2_params', data_in=('wltp_h', 'wltp_l'), data_out=('wltp_h', 'wltp_l'), setting=None): """ Defines the co2_params model selector. .. dispatcher:: d >>> d = co2_params_selector() :return: The co2_params model selector. :rtype: SubDispatch """ from . import _selector d = _selector(name, data_in + ('ALL', ), data_out, setting).dsp n = d.get_node('sort_models', node_attr=None)[0] errors, sort_models = n['inputs'], n['function'] d.dmap.remove_node('sort_models') d.add_function(function=sort_models, inputs=errors[:-1], outputs=['rank<0>']) d.add_data('enable_all', True) d.add_function(function=functools.partial(calibrate_co2_params_all, data_id=data_in), inputs=['enable_all', 'rank<0>'] + errors[:-1], outputs=['ALL']) d.add_function(function=functools.partial(co2_sort_models, **sort_models.keywords), inputs=['rank<0>'] + [errors[-1]], outputs=['rank']) return sh.SubDispatch(d, outputs=['model', 'errors'], output_type='list')
def setUp(self): ss_dsp = sh.Dispatcher(name='Ciao.') def fun(b, c, d=0): """ Fun description. :param b: Second param. :param int c: Third param. :return: Out param. :rtype: float """ return b + c + d def dom(a, *args): """ :param a: First param. """ return a ss_dsp.add_function(function=sh.add_args(fun), inputs=['a', 'b', 'c', 'd'], outputs=['e'], input_domain=dom) ss_dsp.add_function(function=sh.bypass, inputs=['e'], outputs=['f']) ss_dsp.add_function(function=sh.replicate_value, inputs=['f'], outputs=['g', 'h']) sdspfunc = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction', ['a', 'b', 'c', 'd'], ['g', 'h']) sdsp = sh.SubDispatch(ss_dsp, ['e', 'f'], output_type='list') def fun1(): """""" return sdspfunci = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction', ['a', 'b', 'c', 'd']) s_dsp = sh.Dispatcher(name='Sub-Dispatcher') s_dsp.add_function('3', sdspfunc, ['a', 'b', 'c', 'd'], ['g', 'h']) s_dsp.add_function('8', sdspfunci, ['a', 'b', 'c', 'd'], ['o']) s_dsp.add_function('2', sdsp, ['I'], ['e', 'f']) s_dsp.add_function('4', max, ['e', 'f', 'l'], ['i']) s_dsp.add_function('5', inputs=['i'], outputs=['n']) s_dsp.add_function('6', fun1) s_dsp.add_function('7', sh.bypass, inputs=['p'], input_domain=sh.add_args(lambda p: None)) s_dsp.add_data('i', description='max.') self.dsp = dsp = sh.Dispatcher() dsp.add_dispatcher( dsp_id='1', dsp=s_dsp, inputs=('n', 'd', 'I', 'l', { 'm': ('b', 'c'), ('a', 'a1'): 'a' }), outputs=['g', 'i', { 'f': ('h', 'f') }, { ('e', 'h'): ('e', 'e1') }])
class _Logger(logging.Logger): def setLevel(self, level): super(_Logger, self).setLevel(level) frmt = "%(asctime)-15s:%(levelname)5.5s:%(name)s:%(message)s" logging.basicConfig(level=level, format=frmt) rlog = logging.getLogger() # because `basicConfig()` does not reconfig root-logger when re-invoked. rlog.level = level logging.captureWarnings(True) logger = _Logger('cli') click_log.basic_config(logger) _process = sh.SubDispatch(syncing.dsp, ['written'], output_type='value') @click.group( 'syncing', context_settings=dict(help_option_names=['-h', '--help']) ) @click.version_option(__version__) def cli(): """ SYNCING command line tool. """ @cli.command('template', short_help='Generates sample template file.') @click.argument( 'output-file', default='template.xlsx', required=False,
from files.VBSsetup import VBSsetup return VBSsetup process.add_data(data_id='key_mapping', default_value={}, description='Dictionary renaming keys.') process.add_function(function_id='parse_data', function=sh.map_dict, inputs=['key_mapping', 'raw_data'], outputs=['data'], description='Rename the raw data names.') process.add_function(function_id='execute_VBS', function=sh.SubDispatch(model()), inputs=['data'], outputs=['outputs'], description='Executes the computational model.') @sh.add_function(process) def save_outputs(outputs, output_fpath): """ Save model outputs in an Excel file. :param outputs: Model outputs. :type outputs: dict :param output_fpath:
def run_base(): """ Defines the vehicle-processing model. .. dispatcher:: d >>> d = run_base() :return: The vehicle-processing model. :rtype: Dispatcher """ d = sh.Dispatcher( name='run_base', description='Processes a vehicle from the file path to the write of its' ' outputs.') d.add_data(data_id='engineering_mode', default_value=False) d.add_data(data_id='output_folder', default_value='.') d.add_data(data_id='use_selector', default_value=False) d.add_data(data_id='soft_validation', default_value=False) d.add_function(function=schema.validate_meta, inputs=['meta', 'soft_validation'], outputs=['validated_meta']) d.add_function(function=sh.add_args(schema.validate_base), inputs=[ 'run_base', 'data', 'engineering_mode', 'soft_validation', 'use_selector' ], outputs=['validated_base'], input_domain=check_first_arg, weight=10) d.add_data(data_id='only_summary', default_value=False) d.add_function(function=default_vehicle_name, inputs=['input_file_name'], outputs=['vehicle_name']) d.add_function(function=default_start_time, outputs=['start_time']) d.add_function(function=default_timestamp, inputs=['start_time'], outputs=['timestamp']) d.add_function(function=default_output_file_name, inputs=['output_folder', 'vehicle_name', 'timestamp'], outputs=['output_file_name']) from .model import model d.add_function(function=sh.add_args(sh.SubDispatch(model())), inputs=['validated_meta', 'validated_base'], outputs=['dsp_solution']) d.add_function(function=parse_dsp_solution, inputs=['dsp_solution'], outputs=['output_data']) from .report import report d.add_function( function=report(), inputs=['output_data', 'vehicle_name'], outputs=['report', 'summary'], ) from .io.ta import write_ta_output from .conf import defaults dfl = defaults.io_constants_dfl d.add_data('encrypt_inputs', dfl.ENCRYPT_INPUTS) d.add_data('encryption_keys', dfl.ENCRYPTION_KEYS_PATH) d.add_function(function=sh.add_args(write_ta_output()), inputs=[ 'type_approval_mode', 'encrypt_inputs', 'encryption_keys', 'vehicle_family_id', 'start_time', 'timestamp', 'data', 'meta', 'report', 'output_folder' ], outputs=['output_ta_file'], input_domain=check_first_arg) d.add_function(function=get_template_file_name, inputs=['output_template', 'input_file_name'], outputs=['template_file_name']) d.add_data(data_id='output_template', default_value=_get_co2mpas_output_template_fpath(), initial_dist=10) from .io import write_outputs d.add_function(function=sh.add_args(write_outputs()), inputs=[ 'only_summary', 'output_file_name', 'template_file_name', 'report', 'start_time', 'flag' ], outputs=[sh.SINK], input_domain=lambda *args: not args[0]) d.add_function( function=sh.add_args(plot_model_workflow), inputs=['plot_workflow', 'output_file_name', 'vehicle_name'], outputs=[sh.PLOT], weight=30, input_domain=check_first_arg) return sh.SubDispatch(d)
def setUp(self): ss_dsp = sh.Dispatcher(name='ss_dsp') fun = lambda a: (a + 1, 5, a - 1) dom = lambda kw: True ss_dsp.add_function(function=fun, inputs=['a'], outputs=['b', 'd', 'c'], input_domain=dom, weight=1) sdspfunc = sh.SubDispatchFunction(ss_dsp, 'SubDispatchFunction', ['a'], ['b', 'c']) sdsppipe = sh.SubDispatchPipe(ss_dsp, 'SubDispatchPipe', ['a'], ['b', 'c']) sdsp = sh.SubDispatch(ss_dsp, ['b', 'c'], output_type='list') s_dsp = sh.Dispatcher(name='s_dsp') s_dsp.add_function(None, sdspfunc, ['a'], ['b', 'c']) s_dsp.add_function(None, sdsppipe, ['a'], ['g']) s_dsp.add_function('SubDispatch', sdsp, ['d'], ['e', 'f']) dsp = sh.Dispatcher(name='model') dsp.add_data('A', default_value=0) dsp.add_data('D', default_value={'a': 3}) dsp.add_dispatcher(dsp=s_dsp, inputs={ 'A': 'a', 'D': 'd' }, outputs={ 'b': 'B', 'c': 'C', 'e': 'E', 'f': 'F', 'g': 'G' }, inp_weight={'A': 3}) self.dsp = dsp self.sol = sol = dsp.dispatch() sites = set() webmap = dsp.web(node_data=('+set_value', ), run=True, sites=sites) self.site = sites.pop() self.url = '%s/' % self.site.url rules = webmap.rules() self.io = io = [] for rule in rules.values(): n = rule.split('/')[1:] if not n: continue s, k = sol.get_node(*n, node_attr='sol') k = k[-1] try: v = s.workflow.node[k] except KeyError: continue if 'results' not in v: continue inputs = s._wf_pred[k] # List of the function's arguments. inputs = sh.bypass( *[inputs[k]['value'] for k in s.nodes[k]['inputs']]) io.append((rule, inputs, v['results'])) self.sol1 = sol = dsp.dispatch({'A': 1}) self.io1 = io = [] for rule in rules.values(): n = rule.split('/')[1:] if not n: continue s, k = sol.get_node(*n, node_attr='sol') k = k[-1] try: v = s.workflow.node[k] except KeyError: continue if 'results' not in v: continue inputs = s._wf_pred[k] # List of the function's arguments. inputs = sh.bypass( *[inputs[k]['value'] for k in s.nodes[k]['inputs']]) io.append((rule, inputs, v['results']))
MODELS = [ v.name for v in pkgutil.iter_modules([osp.join(osp.dirname(__file__), 'models')]) if not v.name.startswith('_') ] dsp.add_data('enable_selector', False, filters=[lambda x: { 'enable_selector': x }]) for name in MODELS: dsp.add_function(function=sh.SubDispatch(mdl_selector( '.models.%s' % name, __name__), outputs=['model', 'score'], output_type='list'), function_id='%s selector' % name, inputs=['CO2MPAS_results', 'enable_selector'], outputs=['models', 'scores']) def merge_model_selection(adict): """ Merge models or scores selection. :param adict: Models or scores selection. :type adict: dict :return:
def _error(name, setting): d = sh.Dispatcher( name=name, description='Calculates the error of calibrated model of a reference.', ) default_settings = { 'inputs_map': {}, 'targets': [], 'metrics_inputs': {}, 'up_limit': None, 'dn_limit': None } default_settings.update(setting) default_settings['names'] = default_settings.get( 'names', default_settings['targets']) it = sh.selector(['up_limit', 'dn_limit'], default_settings).items() for k, v in it: if v is not None: default_settings[k] = sh.map_list(setting['names'], *v) d.add_function(function_id='select_inputs', function=sh.map_dict, inputs=['inputs_map', 'data'], outputs=['inputs<0>']) d.add_function(function_id='select_inputs', function=functools.partial(sh.selector, allow_miss=True), inputs=['inputs', 'inputs<0>'], outputs=['inputs<1>']) d.add_function(function=sh.combine_dicts, inputs=['calibrated_models', 'inputs<1>'], outputs=['prediction_inputs']) d.add_function(function=select_targets, inputs=['names', 'targets', 'data'], outputs=['references']) d.add_function(function=functools.partial( default_settings.pop('dsp', lambda x: x), {}), inputs=['prediction_inputs', 'calibrated_models'], outputs=['results']) d.add_function(function=select_outputs, inputs=['names', 'outputs', 'targets', 'results'], outputs=['predictions']) d.add_function(function_id='select_metrics_inputs', function=functools.partial(sh.selector, allow_miss=True), inputs=['metrics_inputs', 'data'], outputs=['metrics_args']) d.add_function( function=make_metrics, inputs=['metrics', 'references', 'predictions', 'metrics_args'], outputs=['errors']) d.add_function(function=check_limits, inputs=['errors', 'up_limit', 'dn_limit'], outputs=['status']) for k, v in default_settings.items(): d.add_data(k, v) func = sh.SubDispatch(dsp=d, outputs=['errors', 'status'], output_type='list') return func
""" import matplotlib.pyplot as plt import schedula as sh from co2mpas_driver.load import dsp as _load from co2mpas_driver.model import dsp as _model from co2mpas_driver.plot import dsp as _plot dsp = sh.Dispatcher() dsp.add_dispatcher( dsp=_load, inputs=['inputs', 'vehicle_id', 'db_path', 'input_path'], outputs=['data'] ) dsp.add_function( function=sh.SubDispatch(_model), inputs=['data'], outputs=['outputs'] ) dsp.add_function( function_id='write', inputs=['output_path', 'outputs'] ) dsp.add_function( function=sh.SubDispatch(_plot), inputs=['output_plot_folder', 'outputs'] ) if __name__ == '__main__':
def model(): """ Defines the CO2MPAS model. .. dispatcher:: d >>> d = model() :return: The CO2MPAS model. :rtype: schedula.Dispatcher """ from .physical import physical ph = sh.SubDispatch(physical()) d = sh.Dispatcher( name='CO2MPAS model', description='Calibrates the models with WLTP data and predicts NEDC ' 'cycle.') ############################################################################ # PRECONDITIONING CYCLE ############################################################################ d.add_data( data_id='input.precondition.wltp_p', description='Dictionary that has all inputs of the calibration cycle.', default_value={}) d.add_function( function_id='calculate_precondition_output', function=ph, inputs=['input.precondition.wltp_p'], outputs=['output.precondition.wltp_p'], description='Wraps all functions needed to calculate the precondition ' 'outputs.') ############################################################################ # WLTP - HIGH CYCLE ############################################################################ d.add_data(data_id='input.calibration.wltp_h', default_value={}) d.add_function( function=select_calibration_data, inputs=['input.calibration.wltp_h', 'output.precondition.wltp_p'], outputs=['data.calibration.wltp_h'], ) d.add_function( function_id='calibrate_with_wltp_h', function=ph, inputs=['data.calibration.wltp_h'], outputs=['output.calibration.wltp_h'], description='Wraps all functions needed to calibrate the models to ' 'predict light-vehicles\' CO2 emissions.') d.add_data(data_id='input.prediction.wltp_h', default_value={}) d.add_function(function=select_prediction_data, inputs=[ 'output.calibration.wltp_h', 'data.prediction.models_wltp_h', 'input.prediction.wltp_h' ], outputs=['data.prediction.wltp_h']) d.add_function( function_id='predict_wltp_h', function=ph, inputs=['data.prediction.wltp_h'], outputs=['output.prediction.wltp_h'], description='Wraps all functions needed to predict CO2 emissions.') ############################################################################ # WLTP - LOW CYCLE ############################################################################ d.add_data(data_id='input.calibration.wltp_l', default_value={}) d.add_function( function=select_calibration_data, inputs=['input.calibration.wltp_l', 'output.precondition.wltp_p'], outputs=['data.calibration.wltp_l'], ) d.add_function( function_id='calibrate_with_wltp_l', function=ph, inputs=['data.calibration.wltp_l'], outputs=['output.calibration.wltp_l'], description='Wraps all functions needed to calibrate the models to ' 'predict light-vehicles\' CO2 emissions.') d.add_data(data_id='input.prediction.wltp_l', default_value={}) d.add_function(function=select_prediction_data, inputs=[ 'output.calibration.wltp_l', 'data.prediction.models_wltp_l', 'input.prediction.wltp_l' ], outputs=['data.prediction.wltp_l']) d.add_function( function_id='predict_wltp_l', function=ph, inputs=['data.prediction.wltp_l'], outputs=['output.prediction.wltp_l'], description='Wraps all functions needed to predict CO2 emissions.') ############################################################################ # MODEL SELECTOR ############################################################################ from .selector import selector pred_cyl_ids = ('nedc_h', 'nedc_l', 'wltp_h', 'wltp_l') sel = selector('wltp_h', 'wltp_l', pred_cyl_ids=pred_cyl_ids) d.add_data(data_id='config.selector.all', default_value={}) d.add_data(data_id='input.prediction.models', default_value={}) d.add_function(function_id='extract_calibrated_models', function=sel, inputs=[ 'config.selector.all', 'input.prediction.models', 'output.calibration.wltp_h', 'output.calibration.wltp_l' ], outputs=['data.calibration.model_scores'] + ['data.prediction.models_%s' % k for k in pred_cyl_ids]) ############################################################################ # NEDC - HIGH CYCLE ############################################################################ d.add_function( function_id='predict_nedc_h', function=ph, inputs=['data.prediction.models_nedc_h', 'input.prediction.nedc_h'], outputs=['output.prediction.nedc_h'], ) ############################################################################ # NEDC - LOW CYCLE ############################################################################ d.add_function( function_id='predict_nedc_l', function=ph, inputs=['data.prediction.models_nedc_l', 'input.prediction.nedc_l'], outputs=['output.prediction.nedc_l'], ) return d
dsp.add_data('x_label', 'x') dsp.add_data('y_label', 'y') dsp.add_data('labels', None, sh.inf(1, 0)) dsp.add_data('methods', None, sh.inf(1, 0)) dsp.add_data('sets_mapping', None, sh.inf(1, 0)) input_keys = [ 'methods', 'data', 'reference_name', 'labels', 'x_label', 'y_label', 'interpolation_method' ] dsp.add_function( function_id='map_inputs', function=functools.partial(sh.map_list, input_keys), filters=[lambda x: {k: v for k, v in x.items() if v is not None}], inputs=input_keys, outputs=['inputs']) dsp.add_function(function_id='compute_outputs', function=sh.SubDispatch(model.dsp), inputs=['inputs'], outputs=['outputs'], description='Executes the computational model.') dsp.add_dispatcher(write.dsp, inputs=['outputs', 'output_fpath', 'template_fpath'], outputs=['written']) if __name__ == '__main__': dsp.register().plot()
enable_selector=enable_selector).items() if v is not None } data = sh.combine_dicts(raw_data, flag) data['flag'] = sh.combine_dicts(data.get('flag', {}), cmd_flags or {}, flag) return data def check_validation(sol): """ Check if the data are verified. :param sol: Validation solution. :type sol: schedula.Solution :return: Validated data :rtype: list[dict] """ sol = sol.get('verified') and sol or {} keys = 'plan', 'flag', 'dice', 'meta', 'base' return [sol.get('validated_%s' % k, sh.NONE) for k in keys] dsp.add_function(function=sh.SubDispatch(_validate), inputs=['data'], outputs=['plan', 'flag', 'dice', 'meta', 'base'], filters=[check_validation])
plot """ import matplotlib.pyplot as plt import schedula as sh from co2mpas_driver.load import dsp as _load from co2mpas_driver.model import dsp as _model from co2mpas_driver.plot import dsp as _plot dsp = sh.Dispatcher() dsp.add_dispatcher( dsp=_load, inputs=["inputs", "vehicle_id", "db_path", "input_path"], outputs=["data"], ) dsp.add_function(function=sh.SubDispatch(_model), inputs=["data"], outputs=["outputs"]) dsp.add_function(function_id="write", inputs=["output_path", "outputs"]) dsp.add_function(function=sh.SubDispatch(_plot), inputs=["output_plot_folder", "outputs"]) if __name__ == "__main__": sol = dsp( dict( vehicle_id=39393, inputs=dict(inputs=dict( gear_shifting_style=0.9, desired_velocity=124 / 3.6,
from .scaleTrace import dsp as _scaleTrace from .calculateShiftpointsNdvFullPC import dsp as _calculateShiftpointsNdvFullPC dsp = sh.BlueDispatcher( name="GEARSHIFT model", description= "Calculates the speed trace with scaleTrace and predict the gearshift " "with calculateShiftpointsNdvFullPC", ) dsp.add_data(data_id="execution_case", description="User input data of PYCSIS calibration stage.") dsp.add_function( function_id="speedTrace", function=sh.SubDispatch(_scaleTrace), inputs=["execution_case"], outputs=["speed_trace"], description= "This function calibrates the speed trance, following the Sub-Annex 1", ) dsp.add_function( function_id="calculateShiftpointsNdvFullPC", function=sh.SubDispatch(_calculateShiftpointsNdvFullPC), inputs=["execution_case", "speed_trace"], outputs=["shift_points"], description= "This function calibrates the speed trance, following the Sub-Annex 1", )
def sub_models(): models = {} from ..physical.engine.thermal import thermal models['engine_coolant_temperature_model'] = { 'd': thermal(), 'models': [ 'engine_temperature_regression_model', 'max_engine_coolant_temperature' ], 'inputs': [ 'times', 'accelerations', 'final_drive_powers_in', 'engine_speeds_out_hot', 'initial_engine_temperature' ], 'outputs': ['engine_coolant_temperatures'], 'targets': ['engine_coolant_temperatures'], 'metrics': [sk_met.mean_absolute_error], 'up_limit': [3], } from ..physical.engine.start_stop import start_stop models['start_stop_model'] = { 'd': start_stop(), 'models': ['start_stop_model', 'use_basic_start_stop'], 'inputs': [ 'times', 'velocities', 'accelerations', 'engine_coolant_temperatures', 'state_of_charges', 'gears', 'correct_start_stop_with_gears', 'start_stop_activation_time', 'min_time_engine_on_after_start', 'has_start_stop' ], 'outputs': ['on_engine', 'engine_starts'], 'targets': ['on_engine', 'engine_starts'], 'metrics': [sk_met.accuracy_score] * 2, 'weights': [-1, -1], 'dn_limit': [0.7] * 2, } from ..physical import physical models['engine_speed_model'] = { 'd': physical(), 'select_models': tyre_models_selector, 'models': [ 'final_drive_ratios', 'gear_box_ratios', 'idle_engine_speed_median', 'idle_engine_speed_std', 'CVT', 'max_speed_velocity_ratio', 'tyre_dynamic_rolling_coefficient' ], 'inputs': [ 'velocities', 'gears', 'times', 'on_engine', 'gear_box_type', 'accelerations', 'final_drive_powers_in', 'engine_thermostat_temperature', 'tyre_code' ], 'outputs': ['engine_speeds_out_hot'], 'targets': ['engine_speeds_out'], 'metrics_inputs': ['times', 'velocities', 'gear_shifts', 'on_engine', 'stop_velocity'], 'metrics': [metric_engine_speed_model], 'up_limit': [40], } from ..physical.engine import calculate_engine_speeds_out from ..physical.engine.cold_start import cold_start d = cold_start() d.add_function(function=calculate_engine_speeds_out, inputs=[ 'on_engine', 'idle_engine_speed', 'engine_speeds_out_hot', 'cold_start_speeds_delta' ], outputs=['engine_speeds_out']) models['engine_cold_start_speed_model'] = { 'd': d, 'models': ['cold_start_speed_model'], 'inputs': [ 'engine_speeds_out_hot', 'engine_coolant_temperatures', 'on_engine', 'idle_engine_speed' ], 'outputs': ['engine_speeds_out'], 'targets': ['engine_speeds_out'], 'metrics_inputs': ['cold_start_speeds_phases', 'engine_coolant_temperatures'], 'metrics': [metric_engine_cold_start_speed_model], 'up_limit': [160], } from ..physical.clutch_tc import clutch_torque_converter d = clutch_torque_converter() d.add_function(function=calculate_engine_speeds_out, inputs=[ 'on_engine', 'idle_engine_speed', 'engine_speeds_out_hot', 'clutch_tc_speeds_delta' ], outputs=['engine_speeds_out']) models['clutch_torque_converter_model'] = { 'd': d, 'models': ['clutch_window', 'clutch_model', 'torque_converter_model'], 'inputs': [ 'gear_box_speeds_in', 'on_engine', 'idle_engine_speed', 'gear_box_type', 'gears', 'accelerations', 'times', 'gear_shifts', 'engine_speeds_out_hot', 'velocities', 'lock_up_tc_limits', 'has_torque_converter' ], 'define_sub_model': lambda d, **kwargs: sh.SubDispatch(d), 'outputs': ['engine_speeds_out'], 'targets': ['engine_speeds_out'], 'metrics_inputs': ['on_engine'], 'metrics': [metric_clutch_torque_converter_model], 'up_limit': [100], } from ..physical.engine.co2_emission import co2_emission from .co2_params import co2_params_selector models['co2_params'] = { 'd': co2_emission(), 'model_selector': co2_params_selector, 'models': [ 'co2_params_calibrated', 'calibration_status', 'initial_friction_params', 'engine_idle_fuel_consumption' ], 'inputs': ['co2_emissions_model'], 'outputs': ['co2_emissions', 'calibration_status'], 'targets': ['identified_co2_emissions', 'calibration_status'], 'metrics': [sk_met.mean_absolute_error, metric_calibration_status], 'up_limit': [0.5, None], 'weights': [1, None] } from ..physical.electrics import electrics models['alternator_model'] = { 'd': electrics(), 'models': [ 'alternator_status_model', 'alternator_nominal_voltage', 'alternator_current_model', 'max_battery_charging_current', 'start_demand', 'electric_load', 'alternator_nominal_power', 'alternator_efficiency', 'alternator_initialization_time' ], 'inputs': [ 'battery_capacity', 'alternator_nominal_voltage', 'initial_state_of_charge', 'times', 'gear_box_powers_in', 'on_engine', 'engine_starts', 'accelerations' ], 'outputs': [ 'alternator_currents', 'battery_currents', 'state_of_charges', 'alternator_statuses' ], 'targets': [ 'alternator_currents', 'battery_currents', 'state_of_charges', 'alternator_statuses' ], 'metrics': [sk_met.mean_absolute_error] * 3 + [sk_met.accuracy_score], 'up_limit': [60, 60, None, None], 'weights': [1, 1, 0, 0] } from ..physical.gear_box.at_gear import at_gear at_pred_inputs = [ 'idle_engine_speed', 'full_load_curve', 'road_loads', 'vehicle_mass', 'accelerations', 'motive_powers', 'engine_speeds_out', 'engine_coolant_temperatures', 'time_cold_hot_transition', 'times', 'use_dt_gear_shifting', 'specific_gear_shifting', 'velocity_speed_ratios', 'velocities', 'MVL', 'fuel_saving_at_strategy', 'change_gear_window_width', 'stop_velocity', 'plateau_acceleration', 'max_velocity_full_load_correction', 'cycle_type' ] models['at_model'] = { 'd': at_gear(), 'select_models': functools.partial(at_models_selector, at_gear(), at_pred_inputs), 'models': [ 'MVL', 'CMV', 'CMV_Cold_Hot', 'DTGS', 'GSPV', 'GSPV_Cold_Hot', 'specific_gear_shifting', 'change_gear_window_width', 'max_velocity_full_load_correction', 'plateau_acceleration' ], 'inputs': at_pred_inputs, 'define_sub_model': lambda d, **kwargs: sh.SubDispatch(d), 'outputs': ['gears', 'max_gear'], 'targets': ['gears', 'max_gear'], 'metrics': [sk_met.accuracy_score, None], 'weights': [-1, 0] } return models