def select_prediction_data(data, *new_data): """ Selects the data required to predict the CO2 emissions with CO2MPAS model. :param data: Output data. :type data: dict :param new_data: New data. :type new_data: dict :return: Data required to predict the CO2 emissions with CO2MPAS model. :rtype: dict """ ids = _prediction_data from .physical.defaults import dfl if not dfl.functions.select_prediction_data.theoretical: ids = ids + _prediction_data_ts data = sh.selector(ids, data, allow_miss=True) if new_data: new_data = sh.combine_dicts(*new_data) data = sh.combine_dicts(data, new_data) if 'gears' in data and 'gears' not in new_data: if data.get('gear_box_type', 0) == 'automatic' or \ len(data.get('velocities', ())) != len(data['gears']): data.pop('gears') return data
def _add2summary(total_summary, summary, base_keys=None): base_keys = base_keys or {} for k, v in sh.stack_nested_keys(summary, depth=3): d = sh.get_nested_dicts(total_summary, *k, default=list) if isinstance(v, list): for j in v: d.append(sh.combine_dicts(j, base_keys)) else: d.append(sh.combine_dicts(v, base_keys))
def _summary2df(data): res = [] summary = data.get('summary', {}) if 'results' in summary: r = {} index = ['cycle', 'stage', 'usage'] for k, v in sh.stack_nested_keys(summary['results'], depth=4): l = sh.get_nested_dicts(r, k[0], default=list) l.append(sh.combine_dicts(sh.map_list(index, *k[1:]), v)) if r: df = _dd2df(r, index=index, depth=2, col_key=functools.partial(_sort_key, p_keys=('param', ) * 2), row_key=functools.partial(_sort_key, p_keys=index)) import pandas as pd df.columns = pd.MultiIndex.from_tuples(_add_units(df.columns)) setattr(df, 'name', 'results') res.append(df) if 'selection' in summary: df = _dd2df(summary['selection'], ['model_id'], depth=2, col_key=functools.partial(_sort_key, p_keys=('stage', 'cycle')), row_key=functools.partial(_sort_key, p_keys=())) setattr(df, 'name', 'selection') res.append(df) if 'comparison' in summary: r = {} for k, v in sh.stack_nested_keys(summary['comparison'], depth=3): v = sh.combine_dicts(v, base={'param_id': k[-1]}) sh.get_nested_dicts(r, *k[:-1], default=list).append(v) if r: df = _dd2df(r, ['param_id'], depth=2, col_key=functools.partial(_sort_key, p_keys=('stage', 'cycle')), row_key=functools.partial(_sort_key, p_keys=())) setattr(df, 'name', 'comparison') res.append(df) if res: return {'summary': res} return {}
def _define_inputs(sol, inputs): kw = dict( sources=inputs, check_inputs=False, graph=sol.dsp.dmap, _update_links=False ) keys = set(sol) - set(sol.dsp.get_sub_dsp_from_workflow(**kw).data_nodes) return sh.combine_dicts({k: sol[k] for k in keys}, inputs)
def append(self, token): if isinstance(token, (Operator, Function)): try: tokens = [self.pop() for _ in range(token.get_n_args)][::-1] except IndexError: raise FormulaError() token.update_input_tokens(*tokens) inputs = [self.get_node_id(i) for i in tokens] token.set_expr(*tokens) out, dmap, get_id = token.node_id, self.dsp.dmap, get_unused_node_id if out not in self.dsp.nodes: func = token.compile() kw = dict( function_id=get_id(dmap, token.name), function=func, inputs=inputs or None, outputs=[out], ) if isinstance(func, dict): _inputs = func.get('extra_inputs', {}) for k, v in _inputs.items(): if v is not sh.NONE: self.dsp.add_data(k, v) kw = sh.combine_dicts( {'inputs': (list(_inputs) + inputs) or None}, func, base=kw ) self.dsp.add_function(**kw) else: self.nodes[token] = n_id = get_id(dmap, out, 'c%d>{}') self.dsp.add_function(None, sh.bypass, [out], [n_id]) elif isinstance(token, Operand): self.missing_operands.add(token) self._deque.append(token)
def add_sheet(self, worksheet, context): get_in = sh.get_nested_dicts if isinstance(worksheet, str): book = get_in(self.books, context['excel'], BOOK) worksheet = book[_get_name(worksheet, book.sheetnames)] context = sh.combine_dicts(context, base={'sheet': worksheet.title.upper()}) d = get_in(self.books, context['excel'], SHEETS, context['sheet']) if 'formula_references' not in d: d['formula_references'] = formula_references = { k: v['ref'] for k, v in worksheet.formula_attributes.items() if v.get('t') == 'array' and 'ref' in v } else: formula_references = d['formula_references'] if 'formula_ranges' not in d: d['formula_ranges'] = { Ranges().push(ref, context=context) for ref in formula_references.values() } return worksheet, context
class DispatcherSphinxDirective(Graphviz): required_arguments = 1 img_opt = { 'height': directives.length_or_unitless, 'width': directives.length_or_percentage_or_unitless, } option_spec = { 'graphviz_dot': directives.unchanged, # sphinx==1.3.5 'index': bool_option } sh.combine_dicts(img_opt, Graphviz.option_spec, base=option_spec) def run(self): node = super(DispatcherSphinxDirective, self).run()[0] # noinspection PyUnresolvedReferences node = dsp(node.rawsource, *node.children, **node.attributes) node['img_opt'] = sh.selector(self.img_opt, self.options, allow_miss=True) node['index'] = self.options.get('index', False) env = self.state.document.settings.env argument = search_image_for_language(self.arguments[0], env) dirpath = osp.dirname(env.relfn2path(argument)[1]) node['dirpath'] = dirpath if osp.isdir(dirpath) else None return [node]
def _calibrate_gsm( velocity_speed_ratios, on_engine, anomalies, gear, velocities, stop_velocity, idle_engine_speed): # noinspection PyProtectedMember from .at_gear.cmv import CMV, _filter_gear_shifting_velocity as filter_gs idle = idle_engine_speed[0] - idle_engine_speed[1] _vsr = sh.combine_dicts(velocity_speed_ratios, base={0: 0}) limits = { 0: {False: [0]}, 1: {True: [stop_velocity]}, max(_vsr): {True: [dfl.INF]} } shifts = np.unique(sum(map(_shift, (on_engine, anomalies)), [])) for i, j in sh.pairwise(shifts): if on_engine[i:j].all() and not anomalies[i:j].any(): for v in np.array(list(sh.pairwise(_shift(gear[i:j])))) + i: if j != v[1]: v, (g, ng) = velocities[slice(*v)], gear[[v[1] - 1, v[1]]] up = g < ng sh.get_nested_dicts(limits, g, up, default=list).append( v.max() if up else v.min() ) for k, v in list(limits.items()): limits[k] = v.get(False, [_vsr[k] * idle] * 2), v.get(True, []) d = {j: i for i, j in enumerate(sorted(limits))} gsm = CMV(filter_gs(sh.map_dict(d, limits), stop_velocity)) gsm.velocity_speed_ratios = sh.selector(gsm, sh.map_dict(d, _vsr)) gsm.convert(_vsr) return gsm
def parse_cmd_flags(cmd_flags=None): """ Parses the command line options. :param cmd_flags: Command line options. :type cmd_flags: dict :return: Default parameters of process model. :rtype: tuple """ flags = sh.combine_dicts(cmd_flags or {}, base={ 'only_summary': False, 'hard_validation': False, 'declaration_mode': False, 'enable_selector': False, 'type_approval_mode': False, 'encryption_keys': None, 'sign_key': None, 'output_template': sh.NONE, 'encryption_keys_passwords': None, 'output_folder': './outputs', 'augmented_summary': False }) flags['declaration_mode'] |= flags['type_approval_mode'] flags['hard_validation'] |= flags['declaration_mode'] if flags['declaration_mode'] and not flags['type_approval_mode'] and \ flags['enable_selector']: log.info('Since CO2MPAS is launched in declaration mode the option ' '--enable-selector is not used.\n' 'If you want to use it remove -DM from the cmd.') flags['enable_selector'] = False return sh.selector(_cmd_flags, flags, output_type='list')
def parse_solution(solution): """ Parse the CO2MPAS model solution. :param solution: CO2MPAS model solution. :type solution: schedula.Solution :return: CO2MPAS outputs. :rtype: dict[dict] """ res = {} for k, v in solution.items(): k = k.split('.') sh.get_nested_dicts(res, *k[:-1])[k[-1]] = v for k, v in list(sh.stack_nested_keys(res, depth=3)): n, k = k[:-1], k[-1] if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'): v = sh.selector(('co2_emission_value', ), v, allow_miss=True) if v: d = sh.get_nested_dicts(res, 'target', 'prediction') d[k] = sh.combine_dicts(v, d.get(k, {})) res['pipe'] = solution.pipe return res
def _eval(s, error=None, usersyms=None, **kwargs): error = error or 'cannot be eval!' from asteval import Interpreter usersyms = sh.combine_dicts(_usersyms, usersyms or {}) return Or(And(str, Use(Interpreter(usersyms=usersyms).eval), s), s, error=error)
def xsingle(cell, rng): if len(rng.ranges) == 1 and not rng.is_set and rng.value.shape[1] == 1: rng = rng & Ranges((sh.combine_dicts( rng.ranges[0], sh.selector(('r1', 'r2'), cell.ranges[0])), )) if rng.ranges: return rng return Error.errors['#VALUE!']
def parse_dsp_solution(solution): """ Parses the co2mpas model results. :param solution: Co2mpas model after dispatching. :type solution: schedula.Solution :return: Mapped outputs. :rtype: dict[dict] """ res = {} for k, v in solution.items(): sh.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v)) for k, v in list(sh.stack_nested_keys(res, depth=3)): n, k = k[:-1], k[-1] if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'): v = sh.selector(('co2_emission_value', ), v, allow_miss=True) if v: d = sh.get_nested_dicts(res, 'target', 'prediction') d[k] = sh.combine_dicts(v, d.get(k, {})) res['pipe'] = solution.pipe return res
def __and__(self, other): # Intersection. r = [] for rng in other.ranges: r.extend( _intersect(rng, self.ranges, format_range=self.format_range)) values = sh.combine_dicts(self.values, other.values) is_set = self.is_set or other.is_set return Ranges(r, values, is_set, self.all_values and other.all_values)
def get_range(format_range, ref, context=None): context = context or {} m = _re_range.match(ref).groupdict().items() m = {k: v for k, v in m if v is not None} if 'ref' in m: raise ValueError i = sh.combine_dicts(context, m) return dict(format_range(('name', 'n1', 'n2'), **i))
def write_to_excel(dfs, output_template): """ Writes DataFrames to excel. :param dfs: DataFrames of vehicle output report. :type dfs: dict[str, pandas.DataFrame] :param output_template: Template output. :type output_template: str :return: Excel output file. :rtype: io.BytesIO """ import pandas as pd log.debug('Writing into xl-file based on template(%s)...', output_template) writer, fd = _clone_excel(output_template) xlref, calculate_sheets, charts = [], sorted(writer.sheets), [] for k, v in sorted(dfs.items(), key=_sort_sheets): if not k.startswith('graphs.'): down = True if k.endswith('pa'): kw = {'named_ranges': ('rows', ), 'index': True, 'k0': 1} elif k.endswith('ts'): kw = {'named_ranges': ('columns', ), 'index': False, 'k0': 1} elif k.endswith('proc_info'): down = False kw = {'named_ranges': ()} else: kw = {} xlref.extend(_write_sheets(writer, k, v, down=down, **kw)) else: try: sheet = writer.book.add_worksheet(k) except AttributeError: sheet = writer.book.create_sheet(title=k) charts.append((sheet, v)) for sheet, v in charts: _chart2excel(writer, sheet, v) if xlref: xlref = sorted(sh.combine_dicts(*[x[1] for x in xlref]).items()) xlref = pd.DataFrame(xlref) xlref.set_index([0], inplace=True) _df2excel(writer, 'xlref', xlref, named_ranges=(), index=True, header=False) writer.save() return fd
def _err(model_id, model): gears = func(inputs=sh.combine_dicts(inp, { sgs: model_id, model_id: model }), outputs=['gears'])['gears'] eng = calculate_gear_box_speeds_in(gears, vel, vsr, sv) return calculate_error_coefficients(t_gears, gears, t_eng, eng, vel, sv)
def fast_range2parts(**kw): inputs = sh.selector(_keys, kw, allow_miss=True) for func in (fast_range2parts_v1, fast_range2parts_v2, fast_range2parts_v3): try: return sh.combine_dicts(kw, base=func(**inputs)) except TypeError: pass else: raise ValueError
def __or__(self, other): # Union. base = self.ranges for r0 in other.ranges: stack = [r0] for b in base: s = stack.copy() stack = [] for r in s: stack.extend(_split(b, r, format_range=self.format_range)) base += tuple(stack) values = sh.combine_dicts(self.values, other.values) return Ranges(base, values, True, self.all_values and other.all_values)
def process(self, match, context=None): d = super(Range, self).process(match) if len(d) <= 1 and 'indirect' not in d and 'ref' in d: try: from .function import Function if Function(self.source).name == d['ref']: return {} except TokenError: pass if 'ref' in d: self.attr['is_reference'] = True return range2parts(None, **sh.combine_dicts(context or {}, d))
def write_to_excel(data, output_file_name, template_file_name): import pandas as pd if template_file_name: log.debug('Writing into xl-file(%s) based on template(%s)...', output_file_name, template_file_name) writer = clone_excel(template_file_name, output_file_name) else: log.debug('Writing into xl-file(%s)...', output_file_name) writer = pd.ExcelWriter(output_file_name, engine='xlsxwriter') xlref, calculate_sheets, charts = [], sorted(writer.sheets), [] for k, v in sorted(data.items(), key=_sort_sheets): if not k.startswith('graphs.'): down = True if k.endswith('pa'): kw = {'named_ranges': ('rows', ), 'index': True, 'k0': 1} elif k.endswith('ts'): kw = {'named_ranges': ('columns', ), 'index': False, 'k0': 1} elif k.endswith('proc_info'): down = False kw = {'named_ranges': ()} else: kw = {} xlref.extend(_write_sheets(writer, k, v, down=down, **kw)) else: try: sheet = writer.book.add_worksheet(k) except AttributeError: sheet = writer.book.create_sheet(title=k) charts.append((sheet, v)) for sheet, v in charts: _chart2excel(writer, sheet, v) if xlref: xlref = sorted(sh.combine_dicts(*[x[1] for x in xlref]).items()) xlref = pd.DataFrame(xlref) xlref.set_index([0], inplace=True) _df2excel(writer, 'xlref', xlref, 0, (), index=True, header=False) if calculate_sheets: import formulas xl_model = formulas.ExcelModel() context = xl_model.add_book( writer.book, {'excel': osp.basename(output_file_name)})[1] xl_model.pushes(*calculate_sheets, context=context).finish().calculate() xl_model.write(xl_model.books) writer.save() log.info('Written into xl-file(%s)...', output_file_name)
def _split(base, rng, intersect=None, format_range=range2parts): z = _have_intersect(base, rng) if not z: return rng, if intersect is not None: intersect.update(z) ranges = [] rng = sh.selector(('excel', 'sheet', 'n1', 'n2', 'r1', 'r2'), rng) it = ('n1', 'n2', 1), ('n2', 'n1', -1), ('r1', 'r2', 1), ('r2', 'r1', -1) for i, j, n in it: if z[i] != rng[i]: if j[0] == 'r': r = sh.combine_dicts(rng, {j: str(int(z[i]) - n)}) else: r = sh.combine_dicts(rng, {j: z[i] - n}) r = dict(format_range(('name', 'n1', 'n2'), **r)) ranges.append(r) rng[i] = z[i] return tuple(ranges)
def define_wltp_base_model(wltp_base_model): """ Defines WLTP base model. :param wltp_base_model: WLTP base model params. :type wltp_base_model: dict :return: WLTP base model. :rtype: dict """ import wltp.model as wltp_mdl # noinspection PyProtectedMember return sh.combine_dicts(wltp_mdl._get_model_base(), wltp_base_model)
def _yield_refs(book, context=None): for n in book.defined_names.definedName: if n.value == '#REF!': continue ref, i = n.name.upper(), n.localSheetId rng = Ranges().push(n.value, context=context).ranges[0]['name'] sheet_names = book.sheetnames if i is not None: sheet_names = sheet_names[i:i + 1] for sn in sheet_names: name = range2parts( None, **sh.combine_dicts(context, { 'sheet': sn, 'ref': ref })) yield name['name'], rng
def _parse_values(data, default=None, where=''): default = default or {} for k, v in data.items(): match = _re_params_name.match(k) if k is not None else None if not match and default.get('scope') == 'meta': match = _re_params_name.match('.'.join( filter(bool, ('meta', default.get('meta'), k)))) if not match: log.warning("Parameter '%s' %s cannot be parsed!", k, where) continue elif _isempty(v): continue match = {i: j.lower() for i, j in match.groupdict().items() if j} for key in _parse_key(**sh.combine_dicts(default, match)): yield key, v
def _convert_limits(lu, vsr, n_vsr, stop_velocity=dfl.values.stop_velocity): _r, _l, _u = np.array( sorted([vsr.get(k, 0)] + list(v) for k, v in lu.items())).T nk, nr = np.array(sorted(sh.combine_dicts(n_vsr, base={0: 0}).items())).T klu = sorted( zip(nk.astype(int), _interp(_r[2:], _l[2:], nr), _interp(_r[1:-1], _u[1:-1], nr))) res, n0, n1 = {}, min(k for k in n_vsr if k > 0), max(n_vsr) for k, l, u in klu: if k == 0: l, u = list(lu.get(k, [0, stop_velocity + 1])) else: if k == n0: l = lu.get(k, (stop_velocity, ))[0] if k == n1: u = lu.get(k, (None, dfl.INF))[1] res[k] = [l, u] return res
def get_summary(solutions): """ Extract summary data from model solutions. :param solutions: All model solutions. :type solutions: list[schedula.Solution] :return: Summary data. :rtype: list """ return [sh.combine_dicts( dict(sh.stack_nested_keys(sol.get('summary', {}), depth=4)), base={ 'id': sol['vehicle_name'], 'base': sol['input_file_name'] } ) for sol in solutions]
def _get_theoretical(profile): defaults = { 'cycle_type': 'WLTP', 'gear_box_type': 'manual', 'wltp_class': 'class3b', 'downscale_factor': 0 } profile = {k: v for k, v in profile.items() if v} profile = sh.combine_dicts(defaults, profile) profile['cycle_type'] = profile['cycle_type'].upper() profile['wltp_class'] = profile['wltp_class'].lower() profile['gear_box_type'] = profile['gear_box_type'].lower() from co2mpas.model.physical.cycle import cycle res = cycle().dispatch(inputs=profile, outputs=['times', 'velocities'], shrink=True) data = sh.selector(['times', 'velocities'], res, output_type='list') return pd.DataFrame(data).T
def make_simulation_plan(plan, timestamp, variation, flag, model=None): model, summary = model or batch.vehicle_processing_model(), {} run_base = model.get_node('run_base')[0].dsp run_modes = tuple( run_base.get_sub_dsp_from_workflow( ('data', 'vehicle_name'), check_inputs=False, graph=run_base.dmap).data_nodes) + ('start_time', 'vehicle_name') var = json.dumps(variation, sort_keys=True) o_cache, o_folder = flag['overwrite_cache'], flag['output_folder'] modelconf = flag.get('modelconf', None) kw, bases = sh.combine_dicts(flag, {'run_base': True}), set() for (i, base_fpath, run), p in tqdm.tqdm(plan, disable=False): try: base = get_results(model, o_cache, base_fpath, timestamp, run, var, o_folder, modelconf) except KeyError: log.warning('Base model "%s" of variation "%s" cannot be parsed!', base_fpath, i) continue name = base['vehicle_name'] if 'summary' in base and name not in bases: batch._add2summary(summary, base['summary']) bases.add(name) name = '{}-{}'.format(name, i) new_base, o = define_new_inputs(p, base) inputs = batch.prepare_data(new_base, {}, base_fpath, o_cache, o_folder, timestamp, False, modelconf)[0] inputs.update(sh.selector(set(base).difference(run_modes), base)) inputs['vehicle_name'] = name inputs.update(kw) res = run_base.dispatch(inputs) batch.notify_result_listener(plan_listener, {'solution': res}) s = filter_summary(p, o, res.get('summary', {})) base_keys = { 'vehicle_name': (base_fpath, name, run), } batch._add2summary(summary, s, base_keys) return summary
def add_cell(self, cell, context, references=None, formula_references=None, formula_ranges=None, external_links=None): get_in = sh.get_nested_dicts if formula_references is None: formula_references = get_in(self.books, context['excel'], SHEETS, context['sheet'], 'formula_references') if formula_ranges is None: formula_ranges = get_in(self.books, context['excel'], SHEETS, context['sheet'], 'formula_ranges') if references is None: references = get_in(self.books, context['excel'], 'references') if external_links is None: external_links = get_in(self.books, context['excel'], 'external_links') context = sh.combine_dicts(context, base={'external_links': external_links}) crd = cell.coordinate crd = formula_references.get(crd, crd) #cell = Cell(crd, cell.value, context=context).compile() try: cell = Cell(crd, cell.value, context=context).compile() except FormulaError: # There was an error so set this value with the NA error using the excel function cell = Cell(crd, "=NA()", context=context).compile() if cell.output in self.cells: return if cell.value is not sh.EMPTY: if any(not (cell.range - rng).ranges for rng in formula_ranges): return cell.update_inputs(references=references) if cell.add(self.dsp, context=context): self.cells[cell.output] = cell return cell
def test_combine_dicts(self): res = sh.combine_dicts({'a': 3, 'c': 3}, {'a': 1, 'b': 2}) self.assertEqual(res, {'a': 1, 'b': 2, 'c': 3})