def get_report_output_data(data): """ Produces a vehicle report from CO2MPAS outputs. :param data: :return: """ data = data.copy() report = {} if 'pipe' in data: report['pipe'] = data['pipe'] target = re_sample_targets(data) if target: data['target'] = target summary = format_report_summary(data) if summary: report['summary'] = summary output = format_report_output(data) if output: report['output'] = output scores = format_report_scores(data) if scores: sh.combine_nested_dicts(scores, base=report) graphs = get_chart_reference(report) if graphs: report['graphs'] = graphs return report
def _run_variations(plan, bases, core_model, timestamp): for r in _ProgressBar(plan, _format_meter=_format_meter): sol, data = bases[r['base']], r['data'] if 'solution' in sol: s = sol['solution'] base = _define_inputs(s, sh.combine_nested_dicts(sh.selector( data, s, allow_miss=True ), data)) elif 'base' in sol: base = sh.combine_nested_dicts(sol['base'], data, depth=2) else: continue for i, d in base.items(): if hasattr(d, 'items'): base[i] = {k: v for k, v in d.items() if v is not sh.EMPTY} sol = core_model(_define_inputs(sol, dict( base=base, vehicle_name='-'.join((str(r['id']), sol['vehicle_name'])), timestamp=timestamp ))) summary, keys = {}, { tuple(k.split('.')[:0:-1]) for k in base if k.startswith('output.') } for k, v in data.items(): k = ('plan %s' % k).split('.')[::-1] sh.get_nested_dicts(summary, *k).update(v) for k, v in sh.stack_nested_keys(sol['summary'], depth=3): if k[:-1] not in keys: sh.get_nested_dicts(summary, *k).update(v) sol['summary'] = summary yield sol
def setUp(self): res_dir = osp.join(osp.dirname(__file__), 'results') out_fpath = 'gear_4degree_curves_with_linear_gs.yaml' with open(osp.join(res_dir, out_fpath)) as f: self.data = dict(outputs=yaml.load(f, yaml.CLoader)) with open(osp.join(res_dir, 'core.yaml')) as f: sh.combine_nested_dicts(yaml.load(f, yaml.CLoader), base=self.data, depth=2)
def test_combine_nested_dicts(self): d1 = {'a': {'b': {'c': ('d',), 0: 1}}, 'A': {'B': ('C',), 0: 1}, 0: 1} d2 = {'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}} base = {0: 0, 1: 2} output = sh.combine_nested_dicts(d1, d2, base=base) result = {0: 1, 1: 2, 'A': {0: 1, 'B': {'C': 'D'}}, 'a': {'b': 'c'}} self.assertEqual(output, result) self.assertIs(output, base) output = sh.combine_nested_dicts(d1, d2, depth=1) result = {0: 1, 'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}} self.assertEqual(output, result)
def parse_excel_file(file_path): """ Reads cycle's data and simulation plans. :param file_path: Excel file path. :type file_path: str :return: A pandas DataFrame with cycle's time series. :rtype: dict, pandas.DataFrame """ import pandas as pd try: excel_file = pd.ExcelFile(file_path) except FileNotFoundError: log.error("No such file or directory: '%s'", file_path) return sh.NONE res, plans = {}, [] book = excel_file.book import pandalone.xleash.io._xlrd as pnd_xlrd for sheet_name in excel_file.sheet_names: match = _re_input_sheet_name.match(sheet_name) if not match: log.debug("Sheet name '%s' cannot be parsed!", sheet_name) continue match = {k: v.lower() for k, v in match.groupdict().items() if v} sheet = pnd_xlrd._open_sheet_by_name_or_index(book, 'book', sheet_name) is_plan = match.get('scope', None) == 'plan' if is_plan: r = {'plan': pd.DataFrame()} else: r = {} r = _parse_sheet(match, sheet, sheet_name, res=r) if is_plan: plans.append(r['plan']) else: _add_times_base(r, **match) sh.combine_nested_dicts(r, depth=5, base=res) for k, v in sh.stack_nested_keys(res.get('base', {}), depth=3): if k[0] != 'target': v['cycle_type'] = v.get('cycle_type', k[-1].split('_')[0]).upper() v['cycle_name'] = v.get('cycle_name', k[-1]).upper() res['plan'] = _finalize_plan(res, plans, file_path).to_dict('split') return res
def parse_excel_file(input_file_name, input_file): """ Reads cycle's data and simulation plans. :param input_file_name: Input file name. :type input_file_name: str :param input_file: Input file. :type input_file: io.BytesIO :return: Raw input data. :rtype: dict """ import pandas as pd # noinspection PyProtectedMember from pandalone.xleash.io._xlrd import _open_sheet_by_name_or_index xl_file, res, plans = pd.ExcelFile(input_file), {'base': {}}, [] for sheet_name in xl_file.sheet_names: match = _re_input_sheet_name.match(sheet_name.strip(' ')) if not match: log.debug("Sheet name '%s' cannot be parsed!", sheet_name) continue match = {k: v.lower() for k, v in match.groupdict().items() if v} # noinspection PyProtectedMember sheet = _open_sheet_by_name_or_index(xl_file.book, 'book', sheet_name) is_plan = match.get('scope', None) == 'plan' if is_plan: r = {'plan': pd.DataFrame()} else: r = {} r = _parse_sheet(match, sheet, sheet_name, res=r) if is_plan: plans.append(r['plan']) else: _add_times_base(r, **match) sh.combine_nested_dicts(r, depth=5, base=res) for k, v in sh.stack_nested_keys(res['base'], depth=3): if k[0] != 'target': v['cycle_type'] = v.get('cycle_type', k[-1].split('_')[0]).upper() v['cycle_name'] = v.get('cycle_name', k[-1]).upper() res['plan'] = _finalize_plan(res, plans, input_file_name).to_dict('records') return res
def format_report_output_data(output_data): """ Produces a vehicle output report from CO2MPAS outputs. :param output_data: CO2MPAS outputs. :type output_data: dict :return: Vehicle output report. :rtype: dict """ output_data = output_data.copy() report = {} if 'pipe' in output_data: report['pipe'] = output_data['pipe'] target = _re_sample_targets(output_data) if target: output_data['target'] = target summary = _format_report_summary(output_data) if summary: report['summary'] = summary output = _format_report_output(output_data) if output: report['output'] = output scores = _format_report_scores(output_data) if scores: sh.combine_nested_dicts(scores, base=report) graphs = _get_chart_reference(report) if graphs: report['graphs'] = graphs return report
def overwrite_declaration_config_data(data): config = constants.con_vals.DECLARATION_SELECTOR_CONFIG res = sh.combine_nested_dicts(data, depth=3) key = ('config', 'selector', 'all') d = copy.deepcopy(sh.get_nested_dicts(res, *key)) for k, v in sh.stack_nested_keys(config): sh.get_nested_dicts(d, *k, default=co2_utl.ret_v(v)) sh.get_nested_dicts(res, *key[:-1])[key[-1]] = d return res
def test_combine_nested_dicts(self): d1 = { 'a': { 'b': { 'c': ('d', ), 0: 1 } }, 'A': { 'B': ('C', ), 0: 1 }, 0: 1 } d2 = {'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}} base = {0: 0, 1: 2} output = sh.combine_nested_dicts(d1, d2, base=base) result = {0: 1, 1: 2, 'A': {0: 1, 'B': {'C': 'D'}}, 'a': {'b': 'c'}} self.assertEqual(output, result) self.assertIs(output, base) output = sh.combine_nested_dicts(d1, d2, depth=1) result = {0: 1, 'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}} self.assertEqual(output, result)
def merge_data(vehicle_inputs, raw_data, inputs): """ Merge data. :param vehicle_inputs: Vehicle inputs. :type vehicle_inputs: dict :param raw_data: Raw data of input file. :type raw_data: dict :param inputs: User inputs. :type inputs: :return: Merged data. :rtype: dict """ d = {"vehicle_inputs": vehicle_inputs} d = sh.combine_nested_dicts(d, raw_data, inputs, depth=2) return sh.combine_dicts(d.pop("time_series", {}), d.pop("vehicle_inputs", {}), d.get("inputs", {}))
def prepare_data(raw_data, variation, input_file_name, overwrite_cache, output_folder, timestamp, type_approval_mode, modelconf): """ Prepare the data to be processed. :param raw_data: Raw data from the input file. :type raw_data: dict :param variation: Variations to be applied. :type variation: dict :param input_file_name: Input file name. :type input_file_name: str :param overwrite_cache: Overwrite saved cache? :type overwrite_cache: bool :param output_folder: Output folder. :type output_folder: str :param timestamp: Run timestamp. :type timestamp: str :param type_approval_mode: Is launched for TA? :type type_approval_mode: bool :param modelconf: Path of modelconf that has modified the defaults. :type modelconf: str :return: Prepared data. :rtype: dict """ raw_data = raw_data.copy() import pandas as pd raw_data['plan'] = pd.DataFrame(**raw_data.get('plan', {})) has_plan = not raw_data['plan'].empty match = { 'scope': 'plan' if has_plan else 'base', } r = {} from pandalone.xleash import SheetsFactory, lasso from co2mpas.io import check_xlasso import pandas as pd sheets_factory = SheetsFactory() for k, v in excel._parse_values(variation, match, "in variations"): if isinstance(v, str) and check_xlasso(v): v = lasso(v, sheets_factory, url_file=input_file_name) sh.get_nested_dicts(r, *k[:-1])[k[-1]] = v if 'plan' in r: if has_plan: plan = raw_data['plan'].copy() for k, v in sh.stack_nested_keys(r['plan'], depth=4): plan['.'.join(k)] = v else: gen = sh.stack_nested_keys(r['plan'], depth=4) plan = pd.DataFrame([{'.'.join(k): v for k, v in gen}]) excel._add_index_plan(plan, input_file_name) r['plan'] = plan has_plan = True if 'base' in r: r['base'] = sh.combine_nested_dicts(raw_data.get('base', {}), r['base'], depth=4) if 'flag' in r: r['flag'] = sh.combine_nested_dicts(raw_data.get('flag', {}), r['flag'], depth=1) if 'meta' in r: r['meta'] = sh.combine_nested_dicts(raw_data.get('meta', {}), r['meta'], depth=2) data = sh.combine_dicts(raw_data, r) if type_approval_mode: variation, has_plan = {}, False if not schema._ta_mode(data): return {}, pd.DataFrame([]) flag = data.get('flag', {}).copy() if 'run_base' not in flag: flag['run_base'] = not has_plan if 'run_plan' not in flag: flag['run_plan'] = has_plan flag['type_approval_mode'] = type_approval_mode flag['output_folder'] = output_folder flag['overwrite_cache'] = overwrite_cache if modelconf: flag['modelconf'] = modelconf if timestamp is not None: flag['timestamp'] = timestamp flag = schema.validate_flags(flag) if flag is sh.NONE: return {}, pd.DataFrame([]) schema.check_data_version(flag) res = { 'flag': flag, 'meta': data.get('meta', {}), 'variation': variation, 'input_file_name': input_file_name, } res = sh.combine_dicts(flag, res) base = sh.combine_dicts(res, {'data': data.get('base', {})}) plan = sh.combine_dicts(res, {'data': data.get('plan', pd.DataFrame([]))}) return base, plan