def get_report_output_data(data):
"""
Produces a vehicle report from CO2MPAS outputs.
:param data:
:return:
"""
data = data.copy()
report = {}
if 'pipe' in data:
report['pipe'] = data['pipe']
target = re_sample_targets(data)
if target:
data['target'] = target
summary = format_report_summary(data)
if summary:
report['summary'] = summary
output = format_report_output(data)
if output:
report['output'] = output
scores = format_report_scores(data)
if scores:
sh.combine_nested_dicts(scores, base=report)
graphs = get_chart_reference(report)
if graphs:
report['graphs'] = graphs
return report
def _run_variations(plan, bases, core_model, timestamp):
for r in _ProgressBar(plan, _format_meter=_format_meter):
sol, data = bases[r['base']], r['data']
if 'solution' in sol:
s = sol['solution']
base = _define_inputs(s, sh.combine_nested_dicts(sh.selector(
data, s, allow_miss=True
), data))
elif 'base' in sol:
base = sh.combine_nested_dicts(sol['base'], data, depth=2)
else:
continue
for i, d in base.items():
if hasattr(d, 'items'):
base[i] = {k: v for k, v in d.items() if v is not sh.EMPTY}
sol = core_model(_define_inputs(sol, dict(
base=base,
vehicle_name='-'.join((str(r['id']), sol['vehicle_name'])),
timestamp=timestamp
)))
summary, keys = {}, {
tuple(k.split('.')[:0:-1]) for k in base if k.startswith('output.')
}
for k, v in data.items():
k = ('plan %s' % k).split('.')[::-1]
sh.get_nested_dicts(summary, *k).update(v)
for k, v in sh.stack_nested_keys(sol['summary'], depth=3):
if k[:-1] not in keys:
sh.get_nested_dicts(summary, *k).update(v)
sol['summary'] = summary
yield sol
def setUp(self):
res_dir = osp.join(osp.dirname(__file__), 'results')
out_fpath = 'gear_4degree_curves_with_linear_gs.yaml'
with open(osp.join(res_dir, out_fpath)) as f:
self.data = dict(outputs=yaml.load(f, yaml.CLoader))
with open(osp.join(res_dir, 'core.yaml')) as f:
sh.combine_nested_dicts(yaml.load(f, yaml.CLoader),
base=self.data,
depth=2)
def test_combine_nested_dicts(self):
d1 = {'a': {'b': {'c': ('d',), 0: 1}}, 'A': {'B': ('C',), 0: 1}, 0: 1}
d2 = {'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}}
base = {0: 0, 1: 2}
output = sh.combine_nested_dicts(d1, d2, base=base)
result = {0: 1, 1: 2, 'A': {0: 1, 'B': {'C': 'D'}}, 'a': {'b': 'c'}}
self.assertEqual(output, result)
self.assertIs(output, base)
output = sh.combine_nested_dicts(d1, d2, depth=1)
result = {0: 1, 'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}}
self.assertEqual(output, result)
def parse_excel_file(file_path):
"""
Reads cycle's data and simulation plans.
:param file_path:
Excel file path.
:type file_path: str
:return:
A pandas DataFrame with cycle's time series.
:rtype: dict, pandas.DataFrame
"""
import pandas as pd
try:
excel_file = pd.ExcelFile(file_path)
except FileNotFoundError:
log.error("No such file or directory: '%s'", file_path)
return sh.NONE
res, plans = {}, []
book = excel_file.book
import pandalone.xleash.io._xlrd as pnd_xlrd
for sheet_name in excel_file.sheet_names:
match = _re_input_sheet_name.match(sheet_name)
if not match:
log.debug("Sheet name '%s' cannot be parsed!", sheet_name)
continue
match = {k: v.lower() for k, v in match.groupdict().items() if v}
sheet = pnd_xlrd._open_sheet_by_name_or_index(book, 'book', sheet_name)
is_plan = match.get('scope', None) == 'plan'
if is_plan:
r = {'plan': pd.DataFrame()}
else:
r = {}
r = _parse_sheet(match, sheet, sheet_name, res=r)
if is_plan:
plans.append(r['plan'])
else:
_add_times_base(r, **match)
sh.combine_nested_dicts(r, depth=5, base=res)
for k, v in sh.stack_nested_keys(res.get('base', {}), depth=3):
if k[0] != 'target':
v['cycle_type'] = v.get('cycle_type', k[-1].split('_')[0]).upper()
v['cycle_name'] = v.get('cycle_name', k[-1]).upper()
res['plan'] = _finalize_plan(res, plans, file_path).to_dict('split')
return res
def parse_excel_file(input_file_name, input_file):
"""
Reads cycle's data and simulation plans.
:param input_file_name:
Input file name.
:type input_file_name: str
:param input_file:
Input file.
:type input_file: io.BytesIO
:return:
Raw input data.
:rtype: dict
"""
import pandas as pd
# noinspection PyProtectedMember
from pandalone.xleash.io._xlrd import _open_sheet_by_name_or_index
xl_file, res, plans = pd.ExcelFile(input_file), {'base': {}}, []
for sheet_name in xl_file.sheet_names:
match = _re_input_sheet_name.match(sheet_name.strip(' '))
if not match:
log.debug("Sheet name '%s' cannot be parsed!", sheet_name)
continue
match = {k: v.lower() for k, v in match.groupdict().items() if v}
# noinspection PyProtectedMember
sheet = _open_sheet_by_name_or_index(xl_file.book, 'book', sheet_name)
is_plan = match.get('scope', None) == 'plan'
if is_plan:
r = {'plan': pd.DataFrame()}
else:
r = {}
r = _parse_sheet(match, sheet, sheet_name, res=r)
if is_plan:
plans.append(r['plan'])
else:
_add_times_base(r, **match)
sh.combine_nested_dicts(r, depth=5, base=res)
for k, v in sh.stack_nested_keys(res['base'], depth=3):
if k[0] != 'target':
v['cycle_type'] = v.get('cycle_type', k[-1].split('_')[0]).upper()
v['cycle_name'] = v.get('cycle_name', k[-1]).upper()
res['plan'] = _finalize_plan(res, plans, input_file_name).to_dict('records')
return res
def format_report_output_data(output_data):
"""
Produces a vehicle output report from CO2MPAS outputs.
:param output_data:
CO2MPAS outputs.
:type output_data: dict
:return:
Vehicle output report.
:rtype: dict
"""
output_data = output_data.copy()
report = {}
if 'pipe' in output_data:
report['pipe'] = output_data['pipe']
target = _re_sample_targets(output_data)
if target:
output_data['target'] = target
summary = _format_report_summary(output_data)
if summary:
report['summary'] = summary
output = _format_report_output(output_data)
if output:
report['output'] = output
scores = _format_report_scores(output_data)
if scores:
sh.combine_nested_dicts(scores, base=report)
graphs = _get_chart_reference(report)
if graphs:
report['graphs'] = graphs
return report
def overwrite_declaration_config_data(data):
config = constants.con_vals.DECLARATION_SELECTOR_CONFIG
res = sh.combine_nested_dicts(data, depth=3)
key = ('config', 'selector', 'all')
d = copy.deepcopy(sh.get_nested_dicts(res, *key))
for k, v in sh.stack_nested_keys(config):
sh.get_nested_dicts(d, *k, default=co2_utl.ret_v(v))
sh.get_nested_dicts(res, *key[:-1])[key[-1]] = d
return res
def test_combine_nested_dicts(self):
d1 = {
'a': {
'b': {
'c': ('d', ),
0: 1
}
},
'A': {
'B': ('C', ),
0: 1
},
0: 1
}
d2 = {'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}}
base = {0: 0, 1: 2}
output = sh.combine_nested_dicts(d1, d2, base=base)
result = {0: 1, 1: 2, 'A': {0: 1, 'B': {'C': 'D'}}, 'a': {'b': 'c'}}
self.assertEqual(output, result)
self.assertIs(output, base)
output = sh.combine_nested_dicts(d1, d2, depth=1)
result = {0: 1, 'A': {'B': {'C': 'D'}}, 'a': {'b': 'c'}}
self.assertEqual(output, result)
def merge_data(vehicle_inputs, raw_data, inputs):
"""
Merge data.
:param vehicle_inputs:
Vehicle inputs.
:type vehicle_inputs: dict
:param raw_data:
Raw data of input file.
:type raw_data: dict
:param inputs:
User inputs.
:type inputs:
:return:
Merged data.
:rtype: dict
"""
d = {"vehicle_inputs": vehicle_inputs}
d = sh.combine_nested_dicts(d, raw_data, inputs, depth=2)
return sh.combine_dicts(d.pop("time_series", {}),
d.pop("vehicle_inputs", {}), d.get("inputs", {}))
def prepare_data(raw_data, variation, input_file_name, overwrite_cache,
output_folder, timestamp, type_approval_mode, modelconf):
"""
Prepare the data to be processed.
:param raw_data:
Raw data from the input file.
:type raw_data: dict
:param variation:
Variations to be applied.
:type variation: dict
:param input_file_name:
Input file name.
:type input_file_name: str
:param overwrite_cache:
Overwrite saved cache?
:type overwrite_cache: bool
:param output_folder:
Output folder.
:type output_folder: str
:param timestamp:
Run timestamp.
:type timestamp: str
:param type_approval_mode:
Is launched for TA?
:type type_approval_mode: bool
:param modelconf:
Path of modelconf that has modified the defaults.
:type modelconf: str
:return:
Prepared data.
:rtype: dict
"""
raw_data = raw_data.copy()
import pandas as pd
raw_data['plan'] = pd.DataFrame(**raw_data.get('plan', {}))
has_plan = not raw_data['plan'].empty
match = {
'scope': 'plan' if has_plan else 'base',
}
r = {}
from pandalone.xleash import SheetsFactory, lasso
from co2mpas.io import check_xlasso
import pandas as pd
sheets_factory = SheetsFactory()
for k, v in excel._parse_values(variation, match, "in variations"):
if isinstance(v, str) and check_xlasso(v):
v = lasso(v, sheets_factory, url_file=input_file_name)
sh.get_nested_dicts(r, *k[:-1])[k[-1]] = v
if 'plan' in r:
if has_plan:
plan = raw_data['plan'].copy()
for k, v in sh.stack_nested_keys(r['plan'], depth=4):
plan['.'.join(k)] = v
else:
gen = sh.stack_nested_keys(r['plan'], depth=4)
plan = pd.DataFrame([{'.'.join(k): v for k, v in gen}])
excel._add_index_plan(plan, input_file_name)
r['plan'] = plan
has_plan = True
if 'base' in r:
r['base'] = sh.combine_nested_dicts(raw_data.get('base', {}),
r['base'],
depth=4)
if 'flag' in r:
r['flag'] = sh.combine_nested_dicts(raw_data.get('flag', {}),
r['flag'],
depth=1)
if 'meta' in r:
r['meta'] = sh.combine_nested_dicts(raw_data.get('meta', {}),
r['meta'],
depth=2)
data = sh.combine_dicts(raw_data, r)
if type_approval_mode:
variation, has_plan = {}, False
if not schema._ta_mode(data):
return {}, pd.DataFrame([])
flag = data.get('flag', {}).copy()
if 'run_base' not in flag:
flag['run_base'] = not has_plan
if 'run_plan' not in flag:
flag['run_plan'] = has_plan
flag['type_approval_mode'] = type_approval_mode
flag['output_folder'] = output_folder
flag['overwrite_cache'] = overwrite_cache
if modelconf:
flag['modelconf'] = modelconf
if timestamp is not None:
flag['timestamp'] = timestamp
flag = schema.validate_flags(flag)
if flag is sh.NONE:
return {}, pd.DataFrame([])
schema.check_data_version(flag)
res = {
'flag': flag,
'meta': data.get('meta', {}),
'variation': variation,
'input_file_name': input_file_name,
}
res = sh.combine_dicts(flag, res)
base = sh.combine_dicts(res, {'data': data.get('base', {})})
plan = sh.combine_dicts(res, {'data': data.get('plan', pd.DataFrame([]))})
return base, plan
