def _add2summary(total_summary, summary, base_keys=None):
base_keys = base_keys or {}
for k, v in dsp_utl.stack_nested_keys(summary, depth=3):
d = dsp_utl.get_nested_dicts(total_summary, *k, default=list)
if isinstance(v, list):
for j in v:
d.append(dsp_utl.combine_dicts(j, base_keys))
else:
d.append(dsp_utl.combine_dicts(v, base_keys))
def _summary2df(data):
res = []
summary = data.get('summary', {})
if 'results' in summary:
r = {}
index = ['cycle', 'stage', 'usage']
for k, v in dsp_utl.stack_nested_keys(summary['results'], depth=4):
l = dsp_utl.get_nested_dicts(r, k[0], default=list)
l.append(dsp_utl.combine_dicts(dsp_utl.map_list(index, *k[1:]), v))
if r:
df = _dd2df(r,
index=index,
depth=2,
col_key=functools.partial(_sort_key,
p_keys=('param', ) * 2),
row_key=functools.partial(_sort_key, p_keys=index))
df.columns = pd.MultiIndex.from_tuples(_add_units(df.columns))
setattr(df, 'name', 'results')
res.append(df)
if 'selection' in summary:
df = _dd2df(summary['selection'], ['model_id'],
depth=2,
col_key=functools.partial(_sort_key,
p_keys=('stage', 'cycle')),
row_key=functools.partial(_sort_key, p_keys=()))
setattr(df, 'name', 'selection')
res.append(df)
if 'comparison' in summary:
r = {}
for k, v in dsp_utl.stack_nested_keys(summary['comparison'], depth=3):
v = dsp_utl.combine_dicts(v, base={'param_id': k[-1]})
dsp_utl.get_nested_dicts(r, *k[:-1], default=list).append(v)
if r:
df = _dd2df(r, ['param_id'],
depth=2,
col_key=functools.partial(_sort_key,
p_keys=('stage', 'cycle')),
row_key=functools.partial(_sort_key, p_keys=()))
setattr(df, 'name', 'comparison')
res.append(df)
if res:
return {'summary': res}
return {}
def write_to_excel(data, output_file_name, template_file_name):
if template_file_name:
log.debug('Writing into xl-file(%s) based on template(%s)...',
output_file_name, template_file_name)
shutil.copy(template_file_name, output_file_name)
writer = clone_excel(template_file_name, output_file_name)
else:
log.debug('Writing into xl-file(%s)...', output_file_name)
writer = pd.ExcelWriter(output_file_name, engine='xlsxwriter')
xlref = []
for k, v in sorted(data.items(), key=_sort_sheets):
if not k.startswith('graphs.'):
if k.endswith('pa'):
kw = {'named_ranges': ('rows', ), 'index': True, 'k0': 1}
elif k.endswith('ts'):
kw = {'named_ranges': ('columns', ), 'index': False, 'k0': 1}
else:
kw = {}
down = not k.endswith('proc_info')
xlref.extend(_write_sheets(writer, k, v, down=down, **kw))
else:
_chart2excel(writer, k, v)
if xlref:
xlref = sorted(dsp_utl.combine_dicts(*[x[1] for x in xlref]).items())
xlref = pd.DataFrame(xlref)
xlref.set_index([0], inplace=True)
_df2excel(writer, 'xlref', xlref, 0, (), index=True, header=False)
writer.save()
log.info('Written into xl-file(%s)...', output_file_name)
def select_prediction_data(data, new_data=(), theoretical=True):
"""
Selects the data required to predict the CO2 emissions with CO2MPAS model.
:param data:
Output data.
:type data: dict
:param new_data:
New data.
:type new_data: dict
:param theoretical:
If false
:type theoretical: bool
:return:
Data required to predict the CO2 emissions with CO2MPAS model.
:rtype: dict
"""
ids = [
'angle_slope', 'alternator_nominal_voltage', 'alternator_efficiency',
'battery_capacity', 'cycle_type', 'cycle_name', 'engine_capacity',
'engine_stroke', 'engine_thermostat_temperature',
'final_drive_efficiency', 'frontal_area',
'aerodynamic_drag_coefficient', 'fuel_type', 'ignition_type',
'gear_box_type', 'engine_max_power', 'engine_max_speed_at_max_power',
'rolling_resistance_coeff', 'time_cold_hot_transition',
'engine_idle_fuel_consumption', 'engine_type', 'engine_is_turbo',
'engine_fuel_lower_heating_value', 'has_start_stop',
'has_energy_recuperation', 'fuel_carbon_content_percentage', 'f0',
'f1', 'f2', 'vehicle_mass', 'full_load_speeds', 'plateau_acceleration',
'full_load_powers', 'fuel_saving_at_strategy',
'stand_still_torque_ratio', 'lockup_speed_ratio',
'change_gear_window_width', 'alternator_start_window_width',
'stop_velocity', 'min_time_engine_on_after_start',
'min_engine_on_speed', 'max_velocity_full_load_correction',
'is_hybrid', 'tyre_code', 'engine_has_cylinder_deactivation',
'active_cylinder_ratios', 'engine_has_variable_valve_actuation',
'has_torque_converter', 'has_gear_box_thermal_management',
'has_lean_burn', 'ki_factor', 'n_wheel_drive',
'has_periodically_regenerating_systems',
'has_selective_catalytic_reduction', 'has_exhausted_gas_recirculation'
]
if not theoretical:
ids += ['times', 'velocities', 'gears']
data = dsp_utl.selector(ids, data, allow_miss=True)
if new_data:
data = dsp_utl.combine_dicts(data, new_data)
if 'gears' in data and 'gears' not in new_data:
if data.get('gear_box_type', 0) == 'automatic' or \
len(data.get('velocities', ())) != len(data['gears']):
data.pop('gears')
return data
def _parse_plan_data(plans,
match,
sheet,
sheet_name,
re_params_name=_re_params_name):
# noinspection PyBroadException
xl_ref = '#%s!A1(R):._:R:"recurse"'
data = lasso(xl_ref % sheet_name, sheet=sheet)
try:
data = pd.DataFrame(data[1:], columns=data[0])
except IndexError:
return None
if 'id' not in data:
data['id'] = data.index + 1
data.set_index(['id'], inplace=True)
data.dropna(how='all', inplace=True)
data.dropna(axis=1, how='all', inplace=True)
plan = pd.DataFrame()
defaults = {'usage': 'input', 'stage': 'calibration'}
match = dsp_utl.combine_dicts(defaults, match)
for k, v in parse_values(data, match, re_params_name):
k = k[-1] if k[-1] in ('base', 'defaults') else '.'.join(k[1:])
plan[k] = v
plans.append(plan)
def parse_dsp_solution(solution):
"""
Parses the co2mpas model results.
:param solution:
Co2mpas model after dispatching.
:type solution: co2mpas.dispatcher.Solution
:return:
Mapped outputs.
:rtype: dict[dict]
"""
res = {}
for k, v in solution.items():
dsp_utl.get_nested_dicts(res, *k.split('.'), default=co2_utl.ret_v(v))
for k, v in list(dsp_utl.stack_nested_keys(res, depth=3)):
n, k = k[:-1], k[-1]
if n == ('output', 'calibration') and k in ('wltp_l', 'wltp_h'):
v = dsp_utl.selector(('co2_emission_value', ), v, allow_miss=True)
if v:
d = dsp_utl.get_nested_dicts(res, 'target', 'prediction')
d[k] = dsp_utl.combine_dicts(v, d.get(k, {}))
res['pipe'] = solution.pipe
return res
def _process_vehicle(model, plot_workflow=False, **kw):
inputs = {'plot_workflow': plot_workflow}
res = model.dispatch(inputs=dsp_utl.combine_dicts(inputs, kw))
plot_model_workflow(model, **res)
return res
def _comparison2df(comparison):
res = {}
it = co2_utl.stack_nested_keys(comparison, depth=3)
keys = ['usage', 'cycle', 'param']
gen = [(dsp_utl.map_list(keys, *k), k, v) for k, v in it]
for s, k, v in _yield_sorted_params(gen, keys=keys):
l = co2_utl.get_nested_dicts(res, *k[:-1], default=list)
l.append(dsp_utl.combine_dicts({'param_id': k[-1]}, v))
if res:
return _dd2df(res, 'param_id', depth=2)
def _parse_base_data(res,
match,
sheet,
sheet_name,
re_params_name=_re_params_name):
r = {}
defaults = {'usage': 'input', 'stage': 'calibration'}
if 'type' not in match:
match['type'] = 'pa' if 'cycle' not in match else 'ts'
match = dsp_utl.combine_dicts(defaults, match)
if match['type'] == 'pa':
xl_ref = '#%s!B2:C_:["pipe", ["dict", "recurse"]]' % sheet_name
data = lasso(xl_ref, sheet=sheet)
else:
# noinspection PyBroadException
try:
xl_ref = '#%s!A2(R):.3:RD:["df", {"header": 0}]' % sheet_name
data = lasso(xl_ref, sheet=sheet)
except:
return {}
data.dropna(how='all', inplace=True)
data.dropna(axis=1, how='all', inplace=True)
mask = data.count(0) == len(data._get_axis(0))
# noinspection PyUnresolvedReferences
drop = [k for k, v in mask.items() if not v]
if drop:
msg = 'Columns {} in {} sheet contains nan.\n ' \
'Please correct the inputs!'
raise ValueError(msg.format(drop, sheet_name))
for k, v in parse_values(data, match, re_params_name):
co2_utl.get_nested_dicts(r, *k[:-1])[k[-1]] = v
n = (match['scope'], 'target')
if match['type'] == 'ts' and co2_utl.are_in_nested_dicts(r, *n):
t = co2_utl.get_nested_dicts(r, *n)
for k, v in co2_utl.stack_nested_keys(t, key=n, depth=2):
if 'times' not in v:
n = list(k + ('times', ))
n[1] = match['usage']
if co2_utl.are_in_nested_dicts(r, *n):
v['times'] = co2_utl.get_nested_dicts(r, *n)
else:
for i, j in co2_utl.stack_nested_keys(r, depth=4):
if 'times' in j:
v['times'] = j['times']
break
co2_utl.combine_nested_dicts(r, depth=5, base=res)
def parse_excel_file(file_path,
re_sheet_name=_re_input_sheet_name,
re_params_name=_re_params_name):
"""
Reads cycle's data and simulation plans.
:param file_path:
Excel file path.
:type file_path: str
:param re_sheet_name:
Regular expression to parse sheet names.
:type re_sheet_name: regex.Regex
:param re_params_name:
Regular expression to parse param names.
:type re_params_name: regex.Regex
:return:
A pandas DataFrame with cycle's time series.
:rtype: dict, pandas.DataFrame
"""
excel_file = pd.ExcelFile(file_path)
res, plans = {}, []
defaults = {'scope': 'base'}
book = excel_file.book
for sheet_name in excel_file.sheet_names:
match = re_sheet_name.match(sheet_name)
if not match:
continue
match = {k: v.lower() for k, v in match.groupdict().items() if v}
match = dsp_utl.combine_dicts(defaults, match)
sheet = _open_sheet_by_name_or_index(book, 'book', sheet_name)
if match['scope'] == 'base':
_parse_base_data(res, match, sheet, sheet_name, re_params_name)
elif match['scope'] == 'plan':
_parse_plan_data(plans, match, sheet, sheet_name, re_params_name)
for k, v in co2_utl.stack_nested_keys(res.get('base', {}), depth=3):
if k[0] != 'target':
v['cycle_type'] = v.get('cycle_type', k[-1].split('_')[0]).upper()
v['cycle_name'] = v.get('cycle_name', k[-1]).upper()
res['plan'] = _finalize_plan(res, plans, file_path)
return res
def _parse_values(data, default=None, where=''):
default = default or {}
for k, v in data.items():
match = _re_params_name.match(k) if k is not None else None
if not match:
log.warning("Parameter '%s' %s cannot be parsed!", k, where)
continue
elif _isempty(v):
continue
match = {i: j.lower() for i, j in match.groupdict().items() if j}
for key in _parse_key(**dsp_utl.combine_dicts(default, match)):
yield key, v
def _get_theoretical(profile):
defaults = {
'cycle_type': 'WLTP',
'gear_box_type': 'manual',
'wltp_class': 'class3b',
'downscale_factor': 0
}
profile = {k: v for k, v in profile.items() if v}
profile = dsp_utl.combine_dicts(defaults, profile)
profile['cycle_type'] = profile['cycle_type'].upper()
profile['wltp_class'] = profile['wltp_class'].lower()
profile['gear_box_type'] = profile['gear_box_type'].lower()
from co2mpas.model.physical.cycle import cycle
res = cycle().dispatch(inputs=profile, outputs=['times', 'velocities'])
data = dsp_utl.selector(['times', 'velocities'], res, output_type='list')
return pd.DataFrame(data).T
def parse_values(data, default=None, re_params_name=_re_params_name):
default = default or {'scope': 'base'}
if 'usage' not in default:
default['usage'] = 'input'
if 'cycle' not in default or default['cycle'] == 'all':
default['cycle'] = ('nedc_h', 'nedc_l', 'wltp_p', 'wltp_h', 'wltp_l')
elif default['cycle'] == 'wltp':
default['cycle'] = ('wltp_h', 'wltp_l')
elif default['cycle'] == 'nedc':
default['cycle'] = ('nedc_h', 'nedc_l')
else:
default['cycle'] = default['cycle'].replace('-', '_')
for k, v in data.items():
match = re_params_name.match(k) if k is not None else None
if not match or _isempty(v):
continue
match = {i: j.lower() for i, j in match.groupdict().items() if j}
if 'stage' not in match and match.get('usage', None) == 'target':
match['stage'] = 'prediction'
match = dsp_utl.combine_dicts(default, match)
match['stage'] = match['stage'].replace(' ', '')
if match['stage'] == 'input':
match['stage'] = 'calibration'
i = match['param']
if match['cycle'] == 'wltp':
match['cycle'] = ('wltp_h', 'wltp_l')
elif match['cycle'] == 'nedc':
match['cycle'] = ('nedc_h', 'nedc_l')
elif match['cycle'] == 'all':
match['cycle'] = ('nedc_h', 'nedc_l', 'wltp_p', 'wltp_h', 'wltp_l')
for c in stlp(match['cycle']):
c = c.replace('-', '_')
if c == 'wltp_p':
stage = 'precondition'
elif 'nedc' in c:
stage = 'prediction'
else:
stage = match['stage']
yield (match['scope'], match['usage'], stage, c, i), v
def make_simulation_plan(plan, timestamp, variation, flag, model=None):
model, summary = model or batch.vehicle_processing_model(), {}
run_base = model.get_node('run_base')[0].dsp
run_modes = tuple(
run_base.get_sub_dsp_from_workflow(
('data', 'vehicle_name'), check_inputs=False,
graph=run_base.dmap).data_nodes) + ('start_time', 'vehicle_name')
var = json.dumps(variation, sort_keys=True)
o_cache, o_folder = flag['overwrite_cache'], flag['output_folder']
modelconf = flag.get('modelconf', None)
kw, bases = dsp_utl.combine_dicts(flag, {'run_base': True}), set()
for (i, base_fpath, run), p in tqdm.tqdm(plan, disable=False):
try:
base = get_results(model, o_cache, base_fpath, timestamp, run, var,
o_folder, modelconf)
except KeyError:
log.warn('Base model "%s" of variation "%s" cannot be parsed!',
base_fpath, i)
continue
name = base['vehicle_name']
if 'summary' in base and name not in bases:
batch._add2summary(summary, base['summary'])
bases.add(name)
name = '{}-{}'.format(name, i)
new_base, o = define_new_inputs(p, base)
inputs = batch.prepare_data(new_base, {}, base_fpath, o_cache,
o_folder, timestamp, False, modelconf)[0]
inputs.update(dsp_utl.selector(set(base).difference(run_modes), base))
inputs['vehicle_name'] = name
inputs.update(kw)
res = run_base.dispatch(inputs)
batch.notify_result_listener(plan_listener, res)
s = filter_summary(p, o, res.get('summary', {}))
base_keys = {
'vehicle_name': (base_fpath, name, run),
}
batch._add2summary(summary, s, base_keys)
return summary
def make_simulation_plan(plan, timestamp, output_folder, main_flags):
model, summary = vehicle_processing_model(), {}
run_modes = tuple(model.get_sub_dsp_from_workflow(
('validated_data', 'vehicle_name'), check_inputs=False, graph=model.dmap
).data_nodes) + ('start_time', 'vehicle_name')
kw = {
'output_folder': output_folder,
'plan': False,
'timestamp': timestamp,
}
kw, bases = dsp_utl.combine_dicts(main_flags, kw), set()
for (i, base_fpath, defaults_fpats), p in tqdm(plan, disable=False):
base = get_results(model, base_fpath, **kw)
name = base['vehicle_name']
if name not in bases:
_add2summary(summary, base.get('summary', {}))
bases.add(name)
name = '{}-{}'.format(name, i)
inputs = dsp_utl.selector(set(base).difference(run_modes), base)
inputs['vehicle_name'] = name
dsp_model = base['dsp_model']
outputs = dsp_model.data_output
dfl = build_default_models(model, defaults_fpats, **kw)
if dfl:
dfl = {'data.prediction.models': dfl}
outputs = co2_utl.combine_nested_dicts(dfl, outputs, depth=2)
inputs['validated_data'] = define_new_inputs(p, outputs, dsp_model)
inputs.update(kw)
res = _process_vehicle(model, **inputs)
s = filter_summary(p, res.get('summary', {}))
base_keys = {
'vehicle_name': (defaults_fpats, base_fpath, name),
}
_add2summary(summary, s, base_keys)
return summary
def re_sample_targets(data):
res = {}
for k, v in co2_utl.stack_nested_keys(data.get('target', {}), depth=2):
if co2_utl.are_in_nested_dicts(data, 'output', *k):
o = co2_utl.get_nested_dicts(data, 'output', *k)
o = _split_by_data_format(o)
t = dsp_utl.selector(o, _split_by_data_format(v), allow_miss=True)
if 'times' not in t.get('ts', {}) or 'times' not in o['ts']:
t.pop('ts', None)
else:
time_series = t['ts']
x, xp = o['ts']['times'], time_series.pop('times')
if not _is_equal(x, xp):
for i, fp in time_series.items():
time_series[i] = np.interp(x, xp, fp)
v = dsp_utl.combine_dicts(*t.values())
co2_utl.get_nested_dicts(res, *k, default=co2_utl.ret_v(v))
return res
def define_wltp_base_model(base_model):
return dsp_utl.combine_dicts(_get_model_base(), base_model)
def prepare_data(raw_data, variation, input_file_name, overwrite_cache,
output_folder, timestamp, type_approval_mode, modelconf):
"""
Prepare the data to be processed.
:param raw_data:
Raw data from the input file.
:type raw_data: dict
:param variation:
Variations to be applied.
:type variation: dict
:param input_file_name:
Input file name.
:type input_file_name: str
:param overwrite_cache:
Overwrite saved cache?
:type overwrite_cache: bool
:param output_folder:
Output folder.
:type output_folder: str
:param timestamp:
Run timestamp.
:type timestamp: str
:param type_approval_mode:
Is launched for TA?
:type type_approval_mode: bool
:param modelconf:
Path of modelconf that has modified the defaults.
:type modelconf: str
:return:
Prepared data.
:rtype: dict
"""
has_plan = 'plan' in raw_data and (not raw_data['plan'].empty)
match = {
'scope': 'plan' if has_plan else 'base',
}
r = {}
sheets_factory = xleash.SheetsFactory()
from co2mpas.io import check_xlasso
for k, v in excel._parse_values(variation, match, "in variations"):
if isinstance(v, str) and check_xlasso(v):
v = xleash.lasso(v, sheets_factory, url_file=input_file_name)
dsp_utl.get_nested_dicts(r, *k[:-1])[k[-1]] = v
if 'plan' in r:
if has_plan:
plan = raw_data['plan'].copy()
for k, v in dsp_utl.stack_nested_keys(r['plan'], 4):
plan['.'.join(k)] = v
else:
gen = dsp_utl.stack_nested_keys(r['plan'], 4)
plan = pd.DataFrame([{'.'.join(k): v for k, v in gen}])
excel._add_index_plan(plan, input_file_name)
r['plan'] = plan
has_plan = True
if 'base' in r:
r['base'] = dsp_utl.combine_nested_dicts(raw_data.get('base', {}),
r['base'],
depth=4)
if 'flag' in r:
r['flag'] = dsp_utl.combine_nested_dicts(raw_data.get('flag', {}),
r['flag'],
depth=1)
data = dsp_utl.combine_dicts(raw_data, r)
if type_approval_mode:
variation, has_plan = {}, False
if not schema._ta_mode(data):
return {}, pd.DataFrame([])
flag = data.get('flag', {}).copy()
if 'run_base' not in flag:
flag['run_base'] = not has_plan
if 'run_plan' not in flag:
flag['run_plan'] = has_plan
flag['type_approval_mode'] = type_approval_mode
flag['output_folder'] = output_folder
flag['overwrite_cache'] = overwrite_cache
if modelconf:
flag['modelconf'] = modelconf
if timestamp is not None:
flag['timestamp'] = timestamp
flag = schema.validate_flags(flag)
if flag is dsp_utl.NONE:
return {}, pd.DataFrame([])
schema.check_data_version(flag)
res = {
'flag': flag,
'variation': variation,
'input_file_name': input_file_name,
}
res = dsp_utl.combine_dicts(flag, res)
base = dsp_utl.combine_dicts(res, {'data': data.get('base', {})})
plan = dsp_utl.combine_dicts(res,
{'data': data.get('plan', pd.DataFrame([]))})
return base, plan
def combine_outputs(models):
return dsp_utl.combine_dicts(*models.values())
