def action_VAR(name, title, action_params, _):
"""Load a variable.
This action is used for parameters without constraints. If one configuration
element is given, the parameter doesn't have limits. If three are given, the last two
specify the low and upper limits. Parameter is set to not constant.
"""
# Take numerical value or load from file
if ':' in action_params[0]: # We want to load a fit result
from analysis.fit.result import FitResult
fit_name, var_name = action_params[0].split(':')
result = FitResult.from_yaml_file(fit_name)
try:
value, _, _, _ = result.get_fit_parameter(var_name)
except KeyError:
value = result.get_const_parameter(var_name)
else:
try:
value = float(action_params[0])
except ValueError:
print("error, action params[0]", action_params[0])
parameter = ROOT.RooRealVar(name, title, value)
parameter.setConstant(False)
if len(action_params) > 1:
try:
_, min_val, max_val = action_params
except ValueError:
raise ValueError(
"Wrongly specified var (need to give 1 or 3 arguments) "
"-> {}".format(action_params))
parameter.setMin(float(min_val))
parameter.setMax(float(max_val))
parameter.setConstant(False)
return parameter, None
def action_BLINDRATIO(name, title, action_params, external_vars):
"""Configure a ratio between two variables.
The first is the numerator, the second the denominator. At least needs to be a shared variable,
and currently two constrained variables are not possible.
The third is a randomization string; the fourth a mean and the fifth a width.
"""
from analysis.utils.pdf import load_pdf_by_name
Ratio = load_pdf_by_name('RooRatio')
if len(action_params) != 5:
raise ValueError(
"Wrong number of arguments for BLINDRATIO -> {}".format(
action_params))
if not any(v.startswith('@') for v in action_params):
raise ValueError(
"At least one parameter of a BLINDRATIO must be a reference")
constraint = None
processed_vars = []
for variable in action_params[:2]:
if variable.startswith('@'):
processed_variable, const = external_vars[variable[1:]]
if const:
if not constraint:
constraint = const
else:
raise NotImplementedError(
"Two constrained variables in SCALE are not allowed")
elif ':' in variable:
from analysis.fit.result import FitResult
fit_name, var_name = variable.split(':')
result = FitResult.from_yaml_file(fit_name)
try:
value = result.get_fit_parameter(var_name)[0]
except KeyError:
value = result.get_const_parameter(var_name)
processed_variable = ROOT.RooFit.RooConst(value)
else:
processed_variable = ROOT.RooFit.RooConst(float(variable))
processed_vars.append(processed_variable)
parameter = Ratio(name, title, *processed_vars)
blind_str, blind_central, blind_sigma = action_params[2:]
ref_var = deepcopy(parameter)
parameter = ROOT.RooUnblindPrecision(name + "_blind", title + "_blind",
blind_str, float(blind_central),
float(blind_sigma), ref_var)
return parameter, constraint
def action_SCALE(name, title, action_params, external_vars):
"""Configure a constant scaling to a variable.
The first param must be a shared variable, the second can be a number or a shared variable.
"""
try:
ref_var, second_var = action_params
except ValueError:
raise ValueError(
"Wrong number of arguments for SCALE -> {}".format(action_params))
try:
if ref_var.startswith('@'):
ref_var = ref_var[1:]
else:
raise ValueError(
"The first value for a SCALE must be a reference.")
ref_var, constraint = external_vars[ref_var]
if second_var.startswith('@'):
second_var = second_var[1:]
second_var, const = external_vars[second_var]
if const:
if not constraint:
constraint = const
else:
raise NotImplementedError(
"Two constrained variables in SCALE are not allowed")
elif ':' in second_var:
from analysis.fit.result import FitResult
fit_name, var_name = second_var.split(':')
result = FitResult.from_yaml_file(fit_name)
try:
value = result.get_fit_parameter(var_name)[0]
except KeyError:
value = result.get_const_parameter(var_name)
second_var = ROOT.RooFit.RooConst(value)
else:
second_var = ROOT.RooFit.RooConst(float(second_var))
except KeyError as error:
raise ValueError("Missing parameter definition -> {}".format(error))
parameter = ROOT.RooProduct(name, title,
ROOT.RooArgList(ref_var, second_var))
return parameter, constraint
def action_RATIO(name, title, action_params, external_vars):
"""Configure a ratio between two variables.
The first is the numerator, the second the denomiator. At least needs to be a shared variable,
and currently two constrained variables are not possible.
"""
from analysis.utils.pdf import load_pdf_by_name
Ratio = load_pdf_by_name('RooRatio')
if len(action_params) != 2:
raise ValueError(
"Wrong number of arguments for RATIO -> {}".format(action_params))
if not any(v.startswith('@') for v in action_params):
raise ValueError(
"At least one parameter of a RATIO must be a reference")
constraint = None
processed_vars = []
for variable in action_params:
if variable.startswith('@'):
processed_variable, const = external_vars[variable[1:]]
if const:
if not constraint:
constraint = const
else:
raise NotImplementedError(
"Two constrained variables in SCALE are not allowed")
elif ':' in variable:
from analysis.fit.result import FitResult
fit_name, var_name = variable.split(':')
result = FitResult.from_yaml_file(fit_name)
try:
value = result.get_fit_parameter(var_name)[0]
except KeyError:
value = result.get_const_parameter(var_name)
processed_variable = ROOT.RooFit.RooConst(value)
else:
processed_variable = ROOT.RooFit.RooConst(float(variable))
processed_vars.append(processed_variable)
parameter = Ratio(name, title, *processed_vars)
return parameter, constraint
def action_CONST(name, title, action_params, _):
"""Load a constant variable.
Its argument indicates at which value to fix it.
"""
# Take numerical value or load from file
if ':' in action_params[0]: # We want to load a fit result
from analysis.fit.result import FitResult
fit_name, var_name = action_params[0].split(':')
result = FitResult.from_yaml_file(fit_name)
try:
value, _, _, _ = result.get_fit_parameter(var_name)[0]
except KeyError:
value = result.get_const_parameter(var_name)
else:
try:
value = float(action_params[0])
except ValueError:
print("error, action params[0]", action_params[0])
parameter = ROOT.RooRealVar(name, title, value)
parameter.setConstant(True)
return parameter, None
def action_GAUSS(name, title, action_params, _):
"""Load a variable with a Gaussian constraint.
The arguments of that Gaussian, ie, its mean and sigma, are to be given
as action parameters.
"""
value_error = None
if ':' in action_params[0]: # We want to load a fit result
from analysis.fit.result import FitResult
fit_name, var_name = action_params[0].split(':')
result = FitResult.from_yaml_file(fit_name)
try:
value, value_error, _, _ = result.get_fit_parameter(var_name)
except KeyError:
value = result.get_const_parameter(var_name)
else:
try:
value = float(action_params[0])
except ValueError:
print("error, action params[0]", action_params[0])
parameter = ROOT.RooRealVar(name, title, value)
try:
if len(action_params) == 1 and value_error is None:
raise ValueError
elif len(action_params) == 2:
value_error = float(action_params[1])
else:
raise ValueError
except ValueError:
raise ValueError("Wrongly specified Gaussian constraint -> {}".format(
action_params))
constraint = ROOT.RooGaussian(name + 'Constraint', name + 'Constraint',
parameter, ROOT.RooFit.RooConst(value),
ROOT.RooFit.RooConst(value_error))
parameter.setConstant(False)
return parameter, constraint
def __init__(self, model, acceptance, config):
"""Configure randomizer.
To specify where the parameters come from, `config` needs a `params` key which contains
a list of results and parameter name correspondences to be used to translate from the
fit result to `model`.
{'params': [{'result': result_name,
'param_names': {'fit_result_name': 'model_parameter_name',
...}},
...]}
Arguments:
model (`analysis.physics.PhysicsFactory`): Factory used for generation and
fitting.
config (dict): Configuration.
Raise:
KeyError: If some configuration parameter is missing.
RuntimeError: If the parameter names are badly specified.
ValueError: If no yield is specified, either through the PDF model or the
configuration.
"""
def make_block(*matrices):
"""Make bloc-diagonal matrix.
Arguments:
matrices (list): Matrices to combine.
Return:
numpy.ndarray: Block-diagonal matrix.
"""
dimensions = sum(mat.shape[0] for mat in matrices), sum(mat.shape[1] for mat in matrices)
output_mat = np.zeros(dimensions)
row = 0
column = 0
for mat in matrices:
output_mat[row:row + mat.shape[0], column:column + mat.shape[1]] = mat
row = row + mat.shape[0]
column = column + mat.shape[1]
return output_mat
super(FixedParamsRandomizer, self).__init__(model, config=config, acceptance=acceptance)
cov_matrices = []
central_values = []
param_translation = OrderedDict()
# Load fit results and their covariance matrices
rand_config = config['params']
if not isinstance(rand_config, (list, tuple)):
rand_config = [rand_config]
for result_config in rand_config:
fit_result = FitResult.from_yaml_file(result_config['result'])
param_translation.update(result_config['param_names'])
cov_matrices.append(fit_result.get_covariance_matrix(param_translation.keys()))
central_values.extend([float(fit_result.get_fit_parameter(param)[0])
for param in param_translation.keys()])
# Check that there is a correspondence between the fit result and parameters in the generation PDF
self._cov_matrix = make_block(*cov_matrices)
self._central_values = np.array(central_values)
self._pdf_index = OrderedDict()
for fit_param in param_translation.values():
found = False
for label, pdf_list in self._gen_pdfs.items():
for pdf_num, pdf in enumerate(pdf_list):
if fit_param in [var.GetName() for var in iterate_roocollection(pdf.getVariables())]:
self._pdf_index[fit_param] = (label, pdf_num)
found = True
break
if found:
break
if not found:
raise RuntimeError("Cannot find parameter {} in the physics model".format(fit_param))
def run(config_files, link_from, verbose):
"""Run the script.
Run a generate/fit sequence as many times as requested.
Arguments:
config_files (list[str]): Path to the configuration files.
link_from (str): Path to link the results from.
verbose (bool): Give verbose output?
Raise:
OSError: If the configuration file or some other input does not exist.
AttributeError: If the input data are incompatible with a previous fit.
KeyError: If some configuration data are missing.
ValueError: If there is any problem in configuring the PDF factories.
RuntimeError: If there is a problem during the fitting.
"""
try:
config = _config.load_config(
*config_files, validate=['syst/ntoys', 'name', 'randomizer'])
except OSError:
raise OSError(
"Cannot load configuration files: {}".format(config_files))
except _config.ConfigError as error:
if 'syst/ntoys' in error.missing_keys:
logger.error("Number of toys not specified")
if 'name' in error.missing_keys:
logger.error("No name was specified in the config file!")
if 'randomizer' in error.missing_keys:
logger.error(
"No randomizer configuration specified in config file!")
raise KeyError("ConfigError raised -> {}".format(error.missing_keys))
except KeyError as error:
logger.error("YAML parsing error -> %s", error)
raise
model_name = config['syst'].get('model',
'model') # TODO: 'model' returns name?
try:
model_config = config[model_name]
except KeyError as error:
logger.error("Missing model configuration -> %s", str(error))
raise KeyError("Missing model configuration")
# Load fit model
try:
fit_model = configure_model(copy.deepcopy(model_config))
randomizer_model = configure_model(copy.deepcopy(model_config))
except KeyError:
logger.exception('Error loading model')
raise ValueError('Error loading model')
# Some info
ntoys = config['syst'].get('ntoys-per-job', config['syst']['ntoys'])
logger.info("Doing %s generate/fit sequences", ntoys)
logger.info("Systematics job name: %s", config['name'])
if link_from:
config['link-from'] = link_from
if 'link-from' in config:
logger.info("Linking toy data from %s", config['link-from'])
else:
logger.debug("No linking specified")
# Now load the acceptance
try:
acceptance = get_acceptance(config['acceptance']) \
if 'acceptance' in config \
else None
except _config.ConfigError as error:
raise KeyError("Error loading acceptance -> {}".format(error))
# Fit strategy
fit_strategy = config['syst'].get('strategy', 'simple')
# Load randomizer configuration
randomizer = get_randomizer(config['randomizer'])(
model=randomizer_model,
config=config['randomizer'],
acceptance=acceptance)
# Set seed
job_id = get_job_id()
# Start looping
fit_results = {}
logger.info("Starting sampling-fit loop (print frequency is 20)")
initial_mem = memory_usage()
initial_time = default_timer()
do_extended = config['syst'].get('extended', False)
do_minos = config['syst'].get('minos', False)
for fit_num in range(ntoys):
# Logging
if (fit_num + 1) % 20 == 0:
logger.info(" Fitting event %s/%s", fit_num + 1, ntoys)
# Generate a dataset
seed = get_urandom_int(4)
np.random.seed(seed=seed)
ROOT.RooRandom.randomGenerator().SetSeed(seed)
try:
# Get a randomized dataset and fit it with the nominal fit
dataset = randomizer.get_dataset(randomize=True)
gen_values = randomizer.get_current_values()
fit_result_nominal = fit(fit_model,
model_name,
fit_strategy,
dataset,
verbose,
Extended=do_extended,
Minos=do_minos)
# Fit the randomized dataset with the randomized values as nominal
fit_result_rand = fit(randomizer_model,
model_name,
fit_strategy,
dataset,
verbose,
Extended=do_extended,
Minos=do_minos)
randomizer.reset_values(
) # Needed to avoid generating unphysical values
except ValueError:
raise RuntimeError()
except Exception:
# logger.exception()
raise RuntimeError() # TODO: provide more information?
result = {}
result['fitnum'] = fit_num
result['seed'] = seed
# Save the results of the randomized fit
result_roofit_rand = FitResult.from_roofit(fit_result_rand)
result['param_names'] = result_roofit_rand.get_fit_parameters().keys()
result['rand'] = result_roofit_rand.to_plain_dict()
result['rand_cov'] = result_roofit_rand.get_covariance_matrix()
_root.destruct_object(fit_result_rand)
# Save the results of the nominal fit
result_roofit_nominal = FitResult.from_roofit(fit_result_nominal)
result['nominal'] = result_roofit_nominal.to_plain_dict()
result['nominal_cov'] = result_roofit_nominal.get_covariance_matrix()
result['gen'] = gen_values
_root.destruct_object(result_roofit_nominal)
_root.destruct_object(dataset)
fit_results[fit_num] = result
logger.debug("Cleaning up")
logger.info("Fitting loop over")
logger.info("--> Memory leakage: %.2f MB/sample-fit",
(memory_usage() - initial_mem) / ntoys)
logger.info("--> Spent %.0f ms/sample-fit",
(default_timer() - initial_time) * 1000.0 / ntoys)
logger.info("Saving to disk")
data_res = []
cov_matrices = {}
# Get covariance matrices
for fit_num, fit_res_i in fit_results.items():
fit_res = {
'fitnum': fit_res_i['fitnum'],
'seed': fit_res_i['seed'],
'model_name': model_name,
'fit_strategy': fit_strategy
}
param_names = fit_res_i['param_names']
cov_folder_rand = os.path.join(str(job_id), str(fit_res['fitnum']),
'rand')
cov_matrices[cov_folder_rand] = pd.DataFrame(fit_res_i['rand_cov'],
index=param_names,
columns=param_names)
cov_folder_nominal = os.path.join(str(job_id), str(fit_res['fitnum']),
'nominal')
cov_matrices[cov_folder_nominal] = pd.DataFrame(
fit_res_i['nominal_cov'], index=param_names, columns=param_names)
for res_name, res_value in fit_res_i['rand'].items():
fit_res['{}_rand'.format(res_name)] = res_value
for res_name, res_value in fit_res_i['nominal'].items():
fit_res['{}_nominal'.format(res_name)] = res_value
for res_name, res_value in fit_res_i['gen'].items():
fit_res['{}_gen'.format(res_name)] = res_value
data_res.append(fit_res)
data_frame = pd.DataFrame(data_res)
fit_result_frame = pd.concat([
data_frame,
pd.concat([pd.DataFrame({'jobid': [job_id]})] *
data_frame.shape[0]).reset_index(drop=True)
],
axis=1)
try:
# pylint: disable=E1101
with _paths.work_on_file(config['name'],
path_func=_paths.get_toy_fit_path,
link_from=config.get('link-from',
None)) as toy_fit_file:
with modify_hdf(toy_fit_file) as hdf_file:
# First fit results
hdf_file.append('fit_results', fit_result_frame)
# Save covarinance matrix under 'covariance/jobid/fitnum
for cov_folder, cov_matrix in cov_matrices.items():
cov_path = os.path.join('covariance', cov_folder)
hdf_file.append(cov_path, cov_matrix)
# Generator info
hdf_file.append(
'input_values',
pd.DataFrame.from_dict(randomizer.get_input_values(),
orient='index'))
logger.info("Written output to %s", toy_fit_file)
if 'link-from' in config:
logger.info("Linked to %s", config['link-from'])
except OSError as excp:
logger.error(str(excp))
raise
except ValueError as error:
logger.exception("Exception on dataset saving")
raise RuntimeError(str(error))
def test_fitresult_yaml_conversion(fit_result):
"""Test YAML conversion."""
res = FitResult.from_roofit(fit_result)
res_conv = FitResult.from_yaml(res.to_yaml())
assert (
res_conv.get_covariance_matrix() == res.get_covariance_matrix()).all()
def test_fitresult_convergence(fit_result):
"""Test fit result convergence."""
assert FitResult.from_roofit(fit_result).has_converged()
def run(config_files, link_from, verbose):
"""Run the script.
Run a sample/fit sequence as many times as requested.
Arguments:
config_files (list[str]): Path to the configuration files.
link_from (str): Path to link the results from.
verbose (bool): Give verbose output?
Raise:
OSError: If there either the configuration file does not exist some
of the input toys cannot be found.
AttributeError: If the input data are incompatible with a previous fit.
KeyError: If some configuration data are missing.
ValueError: If there is any problem in configuring the PDF factories.
RuntimeError: If there is a problem during the fitting.
"""
try:
config = _config.load_config(*config_files,
validate=['fit/nfits', 'name', 'data'])
except OSError:
raise OSError(
"Cannot load configuration files: {}".format(config_files))
except ConfigError as error:
if 'fit/nfits' in error.missing_keys:
logger.error("Number of fits not specified")
if 'name' in error.missing_keys:
logger.error("No name was specified in the config file!")
if 'data' in error.missing_keys:
logger.error("No input data specified in the config file!")
raise KeyError("ConfigError raised -> {}".format(error.missing_keys))
except KeyError as error:
logger.error("YAML parsing error -> %s", error)
try:
models = {
model_name: config[model_name]
for model_name in config['fit'].get('models', ['model'])
}
except KeyError as error:
logger.error("Missing model configuration -> %s", str(error))
raise KeyError("Missing model configuration")
if not models:
logger.error(
"Empty list specified in the config file under 'fit/models'!")
raise KeyError()
fit_strategies = config['fit'].get('strategies', ['simple'])
if not fit_strategies:
logger.error("Empty fit strategies were specified in the config file!")
raise KeyError()
# Some info
nfits = config['fit'].get('nfits-per-job', config['fit']['nfits'])
logger.info("Doing %s sample/fit sequences", nfits)
logger.info("Fit job name: %s", config['name'])
if link_from:
config['link-from'] = link_from
if 'link-from' in config:
logger.info("Linking toy data from %s", config['link-from'])
else:
logger.debug("No linking specified")
# Analyze data requirements
logger.info("Loading input data")
data = {}
gen_values = {}
if len(set('category' in data_source
for data_source in config['data'])) > 1:
raise KeyError("Categories in 'data' not consistently specified.")
for data_id, data_source in config['data'].items():
try:
source_toy = data_source['source']
except KeyError:
logger.error("Data source not specified")
raise
data[data_id] = (get_data({
'source': source_toy,
'source-type': 'toy',
'tree': 'data',
'output-format': 'pandas',
'selection': data_source.get('selection')
}), data_source['nevents'], data_source.get('poisson'),
data_source.get('category'))
# Generator values
toy_info = get_data({
'source': source_toy,
'source-type': 'toy',
'tree': 'toy_info',
'output-format': 'pandas'
})
gen_values[data_id] = {}
for var_name in toy_info.columns:
if var_name in ('seed', 'jobid', 'nevents'):
continue
gen_values[data_id][var_name] = toy_info[var_name].iloc[0]
try:
fit_models = {}
for model_name in models:
if model_name not in config:
raise KeyError(
"Missing model definition -> {}".format(model_name))
fit_models[model_name] = configure_model(config[model_name])
if any(yield_.isConstant()
for yield_ in fit_models[model_name].get_yield_vars()
if yield_):
logger.warning(
"Model %s has constant yields. "
"Be careful when configuring the input data, you may need to disable poisson sampling",
model_name)
except KeyError:
logger.exception("Error loading model")
raise ValueError("Error loading model")
if len(set(model.is_extended() for model in fit_models.values())) == 2:
logger.error("Mix of extended and non-extended models!")
raise ValueError("Error loading fit models")
# Let's check these generator values against the output file
try:
gen_values_frame = {}
# pylint: disable=E1101
with _paths.work_on_file(config['name'], _paths.get_toy_fit_path,
config.get('link-from')) as toy_fit_file:
with modify_hdf(toy_fit_file) as hdf_file:
logger.debug("Checking generator values")
test_gen = [('gen_{}'.format(data_source)) in hdf_file
for data_source in gen_values]
if all(test_gen
): # The data were written already, crosscheck values
for source_id, gen_value in gen_values.items():
if not all(
hdf_file['gen_{}'.format(data_source)]
[var_name].iloc[0] == var_value
for var_name, var_value in gen_value.items()):
raise AttributeError(
"Generated and stored values don't match for source '{}'"
.format(source_id))
elif not any(test_gen): # No data were there, just overwrite
for source_id, gen_values in gen_values.items():
gen_data = {
'id':
source_id,
'source':
_paths.get_toy_path(
config['data'][source_id]['source']),
'nevents':
config['data'][source_id]['nevents']
}
gen_data.update(gen_values)
gen_values_frame['gen_{}'.format(
source_id)] = pd.DataFrame([gen_data])
else:
raise AttributeError("Inconsistent number of data sources")
except OSError as excp:
logger.error(str(excp))
raise
# Now load the acceptance
try:
acceptance = get_acceptance(config['acceptance']) \
if 'acceptance' in config \
else None
except ConfigError as error:
raise KeyError("Error loading acceptance -> {}".format(error))
# Prepare output
gen_events = defaultdict(list)
# Set seed
job_id = get_job_id()
if job_id:
seed = int(job_id.split('.')[0])
else:
import random
job_id = 'local'
seed = random.randint(0, 100000)
np.random.seed(seed=seed)
ROOT.RooRandom.randomGenerator().SetSeed(seed)
# Start looping
fit_results = defaultdict(list)
logger.info("Starting sampling-fit loop (print frequency is 20)")
initial_mem = memory_usage()
initial_time = default_timer()
for fit_num in range(nfits):
# Logging
if (fit_num + 1) % 20 == 0:
logger.info(" Fitting event %s/%s", fit_num + 1, nfits)
# Get a compound dataset
seed = get_urandom_int(4)
np.random.seed(seed=seed)
ROOT.RooRandom.randomGenerator().SetSeed(seed)
try:
logger.debug("Sampling input data")
datasets, sample_sizes = get_datasets(data, acceptance, fit_models)
for sample_name, sample_size in sample_sizes.items():
gen_events['N^{{{}}}_{{gen}}'.format(sample_name)].append(
sample_size)
logger.debug("Sampling finalized")
except KeyError:
logger.exception("Bad data configuration")
raise
logger.debug("Fitting")
for model_name in models:
dataset = datasets.pop(model_name)
fit_model = fit_models[model_name]
# Now fit
for fit_strategy in fit_strategies:
toy_key = (model_name, fit_strategy)
try:
fit_result = fit(fit_model,
model_name,
fit_strategy,
dataset,
verbose,
Extended=config['fit'].get(
'extended', False),
Minos=config['fit'].get('minos', False))
except ValueError:
raise RuntimeError()
# Now results are in fit_parameters
result_roofit = FitResult.from_roofit(fit_result)
result = result_roofit.to_plain_dict()
result['cov_matrix'] = result_roofit.get_covariance_matrix()
result['param_names'] = result_roofit.get_fit_parameters(
).keys()
result['fitnum'] = fit_num
result['seed'] = seed
fit_results[toy_key].append(result)
_root.destruct_object(fit_result)
_root.destruct_object(dataset)
logger.debug("Cleaning up")
logger.info("Fitting loop over")
logger.info("--> Memory leakage: %.2f MB/sample-fit",
(memory_usage() - initial_mem) / nfits)
logger.info("--> Spent %.0f ms/sample-fit",
(default_timer() - initial_time) * 1000.0 / nfits)
logger.info("Saving to disk")
data_res = []
cov_matrices = {}
# Get gen values for this model
for (model_name, fit_strategy), fits in fit_results.items():
for fit_res in fits:
fit_res = fit_res.copy()
fit_res['model_name'] = model_name
fit_res['fit_strategy'] = fit_strategy
cov_folder = os.path.join(str(job_id), str(fit_res['fitnum']))
param_names = fit_res.pop('param_names')
cov_matrices[cov_folder] = pd.DataFrame(fit_res.pop('cov_matrix'),
index=param_names,
columns=param_names)
data_res.append(fit_res)
data_frame = pd.DataFrame(data_res)
fit_result_frame = pd.concat([
pd.DataFrame(gen_events), data_frame,
pd.concat([pd.DataFrame({'jobid': [job_id]})] *
data_frame.shape[0]).reset_index(drop=True)
],
axis=1)
try:
# pylint: disable=E1101
with _paths.work_on_file(
config['name'],
path_func=_paths.get_toy_fit_path,
link_from=config.get('link-from')) as toy_fit_file:
with modify_hdf(toy_fit_file) as hdf_file:
# First fit results
hdf_file.append('fit_results', fit_result_frame)
# Save covarinance matrix under 'covariance/jobid/fitnum
for cov_folder, cov_matrix in cov_matrices.items():
cov_path = os.path.join('covariance', cov_folder)
hdf_file.append(cov_path, cov_matrix)
# Generator info
for key_name, gen_frame in gen_values_frame.items():
hdf_file.append(key_name, gen_frame)
logger.info("Written output to %s", toy_fit_file)
if 'link-from' in config:
logger.info("Linked to %s", config['link-from'])
except OSError as excp:
logger.error(str(excp))
raise
except ValueError as error:
logger.exception("Exception on dataset saving")
raise RuntimeError(str(error))