def _load_score_model(filename):
if os.path.isdir(filename):
model = Ensemble()
model.load(filename)
else:
model = ScoreEstimator()
model.load(filename)
return model
)
os.makedirs(f"{results_dir}/{gen_method}", exist_ok=True)
np.save(file=f"{results_dir}/{gen_method}/llr.npy", arr=llr)
np.save(file=f"{results_dir}/{gen_method}/scores.npy", arr=scores)
elif gen_method in score_estimator_methods:
if fisher_info["bool"]:
print_fisher_info()
# Testing data is generated
generate_test_data_score(gen_method)
# The trained model, theta grid and the test data are loaded
estimator = ScoreEstimator()
estimator.load(f"{eval_folder}/{gen_method}")
grid = np.load(f"{rates_dir}/grid.npy")
test = np.load(f"{tests_dir}/{gen_method}/x_test.npy")
scores = estimator.evaluate_score(
x=test,
theta=grid,
)
os.makedirs(f"{results_dir}/{gen_method}", exist_ok=True)
np.save(file=f"{results_dir}/{gen_method}/scores.npy", arr=scores)
else:
raise ValueError("Invalid generation method")
estimator = ParameterizedRatioEstimator(n_hidden=(100, 100, 100))
estimator.train(
method=method,
x=f'{samples_path}/x_{method}_train.npy',
y=f'{samples_path}/y_{method}_train.npy',
theta=f'{samples_path}/theta0_{method}_train.npy',
r_xz=f'{samples_path}/r_xz_{method}_train.npy',
t_xz=f'{samples_path}/t_xz_{method}_train.npy',
alpha=alpha,
n_epochs=num_epochs,
batch_size=batch_size,
validation_split=valid_split,
)
elif method in score_estimator_methods:
estimator = ScoreEstimator()
estimator.train(
method=method,
x=f'{samples_path}/x_{method}_train.npy',
t_xz=f'{samples_path}/t_xz_{method}_train.npy',
)
else:
raise ValueError('Invalid training method')
############################
#### Save trained model ####
############################
model_folder_name = method
model_folder_path = f'{models_dir}/{model_folder_name}'
# _ = sampler.sample_test(
# theta=sampling.benchmark('sm'),
# n_samples=1*10**6,
# folder='./data/samples',
# filename='test'
# )
# ## 2. Train score estimator
# It's now time to build a neural network. Only this time, instead of the likelihood ratio itself, we will estimate the gradient of the log likelihood with respect to the theory parameters -- the score. To be precise, the output of the neural network is an estimate of the score at some reference parameter point, for instance the Standard Model. A neural network that estimates this "local" score can be used to calculate the Fisher information at that point. The estimated score can also be used as a machine learning version of Optimal Observables, and likelihoods can be estimated based on density estimation in the estimated score space. This method for likelihood ratio estimation is called SALLY, and there is a closely related version called SALLINO. Both are explained in ["Constraining Effective Field Theories With Machine Learning"](https://arxiv.org/abs/1805.00013) and ["A Guide to Constraining Effective Field Theories With Machine Learning"](https://arxiv.org/abs/1805.00020).
#
# The central object for this is the `madminer.ml.ScoreEstimator` class:
# In[6]:
estimator = ScoreEstimator(n_hidden=(100, ))
# In[ ]:
estimator.train(
method='sally',
x='data/samples/x_train_score.npy',
t_xz='data/samples/t_xz_train_score.npy',
)
estimator.save('models/sally')
# # ## 3. Evaluate score estimator
# # Let's evaluate the SM score on the test data
estimator = estimator_cls(n_hidden=(100, 100, 100))
estimator.train(
method=method,
x=f"{samples_path}/x_{method}_train.npy",
y=f"{samples_path}/y_{method}_train.npy",
theta=f"{samples_path}/theta0_{method}_train.npy",
r_xz=f"{samples_path}/r_xz_{method}_train.npy",
t_xz=f"{samples_path}/t_xz_{method}_train.npy",
alpha=alpha,
n_epochs=num_epochs,
batch_size=batch_size,
validation_split=valid_split,
)
elif method in score_estimator_methods:
estimator = ScoreEstimator()
estimator.train(
method=method,
x=f"{samples_path}/x_{method}_train.npy",
t_xz=f"{samples_path}/t_xz_{method}_train.npy",
)
else:
raise ValueError("Invalid training method")
############################
#### Save trained model ####
############################
model_folder_name = method
input_file = sys.argv[2]
with open(input_file) as f:
inputs = yaml.safe_load(f)
# get method from inputs
path_split = os.path.split(os.path.abspath(samples_path))
sub_folder = path_split[1]
method = str(sub_folder.split("_", 3)[1])
# training options
if (method in ['sally', 'sallino']):
estimator = ScoreEstimator()
estimator.train(
method=method,
x=samples_path + '/x_' + method + '_train.npy',
t_xz=samples_path + '/t_xz_' + method + '_train.npy',
)
os.mkdir('/home/models/' + method)
estimator.save('/home/models/' + method + '/' + method)
if (method in ['alice', 'alices', 'cascal', 'carl', 'rolr', 'rascal']):
estimator = ParameterizedRatioEstimator(n_hidden=(100, 100, 100))
estimator.train(method=method,
alpha=float(inputs['alpha']),
theta=samples_path + '/theta0_' + method + '_train.npy',
x=samples_path + '/x_' + method + '_train.npy',
y=samples_path + '/y_' + method + '_train.npy',