def getAltHierarchyBestFit(asimov_data, template_maker, params, minimizer_settings,
hypo_normal, check_octant):
"""
Finds the best fit value of alternative hierarchy to that which
was used to produce the asimov data set.
\Params:
* asimov_data - array of values of asimov data set (float)
* template_maker - instance of class TemplateMaker service.
* params - parameters with values, fixed, range, etc. of systematics
* minimizer_settings - used with bfgs_b minimizer
* hypo_normal - bool for Mass hierarchy being Normal (True)
or inverted (False)
* check_octant - bool to check the opposite octant for a solution
to the minimization of the LLH.
"""
llh_data = find_alt_hierarchy_fit(
asimov_data, template_maker, params, hypo_normal,
minimizer_settings, only_atm_params=True, check_octant=check_octant)
alt_params = get_values(select_hierarchy(params, normal_hierarchy=hypo_normal))
for key in llh_data.keys():
if key == 'llh': continue
alt_params[key] = llh_data[key][-1]
return alt_params, llh_data
def getAltHierarchyBestFit(asimov_data, template_maker, params,
minimizer_settings, hypo_normal, check_octant):
"""
Finds the best fit value of alternative hierarchy to that which
was used to produce the asimov data set.
\Params:
* asimov_data - array of values of asimov data set (float)
* template_maker - instance of class TemplateMaker service.
* params - parameters with values, fixed, range, etc. of systematics
* minimizer_settings - used with bfgs_b minimizer
* hypo_normal - bool for Mass hierarchy being Normal (True)
or inverted (False)
* check_octant - bool to check the opposite octant for a solution
to the minimization of the LLH.
"""
llh_data = find_alt_hierarchy_fit(asimov_data,
template_maker,
params,
hypo_normal,
minimizer_settings,
only_atm_params=True,
check_octant=check_octant)
alt_params = get_values(
select_hierarchy(params, normal_hierarchy=hypo_normal))
for key in llh_data.keys():
if key == 'llh': continue
alt_params[key] = llh_data[key][-1]
return alt_params, llh_data
def find_alt_hierarchy_fit(asimov_data_set, template_maker,hypo_params,hypo_normal,
minimizer_settings,only_atm_params=True,check_octant=False):
"""
For the hypothesis of the mass hierarchy being NMH
('normal_hierarchy'=True) or IMH ('normal_hierarchy'=False), finds the
best fit for the free params in 'param_values' for the alternative
(opposite) hierarchy that maximizes the LLH of the Asimov data set.
\params:
* asimov_data_set - asimov data set to find best fit llh
* template_maker - instance of class pisa.analysis.TemplateMaker, used to
generate the asimov data set.
* hypo_params - parameters for template generation
* hypo_normal - boolean for NMH (True) or IMH (False)
* minimizer_settings - settings for bfgs minimization
* only_atm_params - boolean to denote whether the fit will be over the
atmospheric oscillation parameters only or over all the free params
in params
"""
# Find best fit of the alternative hierarchy to the
# hypothesized asimov data set.
#hypo_types = [('hypo_IMH',False)] if data_normal else [('hypo_NMH',True)]
hypo_params = select_hierarchy(hypo_params,normal_hierarchy=hypo_normal)
with Timer() as t:
llh_data = find_max_llh_bfgs(
asimov_data_set,template_maker,hypo_params,minimizer_settings,
normal_hierarchy=hypo_normal,check_octant=check_octant)
profile.info("==> elapsed time for optimizer: %s sec"%t.secs)
return llh_data
def get_col_info(col_name, dkey, hkey, template_settings):
if dkey == "data_NMH":
injected_vals = select_hierarchy(template_settings, normal_hierarchy=True)
else:
injected_vals = select_hierarchy(template_settings, normal_hierarchy=False)
if hkey == "hypo_NMH":
fit_vals = select_hierarchy(template_settings, normal_hierarchy=True)
else:
fit_vals = select_hierarchy(template_settings, normal_hierarchy=False)
value = injected_vals[col_name]["value"]
scale = injected_vals[col_name]["scale"]
prior = fit_vals[col_name]["prior"]
prange = fit_vals[col_name]["range"]
return prior, value, prange, scale
def createStepList(params, true_normal, grid_settings):
"""
No matter how many params are listed as free, force only theta23 to be free.
"""
new_params = fixAllButTheta23(params)
free_params = select_hierarchy(get_free_params(new_params),
normal_hierarchy=true_normal)
fixed_params = select_hierarchy(get_fixed_params(new_params),
normal_hierarchy=true_normal)
calc_steps(free_params, grid_settings['steps'])
# Form list from all parameters that holds a list of (name,step) tuples.
steplist = free_params['theta23']['steps']
return steplist
def createStepList(params,true_normal,grid_settings):
"""
No matter how many params are listed as free, force only theta23 to be free.
"""
new_params = fixAllButTheta23(params)
free_params = select_hierarchy(get_free_params(new_params),
normal_hierarchy=true_normal)
fixed_params = select_hierarchy(get_fixed_params(new_params),
normal_hierarchy=true_normal)
calc_steps(free_params, grid_settings['steps'])
# Form list from all parameters that holds a list of (name,step) tuples.
steplist = free_params['theta23']['steps']
return steplist
def get_col_info(col_name, tkey, hkey, template_settings, mctrue=False):
# REMEMBER: Since the "true_NH" corresponds to "pseudo data IH",
# the normal hierarchy input is opposite!
# If mctrue --> This means that the LLR distributions are of the TH,
# rather than the AH, null hypothesis.
#print "mctrue: ",mctrue
if 'true_N' in tkey:
mh = True if mctrue else False
injected_vals = select_hierarchy(template_settings,
normal_hierarchy=mh)
else:
mh = False if mctrue else True
injected_vals = select_hierarchy(template_settings,
normal_hierarchy=mh)
if 'hypo_N' in hkey:
mh = True if mctrue else False
fit_vals = select_hierarchy(template_settings,
normal_hierarchy=mh)
else:
mh = False if mctrue else True
fit_vals = select_hierarchy(template_settings,
normal_hierarchy=mh)
value = injected_vals[col_name]['value']
scale = injected_vals[col_name]['scale']
prange = fit_vals[col_name]['range']
#prior = fit_vals[col_name]['prior']
if injected_vals[col_name]['prior']['kind'] == "gaussian":
prior_val = injected_vals[col_name]['prior']["sigma"]
else:
prior_val = None
return prior_val, value, prange, scale
def getAsimovData(template_maker, params, data_normal):
"""
Generates the asimov data set (expected counts distribution) at
parameters assuming hierarchy of data_normal
\Params:
* template_maker - instance of class TemplateMaker service.
* params - parameters with values, fixed, range, etc. of systematics
* data_normal - bool for Mass hierarchy being Normal (True)
or inverted (False)
"""
fiducial_params = get_values(select_hierarchy(
params, normal_hierarchy=data_normal))
return get_asimov_fmap(template_maker, fiducial_params,
channel=fiducial_params['channel'])
def plot_posterior_params(frames, template_settings, plot_llh=True,
plot_param_info=True, pbins=20, mctrue=False,
**kwargs):
"""Plot posterior parameter distributions, and related data"""
good_columns = get_free_params(
select_hierarchy(template_settings, normal_hierarchy=True)).keys()
#good_columns = [col for col in frames[0].columns
# if col not in ['hypo','mctrue']]
if plot_llh: good_columns.append('llh')
print "good_columns: \n",good_columns
max_plots_per_fig = 4
nfigs = (len(good_columns)-1)/max_plots_per_fig + 1
logging.info("len(good_cols): %d, nfigs: %d"%(len(good_columns),nfigs))
figs = []
fig_names = []
colors = ['b','r','g','k','c','m']
for frame in frames:
ifig = 0
true_key = frame['mctrue'][0]
hypo_key = frame['hypo'][0]
for icol,col_name in enumerate(good_columns):
column = frame[col_name]
# Create new fig if needed:
if (icol%max_plots_per_fig) == 0:
ifig += 1
fig = plt.figure(figsize=(10,10))
fig_names.append(true_key+"_"+hypo_key+"_"+str(ifig)+".png")
figs.append(fig)
fig.suptitle('Posteriors for %s, %s'%(true_key,hypo_key))
#fontsize='large')
# Why is this not adding subplot?...
subplot = (icol%max_plots_per_fig + 1)
color = 'k' if plot_param_info else colors[icol%len(colors)]
plot_column(
true_key, hypo_key, subplot, column, template_settings,
color,plot_param_info=plot_param_info,pbins=pbins,
mctrue=mctrue)
return figs,fig_names
def find_alt_hierarchy_fit(asimov_data_set,
template_maker,
hypo_params,
hypo_normal,
minimizer_settings,
only_atm_params=True,
check_octant=False):
"""
For the hypothesis of the mass hierarchy being NMH
('normal_hierarchy'=True) or IMH ('normal_hierarchy'=False), finds the
best fit for the free params in 'param_values' for the alternative
(opposite) hierarchy that maximizes the LLH of the Asimov data set.
\params:
* asimov_data_set - asimov data set to find best fit llh
* template_maker - instance of class pisa.analysis.TemplateMaker, used to
generate the asimov data set.
* hypo_params - parameters for template generation
* hypo_normal - boolean for NMH (True) or IMH (False)
* minimizer_settings - settings for bfgs minimization
* only_atm_params - boolean to denote whether the fit will be over the
atmospheric oscillation parameters only or over all the free params
in params
"""
# Find best fit of the alternative hierarchy to the
# hypothesized asimov data set.
#hypo_types = [('hypo_IMH',False)] if data_normal else [('hypo_NMH',True)]
hypo_params = select_hierarchy(hypo_params, normal_hierarchy=hypo_normal)
with Timer() as t:
llh_data = find_max_llh_bfgs(asimov_data_set,
template_maker,
hypo_params,
minimizer_settings,
normal_hierarchy=hypo_normal,
check_octant=check_octant)
profile.info("==> elapsed time for optimizer: %s sec" % t.secs)
return llh_data
def get_fisher_matrices(template_settings,
grid_settings,
IMH=True,
NMH=False,
dump_all_stages=False,
save_templates=False,
outdir=None):
'''
Main function that runs the Fisher analysis for the chosen hierarchy(ies) (inverted by default).
Returns a dictionary of Fisher matrices, in the format:
{'IMH': {'cscd': [...],
'trck': [...],
'comb': [...],
},
'NMH': {'cscd': [...],
'trck': [...],
'comb': [...],
}
}
If save_templates=True and no hierarchy is given, only fiducial templates will be written out;
if one is given, then the templates used to obtain the gradients will be written out in
addition.
'''
if outdir is None and (save_templates or dump_all_stages):
logging.info(
"No output directory specified. Will save templates to current working directory."
)
outdir = os.getcwd()
tprofile.info("start initializing")
# Get the parameters
params = template_settings['params']
bins = template_settings['binning']
# Artifically add the hierarchy parameter to the list of parameters
# The method get_hierarchy_gradients below will know how to deal with it
params['hierarchy_nh'] = {
"value": 1.,
"range": [0., 1.],
"fixed": False,
"prior": None
}
params['hierarchy_ih'] = {
"value": 0.,
"range": [0., 1.],
"fixed": False,
"prior": None
}
chosen_data = []
if IMH:
chosen_data.append(('IMH', False))
logging.info("Fisher matrix will be built for IMH.")
if NMH:
chosen_data.append(('NMH', True))
logging.info("Fisher matrix will be built for NMH.")
if chosen_data == []:
# In this case, only the fiducial maps (for both hierarchies) will be written
logging.info("No Fisher matrices will be built.")
# There is no sense in performing any of the following steps if no Fisher matrices are to be built
# and no templates are to be saved.
if chosen_data != [] or dump_all_stages or save_templates:
# Initialise return dict to hold Fisher matrices
fisher = {
data_tag: {
'cscd': [],
'trck': [],
'comb': []
}
for data_tag, data_normal in chosen_data
}
# Get a template maker with the settings used to initialize
template_maker = TemplateMaker(get_values(params), **bins)
tprofile.info("stop initializing\n")
# Generate fiducial templates for both hierarchies (needed for partial derivatives
# w.r.t. hierarchy parameter)
fiducial_maps = {}
for hierarchy in ['NMH', 'IMH']:
logging.info("Generating fiducial templates for %s." % hierarchy)
# Get the fiducial parameter values corresponding to this hierarchy
fiducial_params = select_hierarchy(
params, normal_hierarchy=(hierarchy == 'NMH'))
# Generate fiducial maps, either all of them or only the ultimate one
tprofile.info("start template calculation")
with Timer() as t:
fid_maps = template_maker.get_template(
get_values(fiducial_params), return_stages=dump_all_stages)
tprofile.info("==> elapsed time for template: %s sec" % t.secs)
fiducial_maps[
hierarchy] = fid_maps[4] if dump_all_stages else fid_maps
# save fiducial map(s)
# all stages
if dump_all_stages:
stage_names = ("0_unoscillated_flux", "1_oscillated_flux",
"2_oscillated_counts", "3_reco", "4_pid")
stage_maps = {}
for stage in xrange(0, len(fid_maps)):
stage_maps[stage_names[stage]] = fid_maps[stage]
logging.info(
"Writing fiducial maps (all stages) for %s to %s." %
(hierarchy, outdir))
to_json(stage_maps,
os.path.join(outdir, "fid_map_" + hierarchy + ".json"))
# only the final stage
elif save_templates:
logging.info(
"Writing fiducial map (final stage) for %s to %s." %
(hierarchy, outdir))
to_json(fiducial_maps[hierarchy],
os.path.join(outdir, "fid_map_" + hierarchy + ".json"))
# Get_gradients and get_hierarchy_gradients will both (temporarily)
# store the templates used to generate the gradient maps
store_dir = outdir if save_templates else tempfile.gettempdir()
# Calculate Fisher matrices for the user-defined cases (NHM true and/or IMH true)
for data_tag, data_normal in chosen_data:
logging.info("Running Fisher analysis for %s." % (data_tag))
# The fiducial params are selected from the hierarchy case that does NOT match
# the data, as we are varying from this model to find the 'best fit'
fiducial_params = select_hierarchy(params, not data_normal)
# Get the free parameters (i.e. those for which the gradients should be calculated)
free_params = select_hierarchy(get_free_params(params),
not data_normal)
gradient_maps = {}
for param in free_params.keys():
# Special treatment for the hierarchy parameter
if param == 'hierarchy':
gradient_maps[param] = get_hierarchy_gradients(
data_tag=data_tag,
fiducial_maps=fiducial_maps,
fiducial_params=fiducial_params,
grid_settings=grid_settings,
store_dir=store_dir)
else:
gradient_maps[param] = get_gradients(
data_tag=data_tag,
param=param,
template_maker=template_maker,
fiducial_params=fiducial_params,
grid_settings=grid_settings,
store_dir=store_dir)
logging.info("Building Fisher matrix for %s." % (data_tag))
# Build Fisher matrices for the given hierarchy
fisher[data_tag] = build_fisher_matrix(
gradient_maps=gradient_maps,
fiducial_map=fiducial_maps['IMH']
if data_normal else fiducial_maps['NMH'],
template_settings=fiducial_params)
# If Fisher matrices exist for both channels, add the matrices to obtain
# the combined one.
if len(fisher[data_tag].keys()) > 1:
fisher[data_tag]['comb'] = FisherMatrix(
matrix=np.array([
f.matrix for f in fisher[data_tag].itervalues()
]).sum(axis=0),
parameters=gradient_maps.keys(), #order is important here!
best_fits=[
fiducial_params[par]['value']
for par in gradient_maps.keys()
],
priors=[
Prior.from_param(fiducial_params[par])
for par in gradient_maps.keys()
],
)
return fisher
else:
logging.info("Nothing to be done.")
return {}
def getAsimovParams(params, true_normal, th23_val):
asimov_params = select_hierarchy(params, normal_hierarchy=true_normal)
asimov_params['theta23']['value'] = th23_val
return asimov_params
profile.info("==> elapsed time to initialize templates: %s sec"%t.secs)
#data_types = [('data_NMH',True),('data_IMH',False)]
data_types = [('data_NMH',True)]
results = {}
# Store for future checking:
results['template_settings'] = template_settings
results['minimizer_settings'] = minimizer_settings
results['grid_settings'] = grid_settings
try:
for data_tag, data_normal in data_types:
results[data_tag] = {}
data_params = select_hierarchy(params,normal_hierarchy=data_normal)
asimov_data_set = get_asimov_fmap(template_maker,get_values(data_params),
chan=channel)
results[data_tag]['asimov_data'] = asimov_data_set
hypo_types = [('hypo_NMH',True)]
#hypo_types = [('hypo_NMH',True),('hypo_IMH',False)]
for hypo_tag, hypo_normal in hypo_types:
hypo_params = select_hierarchy(params,normal_hierarchy=hypo_normal)
# Now scan over theta23,deltam31 values and fix params to
# these values:
# Calculate steps for all free parameters
atm_params = get_atm_params(hypo_params)
calc_steps(atm_params, grid_settings['steps'])
# Build a list from all parameters that holds a list of (name, step) tuples
fiducial_params=alt_mh_settings,
channel=alt_mh_settings['channel'])
# Store all data tag related inputs:
output[data_tag]['asimov_data'] = asimov_data
output[data_tag]['asimov_data_null'] = asimov_data_null
output[data_tag]['alt_mh_settings'] = alt_mh_settings
output[data_tag]['llh_null'] = llh_data
# If we are not taking the best fit of the asimov data to the
# alternative hierarchy as the "null hypothesis", then we will use
# the parameters of the alternative hierarchy in the settings
# file, which correspond to the world best fit values.
if args.no_alt_fit:
null_settings = get_values(
select_hierarchy(template_settings['params'],
normal_hierarchy=(not data_normal)))
alt_mh_expectation = get_asimov_fmap(template_maker,
null_settings,
channel=null_settings['channel'])
else:
alt_mh_expectation = asimov_data_null
trials = []
for itrial in xrange(1, args.ntrials + 1):
results = {} # one trial of results
tprofile.info("start trial %d" % itrial)
logging.info(">" * 10 + "Running trial: %05d" % itrial + "<" * 10)
results['seed'] = get_seed()
logging.info(" RNG seed: %ld" % results['seed'])
template_settings = from_json(args.template_settings)
czbin_edges = template_settings['binning']['czbins']
ebin_edges = template_settings['binning']['ebins']
channel = template_settings['params']['channel']['value']
x_steps = 0.0001
if args.sim == '4digit':
MC_name = '1XXX'
elif args.sim == '5digit':
MC_name = '1XXXX'
elif args.sim == 'dima':
MC_name = 'Dima'
else:
MC_name = 'Other'
params = get_values(select_hierarchy(template_settings['params'],normal_hierarchy=True))
run_list = params['run_list']
run_nominal = params['run_nominal']
run_dict = params['run_dict']
for norm in [False]:
if norm:
HierarchyPrefix = 'NH'
DMRange = np.linspace(template_settings['params']['deltam31_nh']['range'][0],template_settings['params']['deltam31_nh']['range'][1],21)
THRange = np.linspace(0.35,1.25,10)
else:
HierarchyPrefix = 'IH'
DMRange = np.linspace(template_settings['params']['deltam31_ih']['range'][0],template_settings['params']['deltam31_ih']['range'][1],21)
THRange = np.linspace(0.35,1.25,10)
# For each trial, generate two pseudo-data experiemnts (one for each
# hierarchy), and for each find the best matching template in each of the
# hierarchy hypotheses.
# //////////////////////////////////////////////////////////////////////
results = {}
for data_tag, data_normal in [('data_NMH', True), ('data_IMH', False)]:
results[data_tag] = {}
# 0) get a random seed and store with the data
results[data_tag]['seed'] = get_seed()
logging.info(" RNG seed: %ld" % results[data_tag]['seed'])
# 1) get a pseudo data fmap from fiducial model (best fit vals of params).
fmap = get_pseudo_data_fmap(pseudo_data_template_maker,
get_values(
select_hierarchy(
pseudo_data_settings['params'],
normal_hierarchy=data_normal)),
seed=results[data_tag]['seed'],
chan=channel)
# 2) find max llh (and best fit free params) from matching pseudo data
# to templates.
for hypo_tag, hypo_normal in [('hypo_NMH', True),
('hypo_IMH', False)]:
physics.info("Finding best fit for %s under %s assumption" %
(data_tag, hypo_tag))
with Timer() as t:
llh_data = find_max_llh_bfgs(fmap,
template_maker,
template_settings['params'],
else:
pseudo_data_template_maker = template_maker
# //////////////////////////////////////////////////////////////////////
# Generate two pseudo-data experiments (one for each hierarchy),
# and for each experiment, find the best matching template in each
# of the hierarchy hypotheses.
# //////////////////////////////////////////////////////////////////////
results = {}
for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]:
results[data_tag] = {}
# 1) get "Asimov" average fiducial template:
asimov_fmap = get_asimov_fmap(pseudo_data_template_maker,
get_values(select_hierarchy(pseudo_data_settings['params'],
normal_hierarchy=data_normal)),
channel=channel)
# 2) find max llh or min chisquare (and best fit free params) from matching pseudo data
# to templates.
for hypo_tag, hypo_normal in [('hypo_NMH',True),('hypo_IMH',False)]:
physics.info("Finding best fit for %s under %s assumption"%(data_tag,hypo_tag))
tprofile.info("start optimizer")
tprofile.info("Using %s"%metric_name)
opt_data = find_opt_bfgs(asimov_fmap,template_maker,template_settings['params'],
minimizer_settings,args.save_steps,
normal_hierarchy=hypo_normal, check_octant=args.check_octant,
metric_name=metric_name)
tprofile.info("stop optimizer")
parser.add_argument('-o','--outdir',metavar='DIR',default="",
help="Directory to save the output figures.")
parser.add_argument('-v', '--verbose', action='count', default=0,
help='set verbosity level')
args = parser.parse_args()
set_verbosity(args.verbose)
template_settings = from_json(args.template_settings)
with Timer() as t:
template_maker = TemplateMaker(get_values(template_settings['params']),
**template_settings['binning'])
profile.info("==> elapsed time to initialize templates: %s sec"%t.secs)
# Make nmh template:
nmh_params = select_hierarchy(template_settings['params'],
normal_hierarchy=True)
imh_params = select_hierarchy(template_settings['params'],
normal_hierarchy=False)
with Timer(verbose=False) as t:
nmh = template_maker.get_template(get_values(nmh_params), return_stages=args.all)
profile.info("==> elapsed time to get NMH template: %s sec"%t.secs)
with Timer(verbose=False) as t:
imh = template_maker.get_template(get_values(imh_params), return_stages=args.all)
profile.info("==> elapsed time to get IMH template: %s sec"%t.secs)
# Or equivalently, if args.all:
if type(nmh) is tuple:
plot_stages(nmh, imh, title=args.title, save=args.save, outdir=args.outdir)
else: plot_pid_stage(nmh, imh, title=args.title, save=args.save, outdir=args.outdir)
if not args.save: plt.show()
def find_max_grid(fmap,template_maker,params,grid_settings,save_steps=True,
normal_hierarchy=True):
'''
Finds the template (and free systematic params) that maximize
likelihood that the data came from the chosen template of true
params, using a brute force grid scan over the whole parameter space.
returns a dictionary of llh data and best fit params, in the format:
{'llh': [...],
'param1': [...],
'param2': [...],
...}
where 'param1', 'param2', ... are the free params that are varied in the
scan. If save_steps is False, all lists only contain the best-fit parameters
and llh values.
'''
#print "NOW INSIDE find_max_grid:"
#print "After fixing to their true values, params dict is now: "
#for key in params.keys():
# try: print " >>param: %s value: %s"%(key,str(params[key]['best']))
# except: continue
# Get params dict which will be optimized (free_params) and which
# won't be (fixed_params) but are still needed for get_template()
fixed_params = get_fixed_params(select_hierarchy(params,normal_hierarchy))
free_params = get_free_params(select_hierarchy(params,normal_hierarchy))
#Obtain just the priors
priors = get_param_priors(free_params)
#Calculate steps for all free parameters
calc_steps(free_params, grid_settings['steps'])
#Build a list from all parameters that holds a list of (name, step) tuples
steplist = [ [(name,step) for step in param['steps']] for name, param in sorted(free_params.items())]
#Prepare to store all the steps
steps = {key:[] for key in free_params.keys()}
steps['llh'] = []
#Iterate over the cartesian product
for pos in product(*steplist):
#Get a dict with all parameter values at this position
#including the fixed parameters
template_params = dict(list(pos) + get_values(fixed_params).items())
#print " >> NOW IN LOOP: "
#for key in template_params.keys():
# try: print " >>param: %s value: %s"%(key,str(template_params[key]['value']))
# except: continue
# Now get true template
profile.info('start template calculation')
true_template = template_maker.get_template(template_params)
profile.info('stop template calculation')
true_fmap = flatten_map(true_template)
#and calculate the likelihood
llh = -get_binwise_llh(fmap,true_fmap)
#get sorted vals to match with priors
vals = [ v for k,v in sorted(pos) ]
llh -= sum([ get_prior_llh(vals,sigma,value) for (vals,(sigma,value)) in zip(vals,priors)])
# Save all values to steps and report
steps['llh'].append(llh)
physics.debug("LLH is %.2f at: "%llh)
for key, val in pos:
steps[key].append(val)
physics.debug(" %20s = %6.4f" %(key, val))
#Find best fit value
maxllh = min(steps['llh'])
maxpos = steps['llh'].index(maxllh)
#Report best fit
physics.info('Found best LLH = %.2f in %d calls at:'
%(maxllh,len(steps['llh'])))
for name, vals in steps.items():
physics.info(' %20s = %6.4f'%(name,vals[maxpos]))
#only save this maximum if asked for
if not save_steps:
steps[name]=vals[maxpos]
return steps
profile.info("==> elapsed time to initialize templates: %s sec"%t.secs)
#data_types = [('data_NMH',True),('data_IMH',False)]
data_types = [('data_NMH',True)]
results = {}
# Store for future checking:
results['template_settings'] = template_settings
results['minimizer_settings'] = minimizer_settings
results['grid_settings'] = grid_settings
try:
for data_tag, data_normal in data_types:
results[data_tag] = {}
data_params = select_hierarchy(params,normal_hierarchy=data_normal)
asimov_data_set = get_asimov_fmap(template_maker,get_values(data_params),
chan=channel)
results[data_tag]['asimov_data'] = asimov_data_set
hypo_types = [('hypo_NMH',True)]
#hypo_types = [('hypo_NMH',True),('hypo_IMH',False)]
for hypo_tag, hypo_normal in hypo_types:
hypo_params = select_hierarchy(params,normal_hierarchy=hypo_normal)
# Now scan over theta23,deltam31 values and fix params to
# these values:
# Calculate steps for all free parameters
atm_params = get_atm_params(hypo_params)
calc_steps(atm_params, grid_settings['steps'])
# Build a list from all parameters that holds a list of (name, step) tuples
def getAsimovParams(params,true_normal,th23_val):
asimov_params = select_hierarchy(params,normal_hierarchy=true_normal)
asimov_params['theta23']['value'] = th23_val
return asimov_params
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=True):
"""
Finds the template (and free systematic params) that maximize
likelihood that the data came from the chosen template of true
params, using the limited memory BFGS algorithm subject to bounds
(l_bfgs_b).
returns a dictionary of llh data and best fit params, in the format:
{'llh': [...],
'param1': [...],
'param2': [...],
...}
where 'param1', 'param2', ... are the free params varied by
optimizer, and they hold a list of all the values tested in
optimizer algorithm, unless save_steps is False, in which case
they are one element in length-the best fit params and best fit llh.
"""
# Get params dict which will be optimized (free_params) and which
# won't be (fixed_params) but are still needed for get_template()
fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy))
free_params = get_free_params(select_hierarchy(params, normal_hierarchy))
init_vals = get_param_values(free_params)
scales = get_param_scales(free_params)
bounds = get_param_bounds(free_params)
priors = get_param_priors(free_params)
names = sorted(free_params.keys())
# Scale init-vals and bounds to work with bfgs opt:
init_vals = np.array(init_vals) * np.array(scales)
bounds = [bounds[i] * scales[i] for i in range(len(bounds))]
opt_steps_dict = {key: [] for key in names}
opt_steps_dict["llh"] = []
const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors)
physics.info("%d parameters to be optimized" % len(free_params))
for name, init, (down, up), (prior, best) in zip(names, init_vals, bounds, priors):
physics.info(
("%20s : init = %6.4f, bounds = [%6.4f,%6.4f], " "best = %6.4f, prior = " + ("%6.4f" if prior else "%s"))
% (name, init, up, down, best, prior)
)
physics.debug("Optimizer settings:")
for key, item in bfgs_settings.items():
physics.debug(" %s -> `%s` = %.2e" % (item["desc"], key, item["value"]))
best_fit_vals, llh, dict_flags = opt.fmin_l_bfgs_b(
llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)
)
best_fit_params = {name: value for name, value in zip(names, best_fit_vals)}
# Report best fit
physics.info("Found best LLH = %.2f in %d calls at:" % (llh, dict_flags["funcalls"]))
for name, val in best_fit_params.items():
physics.info(" %20s = %6.4f" % (name, val))
# Report any warnings if there are
lvl = logging.WARN if (dict_flags["warnflag"] != 0) else logging.DEBUG
for name, val in dict_flags.items():
physics.log(lvl, " %s : %s" % (name, val))
if not save_steps:
# Do not store the extra history of opt steps:
for key in opt_steps_dict.keys():
opt_steps_dict[key] = [opt_steps_dict[key][-1]]
return opt_steps_dict
def find_max_grid(fmap,
template_maker,
params,
grid_settings,
save_steps=True,
normal_hierarchy=True):
'''
Finds the template (and free systematic params) that maximize
likelihood that the data came from the chosen template of true
params, using a brute force grid scan over the whole parameter space.
returns a dictionary of llh data and best fit params, in the format:
{'llh': [...],
'param1': [...],
'param2': [...],
...}
where 'param1', 'param2', ... are the free params that are varied in the
scan. If save_steps is False, all lists only contain the best-fit parameters
and llh values.
'''
#print "NOW INSIDE find_max_grid:"
#print "After fixing to their true values, params dict is now: "
#for key in params.keys():
# try: print " >>param: %s value: %s"%(key,str(params[key]['best']))
# except: continue
# Get params dict which will be optimized (free_params) and which
# won't be (fixed_params) but are still needed for get_template()
fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy))
free_params = get_free_params(select_hierarchy(params, normal_hierarchy))
# Obtain just the priors
priors = get_param_priors(free_params)
# Calculate steps [(prior,value),...] for all free parameters
calc_steps(free_params, grid_settings['steps'])
# Build a list from all parameters that holds a list of (name, step) tuples
steplist = [[(name, step) for step in param['steps']]
for name, param in sorted(free_params.items())]
# Prepare to store all the steps
steps = {key: [] for key in free_params.keys()}
steps['llh'] = []
# Iterate over the cartesian product
for pos in product(*steplist):
# Get a dict with all parameter values at this position
# including the fixed parameters
template_params = dict(list(pos) + get_values(fixed_params).items())
#print " >> NOW IN LOOP: "
#for key in template_params.keys():
# try: print " >>param: %s value: %s"%(key,str(template_params[key]['value']))
# except: continue
# Now get true template
tprofile.info('start template calculation')
true_template = template_maker.get_template(template_params)
tprofile.info('stop template calculation')
true_fmap = flatten_map(true_template)
# and calculate the likelihood
llh = -get_binwise_llh(fmap, true_fmap)
# get sorted vals to match with priors
vals = [v for k, v in sorted(pos)]
llh -= sum([prior.llh(val) for val, prior in zip(vals, priors)])
# Save all values to steps and report
steps['llh'].append(llh)
physics.debug("LLH is %.2f at: " % llh)
for key, val in pos:
steps[key].append(val)
physics.debug(" %20s = %6.4f" % (key, val))
# Find best fit value
maxllh = min(steps['llh'])
maxpos = steps['llh'].index(maxllh)
# Report best fit
physics.info('Found best LLH = %.2f in %d calls at:' %
(maxllh, len(steps['llh'])))
for name, vals in steps.items():
physics.info(' %20s = %6.4f' % (name, vals[maxpos]))
# only save this maximum if asked for
if not save_steps:
steps[name] = vals[maxpos]
return steps
type=str,
action='store',
default="template.json",
help='file to store the output')
args = parser.parse_args()
set_verbosity(args.verbose)
with Timer() as t:
#Load all the settings
model_settings = from_json(args.template_settings)
#Select a hierarchy
logging.info('Selected %s hierarchy' %
('normal' if args.normal else 'inverted'))
params = select_hierarchy(model_settings['params'],
normal_hierarchy=args.normal)
#Intialize template maker
template_maker = TemplateMaker(get_values(params),
**model_settings['binning'])
tprofile.info(" ==> elapsed time to initialize templates: %s sec" %
t.secs)
#Now get the actual template
with Timer(verbose=False) as t:
template_maps = template_maker.get_template(
get_values(params), return_stages=args.save_all)
tprofile.info("==> elapsed time to get template: %s sec" % t.secs)
logging.info("Saving file to %s" % args.outfile)
to_json(template_maps, args.outfile)
for itrial in xrange(1, args.ntrials+1):
profile.info("start trial %d"%itrial)
logging.info(">"*10 + "Running trial: %05d"%itrial + "<"*10)
# //////////////////////////////////////////////////////////////////////
# For each trial, generate two pseudo-data experiemnts (one for each
# hierarchy), and for each find the best matching template in each of the
# hierarchy hypothesis.
# //////////////////////////////////////////////////////////////////////
results = {}
for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]:
results[data_tag] = {}
# 1) get a pseudo data fmap from fiducial model (best fit vals of params).
fiducial_param_values = get_values(select_hierarchy(params, normal_hierarchy=data_normal))
fmap = get_pseudo_data_fmap(template_maker=template_maker,
fiducial_params=fiducial_param_values,
channel=fiducial_param_values['channel'])
# 2) find max llh (and best fit free params) from matching pseudo data
# to templates.
for hypo_tag, hypo_normal in [('hypo_NMH',True),('hypo_IMH',False)]:
physics.info("Finding best fit for %s under %s assumption"%(data_tag,hypo_tag))
profile.info("start scan")
llh_data = find_max_grid(fmap=fmap,
template_maker=template_maker,
params=params,
grid_settings=grid_settings,
save_steps=args.save_steps,
normal_hierarchy=hypo_normal)
# //////////////////////////////////////////////////////////////////////
# For each trial, generate two pseudo-data experiemnts (one for each
# hierarchy), and for each find the best matching template in each of the
# hierarchy hypothesis.
# //////////////////////////////////////////////////////////////////////
results = {}
for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]:
results[data_tag] = {}
# 0) get a random seed and store with the data
results[data_tag]['seed'] = get_seed()
# 1) get a pseudo data fmap from fiducial model (best fit vals of params).
fmap = get_pseudo_data_fmap(template_maker,
get_values(select_hierarchy(params,
normal_hierarchy=data_normal)),
seed=results[data_tag]['seed'])
# 2) find max llh (and best fit free params) from matching pseudo data
# to templates.
for hypo_tag, hypo_normal in [('hypo_NMH',True),('hypo_IMH',False)]:
physics.info("Finding best fit for %s under %s assumption"%(data_tag,hypo_tag))
profile.info("start optimizer")
llh_data = find_max_llh_bfgs(fmap,template_maker,params,
minimizer_settings,args.save_steps,normal_hierarchy=hypo_normal)
profile.info("stop optimizer")
#Store the LLH data
results[data_tag][hypo_tag] = llh_data
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False,
normal_hierarchy=None, check_octant=False):
"""
Finds the template (and free systematic params) that maximize
likelihood that the data came from the chosen template of true
params, using the limited memory BFGS algorithm subject to bounds
(l_bfgs_b).
returns a dictionary of llh data and best fit params, in the format:
{'llh': [...],
'param1': [...],
'param2': [...],
...}
where 'param1', 'param2', ... are the free params varied by
optimizer, and they hold a list of all the values tested in
optimizer algorithm, unless save_steps is False, in which case
they are one element in length-the best fit params and best fit llh.
"""
# Get params dict which will be optimized (free_params) and which
# won't be (fixed_params) but are still needed for get_template()
fixed_params = get_fixed_params(select_hierarchy(params,normal_hierarchy))
free_params = get_free_params(select_hierarchy(params,normal_hierarchy))
if len(free_params) == 0:
logging.warn("NO FREE PARAMS, returning LLH")
true_template = template_maker.get_template(get_values(fixed_params))
channel = params['channel']['value']
true_fmap = flatten_map(true_template,chan=channel)
return {'llh': [-get_binwise_llh(fmap,true_fmap)]}
init_vals = get_param_values(free_params)
scales = get_param_scales(free_params)
bounds = get_param_bounds(free_params)
priors = get_param_priors(free_params)
names = sorted(free_params.keys())
# Scale init-vals and bounds to work with bfgs opt:
init_vals = np.array(init_vals)*np.array(scales)
bounds = [bounds[i]*scales[i] for i in range(len(bounds))]
opt_steps_dict = {key:[] for key in names}
opt_steps_dict['llh'] = []
const_args = (names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors)
display_optimizer_settings(free_params, names, init_vals, bounds, priors,
bfgs_settings)
best_fit_vals,llh,dict_flags = opt.fmin_l_bfgs_b(
llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0,
bounds=bounds, **get_values(bfgs_settings))
# If needed, run optimizer again, checking for second octant solution:
if check_octant and ('theta23' in free_params.keys()):
physics.info("Checking alternative octant solution")
old_th23_val = free_params['theta23']['value']
delta = np.pi - old_th23_val
free_params['theta23']['value'] = np.pi + delta
init_vals = get_param_values(free_params)
const_args = (names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors)
display_optimizer_settings(free_params, names, init_vals, bounds, priors,
bfgs_settings)
alt_fit_vals,alt_llh,alt_dict_flags = opt.fmin_l_bfgs_b(
llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0,
bounds=bounds, **get_values(bfgs_settings))
# Alternative octant solution is optimal:
if alt_llh < llh:
best_fit_vals = alt_fit_vals
llh = alt_llh
dict_flags = alt_dict_flags
best_fit_params = { name: value for name, value in zip(names, best_fit_vals) }
#Report best fit
physics.info('Found best LLH = %.2f in %d calls at:'
%(llh,dict_flags['funcalls']))
for name, val in best_fit_params.items():
physics.info(' %20s = %6.4f'%(name,val))
#Report any warnings if there are
lvl = logging.WARN if (dict_flags['warnflag'] != 0) else logging.DEBUG
for name, val in dict_flags.items():
physics.log(lvl," %s : %s"%(name,val))
if not save_steps:
# Do not store the extra history of opt steps:
for key in opt_steps_dict.keys():
opt_steps_dict[key] = [opt_steps_dict[key][-1]]
return opt_steps_dict
params = template_settings['params']
mctrue_types = [('true_NMH',True),('true_IMH',False)]
results = {}
# Store for future checking:
results['template_settings'] = template_settings
results['minimizer_settings'] = minimizer_settings
results['grid_settings'] = grid_settings
for true_tag, true_normal in mctrue_types:
results[true_tag] = {}
result = {}
steplist = createStepList(params,true_normal,grid_settings)
free_params = select_hierarchy(get_free_params(params),
normal_hierarchy=true_normal)
# Set up the arrays to store the true/fit values in:
for key in free_params.keys():
result['true_'+key] = []
result['fit_'+key] = []
result['asimov_data'] = []
# This will actually only iterate over theta23 (for now), changing
# the asimov data set:
for step in steplist:
print "Running at asimov parameters: %s"%step
asimov_params = get_values(getAsimovParams(params,true_normal,step))
asimov_data_set = get_asimov_fmap(
template_maker, asimov_params,
'--verbose',
action='count',
default=0,
help='set verbosity level')
args = parser.parse_args()
set_verbosity(args.verbose)
template_settings = from_json(args.template_settings)
with Timer() as t:
template_maker = TemplateMaker(get_values(template_settings['params']),
**template_settings['binning'])
tprofile.info("==> elapsed time to initialize templates: %s sec" % t.secs)
# Make nmh template:
nmh_params = select_hierarchy(template_settings['params'],
normal_hierarchy=True)
imh_params = select_hierarchy(template_settings['params'],
normal_hierarchy=False)
with Timer(verbose=False) as t:
nmh = template_maker.get_template(get_values(nmh_params),
return_stages=args.all)
tprofile.info("==> elapsed time to get NMH template: %s sec" % t.secs)
with Timer(verbose=False) as t:
imh = template_maker.get_template(get_values(imh_params),
return_stages=args.all)
tprofile.info("==> elapsed time to get IMH template: %s sec" % t.secs)
# Or equivalently, if args.all:
if type(nmh) is tuple:
plot_stages(nmh,
imh,
def find_max_llh_bfgs(fmap,
template_maker,
params,
bfgs_settings,
save_steps=False,
normal_hierarchy=None,
check_octant=False):
"""
Finds the template (and free systematic params) that maximize
likelihood that the data came from the chosen template of true
params, using the limited memory BFGS algorithm subject to bounds
(l_bfgs_b).
returns a dictionary of llh data and best fit params, in the format:
{'llh': [...],
'param1': [...],
'param2': [...],
...}
where 'param1', 'param2', ... are the free params varied by
optimizer, and they hold a list of all the values tested in
optimizer algorithm, unless save_steps is False, in which case
they are one element in length-the best fit params and best fit llh.
"""
# Get params dict which will be optimized (free_params) and which
# won't be (fixed_params) but are still needed for get_template()
fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy))
free_params = get_free_params(select_hierarchy(params, normal_hierarchy))
if len(free_params) == 0:
logging.warn("NO FREE PARAMS, returning LLH")
true_template = template_maker.get_template(get_values(fixed_params))
channel = params['channel']['value']
true_fmap = flatten_map(true_template, chan=channel)
return {'llh': [-get_binwise_llh(fmap, true_fmap)]}
init_vals = get_param_values(free_params)
scales = get_param_scales(free_params)
bounds = get_param_bounds(free_params)
priors = get_param_priors(free_params)
names = sorted(free_params.keys())
# Scale init-vals and bounds to work with bfgs opt:
init_vals = np.array(init_vals) * np.array(scales)
bounds = [bounds[i] * scales[i] for i in range(len(bounds))]
opt_steps_dict = {key: [] for key in names}
opt_steps_dict['llh'] = []
const_args = (names, scales, fmap, fixed_params, template_maker,
opt_steps_dict, priors)
display_optimizer_settings(free_params, names, init_vals, bounds, priors,
bfgs_settings)
best_fit_vals, llh, dict_flags = opt.fmin_l_bfgs_b(
llh_bfgs,
init_vals,
args=const_args,
approx_grad=True,
iprint=0,
bounds=bounds,
**get_values(bfgs_settings))
# If needed, run optimizer again, checking for second octant solution:
if check_octant and ('theta23' in free_params.keys()):
physics.info("Checking alternative octant solution")
old_th23_val = free_params['theta23']['value']
delta = np.pi - old_th23_val
free_params['theta23']['value'] = np.pi + delta
init_vals = get_param_values(free_params)
const_args = (names, scales, fmap, fixed_params, template_maker,
opt_steps_dict, priors)
display_optimizer_settings(free_params, names, init_vals, bounds,
priors, bfgs_settings)
alt_fit_vals, alt_llh, alt_dict_flags = opt.fmin_l_bfgs_b(
llh_bfgs,
init_vals,
args=const_args,
approx_grad=True,
iprint=0,
bounds=bounds,
**get_values(bfgs_settings))
# Alternative octant solution is optimal:
if alt_llh < llh:
best_fit_vals = alt_fit_vals
llh = alt_llh
dict_flags = alt_dict_flags
best_fit_params = {
name: value
for name, value in zip(names, best_fit_vals)
}
#Report best fit
physics.info('Found best LLH = %.2f in %d calls at:' %
(llh, dict_flags['funcalls']))
for name, val in best_fit_params.items():
physics.info(' %20s = %6.4f' % (name, val))
#Report any warnings if there are
lvl = logging.WARN if (dict_flags['warnflag'] != 0) else logging.DEBUG
for name, val in dict_flags.items():
physics.log(lvl, " %s : %s" % (name, val))
if not save_steps:
# Do not store the extra history of opt steps:
for key in opt_steps_dict.keys():
opt_steps_dict[key] = [opt_steps_dict[key][-1]]
return opt_steps_dict
for itrial in xrange(1, args.ntrials + 1):
profile.info("start trial %d" % itrial)
logging.info(">" * 10 + "Running trial: %05d" % itrial + "<" * 10)
# //////////////////////////////////////////////////////////////////////
# For each trial, generate two pseudo-data experiemnts (one for each
# hierarchy), and for each find the best matching template in each of the
# hierarchy hypothesis.
# //////////////////////////////////////////////////////////////////////
results = {}
for data_tag, data_normal in [("data_NMH", True), ("data_IMH", False)]:
results[data_tag] = {}
# 1) get a pseudo data fmap from fiducial model (best fit vals of params).
fmap = get_pseudo_data_fmap(template_maker, get_values(select_hierarchy(params, normal_hierarchy=data_normal)))
# 2) find max llh (and best fit free params) from matching pseudo data
# to templates.
for hypo_tag, hypo_normal in [("hypo_NMH", True), ("hypo_IMH", False)]:
physics.info("Finding best fit for %s under %s assumption" % (data_tag, hypo_tag))
profile.info("start scan")
llh_data = find_max_grid(
fmap, template_maker, params, grid_settings, args.save_steps, normal_hierarchy=hypo_normal
)
profile.info("stop scan")
# Store the LLH data
results[data_tag][hypo_tag] = llh_data
help="Save all stages.")
parser.add_argument('-o', '--outfile', dest='outfile', metavar='FILE',
type=str, action='store',default="template.json",
help='file to store the output')
args = parser.parse_args()
set_verbosity(args.verbose)
with Timer() as t:
#Load all the settings
model_settings = from_json(args.template_settings)
#Select a hierarchy
logging.info('Selected %s hierarchy'%
('normal' if args.normal else 'inverted'))
params = select_hierarchy(model_settings['params'],
normal_hierarchy=args.normal)
#Intialize template maker
template_maker = TemplateMaker(get_values(params),
**model_settings['binning'])
profile.info(" ==> elapsed time to initialize templates: %s sec"%t.secs)
#Now get the actual template
with Timer(verbose=False) as t:
template_maps = template_maker.get_template(get_values(params),
return_stages=args.save_all)
profile.info("==> elapsed time to get template: %s sec"%t.secs)
logging.info("Saving file to %s"%args.outfile)
to_json(template_maps, args.outfile)
fiducial_params=alt_mh_settings,
channel=alt_mh_settings['channel'])
# Store all data tag related inputs:
output[data_tag]['asimov_data'] = asimov_data
output[data_tag]['asimov_data_null'] = asimov_data_null
output[data_tag]['alt_mh_settings'] = alt_mh_settings
output[data_tag]['llh_null'] = llh_data
# If we are not taking the best fit of the asimov data to the
# alternative hierarchy as the "null hypothesis", then we will use
# the parameters of the alternative hierarchy in the settings
# file, which correspond to the world best fit values.
if args.no_alt_fit:
null_settings = get_values(
select_hierarchy(template_settings['params'],
normal_hierarchy= (not data_normal)))
alt_mh_expectation = get_asimov_fmap(
template_maker, null_settings, channel=null_settings['channel'] )
else:
alt_mh_expectation = asimov_data_null
trials = []
for itrial in xrange(1,args.ntrials+1):
results = {} # one trial of results
tprofile.info("start trial %d"%itrial)
logging.info(">"*10 + "Running trial: %05d"%itrial + "<"*10)
results['seed'] = get_seed()
logging.info(" RNG seed: %ld"%results['seed'])
params = template_settings['params']
mctrue_types = [('true_NMH', True), ('true_IMH', False)]
results = {}
# Store for future checking:
results['template_settings'] = template_settings
results['minimizer_settings'] = minimizer_settings
results['grid_settings'] = grid_settings
for true_tag, true_normal in mctrue_types:
results[true_tag] = {}
result = {}
steplist = createStepList(params, true_normal, grid_settings)
free_params = select_hierarchy(get_free_params(params),
normal_hierarchy=true_normal)
# Set up the arrays to store the true/fit values in:
for key in free_params.keys():
result['true_' + key] = []
result['fit_' + key] = []
result['asimov_data'] = []
# This will actually only iterate over theta23 (for now), changing
# the asimov data set:
for step in steplist:
print "Running at asimov parameters: %s" % step
asimov_params = get_values(getAsimovParams(params, true_normal, step))
asimov_data_set = get_asimov_fmap(template_maker,
asimov_params,
def get_fisher_matrices(template_settings, grid_settings, IMH=True, NMH=False, dump_all_stages=False,
save_templates=False, outdir=None):
'''
Main function that runs the Fisher analysis for the chosen hierarchy(ies) (inverted by default).
Returns a dictionary of Fisher matrices, in the format:
{'IMH': {'cscd': [...],
'trck': [...],
'comb': [...],
},
'NMH': {'cscd': [...],
'trck': [...],
'comb': [...],
}
}
If save_templates=True and no hierarchy is given, only fiducial templates will be written out;
if one is given, then the templates used to obtain the gradients will be written out in
addition.
'''
if outdir is None and (save_templates or dump_all_stages):
logging.info("No output directory specified. Will save templates to current working directory.")
outdir = os.getcwd()
profile.info("start initializing")
# Get the parameters
params = template_settings['params']
bins = template_settings['binning']
# Artifically add the hierarchy parameter to the list of parameters
# The method get_hierarchy_gradients below will know how to deal with it
params['hierarchy_nh'] = { "value": 1., "range": [0.,1.],
"fixed": False, "prior": None}
params['hierarchy_ih'] = { "value": 0., "range": [0.,1.],
"fixed": False, "prior": None}
chosen_data = []
if IMH:
chosen_data.append(('IMH',False))
logging.info("Fisher matrix will be built for IMH.")
if NMH:
chosen_data.append(('NMH',True))
logging.info("Fisher matrix will be built for NMH.")
if chosen_data == []:
# In this case, only the fiducial maps (for both hierarchies) will be written
logging.info("No Fisher matrices will be built.")
# There is no sense in performing any of the following steps if no Fisher matrices are to be built
# and no templates are to be saved.
if chosen_data!=[] or dump_all_stages or save_templates:
# Initialise return dict to hold Fisher matrices
fisher = { data_tag:{'cscd':[],'trck':[],'comb':[]} for data_tag, data_normal in chosen_data }
# Get a template maker with the settings used to initialize
template_maker = TemplateMaker(get_values(params),**bins)
profile.info("stop initializing\n")
# Generate fiducial templates for both hierarchies (needed for partial derivatives
# w.r.t. hierarchy parameter)
fiducial_maps = {}
for hierarchy in ['NMH','IMH']:
logging.info("Generating fiducial templates for %s."%hierarchy)
# Get the fiducial parameter values corresponding to this hierarchy
fiducial_params = select_hierarchy(params,normal_hierarchy=(hierarchy=='NMH'))
# Generate fiducial maps, either all of them or only the ultimate one
profile.info("start template calculation")
with Timer() as t:
fid_maps = template_maker.get_template(get_values(fiducial_params),
return_stages=dump_all_stages)
profile.info("==> elapsed time for template: %s sec"%t.secs)
fiducial_maps[hierarchy] = fid_maps[4] if dump_all_stages else fid_maps
# save fiducial map(s)
# all stages
if dump_all_stages:
stage_names = ("0_unoscillated_flux","1_oscillated_flux","2_oscillated_counts","3_reco","4_pid")
stage_maps = {}
for stage in xrange(0,len(fid_maps)):
stage_maps[stage_names[stage]] = fid_maps[stage]
logging.info("Writing fiducial maps (all stages) for %s to %s."%(hierarchy,outdir))
to_json(stage_maps,os.path.join(outdir,"fid_map_"+hierarchy+".json"))
# only the final stage
elif save_templates:
logging.info("Writing fiducial map (final stage) for %s to %s."%(hierarchy,outdir))
to_json(fiducial_maps[hierarchy],os.path.join(outdir,"fid_map_"+hierarchy+".json"))
# Get_gradients and get_hierarchy_gradients will both (temporarily)
# store the templates used to generate the gradient maps
store_dir = outdir if save_templates else tempfile.gettempdir()
# Calculate Fisher matrices for the user-defined cases (NHM true and/or IMH true)
for data_tag, data_normal in chosen_data:
logging.info("Running Fisher analysis for %s."%(data_tag))
# The fiducial params are selected from the hierarchy case that does NOT match
# the data, as we are varying from this model to find the 'best fit'
fiducial_params = select_hierarchy(params,not data_normal)
# Get the free parameters (i.e. those for which the gradients should be calculated)
free_params = select_hierarchy(get_free_params(params),not data_normal)
gradient_maps = {}
for param in free_params.keys():
# Special treatment for the hierarchy parameter
if param=='hierarchy':
gradient_maps[param] = get_hierarchy_gradients(data_tag,
fiducial_maps,
fiducial_params,
grid_settings,
store_dir,
)
else:
gradient_maps[param] = get_gradients(data_tag,
param,
template_maker,
fiducial_params,
grid_settings,
store_dir
)
logging.info("Building Fisher matrix for %s."%(data_tag))
# Build Fisher matrices for the given hierarchy
fisher[data_tag] = build_fisher_matrix(gradient_maps,fiducial_maps['IMH'] if data_normal else fiducial_maps['NMH'],fiducial_params)
# If Fisher matrices exist for both channels, add the matrices to obtain the combined one.
if len(fisher[data_tag].keys()) > 1:
fisher[data_tag]['comb'] = FisherMatrix(matrix=np.array([f.matrix for f in fisher[data_tag].itervalues()]).sum(axis=0),
parameters=gradient_maps.keys(), #order is important here!
best_fits=[fiducial_params[par]['value'] for par in gradient_maps.keys()],
priors=[fiducial_params[par]['prior'] for par in gradient_maps.keys()],
)
return fisher
else:
logging.info("Nothing to be done.")
return {}
profile.info("start trial %d" % itrial)
logging.info(">" * 10 + "Running trial: %05d" % itrial + "<" * 10)
# //////////////////////////////////////////////////////////////////////
# For each trial, generate two pseudo-data experiemnts (one for each
# hierarchy), and for each find the best matching template in each of the
# hierarchy hypothesis.
# //////////////////////////////////////////////////////////////////////
results = {}
for data_tag, data_normal in [('data_NMH', True), ('data_IMH', False)]:
results[data_tag] = {}
# 1) get a pseudo data fmap from fiducial model (best fit vals of params).
fmap = get_pseudo_data_fmap(
template_maker,
get_values(select_hierarchy(params, normal_hierarchy=data_normal)))
# 2) find max llh (and best fit free params) from matching pseudo data
# to templates.
for hypo_tag, hypo_normal in [('hypo_NMH', True), ('hypo_IMH', False)]:
physics.info("Finding best fit for %s under %s assumption" %
(data_tag, hypo_tag))
profile.info("start scan")
llh_data = find_max_grid(fmap,
template_maker,
params,
grid_settings,
args.save_steps,
normal_hierarchy=hypo_normal)
profile.info("stop scan")
