def getAltHierarchyBestFit(asimov_data, template_maker, params, minimizer_settings, hypo_normal, check_octant): """ Finds the best fit value of alternative hierarchy to that which was used to produce the asimov data set. \Params: * asimov_data - array of values of asimov data set (float) * template_maker - instance of class TemplateMaker service. * params - parameters with values, fixed, range, etc. of systematics * minimizer_settings - used with bfgs_b minimizer * hypo_normal - bool for Mass hierarchy being Normal (True) or inverted (False) * check_octant - bool to check the opposite octant for a solution to the minimization of the LLH. """ llh_data = find_alt_hierarchy_fit( asimov_data, template_maker, params, hypo_normal, minimizer_settings, only_atm_params=True, check_octant=check_octant) alt_params = get_values(select_hierarchy(params, normal_hierarchy=hypo_normal)) for key in llh_data.keys(): if key == 'llh': continue alt_params[key] = llh_data[key][-1] return alt_params, llh_data
def getAltHierarchyBestFit(asimov_data, template_maker, params, minimizer_settings, hypo_normal, check_octant): """ Finds the best fit value of alternative hierarchy to that which was used to produce the asimov data set. \Params: * asimov_data - array of values of asimov data set (float) * template_maker - instance of class TemplateMaker service. * params - parameters with values, fixed, range, etc. of systematics * minimizer_settings - used with bfgs_b minimizer * hypo_normal - bool for Mass hierarchy being Normal (True) or inverted (False) * check_octant - bool to check the opposite octant for a solution to the minimization of the LLH. """ llh_data = find_alt_hierarchy_fit(asimov_data, template_maker, params, hypo_normal, minimizer_settings, only_atm_params=True, check_octant=check_octant) alt_params = get_values( select_hierarchy(params, normal_hierarchy=hypo_normal)) for key in llh_data.keys(): if key == 'llh': continue alt_params[key] = llh_data[key][-1] return alt_params, llh_data
def get_gradients(data_tag, param, template_maker, fiducial_params, grid_settings, store_dir): """ Use the template maker to create all the templates needed to obtain the gradients. """ logging.info("Working on parameter %s."%param) steps = get_steps(param, grid_settings, fiducial_params) pmaps = {} # Generate one template for each value of the parameter in question and store in pmaps for param_value in steps: # Make the template corresponding to the current value of the parameter with Timer() as t: maps = template_maker.get_template( get_values(dict(fiducial_params,**{param:dict(fiducial_params[param], **{'value': param_value})}))) tprofile.info("==> elapsed time for template: %s sec"%t.secs) pmaps[param_value] = maps # Store the maps used to calculate partial derivatives if store_dir != tempfile.gettempdir(): logging.info("Writing maps for parameter %s to %s"%(param,store_dir)) to_json(pmaps, os.path.join(store_dir,param+"_"+data_tag+".json")) gradient_map = get_derivative_map(pmaps,fiducial_params[param],degree=2) return gradient_map
def getAsimovData(template_maker, params, data_normal): """ Generates the asimov data set (expected counts distribution) at parameters assuming hierarchy of data_normal \Params: * template_maker - instance of class TemplateMaker service. * params - parameters with values, fixed, range, etc. of systematics * data_normal - bool for Mass hierarchy being Normal (True) or inverted (False) """ fiducial_params = get_values(select_hierarchy( params, normal_hierarchy=data_normal)) return get_asimov_fmap(template_maker, fiducial_params, channel=fiducial_params['channel'])
def get_gradients(data_tag, param, template_maker, fiducial_params, grid_settings, store_dir): """ Use the template maker to create all the templates needed to obtain the gradients. """ logging.info("Working on parameter %s." % param) steps = get_steps(param, grid_settings, fiducial_params) pmaps = {} # Generate one template for each value of the parameter in question and store in pmaps for param_value in steps: # Make the template corresponding to the current value of the parameter with Timer() as t: maps = template_maker.get_template( get_values( dict( fiducial_params, **{ param: dict(fiducial_params[param], **{'value': param_value}) }))) tprofile.info("==> elapsed time for template: %s sec" % t.secs) pmaps[param_value] = maps # Store the maps used to calculate partial derivatives if store_dir != tempfile.gettempdir(): logging.info("Writing maps for parameter %s to %s" % (param, store_dir)) to_json(pmaps, os.path.join(store_dir, param + "_" + data_tag + ".json")) gradient_map = get_derivative_map(pmaps, fiducial_params[param], degree=2) return gradient_map
set_verbosity(args.verbose) # Read in the settings template_settings = from_json(args.template_settings) minimizer_settings = from_json(args.minimizer_settings) # Change this throughout code later? check_octant = not args.single_octant check_scipy_version(minimizer_settings) if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) # Assemble output dict output = { 'template_settings': template_settings, 'minimizer_settings': minimizer_settings } asimov_data = {} asimov_data_null = {} alt_mh_settings = {} for data_tag, data_normal in [('true_NMH', True), ('true_IMH', False)]: tprofile.info("Assuming: %s" % data_tag) output[data_tag] = {}
parser.add_argument('-o', '--outfile', dest='outfile', metavar='FILE', type=str, action='store',default="template.json", help='file to store the output') args = parser.parse_args() set_verbosity(args.verbose) with Timer() as t: #Load all the settings model_settings = from_json(args.template_settings) #Select a hierarchy logging.info('Selected %s hierarchy'% ('normal' if args.normal else 'inverted')) params = select_hierarchy(model_settings['params'], normal_hierarchy=args.normal) #Intialize template maker template_maker = TemplateMaker(get_values(params), **model_settings['binning']) profile.info(" ==> elapsed time to initialize templates: %s sec"%t.secs) #Now get the actual template with Timer(verbose=False) as t: template_maps = template_maker.get_template(get_values(params), return_stages=args.save_all) profile.info("==> elapsed time to get template: %s sec"%t.secs) logging.info("Saving file to %s"%args.outfile) to_json(template_maps, args.outfile)
if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True #Get the parameters params = template_settings['params'] # Make sure that atmospheric parameters are fixed: logging.warn("Ensuring that atmospheric parameters are fixed for this analysis") params = fix_atm_params(params) #print "params: ",params.items() with Timer() as t: template_maker = TemplateMaker(get_values(params),**template_settings['binning']) profile.info("==> elapsed time to initialize templates: %s sec"%t.secs) #data_types = [('data_NMH',True),('data_IMH',False)] data_types = [('data_NMH',True)] results = {} # Store for future checking: results['template_settings'] = template_settings results['minimizer_settings'] = minimizer_settings results['grid_settings'] = grid_settings try: for data_tag, data_normal in data_types: results[data_tag] = {}
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=None, check_octant=False): """ Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using the limited memory BFGS algorithm subject to bounds (l_bfgs_b). returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params varied by optimizer, and they hold a list of all the values tested in optimizer algorithm, unless save_steps is False, in which case they are one element in length-the best fit params and best fit llh. """ # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params,normal_hierarchy)) free_params = get_free_params(select_hierarchy(params,normal_hierarchy)) if len(free_params) == 0: logging.warn("NO FREE PARAMS, returning LLH") true_template = template_maker.get_template(get_values(fixed_params)) channel = params['channel']['value'] true_fmap = flatten_map(true_template,chan=channel) return {'llh': [-get_binwise_llh(fmap,true_fmap)]} init_vals = get_param_values(free_params) scales = get_param_scales(free_params) bounds = get_param_bounds(free_params) priors = get_param_priors(free_params) names = sorted(free_params.keys()) # Scale init-vals and bounds to work with bfgs opt: init_vals = np.array(init_vals)*np.array(scales) bounds = [bounds[i]*scales[i] for i in range(len(bounds))] opt_steps_dict = {key:[] for key in names} opt_steps_dict['llh'] = [] const_args = (names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) best_fit_vals,llh,dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # If needed, run optimizer again, checking for second octant solution: if check_octant and ('theta23' in free_params.keys()): physics.info("Checking alternative octant solution") old_th23_val = free_params['theta23']['value'] delta = np.pi - old_th23_val free_params['theta23']['value'] = np.pi + delta init_vals = get_param_values(free_params) const_args = (names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) alt_fit_vals,alt_llh,alt_dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # Alternative octant solution is optimal: if alt_llh < llh: best_fit_vals = alt_fit_vals llh = alt_llh dict_flags = alt_dict_flags best_fit_params = { name: value for name, value in zip(names, best_fit_vals) } #Report best fit physics.info('Found best LLH = %.2f in %d calls at:' %(llh,dict_flags['funcalls'])) for name, val in best_fit_params.items(): physics.info(' %20s = %6.4f'%(name,val)) #Report any warnings if there are lvl = logging.WARN if (dict_flags['warnflag'] != 0) else logging.DEBUG for name, val in dict_flags.items(): physics.log(lvl," %s : %s"%(name,val)) if not save_steps: # Do not store the extra history of opt steps: for key in opt_steps_dict.keys(): opt_steps_dict[key] = [opt_steps_dict[key][-1]] return opt_steps_dict
def find_max_grid(fmap, template_maker, params, grid_settings, save_steps=True, normal_hierarchy=True): ''' Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using a brute force grid scan over the whole parameter space. returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params that are varied in the scan. If save_steps is False, all lists only contain the best-fit parameters and llh values. ''' #print "NOW INSIDE find_max_grid:" #print "After fixing to their true values, params dict is now: " #for key in params.keys(): # try: print " >>param: %s value: %s"%(key,str(params[key]['best'])) # except: continue # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy)) free_params = get_free_params(select_hierarchy(params, normal_hierarchy)) # Obtain just the priors priors = get_param_priors(free_params) # Calculate steps [(prior,value),...] for all free parameters calc_steps(free_params, grid_settings['steps']) # Build a list from all parameters that holds a list of (name, step) tuples steplist = [[(name, step) for step in param['steps']] for name, param in sorted(free_params.items())] # Prepare to store all the steps steps = {key: [] for key in free_params.keys()} steps['llh'] = [] # Iterate over the cartesian product for pos in product(*steplist): # Get a dict with all parameter values at this position # including the fixed parameters template_params = dict(list(pos) + get_values(fixed_params).items()) #print " >> NOW IN LOOP: " #for key in template_params.keys(): # try: print " >>param: %s value: %s"%(key,str(template_params[key]['value'])) # except: continue # Now get true template tprofile.info('start template calculation') true_template = template_maker.get_template(template_params) tprofile.info('stop template calculation') true_fmap = flatten_map(true_template) # and calculate the likelihood llh = -get_binwise_llh(fmap, true_fmap) # get sorted vals to match with priors vals = [v for k, v in sorted(pos)] llh -= sum([prior.llh(val) for val, prior in zip(vals, priors)]) # Save all values to steps and report steps['llh'].append(llh) physics.debug("LLH is %.2f at: " % llh) for key, val in pos: steps[key].append(val) physics.debug(" %20s = %6.4f" % (key, val)) # Find best fit value maxllh = min(steps['llh']) maxpos = steps['llh'].index(maxllh) # Report best fit physics.info('Found best LLH = %.2f in %d calls at:' % (maxllh, len(steps['llh']))) for name, vals in steps.items(): physics.info(' %20s = %6.4f' % (name, vals[maxpos])) # only save this maximum if asked for if not save_steps: steps[name] = vals[maxpos] return steps
def bfgs_metric(opt_vals, names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors, metric_name='llh'): """ Function that the bfgs algorithm tries to minimize: wraps get_template() and get_binwise_llh() (or get_binwise_chisquare()), and returns the negative log likelihood (the chisquare). This function is set up this way because the fmin_l_bfgs_b algorithm must take a function with two inputs: params & *args, where 'params' are the actual VALUES to be varied, and must correspond to the limits in 'bounds', and 'args' are arguments which are not varied and optimized, but needed by the get_template() function here. Parameters ---------- opt_vals : sequence of scalars Systematics varied in the optimization. Format: [param1, param2, ... , paramN] names : sequence of str Dictionary keys corresponding to param1, param2, ... scales : sequence of float Scales to be applied before passing to get_template [IMPORTANT! In the optimizer, all parameters must be ~ the same order. Here, we keep them between 0.1,1 so the "epsilon" step size will vary the parameters with roughly the same precision.] fmap : sequence of float Pseudo data flattened map fixed_params : dict Other paramters needed by the get_template() function. template_maker : template maker object opt_steps_dict: dict Dictionary recording information regarding the steps taken for each trial of the optimization process. priors : sequence of pisa.utils.params.Prior objects Priors corresponding to opt_vals list. metric_name : string Returns chisquare instead of negative llh if metric_name is 'chisquare'. Note: this string has to be present as a key in opt_steps_dict Returns ------- metric_val : float either minimum negative llh or chisquare found by BFGS minimizer """ # free parameters being "optimized" by minimizer re-scaled to their true # values. unscaled_opt_vals = [ opt_vals[i] / scales[i] for i in xrange(len(opt_vals)) ] unscaled_free_params = { names[i]: val for i, val in enumerate(unscaled_opt_vals) } template_params = dict(unscaled_free_params.items() + get_values(fixed_params).items()) # Now get true template, and compute metric with Timer() as t: if template_params['theta23'] == 0.0: logging.info( "Zero theta23, so generating no oscillations template...") true_template = template_maker.get_template_no_osc(template_params) else: true_template = template_maker.get_template(template_params) tprofile.info("==> elapsed time for template maker: %s sec" % t.secs) true_fmap = flatten_map(template=true_template, channel=template_params['channel']) # NOTE: The minus sign is present on both of these next two lines # because the optimizer finds a minimum rather than maximum, so we # have to minimize the negative of the log likelhood. if metric_name == 'chisquare': metric_val = get_binwise_chisquare(fmap, true_fmap) metric_val += sum([ prior.chi2(opt_val) for (opt_val, prior) in zip(unscaled_opt_vals, priors) ]) elif metric_name == 'llh': metric_val = -get_binwise_llh(fmap, true_fmap) metric_val -= sum([ prior.llh(opt_val) for (opt_val, prior) in zip(unscaled_opt_vals, priors) ]) # Save all optimizer-tested values to opt_steps_dict, to see # optimizer history later for key in names: opt_steps_dict[key].append(template_params[key]) opt_steps_dict[metric_name].append(metric_val) physics.debug("%s is %.2f at: " % (metric_name, metric_val)) for name, val in zip(names, opt_vals): physics.debug(" %20s = %6.4f" % (name, val)) return metric_val
def find_max_grid(fmap,template_maker,params,grid_settings,save_steps=True, normal_hierarchy=True): ''' Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using a brute force grid scan over the whole parameter space. returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params that are varied in the scan. If save_steps is False, all lists only contain the best-fit parameters and llh values. ''' #print "NOW INSIDE find_max_grid:" #print "After fixing to their true values, params dict is now: " #for key in params.keys(): # try: print " >>param: %s value: %s"%(key,str(params[key]['best'])) # except: continue # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params,normal_hierarchy)) free_params = get_free_params(select_hierarchy(params,normal_hierarchy)) #Obtain just the priors priors = get_param_priors(free_params) #Calculate steps for all free parameters calc_steps(free_params, grid_settings['steps']) #Build a list from all parameters that holds a list of (name, step) tuples steplist = [ [(name,step) for step in param['steps']] for name, param in sorted(free_params.items())] #Prepare to store all the steps steps = {key:[] for key in free_params.keys()} steps['llh'] = [] #Iterate over the cartesian product for pos in product(*steplist): #Get a dict with all parameter values at this position #including the fixed parameters template_params = dict(list(pos) + get_values(fixed_params).items()) #print " >> NOW IN LOOP: " #for key in template_params.keys(): # try: print " >>param: %s value: %s"%(key,str(template_params[key]['value'])) # except: continue # Now get true template profile.info('start template calculation') true_template = template_maker.get_template(template_params) profile.info('stop template calculation') true_fmap = flatten_map(true_template) #and calculate the likelihood llh = -get_binwise_llh(fmap,true_fmap) #get sorted vals to match with priors vals = [ v for k,v in sorted(pos) ] llh -= sum([ get_prior_llh(vals,sigma,value) for (vals,(sigma,value)) in zip(vals,priors)]) # Save all values to steps and report steps['llh'].append(llh) physics.debug("LLH is %.2f at: "%llh) for key, val in pos: steps[key].append(val) physics.debug(" %20s = %6.4f" %(key, val)) #Find best fit value maxllh = min(steps['llh']) maxpos = steps['llh'].index(maxllh) #Report best fit physics.info('Found best LLH = %.2f in %d calls at:' %(maxllh,len(steps['llh']))) for name, vals in steps.items(): physics.info(' %20s = %6.4f'%(name,vals[maxpos])) #only save this maximum if asked for if not save_steps: steps[name]=vals[maxpos] return steps
def get_fisher_matrices(template_settings, grid_settings, IMH=True, NMH=False, dump_all_stages=False, save_templates=False, outdir=None): ''' Main function that runs the Fisher analysis for the chosen hierarchy(ies) (inverted by default). Returns a dictionary of Fisher matrices, in the format: {'IMH': {'cscd': [...], 'trck': [...], 'comb': [...], }, 'NMH': {'cscd': [...], 'trck': [...], 'comb': [...], } } If save_templates=True and no hierarchy is given, only fiducial templates will be written out; if one is given, then the templates used to obtain the gradients will be written out in addition. ''' if outdir is None and (save_templates or dump_all_stages): logging.info( "No output directory specified. Will save templates to current working directory." ) outdir = os.getcwd() tprofile.info("start initializing") # Get the parameters params = template_settings['params'] bins = template_settings['binning'] # Artifically add the hierarchy parameter to the list of parameters # The method get_hierarchy_gradients below will know how to deal with it params['hierarchy_nh'] = { "value": 1., "range": [0., 1.], "fixed": False, "prior": None } params['hierarchy_ih'] = { "value": 0., "range": [0., 1.], "fixed": False, "prior": None } chosen_data = [] if IMH: chosen_data.append(('IMH', False)) logging.info("Fisher matrix will be built for IMH.") if NMH: chosen_data.append(('NMH', True)) logging.info("Fisher matrix will be built for NMH.") if chosen_data == []: # In this case, only the fiducial maps (for both hierarchies) will be written logging.info("No Fisher matrices will be built.") # There is no sense in performing any of the following steps if no Fisher matrices are to be built # and no templates are to be saved. if chosen_data != [] or dump_all_stages or save_templates: # Initialise return dict to hold Fisher matrices fisher = { data_tag: { 'cscd': [], 'trck': [], 'comb': [] } for data_tag, data_normal in chosen_data } # Get a template maker with the settings used to initialize template_maker = TemplateMaker(get_values(params), **bins) tprofile.info("stop initializing\n") # Generate fiducial templates for both hierarchies (needed for partial derivatives # w.r.t. hierarchy parameter) fiducial_maps = {} for hierarchy in ['NMH', 'IMH']: logging.info("Generating fiducial templates for %s." % hierarchy) # Get the fiducial parameter values corresponding to this hierarchy fiducial_params = select_hierarchy( params, normal_hierarchy=(hierarchy == 'NMH')) # Generate fiducial maps, either all of them or only the ultimate one tprofile.info("start template calculation") with Timer() as t: fid_maps = template_maker.get_template( get_values(fiducial_params), return_stages=dump_all_stages) tprofile.info("==> elapsed time for template: %s sec" % t.secs) fiducial_maps[ hierarchy] = fid_maps[4] if dump_all_stages else fid_maps # save fiducial map(s) # all stages if dump_all_stages: stage_names = ("0_unoscillated_flux", "1_oscillated_flux", "2_oscillated_counts", "3_reco", "4_pid") stage_maps = {} for stage in xrange(0, len(fid_maps)): stage_maps[stage_names[stage]] = fid_maps[stage] logging.info( "Writing fiducial maps (all stages) for %s to %s." % (hierarchy, outdir)) to_json(stage_maps, os.path.join(outdir, "fid_map_" + hierarchy + ".json")) # only the final stage elif save_templates: logging.info( "Writing fiducial map (final stage) for %s to %s." % (hierarchy, outdir)) to_json(fiducial_maps[hierarchy], os.path.join(outdir, "fid_map_" + hierarchy + ".json")) # Get_gradients and get_hierarchy_gradients will both (temporarily) # store the templates used to generate the gradient maps store_dir = outdir if save_templates else tempfile.gettempdir() # Calculate Fisher matrices for the user-defined cases (NHM true and/or IMH true) for data_tag, data_normal in chosen_data: logging.info("Running Fisher analysis for %s." % (data_tag)) # The fiducial params are selected from the hierarchy case that does NOT match # the data, as we are varying from this model to find the 'best fit' fiducial_params = select_hierarchy(params, not data_normal) # Get the free parameters (i.e. those for which the gradients should be calculated) free_params = select_hierarchy(get_free_params(params), not data_normal) gradient_maps = {} for param in free_params.keys(): # Special treatment for the hierarchy parameter if param == 'hierarchy': gradient_maps[param] = get_hierarchy_gradients( data_tag=data_tag, fiducial_maps=fiducial_maps, fiducial_params=fiducial_params, grid_settings=grid_settings, store_dir=store_dir) else: gradient_maps[param] = get_gradients( data_tag=data_tag, param=param, template_maker=template_maker, fiducial_params=fiducial_params, grid_settings=grid_settings, store_dir=store_dir) logging.info("Building Fisher matrix for %s." % (data_tag)) # Build Fisher matrices for the given hierarchy fisher[data_tag] = build_fisher_matrix( gradient_maps=gradient_maps, fiducial_map=fiducial_maps['IMH'] if data_normal else fiducial_maps['NMH'], template_settings=fiducial_params) # If Fisher matrices exist for both channels, add the matrices to obtain # the combined one. if len(fisher[data_tag].keys()) > 1: fisher[data_tag]['comb'] = FisherMatrix( matrix=np.array([ f.matrix for f in fisher[data_tag].itervalues() ]).sum(axis=0), parameters=gradient_maps.keys(), #order is important here! best_fits=[ fiducial_params[par]['value'] for par in gradient_maps.keys() ], priors=[ Prior.from_param(fiducial_params[par]) for par in gradient_maps.keys() ], ) return fisher else: logging.info("Nothing to be done.") return {}
if scipy.__version__ < '0.12.0': logging.warn('Detected scipy version %s < 0.12.0'%scipy.__version__) if 'maxiter' in minimizer_settings: logging.warn('Optimizer settings for \"maxiter\" will be ignored') minimizer_settings.pop('maxiter') #Get the parameters params = template_settings['params'] # Make sure that atmospheric parameters are fixed: logging.warn("Ensuring that atmospheric parameters are fixed for this analysis") params = fix_atm_params(params) #print "params: ",params.items() with Timer() as t: template_maker = TemplateMaker(get_values(params),**template_settings['binning']) profile.info("==> elapsed time to initialize templates: %s sec"%t.secs) #data_types = [('data_NMH',True),('data_IMH',False)] data_types = [('data_NMH',True)] results = {} # Store for future checking: results['template_settings'] = template_settings results['minimizer_settings'] = minimizer_settings results['grid_settings'] = grid_settings try: for data_tag, data_normal in data_types: results[data_tag] = {}
parser.add_argument('-o', '--outfile', dest='outfile', metavar='FILE', type=str, action='store',default="template.json", help='file to store the output') args = parser.parse_args() set_verbosity(args.verbose) profile.info("start initializing") #Load all the settings model_settings = from_json(args.template_settings) #Select a hierarchy logging.info('Selected %s hierarchy'% ('normal' if args.normal else 'inverted')) params = select_hierarchy(model_settings['params'],normal_hierarchy=args.normal) #Intialize template maker template_maker = TemplateMaker(get_values(params),**model_settings['binning']) profile.info("stop initializing") #Now get the actual template profile.info("start template calculation") template = template_maker.get_template(get_values(params)) profile.info("stop template calculation") #Write out logging.info("Saving output to: %s",args.outfile) to_json(template, args.outfile)
args = parser.parse_args() set_verbosity(args.verbose) #Read in the settings template_settings = from_json(args.template_settings) minimizer_settings = from_json(args.minimizer_settings) grid_settings = from_json(args.grid_settings) if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True with Timer() as t: template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) profile.info("==> elapsed time to initialize templates: %s sec"%t.secs) #Get the parameters params = template_settings['params'] mctrue_types = [('true_NMH',True),('true_IMH',False)] results = {} # Store for future checking: results['template_settings'] = template_settings results['minimizer_settings'] = minimizer_settings results['grid_settings'] = grid_settings
template_settings = from_json(args.template_settings) czbin_edges = template_settings['binning']['czbins'] ebin_edges = template_settings['binning']['ebins'] channel = template_settings['params']['channel']['value'] x_steps = 0.0001 if args.sim == '4digit': MC_name = '1XXX' elif args.sim == '5digit': MC_name = '1XXXX' elif args.sim == 'dima': MC_name = 'Dima' else: MC_name = 'Other' params = get_values(select_hierarchy(template_settings['params'],normal_hierarchy=True)) run_list = params['run_list'] run_nominal = params['run_nominal'] run_dict = params['run_dict'] for norm in [False]: if norm: HierarchyPrefix = 'NH' DMRange = np.linspace(template_settings['params']['deltam31_nh']['range'][0],template_settings['params']['deltam31_nh']['range'][1],21) THRange = np.linspace(0.35,1.25,10) else: HierarchyPrefix = 'IH' DMRange = np.linspace(template_settings['params']['deltam31_ih']['range'][0],template_settings['params']['deltam31_ih']['range'][1],21) THRange = np.linspace(0.35,1.25,10)
#Read in the settings template_settings = from_json(args.template_settings) minimizer_settings = from_json(args.minimizer_settings) # Change this throughout code later? check_octant = not args.single_octant check_scipy_version(minimizer_settings) if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) # Assemble output dict output = {'template_settings' : template_settings, 'minimizer_settings' : minimizer_settings} for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]: tprofile.info("Assuming: %s"%data_tag) output[data_tag] = {} # Get Asimov data set for assuming true: data_tag, and store for # later comparison asimov_data = getAsimovData(
set_verbosity(args.verbose) # Read in the settings template_settings = from_json(args.template_settings) minimizer_settings = from_json(args.minimizer_settings) # Change this throughout code later? check_octant = not args.single_octant check_scipy_version(minimizer_settings) if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) # Assemble output dict output = {'template_settings' : template_settings, 'minimizer_settings' : minimizer_settings} asimov_data = {} asimov_data_null = {} alt_mh_settings = {} for data_tag, data_normal in [('true_NMH',True),('true_IMH',False)]: tprofile.info("Assuming: %s"%data_tag) output[data_tag] = {}
help="Plot all stages 1-5 of templates and Asymmetry") parser.add_argument('--title',metavar="str",default='', help="Title of the geometry or test in plots") parser.add_argument('--save',action='store_true',default=False, help="Save plots in cwd") parser.add_argument('-o','--outdir',metavar='DIR',default="", help="Directory to save the output figures.") parser.add_argument('-v', '--verbose', action='count', default=0, help='set verbosity level') args = parser.parse_args() set_verbosity(args.verbose) template_settings = from_json(args.template_settings) with Timer() as t: template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) profile.info("==> elapsed time to initialize templates: %s sec"%t.secs) # Make nmh template: nmh_params = select_hierarchy(template_settings['params'], normal_hierarchy=True) imh_params = select_hierarchy(template_settings['params'], normal_hierarchy=False) with Timer(verbose=False) as t: nmh = template_maker.get_template(get_values(nmh_params), return_stages=args.all) profile.info("==> elapsed time to get NMH template: %s sec"%t.secs) with Timer(verbose=False) as t: imh = template_maker.get_template(get_values(imh_params), return_stages=args.all) profile.info("==> elapsed time to get IMH template: %s sec"%t.secs)
args = parser.parse_args() set_verbosity(args.verbose) #Read in the settings template_settings = from_json(args.template_settings) minimizer_settings = from_json(args.minimizer_settings) grid_settings = from_json(args.grid_settings) if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True with Timer() as t: template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) profile.info("==> elapsed time to initialize templates: %s sec" % t.secs) #Get the parameters params = template_settings['params'] mctrue_types = [('true_NMH', True), ('true_IMH', False)] results = {} # Store for future checking: results['template_settings'] = template_settings results['minimizer_settings'] = minimizer_settings results['grid_settings'] = grid_settings for true_tag, true_normal in mctrue_types:
parser.add_argument('-o','--outfile',type=str,default='llh_data.json',metavar='JSONFILE', help="Output filename.") parser.add_argument('-th','--theta23',type=int, help='''Bin number to test in theta23''') parser.add_argument('-de','--deltam31',type=int, help='''Bin number to test in deltam31''') parser.add_argument('-v', '--verbose', action='count', default=0, help='set verbosity level') args = parser.parse_args() set_verbosity(args.verbose) fh = json.load(open(args.llh_file)) all_data = fh['trials'][0] template_settings = from_json(args.template_settings) template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) output = {'template_settings' : template_settings} output['seed'] = 0 trials = [] for dkey in all_data.keys(): if dkey in ['data_NMH', 'hypo_NMH']: output[dkey] = {} if dkey == 'data_NMH': data_normal = True else:
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=True): """ Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using the limited memory BFGS algorithm subject to bounds (l_bfgs_b). returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params varied by optimizer, and they hold a list of all the values tested in optimizer algorithm, unless save_steps is False, in which case they are one element in length-the best fit params and best fit llh. """ # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy)) free_params = get_free_params(select_hierarchy(params, normal_hierarchy)) init_vals = get_param_values(free_params) scales = get_param_scales(free_params) bounds = get_param_bounds(free_params) priors = get_param_priors(free_params) names = sorted(free_params.keys()) # Scale init-vals and bounds to work with bfgs opt: init_vals = np.array(init_vals) * np.array(scales) bounds = [bounds[i] * scales[i] for i in range(len(bounds))] opt_steps_dict = {key: [] for key in names} opt_steps_dict["llh"] = [] const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors) physics.info("%d parameters to be optimized" % len(free_params)) for name, init, (down, up), (prior, best) in zip(names, init_vals, bounds, priors): physics.info( ("%20s : init = %6.4f, bounds = [%6.4f,%6.4f], " "best = %6.4f, prior = " + ("%6.4f" if prior else "%s")) % (name, init, up, down, best, prior) ) physics.debug("Optimizer settings:") for key, item in bfgs_settings.items(): physics.debug(" %s -> `%s` = %.2e" % (item["desc"], key, item["value"])) best_fit_vals, llh, dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings) ) best_fit_params = {name: value for name, value in zip(names, best_fit_vals)} # Report best fit physics.info("Found best LLH = %.2f in %d calls at:" % (llh, dict_flags["funcalls"])) for name, val in best_fit_params.items(): physics.info(" %20s = %6.4f" % (name, val)) # Report any warnings if there are lvl = logging.WARN if (dict_flags["warnflag"] != 0) else logging.DEBUG for name, val in dict_flags.items(): physics.log(lvl, " %s : %s" % (name, val)) if not save_steps: # Do not store the extra history of opt steps: for key in opt_steps_dict.keys(): opt_steps_dict[key] = [opt_steps_dict[key][-1]] return opt_steps_dict
default="template.json", help='file to store the output') args = parser.parse_args() set_verbosity(args.verbose) with Timer() as t: #Load all the settings model_settings = from_json(args.template_settings) #Select a hierarchy logging.info('Selected %s hierarchy' % ('normal' if args.normal else 'inverted')) params = select_hierarchy(model_settings['params'], normal_hierarchy=args.normal) #Intialize template maker template_maker = TemplateMaker(get_values(params), **model_settings['binning']) tprofile.info(" ==> elapsed time to initialize templates: %s sec" % t.secs) #Now get the actual template with Timer(verbose=False) as t: template_maps = template_maker.get_template( get_values(params), return_stages=args.save_all) tprofile.info("==> elapsed time to get template: %s sec" % t.secs) logging.info("Saving file to %s" % args.outfile) to_json(template_maps, args.outfile)
# make sure that both pseudo data and template are using the same # channel. Raise Exception and quit otherwise channel = template_settings['params']['channel']['value'] if channel != pseudo_data_settings['params']['channel']['value']: error_msg = "Both template and pseudo data must have same channel!\n" error_msg += " pseudo_data_settings chan: '%s', template chan: '%s' " % ( pseudo_data_settings['params']['channel']['value'], channel) raise ValueError(error_msg) if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) if args.pseudo_data_settings: pseudo_data_template_maker = TemplateMaker( get_values(pseudo_data_settings['params']), **pseudo_data_settings['binning']) else: pseudo_data_template_maker = template_maker # Put in try/except block? #store results from all the trials trials = [] try: for itrial in xrange(1, args.ntrials + 1):
#Workaround for old scipy versions import scipy if scipy.__version__ < '0.12.0': logging.warn('Detected scipy version %s < 0.12.0'%scipy.__version__) if 'maxiter' in minimizer_settings: logging.warn('Optimizer settings for \"maxiter\" will be ignored') minimizer_settings.pop('maxiter') #Get the parameters params = template_settings['params'] #store results from all the trials trials = [] template_maker = TemplateMaker(get_values(params),**template_settings['binning']) for itrial in xrange(1,args.ntrials+1): profile.info("start trial %d"%itrial) logging.info(">"*10 + "Running trial: %05d"%itrial + "<"*10) # ////////////////////////////////////////////////////////////////////// # For each trial, generate two pseudo-data experiemnts (one for each # hierarchy), and for each find the best matching template in each of the # hierarchy hypothesis. # ////////////////////////////////////////////////////////////////////// results = {} for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]: results[data_tag] = {}
logging.warn('Detected scipy version %s < 0.12.0'%scipy.__version__) if 'maxiter' in minimizer_settings: logging.warn('Optimizer settings for \"maxiter\" will be ignored') minimizer_settings.pop('maxiter') # Make sure that both pseudo data and template are using the same # channel. Raise Exception and quit otherwise channel = template_settings['params']['channel']['value'] if channel != pseudo_data_settings['params']['channel']['value']: error_msg = "Both template and pseudo data must have same channel!\n" error_msg += " pseudo_data_settings chan: '%s', template chan: '%s' "%(pseudo_data_settings['params']['channel']['value'],channel) raise ValueError(error_msg) template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) if args.pseudo_data_settings: pseudo_data_template_maker = TemplateMaker(get_values(pseudo_data_settings['params']), **pseudo_data_settings['binning']) else: pseudo_data_template_maker = template_maker # ////////////////////////////////////////////////////////////////////// # Generate two pseudo-data experiments (one for each hierarchy), # and for each experiment, find the best matching template in each # of the hierarchy hypotheses. # ////////////////////////////////////////////////////////////////////// results = {} for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]:
if args.gpu_id is not None: template_settings['params']['gpu_id'] = {} template_settings['params']['gpu_id']['value'] = args.gpu_id template_settings['params']['gpu_id']['fixed'] = True #Get the parameters params = template_settings['params'] # Make sure that atmospheric parameters are fixed: logging.warn( "Ensuring that atmospheric parameters are fixed for this analysis") params = fix_atm_params(params) with Timer() as t: template_maker = TemplateMaker(get_values(params), **template_settings['binning']) tprofile.info("==> elapsed time to initialize templates: %s sec" % t.secs) results = {} # Store for future checking: results['template_settings'] = template_settings results['minimizer_settings'] = minimizer_settings results['grid_settings'] = grid_settings # Set up data/hypo nmh or imh if args.data_nmh: data_tag = 'data_NMH' data_normal = True else: data_tag = 'data_IMH'
'--outdir', metavar='DIR', default="", help="Directory to save the output figures.") parser.add_argument('-v', '--verbose', action='count', default=0, help='set verbosity level') args = parser.parse_args() set_verbosity(args.verbose) template_settings = from_json(args.template_settings) with Timer() as t: template_maker = TemplateMaker(get_values(template_settings['params']), **template_settings['binning']) tprofile.info("==> elapsed time to initialize templates: %s sec" % t.secs) # Make nmh template: nmh_params = select_hierarchy(template_settings['params'], normal_hierarchy=True) imh_params = select_hierarchy(template_settings['params'], normal_hierarchy=False) with Timer(verbose=False) as t: nmh = template_maker.get_template(get_values(nmh_params), return_stages=args.all) tprofile.info("==> elapsed time to get NMH template: %s sec" % t.secs) with Timer(verbose=False) as t: imh = template_maker.get_template(get_values(imh_params), return_stages=args.all)
def llh_bfgs(opt_vals, *args): """ Function that the bfgs algorithm tries to minimize. Essentially, it is a wrapper function around get_template() and get_binwise_llh(). This fuction is set up this way, because the fmin_l_bfgs_b algorithm must take a function with two inputs: params & *args, where 'params' are the actual VALUES to be varied, and must correspond to the limits in 'bounds', and 'args' are arguments which are not varied and optimized, but needed by the get_template() function here. Thus, we pass the arguments to this function as follows: --opt_vals: [param1,param2,...,paramN] - systematics varied in the optimization. --args: [names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors] where names: are the dict keys corresponding to param1, param2,... scales: the scales to be applied before passing to get_template [IMPORTANT! In the optimizer, all parameters must be ~ the same order. Here, we keep them between 0.1,1 so the "epsilon" step size will vary the parameters in roughly the same precision.] fmap: pseudo data flattened map fixed_params: dictionary of other paramters needed by the get_template() function template_maker: template maker object opt_steps_dict: dictionary recording information regarding the steps taken for each trial of the optimization process. priors: gaussian priors corresponding to opt_vals list. Format: [(prior1,best1),(prior2,best2),...,(priorN,bestN)] """ names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors = args # free parameters being "optimized" by minimizer re-scaled to their true values. unscaled_opt_vals = [ opt_vals[i] / scales[i] for i in xrange(len(opt_vals)) ] unscaled_free_params = { names[i]: val for i, val in enumerate(unscaled_opt_vals) } template_params = dict(unscaled_free_params.items() + get_values(fixed_params).items()) # Now get true template, and compute LLH with Timer() as t: if template_params['theta23'] == 0.0: logging.info( "Zero theta23, so generating no oscillations template...") true_template = template_maker.get_template_no_osc(template_params) else: true_template = template_maker.get_template(template_params) profile.info("==> elapsed time for template maker: %s sec" % t.secs) true_fmap = flatten_map(true_template, chan=template_params['channel']) # NOTE: The minus sign is present on both of these next two lines # to reflect the fact that the optimizer finds a minimum rather # than maximum. llh = -get_binwise_llh(fmap, true_fmap) llh -= sum([ get_prior_llh(opt_val, sigma, value) for (opt_val, (sigma, value)) in zip(unscaled_opt_vals, priors) ]) # Save all optimizer-tested values to opt_steps_dict, to see # optimizer history later for key in names: opt_steps_dict[key].append(template_params[key]) opt_steps_dict['llh'].append(llh) physics.debug("LLH is %.2f at: " % llh) for name, val in zip(names, opt_vals): physics.debug(" %20s = %6.4f" % (name, val)) return llh
parser.add_argument("-v", "--verbose", action="count", default=None, help="set verbosity level") args = parser.parse_args() set_verbosity(args.verbose) # Read in the settings template_settings = from_json(args.template_settings) grid_settings = from_json(args.grid_settings) # Get the parameters params = template_settings["params"] # store results from all the trials trials = [] template_maker = TemplateMaker(get_values(params), **template_settings["binning"]) for itrial in xrange(1, args.ntrials + 1): profile.info("start trial %d" % itrial) logging.info(">" * 10 + "Running trial: %05d" % itrial + "<" * 10) # ////////////////////////////////////////////////////////////////////// # For each trial, generate two pseudo-data experiemnts (one for each # hierarchy), and for each find the best matching template in each of the # hierarchy hypothesis. # ////////////////////////////////////////////////////////////////////// results = {} for data_tag, data_normal in [("data_NMH", True), ("data_IMH", False)]: results[data_tag] = {} # 1) get a pseudo data fmap from fiducial model (best fit vals of params).
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=None, check_octant=False): """ Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using the limited memory BFGS algorithm subject to bounds (l_bfgs_b). returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params varied by optimizer, and they hold a list of all the values tested in optimizer algorithm, unless save_steps is False, in which case they are one element in length-the best fit params and best fit llh. """ # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy)) free_params = get_free_params(select_hierarchy(params, normal_hierarchy)) if len(free_params) == 0: logging.warn("NO FREE PARAMS, returning LLH") true_template = template_maker.get_template(get_values(fixed_params)) channel = params['channel']['value'] true_fmap = flatten_map(true_template, chan=channel) return {'llh': [-get_binwise_llh(fmap, true_fmap)]} init_vals = get_param_values(free_params) scales = get_param_scales(free_params) bounds = get_param_bounds(free_params) priors = get_param_priors(free_params) names = sorted(free_params.keys()) # Scale init-vals and bounds to work with bfgs opt: init_vals = np.array(init_vals) * np.array(scales) bounds = [bounds[i] * scales[i] for i in range(len(bounds))] opt_steps_dict = {key: [] for key in names} opt_steps_dict['llh'] = [] const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) best_fit_vals, llh, dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # If needed, run optimizer again, checking for second octant solution: if check_octant and ('theta23' in free_params.keys()): physics.info("Checking alternative octant solution") old_th23_val = free_params['theta23']['value'] delta = np.pi - old_th23_val free_params['theta23']['value'] = np.pi + delta init_vals = get_param_values(free_params) const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) alt_fit_vals, alt_llh, alt_dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # Alternative octant solution is optimal: if alt_llh < llh: best_fit_vals = alt_fit_vals llh = alt_llh dict_flags = alt_dict_flags best_fit_params = { name: value for name, value in zip(names, best_fit_vals) } #Report best fit physics.info('Found best LLH = %.2f in %d calls at:' % (llh, dict_flags['funcalls'])) for name, val in best_fit_params.items(): physics.info(' %20s = %6.4f' % (name, val)) #Report any warnings if there are lvl = logging.WARN if (dict_flags['warnflag'] != 0) else logging.DEBUG for name, val in dict_flags.items(): physics.log(lvl, " %s : %s" % (name, val)) if not save_steps: # Do not store the extra history of opt steps: for key in opt_steps_dict.keys(): opt_steps_dict[key] = [opt_steps_dict[key][-1]] return opt_steps_dict
def llh_bfgs(opt_vals,*args): """ Function that the bfgs algorithm tries to minimize. Essentially, it is a wrapper function around get_template() and get_binwise_llh(). This fuction is set up this way, because the fmin_l_bfgs_b algorithm must take a function with two inputs: params & *args, where 'params' are the actual VALUES to be varied, and must correspond to the limits in 'bounds', and 'args' are arguments which are not varied and optimized, but needed by the get_template() function here. Thus, we pass the arguments to this function as follows: --opt_vals: [param1,param2,...,paramN] - systematics varied in the optimization. --args: [names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors] where names: are the dict keys corresponding to param1, param2,... scales: the scales to be applied before passing to get_template [IMPORTANT! In the optimizer, all parameters must be ~ the same order. Here, we keep them between 0.1,1 so the "epsilon" step size will vary the parameters in roughly the same precision.] fmap: pseudo data flattened map fixed_params: dictionary of other paramters needed by the get_template() function template_maker: template maker object opt_steps_dict: dictionary recording information regarding the steps taken for each trial of the optimization process. priors: gaussian priors corresponding to opt_vals list. Format: [(prior1,best1),(prior2,best2),...,(priorN,bestN)] """ names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors = args # free parameters being "optimized" by minimizer re-scaled to their true values. unscaled_opt_vals = [opt_vals[i]/scales[i] for i in xrange(len(opt_vals))] unscaled_free_params = { names[i]: val for i,val in enumerate(unscaled_opt_vals) } template_params = dict(unscaled_free_params.items() + get_values(fixed_params).items()) # Now get true template, and compute LLH with Timer() as t: if template_params['theta23'] == 0.0: logging.info("Zero theta23, so generating no oscillations template...") true_template = template_maker.get_template_no_osc(template_params) else: true_template = template_maker.get_template(template_params) profile.info("==> elapsed time for template maker: %s sec"%t.secs) true_fmap = flatten_map(true_template,chan=template_params['channel']) # NOTE: The minus sign is present on both of these next two lines # to reflect the fact that the optimizer finds a minimum rather # than maximum. llh = -get_binwise_llh(fmap,true_fmap) llh -= sum([ get_prior_llh(opt_val,sigma,value) for (opt_val,(sigma,value)) in zip(unscaled_opt_vals,priors)]) # Save all optimizer-tested values to opt_steps_dict, to see # optimizer history later for key in names: opt_steps_dict[key].append(template_params[key]) opt_steps_dict['llh'].append(llh) physics.debug("LLH is %.2f at: "%llh) for name, val in zip(names, opt_vals): physics.debug(" %20s = %6.4f" %(name,val)) return llh
def bfgs_metric(opt_vals, names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors, metric_name='llh'): """ Function that the bfgs algorithm tries to minimize: wraps get_template() and get_binwise_llh() (or get_binwise_chisquare()), and returns the negative log likelihood (the chisquare). This function is set up this way because the fmin_l_bfgs_b algorithm must take a function with two inputs: params & *args, where 'params' are the actual VALUES to be varied, and must correspond to the limits in 'bounds', and 'args' are arguments which are not varied and optimized, but needed by the get_template() function here. Parameters ---------- opt_vals : sequence of scalars Systematics varied in the optimization. Format: [param1, param2, ... , paramN] names : sequence of str Dictionary keys corresponding to param1, param2, ... scales : sequence of float Scales to be applied before passing to get_template [IMPORTANT! In the optimizer, all parameters must be ~ the same order. Here, we keep them between 0.1,1 so the "epsilon" step size will vary the parameters with roughly the same precision.] fmap : sequence of float Pseudo data flattened map fixed_params : dict Other paramters needed by the get_template() function. template_maker : template maker object opt_steps_dict: dict Dictionary recording information regarding the steps taken for each trial of the optimization process. priors : sequence of pisa.utils.params.Prior objects Priors corresponding to opt_vals list. metric_name : string Returns chisquare instead of negative llh if metric_name is 'chisquare'. Note: this string has to be present as a key in opt_steps_dict Returns ------- metric_val : float either minimum negative llh or chisquare found by BFGS minimizer """ # free parameters being "optimized" by minimizer re-scaled to their true # values. unscaled_opt_vals = [opt_vals[i]/scales[i] for i in xrange(len(opt_vals))] unscaled_free_params = { names[i]: val for i,val in enumerate(unscaled_opt_vals) } template_params = dict(unscaled_free_params.items() + get_values(fixed_params).items()) # Now get true template, and compute metric with Timer() as t: if template_params['theta23'] == 0.0: logging.info("Zero theta23, so generating no oscillations template...") true_template = template_maker.get_template_no_osc(template_params) else: true_template = template_maker.get_template(template_params) tprofile.info("==> elapsed time for template maker: %s sec"%t.secs) true_fmap = flatten_map(template=true_template, channel=template_params['channel']) # NOTE: The minus sign is present on both of these next two lines # because the optimizer finds a minimum rather than maximum, so we # have to minimize the negative of the log likelhood. if metric_name=='chisquare': metric_val = get_binwise_chisquare(fmap, true_fmap) metric_val += sum([prior.chi2(opt_val) for (opt_val, prior) in zip(unscaled_opt_vals, priors)]) elif metric_name=='llh': metric_val = -get_binwise_llh(fmap, true_fmap) metric_val -= sum([prior.llh(opt_val) for (opt_val, prior) in zip(unscaled_opt_vals, priors)]) #prior_list = [prior.llh(opt_val) # for (opt_val, prior) in zip(unscaled_opt_vals, priors)] #print(" prior sum: ",sum(prior_list)) #neg_llh -= sum(prior_list) # Save all optimizer-tested values to opt_steps_dict, to see # optimizer history later for key in names: opt_steps_dict[key].append(template_params[key]) opt_steps_dict[metric_name].append(metric_val) physics.debug("%s is %.2f at: "%(metric_name, metric_val)) for name, val in zip(names, opt_vals): physics.debug(" %20s = %6.4f" %(name,val)) return metric_val
def get_fisher_matrices(template_settings, grid_settings, IMH=True, NMH=False, dump_all_stages=False, save_templates=False, outdir=None): ''' Main function that runs the Fisher analysis for the chosen hierarchy(ies) (inverted by default). Returns a dictionary of Fisher matrices, in the format: {'IMH': {'cscd': [...], 'trck': [...], 'comb': [...], }, 'NMH': {'cscd': [...], 'trck': [...], 'comb': [...], } } If save_templates=True and no hierarchy is given, only fiducial templates will be written out; if one is given, then the templates used to obtain the gradients will be written out in addition. ''' if outdir is None and (save_templates or dump_all_stages): logging.info("No output directory specified. Will save templates to current working directory.") outdir = os.getcwd() profile.info("start initializing") # Get the parameters params = template_settings['params'] bins = template_settings['binning'] # Artifically add the hierarchy parameter to the list of parameters # The method get_hierarchy_gradients below will know how to deal with it params['hierarchy_nh'] = { "value": 1., "range": [0.,1.], "fixed": False, "prior": None} params['hierarchy_ih'] = { "value": 0., "range": [0.,1.], "fixed": False, "prior": None} chosen_data = [] if IMH: chosen_data.append(('IMH',False)) logging.info("Fisher matrix will be built for IMH.") if NMH: chosen_data.append(('NMH',True)) logging.info("Fisher matrix will be built for NMH.") if chosen_data == []: # In this case, only the fiducial maps (for both hierarchies) will be written logging.info("No Fisher matrices will be built.") # There is no sense in performing any of the following steps if no Fisher matrices are to be built # and no templates are to be saved. if chosen_data!=[] or dump_all_stages or save_templates: # Initialise return dict to hold Fisher matrices fisher = { data_tag:{'cscd':[],'trck':[],'comb':[]} for data_tag, data_normal in chosen_data } # Get a template maker with the settings used to initialize template_maker = TemplateMaker(get_values(params),**bins) profile.info("stop initializing\n") # Generate fiducial templates for both hierarchies (needed for partial derivatives # w.r.t. hierarchy parameter) fiducial_maps = {} for hierarchy in ['NMH','IMH']: logging.info("Generating fiducial templates for %s."%hierarchy) # Get the fiducial parameter values corresponding to this hierarchy fiducial_params = select_hierarchy(params,normal_hierarchy=(hierarchy=='NMH')) # Generate fiducial maps, either all of them or only the ultimate one profile.info("start template calculation") with Timer() as t: fid_maps = template_maker.get_template(get_values(fiducial_params), return_stages=dump_all_stages) profile.info("==> elapsed time for template: %s sec"%t.secs) fiducial_maps[hierarchy] = fid_maps[4] if dump_all_stages else fid_maps # save fiducial map(s) # all stages if dump_all_stages: stage_names = ("0_unoscillated_flux","1_oscillated_flux","2_oscillated_counts","3_reco","4_pid") stage_maps = {} for stage in xrange(0,len(fid_maps)): stage_maps[stage_names[stage]] = fid_maps[stage] logging.info("Writing fiducial maps (all stages) for %s to %s."%(hierarchy,outdir)) to_json(stage_maps,os.path.join(outdir,"fid_map_"+hierarchy+".json")) # only the final stage elif save_templates: logging.info("Writing fiducial map (final stage) for %s to %s."%(hierarchy,outdir)) to_json(fiducial_maps[hierarchy],os.path.join(outdir,"fid_map_"+hierarchy+".json")) # Get_gradients and get_hierarchy_gradients will both (temporarily) # store the templates used to generate the gradient maps store_dir = outdir if save_templates else tempfile.gettempdir() # Calculate Fisher matrices for the user-defined cases (NHM true and/or IMH true) for data_tag, data_normal in chosen_data: logging.info("Running Fisher analysis for %s."%(data_tag)) # The fiducial params are selected from the hierarchy case that does NOT match # the data, as we are varying from this model to find the 'best fit' fiducial_params = select_hierarchy(params,not data_normal) # Get the free parameters (i.e. those for which the gradients should be calculated) free_params = select_hierarchy(get_free_params(params),not data_normal) gradient_maps = {} for param in free_params.keys(): # Special treatment for the hierarchy parameter if param=='hierarchy': gradient_maps[param] = get_hierarchy_gradients(data_tag, fiducial_maps, fiducial_params, grid_settings, store_dir, ) else: gradient_maps[param] = get_gradients(data_tag, param, template_maker, fiducial_params, grid_settings, store_dir ) logging.info("Building Fisher matrix for %s."%(data_tag)) # Build Fisher matrices for the given hierarchy fisher[data_tag] = build_fisher_matrix(gradient_maps,fiducial_maps['IMH'] if data_normal else fiducial_maps['NMH'],fiducial_params) # If Fisher matrices exist for both channels, add the matrices to obtain the combined one. if len(fisher[data_tag].keys()) > 1: fisher[data_tag]['comb'] = FisherMatrix(matrix=np.array([f.matrix for f in fisher[data_tag].itervalues()]).sum(axis=0), parameters=gradient_maps.keys(), #order is important here! best_fits=[fiducial_params[par]['value'] for par in gradient_maps.keys()], priors=[fiducial_params[par]['prior'] for par in gradient_maps.keys()], ) return fisher else: logging.info("Nothing to be done.") return {}
help='set verbosity level') args = parser.parse_args() set_verbosity(args.verbose) #Read in the settings template_settings = from_json(args.template_settings) grid_settings = from_json(args.grid_settings) #Get the parameters params = template_settings['params'] #store results from all the trials trials = [] template_maker = TemplateMaker(get_values(params), **template_settings['binning']) for itrial in xrange(1, args.ntrials + 1): profile.info("start trial %d" % itrial) logging.info(">" * 10 + "Running trial: %05d" % itrial + "<" * 10) # ////////////////////////////////////////////////////////////////////// # For each trial, generate two pseudo-data experiemnts (one for each # hierarchy), and for each find the best matching template in each of the # hierarchy hypothesis. # ////////////////////////////////////////////////////////////////////// results = {} for data_tag, data_normal in [('data_NMH', True), ('data_IMH', False)]: results[data_tag] = {}