def get_llh_hypothesis( data_tag, asimov_data, ntrials, template_maker, template_params, minimizer_settings, save_steps, check_octant): """ Runs the llh fitter ntrials number of times, pulling pseudo data sets from asimov_data. \Params: * data_tag - hierarchy type running for assumed true. * asimov_data - asimov (unfluctuated) data from which to generate poisson fluctuated pseudo data * ntrials - number of trials to run for each hierarchy hypothesis * template_maker - instance of TemplateMaker class, from which to fit pseudo data to * template_params - dictionary of parameters at which to test the pseudo data and find the best match llh * minimizer_settings - settings for bfgs minimizer in llh fit * save_steps - flag to save the optimizer steps * check_octant - boolean to check both octants of theta23 \returns - trials list that holds the dictionaries of llh results. """ trials = [] for itrial in xrange(1,ntrials+1): results = {} # one trial of results tprofile.info("start trial %d"%itrial) logging.info(">"*10 + "Running trial: %05d"%itrial + "<"*10) results['seed'] = get_seed() logging.info(" RNG seed: %ld"%results['seed']) # Get random map generated from asimov data (or from data_tag). fmap = get_random_map(asimov_data, seed=results['seed']) for hypo_tag, hypo_normal in [('hypo_NMH',True),('hypo_IMH',False)]: physics.info( "Finding best fit for %s under %s assumption"%(data_tag,hypo_tag)) with Timer() as t: llh_data = find_max_llh_bfgs( fmap, template_maker, template_params, minimizer_settings, save_steps, normal_hierarchy=hypo_normal, check_octant=check_octant) tprofile.info("==> elapsed time for optimizer: %s sec"%t.secs) # Store the LLH data results[hypo_tag] = llh_data trials += [results] tprofile.info("stop trial %d"%itrial) return trials
def display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings): """ Displays parameters and optimization settings that minimizer will run. """ physics.info('%d parameters to be optimized'%len(free_params)) for name, init_val, bound, prior in zip(names, init_vals, bounds, priors): physics.info(('%20s : init = %6.4f, bounds = [%6.4f,%6.4f], prior = %s') %(name, init_val, bound[0], bound[1], prior)) physics.debug("Optimizer settings:") for key,item in bfgs_settings.items(): physics.debug(" %s -> `%s` = %.2e"%(item['desc'],key,item['value'])) return
def display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings): """ Displays parameters and optimization settings that minimizer will run. """ physics.info('%d parameters to be optimized'%len(free_params)) for name,init,(down,up),(prior, best) in zip(names, init_vals, bounds, priors): physics.info(('%20s : init = %6.4f, bounds = [%6.4f,%6.4f], ' 'best = %6.4f, prior = '+ ('%6.4f' if prior else "%s"))% (name, init, up, down, best, prior)) physics.debug("Optimizer settings:") for key,item in bfgs_settings.items(): physics.debug(" %s -> `%s` = %.2e"%(item['desc'],key,item['value'])) return
def display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings): """ Displays parameters and optimization settings that minimizer will run. """ physics.info('%d parameters to be optimized' % len(free_params)) for name, init_val, bound, prior in zip(names, init_vals, bounds, priors): physics.info( ('%20s : init = %6.4f, bounds = [%6.4f,%6.4f], prior = %s') % (name, init_val, bound[0], bound[1], prior)) physics.debug("Optimizer settings:") for key, item in bfgs_settings.items(): physics.debug(" %s -> `%s` = %.2e" % (item['desc'], key, item['value'])) return
def display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings): """ Displays parameters and optimization settings that minimizer will run. """ physics.info('%d parameters to be optimized' % len(free_params)) for name, init, (down, up), (prior, best) in zip(names, init_vals, bounds, priors): physics.info( ('%20s : init = %6.4f, bounds = [%6.4f,%6.4f], ' 'best = %6.4f, prior = ' + ('%6.4f' if prior else "%s")) % (name, init, up, down, best, prior)) physics.debug("Optimizer settings:") for key, item in bfgs_settings.items(): physics.debug(" %s -> `%s` = %.2e" % (item['desc'], key, item['value'])) return
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=None, check_octant=False): """ Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using the limited memory BFGS algorithm subject to bounds (l_bfgs_b). returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params varied by optimizer, and they hold a list of all the values tested in optimizer algorithm, unless save_steps is False, in which case they are one element in length-the best fit params and best fit llh. """ # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params,normal_hierarchy)) free_params = get_free_params(select_hierarchy(params,normal_hierarchy)) if len(free_params) == 0: logging.warn("NO FREE PARAMS, returning LLH") true_template = template_maker.get_template(get_values(fixed_params)) channel = params['channel']['value'] true_fmap = flatten_map(true_template,chan=channel) return {'llh': [-get_binwise_llh(fmap,true_fmap)]} init_vals = get_param_values(free_params) scales = get_param_scales(free_params) bounds = get_param_bounds(free_params) priors = get_param_priors(free_params) names = sorted(free_params.keys()) # Scale init-vals and bounds to work with bfgs opt: init_vals = np.array(init_vals)*np.array(scales) bounds = [bounds[i]*scales[i] for i in range(len(bounds))] opt_steps_dict = {key:[] for key in names} opt_steps_dict['llh'] = [] const_args = (names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) best_fit_vals,llh,dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # If needed, run optimizer again, checking for second octant solution: if check_octant and ('theta23' in free_params.keys()): physics.info("Checking alternative octant solution") old_th23_val = free_params['theta23']['value'] delta = np.pi - old_th23_val free_params['theta23']['value'] = np.pi + delta init_vals = get_param_values(free_params) const_args = (names,scales,fmap,fixed_params,template_maker,opt_steps_dict,priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) alt_fit_vals,alt_llh,alt_dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # Alternative octant solution is optimal: if alt_llh < llh: best_fit_vals = alt_fit_vals llh = alt_llh dict_flags = alt_dict_flags best_fit_params = { name: value for name, value in zip(names, best_fit_vals) } #Report best fit physics.info('Found best LLH = %.2f in %d calls at:' %(llh,dict_flags['funcalls'])) for name, val in best_fit_params.items(): physics.info(' %20s = %6.4f'%(name,val)) #Report any warnings if there are lvl = logging.WARN if (dict_flags['warnflag'] != 0) else logging.DEBUG for name, val in dict_flags.items(): physics.log(lvl," %s : %s"%(name,val)) if not save_steps: # Do not store the extra history of opt steps: for key in opt_steps_dict.keys(): opt_steps_dict[key] = [opt_steps_dict[key][-1]] return opt_steps_dict
for data_tag, data_normal in [('data_NMH',True),('data_IMH',False)]: results[data_tag] = {} # 0) get a random seed and store with the data results[data_tag]['seed'] = get_seed() # 1) get a pseudo data fmap from fiducial model (best fit vals of params). fmap = get_pseudo_data_fmap(template_maker, get_values(select_hierarchy(params, normal_hierarchy=data_normal)), seed=results[data_tag]['seed']) # 2) find max llh (and best fit free params) from matching pseudo data # to templates. for hypo_tag, hypo_normal in [('hypo_NMH',True),('hypo_IMH',False)]: physics.info("Finding best fit for %s under %s assumption"%(data_tag,hypo_tag)) profile.info("start optimizer") llh_data = find_max_llh_bfgs(fmap,template_maker,params, minimizer_settings,args.save_steps,normal_hierarchy=hypo_normal) profile.info("stop optimizer") #Store the LLH data results[data_tag][hypo_tag] = llh_data #Store this trial trials += [results] profile.info("stop trial %d"%itrial) #Assemble output dict output = {'trials' : trials,
logging.info(" RNG seed: %ld" % results[data_tag]['seed']) # 1) get a pseudo data fmap from fiducial model (best fit vals of params). fmap = get_pseudo_data_fmap(pseudo_data_template_maker, get_values( select_hierarchy( pseudo_data_settings['params'], normal_hierarchy=data_normal)), seed=results[data_tag]['seed'], chan=channel) # 2) find max llh (and best fit free params) from matching pseudo data # to templates. for hypo_tag, hypo_normal in [('hypo_NMH', True), ('hypo_IMH', False)]: physics.info("Finding best fit for %s under %s assumption" % (data_tag, hypo_tag)) with Timer() as t: llh_data = find_max_llh_bfgs(fmap, template_maker, template_settings['params'], minimizer_settings, args.save_steps, normal_hierarchy=hypo_normal) profile.info("==> elapsed time for optimizer: %s sec" % t.secs) # Store the LLH data results[data_tag][hypo_tag] = llh_data # Store this trial trials += [results] profile.info("stop trial %d" % itrial)
def find_max_grid(fmap, template_maker, params, grid_settings, save_steps=True, normal_hierarchy=True): ''' Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using a brute force grid scan over the whole parameter space. returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params that are varied in the scan. If save_steps is False, all lists only contain the best-fit parameters and llh values. ''' #print "NOW INSIDE find_max_grid:" #print "After fixing to their true values, params dict is now: " #for key in params.keys(): # try: print " >>param: %s value: %s"%(key,str(params[key]['best'])) # except: continue # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy)) free_params = get_free_params(select_hierarchy(params, normal_hierarchy)) # Obtain just the priors priors = get_param_priors(free_params) # Calculate steps [(prior,value),...] for all free parameters calc_steps(free_params, grid_settings['steps']) # Build a list from all parameters that holds a list of (name, step) tuples steplist = [[(name, step) for step in param['steps']] for name, param in sorted(free_params.items())] # Prepare to store all the steps steps = {key: [] for key in free_params.keys()} steps['llh'] = [] # Iterate over the cartesian product for pos in product(*steplist): # Get a dict with all parameter values at this position # including the fixed parameters template_params = dict(list(pos) + get_values(fixed_params).items()) #print " >> NOW IN LOOP: " #for key in template_params.keys(): # try: print " >>param: %s value: %s"%(key,str(template_params[key]['value'])) # except: continue # Now get true template tprofile.info('start template calculation') true_template = template_maker.get_template(template_params) tprofile.info('stop template calculation') true_fmap = flatten_map(true_template) # and calculate the likelihood llh = -get_binwise_llh(fmap, true_fmap) # get sorted vals to match with priors vals = [v for k, v in sorted(pos)] llh -= sum([prior.llh(val) for val, prior in zip(vals, priors)]) # Save all values to steps and report steps['llh'].append(llh) physics.debug("LLH is %.2f at: " % llh) for key, val in pos: steps[key].append(val) physics.debug(" %20s = %6.4f" % (key, val)) # Find best fit value maxllh = min(steps['llh']) maxpos = steps['llh'].index(maxllh) # Report best fit physics.info('Found best LLH = %.2f in %d calls at:' % (maxllh, len(steps['llh']))) for name, vals in steps.items(): physics.info(' %20s = %6.4f' % (name, vals[maxpos])) # only save this maximum if asked for if not save_steps: steps[name] = vals[maxpos] return steps
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=True): """ Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using the limited memory BFGS algorithm subject to bounds (l_bfgs_b). returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params varied by optimizer, and they hold a list of all the values tested in optimizer algorithm, unless save_steps is False, in which case they are one element in length-the best fit params and best fit llh. """ # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy)) free_params = get_free_params(select_hierarchy(params, normal_hierarchy)) init_vals = get_param_values(free_params) scales = get_param_scales(free_params) bounds = get_param_bounds(free_params) priors = get_param_priors(free_params) names = sorted(free_params.keys()) # Scale init-vals and bounds to work with bfgs opt: init_vals = np.array(init_vals) * np.array(scales) bounds = [bounds[i] * scales[i] for i in range(len(bounds))] opt_steps_dict = {key: [] for key in names} opt_steps_dict["llh"] = [] const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors) physics.info("%d parameters to be optimized" % len(free_params)) for name, init, (down, up), (prior, best) in zip(names, init_vals, bounds, priors): physics.info( ("%20s : init = %6.4f, bounds = [%6.4f,%6.4f], " "best = %6.4f, prior = " + ("%6.4f" if prior else "%s")) % (name, init, up, down, best, prior) ) physics.debug("Optimizer settings:") for key, item in bfgs_settings.items(): physics.debug(" %s -> `%s` = %.2e" % (item["desc"], key, item["value"])) best_fit_vals, llh, dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings) ) best_fit_params = {name: value for name, value in zip(names, best_fit_vals)} # Report best fit physics.info("Found best LLH = %.2f in %d calls at:" % (llh, dict_flags["funcalls"])) for name, val in best_fit_params.items(): physics.info(" %20s = %6.4f" % (name, val)) # Report any warnings if there are lvl = logging.WARN if (dict_flags["warnflag"] != 0) else logging.DEBUG for name, val in dict_flags.items(): physics.log(lvl, " %s : %s" % (name, val)) if not save_steps: # Do not store the extra history of opt steps: for key in opt_steps_dict.keys(): opt_steps_dict[key] = [opt_steps_dict[key][-1]] return opt_steps_dict
def find_max_llh_bfgs(fmap, template_maker, params, bfgs_settings, save_steps=False, normal_hierarchy=None, check_octant=False): """ Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using the limited memory BFGS algorithm subject to bounds (l_bfgs_b). returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params varied by optimizer, and they hold a list of all the values tested in optimizer algorithm, unless save_steps is False, in which case they are one element in length-the best fit params and best fit llh. """ # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params, normal_hierarchy)) free_params = get_free_params(select_hierarchy(params, normal_hierarchy)) if len(free_params) == 0: logging.warn("NO FREE PARAMS, returning LLH") true_template = template_maker.get_template(get_values(fixed_params)) channel = params['channel']['value'] true_fmap = flatten_map(true_template, chan=channel) return {'llh': [-get_binwise_llh(fmap, true_fmap)]} init_vals = get_param_values(free_params) scales = get_param_scales(free_params) bounds = get_param_bounds(free_params) priors = get_param_priors(free_params) names = sorted(free_params.keys()) # Scale init-vals and bounds to work with bfgs opt: init_vals = np.array(init_vals) * np.array(scales) bounds = [bounds[i] * scales[i] for i in range(len(bounds))] opt_steps_dict = {key: [] for key in names} opt_steps_dict['llh'] = [] const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) best_fit_vals, llh, dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # If needed, run optimizer again, checking for second octant solution: if check_octant and ('theta23' in free_params.keys()): physics.info("Checking alternative octant solution") old_th23_val = free_params['theta23']['value'] delta = np.pi - old_th23_val free_params['theta23']['value'] = np.pi + delta init_vals = get_param_values(free_params) const_args = (names, scales, fmap, fixed_params, template_maker, opt_steps_dict, priors) display_optimizer_settings(free_params, names, init_vals, bounds, priors, bfgs_settings) alt_fit_vals, alt_llh, alt_dict_flags = opt.fmin_l_bfgs_b( llh_bfgs, init_vals, args=const_args, approx_grad=True, iprint=0, bounds=bounds, **get_values(bfgs_settings)) # Alternative octant solution is optimal: if alt_llh < llh: best_fit_vals = alt_fit_vals llh = alt_llh dict_flags = alt_dict_flags best_fit_params = { name: value for name, value in zip(names, best_fit_vals) } #Report best fit physics.info('Found best LLH = %.2f in %d calls at:' % (llh, dict_flags['funcalls'])) for name, val in best_fit_params.items(): physics.info(' %20s = %6.4f' % (name, val)) #Report any warnings if there are lvl = logging.WARN if (dict_flags['warnflag'] != 0) else logging.DEBUG for name, val in dict_flags.items(): physics.log(lvl, " %s : %s" % (name, val)) if not save_steps: # Do not store the extra history of opt steps: for key in opt_steps_dict.keys(): opt_steps_dict[key] = [opt_steps_dict[key][-1]] return opt_steps_dict
def find_max_grid(fmap,template_maker,params,grid_settings,save_steps=True, normal_hierarchy=True): ''' Finds the template (and free systematic params) that maximize likelihood that the data came from the chosen template of true params, using a brute force grid scan over the whole parameter space. returns a dictionary of llh data and best fit params, in the format: {'llh': [...], 'param1': [...], 'param2': [...], ...} where 'param1', 'param2', ... are the free params that are varied in the scan. If save_steps is False, all lists only contain the best-fit parameters and llh values. ''' #print "NOW INSIDE find_max_grid:" #print "After fixing to their true values, params dict is now: " #for key in params.keys(): # try: print " >>param: %s value: %s"%(key,str(params[key]['best'])) # except: continue # Get params dict which will be optimized (free_params) and which # won't be (fixed_params) but are still needed for get_template() fixed_params = get_fixed_params(select_hierarchy(params,normal_hierarchy)) free_params = get_free_params(select_hierarchy(params,normal_hierarchy)) #Obtain just the priors priors = get_param_priors(free_params) #Calculate steps for all free parameters calc_steps(free_params, grid_settings['steps']) #Build a list from all parameters that holds a list of (name, step) tuples steplist = [ [(name,step) for step in param['steps']] for name, param in sorted(free_params.items())] #Prepare to store all the steps steps = {key:[] for key in free_params.keys()} steps['llh'] = [] #Iterate over the cartesian product for pos in product(*steplist): #Get a dict with all parameter values at this position #including the fixed parameters template_params = dict(list(pos) + get_values(fixed_params).items()) #print " >> NOW IN LOOP: " #for key in template_params.keys(): # try: print " >>param: %s value: %s"%(key,str(template_params[key]['value'])) # except: continue # Now get true template profile.info('start template calculation') true_template = template_maker.get_template(template_params) profile.info('stop template calculation') true_fmap = flatten_map(true_template) #and calculate the likelihood llh = -get_binwise_llh(fmap,true_fmap) #get sorted vals to match with priors vals = [ v for k,v in sorted(pos) ] llh -= sum([ get_prior_llh(vals,sigma,value) for (vals,(sigma,value)) in zip(vals,priors)]) # Save all values to steps and report steps['llh'].append(llh) physics.debug("LLH is %.2f at: "%llh) for key, val in pos: steps[key].append(val) physics.debug(" %20s = %6.4f" %(key, val)) #Find best fit value maxllh = min(steps['llh']) maxpos = steps['llh'].index(maxllh) #Report best fit physics.info('Found best LLH = %.2f in %d calls at:' %(maxllh,len(steps['llh']))) for name, vals in steps.items(): physics.info(' %20s = %6.4f'%(name,vals[maxpos])) #only save this maximum if asked for if not save_steps: steps[name]=vals[maxpos] return steps