def batch_gbtxwigglez_data_run(gbt_map_key, wigglez_map_key, wigglez_mock_key, wigglez_selection_key, inifile=None, datapath_db=None, outdir="./plots", output_tag=None, beam_transfer=None, mode_transfer_1d=None, mode_transfer_2d=None, theory_curve=None): r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power""" if datapath_db is None: datapath_db = data_paths.DataPath() cache_path = datapath_db.fetch("quadratic_batch_data") map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment") mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization") funcname = "correlate.batch_quadratic.call_xspec_run" generate = False if output_tag else True if generate: print "REGENERATING the power spectrum result cache: " caller = batch_handler.MemoizeBatch(funcname, cache_path, generate=generate, verbose=True) if output_tag: file_tools.mkparents(outdir) for treatment in map_cases['treatment']: # TODO: make this more elegant # TODO: convert treatment into mode num transfer_2d = None if (mode_transfer_2d is not None) and (beam_transfer is None): transfer_2d = mode_transfer_2d[treatment][2] if (mode_transfer_2d is None) and (beam_transfer is not None): transfer_2d = beam_transfer if (mode_transfer_2d is not None) and (beam_transfer is not None): transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer pwr_1d = [] pwr_2d = [] pwr_1d_from_2d = [] for index in mock_cases['realization']: dbkeydict = {} dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment) dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index) dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \ (gbt_map_key, treatment) dbkeydict['noiseinv2_key'] = wigglez_selection_key files = data_paths.convert_dbkeydict_to_filedict( dbkeydict, datapath_db=datapath_db) pwrspec_out = caller.execute(files['map1_key'], files['map2_key'], files['noiseinv1_key'], files['noiseinv2_key'], inifile=inifile) if output_tag: pwr_1d_from_2d.append( pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d)) pwr_2d.append(pwrspec_out[0]) pwr_1d.append(pwrspec_out[1]) if output_tag: if mode_transfer_1d is not None: transfunc = mode_transfer_1d[treatment][1] else: transfunc = None mtag = output_tag + "_%s_mock" % treatment agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d, mtag, outdir=outdir, apply_1d_transfer=transfunc) mean1dmock = agg_pwrspec["mean_1d"] std1dmock = agg_pwrspec["std_1d"] covmock = agg_pwrspec["covmat_1d"] # now recover the xspec with the real data dbkeydict = {} dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment) dbkeydict['map2_key'] = wigglez_map_key dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment) dbkeydict['noiseinv2_key'] = wigglez_selection_key files = data_paths.convert_dbkeydict_to_filedict( dbkeydict, datapath_db=datapath_db) pwrspec_out_signal = caller.execute(files['map1_key'], files['map2_key'], files['noiseinv1_key'], files['noiseinv2_key'], inifile=inifile) if output_tag: pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0], transfer=transfer_2d) pwr_1d = pwrspec_out_signal[1]['binavg'] pwr_1d_from_2d = pwr_1d_from_2d['binavg'] if mode_transfer_1d is not None: pwr_1d /= mode_transfer_1d[treatment][1] pwr_1d_from_2d /= mode_transfer_1d[treatment][1] # assume that they all have the same binning bin_left = pwrspec_out_signal[1]['bin_left'] bin_center = pwrspec_out_signal[1]['bin_center'] bin_right = pwrspec_out_signal[1]['bin_right'] counts_histo = pwrspec_out_signal[1]['counts_histo'] filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment) outfile = open(filename, "w") for specdata in zip(bin_left, bin_center, bin_right, counts_histo, pwr_1d, pwr_1d_from_2d, mean1dmock, std1dmock): outfile.write(("%10.15g " * 8 + "\n") % specdata) outfile.close() # TODO: kludge to make a fast fit; remove theory_curve = np.genfromtxt( "plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat" ) theory_curve = theory_curve[:, 4] if theory_curve is not None: restrict = np.where( np.logical_and(bin_center > 0.09, bin_center < 1.1)) res_slice = slice(min(restrict[0]), max(restrict[0])) #restrict_alt = np.where(restrict)[0][np.newaxis, :] #restricted_cov = covmock[restrict_alt][0] from core import utils amplitude = utils.ampfit(pwr_1d_from_2d[res_slice], covmock[res_slice, res_slice], theory_curve[res_slice]) print "AMP:", mtag, treatment, amplitude if not output_tag: caller.multiprocess_stack() return None
def batch_gbtxwigglez_data_run(gbt_map_key, wigglez_map_key, wigglez_mock_key, wigglez_selection_key, inifile=None, datapath_db=None, outdir="./plots", output_tag=None, beam_transfer=None, mode_transfer_1d=None, mode_transfer_2d=None, theory_curve=None): r"""assemble the pairs of GBT and WiggleZ and calculate the cross-power""" if datapath_db is None: datapath_db = data_paths.DataPath() cache_path = datapath_db.fetch("quadratic_batch_data") map_cases = datapath_db.fileset_cases(gbt_map_key, "type;treatment") mock_cases = datapath_db.fileset_cases(wigglez_mock_key, "realization") funcname = "correlate.batch_quadratic.call_xspec_run" generate = False if output_tag else True if generate: print "REGENERATING the power spectrum result cache: " caller = batch_handler.MemoizeBatch(funcname, cache_path, generate=generate, verbose=True) if output_tag: file_tools.mkparents(outdir) for treatment in map_cases['treatment']: # TODO: make this more elegant # TODO: convert treatment into mode num transfer_2d = None if (mode_transfer_2d is not None) and (beam_transfer is None): transfer_2d = mode_transfer_2d[treatment][2] if (mode_transfer_2d is None) and (beam_transfer is not None): transfer_2d = beam_transfer if (mode_transfer_2d is not None) and (beam_transfer is not None): transfer_2d = mode_transfer_2d[treatment][2] * beam_transfer pwr_1d = [] pwr_2d = [] pwr_1d_from_2d = [] for index in mock_cases['realization']: dbkeydict = {} dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment) dbkeydict['map2_key'] = "%s:%s" % (wigglez_mock_key, index) dbkeydict['noiseinv1_key'] = "%s:weight;%s" % \ (gbt_map_key, treatment) dbkeydict['noiseinv2_key'] = wigglez_selection_key files = data_paths.convert_dbkeydict_to_filedict(dbkeydict, datapath_db=datapath_db) pwrspec_out = caller.execute(files['map1_key'], files['map2_key'], files['noiseinv1_key'], files['noiseinv2_key'], inifile=inifile) if output_tag: pwr_1d_from_2d.append(pe.convert_2d_to_1d(pwrspec_out[0], transfer=transfer_2d)) pwr_2d.append(pwrspec_out[0]) pwr_1d.append(pwrspec_out[1]) if output_tag: if mode_transfer_1d is not None: transfunc = mode_transfer_1d[treatment][1] else: transfunc = None mtag = output_tag + "_%s_mock" % treatment agg_pwrspec = pe.summarize_agg_pwrspec(pwr_1d, pwr_1d_from_2d, pwr_2d, mtag, outdir=outdir, apply_1d_transfer=transfunc) mean1dmock = agg_pwrspec["mean_1d"] std1dmock = agg_pwrspec["std_1d"] covmock = agg_pwrspec["covmat_1d"] # now recover the xspec with the real data dbkeydict = {} dbkeydict['map1_key'] = "%s:map;%s" % (gbt_map_key, treatment) dbkeydict['map2_key'] = wigglez_map_key dbkeydict['noiseinv1_key'] = "%s:weight;%s" % (gbt_map_key, treatment) dbkeydict['noiseinv2_key'] = wigglez_selection_key files = data_paths.convert_dbkeydict_to_filedict(dbkeydict, datapath_db=datapath_db) pwrspec_out_signal = caller.execute(files['map1_key'], files['map2_key'], files['noiseinv1_key'], files['noiseinv2_key'], inifile=inifile) if output_tag: pwr_1d_from_2d = pe.convert_2d_to_1d(pwrspec_out_signal[0], transfer=transfer_2d) pwr_1d = pwrspec_out_signal[1]['binavg'] pwr_1d_from_2d = pwr_1d_from_2d['binavg'] if mode_transfer_1d is not None: pwr_1d /= mode_transfer_1d[treatment][1] pwr_1d_from_2d /= mode_transfer_1d[treatment][1] # assume that they all have the same binning bin_left = pwrspec_out_signal[1]['bin_left'] bin_center = pwrspec_out_signal[1]['bin_center'] bin_right = pwrspec_out_signal[1]['bin_right'] counts_histo = pwrspec_out_signal[1]['counts_histo'] filename = "%s/%s_%s.dat" % (outdir, output_tag, treatment) outfile = open(filename, "w") for specdata in zip(bin_left, bin_center, bin_right, counts_histo, pwr_1d, pwr_1d_from_2d, mean1dmock, std1dmock): outfile.write(("%10.15g " * 8 + "\n") % specdata) outfile.close() # TODO: kludge to make a fast fit; remove theory_curve = np.genfromtxt("plots/sim_15hr_oldmap_str_temperature_xWigglez/sim_15hr_oldmap_str_temperature_xWigglez_avg_from2d.dat") theory_curve = theory_curve[:, 4] if theory_curve is not None: restrict = np.where(np.logical_and(bin_center > 0.09, bin_center < 1.1)) res_slice = slice(min(restrict[0]), max(restrict[0])) #restrict_alt = np.where(restrict)[0][np.newaxis, :] #restricted_cov = covmock[restrict_alt][0] from core import utils amplitude = utils.ampfit(pwr_1d_from_2d[res_slice], covmock[res_slice, res_slice], theory_curve[res_slice]) print "AMP:", mtag, treatment, amplitude if not output_tag: caller.multiprocess_stack() return None
def make_xcorr_plotdata(): # find the mean brightness used in simulations corrobj = corr21cm.Corr21cm() T_b_sim = corrobj.T_b(1420./800.-1) print T_b_sim #splt.process_batch_correlations(xcorr_real_param, # multiplier=1., cross_power=True) ##splt.process_batch_correlations(xcorr_sim_param, cross_power=True, ## multiplier=1./(T_b_sim/1.e3)) #splt.process_batch_correlations(xcorr_sim_param, cross_power=True, # multiplier=1./(T_b_sim)) #splt.process_batch_correlations(xcorr_loss_sim_param, # multiplier=1./T_b_sim*1.e-3, cross_power=True) #splt.process_batch_correlations(xcorr_variants_param, # multiplier=1., cross_power=True) #splt.plot_batch_correlations(xcorr_real_param, # dir_prefix="plots/xcorr_real/", # color_range=[-0.04, 0.04], cross_power=True) #splt.plot_batch_correlations(xcorr_sim_param, # dir_prefix="plots/xcorr_sim/", # color_range=[-0.04, 0.04], cross_power=True) #splt.plot_batch_correlations(xcorr_loss_param, # dir_prefix="plots/xcorr_loss/", # color_range=[-0.04, 0.04], cross_power=True) #splt.plot_batch_correlations(xcorr_loss_sim_param, # dir_prefix="plots/xcorr_loss_sim/", # color_range=[-0.04, 0.04], cross_power=True) #splt.plot_batch_correlations(xcorr_variants_param, # dir_prefix="plots/xcorr_variants/", # color_range=[-0.04, 0.04], cross_power=True) # find the real xcorr signal with errors xcorr_data = splt.batch_correlations_statistics(xcorr_real_param, randtoken="rand", include_signal=True) # find the simulated xcorr signal with errors xcorr_sim_data = splt.batch_correlations_statistics(xcorr_sim_param, randtoken="rand", include_signal=False) # find the compensation function from simulations compmode = splt.batch_compensation_function(xcorr_loss_sim_param) lagl = xcorr_data[0] lagc = xcorr_data[1] lagr = xcorr_data[2] xcorr_null = xcorr_data[3] xcorr_signal = xcorr_data[6] xcorr_cov = xcorr_data[5] xcorr_err = xcorr_data[4] xcorr_sim = xcorr_sim_data[3] xcorr_sim_err = xcorr_sim_data[4] xcorr_sim_cov = xcorr_sim_data[5] # 0=0, 5=1, 10=2, 15=3, etc. compensation = compmode[3 ,:] (amp, amp_err) = utils.ampfit(xcorr_signal, xcorr_cov + xcorr_sim_cov, xcorr_sim) print amp, amp_err # now convert from the 1e-3 Omega_HI in simulations to 0.5e-3 #xcorr_sim /= 2. #xcorr_sim_err /= 2. xcorr_sim *= amp xcorr_sim_err *= amp for correntry in zip(lagl, lagc, lagr, xcorr_signal, xcorr_null, xcorr_err, xcorr_sim, xcorr_sim_err, compensation): print "%5.3g %5.3g %5.3g %5.3g %5.3g %5.3g %5.3g %5.3g %5.3g" % correntry