def multi_init(n, **kwargs):
llh = MultiPointSourceLLH( #hemispheres=dict(Full=[-np.inf, np.inf]),
seed=np.random.randint(2**32),
**kwargs)
for i in xrange(len(n)):
llh.add_sample(str(i), n[i])
return llh
def multi_init(n, **kwargs):
llh = MultiPointSourceLLH(seed=np.random.randint(2**32), **kwargs)
for i in xrange(len(n)):
llh.add_sample(str(i), n[i])
print("Adding sample: {}".format(str(n[i])))
return llh
def load4yr(sirin=True, nopull=True, mode='box', **kwargs):
print("Mode: " + str(mode))
print("No pull?" + str(nopull))
llh = MultiPointSourceLLH()
print("loading IC40...")
IC40 = init40(mode=mode, nopull=nopull)
print("loading IC59...")
IC59 = init59(mode=mode, nopull=nopull)
if sirin:
print("loading IC79 - sirin's version...")
IC79 = init79_sirin(mode=mode, nopull=nopull)
else:
print("loading IC79...")
#Nopull is not used for kai's sample - the correction happened after his dataset started being used.
IC79 = init79(mode=mode)
print("loading IC86I...")
IC86I = init86I(mode=mode, nopull=nopull)
samples = [IC40, IC59, IC79, IC86I]
arr_exp = []
arr_mc = []
for i in xrange(len(samples)):
llh.add_sample(str(i), samples[i][0])
print("Adding sample" + str(samples[i]))
arr_exp.append(samples[i][1])
arr_mc.append(samples[i][2])
#gotta make exp, MC into dicts.
exp = {0: arr_exp[0], 1: arr_exp[1], 2: arr_exp[2], 3: arr_exp[3]}
MC = {0: arr_mc[0], 1: arr_mc[1], 2: arr_mc[2], 3: arr_mc[3]}
return (llh, exp, MC)
def load7yr_mese(energy=True, sirin=False, mode='all', **kwargs):
print("Energy: " + str(energy))
print("Mode: " + str(mode))
llh = MultiPointSourceLLH()
print("loading IC40...")
IC40 = init40(energy=energy, mode=mode)
print("loading IC59...")
IC59 = init59(energy=energy, mode=mode)
if sirin:
print("loading IC79 - sirin's version...")
IC79 = init79_sirin(energy=energy, mode=mode, decorr=True)
else:
print("loading IC79...")
IC79 = init79(energy=energy, mode=mode, decorr=True)
print("loading IC86I...")
IC86I = init86I(energy=energy, mode=mode, decorr=True)
print("loading IC86II - IC86IV...")
IC86II_III_IV = init3yr(energy=energy, mode=mode, decorr=True)
print("loading MESE 3yr...")
MESE3yr = initMESE3yr(energy=energy, mode=mode)
print("loading MESE followup...")
MESEfollowup = initMESEfollowup(energy=energy, mode=mode)
samples = [IC40, IC59, IC79, IC86I, IC86II_III_IV, MESE3yr, MESEfollowup]
arr_exp = []
arr_mc = []
for i in xrange(len(samples)):
llh.add_sample(str(i), samples[i][0])
print("Adding sample" + str(samples[i]))
arr_exp.append(samples[i][1])
arr_mc.append(samples[i][2])
#gotta make exp, MC into dicts.
exp = {
0: arr_exp[0],
1: arr_exp[1],
2: arr_exp[2],
3: arr_exp[3],
4: arr_exp[4],
5: arr_exp[5],
6: arr_exp[6]
}
MC = {
0: arr_mc[0],
1: arr_mc[1],
2: arr_mc[2],
3: arr_mc[3],
4: arr_mc[4],
5: arr_mc[5],
6: arr_mc[6]
}
return (llh, exp, MC)
def load1yr(nopull=True, mode='box', **kwargs):
print("Mode: " + str(mode))
print("No pull?" + str(nopull))
llh = MultiPointSourceLLH()
print("loading IC86I...")
IC86I = init86I(mode=mode, nopull=nopull)
samples = [IC86I]
arr_exp = []
arr_mc = []
for i in xrange(len(samples)):
llh.add_sample(str(i), samples[i][0])
print("Adding sample" + str(samples[i]))
arr_exp.append(samples[i][1])
arr_mc.append(samples[i][2])
#gotta make exp, MC into dicts.
exp = {0: arr_exp[0]}
MC = {0: arr_mc[0]}
return (llh, exp, MC)
def load3yr_no40(energy=True, sirin=False, nopull=False, mode='all', **kwargs):
print("Energy: " + str(energy))
print("Mode: " + str(mode))
print("No pull?" + str(nopull))
llh = MultiPointSourceLLH()
print("loading IC59...")
IC59 = init59(mode=mode, nopull=nopull)
if sirin:
print("loading IC79 - sirin's version...")
IC79 = init79_sirin(mode=mode, nopull=nopull)
else:
print("loading IC79...")
#Nopull is not used for kai's sample - the correction happened after his dataset started being used.
IC79 = init79(mode=mode)
print("loading IC86I...")
IC86I = init86I(mode=mode, nopull=nopull)
samples = [IC59, IC79, IC86I]
for i in xrange(len(samples)):
llh.add_sample(str(i), samples[i])
print("Adding sample" + str(samples[i]))
return llh
def load7yr(energy=True, sirin=False, mode='all', **kwargs):
print("Energy: " + str(energy))
print("Mode: " + str(mode))
llh = MultiPointSourceLLH()
print("loading IC40...")
IC40 = init40(energy=energy, mode=mode)
print("loading IC59...")
IC59 = init59(energy=energy, mode=mode)
if sirin:
print("loading IC79 - sirin's version...")
IC79 = init79_sirin(energy=energy, mode=mode)
else:
print("loading IC79...")
IC79 = init79(energy=energy, mode=mode)
print("loading IC86I...")
IC86I = init86I(energy=energy, mode=mode)
print("loading IC86II - IC86IV...")
IC86II_III_IV = init3yr(energy=energy, mode=mode)
samples = [IC40, IC59, IC79, IC86I, IC86II_III_IV]
for i in xrange(len(samples)):
llh.add_sample(str(i), samples[i])
print("Adding sample" + str(samples[i]))
return llh
outpath = os.path.join(PATHS.local, "unblinding")
pdf_path = os.path.join(PATHS.local, "bg_pdf", "bg_pdf.json.gz")
check_dir(outpath, ask=False)
# Extract source info, use the same fixed spectral index -2 for all
sources = _loader.source_list_loader()
nsrcs = len(sources)
ra = [src["ra"] for src in sources]
dec = [src["dec"] for src in sources]
# Theo weights need to be normalized manually in skylab?
w = np.ones(nsrcs, dtype=float) / float(nsrcs)
# Create the multi LLH
rnd_seed = 1
multillh = MultiPointSourceLLH(seed=rnd_seed, ncpu=40)
livetimes = _loader.livetime_loader()
for name, livetime in sorted(livetimes.items()):
print("\n# Setting up LLH for sample '{}'".format(name))
# Get exp data and MC
exp = _loader.exp_data_loader(name)[name]
mc = _loader.mc_loader(name)[name]
# Setup the energy LLH model with fixed index, only ns is fitted
settings = _loader.settings_loader(name, skylab_bins=use_skylab_bins)[name]
llh_model = EnergyLLH(**settings["llh_model_opts"])
llh = PointSourceLLH(exp,
mc,
livetime,
llh_model,
if not os.path.isdir(outpath):
os.makedirs(outpath)
# Extract source info
sources = _loader.source_list_loader()
nsrcs = len(sources)
ra = [src["ra"] for src in sources]
dec = [src["dec"] for src in sources]
# Theo weights need to be normalized manually in skylab?
w = np.ones(nsrcs, dtype=float) / float(nsrcs)
# Create the multi LLH
# multillh = MultiPointSourceLLH(seed=rnd_seed, ncpu=40) # For local testing
# multillh = MultiPointSourceLLH(seed=rnd_seed, ncpu=4)
multillh = MultiPointSourceLLH(seed=rnd_seed, ncpu=1)
livetimes = _loader.livetime_loader()
for name, livetime in sorted(livetimes.items()):
print("\n# Setting up LLH for sample '{}'".format(name))
# Get exp data and MC
exp = _loader.exp_data_loader(name)[name]
mc = _loader.mc_loader(name)[name]
# Setup the energy LLH model with fixed index, only ns is fitted
settings = _loader.settings_loader(name, skylab_bins=use_skylab_bins)[name]
llh_model = EnergyLLH(**settings["llh_model_opts"])
llh = PointSourceLLH(exp, mc, livetime, llh_model, scramble=True,
**settings["llh_opts"])
multillh.add_sample(name, llh)
def config(alert_ind,
seed=1,
scramble=True,
e_range=(0, np.inf),
g_range=[1., 5.],
gamma=2.0,
E0=1 * TeV,
remove=False,
ncpu=20,
nside=256,
poisson=False,
injector=True,
verbose=True,
smear=True):
r""" Configure point source likelihood and injector.
Parameters
----------
alert_ind: int
index of IceCube alert event
seed : int
Seed for random number generator
Returns
-------
llh : PointSourceLLH
Point source likelihood object
inj : PriorInjector
Point source injector object
"""
seasons = [("GFUOnline_v001p02", "IC86, 2011-2018"),
("GFUOnline_v001p02", "IC86, 2019")]
#skymaps_path = '/data/user/steinrob/millipede_scan_archive/fits_v3_prob_map/'
#files = glob(skymaps_path + '*.fits')
#skymap_fits = fits.open(files[alert_ind])[0].data
#Turn this into a function read_alert_event()
#skymap_files = glob('/data/ana/realtime/alert_catalog_v2/2yr_prelim/fits_files/Run13*.fits.gz')
skymap_files = glob(
'/data/ana/realtime/alert_catalog_v2/fits_files/Run1*.fits.gz')
#skymap_f = fits.open(skymap_files[alert_ind])
#skymap_fits = skymap_f[1].data
#skymap_header = skymap_f[1].header
skymap_fits, skymap_header = hp.read_map(skymap_files[alert_ind],
h=True,
verbose=False)
skymap_header = {name: val for name, val in skymap_header}
run_id, ev_id = skymap_header['RUNID'], skymap_header['EVENTID']
ev_mjd = skymap_header['EVENTMJD']
ev_iso = skymap_header['START']
signalness = skymap_header['SIGNAL']
ev_en = skymap_header['ENERGY']
ev_ra, ev_dec = np.radians(skymap_header['RA']), np.radians(
skymap_header['DEC'])
ev_stream = skymap_header['I3TYPE']
skymap_llh = skymap_fits.copy()
skymap_fits = np.exp(-1. * skymap_fits /
2.) #Convert from 2LLH to unnormalized probability
skymap_fits = np.where(skymap_fits > 1e-12, skymap_fits, 0.0)
skymap_fits = skymap_fits / np.sum(skymap_fits)
if smear:
ninety_msk = skymap_llh < 64.2
init_nside = hp.get_nside(skymap_llh)
cdf = np.cumsum(np.sort(skymap_fits[ninety_msk][::-1]))
pixs_above_ninety = np.count_nonzero(cdf > 0.1)
original_ninety_area = hp.nside2pixarea(init_nside) * pixs_above_ninety
new_ninety_area = hp.nside2pixarea(init_nside) * np.count_nonzero(
skymap_fits[ninety_msk])
original_ninety_radius = np.sqrt(original_ninety_area / np.pi)
new_ninety_radius = np.sqrt(new_ninety_area / np.pi)
scaled_probs = scale_2d_gauss(skymap_fits, original_ninety_radius,
new_ninety_radius)
skymap_fits = scaled_probs
if hp.pixelfunc.get_nside(skymap_fits) != nside:
skymap_fits = hp.pixelfunc.ud_grade(skymap_fits, nside)
skymap_fits = skymap_fits / skymap_fits.sum()
#print(hp.pixelfunc.get_nside(skymap_fits))
spatial_prior = SpatialPrior(skymap_fits, containment=0.99)
llh = [] # store individual llh as lists to prevent pointer over-writing
multillh = MultiPointSourceLLH(ncpu=1)
if verbose:
print("\n seasons:")
for season in np.atleast_1d(seasons):
sample = season[0]
name = season[1]
exp, mc, livetime = Datasets[sample].season(name,
floor=np.radians(0.2))
sinDec_bins = Datasets[sample].sinDec_bins(name)
energy_bins = Datasets[sample].energy_bins(name)
msg = " - % 15s (" % season
msg += "livetime %7.2f days, %6d events" % (livetime, exp.size)
msg += ", mjd0 %.2f" % min(exp['time'])
msg += ", mjd1 %.2f)" % max(exp['time'])
if verbose:
print(msg)
llh_model = EnergyLLH(twodim_bins=[energy_bins, sinDec_bins],
allow_empty=True,
bounds=g_range,
seed=gamma,
kernel=1,
ncpu=ncpu)
llh.append(
PointSourceLLH(exp,
mc,
livetime,
mode="box",
scramble=scramble,
llh_model=llh_model,
nsource_bounds=(0., 1e3),
nsource=1.))
multillh.add_sample(sample + " : " + name, llh[-1])
# save a little RAM by removing items copied into LLHs
del exp, mc
# END for (season)
######################### REMOVE EVENT
if injector is False:
return multillh, spatial_prior
else:
inj = PriorInjector(spatial_prior,
seed=seed,
gamma=gamma,
E0=1 * TeV,
bunchsize=10)
inj.fill(multillh.exp, multillh.mc, multillh.livetime)
if verbose:
print("\n injected spectrum:")
print(" - %s" % str(inj.spectrum))
return multillh, spatial_prior, inj
sources = _loader.source_list_loader()
srcs_rec = np.lib.recfunctions.drop_fields(make_src_records(sources, 0, 0),
drop_names=["dt0", "dt1"])
nsrcs = len(sources)
src_ra = srcs_rec["ra"]
src_dec = srcs_rec["dec"]
# Theo weights need to be normalized manually in skylab?
src_w = np.ones(nsrcs, dtype=float) / float(nsrcs)
srcs_rec["w_theo"] = src_w
# Load healpy reco map for each source
src_maps = _loader.source_map_loader(sources)
assert nsrcs == len(src_maps)
# Create the multi LLH
rndgen = np.random.RandomState(rnd_seed)
multillh = MultiPointSourceLLH(seed=rnd_seed + 1000, ncpu=1)
multiinj = MultiPowerLawFluxInjector(random_state=rndgen)
# Injector setting
inj_opts = {
"mode": "band",
"inj_sigma": 3.,
"sindec_inj_width": 0.035,
"dec_range": np.array([-np.pi / 2., np.pi / 2.])
}
sig_injs = {}
livetimes = _loader.livetime_loader()
for name, livetime in sorted(livetimes.items()):
print("\n# Setting up LLH for sample '{}'".format(name))
# Get exp data and MC
outpath = os.path.join(PATHS.local, "llh_scan")
pdf_path = os.path.join(PATHS.local, "bg_pdf", "bg_pdf.json.gz")
check_dir(outpath, ask=False)
# Extract source info, use the same fixed spectral index -2 for all
sources = _loader.source_list_loader()
nsrcs = len(sources)
ra = [src["ra"] for src in sources]
dec = [src["dec"] for src in sources]
# Theo weights need to be normalized manually in skylab?
w = np.ones(nsrcs, dtype=float) / float(nsrcs)
# Create the multi LLH
rnd_seed = 1
multillh = MultiPointSourceLLH(seed=rnd_seed, ncpu=40)
livetimes = _loader.livetime_loader()
for name, livetime in sorted(livetimes.items()):
print("\n# Setting up LLH for sample '{}'".format(name))
# Get exp data and MC
exp = _loader.exp_data_loader(name)[name]
mc = _loader.mc_loader(name)[name]
# Setup the energy LLH model with fixed index, only ns is fitted
settings = _loader.settings_loader(name, skylab_bins=use_skylab_bins)[name]
llh_model = EnergyLLH(**settings["llh_model_opts"])
llh = PointSourceLLH(exp, mc, livetime, llh_model, scramble=scramble,
**settings["llh_opts"])
multillh.add_sample(name, llh)
def config(seasons,
seed=1,
scramble=True,
e_range=(0, np.inf),
verbose=True,
gamma=2.0,
dec=0.,
remove=False,
src_w=None):
r""" Configure multi season point source likelihood and injector.
Parameters
----------
seasons : list
List of season names
seed : int
Seed for random number generator
Returns
-------
multillh : MultiPointSourceLLH
Multi year point source likelihood object
inj : PointSourceInjector
Point source injector object
"""
print("Scramble is %s" % str(scramble))
# store individual llh as lists to prevent pointer over-writing
llh = []
multillh = MultiPointSourceLLH(seed=seed, ncpu=25)
# setup likelihoods
if verbose: print("\n seasons:")
for season in np.atleast_1d(seasons):
sample = season[0]
name = season[1]
exp, mc, livetime = Datasets[sample].season(name)
sinDec_bins = Datasets[sample].sinDec_bins(name)
energy_bins = Datasets[sample].energy_bins(name)
if sample == "GFU" and remove:
exp = Datasets['GFU'].remove_ev(
exp, 58018.87118560489) # remove EHE 170922A
mc = mc[mc["logE"] > 1.]
if verbose:
print(" - % 15s : % 15s" % (sample, name))
vals = (livetime, exp.size, min(exp['time']), max(exp['time']))
print(
" (livetime %7.2f days, %6d events, mjd0 %.2f, mjd1 %.2f)"
% vals)
llh_model = EnergyLLH(twodim_bins=[energy_bins, sinDec_bins],
allow_empty=True,
bounds=[1., 4.],
seed=2.,
kernel=1)
llh.append(
StackingPointSourceLLH(exp,
mc,
livetime,
mode="box",
scramble=scramble,
llh_model=llh_model,
nsource_bounds=(0., 1e3),
nsource=15.))
multillh.add_sample(sample + " : " + name, llh[-1])
# save a little RAM by removing items copied into LLHs
del exp, mc
# END for (season)
#######
# LLH #
#######
#############################################################################
############
# INJECTOR #
############
inj = PointSourceInjector(gamma=gamma,
E0=1 * TeV,
seed=seed,
e_range=e_range)
inj.fill(dec, multillh.exp, multillh.mc, multillh.livetime, src_w)
############
# INJECTOR #
############
#############################################################################
if verbose:
print("\n fitted spectrum:")
vals = (inj.E0 / TeV)
print(" - dN/dE = A (E / %.1f TeV)^-index TeV^-1cm^-2s^-1" % vals)
print(" - index is *fit*")
print("\n injected spectrum:")
vals = (inj.E0 / TeV, inj.gamma)
print(" - dN/dE = A (E / %.1f TeV)^-%.2f TeV^-1cm^-2s^-1" % vals)
return (multillh, inj)
# Extract source info
sources = _loader.source_list_loader()
srcs_rec = np.lib.recfunctions.drop_fields(make_src_records(sources, 0, 0),
drop_names=["dt0", "dt1"])
nsrcs = len(sources)
src_ra = srcs_rec["ra"]
src_dec = srcs_rec["dec"]
# Theo weights need to be normalized manually in skylab?
src_w = np.ones(nsrcs, dtype=float) / float(nsrcs)
srcs_rec["w_theo"] = src_w
# Load healpy reco map for each source
src_maps = _loader.source_map_loader(sources)
assert nsrcs == len(src_maps)
# Create the multi LLH
multillh = MultiPointSourceLLH(seed=rnd_seed + 1000, ncpu=1)
multiinj = MultiPowerLawFluxInjector(random_state=rndgen)
# Injector setting
inj_opts = {
"mode": "band",
"inj_sigma": 3.,
"sindec_inj_width": 0.035,
"dec_range": np.array([-np.pi / 2., np.pi / 2.])
}
sig_injs = {}
livetimes = _loader.livetime_loader()
for name, livetime in sorted(livetimes.items()):
print("\n# Setting up LLH for sample '{}'".format(name))
# Get exp data and MC
src_ra=src_ra,
src_dec=src_dec,
alpha=alpha,
beta=beta,
inj=inj,
TSval=TSval,
src_w=modelweights['{}'.format(injweight)],
eps=0.04,
n_iter=100)
else:
sensitivity = MultiPointSourceLLH.weighted_sensitivity(
llhmodel,
src_ra=src_ra,
src_dec=src_dec,
alpha=alpha,
beta=beta,
inj=inj,
TSval=TSval,
src_w=modelweights['{}'.format(injweight)],
eps=0.04,
n_iter=100)
else:
if years == 1:
sensitivity = PointSourceLLH.weighted_sensitivity(llhmodel,
src_ra=src_ra,
src_dec=src_dec,
alpha=alpha,
beta=beta,
inj=inj,
TSval=TSval,
eps=0.04,
elif years == 1:
print("Using IC86")
llh = datascript.init86I(mode='box')
if catalog:
if years == 1:
bckg_trials = PointSourceLLH.do_trials(
llh[0],
n_iter=batchsize,
src_ra=src_ra,
src_dec=src_dec,
src_w=modelweights['{}'.format(llhweight)])
else:
bckg_trials = MultiPointSourceLLH.do_trials(
llh[0],
n_iter=batchsize,
src_ra=src_ra,
src_dec=src_dec,
src_w=modelweights['{}'.format(llhweight)])
else:
if years == 1:
bckg_trials = PointSourceLLH.do_trials(llh[0],
n_iter=batchsize,
src_ra=src_ra,
src_dec=src_dec)
else:
bckg_trials = MultiPointSourceLLH.do_trials(llh[0],
n_iter=batchsize,
src_ra=src_ra,
src_dec=src_dec)
#choose an output dir, and make sure it exists
