def do_signal_trials(n_sig, N=n_sig_trials):
# get trial runner
src = cy.sources(FRB_ra_rad,
FRB_dec_rad,
mjd=FRB_mjd_time,
sigma_t=np.zeros_like(FRB_ra_rad),
t_100=FRB_time_window)
conf = {
'extended': True,
'space': "ps",
'time': "transient",
'sig': "transient",
'flux': cy.hyp.PowerLawFlux(args.gamma),
'rates_by': 'livetime'
}
tr = cy.get_trial_runner(conf, src=src, ana=ana)
# run trials
trials = tr.get_many_fits(N, n_sig, poisson=True, logging=False)
# save to disk
dir = cy.utils.ensure_dir('{}/gamma/{}/dt/{}/n_sig/{}'.format(
sig_dir, args.gamma, args.dt, n_sig))
filename = '{}/sig_trials_{}.npy'.format(dir, args.seed)
print('->', filename)
# notice: trials.as_array is a numpy structured array, not a cy.utils.Arrays
np.save(filename, trials.as_array)
def do_background_trials(N=n_bg_trials):
src = cy.sources(FRB_ra_rad,
FRB_dec_rad,
mjd=FRB_mjd_time,
sigma_t=np.zeros_like(FRB_ra_rad),
t_100=FRB_time_window)
conf = {
'extended': True,
'space': "ps",
'time': "transient",
'sig': "transient",
'flux': cy.hyp.PowerLawFlux(args.gamma)
}
tr = cy.get_trial_runner(conf, src=src, ana=ana)
# run trials
trials = tr.get_many_fits(N, logging=False)
# save to disk
dir = cy.utils.ensure_dir('{}/dt/{}'.format(bg_dir, args.dt))
filename = '{}/bg_trials_seed_{}.npy'.format(dir, args.seed)
print('->', filename)
# notice: trials.as_array is a numpy structured array, not a cy.utils.Arrays
np.save(filename, trials.as_array)
def run_background_trials(args):
"""Run the background trials for the stacked nova analysis
:type args: dict-like
:param args: Command line arguments or dict with keys needed for analysis
"""
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'}
only_gamma = not args.all_nova
weighting_scheme = args.weighting
src, sample_str = get_sources(only_gamma, weighting_scheme, delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
tr = cy.get_trial_runner(conf, ana=greco_ana, src=src)
n_trials = args.ntrials
bg_trials = tr.get_many_fits(n_trials, seed=args.seed, logging=False)
filename = '/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/bg/' \
+ '{}_delta_t_{:.2e}_seed_{}.npy'.format(
sample_str, delta_t, args.seed)
np.save(filename, bg_trials.as_array)
def find_n_sig(beta=0.9, nsigma=None):
# get signal trials, background distribution, and trial runner
sig_trials = cy.bk.get_best(sig, 'gamma', args.gamma, 'dt', args.dt,
'n_sig')
b = cy.bk.get_best(bg, 'gamma', args.gamma, 'dt', args.dt)
src = cy.sources(FRB_ra_rad,
FRB_dec_rad,
mjd=FRB_mjd_time,
sigma_t=np.zeros_like(FRB_ra_rad),
t_100=FRB_time_window)
conf = {
'extended': True,
'space': "ps",
'time': "transient",
'sig': "transient",
'flux': cy.hyp.PowerLawFlux(args.gamma),
'box_mode': 'center'
}
tr = cy.get_trial_runner(conf, src=src, ana=ana)
# determine ts threshold
if nsigma is not None:
ts = b.isf_nsigma(nsigma)
else:
ts = b.median()
# include background trials in calculation
trials = {0: b.trials}
trials.update(sig_trials)
# get number of signal events
# (arguments prevent additional trials from being run)
result = tr.find_n_sig(ts,
beta,
max_batch_size=0,
logging=False,
trials=trials,
n_bootstrap=1)
# return flux
return tr.to_E2dNdE(result, E0=1e5)
cy.CONF['mp_cpus'] = 2
################################################################################
# CSKY SOURCE DESCRIPTION
################################################################################
src = cy.utils.Sources(ra=src_ra,dec=src_dec,weight=wt,deg=False)
flux = cy.hyp.PowerLawFlux(gamma=index,norm=E0)
################################################################################
# BACKGROUND
################################################################################
#conf = dict(fitter_args = _fmin_method='bfgs')
timer = cy.timing.Timer()
time = timer.time
with time('trial runner construction'):
tr = cy.get_trial_runner(src=src, ana=ana,flux=flux, mp_cpus=2, sindec_bandwidth=np.radians(.1))
################################################################################
# N Bias test
################################################################################
n_sig_arr =[1,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,437]
trials = [tr.get_many_fits(100, n_sig=n_sig, logging=False, seed=n_sig*np.random.randint(1,1000)) for n_sig in n_sig_arr]
for (n_sig, t) in zip(n_sig_arr, trials):
t['ntrue'] = np.repeat(n_sig, len(t))
time = timer.time
allt = cy.utils.Arrays.concatenate(trials)
#allt
np.savez(wrkdir+"trials_with_sig_inj_"+str(index)+".npz",allt.as_array)
fig, axs = plt.subplots(1, 2, figsize=(6,3))
result_dict = {'dec': [], 'sens': [], 'sens_nsig': []}
ra = 0.
mjd_start = 57200.
sin_decs = np.linspace(-0.95, 0.95, 20)
for dec in np.arcsin(sin_decs):
src = cy.utils.Sources(ra=ra,
dec=dec,
mjd=mjd_start,
sigma_t=0.,
t_100=delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
tr = cy.get_trial_runner(conf, ana=greco_ana, src=src)
bg = cy.dists.Chi2TSD(tr.get_many_fits(args.ntrials_bg))
tr = cy.get_trial_runner(
conf,
ana=greco_ana,
src=src,
inj_conf={'flux': cy.hyp.PowerLawFlux(args.index)})
########################################################################
# SENSITIVITY CALCULATION
########################################################################
nsig_step = 1. if delta_t < 864000. else 20.
sensitivity = tr.find_n_sig(bg.median(),
0.9,
def run_all_signal_trials(args):
r"""Look at output background trials and run the signal
trials for a stacked nova analysis
"""
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'
}
only_gamma = not args.all_nova
print('only', only_gamma)
print('all', args.all_nova)
print('~all', ~args.all_nova)
weighting_scheme = args.weighting
src, sample_str = get_sources(only_gamma, weighting_scheme, delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
def ndarray_to_Chi2TSD(trials):
return cy.dists.Chi2TSD(cy.utils.Arrays(trials))
bg = cy.bk.get_all(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/bg/',
'{}_delta_t_{:.2e}_seed_*.npy'.format(sample_str, delta_t),
merge=np.concatenate,
post_convert=ndarray_to_Chi2TSD)
tr = cy.get_trial_runner(
conf,
ana=greco_ana,
src=src,
inj_conf={'flux': cy.hyp.PowerLawFlux(args.index)})
result = {}
if delta_t < 1e3:
n_sig_step = 5
elif delta_t <= 86400. and args.index <= 2.5:
n_sig_step = 7
elif delta_t <= 86400:
n_sig_step = 15
elif args.index <= 2.5:
n_sig_step = 15
else:
n_sig_step = 25
########################################################################
# SENSITIVITY CALCULATION
########################################################################
beta = 0.9
sensitivity = tr.find_n_sig(bg.median(),
beta,
batch_size=args.ntrials_sig,
n_sig_step=n_sig_step,
max_batch_size=0,
logging=True,
n_bootstrap=1)
sensitivity['E2dNdE'] = tr.to_E2dNdE(sensitivity, E0=1., unit=1e3)
########################################################################
# DISCOVERY POTENTIAL CALC
########################################################################
thresh_ts = bg.isf_nsigma(5.)
beta = 0.5 # beta = 0.5
discovery = tr.find_n_sig(thresh_ts,
beta,
batch_size=args.ntrials_sig,
n_sig_step=n_sig_step,
max_batch_size=0,
logging=True,
n_bootstrap=1)
discovery['E2dNdE'] = tr.to_E2dNdE(discovery, E0=1., unit=1e3)
discovery['nsigma'] = 5.
discovery['CL'] = beta
########################################################################
# FIT BIAS TRIALS
########################################################################
n_sigs = np.r_[:201:10]
trials = [
tr.get_many_fits(int(args.ntrials_sig / 2),
n_sig=n_sig,
logging=False,
seed=n_sig) for n_sig in n_sigs
]
for (n_sig, t) in zip(n_sigs, trials):
t['ntrue'] = np.repeat(n_sig, len(t))
allt = cy.utils.Arrays.concatenate(trials)
result['bg'] = bg
result['sensitivity'] = sensitivity
result['discovery'] = discovery
result['fit'] = allt
result['settings'] = args
result['source_info'] = {'ra': src.ra, 'dec': src.dec, 'mjd': src.mjd}
with open(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/' +
'signal_results/' + '{}_delta_t_{:.2e}_gamma_{}_.pkl'.format(
sample_str, delta_t, args.index), 'wb') as f:
pickle.dump(result, f)
cy.CONF['mp_cpus'] = n_cpus
src = cy.sources(FRB_ra_rad,
FRB_dec_rad,
mjd=FRB_mjd_time,
sigma_t=np.zeros_like(FRB_ra_rad),
t_100=FRB_time_window)
conf = {
'extended': True,
'space': "ps",
'time': "transient",
'sig': "transient"
} #,'flux': cy.hyp.PowerLawFlux(gamma)}
with time('background estimation'):
allE_tr = cy.get_trial_runner(conf, ana=ana, src=src)
bg = cy.dists.TSD(allE_tr.get_many_fits(10000, mp_cpus=n_cpus))
# you could also use np.logspace()
Ebins = 10**np.r_[3:7.1:.25]
trs = [
cy.get_trial_runner(conf,
ana=ana,
src=src,
flux=cy.hyp.PowerLawFlux(gamma,
energy_range=(Emin, Emax)))
for (Emin, Emax) in zip(Ebins[:-1], Ebins[1:])
]
senss = []
'sig': 'tw',
'sig_kw': dict(t0=mjd_grb, dt=inj_duration, box_mode='post'),
#Note that the box_mode here is set to 'pre'. This is so that we can control injecting the size of pulses.
# hold t0 fixed
'fitter_args': dict(t0=mjd_grb),
# use multiprocessing
'mp_cpus': 10,
'prior': None #prior =None is the key for removing the penalty term
}
afterglow_inj = {}
afterglow_inj.update(afterglow)
del afterglow_inj['fitter_args']
tr_inj = cy.get_trial_runner(afterglow_inj,
src=sources,
inj_conf=dict(src=sources),
extra_keep=['run', 'event'])
#Defining the Multi trial runner to perform the analysis on the same scrambled sky
multr = cy.trial.MultiTrialRunner(
# the Analysis
ana,
# bg+sig injection trial runner (produces trials)
tr_inj,
# llh test trial runners (perform fits given trials)
trs,
# background distrubutions, can also do without this info since was getting some error when i use it
#bgs=bg,
# use multiprocessing
mp_cpus=10,
)
def run_differential_sens(args):
r"""Look at output background trials and run the signal
trials for a stacked nova analysis
"""
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'
}
only_gamma = not args.all_nova
weighting_scheme = args.weighting
src, sample_str = get_sources(only_gamma, weighting_scheme, delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
def ndarray_to_Chi2TSD(trials):
return cy.dists.Chi2TSD(cy.utils.Arrays(trials))
bg = cy.bk.get_all(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/bg/',
'{}_delta_t_{:.2e}_seed_*.npy'.format(sample_str, delta_t),
merge=np.concatenate,
post_convert=ndarray_to_Chi2TSD)
Ebins = 10**np.r_[0.:4.1:1.0]
trs = [
cy.get_trial_runner(conf,
ana=greco_ana,
src=src,
flux=cy.hyp.PowerLawFlux(2.0,
energy_range=(Emin,
Emax)))
for (Emin, Emax) in zip(Ebins[:-1], Ebins[1:])
]
if delta_t < 1e3:
n_sig_step = 5
elif delta_t <= 86400.:
n_sig_step = 7
elif delta_t <= 86400:
n_sig_step = 15
else:
n_sig_step = 15
result = {}
for ii, tr in enumerate(trs):
try:
#################################################################
# SENSITIVITY CALCULATION
#################################################################
beta = 0.9
sensitivity = tr.find_n_sig(bg.median(),
beta,
batch_size=args.ntrials_sig,
n_sig_step=n_sig_step,
max_batch_size=0,
logging=True,
n_bootstrap=1)
sensitivity['E2dNdE'] = tr.to_E2dNdE(sensitivity,
E0=Ebins[ii] / 1e3,
unit=1e3)
tmp_key = f'sensitivity_{Ebins[ii]:.1f}_{Ebins[ii+1]:.1f}'
result[tmp_key] = sensitivity
except Exception as e:
print(f"Could not do energy range {Ebins[ii]} to {Ebins[ii+1]}")
result['bg'] = bg
result['settings'] = args
result['source_info'] = {
'ra': list(src.ra),
'dec': list(src.dec),
'mjd': list(src.mjds)
}
with open(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/' +
'differential_sens/' +
'{}_delta_t_{:.2e}.pkl'.format(sample_str, delta_t), 'wb') as f:
pickle.dump(result, f)
flux_data = np.copy(data['EF30-100'])
src_weights = flux_data / np.sum(flux_data)
srcs = cy.utils.Sources(ra=data['RAdeg'], dec=data['DEdeg'], deg=True, weight=src_weights)
for g in gamma:
#Get Trials Runner
#Set signal flux (for injections)
# Power law from gamma, with or without some cutoff (for determining sensitivity changes and energy range)
if not E_cut:
flux = cy.hyp.PowerLawFlux(g)
elif E_cut:
flux = cy.hyp.PowerLawFlux(g, energy_range=(ethresh_low, ethresh_high))
tr = cy.get_trial_runner(src=srcs, ana=ana, flux=flux, mp_cpus=cpus)
for s in seed:
print('Doing {} BG trials for {} data, {} weight, gamma = {}, seed = {} ...'.format(num_trials,dSetName,w,g,s))
if do_bg_trials:
if g == 2.0:
bg_trials()
if w == 'equal':
if g == 1.75:
n_sigs = np.r_[2:22.1:2, 30:70.1:4]
elif g == 2.0:
n_sigs = np.r_[2:42.1:4, 60:100.1:4]
elif g == 2.25:
n_sigs = np.r_[10:50.1:4, 120:160.1:4]
