def run_background_trials(args):
"""Run the background trials for the stacked nova analysis
:type args: dict-like
:param args: Command line arguments or dict with keys needed for analysis
"""
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'}
only_gamma = not args.all_nova
weighting_scheme = args.weighting
src, sample_str = get_sources(only_gamma, weighting_scheme, delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
tr = cy.get_trial_runner(conf, ana=greco_ana, src=src)
n_trials = args.ntrials
bg_trials = tr.get_many_fits(n_trials, seed=args.seed, logging=False)
filename = '/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/bg/' \
+ '{}_delta_t_{:.2e}_seed_{}.npy'.format(
sample_str, delta_t, args.seed)
np.save(filename, bg_trials.as_array)
################################################################################
wt=[]
for j in range(len(flux_used)):
if equal_weights:
wt.append(1.0)
else:
wt.append(flux_used[j])
wt=wt/np.sum(wt)
print("\n WEIGHTS USED FOR THIS RUN:\n %s \n"%(wt))
################################################################################
# CSKY SETUP
################################################################################
ana_dir = cy.utils.ensure_dir(ana_dir_path)
repo = cy.selections.repo
ana = cy.get_analysis(cy.selections.repo, cy.selections.PSDataSpecs.ps_10yr, dir=ana_dir)
#ana.save(ana_dir) #Un-comment if you are running the analysis for the first time
cy.CONF['ana'] = ana
cy.CONF['mp_cpus'] = 2
################################################################################
# CSKY SOURCE DESCRIPTION
################################################################################
src = cy.utils.Sources(ra=src_ra,dec=src_dec,weight=wt,deg=False)
flux = cy.hyp.PowerLawFlux(gamma=index,norm=E0)
################################################################################
# BACKGROUND
################################################################################
#conf = dict(fitter_args = _fmin_method='bfgs')
timer = cy.timing.Timer()
type=int,
default=10000,
help="Number of background trials")
parser.add_argument('--ntrials_sig',
type=int,
default=500,
help='Number of trials per signal strength')
args = parser.parse_args()
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'
}
result_dict = {'dec': [], 'sens': [], 'sens_nsig': []}
ra = 0.
mjd_start = 57200.
def run_all_signal_trials(args):
r"""Look at output background trials and run the signal
trials for a stacked nova analysis
"""
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'
}
only_gamma = not args.all_nova
print('only', only_gamma)
print('all', args.all_nova)
print('~all', ~args.all_nova)
weighting_scheme = args.weighting
src, sample_str = get_sources(only_gamma, weighting_scheme, delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
def ndarray_to_Chi2TSD(trials):
return cy.dists.Chi2TSD(cy.utils.Arrays(trials))
bg = cy.bk.get_all(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/bg/',
'{}_delta_t_{:.2e}_seed_*.npy'.format(sample_str, delta_t),
merge=np.concatenate,
post_convert=ndarray_to_Chi2TSD)
tr = cy.get_trial_runner(
conf,
ana=greco_ana,
src=src,
inj_conf={'flux': cy.hyp.PowerLawFlux(args.index)})
result = {}
if delta_t < 1e3:
n_sig_step = 5
elif delta_t <= 86400. and args.index <= 2.5:
n_sig_step = 7
elif delta_t <= 86400:
n_sig_step = 15
elif args.index <= 2.5:
n_sig_step = 15
else:
n_sig_step = 25
########################################################################
# SENSITIVITY CALCULATION
########################################################################
beta = 0.9
sensitivity = tr.find_n_sig(bg.median(),
beta,
batch_size=args.ntrials_sig,
n_sig_step=n_sig_step,
max_batch_size=0,
logging=True,
n_bootstrap=1)
sensitivity['E2dNdE'] = tr.to_E2dNdE(sensitivity, E0=1., unit=1e3)
########################################################################
# DISCOVERY POTENTIAL CALC
########################################################################
thresh_ts = bg.isf_nsigma(5.)
beta = 0.5 # beta = 0.5
discovery = tr.find_n_sig(thresh_ts,
beta,
batch_size=args.ntrials_sig,
n_sig_step=n_sig_step,
max_batch_size=0,
logging=True,
n_bootstrap=1)
discovery['E2dNdE'] = tr.to_E2dNdE(discovery, E0=1., unit=1e3)
discovery['nsigma'] = 5.
discovery['CL'] = beta
########################################################################
# FIT BIAS TRIALS
########################################################################
n_sigs = np.r_[:201:10]
trials = [
tr.get_many_fits(int(args.ntrials_sig / 2),
n_sig=n_sig,
logging=False,
seed=n_sig) for n_sig in n_sigs
]
for (n_sig, t) in zip(n_sigs, trials):
t['ntrue'] = np.repeat(n_sig, len(t))
allt = cy.utils.Arrays.concatenate(trials)
result['bg'] = bg
result['sensitivity'] = sensitivity
result['discovery'] = discovery
result['fit'] = allt
result['settings'] = args
result['source_info'] = {'ra': src.ra, 'dec': src.dec, 'mjd': src.mjd}
with open(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/' +
'signal_results/' + '{}_delta_t_{:.2e}_gamma_{}_.pkl'.format(
sample_str, delta_t, args.index), 'wb') as f:
pickle.dump(result, f)
BpBest = BpArr.min()
kBest = BpArr.argmin() + 1
pThresh = pSorted[kBest - 1]
if returnArray:
return pThresh, kBest, BpBest, BpArr
else:
return pThresh, kBest, BpBest
def ndarray_to_Chi2TSD(trials):
return cy.dists.Chi2TSD(cy.utils.Arrays(trials))
repo = selections.Repository()
with time('ana setup'):
ana = cy.get_analysis(cy.selections.repo,
cy.selections.GFUDataSpecs.gfu_IC86)
cy.CONF['ana'] = ana
min_data_set = ana.mjd_min
max_data_set = ana.mjd_max
# Load the complete GRB list
GRB_main_list = np.genfromtxt("GRBs_withoutheader.txt",
dtype=[('GRB_name', 'S20'),
('GRB_name_Fermi', 'S20'),
('t_trigger', 'S20'), ('ra', 'f4'),
('decl', 'float'), ('pos_error', 'f4'),
('T90', 'f4'), ('T90_error', 'f4'),
('T90_start', 'S20'), ('fluence', 'f4'),
('fluence_error', 'f4'),
('redshift', 'f4'), ('T100', 'f4'),
('GBM_located', 'bool'), ('mjd', 'f4')])
def run_differential_sens(args):
r"""Look at output background trials and run the signal
trials for a stacked nova analysis
"""
delta_t = args.deltaT
delta_t_days = delta_t / 86400.
ana_dir = cy.utils.ensure_dir('/data/user/apizzuto/csky_cache/greco_ana')
greco_ana = cy.get_analysis(
cy.selections.repo,
'version-002-p10',
cy.selections.GRECOOnlineDataSpecs.GRECO_IC86_2012_2019,
dir=ana_dir)
conf = {
'ana': greco_ana,
'extended': True,
'space': "ps",
'time': "transient",
'sig': 'transient'
}
only_gamma = not args.all_nova
weighting_scheme = args.weighting
src, sample_str = get_sources(only_gamma, weighting_scheme, delta_t_days)
cy.CONF['src'] = src
cy.CONF['mp_cpus'] = 5
def ndarray_to_Chi2TSD(trials):
return cy.dists.Chi2TSD(cy.utils.Arrays(trials))
bg = cy.bk.get_all(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/bg/',
'{}_delta_t_{:.2e}_seed_*.npy'.format(sample_str, delta_t),
merge=np.concatenate,
post_convert=ndarray_to_Chi2TSD)
Ebins = 10**np.r_[0.:4.1:1.0]
trs = [
cy.get_trial_runner(conf,
ana=greco_ana,
src=src,
flux=cy.hyp.PowerLawFlux(2.0,
energy_range=(Emin,
Emax)))
for (Emin, Emax) in zip(Ebins[:-1], Ebins[1:])
]
if delta_t < 1e3:
n_sig_step = 5
elif delta_t <= 86400.:
n_sig_step = 7
elif delta_t <= 86400:
n_sig_step = 15
else:
n_sig_step = 15
result = {}
for ii, tr in enumerate(trs):
try:
#################################################################
# SENSITIVITY CALCULATION
#################################################################
beta = 0.9
sensitivity = tr.find_n_sig(bg.median(),
beta,
batch_size=args.ntrials_sig,
n_sig_step=n_sig_step,
max_batch_size=0,
logging=True,
n_bootstrap=1)
sensitivity['E2dNdE'] = tr.to_E2dNdE(sensitivity,
E0=Ebins[ii] / 1e3,
unit=1e3)
tmp_key = f'sensitivity_{Ebins[ii]:.1f}_{Ebins[ii+1]:.1f}'
result[tmp_key] = sensitivity
except Exception as e:
print(f"Could not do energy range {Ebins[ii]} to {Ebins[ii+1]}")
result['bg'] = bg
result['settings'] = args
result['source_info'] = {
'ra': list(src.ra),
'dec': list(src.dec),
'mjd': list(src.mjds)
}
with open(
'/data/user/apizzuto/Nova/csky_trials/stacking_sens_res/' +
'differential_sens/' +
'{}_delta_t_{:.2e}.pkl'.format(sample_str, delta_t), 'wb') as f:
pickle.dump(result, f)
trials_dir = cy.utils.ensure_dir('/data/user/mcampana/analysis/Blazar_1FLE/trials_Erange_{}-{}'.format(ethresh_low, ethresh_high))
version = 'version-003-p02'
elif dSetName == 'ps-v4':
selection = cy.selections.PSDataSpecs.ps_v4
trials_dir = cy.utils.ensure_dir('/data/user/mcampana/analysis/Blazar_1FLE/trials_psv4')
version = 'version-004-p00'
print('Getting analysis arrays...')
#Create analysis with data selection (saved in working dir)
#ana = cy.get_analysis(cy.selections.repo, selection)
#ana.save('./')
ana_dir = cy.utils.ensure_dir('/data/user/mcampana/analysis/Blazar_1FLE/subanalyses/{}'.format(dSetName))
ana = cy.get_analysis(cy.selections.repo, version, selection, dir=ana_dir)
#Trial save and load directories
sig_dir = cy.utils.ensure_dir('{}/sig'.format(trials_dir))
if not E_cut:
bg_dir = cy.utils.ensure_dir('{}/bg'.format(trials_dir))
elif E_cut:
bg_dir = cy.utils.ensure_dir('/data/user/mcampana/analysis/Blazar_1FLE/trials_u1FLE/bg')
#=========================================================
# FUNCTION DEFINITIONS
#=========================================================
#Do BG Trials
def bg_trials():