def make_fakes(amp, psrcut):
datadir = './data/partim/'
## Get parameter noise dictionary
noisedir = os.getcwd() + '/data/noisefiles/'
noise_params = {}
for filepath in glob.iglob(noisedir + '/*.json'):
with open(filepath, 'r') as f:
noise_params.update(json.load(f))
# print(noise_params)
# import par and tim files into parfiles and timfiles
tempopsr = []
print('Importing to libstempo....')
for psr in psrcut:
parfiles = sorted(glob.glob(datadir + psr.name + '*.par'))
timfiles = sorted(glob.glob(datadir + psr.name + '*.tim'))
psr = libs.tempopulsar(parfile=parfiles[0],
timfile=timfiles[0],
maxobs=50000)
tempopsr.append(psr)
print('adding efac')
for psr in tempopsr:
LT.make_ideal(psr)
LT.add_efac(psr, efac=1.0)
print('adding red noise')
for psr in tempopsr:
# print(psr.name)
rednoise_toadd = noise_params[str(psr.name) + '_log10_A']
# print(psr.name)
gamma = noise_params[str(psr.name) + '_gamma']
LT.add_rednoise(psr, 10**rednoise_toadd, gamma, components=30)
psr.fit()
print('adding GWB')
LT.createGWB(tempopsr, Amp=amp, gam=13. / 3.)
for psr in tempopsr:
psr.fit()
# save the pulsars
pardir = './fakes_gwb/par/'
timdir = './fakes_gwb/tim/'
if not os.path.exists(pardir):
os.makedirs(pardir)
os.makedirs(timdir)
print('fixing tim files')
for psr in tempopsr:
psr.savepar(pardir + psr.name + '-sim.par')
psr.savetim(timdir + psr.name + '-sim.tim')
libs.purgetim(timdir + psr.name + '-sim.tim') # fix the tim files
parfiles = sorted(glob.glob(args.pardir + '*.par'))
timfiles = sorted(glob.glob(args.timdir + '*.tim'))
if args.psr_list is not None:
parfiles = filter_psr_path(parfiles, args.psr_list, rhs='_')
timfiles = filter_psr_path(timfiles, args.psr_list, rhs='_')
sim = Simulation(parfiles, timfiles, ephem=args.ephem, verbose=True)
if args.process is None:
seed_gwb = None
else:
seed_gwb = 1978 + args.process
for ii, p in enumerate(sim.libs_psrs):
LT.add_efac(p, seed=seed_gwb + ii)
sim.createGWB(A_gwb=args.A_gwb, gamma_gw=args.gamma_gw, seed=seed_gwb)
sim.init_ePulsars()
# noise = {}
# with open(args.noisepath, 'r') as fin:
# noise.update(json.load(fin))
#
# pta = models.model_2a(psrs=sim.psrs, psd='powerlaw', noisedict=noise,
# components=30, gamma_common=args.gamma_gw,
# upper_limit=True, bayesephem=False)
sim.filter_by_mjd(args.end_time)
pta = model_simple(psrs=sim.psrs,
ct = 0
for d, p in zip(delays, parfiles):
print 'Using {0} delay with parfile {1}.'.format(d.split('/')[-1], p.split('/')[-1])
d = np.loadtxt(d)
toas, res = d[:,0], d[:,1]
err = np.ones(len(toas)) * args.rms * 1e-3
# make fake observations
psr = ts.fakepulsar(p, toas, err)
# makes perfect residuals (i.e. no noise)
ts.make_ideal(psr)
# add GWB delays
psr.stoas[:] += res
# adds white noise at level of defined uncertainties above
ts.add_efac(psr, 1, seed=(args.seed*ct))
# fit
psr.fit()
# save par and tim file
psr.savepar(outdir + '/' + psr.name + '.par')
psr.savetim(outdir + '/' + psr.name + '.tim')
# update counter
ct += 1
else:
tempopsr.append(psr)
# remove all noise from the pulsars.
for psr in tempopsr:
print('Adding rednoise to ' + psr.name)
efac_keys, efac_vals, equad_keys, equad_vals, red_noise_gamma_val, red_noise_amp_val = get_noise(
psr.name)
if args.average_epoch:
efac_val = np.mean(efac_vals)
equad_val = np.mean(equad_vals)
LT.add_efac(psr, efac=efac_val)
LT.add_equad(psr, equad=10**equad_val)
LT.add_rednoise(psr, 10**red_noise_amp_val, red_noise_gamma_val)
else:
for i in range(len(equad_vals)):
equad_vals[i] = 10**equad_vals[i]
print(equad_vals)
LT.make_ideal(psr)
LT.add_efac(psr, efac=efac_vals, flags=efac_keys, flagid='f')
LT.add_equad(psr, equad=equad_vals, flags=equad_keys, flagid='f')
LT.add_rednoise(psr, 10**red_noise_amp_val, red_noise_gamma_val)
## Inject fdm.
def create_dataset(dataset, Agwb):
""" Create simulated dataset using 18 pulsars from NANOGrav 9-year
stochastic analysis. Will use 11-year data span and red noise values
with white noise values taken from most recent time-to-detection
simulations.
:param dataset: Name of output dataset.
:param Agwb: Amplitude of injected dipole signal
"""
print 'Getting pulsar parameters for simulated dataset...'
# get simulation data
with open('nano9_simdata.json', 'r') as fp:
pdict = json.load(fp)
# get red noise dictionary
with open('nano_red_dict.json', 'r') as fp:
red_dict = json.load(fp)
# get parfiles
parfiles = glob.glob('../data/nano9_stipped_parfiles/*.par')
parfiles = [
p for p in parfiles if p.split('/')[-1].split('_')[0] in pdict.keys()
]
datadir = '../data/simulated_data/' + dataset
if not os.path.exists(datadir):
os.makedirs(datadir)
print 'Making simulated data in directory ' + datadir
psrs = []
for pf in parfiles:
pname = pf.split('/')[-1].split('_')[0]
psrs.append(LT.fakepulsar(pf, pdict[pname][0], pdict[pname][1]))
for psr in psrs:
# white noise
LT.add_efac(psr)
# red noise
if pname in red_dict:
LT.add_rednoise(psr,
red_dict[pname][0],
red_dict[pname][1],
components=30)
# dipole signal
createdipole(psrs, Agwb, 13. / 3., seed=None)
for psr in psrs:
psr.fit(iters=2)
## Save par and tim files
psr.savepar(datadir + '/{0}_optstatsim.par'.format(psr.name))
psr.savetim(datadir + '/{0}_optstatsim.tim'.format(psr.name))
# make the hdf5 file for the simulated dataset
print 'Making the hdf5 file for the simulated dataset...'
datadir = '../data/simulated_data/' + dataset
h5filename = datadir + '/sim.hdf5'
os.system('python makeH5file.py \
--pardir {0} --timdir {0} \
--h5File {1}'.format(datadir, h5filename))
print 'Finished!'
seed_efac = 1066
seed_equad = 1492
seed_jitter = 1776
seed_red = 1987
seed_gwb_1 = 1667
seed_gwb_2 = 1980
for ii, p in enumerate(t2psr):
## make ideal
LT.make_ideal(p)
## add efacs
LT.add_efac(p,
efac=noise_dict[p.name]['efacs'][:, 1],
flagid='f',
flags=noise_dict[p.name]['efacs'][:, 0],
seed=seed_efac + ii)
## add equads
LT.add_equad(p,
equad=noise_dict[p.name]['equads'][:, 1],
flagid='f',
flags=noise_dict[p.name]['equads'][:, 0],
seed=seed_equad + ii)
## add jitter
try: #Only NANOGrav Pulsars have ECORR
LT.add_jitter(p,
ecorr=noise_dict[p.name]['ecorrs'][:, 1],
flagid='f',
def add_noise(t2pulsar, noise_dict, sim_dm=True, sim_white=True, sim_red=True,
seed=None):
"""
Recognize noise from noise parameter name, and add to t2pulsar.
"""
added_noise_psd_params = dict()
flagid=list()
if 'f' in t2pulsar.flags():
backends = np.unique(t2pulsar.flagvals('f'))
flagid.append('f')
if 'g' in t2pulsar.flags(): #PPTA
backends = np.append(backends, np.unique(t2pulsar.flagvals('g')) )
flagid.append('g')
if 'sys' in t2pulsar.flags() and not 'group' in t2pulsar.flags():
backends = np.unique(t2pulsar.flagvals('sys'))
flagid.append('sys')
if 'sys' in t2pulsar.flags() and 'group' in t2pulsar.flags():
backends = np.unique(t2pulsar.flagvals('group'))
flagid.append('group')
if flagid==[]:
backends = []
raise Exception('Backend convention is not recognized')
used_backends=list()
for noise_param, noise_val in noise_dict.items():
if not np.isscalar(noise_val):
noise_val = noise_val[0]
noise_dict[noise_param] = noise_dict[noise_param][0]
backend_name = ''
param_name = ''
for bcknd in backends:
if bcknd in noise_param:
used_backends.append(bcknd)
backend_name = bcknd
for fid in flagid: # for 2 flags in PPTA
flagid_bcknd = ''
if backend_name in t2pulsar.flagvals(fid):
flagid_bcknd = fid
added_noise_psd_params.setdefault(backend_name,dict())
toaerr_bcknd = t2pulsar.toaerrs[ np.where(\
t2pulsar.flagvals(flagid_bcknd)==backend_name)[0] ]
added_noise_psd_params[backend_name]['rms_toaerr'] = \
(np.sum(toaerr_bcknd**2)/len(toaerr_bcknd))**(0.5)
added_noise_psd_params[backend_name]['mean_toaerr'] = \
np.mean(toaerr_bcknd)
if 'efac' in noise_param.lower() and sim_white:
param_name = 'efac'
if not backend_name == '':
#LT.add_efac(t2pulsar, efac=noise_val, seed=seed,\
# flags=backend_name, flagid=flagid_bcknd)
added_noise_psd_params[backend_name]['efac'] = noise_val
print('Added efac: ',noise_val,backend_name)
elif noise_param==t2pulsar.name+'_efac':
#LT.add_efac(t2pulsar, efac=noise_val, seed=seed)
added_noise_psd_params['efac'] = noise_val
print('Added efac: ',noise_val)
else:
raise Exception('Efac is not recognized. Neither signle, nor per \
backend. Parameter name from noise file: ', \
noise_param,'. Backends: ',backends)
elif 'log10_equad' in noise_param.lower() and sim_white:
param_name = 'log10_equad'
if not backend_name == '':
#LT.add_equad(t2pulsar, equad=10**noise_val, seed=seed, \
# flags=backend_name, flagid=flagid_bcknd)
added_noise_psd_params[backend_name]['equad'] = 10**noise_val
print('Added equad: ',10**noise_val,backend_name)
elif noise_param==t2pulsar.name+'_log10_equad':
#LT.add_equad(t2pulsar, equad=10**noise_val, seed=seed)
added_noise_psd_params['equad'] = 10**noise_val
print('Added efac: ',10**noise_val)
else:
raise Exception('Equad is not recognized. Neither signle, nor per \
backend. Parameter name from noise file: ', \
noise_param,'. Backends: ',backends)
elif 'log10_ecorr' in noise_param.lower() and sim_white:
param_name = 'log10_ecorr'
if not backend_name == '':
#LT.add_jitter(t2pulsar, ecorr=10**noise_val, seed=seed, \
# flags=backend_name, flagid=flagid_bcknd)
added_noise_psd_params[backend_name]['ecorr'] = 10**noise_val
print('Added ecorr: ',10**noise_val,backend_name)
elif noise_param==t2pulsar.name+'_log10_ecorr':
#LT.add_jitter(t2pulsar, ecorr=10**noise_val, seed=seed)
added_noise_psd_params['ecorr'] = 10**noise_val
print('Added efac: ',noise_val)
else:
raise Exception('Ecorr is not recognized. Neither signle, nor per backend. Parameter name from noise file: ',noise_param,'. Backends: ',backends)
elif 'dm_gp_log10_A' in noise_param and sim_dm:
param_name = 'dm_gp_log10_A'
#LT.add_dm(t2pulsar,A=10**noise_val,\
# gamma=noise_dict[t2pulsar.name+'_dm_gp_gamma'],\
# seed=seed,components=30)
added_noise_psd_params.setdefault('dm',dict())
added_noise_psd_params['dm']['A'] = 10**noise_val
print('Added DM noise, A=',10**noise_val,' gamma=',noise_dict[t2pulsar.name+'_dm_gp_gamma'])
elif 'dm_gp_gamma' in noise_param and sim_dm:
added_noise_psd_params.setdefault('dm',dict())
added_noise_psd_params['dm']['gamma'] = noise_val
param_name = 'dm_gp_gamma'
pass
elif noise_param==t2pulsar.name+'_log10_A' and sim_red:
param_name = 'log10_A'
#LT.add_rednoise(t2pulsar,A=10**noise_val,\
# gamma=noise_dict[t2pulsar.name+'_gamma'],\
# seed=seed,components=30)
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['A'] = 10**noise_val
print('Added red noise, A=',10**noise_val,' gamma=',noise_dict[t2pulsar.name+'_gamma'])
elif noise_param==t2pulsar.name+'_gamma':
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['gamma'] = noise_val
param_name = 'gamma'
pass
elif 'log10_P0' in noise_param:
param_name = 'log10_P0'
#LT_custom.add_lorenzianrednoise(t2pulsar,P=10**noise_val,\
# fc=10**noise_dict[t2pulsar.name+'_fc'],\
# alpha=noise_dict[t2pulsar.name+'_alpha'],\
# seed=seed)
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['P'] = 10**noise_val
print('Added red noise, P=',10**noise_val,' fc=',noise_dict[t2pulsar.name+'_fc'], ' alpha=',noise_dict[t2pulsar.name+'_alpha'])
elif 'alpha' in noise_param:
param_name = 'alpha'
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['alpha'] = noise_val
pass
elif 'fc' in noise_param:
param_name = 'fc'
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['fc'] = 10**noise_val
pass
else:
print('Warning: parameter ',noise_param,' is not recognized or \
switched off. It was not used to simulate any data.')
if sim_white:
vector_vals, vector_bckd = added_noise_psd_to_vector( \
added_noise_psd_params,param='efac')
LT.add_efac(t2pulsar, efac=vector_vals, seed=seed, flags=vector_bckd,
flagid=flagid_bcknd)
vector_vals, vector_bckd = added_noise_psd_to_vector( \
added_noise_psd_params,param='equad')
LT.add_equad(t2pulsar, equad=vector_vals, seed=seed, flags=vector_bckd,
flagid=flagid_bcknd)
if sim_red:
Ared = added_noise_psd_params['red']['A']
gred = added_noise_psd_params['red']['gamma']
LT.add_rednoise(t2pulsar,A=Ared,gamma=gred,seed=seed,components=30)
if sim_dm:
Adm = added_noise_psd_params['dm']['A']
gdm = added_noise_psd_params['dm']['gamma']
LT.add_dm(t2pulsar,A=Adm,gamma=gdm,seed=seed,components=30)
used_backends = np.unique(used_backends)
backends = np.unique(backends)
if len(backends)!=len(used_backends):
print('[!] Warning, number of not used backends: ',\
len(backends)-len(used_backends))
return t2pulsar, used_backends, added_noise_psd_params