def simulate_data(parfile, timfile, theta=0.05, idx=0, sigma_out=1e-6):
pt = t2.tempopulsar(parfile, timfile)
# draw error bars from log-normal distribution
err = 10**(-7 + np.random.randn(len(pt.stoas)) * 0.2) * 1e6
# fake pulsar
psr = ts.fakepulsar(parfile, pt.stoas[:], err)
# red noise
ts.add_rednoise(psr, 1e-14, 4.33, components=30)
# outlier
z = scipy.stats.binom.rvs(1, np.ones(len(psr.stoas)) * theta)
psr.stoas[:] += ((1 - z) * err * 1e-6 + z * sigma_out) * np.random.randn(
len(psr.stoas)) / 86400
ind = z.astype(bool)
outdir = 'simulated_data/outlier/{}/{}/'.format(theta, idx)
os.system('mkdir -p {}'.format(outdir))
np.savetxt('{}/outliers.txt'.format(outdir), np.flatnonzero(z), fmt='%d')
psr.savepar(outdir + '{}.par'.format(psr.name))
psr.savetim(outdir + '{}.tim'.format(psr.name))
outdir = 'simulated_data/no_outlier/{}/{}/'.format(theta, idx)
psr.deleted[ind] = 1
os.system('mkdir -p {}'.format(outdir))
psr.savepar(outdir + '{}.par'.format(psr.name))
psr.savetim(outdir + '{}.tim'.format(psr.name))
def make_fakes(amp, psrcut):
datadir = './data/partim/'
## Get parameter noise dictionary
noisedir = os.getcwd() + '/data/noisefiles/'
noise_params = {}
for filepath in glob.iglob(noisedir + '/*.json'):
with open(filepath, 'r') as f:
noise_params.update(json.load(f))
# print(noise_params)
# import par and tim files into parfiles and timfiles
tempopsr = []
print('Importing to libstempo....')
for psr in psrcut:
parfiles = sorted(glob.glob(datadir + psr.name + '*.par'))
timfiles = sorted(glob.glob(datadir + psr.name + '*.tim'))
psr = libs.tempopulsar(parfile=parfiles[0],
timfile=timfiles[0],
maxobs=50000)
tempopsr.append(psr)
print('adding efac')
for psr in tempopsr:
LT.make_ideal(psr)
LT.add_efac(psr, efac=1.0)
print('adding red noise')
for psr in tempopsr:
# print(psr.name)
rednoise_toadd = noise_params[str(psr.name) + '_log10_A']
# print(psr.name)
gamma = noise_params[str(psr.name) + '_gamma']
LT.add_rednoise(psr, 10**rednoise_toadd, gamma, components=30)
psr.fit()
print('adding GWB')
LT.createGWB(tempopsr, Amp=amp, gam=13. / 3.)
for psr in tempopsr:
psr.fit()
# save the pulsars
pardir = './fakes_gwb/par/'
timdir = './fakes_gwb/tim/'
if not os.path.exists(pardir):
os.makedirs(pardir)
os.makedirs(timdir)
print('fixing tim files')
for psr in tempopsr:
psr.savepar(pardir + psr.name + '-sim.par')
psr.savetim(timdir + psr.name + '-sim.tim')
libs.purgetim(timdir + psr.name + '-sim.tim') # fix the tim files
def add_rn(self, rn_psrs, seeds=None):
"""
Add rednoise to a subset of the pulsars.
Parameters
----------
rn_psrs : dict
Dictionary of rednoise parameter entries. The keys are pulsar names
while each entry is an array/list/tuple of RN amplitude, RN spectral
index.
"""
if seeds is None:
seeds=[None for ky in rn_psrs.keys()]
for ii, (p, pars) in enumerate(rn_psrs.items()):
A = pars[0]
gamma = pars[1]
pidx = self.pnames.index(p)
LT.add_rednoise(self.libs_psrs[pidx], A, gamma,
components=30, seed=seeds[ii])
tempopsr.append(psr)
# remove all noise from the pulsars.
for psr in tempopsr:
print('Adding rednoise to ' + psr.name)
efac_keys, efac_vals, equad_keys, equad_vals, red_noise_gamma_val, red_noise_amp_val = get_noise(
psr.name)
if args.average_epoch:
efac_val = np.mean(efac_vals)
equad_val = np.mean(equad_vals)
LT.add_efac(psr, efac=efac_val)
LT.add_equad(psr, equad=10**equad_val)
LT.add_rednoise(psr, 10**red_noise_amp_val, red_noise_gamma_val)
else:
for i in range(len(equad_vals)):
equad_vals[i] = 10**equad_vals[i]
print(equad_vals)
LT.make_ideal(psr)
LT.add_efac(psr, efac=efac_vals, flags=efac_keys, flagid='f')
LT.add_equad(psr, equad=equad_vals, flags=equad_keys, flagid='f')
LT.add_rednoise(psr, 10**red_noise_amp_val, red_noise_gamma_val)
## Inject fdm.
if args.inject_fdm:
print("Injecting Fuzzy Dark Matter!")
def create_dataset(dataset, Agwb):
""" Create simulated dataset using 18 pulsars from NANOGrav 9-year
stochastic analysis. Will use 11-year data span and red noise values
with white noise values taken from most recent time-to-detection
simulations.
:param dataset: Name of output dataset.
:param Agwb: Amplitude of injected dipole signal
"""
print 'Getting pulsar parameters for simulated dataset...'
# get simulation data
with open('nano9_simdata.json', 'r') as fp:
pdict = json.load(fp)
# get red noise dictionary
with open('nano_red_dict.json', 'r') as fp:
red_dict = json.load(fp)
# get parfiles
parfiles = glob.glob('../data/nano9_stipped_parfiles/*.par')
parfiles = [
p for p in parfiles if p.split('/')[-1].split('_')[0] in pdict.keys()
]
datadir = '../data/simulated_data/' + dataset
if not os.path.exists(datadir):
os.makedirs(datadir)
print 'Making simulated data in directory ' + datadir
psrs = []
for pf in parfiles:
pname = pf.split('/')[-1].split('_')[0]
psrs.append(LT.fakepulsar(pf, pdict[pname][0], pdict[pname][1]))
for psr in psrs:
# white noise
LT.add_efac(psr)
# red noise
if pname in red_dict:
LT.add_rednoise(psr,
red_dict[pname][0],
red_dict[pname][1],
components=30)
# dipole signal
createdipole(psrs, Agwb, 13. / 3., seed=None)
for psr in psrs:
psr.fit(iters=2)
## Save par and tim files
psr.savepar(datadir + '/{0}_optstatsim.par'.format(psr.name))
psr.savetim(datadir + '/{0}_optstatsim.tim'.format(psr.name))
# make the hdf5 file for the simulated dataset
print 'Making the hdf5 file for the simulated dataset...'
datadir = '../data/simulated_data/' + dataset
h5filename = datadir + '/sim.hdf5'
os.system('python makeH5file.py \
--pardir {0} --timdir {0} \
--h5File {1}'.format(datadir, h5filename))
print 'Finished!'
## add jitter
try: #Only NANOGrav Pulsars have ECORR
LT.add_jitter(p,
ecorr=noise_dict[p.name]['ecorrs'][:, 1],
flagid='f',
flags=noise_dict[p.name]['ecorrs'][:, 0],
coarsegrain=1.0 / 86400.0,
seed=seed_jitter + ii)
except KeyError:
pass
## add red noise
LT.add_rednoise(p,
noise_dict[p.name]['RN_Amp'],
noise_dict[p.name]['RN_gamma'],
components=30,
seed=seed_red + ii)
print(ii, p.name)
# Create GWB
# Takes a list of libstempo pulsar objects as input.
LT.createGWB(t2psr, Amp=1.3e-15, gam=13. / 3., seed=seed_gwb_1)
LT.createGWB(t2psr, Amp=1.0e-15, gam=7. / 3., seed=seed_gwb_2)
psrs = []
for p in t2psr:
psrs.append(Pulsar(p))
#Save sim pulsars to a pickle file
for ii in range(len(psr)):
white_noise = np.ones(len((psr[ii].toaerrs).copy()))
for jj in range(len(backends[ii])):
white_noise[backends[ii][jj]] *= EQUAD_ML[ii][jj]
psr[ii].stoas[:] += (white_noise / day) * np.random.randn(psr[ii].nobs)
##############################################################################################################################
# Now adding back red noise and DM-variation noise (libstempo has the ST format DM amplitude)
##############################################################################################################################
for ii in range(len(psr)):
LT.add_rednoise(psr[ii],Ared_ML[ii],gam_red_ML[ii],components=args.nmodes)
LT.add_dm(psr[ii],Adm_ML[ii],gam_dm_ML[ii],args.nmodes)
###############################################################
# Do we want to add in a GWB signal?
###############################################################
gwb = GWB(ngw=10000,flow=1e-11,fhigh=1e-5,gwAmp=args.gwamp,alpha=args.gwalpha)
psrDist = np.ones(len(psr)) # positions all pulsars at 1kpc as default
for ii in range(len(psr)):
gwb.add_gwb(psr[ii],psrDist[ii])
###############################################################
# Finally, let's save all the resulting .par and .tim files
def add_noise(t2pulsar, noise_dict, sim_dm=True, sim_white=True, sim_red=True,
seed=None):
"""
Recognize noise from noise parameter name, and add to t2pulsar.
"""
added_noise_psd_params = dict()
flagid=list()
if 'f' in t2pulsar.flags():
backends = np.unique(t2pulsar.flagvals('f'))
flagid.append('f')
if 'g' in t2pulsar.flags(): #PPTA
backends = np.append(backends, np.unique(t2pulsar.flagvals('g')) )
flagid.append('g')
if 'sys' in t2pulsar.flags() and not 'group' in t2pulsar.flags():
backends = np.unique(t2pulsar.flagvals('sys'))
flagid.append('sys')
if 'sys' in t2pulsar.flags() and 'group' in t2pulsar.flags():
backends = np.unique(t2pulsar.flagvals('group'))
flagid.append('group')
if flagid==[]:
backends = []
raise Exception('Backend convention is not recognized')
used_backends=list()
for noise_param, noise_val in noise_dict.items():
if not np.isscalar(noise_val):
noise_val = noise_val[0]
noise_dict[noise_param] = noise_dict[noise_param][0]
backend_name = ''
param_name = ''
for bcknd in backends:
if bcknd in noise_param:
used_backends.append(bcknd)
backend_name = bcknd
for fid in flagid: # for 2 flags in PPTA
flagid_bcknd = ''
if backend_name in t2pulsar.flagvals(fid):
flagid_bcknd = fid
added_noise_psd_params.setdefault(backend_name,dict())
toaerr_bcknd = t2pulsar.toaerrs[ np.where(\
t2pulsar.flagvals(flagid_bcknd)==backend_name)[0] ]
added_noise_psd_params[backend_name]['rms_toaerr'] = \
(np.sum(toaerr_bcknd**2)/len(toaerr_bcknd))**(0.5)
added_noise_psd_params[backend_name]['mean_toaerr'] = \
np.mean(toaerr_bcknd)
if 'efac' in noise_param.lower() and sim_white:
param_name = 'efac'
if not backend_name == '':
#LT.add_efac(t2pulsar, efac=noise_val, seed=seed,\
# flags=backend_name, flagid=flagid_bcknd)
added_noise_psd_params[backend_name]['efac'] = noise_val
print('Added efac: ',noise_val,backend_name)
elif noise_param==t2pulsar.name+'_efac':
#LT.add_efac(t2pulsar, efac=noise_val, seed=seed)
added_noise_psd_params['efac'] = noise_val
print('Added efac: ',noise_val)
else:
raise Exception('Efac is not recognized. Neither signle, nor per \
backend. Parameter name from noise file: ', \
noise_param,'. Backends: ',backends)
elif 'log10_equad' in noise_param.lower() and sim_white:
param_name = 'log10_equad'
if not backend_name == '':
#LT.add_equad(t2pulsar, equad=10**noise_val, seed=seed, \
# flags=backend_name, flagid=flagid_bcknd)
added_noise_psd_params[backend_name]['equad'] = 10**noise_val
print('Added equad: ',10**noise_val,backend_name)
elif noise_param==t2pulsar.name+'_log10_equad':
#LT.add_equad(t2pulsar, equad=10**noise_val, seed=seed)
added_noise_psd_params['equad'] = 10**noise_val
print('Added efac: ',10**noise_val)
else:
raise Exception('Equad is not recognized. Neither signle, nor per \
backend. Parameter name from noise file: ', \
noise_param,'. Backends: ',backends)
elif 'log10_ecorr' in noise_param.lower() and sim_white:
param_name = 'log10_ecorr'
if not backend_name == '':
#LT.add_jitter(t2pulsar, ecorr=10**noise_val, seed=seed, \
# flags=backend_name, flagid=flagid_bcknd)
added_noise_psd_params[backend_name]['ecorr'] = 10**noise_val
print('Added ecorr: ',10**noise_val,backend_name)
elif noise_param==t2pulsar.name+'_log10_ecorr':
#LT.add_jitter(t2pulsar, ecorr=10**noise_val, seed=seed)
added_noise_psd_params['ecorr'] = 10**noise_val
print('Added efac: ',noise_val)
else:
raise Exception('Ecorr is not recognized. Neither signle, nor per backend. Parameter name from noise file: ',noise_param,'. Backends: ',backends)
elif 'dm_gp_log10_A' in noise_param and sim_dm:
param_name = 'dm_gp_log10_A'
#LT.add_dm(t2pulsar,A=10**noise_val,\
# gamma=noise_dict[t2pulsar.name+'_dm_gp_gamma'],\
# seed=seed,components=30)
added_noise_psd_params.setdefault('dm',dict())
added_noise_psd_params['dm']['A'] = 10**noise_val
print('Added DM noise, A=',10**noise_val,' gamma=',noise_dict[t2pulsar.name+'_dm_gp_gamma'])
elif 'dm_gp_gamma' in noise_param and sim_dm:
added_noise_psd_params.setdefault('dm',dict())
added_noise_psd_params['dm']['gamma'] = noise_val
param_name = 'dm_gp_gamma'
pass
elif noise_param==t2pulsar.name+'_log10_A' and sim_red:
param_name = 'log10_A'
#LT.add_rednoise(t2pulsar,A=10**noise_val,\
# gamma=noise_dict[t2pulsar.name+'_gamma'],\
# seed=seed,components=30)
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['A'] = 10**noise_val
print('Added red noise, A=',10**noise_val,' gamma=',noise_dict[t2pulsar.name+'_gamma'])
elif noise_param==t2pulsar.name+'_gamma':
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['gamma'] = noise_val
param_name = 'gamma'
pass
elif 'log10_P0' in noise_param:
param_name = 'log10_P0'
#LT_custom.add_lorenzianrednoise(t2pulsar,P=10**noise_val,\
# fc=10**noise_dict[t2pulsar.name+'_fc'],\
# alpha=noise_dict[t2pulsar.name+'_alpha'],\
# seed=seed)
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['P'] = 10**noise_val
print('Added red noise, P=',10**noise_val,' fc=',noise_dict[t2pulsar.name+'_fc'], ' alpha=',noise_dict[t2pulsar.name+'_alpha'])
elif 'alpha' in noise_param:
param_name = 'alpha'
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['alpha'] = noise_val
pass
elif 'fc' in noise_param:
param_name = 'fc'
added_noise_psd_params.setdefault('red',dict())
added_noise_psd_params['red']['fc'] = 10**noise_val
pass
else:
print('Warning: parameter ',noise_param,' is not recognized or \
switched off. It was not used to simulate any data.')
if sim_white:
vector_vals, vector_bckd = added_noise_psd_to_vector( \
added_noise_psd_params,param='efac')
LT.add_efac(t2pulsar, efac=vector_vals, seed=seed, flags=vector_bckd,
flagid=flagid_bcknd)
vector_vals, vector_bckd = added_noise_psd_to_vector( \
added_noise_psd_params,param='equad')
LT.add_equad(t2pulsar, equad=vector_vals, seed=seed, flags=vector_bckd,
flagid=flagid_bcknd)
if sim_red:
Ared = added_noise_psd_params['red']['A']
gred = added_noise_psd_params['red']['gamma']
LT.add_rednoise(t2pulsar,A=Ared,gamma=gred,seed=seed,components=30)
if sim_dm:
Adm = added_noise_psd_params['dm']['A']
gdm = added_noise_psd_params['dm']['gamma']
LT.add_dm(t2pulsar,A=Adm,gamma=gdm,seed=seed,components=30)
used_backends = np.unique(used_backends)
backends = np.unique(backends)
if len(backends)!=len(used_backends):
print('[!] Warning, number of not used backends: ',\
len(backends)-len(used_backends))
return t2pulsar, used_backends, added_noise_psd_params