def setUpClass(cls):
cls.obstimes = np.arange(53000, 54800, 10, dtype=np.float128)
cls.toaerr = 1e-3
cls.freq = 1440.0
cls.observatory = "ao"
cls.parfile = DATA_PATH + "/J1909-3744_NANOGrav_dfg+12.par"
# create a fake pulsar using fakepulsar
cls.fakepsr = fakepulsar(
parfile=cls.parfile,
obstimes=cls.obstimes,
toaerr=cls.toaerr,
freq=cls.freq,
observatory=cls.observatory,
iters=0,
)
# create a fake pulsar using tempopulsar
cls.fakepsrtp = t2.tempopulsar(
parfile=cls.parfile,
toas=cls.obstimes,
toaerrs=cls.toaerr,
observatory=cls.observatory,
obsfreq=cls.freq,
dofit=False,
)
def simulate_data(parfile, timfile, theta=0.05, idx=0, sigma_out=1e-6):
pt = t2.tempopulsar(parfile, timfile)
# draw error bars from log-normal distribution
err = 10**(-7 + np.random.randn(len(pt.stoas)) * 0.2) * 1e6
# fake pulsar
psr = ts.fakepulsar(parfile, pt.stoas[:], err)
# red noise
ts.add_rednoise(psr, 1e-14, 4.33, components=30)
# outlier
z = scipy.stats.binom.rvs(1, np.ones(len(psr.stoas)) * theta)
psr.stoas[:] += ((1 - z) * err * 1e-6 + z * sigma_out) * np.random.randn(
len(psr.stoas)) / 86400
ind = z.astype(bool)
outdir = 'simulated_data/outlier/{}/{}/'.format(theta, idx)
os.system('mkdir -p {}'.format(outdir))
np.savetxt('{}/outliers.txt'.format(outdir), np.flatnonzero(z), fmt='%d')
psr.savepar(outdir + '{}.par'.format(psr.name))
psr.savetim(outdir + '{}.tim'.format(psr.name))
outdir = 'simulated_data/no_outlier/{}/{}/'.format(theta, idx)
psr.deleted[ind] = 1
os.system('mkdir -p {}'.format(outdir))
psr.savepar(outdir + '{}.par'.format(psr.name))
psr.savetim(outdir + '{}.tim'.format(psr.name))
def make_fake_pulsar(source, pulsar_name, obs_times, toa_err=1e-6):
"""
Make an LT fakepulsar
Parameters
----------
source: str
path to pulsars with .par files in 'pulsar'/'pulsar'.IPTADR2.pa
pulsar_name: str
name of the pulsar (is also the directory with files in source)
obs_times: array-like
times of observation in MJD
toa_err: float
toa error in us
Returns
-------
LT.fakepulsar object
"""
par_path = os.path.join(source, pulsar_name, pulsar_name + '.IPTADR2.par')
return LT.fakepulsar(par_path, obs_times, toa_err)
def init_prediction_psr(self, parfile, nobs=100):
"""Initialize a fake pulsar, just for prediction purposes
Initialize a fake pulsar, with 100 observations (for speed)
:param parfile:
Parfile to base the pulsar on
"""
# TODO: Make this work with PINT!
obstimes = np.linspace(54000, 55000, nobs)
self.predpsr_nobs = nobs
self.predpsr = lts.fakepulsar(parfile, obstimes, 0.1, observatory='ao')
# Remove JUMP parameters
for par in self.predpsr.pars(which='set'):
if par[:4] == 'JUMP':
self.predpsr[par].val = 0.0
self.predpsr[par].err = 0.0
#self.predpsr[par].set = False # TODO: Cannot unset jumps
self.predpsr[par].fit = False
lts.make_ideal(self.predpsr)
try:
os.makedirs(outdir)
except OSError:
pass
ct = 0
for d, p in zip(delays, parfiles):
print 'Using {0} delay with parfile {1}.'.format(d.split('/')[-1], p.split('/')[-1])
d = np.loadtxt(d)
toas, res = d[:,0], d[:,1]
err = np.ones(len(toas)) * args.rms * 1e-3
# make fake observations
psr = ts.fakepulsar(p, toas, err)
# makes perfect residuals (i.e. no noise)
ts.make_ideal(psr)
# add GWB delays
psr.stoas[:] += res
# adds white noise at level of defined uncertainties above
ts.add_efac(psr, 1, seed=(args.seed*ct))
# fit
psr.fit()
# save par and tim file
psr.savepar(outdir + '/' + psr.name + '.par')
tempopsr = []
## Loop through the pulsars and make each a tempo2 object.
for i in range(len(psrcut_parfiles)):
psr = libstempo.tempopulsar(parfile=psrcut_parfiles[i],
timfile=psrcut_timfiles[i],
maxobs=50000)
print('Creating Tempo object for ' + psr.name)
output_params.write(psr.name + '\n')
if args.average_epoch:
print("This psr has " + str(len(psr.toas())) +
" observations before averaging.")
toas, res, err = epoch_average(psr, dt=args.average_epoch)
fp = LT.fakepulsar(psrcut_parfiles[i], toas, err)
fp.stoas[:] += res / 86400
print("This psr has " + str(len(fp.toas())) +
" observations after averaging.")
fp.formbats()
tempopsr.append(fp)
else:
tempopsr.append(psr)
# remove all noise from the pulsars.
for psr in tempopsr:
print('Adding rednoise to ' + psr.name)
efac_keys, efac_vals, equad_keys, equad_vals, red_noise_gamma_val, red_noise_amp_val = get_noise(
def create_dataset(dataset, Agwb):
""" Create simulated dataset using 18 pulsars from NANOGrav 9-year
stochastic analysis. Will use 11-year data span and red noise values
with white noise values taken from most recent time-to-detection
simulations.
:param dataset: Name of output dataset.
:param Agwb: Amplitude of injected dipole signal
"""
print 'Getting pulsar parameters for simulated dataset...'
# get simulation data
with open('nano9_simdata.json', 'r') as fp:
pdict = json.load(fp)
# get red noise dictionary
with open('nano_red_dict.json', 'r') as fp:
red_dict = json.load(fp)
# get parfiles
parfiles = glob.glob('../data/nano9_stipped_parfiles/*.par')
parfiles = [
p for p in parfiles if p.split('/')[-1].split('_')[0] in pdict.keys()
]
datadir = '../data/simulated_data/' + dataset
if not os.path.exists(datadir):
os.makedirs(datadir)
print 'Making simulated data in directory ' + datadir
psrs = []
for pf in parfiles:
pname = pf.split('/')[-1].split('_')[0]
psrs.append(LT.fakepulsar(pf, pdict[pname][0], pdict[pname][1]))
for psr in psrs:
# white noise
LT.add_efac(psr)
# red noise
if pname in red_dict:
LT.add_rednoise(psr,
red_dict[pname][0],
red_dict[pname][1],
components=30)
# dipole signal
createdipole(psrs, Agwb, 13. / 3., seed=None)
for psr in psrs:
psr.fit(iters=2)
## Save par and tim files
psr.savepar(datadir + '/{0}_optstatsim.par'.format(psr.name))
psr.savetim(datadir + '/{0}_optstatsim.tim'.format(psr.name))
# make the hdf5 file for the simulated dataset
print 'Making the hdf5 file for the simulated dataset...'
datadir = '../data/simulated_data/' + dataset
h5filename = datadir + '/sim.hdf5'
os.system('python makeH5file.py \
--pardir {0} --timdir {0} \
--h5File {1}'.format(datadir, h5filename))
print 'Finished!'
def create_gw_delays(parfile, popfile, T=10, cadence=26, seed=55, outdir='./'):
"""
Create a file with delays due to the summed GW sources
from a population.
:param pardir: parfile to be used for simulation
:param popfile: name of the poulation file from Alberto's sims
:param T: Timespan of simulated observations [yr]
:param cadence: Cadence of observations [pts/yr]
:param seed: Random number seed for binary orientation parameters
:param outdir: Output directory to place delay files
"""
if not os.path.exists(outdir):
try:
os.makedirs(outdir)
except OSError:
pass
# read in data
data = np.loadtxt(popfile)
# get number of sources from file
ns = data.shape[0]
# set random number seed and draw orientation parameters
np.random.seed(seed)
theta = np.arccos(np.random.uniform(-1, 1, ns))
phi = np.random.uniform(0, 2*np.pi, ns)
psi = np.random.uniform(0, np.pi, ns)
l0 = np.random.uniform(0, 2*np.pi, ns)
gamma0 = np.random.uniform(0, 2*np.pi, ns)
###### initialize pulsar class #####
# toas (evenly spaced)
toas = np.linspace(0, T*3.16e7, int(T*cadence))
toas /= 86400 # need to be in units of days
toas += 53000 # start at MJD 53000
# uncertainties (mu s) Don't matter here but need them for toasim
err = np.ones(len(toas)) * 0.05
# make fake observations
psr = ts.fakepulsar(parfile, toas, err)
# ideal toas
ts.make_ideal(psr)
# setup index array to grab values from datafile
ind = np.array([0, 1, 2, 3, 4, 8])
# write seed, T, and cadence to file
np.savetxt(outdir + 'setup.txt', np.array([seed, T, cadence]))
# open output file for source parameters
fout = open(outdir + '/sources.txt', 'w')
# loop over sources
for ii in range(ns):
# get values from file
lmc, q, z, lf, e, inc = data[ii, ind]
# get redshifted values
mc, dl = get_redshifted_values(10**lmc, z)
F = 10**lf/2
# injection
ts.add_ecc_cgw(psr, theta[ii], phi[ii], mc, dl, F, inc,
psi[ii], gamma0[ii], e, l0[ii], q,
nmax=100, pd=1.0, psrTerm=True,
tref=53000*86400, check=False,
useFile=True)
# write parameters to file
fout.write('%g %g %g %g %g %g %g %g %g %g %g\n' %
(theta[ii], phi[ii], np.log10(mc), np.log10(dl),
np.log10(F), inc, psi[ii], gamma0[ii], e,
l0[ii], q))
sys.stdout.write('\r')
sys.stdout.write('Finished %2.2f percent of Injections.' % ((ii+1)/ns * 100))
sys.stdout.flush()
fout.close()
# save GW delays
np.savetxt(outdir + '/' + psr.name + '_delay.txt',
np.array([psr.stoas[:], psr.residuals()/86400]).T)
psrlistpath = topdir + '/nano11/psrlist_Tg3yr.txt'
J1909_par = parpath + 'J1909-3744' + '_NANOGrav_11yv0.gls.par'
J1909_tim = timpath + 'J1909-3744' + '_NANOGrav_11yv0.tim'
J1909 = T2.tempopulsar(parfile=J1909_par,
timfile=J1909_tim,
maxobs=30000,
ephem='DE436',
clk=None)
t = np.arange(53000, 56650, 30.0) #observing dates for 10 years
t += np.random.randn(len(t)) #observe every 30+/-1 days
J0613_par = parpath + 'J0613-0200' + '_NANOGrav_11yv0.gls.par'
fake_J0613 = LT.fakepulsar(J0613_par, obstimes=t, toaerr=0.5)
encoding = "utf-8"
psrlist_bytes = np.loadtxt(psrlistpath, dtype='S42')
psrlist = []
for psr in psrlist_bytes:
psrlist.append(psr.decode(encoding))
parfiles = sorted(glob.glob(parpath + '*.par'))
timfiles = sorted(glob.glob(timpath + '*.tim'))
noisefiles = sorted(glob.glob(noisepath + '*.txt'))
parfiles = [
x for x in parfiles
if x.split('/')[-1].split('.')[0].split('_')[0] in psrlist
]
###################################################################################
for ranseed in seeds:
for rho in snrs:
for forb in orb_freqs:
for e0 in eccs:
# Define sampling and add noise
psr_noise = []
psr_signal = []
for ii, p in enumerate(psr):
fake_obs = tsamp[p.name]
psr_noise.append(
LT.fakepulsar(parfile=ideal_par_file_dict[p.name],
obstimes=fake_obs,
toaerr=active_psrs[p.name][2]))
psr_signal.append(
LT.fakepulsar(parfile=ideal_par_file_dict[p.name],
obstimes=fake_obs,
toaerr=active_psrs[p.name][2]))
tref_fake = np.min([p.toas().min() for p in psr_noise
]) * 86400.0 # needs to be in seconds
print "--> Performing SVD of design matrix to get G-matrix..."
Gcmat = []
for ii, p in enumerate(psr_signal):
u, s, v = np.linalg.svd(p.designmatrix())
Gcmat.append(u[:, :len(s)])
###################################################################################
for ranseed in seeds:
for rho in snrs:
for forb in orb_freqs:
for e0 in eccs:
# Define sampling and add noise
psr_noise=[]
psr_signal=[]
for ii,p in enumerate(psr):
fake_obs = tsamp[p.name]
psr_noise.append(LT.fakepulsar(parfile=ideal_par_file_dict[p.name],
obstimes = fake_obs,
toaerr = active_psrs[p.name][2]))
psr_signal.append(LT.fakepulsar(parfile=ideal_par_file_dict[p.name],
obstimes = fake_obs,
toaerr = active_psrs[p.name][2]))
tref_fake = np.min([p.toas().min() for p in psr_noise])*86400.0 # needs to be in seconds
print "--> Performing SVD of design matrix to get G-matrix..."
Gcmat = []
for ii,p in enumerate(psr_signal):
u,s,v = np.linalg.svd(p.designmatrix())
Gcmat.append(u[:,:len(s)])