def addpulsar(self, parfile, timfile, DMOFF=None, DMXOFF=None, dailyAverage=False): """ Add another pulsar to the HDF5 file, given a tempo2 par and tim file. @param parfile: tempo2 par file @param timfile: tempo2 tim file @param DMOFF: Option to turn off DMMODEL fitting @param DMOFF: Option to turn off DMX fitting @param dailyAverage: Option to perform daily averaging to reduce the number of points by consructing daily averaged TOAs that have one residual per day per frequency band. (This has only been tested on NANOGrav data thus far.) """ # Check whether the two files exist if not os.path.isfile(parfile) or not os.path.isfile(timfile): raise IOError, "Cannot find parfile (%s) or timfile (%s)!" % (parfile, timfile) assert(self.filename != None), "ERROR: HDF5 file not set!" # 'a' means: read/write if exists, create otherwise self.h5file = h5.File(self.filename, 'a') if "Model" in self.h5file: self.h5file.close() self.h5file = None raise IOError, "model already available in '%s'. Refusing to add data" % (self.filename) # Create the data subgroup if it does not exist if "Data" in self.h5file: datagroup = self.h5file["Data"] else: datagroup = self.h5file.create_group("Data") # Load pulsar data from the JPL Cython tempo2 library t2pulsar = t2.tempopulsar(parfile, timfile) # do multiple fits #t2pulsar.fit(iters=10) # turn off DMMODEL fitting if DMOFF is not None: t2pulsar['DMMODEL'].fit = False # turn off DMX fitting if DMXOFF is not None: DMXFlag = False print 'Turning off DMX fitting and turning DM fitting on' for par in t2pulsar.pars: if 'DMX' in par: print par t2pulsar[par].fit = False t2pulsar['DM'].fit = True DMXFlag = True if DMXFlag== False: print 'NO DMX for pulsar {0}'.format(t2pulsar.name) # refit 5 times to make sure we are converged t2pulsar.fit(iters=5) # Create the pulsar subgroup if it does not exist if "Pulsars" in datagroup: pulsarsgroup = datagroup["Pulsars"] else: pulsarsgroup = datagroup.create_group("Pulsars") # Look up the name of the pulsar, and see if it exist if t2pulsar.name in pulsarsgroup: self.h5file.close() raise IOError, "%s already exists in %s!" % (t2pulsar.name, self.filename) pulsarsgroup = pulsarsgroup.create_group(t2pulsar.name) # Read the data from the tempo2 structure. designmatrix = np.double(t2pulsar.designmatrix()) residuals = np.double(t2pulsar.residuals()) toas = np.double(t2pulsar.toas()) errs = np.double(t2pulsar.toaerrs*1e-6) pname = t2pulsar.name try: # if tim file has frequencies freqs = np.double(t2pulsar.freqs) except AttributeError: freqs = 0 try: # if tim file has frequency band flags bands = t2pulsar.flags['B'] except KeyError: bands = 0 # if doing daily averaging if dailyAverage: # get average quantities toas, qmatrix, errs, dmatrix, freqs, bands = PALutils.dailyAverage(t2pulsar) # construct new daily averaged residuals and designmatrix residuals = np.dot(qmatrix, residuals) designmatrix = np.dot(qmatrix, dmatrix) # Write the TOAs, residuals, and uncertainties. spd = 24.0*3600 # seconds per day pulsarsgroup.create_dataset('TOAs', data = toas*spd) # days (MJD) * sec per day pulsarsgroup.create_dataset('residuals', data = residuals) # seconds pulsarsgroup.create_dataset('toaErr', data = errs) # seconds pulsarsgroup.create_dataset('freqs', data = freqs*1e6) # Hz pulsarsgroup.create_dataset('bands', data = bands) # Hz # add tim and par file paths pulsarsgroup.create_dataset('parFile', data = parfile) # string pulsarsgroup.create_dataset('timFile', data = timfile) # string # Write the full design matrix pulsarsgroup.create_dataset('designmatrix', data = designmatrix) # Obtain the timing model parameters tmpname = np.array(t2pulsar.pars) tmpvalpre = np.double([t2pulsar.prefit[parname].val for parname in t2pulsar.pars]) tmpvalpost = np.double([t2pulsar[parname].val for parname in t2pulsar.pars]) tmperrpre = np.double([t2pulsar.prefit[parname].err for parname in t2pulsar.pars]) tmperrpost = np.double([t2pulsar[parname].err for parname in t2pulsar.pars]) # Write the timing model parameter (TMP) descriptions pulsarsgroup.create_dataset('pname', data=pname) # pulsar name pulsarsgroup.create_dataset('tmp_name', data=tmpname) # TMP name pulsarsgroup.create_dataset('tmp_valpre', data=tmpvalpre) # TMP pre-fit value pulsarsgroup.create_dataset('tmp_valpost', data=tmpvalpost) # TMP post-fit value pulsarsgroup.create_dataset('tmp_errpre', data=tmperrpre) # TMP pre-fit error pulsarsgroup.create_dataset('tmp_errpost', data=tmperrpost) # TMP post-fit error # Close the hdf5 file self.h5file.close()
def addpulsar(self, parfile, timfile, DMOFF=None, DMXOFF=None, dailyAverage=False): """ Add another pulsar to the HDF5 file, given a tempo2 par and tim file. @param parfile: tempo2 par file @param timfile: tempo2 tim file @param DMOFF: Option to turn off DMMODEL fitting @param DMOFF: Option to turn off DMX fitting @param dailyAverage: Option to perform daily averaging to reduce the number of points by consructing daily averaged TOAs that have one residual per day per frequency band. (This has only been tested on NANOGrav data thus far.) """ # Check whether the two files exist if not os.path.isfile(parfile) or not os.path.isfile(timfile): raise IOError, "Cannot find parfile (%s) or timfile (%s)!" % ( parfile, timfile) assert (self.filename != None), "ERROR: HDF5 file not set!" # 'a' means: read/write if exists, create otherwise self.h5file = h5.File(self.filename, 'a') if "Model" in self.h5file: self.h5file.close() self.h5file = None raise IOError, "model already available in '%s'. Refusing to add data" % ( self.filename) # Create the data subgroup if it does not exist if "Data" in self.h5file: datagroup = self.h5file["Data"] else: datagroup = self.h5file.create_group("Data") # Load pulsar data from the JPL Cython tempo2 library t2pulsar = t2.tempopulsar(parfile, timfile) # do multiple fits #t2pulsar.fit(iters=10) # turn off DMMODEL fitting if DMOFF is not None: t2pulsar['DMMODEL'].fit = False # turn off DMX fitting if DMXOFF is not None: DMXFlag = False print 'Turning off DMX fitting and turning DM fitting on' for par in t2pulsar.pars: if 'DMX' in par: print par t2pulsar[par].fit = False t2pulsar['DM'].fit = True DMXFlag = True if DMXFlag == False: print 'NO DMX for pulsar {0}'.format(t2pulsar.name) # refit 5 times to make sure we are converged t2pulsar.fit(iters=5) # Create the pulsar subgroup if it does not exist if "Pulsars" in datagroup: pulsarsgroup = datagroup["Pulsars"] else: pulsarsgroup = datagroup.create_group("Pulsars") # Look up the name of the pulsar, and see if it exist if t2pulsar.name in pulsarsgroup: self.h5file.close() raise IOError, "%s already exists in %s!" % (t2pulsar.name, self.filename) pulsarsgroup = pulsarsgroup.create_group(t2pulsar.name) # Read the data from the tempo2 structure. designmatrix = np.double(t2pulsar.designmatrix()) residuals = np.double(t2pulsar.residuals()) toas = np.double(t2pulsar.toas()) errs = np.double(t2pulsar.toaerrs * 1e-6) pname = t2pulsar.name try: # if tim file has frequencies freqs = np.double(t2pulsar.freqs) except AttributeError: freqs = 0 try: # if tim file has frequency band flags bands = t2pulsar.flags['B'] except KeyError: bands = 0 # if doing daily averaging if dailyAverage: # get average quantities toas, qmatrix, errs, dmatrix, freqs, bands = PALutils.dailyAverage( t2pulsar) # construct new daily averaged residuals and designmatrix residuals = np.dot(qmatrix, residuals) designmatrix = np.dot(qmatrix, dmatrix) # Write the TOAs, residuals, and uncertainties. spd = 24.0 * 3600 # seconds per day pulsarsgroup.create_dataset('TOAs', data=toas * spd) # days (MJD) * sec per day pulsarsgroup.create_dataset('residuals', data=residuals) # seconds pulsarsgroup.create_dataset('toaErr', data=errs) # seconds pulsarsgroup.create_dataset('freqs', data=freqs * 1e6) # Hz pulsarsgroup.create_dataset('bands', data=bands) # Hz # add tim and par file paths pulsarsgroup.create_dataset('parFile', data=parfile) # string pulsarsgroup.create_dataset('timFile', data=timfile) # string # Write the full design matrix pulsarsgroup.create_dataset('designmatrix', data=designmatrix) # Obtain the timing model parameters tmpname = np.array(t2pulsar.pars) tmpvalpre = np.double( [t2pulsar.prefit[parname].val for parname in t2pulsar.pars]) tmpvalpost = np.double( [t2pulsar[parname].val for parname in t2pulsar.pars]) tmperrpre = np.double( [t2pulsar.prefit[parname].err for parname in t2pulsar.pars]) tmperrpost = np.double( [t2pulsar[parname].err for parname in t2pulsar.pars]) # Write the timing model parameter (TMP) descriptions pulsarsgroup.create_dataset('pname', data=pname) # pulsar name pulsarsgroup.create_dataset('tmp_name', data=tmpname) # TMP name pulsarsgroup.create_dataset('tmp_valpre', data=tmpvalpre) # TMP pre-fit value pulsarsgroup.create_dataset('tmp_valpost', data=tmpvalpost) # TMP post-fit value pulsarsgroup.create_dataset('tmp_errpre', data=tmperrpre) # TMP pre-fit error pulsarsgroup.create_dataset('tmp_errpost', data=tmperrpost) # TMP post-fit error # Close the hdf5 file self.h5file.close()
def addInverseCovFromNoiseFile(self, parfile, timfile, noisefile, DMOFF=None, DMXOFF=None, dailyAverage=False): """ Add noise covariance matrix after timing model subtraction. """ # Check whether the two files exist if not os.path.isfile(parfile) or not os.path.isfile(timfile): raise IOError, "Cannot find parfile (%s) or timfile (%s)!" % (parfile, timfile) assert(self.filename != None), "ERROR: HDF5 file not set!" # 'a' means: read/write if exists, create otherwise self.h5file = h5.File(self.filename, 'a') # Create the data subgroup if it does not exist if "Data" in self.h5file: datagroup = self.h5file["Data"] else: raise IOError, "Cannot add noise parameters if Data group does not exist!" # Load pulsar data from the JPL Cython tempo2 library t2pulsar = t2.tempopulsar(parfile, timfile) # turn off DMMODEL fitting if DMOFF is not None: t2pulsar['DMMODEL'].fit = False # turn off DMX fitting if DMXOFF is not None: DMXFlag = False print 'Turning off DMX fitting and turning DM fitting on' for par in t2pulsar.pars: if 'DMX' in par: t2pulsar[par].fit = False t2pulsar['DM'].fit = True DMXFlag = True if DMXFlag== False: print 'NO DMX for pulsar {0}'.format(t2pulsar.name) # refit 5 times to make sure we are converged t2pulsar.fit(iters=5) # Create the pulsar subgroup if it does not exist if "Pulsars" in datagroup: pulsarsgroup = datagroup["Pulsars"] else: raise IOError, "Cannot add noise parameters if pulsar group does not exist!" # Look up the name of the pulsar, and see if it exist if t2pulsar.name in pulsarsgroup: pass else: raise IOError, "%s must already exists in %s to add noise parameters!"\ % (t2pulsar.name, self.filename) pulsarsgroup = pulsarsgroup[t2pulsar.name] # first create G matrix from design matrix and toas designmatrix = np.double(t2pulsar.designmatrix()) toas = np.double(t2pulsar.toas()*86400) errs = np.double(t2pulsar.toaerrs*1e-6) # if doing daily averaging if dailyAverage: # get average quantities toas, qmatrix, errs, dmatrix, freqs, bands = PALutils.dailyAverage(t2pulsar) # construct new daily averaged residuals and designmatrix toas *= 86400 designmatrix = np.dot(qmatrix, dmatrix) G = PALutils.createGmatrix(designmatrix) # create matrix of time lags tm = PALutils.createTimeLags(toas, toas, round=True) # now read noise file to get model and parameters file = open(noisefile,'r') fH = None tau = None DMAmp = None DMgam = None for line in file.readlines(): # default parameters for different models other than pure PL key = line.split()[0] # get amplitude if "Amp" == key: Amp = float(line.split()[-1]) # get spectral index elif "gam" == key: gam = float(line.split()[-1]) # get efac elif "efac" == key: efac = float(line.split()[-1]) # get quad elif "equad" == key: equad = float(line.split()[-1]) # get high frequency cutoff if available elif "fH" == key: fH = float(line.split()[-1]) # get correlation time scale if available elif "tau" == key: tau = float(line.split()[-1]) # get DM Amplitude if available elif "DMAmp" == key: DMAmp = float(line.split()[-1]) # get DM Spectral Index if available elif "DMgam" == key: DMgam = float(line.split()[-1]) # cosstruct red and white noise covariance matrices red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) white = PALutils.createWhiteNoiseCovarianceMatrix(errs, efac, equad, tau=tau) # construct post timing model marginalization covariance matrix cov = red + white pcov = np.dot(G.T, np.dot(cov, G)) # finally construct "inverse" invCov = np.dot(G, np.dot(np.linalg.inv(pcov), G.T)) # create dataset for inverse covariance matrix pulsarsgroup.create_dataset('invCov', data = invCov) # create dataset for G matrix pulsarsgroup.create_dataset('Gmatrix', data = G) # record noise parameter values pulsarsgroup.create_dataset('Amp', data = Amp) pulsarsgroup.create_dataset('gam', data = gam) pulsarsgroup.create_dataset('efac', data = efac) pulsarsgroup.create_dataset('equad', data = equad) if fH is not None: pulsarsgroup.create_dataset('fH', data = fH) if tau is not None: pulsarsgroup.create_dataset('tau', data = tau) if DMAmp is not None: pulsarsgroup.create_dataset('DMAmp', data = DMAmp) if DMgam is not None: pulsarsgroup.create_dataset('DMgam', data = DMgam) # Close the hdf5 file self.h5file.close()
def addInverseCovFromNoiseFile(self, parfile, timfile, noisefile, DMOFF=None, DMXOFF=None, dailyAverage=False): """ Add noise covariance matrix after timing model subtraction. """ # Check whether the two files exist if not os.path.isfile(parfile) or not os.path.isfile(timfile): raise IOError, "Cannot find parfile (%s) or timfile (%s)!" % ( parfile, timfile) assert (self.filename != None), "ERROR: HDF5 file not set!" # 'a' means: read/write if exists, create otherwise self.h5file = h5.File(self.filename, 'a') # Create the data subgroup if it does not exist if "Data" in self.h5file: datagroup = self.h5file["Data"] else: raise IOError, "Cannot add noise parameters if Data group does not exist!" # Load pulsar data from the JPL Cython tempo2 library t2pulsar = t2.tempopulsar(parfile, timfile) # turn off DMMODEL fitting if DMOFF is not None: t2pulsar['DMMODEL'].fit = False # turn off DMX fitting if DMXOFF is not None: DMXFlag = False print 'Turning off DMX fitting and turning DM fitting on' for par in t2pulsar.pars: if 'DMX' in par: t2pulsar[par].fit = False t2pulsar['DM'].fit = True DMXFlag = True if DMXFlag == False: print 'NO DMX for pulsar {0}'.format(t2pulsar.name) # refit 5 times to make sure we are converged t2pulsar.fit(iters=5) # Create the pulsar subgroup if it does not exist if "Pulsars" in datagroup: pulsarsgroup = datagroup["Pulsars"] else: raise IOError, "Cannot add noise parameters if pulsar group does not exist!" # Look up the name of the pulsar, and see if it exist if t2pulsar.name in pulsarsgroup: pass else: raise IOError, "%s must already exists in %s to add noise parameters!"\ % (t2pulsar.name, self.filename) pulsarsgroup = pulsarsgroup[t2pulsar.name] # first create G matrix from design matrix and toas designmatrix = np.double(t2pulsar.designmatrix()) toas = np.double(t2pulsar.toas() * 86400) errs = np.double(t2pulsar.toaerrs * 1e-6) # if doing daily averaging if dailyAverage: # get average quantities toas, qmatrix, errs, dmatrix, freqs, bands = PALutils.dailyAverage( t2pulsar) # construct new daily averaged residuals and designmatrix toas *= 86400 designmatrix = np.dot(qmatrix, dmatrix) G = PALutils.createGmatrix(designmatrix) # create matrix of time lags tm = PALutils.createTimeLags(toas, toas, round=True) # now read noise file to get model and parameters file = open(noisefile, 'r') fH = None tau = None DMAmp = None DMgam = None for line in file.readlines(): # default parameters for different models other than pure PL key = line.split()[0] # get amplitude if "Amp" == key: Amp = float(line.split()[-1]) # get spectral index elif "gam" == key: gam = float(line.split()[-1]) # get efac elif "efac" == key: efac = float(line.split()[-1]) # get quad elif "equad" == key: equad = float(line.split()[-1]) # get high frequency cutoff if available elif "fH" == key: fH = float(line.split()[-1]) # get correlation time scale if available elif "tau" == key: tau = float(line.split()[-1]) # get DM Amplitude if available elif "DMAmp" == key: DMAmp = float(line.split()[-1]) # get DM Spectral Index if available elif "DMgam" == key: DMgam = float(line.split()[-1]) # cosstruct red and white noise covariance matrices red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) white = PALutils.createWhiteNoiseCovarianceMatrix(errs, efac, equad, tau=tau) # construct post timing model marginalization covariance matrix cov = red + white pcov = np.dot(G.T, np.dot(cov, G)) # finally construct "inverse" invCov = np.dot(G, np.dot(np.linalg.inv(pcov), G.T)) # create dataset for inverse covariance matrix pulsarsgroup.create_dataset('invCov', data=invCov) # create dataset for G matrix pulsarsgroup.create_dataset('Gmatrix', data=G) # record noise parameter values pulsarsgroup.create_dataset('Amp', data=Amp) pulsarsgroup.create_dataset('gam', data=gam) pulsarsgroup.create_dataset('efac', data=efac) pulsarsgroup.create_dataset('equad', data=equad) if fH is not None: pulsarsgroup.create_dataset('fH', data=fH) if tau is not None: pulsarsgroup.create_dataset('tau', data=tau) if DMAmp is not None: pulsarsgroup.create_dataset('DMAmp', data=DMAmp) if DMgam is not None: pulsarsgroup.create_dataset('DMgam', data=DMgam) # Close the hdf5 file self.h5file.close()