def addInverseCovFromNoiseFile(self, parfile, timfile, noisefile, DMOFF=None, DMXOFF=None, dailyAverage=False): """ Add noise covariance matrix after timing model subtraction. """ # Check whether the two files exist if not os.path.isfile(parfile) or not os.path.isfile(timfile): raise IOError, "Cannot find parfile (%s) or timfile (%s)!" % (parfile, timfile) assert(self.filename != None), "ERROR: HDF5 file not set!" # 'a' means: read/write if exists, create otherwise self.h5file = h5.File(self.filename, 'a') # Create the data subgroup if it does not exist if "Data" in self.h5file: datagroup = self.h5file["Data"] else: raise IOError, "Cannot add noise parameters if Data group does not exist!" # Load pulsar data from the JPL Cython tempo2 library t2pulsar = t2.tempopulsar(parfile, timfile) # turn off DMMODEL fitting if DMOFF is not None: t2pulsar['DMMODEL'].fit = False # turn off DMX fitting if DMXOFF is not None: DMXFlag = False print 'Turning off DMX fitting and turning DM fitting on' for par in t2pulsar.pars: if 'DMX' in par: t2pulsar[par].fit = False t2pulsar['DM'].fit = True DMXFlag = True if DMXFlag== False: print 'NO DMX for pulsar {0}'.format(t2pulsar.name) # refit 5 times to make sure we are converged t2pulsar.fit(iters=5) # Create the pulsar subgroup if it does not exist if "Pulsars" in datagroup: pulsarsgroup = datagroup["Pulsars"] else: raise IOError, "Cannot add noise parameters if pulsar group does not exist!" # Look up the name of the pulsar, and see if it exist if t2pulsar.name in pulsarsgroup: pass else: raise IOError, "%s must already exists in %s to add noise parameters!"\ % (t2pulsar.name, self.filename) pulsarsgroup = pulsarsgroup[t2pulsar.name] # first create G matrix from design matrix and toas designmatrix = np.double(t2pulsar.designmatrix()) toas = np.double(t2pulsar.toas()*86400) errs = np.double(t2pulsar.toaerrs*1e-6) # if doing daily averaging if dailyAverage: # get average quantities toas, qmatrix, errs, dmatrix, freqs, bands = PALutils.dailyAverage(t2pulsar) # construct new daily averaged residuals and designmatrix toas *= 86400 designmatrix = np.dot(qmatrix, dmatrix) G = PALutils.createGmatrix(designmatrix) # create matrix of time lags tm = PALutils.createTimeLags(toas, toas, round=True) # now read noise file to get model and parameters file = open(noisefile,'r') fH = None tau = None DMAmp = None DMgam = None for line in file.readlines(): # default parameters for different models other than pure PL key = line.split()[0] # get amplitude if "Amp" == key: Amp = float(line.split()[-1]) # get spectral index elif "gam" == key: gam = float(line.split()[-1]) # get efac elif "efac" == key: efac = float(line.split()[-1]) # get quad elif "equad" == key: equad = float(line.split()[-1]) # get high frequency cutoff if available elif "fH" == key: fH = float(line.split()[-1]) # get correlation time scale if available elif "tau" == key: tau = float(line.split()[-1]) # get DM Amplitude if available elif "DMAmp" == key: DMAmp = float(line.split()[-1]) # get DM Spectral Index if available elif "DMgam" == key: DMgam = float(line.split()[-1]) # cosstruct red and white noise covariance matrices red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) white = PALutils.createWhiteNoiseCovarianceMatrix(errs, efac, equad, tau=tau) # construct post timing model marginalization covariance matrix cov = red + white pcov = np.dot(G.T, np.dot(cov, G)) # finally construct "inverse" invCov = np.dot(G, np.dot(np.linalg.inv(pcov), G.T)) # create dataset for inverse covariance matrix pulsarsgroup.create_dataset('invCov', data = invCov) # create dataset for G matrix pulsarsgroup.create_dataset('Gmatrix', data = G) # record noise parameter values pulsarsgroup.create_dataset('Amp', data = Amp) pulsarsgroup.create_dataset('gam', data = gam) pulsarsgroup.create_dataset('efac', data = efac) pulsarsgroup.create_dataset('equad', data = equad) if fH is not None: pulsarsgroup.create_dataset('fH', data = fH) if tau is not None: pulsarsgroup.create_dataset('tau', data = tau) if DMAmp is not None: pulsarsgroup.create_dataset('DMAmp', data = DMAmp) if DMgam is not None: pulsarsgroup.create_dataset('DMgam', data = DMgam) # Close the hdf5 file self.h5file.close()
print "Maximum likelihood from f-stat search = {0}\n".format(fmaxlike) # get determinant of covariance matrix for use in likelihood logdetTerm = [] for ct, p in enumerate(psr): efac = p.efac equad = p.equad Amp = p.Amp gam = p.gam fH = p.fH print p.name, efac, equad, Amp, gam # get white noise covariance matrix white = PALutils.createWhiteNoiseCovarianceMatrix(p.err, efac, equad) # get red noise covariance matrix tm = PALutils.createTimeLags(p.toas, p.toas, round=True) red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) C = np.dot(p.G.T, np.dot(red + white, p.G)) cf = sl.cho_factor(C) logdetTerm.append(np.sum(2 * np.log(np.diag(cf[0])))) # + p.G.shape[1]*np.log(2*np.pi)) # get null model log-likelihood nullLike = 0 for ct, p in enumerate(psr): nullLike += -0.5 * logdetTerm[ct] nullLike += -0.5 * np.dot(p.res, np.dot(p.invCov, p.res))
def addInverseCovFromNoiseFile(self, parfile, timfile, noisefile, DMOFF=None, DMXOFF=None, dailyAverage=False): """ Add noise covariance matrix after timing model subtraction. """ # Check whether the two files exist if not os.path.isfile(parfile) or not os.path.isfile(timfile): raise IOError, "Cannot find parfile (%s) or timfile (%s)!" % ( parfile, timfile) assert (self.filename != None), "ERROR: HDF5 file not set!" # 'a' means: read/write if exists, create otherwise self.h5file = h5.File(self.filename, 'a') # Create the data subgroup if it does not exist if "Data" in self.h5file: datagroup = self.h5file["Data"] else: raise IOError, "Cannot add noise parameters if Data group does not exist!" # Load pulsar data from the JPL Cython tempo2 library t2pulsar = t2.tempopulsar(parfile, timfile) # turn off DMMODEL fitting if DMOFF is not None: t2pulsar['DMMODEL'].fit = False # turn off DMX fitting if DMXOFF is not None: DMXFlag = False print 'Turning off DMX fitting and turning DM fitting on' for par in t2pulsar.pars: if 'DMX' in par: t2pulsar[par].fit = False t2pulsar['DM'].fit = True DMXFlag = True if DMXFlag == False: print 'NO DMX for pulsar {0}'.format(t2pulsar.name) # refit 5 times to make sure we are converged t2pulsar.fit(iters=5) # Create the pulsar subgroup if it does not exist if "Pulsars" in datagroup: pulsarsgroup = datagroup["Pulsars"] else: raise IOError, "Cannot add noise parameters if pulsar group does not exist!" # Look up the name of the pulsar, and see if it exist if t2pulsar.name in pulsarsgroup: pass else: raise IOError, "%s must already exists in %s to add noise parameters!"\ % (t2pulsar.name, self.filename) pulsarsgroup = pulsarsgroup[t2pulsar.name] # first create G matrix from design matrix and toas designmatrix = np.double(t2pulsar.designmatrix()) toas = np.double(t2pulsar.toas() * 86400) errs = np.double(t2pulsar.toaerrs * 1e-6) # if doing daily averaging if dailyAverage: # get average quantities toas, qmatrix, errs, dmatrix, freqs, bands = PALutils.dailyAverage( t2pulsar) # construct new daily averaged residuals and designmatrix toas *= 86400 designmatrix = np.dot(qmatrix, dmatrix) G = PALutils.createGmatrix(designmatrix) # create matrix of time lags tm = PALutils.createTimeLags(toas, toas, round=True) # now read noise file to get model and parameters file = open(noisefile, 'r') fH = None tau = None DMAmp = None DMgam = None for line in file.readlines(): # default parameters for different models other than pure PL key = line.split()[0] # get amplitude if "Amp" == key: Amp = float(line.split()[-1]) # get spectral index elif "gam" == key: gam = float(line.split()[-1]) # get efac elif "efac" == key: efac = float(line.split()[-1]) # get quad elif "equad" == key: equad = float(line.split()[-1]) # get high frequency cutoff if available elif "fH" == key: fH = float(line.split()[-1]) # get correlation time scale if available elif "tau" == key: tau = float(line.split()[-1]) # get DM Amplitude if available elif "DMAmp" == key: DMAmp = float(line.split()[-1]) # get DM Spectral Index if available elif "DMgam" == key: DMgam = float(line.split()[-1]) # cosstruct red and white noise covariance matrices red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) white = PALutils.createWhiteNoiseCovarianceMatrix(errs, efac, equad, tau=tau) # construct post timing model marginalization covariance matrix cov = red + white pcov = np.dot(G.T, np.dot(cov, G)) # finally construct "inverse" invCov = np.dot(G, np.dot(np.linalg.inv(pcov), G.T)) # create dataset for inverse covariance matrix pulsarsgroup.create_dataset('invCov', data=invCov) # create dataset for G matrix pulsarsgroup.create_dataset('Gmatrix', data=G) # record noise parameter values pulsarsgroup.create_dataset('Amp', data=Amp) pulsarsgroup.create_dataset('gam', data=gam) pulsarsgroup.create_dataset('efac', data=efac) pulsarsgroup.create_dataset('equad', data=equad) if fH is not None: pulsarsgroup.create_dataset('fH', data=fH) if tau is not None: pulsarsgroup.create_dataset('tau', data=tau) if DMAmp is not None: pulsarsgroup.create_dataset('DMAmp', data=DMAmp) if DMgam is not None: pulsarsgroup.create_dataset('DMgam', data=DMgam) # Close the hdf5 file self.h5file.close()
def firstOrderLikelihood(psr, ORF, Agw, gamgw, Ared, gred, efac, equad, \ interpolate=False): """ Compute the value of the first-order likelihood as defined in Ellis, Siemens, van Haasteren (2013). @param psr: List of pulsar object instances @param ORF: Vector of pairwise overlap reduction values @param Agw: Amplitude of GWB in standard strain amplitude units @param gamgw: Power spectral index of GWB @param Ared: Vector of amplitudes of intrinsic red noise in GWB strain units @param gamgw: Vector of power spectral index of red noise @param efac: Vector of efacs @param equad: Vector of equads @param interpolate: Boolean to perform interpolation only with compressed data. (default = False) @return: Log-likelihood value """ npsr = len(psr) loglike = 0 tmp = [] # start loop to evaluate auto-terms for ll in range(npsr): r1 = np.dot(psr[ll].G.T, psr[ll].res) # create time lags tm = PALutils.createTimeLags(psr[ll].toas, psr[ll].toas) #TODO: finish option to do interpolation when using compression # calculate auto GW covariance matrix SC = PALutils.createRedNoiseCovarianceMatrix(tm, Agw, gamgw) # calculate auto red noise covariance matrix SA = PALutils.createRedNoiseCovarianceMatrix(tm, Ared[ll], gred[ll]) # create white noise covariance matrix #TODO: add ability to use multiple efacs for different backends white = PALutils.createWhiteNoiseCovarianceMatrix(psr[ll].err, efac[ll], equad[ll]) # total auto-covariance matrix P = SC + SA + white # sandwich with G matrices Ppost = np.dot(psr[ll].G.T, np.dot(P, psr[ll].G)) # do cholesky solve cf = sl.cho_factor(Ppost) # solution vector P^_1 r rr = sl.cho_solve(cf, r1) # temporarily store P^-1 r tmp.append(np.dot(psr[ll].G, rr)) # add to log-likelihood loglike += -0.5 * (np.sum(np.log(2*np.pi*np.diag(cf[0])**2)) + np.dot(r1, rr)) # now compute cross terms k = 0 for ll in range(npsr): for kk in range(ll+1, npsr): # create time lags tm = PALutils.createTimeLags(psr[ll].toas, psr[kk].toas) # create cross covariance matrix SIJ = PALutils.createRedNoiseCovarianceMatrix(tm, 1, gamgw) # carry out matrix-vetor operations tmp1 = np.dot(SIJ, tmp[kk]) # add to likelihood loglike += ORF[k]/2 * Agw**2 * np.dot(tmp[ll], tmp1) # increment ORF counter k += 1 return loglike
print 'Maximum likelihood from f-stat search = {0}\n'.format(fmaxlike) # get determinant of covariance matrix for use in likelihood logdetTerm = [] for ct, p in enumerate(psr): efac = p.efac equad = p.equad Amp = p.Amp gam = p.gam fH = p.fH print p.name, efac, equad, Amp, gam # get white noise covariance matrix white = PALutils.createWhiteNoiseCovarianceMatrix(p.err, efac, equad) # get red noise covariance matrix tm = PALutils.createTimeLags(p.toas, p.toas, round=True) red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) C = np.dot(p.G.T, np.dot(red+white, p.G)) cf = sl.cho_factor(C) logdetTerm.append(np.sum(2*np.log(np.diag(cf[0])))) #+ p.G.shape[1]*np.log(2*np.pi)) # get null model log-likelihood nullLike = 0 for ct, p in enumerate(psr): nullLike += -0.5 * logdetTerm[ct] nullLike += -0.5 * np.dot(p.res, np.dot(p.invCov, p.res))
try: fH = pfile['Data']['Pulsars'][p.name]['fH'].value except KeyError: fH = None # get values from hdf5 file try: Amp = pfile['Data']['Pulsars'][p.name]['Amp'].value gam = pfile['Data']['Pulsars'][p.name]['gam'].value efac = pfile['Data']['Pulsars'][p.name]['efac'].value equad = pfile['Data']['Pulsars'][p.name]['equad'].value tm = PALutils.createTimeLags(p.toas, p.toas) red = PALutils.createRedNoiseCovarianceMatrix(tm, Amp, gam, fH=fH) white = PALutils.createWhiteNoiseCovarianceMatrix(p.err, efac, equad) cov = red + white # cholesky decomp L = np.linalg.cholesky(cov) # set random number seed if args.seed: print 'Using fixed random number seed!' np.random.seed(seed=args.seed*(ct+1)) # zero mean unit variance w = np.random.randn(p.ntoa) # get induced residuals
def firstOrderLikelihood(psr, ORF, Agw, gamgw, Ared, gred, efac, equad, \ interpolate=False): """ Compute the value of the first-order likelihood as defined in Ellis, Siemens, van Haasteren (2013). @param psr: List of pulsar object instances @param ORF: Vector of pairwise overlap reduction values @param Agw: Amplitude of GWB in standard strain amplitude units @param gamgw: Power spectral index of GWB @param Ared: Vector of amplitudes of intrinsic red noise in GWB strain units @param gamgw: Vector of power spectral index of red noise @param efac: Vector of efacs @param equad: Vector of equads @param interpolate: Boolean to perform interpolation only with compressed data. (default = False) @return: Log-likelihood value """ npsr = len(psr) loglike = 0 tmp = [] # start loop to evaluate auto-terms for ll in range(npsr): r1 = np.dot(psr[ll].G.T, psr[ll].res) # create time lags tm = PALutils.createTimeLags(psr[ll].toas, psr[ll].toas) #TODO: finish option to do interpolation when using compression # calculate auto GW covariance matrix SC = PALutils.createRedNoiseCovarianceMatrix(tm, Agw, gamgw) # calculate auto red noise covariance matrix SA = PALutils.createRedNoiseCovarianceMatrix(tm, Ared[ll], gred[ll]) # create white noise covariance matrix #TODO: add ability to use multiple efacs for different backends white = PALutils.createWhiteNoiseCovarianceMatrix( psr[ll].err, efac[ll], equad[ll]) # total auto-covariance matrix P = SC + SA + white # sandwich with G matrices Ppost = np.dot(psr[ll].G.T, np.dot(P, psr[ll].G)) # do cholesky solve cf = sl.cho_factor(Ppost) # solution vector P^_1 r rr = sl.cho_solve(cf, r1) # temporarily store P^-1 r tmp.append(np.dot(psr[ll].G, rr)) # add to log-likelihood loglike += -0.5 * (np.sum(np.log(2 * np.pi * np.diag(cf[0])**2)) + np.dot(r1, rr)) # now compute cross terms k = 0 for ll in range(npsr): for kk in range(ll + 1, npsr): # create time lags tm = PALutils.createTimeLags(psr[ll].toas, psr[kk].toas) # create cross covariance matrix SIJ = PALutils.createRedNoiseCovarianceMatrix(tm, 1, gamgw) # carry out matrix-vetor operations tmp1 = np.dot(SIJ, tmp[kk]) # add to likelihood loglike += ORF[k] / 2 * Agw**2 * np.dot(tmp[ll], tmp1) # increment ORF counter k += 1 return loglike