Exemple #1
0
def do_RRLyrae():

    dtype = np.dtype([("mjd", np.float), ("filt", np.str, 1), ("mag", np.float), ("dmag", np.float)])
    data = np.loadtxt(data_dir + 'RRLyrae.txt', comments="#", dtype=dtype)

    # do g-band light curve
    gIdx = np.where(data["filt"] == "g")[0]
    jdate = data['mjd'][gIdx]
    gmag = data['mag'][gIdx]
    gerr = data['dmag'][gIdx]

    load_pickle = True
    if load_pickle:
        time = jdate - jdate.min()
        ysig = gerr
        carma_sample = cPickle.load(open(data_dir + 'RRLyrae.pickle', 'rb'))
        froot = base_dir + 'plots/' + 'RRLyrae_'
        ax = plt.subplot(111)
        print 'Getting bounds on PSD...'
        psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                                 color='SkyBlue', nsamples=5000)
        psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                    ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
        ax.loglog(frequencies, psd_mle, '--b', lw=2)
        dt = time[1:] - time[0:-1]
        noise_level = 2.0 * np.median(dt) * np.mean(ysig ** 2)
        ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
        ax.set_ylim(bottom=noise_level / 100.0)
        ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
        ax.annotate("C", (1.0 / 0.5, 1e3))
        ax.annotate("B", (1.0 / 1.3, 2.0))
        ax.annotate("A", (1.0 / 2.49, 1.0))
        ax.set_xlabel('Frequency [1 / day]')
        ax.set_ylabel('Power Spectral Density [mag$^2$ day]')
        plt.title("RR Lyrae, g-band")
        plt.savefig(froot + 'psd.eps')

        print 'Assessing the fit quality...'
        fig = carma_sample.assess_fit(doShow=False)
        ax_again = fig.add_subplot(2, 2, 1)
        ax_again.set_title("RR Lyrae, g-band")
        ylims = ax_again.get_ylim()
        ax_again.set_ylim(ylims[1], ylims[0])
        ax_again.set_ylabel('magnitude')
        ax_again.set_xlabel('Time [days]')
        ax_again = fig.add_subplot(2, 2, 2)
        ax_again.set_xlabel('Time [days]')
        ax_again = fig.add_subplot(2, 2, 3)
        ax_again.set_xlabel('Lag')
        ax_again = fig.add_subplot(2, 2, 4)
        ax_again.set_xlabel('Lag')
        plt.savefig(froot + 'fit_quality.eps')

    else:
        carma_sample = make_sampler_plots(jdate - jdate.min(), gmag, gerr, 7, 'RRLyrae_', 'RR Lyrae, g-band', do_mags=True,
                                          njobs=1)
        pfile = open(data_dir + 'RRLyrae.pickle', 'wb')
        cPickle.dump(carma_sample, pfile)
        pfile.close()
Exemple #2
0
def do_AGN_Xray():

    sname = 'MCG-6-30-15, X-ray'
    data = np.genfromtxt(data_dir + 'mcg-6-30-15_rxte_xmm.txt')
    jdate = data[:, 0]
    flux = data[:, 1] * np.log(10.0)  # convert to natural logarithm
    ferr = data[:, 2] * np.log(10.0)

    jdate = jdate - jdate.min()
    time = jdate * 86.4e3  # convert to seconds

    dt = time[1:] - time[0:-1]
    rxte = np.where(dt > 50.0)[0]
    dt_rxte = np.median(dt[rxte])
    xmm = np.where(dt < 50.0)[0]
    dt_xmm = 48.0

    carma_sample = make_sampler_plots(time[rxte], flux[rxte], ferr[rxte] / 1e6, 7, 'mcg63015_rxte_', sname, njobs=4)

    measerr_scale = carma_sample.get_samples('measerr_scale')
    print "95% credibility interval on Kepler measurement error scale parameter:", np.percentile(measerr_scale, 2.5), \
        np.percentile(measerr_scale, 97.5)

    pfile = open(data_dir + 'mcg63015.pickle', 'wb')
    cPickle.dump(carma_sample, pfile)
    pfile.close()

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                             color='SkyBlue', nsamples=5000)
    psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
    ax.loglog(frequencies, psd_mle, '--b', lw=2)
    noise_level_rxte = 2.0 * dt_rxte * np.mean(ferr[rxte] ** 2)
    noise_level_xmm = 2.0 * dt_xmm * np.mean(ferr[xmm] ** 2)
    rxte_frange = np.array([1.0 / time[rxte].max(), 1.0 / dt_rxte])
    xmm_frange = np.array([1.0 / (time[xmm].max() - time[xmm].min()), 1.0 / dt_xmm])
    ax.loglog(rxte_frange, np.ones(2) * noise_level_rxte, color='grey', lw=2)
    ax.loglog(xmm_frange, np.ones(2) * noise_level_xmm, color='grey', lw=2)
    noise_level = np.min([noise_level_rxte, noise_level_xmm])
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level, RXTE", (2.0 * ax.get_xlim()[0], noise_level_rxte / 2.5))
    ax.annotate("Noise Level, XMM", (xmm_frange[0], noise_level_xmm / 2.5))
    ax.set_xlabel('Frequency [Hz]')
    ax.set_ylabel('Power Spectral Density [fraction$^2$ / Hz]')
    plt.title(sname)
    plt.savefig(base_dir + 'plots/mcg63015_psd.eps')
Exemple #3
0
    def power_spectrum(self, fs, p):
        p = self.to_params(p)

        ar_roots = self.ar_roots(p)
        ma_roots = self.ma_roots(p)

        ar_coefs = np.real(np.poly(ar_roots))

        ma_coefs = np.real(np.poly(ma_roots))
        ma_coefs /= ma_coefs[-1]
        ma_coefs = ma_coefs[::-1]

        s = par.bounded_values(p['logit_sigma'],
                               low=self.sigma_min,
                               high=self.sigma_max)

        sigma = s / np.sqrt(cm.carma_variance(1.0, ar_roots, ma_coefs))

        return cm.power_spectrum(fs, sigma, ar_coefs, ma_coefs)
Exemple #4
0
def do_OGLE_LPV():
    sname = 'LPV, RGB, i-band'
    data = np.genfromtxt(data_dir + 'OGLE-LMC-LPV-00007.dat')
    jdate = data[:, 0]
    imag = data[:, 1]
    ierr = data[:, 2]

    load_pickle = False
    if load_pickle:
        time = jdate
        ysig = ierr
        carma_sample = cPickle.load(open(data_dir + 'ogle_lpv_rgb.pickle', 'rb'))
        froot = base_dir + 'plots/' + 'ogle_lpv_rgb_'
        ax = plt.subplot(111)
        print 'Getting bounds on PSD...'
        psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                                 color='SkyBlue', nsamples=5000)
        psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                    ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
        ax.loglog(frequencies, psd_mle, '--b', lw=2)
        dt = time[1:] - time[0:-1]
        noise_level = 2.0 * np.median(dt) * np.mean(ysig ** 2)
        ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
        ax.set_ylim(bottom=noise_level / 100.0)
        ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
        ax.annotate("A", (1.0 / 25.0, 2.5e-3))
        ax.annotate("B", (1.0 / 16.0, 2.5e-3))
        ax.set_xlabel('Frequency [1 / day]')
        ax.set_ylabel('Power Spectral Density [mag$^2$ day]')
        plt.title(sname)
        plt.savefig(froot + 'psd.eps')
    else:
        carma_sample = make_sampler_plots(jdate - jdate.min(), imag, ierr, 7, 'ogle_lpv_rgb_', sname, do_mags=True,
                                          njobs=1)
        pfile = open(data_dir + 'ogle_lpv_rgb.pickle', 'wb')
        cPickle.dump(carma_sample, pfile)
        pfile.close()
Exemple #5
0
def do_simulated_regular():

    # first generate some data assuming a CAR(5) process on a uniform grid
    sigmay = 2.3  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    qpo_width = np.array([1.0 / 100.0, 1.0 / 100.0, 1.0 / 500.0])
    qpo_cent = np.array([1.0 / 5.0, 1.0 / 50.0])
    ar_roots = cm.get_ar_roots(qpo_width, qpo_cent)
    ma_coefs = np.zeros(p)
    ma_coefs[0] = 1.0
    ma_coefs[1] = 4.5
    ma_coefs[2] = 1.25
    sigsqr = sigmay**2 / cm.carma_variance(1.0, ar_roots, ma_coefs=ma_coefs)

    ny = 1028
    time = np.arange(0.0, ny)
    y0 = cm.carma_process(time, sigsqr, ar_roots, ma_coefs=ma_coefs)

    ysig = np.ones(ny) * np.sqrt(1e-3 / 2.0)

    y = y0 + ysig * np.random.standard_normal(ny)

    data = (time, y, ysig)

    froot = base_dir + 'car5_regular_'

    plt.subplot(111)
    plt.plot(time, y0, 'k-')
    plt.plot(time, y, '.')
    plt.xlim(time.min(), time.max())
    plt.xlabel('Time')
    plt.ylabel('CAR(5) Process')
    plt.savefig(froot + 'tseries.eps')

    ar_coef = np.poly(ar_roots)

    pool = mp.Pool(mp.cpu_count() - 1)

    args = []
    maxp = 8
    for p in xrange(1, maxp + 1):
        for q in xrange(p):
            args.append(((p, q), data))

    print "Running the CARMA MCMC samplers..."

    carma_run = pool.map(run_carma_sampler, args)

    dic = []
    pmodels = []
    qmodels = []
    for crun in carma_run:
        dic.append(crun.DIC())
        pmodels.append(crun.p)
        qmodels.append(crun.q)

    pmodels = np.array(pmodels)
    qmodels = np.array(qmodels)
    dic = np.array(dic)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max() + 1):
        plt.plot(pmodels[qmodels == i],
                 dic[qmodels == i],
                 label='q=' + str(i),
                 lw=2)

    plt.legend()
    plt.xlabel('p')
    plt.ylabel('DIC')
    print "DIC", dic
    plt.savefig(froot + 'dic.eps')

    carma = carma_run[np.argmin(dic)]

    print "order of best model is", carma.p

    plt.subplot(111)
    pgram, freq = plt.psd(y)
    plt.clf()

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma.plot_power_spectrum(
        percentile=95.0, sp=ax, doShow=False, color='SkyBlue')
    ax.loglog(freq / 2.0, pgram, 'o', color='DarkOrange')
    psd = cm.power_spectrum(frequencies,
                            np.sqrt(sigsqr),
                            ar_coef,
                            ma_coefs=ma_coefs)
    ax.loglog(frequencies, psd, 'k', lw=2)
    ax.loglog(frequencies, psd_mid, '--b', lw=2)
    noise_level = np.mean(ysig**2)
    ax.loglog(frequencies,
              np.ones(frequencies.size) * noise_level,
              color='grey',
              lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level",
                (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency')
    ax.set_ylabel('Power Spectral Density')

    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    carma.assess_fit(doShow=False)
    plt.savefig(froot + 'fit_quality.eps')
Exemple #6
0
def do_simulated_irregular():

    # first generate some data assuming a CAR(5) process on a uniform grid
    sigmay = 2.3  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    mu = 17.0  # mean of time series
    qpo_width = np.array([1.0 / 100.0, 1.0 / 300.0, 1.0 / 200.0])
    qpo_cent = np.array([1.0 / 5.0, 1.0 / 25.0])
    ar_roots = cm.get_ar_roots(qpo_width, qpo_cent)
    ma_coefs = np.zeros(p)
    ma_coefs[0] = 1.0
    ma_coefs[1] = 4.5
    ma_coefs[2] = 1.25
    sigsqr = sigmay**2 / cm.carma_variance(1.0, ar_roots, ma_coefs=ma_coefs)

    ny = 270
    time = np.empty(ny)
    dt = np.random.uniform(1.0, 3.0, ny)
    time[0:90] = np.cumsum(dt[0:90])
    time[90:2 * 90] = 180 + time[90 - 1] + np.cumsum(dt[90:2 * 90])
    time[2 * 90:] = 180 + time[2 * 90 - 1] + np.cumsum(dt[2 * 90:])

    y = mu + cm.carma_process(time, sigsqr, ar_roots, ma_coefs=ma_coefs)

    ysig = np.ones(ny) * y.std() / 5.0
    ysig = np.ones(ny) * 1e-6
    y0 = y.copy()
    y += ysig * np.random.standard_normal(ny)

    data = (time, y, ysig)

    #    car5_model = cm.CarmaMCMC(time, y, ysig, 5, 5000, doZcarma=True, nburnin=1000)
    #    zcar = car5_model.RunMCMC()

    froot = base_dir + 'car5_irregular_'

    plt.subplot(111)
    for i in xrange(3):
        plt.plot(time[90 * i:90 * (i + 1)], y0[90 * i:90 * (i + 1)], 'k', lw=2)
        plt.plot(time[90 * i:90 * (i + 1)], y[90 * i:90 * (i + 1)], 'bo')

    plt.xlim(time.min(), time.max())
    plt.xlabel('Time')
    plt.ylabel('CAR(5) Process')
    plt.savefig(froot + 'tseries.eps')

    ar_coef = np.poly(ar_roots)

    pool = mp.Pool(mp.cpu_count() - 1)

    args = []
    maxp = 9
    for p in xrange(1, maxp + 1):
        for q in xrange(p):
            args.append((p, q, data))

    print "Running the CARMA MCMC samplers..."

    carma_run = pool.map(run_carma_sampler, args)

    dic = []
    pmodels = []
    qmodels = []
    for crun in carma_run:
        dic.append(crun.DIC())
        pmodels.append(crun.p)
        qmodels.append(crun.q)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max() + 1):
        plt.plot(pmodels[qmodels == i],
                 dic[qmodels == i],
                 label='q=' + str(i),
                 lw=2)

    plt.legend()
    plt.xlabel('p')
    plt.ylabel('DIC')
    print "DIC", dic
    plt.savefig(froot + 'dic.eps')

    carma = carma_run[np.argmin(dic)]

    print "order of best model is", carma.p

    plt.clf()
    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma.plot_power_spectrum(
        percentile=95.0, sp=ax, doShow=False, color='SkyBlue')
    psd = cm.power_spectrum(frequencies,
                            np.sqrt(sigsqr),
                            ar_coef,
                            ma_coefs=ma_coefs)
    ax.loglog(frequencies, psd_mid, '--b', lw=2)
    ax.loglog(frequencies, psd, 'k', lw=2)
    noise_level = np.mean(ysig**2) * dt.min()
    ax.loglog(frequencies,
              np.ones(frequencies.size) * noise_level,
              color='grey',
              lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level",
                (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency')
    ax.set_ylabel('Power Spectral Density')

    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    carma.assess_fit(doShow=False)
    plt.savefig(froot + 'fit_quality.eps')

    # compute the marginal mean and variance of the predicted values
    nplot = 1028
    time_predict = np.linspace(time.min(), 1.25 * time.max(), nplot)
    time_predict = time_predict[1:]
    predicted_mean, predicted_var = carma.predict_lightcurve(time_predict,
                                                             bestfit='map')
    predicted_low = predicted_mean - np.sqrt(predicted_var)
    predicted_high = predicted_mean + np.sqrt(predicted_var)

    # plot the time series and the marginal 1-sigma error bands
    plt.clf()
    plt.subplot(111)
    plt.fill_between(time_predict, predicted_low, predicted_high, color='cyan')
    plt.plot(time_predict, predicted_mean, '-b', label='Predicted')
    plt.plot(time[0:90], y0[0:90], 'k', lw=2, label='True')
    plt.plot(time[0:90], y[0:90], 'bo')
    for i in xrange(1, 3):
        plt.plot(time[90 * i:90 * (i + 1)], y0[90 * i:90 * (i + 1)], 'k', lw=2)
        plt.plot(time[90 * i:90 * (i + 1)], y[90 * i:90 * (i + 1)], 'bo')

    plt.xlabel('Time')
    plt.ylabel('CAR(5) Process')
    plt.xlim(time_predict.min(), time_predict.max())
    plt.legend()
    plt.savefig(froot + 'interp.eps')
Exemple #7
0
def do_XRB():
    sname = 'XTE 1550-564'
    data_file = data_dir + 'LC_B_3.35-12.99keV_1div128s_total.fits'
    data = fits.open(data_file)[1].data
    tsecs = data['TIME']
    flux = data['RATE']
    dt = tsecs[1:] - tsecs[:-1]
    gap = np.where(dt > 1)[0]

    tsecs = tsecs[gap[0]+1:gap[1]][:40000]
    flux = flux[gap[0]+1:gap[1]][:40000]

    tsecs0 = tsecs.copy()
    flux0 = flux.copy()

    ndown_sample = 4000
    idx = np.random.permutation(len(flux0))[:ndown_sample]
    idx.sort()
    tsecs = tsecs[idx]
    logflux = np.log(flux[idx])
    ferr = np.sqrt(flux[idx])
    logf_err = ferr / flux[idx]

    # # high-frequency sampling lightcurve
    # high_cutoff = 10000
    # tsecs_high = tsecs[:high_cutoff]
    # logflux_high = np.log(flux[:high_cutoff])
    # ferr_high = np.sqrt(flux[:high_cutoff])
    # logferr_high = ferr_high / flux[:high_cutoff]
    #
    # ndown_sample_high = 1000
    # idx_high = np.random.permutation(len(logflux_high))[:ndown_sample_high]
    # idx_high.sort()
    #
    # # middle-frequency sampling lightcurve
    # tsecs_mid = tsecs[high_cutoff:]
    # logflux_mid = np.log(flux[high_cutoff:])
    # ferr_mid = np.sqrt(flux[high_cutoff:])
    # logf_err_mid = ferr_mid / flux[high_cutoff:]
    # # logf_err = np.sqrt(0.00018002985939372774 / 2.0 / np.median(dt))  # eyeballed from periodogram
    # # logf_err = np.ones(len(tsecs)) * logf_err
    #
    # ndown_sample_mid = 4000 - ndown_sample_high
    # idx_mid = np.random.permutation(len(logflux_mid))[:ndown_sample_mid]
    # idx_mid.sort()
    #
    # tsecs = np.concatenate((tsecs_high[idx_high], tsecs_mid[idx_mid]))
    # logflux = np.concatenate((logflux_high[idx_high], logflux_mid[idx_mid]))
    # logf_err = np.concatenate((logferr_high[idx_high], logf_err_mid[idx_mid]))
    # idx = np.concatenate((idx_high, idx_mid))

    plt.plot(tsecs0, np.log(flux0))
    plt.errorbar(tsecs, logflux, yerr=logf_err)
    print 'Measurement errors are', np.mean(logf_err) / np.std(logflux) * 100, ' % of observed standard deviation.'
    print 'Mean time spacing:', np.mean(tsecs[1:] - tsecs[:-1])
    # print 'Mean time spacing for high-frequency sampling:', np.mean(tsecs_high[idx_high[1:]]-tsecs_high[idx_high[:-1]])
    # print 'Mean time spacing for low-frequency sampling:', np.mean(tsecs_mid[idx_mid[1:]]-tsecs_mid[idx_mid[:-1]])
    plt.show()
    plt.clf()
    plt.plot(tsecs, logflux)
    plt.show()
    plt.hist(logflux, bins=100, normed=True)
    plt.xlabel('log Flux')
    print 'Standard deviation in lightcurve:', np.std(logflux)
    print 'Typical measurement error:', np.mean(logf_err)
    plt.show()
    plt.clf()
    assert np.all(np.isfinite(tsecs))
    assert np.all(np.isfinite(logflux))
    assert np.all(np.isfinite(logf_err))
    dt_idx = tsecs[1:] - tsecs[:-1]
    assert np.all(dt_idx > 0)

    load_pickle = True
    if load_pickle:
        carma_sample = cPickle.load(open(data_dir + 'xte1550_p5q4.pickle', 'rb'))
    else:
        carma_sample = make_sampler_plots(tsecs, logflux, logf_err, 7, 'xte1550_', sname, njobs=7)

    plt.subplot(111)
    pgram, freq = plt.psd(np.log(flux0), 512, 2.0 / np.median(dt), detrend=detrend_mean)
    plt.clf()

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                             color='SkyBlue', nsamples=5000)
    psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
    ax.loglog(freq / 2, pgram, 'o', color='DarkOrange')
    nyquist_freq = np.mean(0.5 / dt_idx)
    nyquist_idx = np.where(frequencies <= nyquist_freq)[0]
    ax.loglog(frequencies, psd_mle, '--b', lw=2)
    # noise_level = 2.0 * np.mean(dt_idx) * np.mean(logf_err ** 2)
    noise_level0 = 0.00018002985939372774 / 2.0  # scale the noise level seen in the PSD
    noise_level = noise_level0 * (0.5 / np.median(dt)) / nyquist_freq
    ax.loglog(frequencies[nyquist_idx], np.ones(len(nyquist_idx)) * noise_level, color='grey', lw=2)
    # ax.loglog(frequencies, np.ones(len(frequencies)) * noise_level0)
    ax.set_ylim(bottom=noise_level0 / 10.0)
    ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 1.5))
    ax.set_xlabel('Frequency [Hz]')
    ax.set_ylabel('Power Spectral Density [fraction$^2$ Hz$^{-1}$]')
    plt.title(sname)

    plt.savefig(base_dir + 'plots/xte1550_psd.eps')

    # plot the standardized residuals and compare with the standard normal
    plt.clf()
    kfilter, mu = carma_sample.makeKalmanFilter('map')
    kfilter.Filter()
    kmean = np.asarray(kfilter.GetMean())
    kvar = np.asarray(kfilter.GetVar())
    standardized_residuals = (carma_sample.y - mu - kmean) / np.sqrt(kvar)
    plt.hist(standardized_residuals, bins=100, normed=True, color='SkyBlue', histtype='stepfilled')
    plt.xlabel('Standardized Residuals')
    plt.ylabel('Probability Distribution')
    xlim = plt.xlim()
    xvalues = np.linspace(xlim[0], xlim[1], num=100)
    expected_pdf = np.exp(-0.5 * xvalues ** 2) / np.sqrt(2.0 * np.pi)
    plt.plot(xvalues, expected_pdf, 'k', lw=3)
    plt.title(sname)
    plt.savefig(base_dir + 'plots/xte1550_resid_dist.eps')

    # plot the autocorrelation function of the residuals and compare with the 95% confidence intervals for white
    # noise
    plt.clf()
    maxlag = 50
    wnoise_upper = 1.96 / np.sqrt(carma_sample.time.size)
    wnoise_lower = -1.96 / np.sqrt(carma_sample.time.size)
    plt.fill_between([0, maxlag], wnoise_upper, wnoise_lower, facecolor='grey')
    lags, acf, not_needed1, not_needed2 = plt.acorr(standardized_residuals, maxlags=maxlag, lw=3)
    plt.xlim(0, maxlag)
    plt.ylim(-0.2, 0.2)
    plt.xlabel('Time Lag')
    plt.ylabel('ACF of Residuals')
    plt.savefig(base_dir + 'plots/xte1550_resid_acf.eps')

    # plot the autocorrelation function of the squared residuals and compare with the 95% confidence intervals for
    # white noise
    plt.clf()
    squared_residuals = standardized_residuals ** 2
    wnoise_upper = 1.96 / np.sqrt(carma_sample.time.size)
    wnoise_lower = -1.96 / np.sqrt(carma_sample.time.size)
    plt.fill_between([0, maxlag], wnoise_upper, wnoise_lower, facecolor='grey')
    lags, acf, not_needed1, not_needed2 = plt.acorr(squared_residuals - squared_residuals.mean(), maxlags=maxlag,
                                                    lw=3)
    plt.xlim(0, maxlag)
    plt.ylim(-0.2, 0.2)
    plt.xlabel('Time Lag')
    plt.ylabel('ACF of Sqrd. Resid.')
    plt.savefig(base_dir + 'plots/xte1550_sqrres_acf.eps')

    if not load_pickle:
        pfile = open(data_dir + 'xte1550_nonoise.pickle', 'wb')
        cPickle.dump(carma_sample, pfile)
        pfile.close()
Exemple #8
0
def do_AGN_Kepler():

    sname = 'Zw 229-15'
    data = fits.open(data_dir + 'kepler_zw229_Q7.fits')[1].data
    jdate = data['time']
    flux = np.array(data['SAP_FLUX'], dtype=float)
    ferr = np.array(data['SAP_FLUX_ERR'], dtype=float)

    keep = np.where(np.logical_and(np.isfinite(jdate), np.isfinite(flux)))[0]
    jdate = jdate[keep]
    jdate -= jdate.min()
    flux = flux[keep]
    ferr = ferr[keep]

    df = flux[1:] - flux[0:-1]  # remove outliers
    keep = np.where(np.abs(df) < 56.0)
    jdate = jdate[keep]
    flux = flux[keep]
    ferr = ferr[keep]

    load_pickle = True
    if load_pickle:
        carma_sample = cPickle.load(open(data_dir + 'zw229.pickle', 'rb'))
        # rerun MLE
        # carma_model = cm.CarmaModel(jdate, flux, ferr, p=carma_sample.p, q=carma_sample.q)
        # mle = carma_model.get_map(carma_sample.p, carma_sample.q)
        # carma_sample.add_map(mle)
    else:
        carma_sample = make_sampler_plots(jdate, flux, ferr, 7, 'zw229_', sname, njobs=1)

    # transform the flux through end matching
    tflux = flux - flux[0]
    slope = (tflux[-1] - tflux[0]) / (jdate[-1] - jdate[0])
    tflux -= slope * jdate

    plt.subplot(111)
    dt = jdate[1] - jdate[0]
    pgram, freq = plt.psd(tflux, 512, 2.0 / dt, detrend=detrend_mean)
    plt.clf()

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                             color='SkyBlue', nsamples=5000)
    psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
    ax.loglog(freq / 2.0, pgram, 'o', color='DarkOrange')
    psd_slope = 3.14
    above_noise = np.where(freq / 2.0 < 1.0)[0]
    psd_norm = np.mean(np.log(pgram[above_noise[1:]]) + 3.14 * np.log(freq[above_noise[1:]] / 2.0))
    psd_plaw = np.exp(psd_norm) / (freq[1:] / 2.0) ** psd_slope
    ax.loglog(freq[1:] / 2.0, psd_plaw, '-', lw=2, color='DarkOrange')
    ax.loglog(frequencies, psd_mid, '--b', lw=2)
    noise_level = 2.0 * dt * np.mean(ferr ** 2)
    ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency [1 / day]')
    ax.set_ylabel('Power Spectral Density [flux$^2$ day]')
    plt.title(sname)
    plt.savefig(base_dir + 'plots/zw229_psd.eps')

    plt.clf()
    carma_sample.plot_1dpdf('measerr_scale')
    plt.savefig(base_dir + 'plots/zw229_measerr_scale.eps')
    measerr_scale = carma_sample.get_samples('measerr_scale')
    print "95% credibility interval on Kepler measurement error scale parameter:", np.percentile(measerr_scale, 2.5), \
        np.percentile(measerr_scale, 97.5)

    pfile = open(data_dir + 'zw229.pickle', 'wb')
    cPickle.dump(carma_sample, pfile)
    pfile.close()
Exemple #9
0
def do_simulated_irregular_nonstationary():

    # generate first half of lightcurve assuming a CARMA(5,3) process on a uniform grid
    sigmay1 = 2.3  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    qpo_width1 = np.array([1.0/100.0, 1.0/300.0, 1.0/200.0])
    qpo_cent1 = np.array([1.0/5.0, 1.0/25.0])
    ar_roots1 = cm.get_ar_roots(qpo_width1, qpo_cent1)
    ma_coefs1 = np.zeros(p)
    ma_coefs1[0] = 1.0
    ma_coefs1[1] = 4.5
    ma_coefs1[2] = 1.25
    sigsqr1 = sigmay1 ** 2 / cm.carma_variance(1.0, ar_roots1, ma_coefs=ma_coefs1)

    ny = 270
    time = np.zeros(ny)
    dt = np.random.uniform(1.0, 3.0, ny)
    time[0:90] = np.cumsum(dt[0:90])
    time[90:2*90] = 180 + time[90-1] + np.cumsum(dt[90:2*90])
    time[2*90:] = 180 + time[2*90-1] + np.cumsum(dt[2*90:])

    y = cm.carma_process(time, sigsqr1, ar_roots1, ma_coefs=ma_coefs1)

    # first generate some data assuming a CARMA(5,3) process on a uniform grid
    sigmay2 = 4.5  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    qpo_width2 = np.array([1.0/100.0, 1.0/100.0, 1.0/500.0])
    qpo_cent2 = np.array([1.0/5.0, 1.0/50.0])
    ar_roots2 = cm.get_ar_roots(qpo_width2, qpo_cent2)
    ma_coefs2 = np.zeros(p)
    ma_coefs2[0] = 1.0
    ma_coefs2[1] = 4.5
    ma_coefs2[2] = 1.25
    sigsqr2 = sigmay2 ** 2 / cm.carma_variance(1.0, ar_roots2, ma_coefs=ma_coefs2)

    ny = 270
    time2 = np.zeros(ny)
    dt = np.random.uniform(1.0, 3.0, ny)
    time2[0:90] = np.cumsum(dt[0:90])
    time2[90:2*90] = 180 + time2[90-1] + np.cumsum(dt[90:2*90])
    time2[2*90:] = 180 + time2[2*90-1] + np.cumsum(dt[2*90:])

    time = np.append(time, time.max() + 180 + time2)

    y2 = cm.carma_process(time2, sigsqr2, ar_roots2, ma_coefs=ma_coefs2)

    y = np.append(y, y2)

    ysig = np.ones(len(y)) * y.std() / 8.0
    # ysig = np.ones(ny) * 1e-6
    y0 = y.copy()
    y += ysig * np.random.standard_normal(len(y))

    froot = base_dir + 'plots/car5_nonstationary_'

    plt.subplot(111)
    for i in xrange(6):
        plt.plot(time[90*i:90*(i+1)], y0[90*i:90*(i+1)], 'k', lw=2)
        plt.plot(time[90*i:90*(i+1)], y[90*i:90*(i+1)], 'bo')

    plt.xlim(time.min(), time.max())
    plt.xlabel('Time')
    plt.ylabel('Non-Stationary Process')
    plt.savefig(froot + 'tseries.eps')
    plt.show()

    ar_coef1 = np.poly(ar_roots1)
    ar_coef2 = np.poly(ar_roots2)

    print 'Getting maximum-likelihood estimates...'

    carma_model = cm.CarmaModel(time, y, ysig)
    pmax = 7
    MAP, pqlist, AIC_list = carma_model.choose_order(pmax, njobs=1)

    # convert lists to a numpy arrays, easier to manipulate
    pqarray = np.array(pqlist)
    pmodels = pqarray[:, 0]
    qmodels = pqarray[:, 1]
    AICc = np.array(AIC_list)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max()+1):
        plt.plot(pmodels[qmodels == i], AICc[qmodels == i], 's-', label='q=' + str(i), lw=2)
    plt.legend()
    plt.xlabel('p')
    plt.ylabel('AICc(p,q)')
    plt.xlim(0, pmodels.max() + 1)
    plt.savefig(froot + 'aic.eps')
    plt.close()

    nsamples = 50000
    carma_sample = carma_model.run_mcmc(nsamples)
    carma_sample.add_mle(MAP)

    cPickle.dump(carma_sample, open(data_dir + 'nonstationary.pickle', 'wb'))

    plt.clf()
    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                             color='SkyBlue', nsamples=5000)
    psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
    psd1 = cm.power_spectrum(frequencies, np.sqrt(sigsqr1), ar_coef1, ma_coefs=ma_coefs1)
    psd2 = cm.power_spectrum(frequencies, np.sqrt(sigsqr2), ar_coef2, ma_coefs=ma_coefs2)
    # ax.loglog(frequencies, psd_mle, '--b', lw=2)
    ax.loglog(frequencies, psd1, 'k', lw=2)
    ax.loglog(frequencies, psd2, '--k', lw=2)
    dt = np.median(time[1:] - time[0:-1])
    noise_level = 2.0 * dt * np.mean(ysig ** 2)
    ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency')
    ax.set_ylabel('Power Spectral Density')

    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    carma_sample.assess_fit(doShow=False)
    plt.savefig(froot + 'fit_quality.eps')

    # compute the marginal mean and variance of the predicted values
    nplot = 1028
    time_predict = np.linspace(time.min(), 1.25 * time.max(), nplot)
    time_predict = time_predict[1:]
    predicted_mean, predicted_var = carma_sample.predict(time_predict, bestfit='map')
    predicted_low = predicted_mean - np.sqrt(predicted_var)
    predicted_high = predicted_mean + np.sqrt(predicted_var)

    # plot the time series and the marginal 1-sigma error bands
    plt.clf()
    plt.subplot(111)
    plt.fill_between(time_predict, predicted_low, predicted_high, color='cyan')
    plt.plot(time_predict, predicted_mean, '-b', label='Predicted')
    plt.plot(time[0:90], y0[0:90], 'k', lw=2, label='True')
    plt.plot(time[0:90], y[0:90], 'bo')
    for i in xrange(1, 3):
        plt.plot(time[90*i:90*(i+1)], y0[90*i:90*(i+1)], 'k', lw=2)
        plt.plot(time[90*i:90*(i+1)], y[90*i:90*(i+1)], 'bo')

    plt.xlabel('Time')
    plt.ylabel('CARMA(5,3) Process')
    plt.xlim(time_predict.min(), time_predict.max())
    plt.legend()
    plt.savefig(froot + 'interp.eps')
Exemple #10
0
def make_sampler_plots(time, y, ysig, pmax, file_root, title, do_mags=False, njobs=-1):

    froot = base_dir + 'plots/' + file_root

    # clean data
    dt = time[1:] - time[0:-1]
    if np.sum(dt <= 0) > 0:
        time = time[dt > 0]
        y = y[dt > 0]
        ysig = ysig[dt > 0]

    good = np.where(np.isfinite(time))[0]
    time = time[good]
    y = y[good]
    ysig = ysig[good]

    good = np.where(np.isfinite(y))[0]
    time = time[good]
    y = y[good]
    ysig = ysig[good]

    good = np.where(np.isfinite(ysig))[0]
    time = time[good]
    y = y[good]
    ysig = ysig[good]

    print 'Getting maximum-likelihood estimates...'

    carma_model = cm.CarmaModel(time, y, ysig)
    MAP, pqlist, AIC_list = carma_model.choose_order(pmax, njobs=njobs)

    # convert lists to a numpy arrays, easier to manipulate
    pqarray = np.array(pqlist)
    pmodels = pqarray[:, 0]
    qmodels = pqarray[:, 1]
    AICc = np.array(AIC_list)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max()+1):
        plt.plot(pmodels[qmodels == i], AICc[qmodels == i], 's-', label='q=' + str(i), lw=2)
    plt.legend(loc='best')
    plt.xlabel('p')
    plt.ylabel('AICc(p,q)')
    plt.xlim(0, pmodels.max() + 1)
    plt.title(title)
    plt.savefig(froot + 'aic.eps')
    plt.close()

    # make sure to change these back!!!!
    # carma_model.p = 7
    # carma_model.q = 3

    nsamples = 50000
    carma_sample = carma_model.run_mcmc(nsamples)
    carma_sample.add_mle(MAP)

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                             color='SkyBlue', nsamples=5000)
    psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
    ax.loglog(frequencies, psd_mle, '--b', lw=2)
    dt = time[1:] - time[0:-1]
    noise_level = 2.0 * np.median(dt) * np.mean(ysig ** 2)
    ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    do_s82 = True
    if do_s82:
        ax.annotate("Measurement Noise Level", (3.0e-2, 1e-2))
    else:
        ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency [1 / day]')
    if do_mags:
        ax.set_ylabel('Power Spectral Density [mag$^2$ day]')
    else:
        ax.set_ylabel('Power Spectral Density [flux$^2$ day]')
    plt.title(title)
    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    fig = carma_sample.assess_fit(doShow=False)
    ax_again = fig.add_subplot(2, 2, 1)
    ax_again.set_title(title)
    if do_mags:
        ylims = ax_again.get_ylim()
        ax_again.set_ylim(ylims[1], ylims[0])
        ax_again.set_ylabel('magnitude')
    else:
        ax_again.set_ylabel('ln Flux')
    plt.savefig(froot + 'fit_quality.eps')

    return carma_sample
Exemple #11
0
def do_simulated_regular():

    # first generate some data assuming a CARMA(5,3) process on a uniform grid
    sigmay = 2.3  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    qpo_width = np.array([1.0/100.0, 1.0/100.0, 1.0/500.0])
    qpo_cent = np.array([1.0/5.0, 1.0/50.0])
    ar_roots = cm.get_ar_roots(qpo_width, qpo_cent)
    ma_coefs = np.zeros(p)
    ma_coefs[0] = 1.0
    ma_coefs[1] = 4.5
    ma_coefs[2] = 1.25
    sigsqr = sigmay ** 2 / cm.carma_variance(1.0, ar_roots, ma_coefs=ma_coefs)

    ny = 1028
    time = np.arange(0.0, ny)
    y0 = cm.carma_process(time, sigsqr, ar_roots, ma_coefs=ma_coefs)

    ysig = np.ones(ny) * np.sqrt(1e-2)
    # ysig = np.ones(ny) * np.sqrt(1e-6)

    y = y0 + ysig * np.random.standard_normal(ny)

    froot = base_dir + 'plots/car5_regular_'

    plt.subplot(111)
    plt.plot(time, y0, 'k-')
    plt.plot(time, y, '.')
    plt.xlim(time.min(), time.max())
    plt.xlabel('Time')
    plt.ylabel('CARMA(5,3) Process')
    plt.savefig(froot + 'tseries.eps')

    ar_coef = np.poly(ar_roots)

    print 'Getting maximum-likelihood estimates...'

    carma_model = cm.CarmaModel(time, y, ysig)
    pmax = 7
    MAP, pqlist, AIC_list = carma_model.choose_order(pmax, njobs=-1)

    # convert lists to a numpy arrays, easier to manipulate
    pqarray = np.array(pqlist)
    pmodels = pqarray[:, 0]
    qmodels = pqarray[:, 1]
    AICc = np.array(AIC_list)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max()+1):
        plt.plot(pmodels[qmodels == i], AICc[qmodels == i], 's-', label='q=' + str(i), lw=2)
    plt.legend()
    plt.xlabel('p')
    plt.ylabel('AICc(p,q)')
    plt.xlim(0, pmodels.max() + 1)
    plt.savefig(froot + 'aic.eps')
    plt.close()

    nsamples = 50000
    carma_sample = carma_model.run_mcmc(nsamples)
    carma_sample.add_mle(MAP)

    plt.subplot(111)
    pgram, freq = plt.psd(y)
    plt.clf()

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                             color='SkyBlue', nsamples=5000)
    psd_mle = cm.power_spectrum(frequencies, carma_sample.mle['sigma'], carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(carma_sample.mle['ma_coefs']))
    ax.loglog(freq / 2.0, pgram, 'o', color='DarkOrange')
    psd = cm.power_spectrum(frequencies, np.sqrt(sigsqr), ar_coef, ma_coefs=ma_coefs)
    ax.loglog(frequencies, psd, 'k', lw=2)
    ax.loglog(frequencies, psd_mle, '--b', lw=2)
    noise_level = 2.0 * np.mean(ysig ** 2)
    ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency')
    ax.set_ylabel('Power Spectral Density')

    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    carma_sample.assess_fit(doShow=False)
    plt.savefig(froot + 'fit_quality.eps')

    pfile = open(data_dir + froot + '.pickle', 'wb')
    cPickle.dump(carma_sample, pfile)
    pfile.close()
data = np.genfromtxt(data_dir + 'zcarma5_test.dat')

Zcarma = carmcmc.ZCarmaSample(data[:, 0], data[:, 1], data[:, 2], filename=fname)

Zcarma.assess_fit()

print "True value of log10(kappa) is: ", np.log10(kappa)
plt.hist(Zcarma.get_samples('kappa'), bins=100)
plt.show()

Zcarma.plot_parameter('kappa', doShow=True)

psd_low, psd_high, psd_mid, freq = Zcarma.plot_power_spectrum(percentile=95.0, nsamples=10000, doShow=False)

ar_coef = np.poly(ar_roots)
true_psd = carmcmc.power_spectrum(freq, np.sqrt(sigsqr), ar_coef, ma_coefs=ma_coefs)
plt.loglog(freq, true_psd, 'r', lw=2)
plt.xlabel('Frequency')
plt.ylabel('PSD, ZCARMA(5)')
plt.show()

fname = data_dir + 'carma_mcmc.dat'
Carma = carmcmc.CarmaSample(data[:, 0], data[:, 1], data[:, 2], filename=fname, q=p-1)

Carma.assess_fit()
print "True values of MA coefs are:", ma_coefs
Carma.plot_1dpdf('ma_coefs', doShow=True)
Carma.posterior_summaries('ma_coefs')
print ''
print "True values of log_widths are", np.log(qpo_width)
Carma.posterior_summaries('log_width')
Exemple #13
0
def do_simulated_regular():

    # first generate some data assuming a CAR(5) process on a uniform grid
    sigmay = 2.3  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    qpo_width = np.array([1.0/100.0, 1.0/100.0, 1.0/500.0])
    qpo_cent = np.array([1.0/5.0, 1.0/50.0])
    ar_roots = cm.get_ar_roots(qpo_width, qpo_cent)
    ma_coefs = np.zeros(p)
    ma_coefs[0] = 1.0
    ma_coefs[1] = 4.5
    ma_coefs[2] = 1.25
    sigsqr = sigmay ** 2 / cm.carma_variance(1.0, ar_roots, ma_coefs=ma_coefs)

    ny = 1028
    time = np.arange(0.0, ny)
    y0 = cm.carma_process(time, sigsqr, ar_roots, ma_coefs=ma_coefs)

    ysig = np.ones(ny) * np.sqrt(1e-3 / 2.0)

    y = y0 + ysig * np.random.standard_normal(ny)

    data = (time, y, ysig)

    froot = base_dir + 'car5_regular_'

    plt.subplot(111)
    plt.plot(time, y0, 'k-')
    plt.plot(time, y, '.')
    plt.xlim(time.min(), time.max())
    plt.xlabel('Time')
    plt.ylabel('CAR(5) Process')
    plt.savefig(froot + 'tseries.eps')

    ar_coef = np.poly(ar_roots)

    pool = mp.Pool(mp.cpu_count()-1)

    args = []
    maxp = 8
    for p in xrange(1, maxp + 1):
        for q in xrange(p):
            args.append(((p, q), data))

    print "Running the CARMA MCMC samplers..."

    carma_run = pool.map(run_carma_sampler, args)

    dic = []
    pmodels = []
    qmodels = []
    for crun in carma_run:
        dic.append(crun.DIC())
        pmodels.append(crun.p)
        qmodels.append(crun.q)

    pmodels = np.array(pmodels)
    qmodels = np.array(qmodels)
    dic = np.array(dic)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max()+1):
        plt.plot(pmodels[qmodels == i], dic[qmodels == i], label='q=' + str(i), lw=2)

    plt.legend()
    plt.xlabel('p')
    plt.ylabel('DIC')
    print "DIC", dic
    plt.savefig(froot + 'dic.eps')

    carma = carma_run[np.argmin(dic)]

    print "order of best model is", carma.p

    plt.subplot(111)
    pgram, freq = plt.psd(y)
    plt.clf()

    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                      color='SkyBlue')
    ax.loglog(freq / 2.0, pgram, 'o', color='DarkOrange')
    psd = cm.power_spectrum(frequencies, np.sqrt(sigsqr), ar_coef, ma_coefs=ma_coefs)
    ax.loglog(frequencies, psd, 'k', lw=2)
    ax.loglog(frequencies, psd_mid, '--b', lw=2)
    noise_level = np.mean(ysig ** 2)
    ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency')
    ax.set_ylabel('Power Spectral Density')

    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    carma.assess_fit(doShow=False)
    plt.savefig(froot + 'fit_quality.eps')
Exemple #14
0
def do_simulated_irregular():

    # first generate some data assuming a CAR(5) process on a uniform grid
    sigmay = 2.3  # dispersion in lightcurve
    p = 5  # order of AR polynomial
    mu = 17.0  # mean of time series
    qpo_width = np.array([1.0/100.0, 1.0/300.0, 1.0/200.0])
    qpo_cent = np.array([1.0/5.0, 1.0/25.0])
    ar_roots = cm.get_ar_roots(qpo_width, qpo_cent)
    ma_coefs = np.zeros(p)
    ma_coefs[0] = 1.0
    ma_coefs[1] = 4.5
    ma_coefs[2] = 1.25
    sigsqr = sigmay ** 2 / cm.carma_variance(1.0, ar_roots, ma_coefs=ma_coefs)

    ny = 270
    time = np.empty(ny)
    dt = np.random.uniform(1.0, 3.0, ny)
    time[0:90] = np.cumsum(dt[0:90])
    time[90:2*90] = 180 + time[90-1] + np.cumsum(dt[90:2*90])
    time[2*90:] = 180 + time[2*90-1] + np.cumsum(dt[2*90:])

    y = mu + cm.carma_process(time, sigsqr, ar_roots, ma_coefs=ma_coefs)

    ysig = np.ones(ny) * y.std() / 5.0
    ysig = np.ones(ny) * 1e-6
    y0 = y.copy()
    y += ysig * np.random.standard_normal(ny)

    data = (time, y, ysig)

#    car5_model = cm.CarmaMCMC(time, y, ysig, 5, 5000, doZcarma=True, nburnin=1000)
#    zcar = car5_model.RunMCMC()

    froot = base_dir + 'car5_irregular_'

    plt.subplot(111)
    for i in xrange(3):
        plt.plot(time[90*i:90*(i+1)], y0[90*i:90*(i+1)], 'k', lw=2)
        plt.plot(time[90*i:90*(i+1)], y[90*i:90*(i+1)], 'bo')

    plt.xlim(time.min(), time.max())
    plt.xlabel('Time')
    plt.ylabel('CAR(5) Process')
    plt.savefig(froot + 'tseries.eps')

    ar_coef = np.poly(ar_roots)

    pool = mp.Pool(mp.cpu_count()-1)

    args = []
    maxp = 9
    for p in xrange(1, maxp + 1):
        for q in xrange(p):
            args.append((p, q, data))

    print "Running the CARMA MCMC samplers..."

    carma_run = pool.map(run_carma_sampler, args)

    dic = []
    pmodels = []
    qmodels = []
    for crun in carma_run:
        dic.append(crun.DIC())
        pmodels.append(crun.p)
        qmodels.append(crun.q)

    plt.clf()
    plt.subplot(111)
    for i in xrange(qmodels.max()+1):
        plt.plot(pmodels[qmodels == i], dic[qmodels == i], label='q=' + str(i), lw=2)

    plt.legend()
    plt.xlabel('p')
    plt.ylabel('DIC')
    print "DIC", dic
    plt.savefig(froot + 'dic.eps')

    carma = carma_run[np.argmin(dic)]

    print "order of best model is", carma.p

    plt.clf()
    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma.plot_power_spectrum(percentile=95.0, sp=ax, doShow=False,
                                                                      color='SkyBlue')
    psd = cm.power_spectrum(frequencies, np.sqrt(sigsqr), ar_coef, ma_coefs=ma_coefs)
    ax.loglog(frequencies, psd_mid, '--b', lw=2)
    ax.loglog(frequencies, psd, 'k', lw=2)
    noise_level = np.mean(ysig ** 2) * dt.min()
    ax.loglog(frequencies, np.ones(frequencies.size) * noise_level, color='grey', lw=2)
    ax.set_ylim(bottom=noise_level / 100.0)
    ax.annotate("Measurement Noise Level", (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency')
    ax.set_ylabel('Power Spectral Density')

    plt.savefig(froot + 'psd.eps')

    print 'Assessing the fit quality...'
    carma.assess_fit(doShow=False)
    plt.savefig(froot + 'fit_quality.eps')

    # compute the marginal mean and variance of the predicted values
    nplot = 1028
    time_predict = np.linspace(time.min(), 1.25 * time.max(), nplot)
    time_predict = time_predict[1:]
    predicted_mean, predicted_var = carma.predict_lightcurve(time_predict, bestfit='map')
    predicted_low = predicted_mean - np.sqrt(predicted_var)
    predicted_high = predicted_mean + np.sqrt(predicted_var)

    # plot the time series and the marginal 1-sigma error bands
    plt.clf()
    plt.subplot(111)
    plt.fill_between(time_predict, predicted_low, predicted_high, color='cyan')
    plt.plot(time_predict, predicted_mean, '-b', label='Predicted')
    plt.plot(time[0:90], y0[0:90], 'k', lw=2, label='True')
    plt.plot(time[0:90], y[0:90], 'bo')
    for i in xrange(1, 3):
        plt.plot(time[90*i:90*(i+1)], y0[90*i:90*(i+1)], 'k', lw=2)
        plt.plot(time[90*i:90*(i+1)], y[90*i:90*(i+1)], 'bo')

    plt.xlabel('Time')
    plt.ylabel('CAR(5) Process')
    plt.xlim(time_predict.min(), time_predict.max())
    plt.legend()
    plt.savefig(froot + 'interp.eps')
Zcarma.assess_fit()

print "True value of log10(kappa) is: ", np.log10(kappa)
plt.hist(Zcarma.get_samples('kappa'), bins=100)
plt.show()

Zcarma.plot_parameter('kappa', doShow=True)

psd_low, psd_high, psd_mid, freq = Zcarma.plot_power_spectrum(percentile=95.0,
                                                              nsamples=10000,
                                                              doShow=False)

ar_coef = np.poly(ar_roots)
true_psd = carmcmc.power_spectrum(freq,
                                  np.sqrt(sigsqr),
                                  ar_coef,
                                  ma_coefs=ma_coefs)
plt.loglog(freq, true_psd, 'r', lw=2)
plt.xlabel('Frequency')
plt.ylabel('PSD, ZCARMA(5)')
plt.show()

fname = data_dir + 'carma_mcmc.dat'
Carma = carmcmc.CarmaSample(data[:, 0],
                            data[:, 1],
                            data[:, 2],
                            filename=fname,
                            q=p - 1)

Carma.assess_fit()
print "True values of MA coefs are:", ma_coefs
Exemple #16
0
def run_CARMA(time,
              y,
              ysig,
              maxp,
              nsamples,
              aic_file,
              carma_sample_file,
              psd_file,
              psd_plot,
              fit_quality_plot,
              pl_plot,
              do_mags=True):
    #to calculate the order p and q of the CARMA(p,q) process, then run CARMA for values of p and q already calculated

    #function to calculate the order p and q of the CARMA(p,q) process
    # maxp: Maximum value allowed for p, maximun value for q is by default p-1

    time = time - time[0]

    model = cm.CarmaModel(time, y, ysig)
    MAP, pqlist, AIC_list = model.choose_order(maxp, njobs=1)

    # convert lists to a numpy arrays, easier to manipulate
    #the results of the AIC test are stored for future references.
    pqarray = np.array(pqlist)
    pmodels = pqarray[:, 0]
    qmodels = pqarray[:, 1]
    AICc = np.array(AIC_list)

    np.savetxt(aic_file,
               np.transpose([pmodels, qmodels, AICc]),
               header='p  q  AICc')

    p = model.p
    q = model.q

    #running the sampler
    carma_model = cm.CarmaModel(time, y, ysig, p=p, q=q)
    carma_sample = carma_model.run_mcmc(nsamples)
    carma_sample.add_mle(MAP)

    #getting the PSD
    ax = plt.subplot(111)
    print 'Getting bounds on PSD...'
    psd_low, psd_hi, psd_mid, frequencies = carma_sample.plot_power_spectrum(
        percentile=95.0, sp=ax, doShow=False, color='SkyBlue', nsamples=5000)

    psd_mle = cm.power_spectrum(frequencies,
                                carma_sample.mle['sigma'],
                                carma_sample.mle['ar_coefs'],
                                ma_coefs=np.atleast_1d(
                                    carma_sample.mle['ma_coefs']))

    #saving the psd
    np.savetxt(psd_file,
               np.transpose([frequencies, psd_low, psd_hi, psd_mid, psd_mle]),
               header='frequencies  psd_low  psd_hi  psd_mid psd_mle')

    ax.loglog(frequencies, psd_mle, '--b', lw=2)
    dt = time[1:] - time[0:-1]
    noise_level = 2.0 * np.median(dt) * np.mean(ysig**2)
    mean_noise_level = noise_level
    median_noise_level = 2.0 * np.median(dt) * np.median(ysig**2)
    ax.loglog(frequencies,
              np.ones(frequencies.size) * noise_level,
              color='grey',
              lw=2)
    ax.loglog(frequencies,
              np.ones(frequencies.size) * median_noise_level,
              color='green',
              lw=2)

    ax.set_ylim(bottom=noise_level / 100.0)

    ax.annotate("Measurement Noise Level",
                (3.0 * ax.get_xlim()[0], noise_level / 2.5))
    ax.set_xlabel('Frequency [1 / day]')
    if do_mags:
        ax.set_ylabel('Power Spectral Density [mag$^2$ day]')
    else:
        ax.set_ylabel('Power Spectral Density [flux$^2$ day]')
    #plt.title(title)
    plt.savefig(psd_plot)
    plt.close('all')

    print 'Assessing the fit quality...'
    fig = carma_sample.assess_fit(doShow=False)
    ax_again = fig.add_subplot(2, 2, 1)
    #ax_again.set_title(title)
    if do_mags:
        ylims = ax_again.get_ylim()
        ax_again.set_ylim(ylims[1], ylims[0])
        ax_again.set_ylabel('magnitude')
    else:
        ax_again.set_ylabel('ln Flux')
    plt.savefig(fit_quality_plot)

    pfile = open(carma_sample_file, 'wb')
    cPickle.dump(carma_sample, pfile)
    pfile.close()

    params = {
        param: carma_sample.get_samples(param)
        for param in carma_sample.parameters
    }
    params['p'] = model.p
    params['q'] = model.q

    print "fitting bending power-law"
    nf = np.where(psd_mid >= median_noise_level)

    psdfreq = frequencies[nf]
    psd_low = psd_low[nf]
    psd_hi = psd_hi[nf]
    psd_mid = psd_mid[nf]

    A, v_bend, a_low, a_high, blpfit = fit_BendingPL(psdfreq, psd_mid)

    pl_init = models.BrokenPowerLaw1D(amplitude=2,
                                      x_break=0.002,
                                      alpha_1=1,
                                      alpha_2=2)
    fit = LevMarLSQFitter()
    pl = fit(pl_init, psdfreq, psd_mid)

    amplitude = pl.amplitude.value
    x_break = pl.x_break.value
    alpha_1 = pl.alpha_1.value
    alpha_2 = pl.alpha_2.value

    print amplitude, x_break, alpha_1, alpha_2

    print "BendingPL fit parameters = ", A, v_bend, a_low, a_high
    print "BrokenPL fit parameters = ", amplitude, x_break, alpha_1, alpha_2

    plt.clf()
    plt.subplot(111)
    plt.loglog(psdfreq, psd_mid, color='green')
    plt.fill_between(psdfreq, psd_low, psd_hi, facecolor='green', alpha=0.3)
    plt.plot(psdfreq, blpfit, 'r--', lw=2)
    plt.plot(psdfreq, pl(psdfreq), 'k--', lw=2)
    plt.savefig(pl_plot)
    plt.close('all')

    return (params, mean_noise_level, median_noise_level, A, v_bend, a_low,
            a_high, amplitude, x_break, alpha_1, alpha_2)