def mcmc(max_iter=50000):
logger = Logger().getLogger()
# Generate a random light curve.
#We don't need to generate time since we know the sampling rate is
#very regular (yes, TAOS II).
random = np.random
random.seed(2)
flux = random.random(50)
# Remember that, for a real light curve, we have to match the length
#of the light curve with the database light curves.
# Connect DB
start = time.time()
logger.info('Connecting..')
conn = psycopg2.connect(database='taos-ii', user='******')
cur = conn.cursor()
end = time.time()
db_conneting_time = end - start
# Search the DB.
logger.info('Start MCMC DB searching..')
start = time.time()
chi2_map = np.zeros((max_iter, 6))
# Search grid.
b = np.arange(41)
t = np.arange(11)
d = np.arange(84)
size = np.arange(52)
starting_point = [int(len(b)/2), int(len(t)/2), int(len(d)/2),
int(len(size)/2)]
starting_row = get_row(cur, starting_point)
pre_val_a, pre_p_a = chisquare(flux, starting_row[4])
pre_val_b, pre_p_b = chisquare(flux, starting_row[5])
pre_val_c, pre_p_c = chisquare(flux, starting_row[6])
pre_val = np.log10(pre_val_a) + np.log10(pre_val_b) + np.log10(pre_val_c)
pre_p = pre_p_a * pre_p_b * pre_p_c
chi2_map[0][0:4] = starting_row[0:4]
chi2_map[0][4] = pre_val
chi2_map[0][5] = pre_p
pre_index = starting_point
steps = 1
rejected = 0
total_query = 0
while steps < max_iter:
total_query += 1
cur_b = get_next_index(b, pre_index[0])
cur_t = get_next_index(t, pre_index[1])
cur_d = get_next_index(d, pre_index[2])
cur_size = get_next_index(size, pre_index[3])
cur_index = [cur_b, cur_t, cur_d, cur_size]
#print cur_index
cur_row = get_row(cur, cur_index)
cur_val_a, cur_p_a = chisquare(flux, cur_row[4])
cur_val_b, cur_p_b = chisquare(flux, cur_row[5])
cur_val_c, cur_p_c = chisquare(flux, cur_row[6])
cur_val = np.log10(cur_val_a) + np.log10(cur_val_b) + np.log10(cur_val_c)
cur_p = cur_p_a * cur_p_b * cur_p_c
#if cur_val > pre_val: # reject
if cur_p < pre_p: # reject
rejected += 1
continue
else:
# accept
criterion = np.random.random()
#if cur_val / pre_val <= criterion:
if cur_p / pre_p >= criterion:
#print cur_val, pre_val, criterion, cur_index, pre_index
pre_index = cur_index
prev_val = cur_val
# Set parameters
chi2_map[steps][0:4] = cur_row[0:4]
chi2_map[steps][4] = cur_val
chi2_map[steps][5] = cur_p
steps += 1
else:
# reject
#print cur_val, pre_val, cur_val / pre_val, criterion
rejected += 1
continue
#print chi2_map
logger.info('MCMC estimating is done.')
acceptance_rate = float(steps) / (steps + rejected)
#chi2_map[::,4] = np.log10(chi2_map[::,4])
end = time.time()
querying_time = end - start
fig = corner.corner(chi2_map[::,:6], labels=[r'$\Omega$', r'$\Delta t$', 'D', 'Size',
r'$\log(\chi^2)$', 'p-value'], show_titles=True)
fig.suptitle(('The number of sources in the contour: %d\n' +
'# The nubmer of total queries: %d\n\n' +
'# DB connecting runtime: %.3f seconds\n' +
'# Total runtime: %.3f seconds\n' +
'# Acceptance rate: %.3f') %
(max_iter, total_query, db_conneting_time, querying_time, acceptance_rate),
fontsize=15)
#pl.show()
pl.savefig('corner_random_mcmc_%d.pdf' % (max_iter))
logger.info('Done.')
cur.close()
conn.close()
def brute_force():
logger = Logger().getLogger()
# Generate a random light curve.
#We don't need to generate time since we know the sampling rate is
#very regular (yes, TAOS II).
random = np.random
random.seed(1)
flux = random.random(50)
# Remember that, for a real light curve, we have to match the length
#of the light curve with the database light curves.
# Connect DB
start = time.time()
logger.info('Connecting..')
conn = psycopg2.connect(database='taos-ii', user='******')
cur = conn.cursor()
end = time.time()
db_conneting_time = end - start
# Search the DB.
logger.info('Start DB searching..')
start = time.time()
cur.execute('''select b_real, t_real, d_real, size_real,
flux_a, flux_b, flux_c from test
where width > 0.2 and width < 0.5 and
depth > 0.2 and depth < 0.5 and
height > 0.2 and height < 0.5''')
rows = cur.fetchall()
end = time.time()
querying_time = end - start
logger.info('The number of rows: %d' % (len(rows)))
# Calculating chi^2
logger.info('Deriving Chi^2 Map..')
# Initialize the map
start = time.time()
chi2_map = np.zeros((len(rows), 12))
for i in range(len(rows)):
# Set parameters
chi2_map[i][0:4] = rows[i][0:4]
# Calculate chi^2
#We assume that the light curves are all normalized.
len_lc = len(rows[i][4])
half_len_lc = len_lc / 2
c_flux = flux[len(flux)/2 - half_len_lc:len(flux)/2 - half_len_lc + len_lc]
chi2_map[i][4], chi2_map[i][5] = chisquare(c_flux, rows[i][4])
len_lc = len(rows[i][5])
half_len_lc = len_lc / 2
c_flux = flux[len(flux)/2 - half_len_lc:len(flux)/2 - half_len_lc + len_lc]
chi2_map[i][6], chi2_map[i][7] = chisquare(c_flux, rows[i][5])
len_lc = len(rows[i][6])
half_len_lc = len_lc / 2
c_flux = flux[len(flux)/2 - half_len_lc:len(flux)/2 - half_len_lc + len_lc]
chi2_map[i][8], chi2_map[i][9] = chisquare(c_flux, rows[i][6])
#print chi2_map
chi2_map[::,4] = np.log10(chi2_map[::,4])
chi2_map[::,6] = np.log10(chi2_map[::,6])
chi2_map[::,8] = np.log10(chi2_map[::,8])
chi2_map[::,10] = chi2_map[::,4] + chi2_map[::,6] + chi2_map[::,8]
chi2_map[::,11] = chi2_map[::,5] * chi2_map[::,7] * chi2_map[::,9]
#print chi2_map
end = time.time()
estimating_time = end - start
fig = corner.corner(chi2_map[::,(0,1,2,3,10,11)],
labels=[r'$\Omega$', r'$\Delta t$', 'D', 'Size',
r'$\log(\chi^2)$', 'p-value'], show_titles=True)
fig.suptitle(('The number of sources in the contour: %d\n\n' +
'# DB connecting runtime: %.3f seconds\n' +
'# Querying runtime: %.3f seconds\n' +
'# Estimating runtime: %.3f seconds') %
(len(rows), db_conneting_time, querying_time, estimating_time),
fontsize=15)
#pl.show()
pl.savefig('corner_random_brute_force.pdf')
logger.info('Done.')
cur.close()
conn.close()
#print chi2_map
#chi2_map[::,4] = np.log10(chi2_map[::,4])
#print chi2_map
return chi2_map, ids
if __name__ == '__main__':
logger = Logger().getLogger()
# Connect to DB
conn = psycopg2.connect(database='taos-ii', user='******')
# loop for different SNRs.
SNRs = [np.inf, 30, 10, 5, 3]
for m in range(len(SNRs)):
logger.info('%s SNRs..' % (str(SNRs[m])))
# set the same seed for each SNR,
#so that the same sources are used.
np.random.seed(1)
true_id = []
pred_id = []
true = []
predicted = []
for i in range(10):
logger.info('\t%dth search..' % (i + 1))
# Select one random event (three light curves).
results, c_id = select_random(conn)
true_id.append(c_id)
__author__='kim'
import os
import sys
import glob
import psycopg2
import pylab as pl
import numpy as np
from occultation.utils.logger import Logger
from occultation.tape.tape3 import EEP
logger = Logger().getLogger()
logger.info('Connet to DB..')
conn = psycopg2.connect(host='localhost', port='5432', database='taos-ii',
user='******')
cur = conn.cursor()
# Drop the table if exists.
logger.info('Drop the existing table..')
cur.execute('DROP TABLE IF EXISTS synthetic2')
# Create the table.
cur.execute('''CREATE TABLE synthetic2 (id serial PRIMARY KEY,
b_index integer, t_index integer, a_index integer, d_index integer,
b float, t float, a float, d float,
width_a float, depth_a float, height_a float,
width_b float, depth_b float, height_b float,
width_c float, depth_c float, height_c float,
flux_a float[], flux_b float[], flux_c float[]);''')
def mcmc(conn, flux_a, flux_b, flux_c, max_iter=100):
logger = Logger().getLogger()
# Connect DB
cur = conn.cursor()
# Search the DB.
logger.info('Start MCMC DB searching..')
start = time.time()
chi2_map = np.zeros((max_iter, 6))
# Search grid.
b = np.linspace(-0.6, 0.6, 41)
t = np.linspace(-0.02, 0.025, 10)
a = [27.0, 28.0, 29.0, 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0,
37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, 44.0, 45.0, 46.0,
47.0, 48.0, 49.0, 50.0, 52.5, 55.0, 57.5, 60.0, 62.5, 65.0, 67.5,
70.0, 72.5, 75.0, 77.5, 80.0, 82.5, 85.0, 87.5, 90.0,
92.5, 95.0, 97.5,100.0,110.0,120.0,130.0,140.0, 150.0,
160.0,170.0,180.0,190.0,200.0,210.0,220.0,230.0,240.0,
250.0,260.0,270.0,280.0,290.0,300.0,310.0,320.0,330.0,
340.0,350.0,360.0,370.0,380.0,390.0,400.0,410.0,420.0,
430.0,440.0,450.0,460.0,470.0,480.0,490.0,500.0,525.0,
550.0,575.0,600.0,625.0,650.0,675.0,700.0,
725.0,750.0,775.0,800.0,825.0,850.0,875.0,
900.0,925.0,950.0,975.0,1000.0]
d = [0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2,
1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0,
2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5,
7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10.0,
11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0,
20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0,
27.0, 28.0, 29.0, 30.0]
#b = np.linspace(-0.6, 0.6, 41)
#t = np.linspace(-0.025, 0.025, 11)
#a = [30, 35, 40, 45, 50, 55, 65, 75, 85, 95,
# 100, 200, 300, 400, 500, 600, 800, 1000]
#d = [0.5, 1.0, 1.5, 2.0, 3.0, 4.5, 6.0, 7.5, 9.0,
# 12, 15, 18, 21, 24, 27, 30]
b = np.arange(len(b))
t = np.arange(len(t))
a = np.arange(len(a))
d = np.arange(len(d))
starting_point = [int(len(b)/2), int(len(t)/2), int(len(a)/2),
int(len(d)/2)]
starting_row = get_row(cur, starting_point)
len_lc = len(starting_row[4])
half_len_lc = len_lc / 2
flux_dummy = flux_a[len(flux_a)/2 - half_len_lc:len(flux_a)/2 - half_len_lc + len_lc]
a_pre_val, a_pre_p = chisquare(flux_dummy, starting_row[4])
len_lc = len(starting_row[5])
half_len_lc = len_lc / 2
flux_dummy = flux_b[len(flux_b)/2 - half_len_lc:len(flux_b)/2 - half_len_lc + len_lc]
b_pre_val, b_pre_p = chisquare(flux_dummy, starting_row[5])
len_lc = len(starting_row[6])
half_len_lc = len_lc / 2
flux_dummy = flux_c[len(flux_c)/2 - half_len_lc:len(flux_c)/2 - half_len_lc + len_lc]
c_pre_val, c_pre_p = chisquare(flux_dummy, starting_row[6])
pre_p = a_pre_p * b_pre_p * c_pre_p
chi2_map[0][0:4] = starting_row[0:4]
chi2_map[0][4] = a_pre_val + b_pre_val + c_pre_val
chi2_map[0][5] = pre_p
#chi2_map[0][4] = b_pre_val
#chi2_map[0][5] = b_pre_p
pre_index = starting_point
steps = 1
rejected = 0
total_query = 0
while steps < max_iter:
total_query += 1
cur_b = get_next_index(b, pre_index[0])
cur_t = get_next_index(t, pre_index[1])
cur_a = get_next_index(a, pre_index[2])
cur_d = get_next_index(d, pre_index[3])
cur_index = [cur_b, cur_t, cur_a, cur_d]
#print cur_index
cur_row = get_row(cur, cur_index)
len_lc = len(cur_row[4])
half_len_lc = len_lc / 2
flux_dummy = flux_a[len(flux_a)/2 - half_len_lc:len(flux_a)/2 - half_len_lc + len_lc]
a_cur_val, a_cur_p = chisquare(flux_dummy, cur_row[4])
len_lc = len(cur_row[5])
half_len_lc = len_lc / 2
flux_dummy = flux_b[len(flux_b)/2 - half_len_lc:len(flux_b)/2 - half_len_lc + len_lc]
b_cur_val, b_cur_p = chisquare(flux_dummy, cur_row[5])
len_lc = len(cur_row[6])
half_len_lc = len_lc / 2
flux_dummy = flux_c[len(flux_c)/2 - half_len_lc:len(flux_c)/2 - half_len_lc + len_lc]
c_cur_val, c_cur_p = chisquare(flux_dummy, cur_row[6])
cur_val = a_cur_val + b_cur_val + c_cur_val
cur_p = a_cur_p * b_cur_p * c_cur_p
#cur_val = a_cur_val + b_cur_val + c_cur_val
#cur_p = b_cur_p
#if cur_val > pre_val: # reject
if cur_p < pre_p: # reject
rejected += 1
continue
else:
# accept
criterion = np.random.random()
#if cur_val / pre_val <= criterion:
if cur_p / pre_p >= criterion:
#print cur_val, pre_val, criterion, cur_index, pre_index
pre_index = cur_index
prev_val = cur_val
# Set parameters
chi2_map[steps][0:4] = cur_row[0:4]
chi2_map[steps][4] = cur_val
chi2_map[steps][5] = cur_p
steps += 1
else:
# reject
#print cur_val, pre_val, cur_val / pre_val, criterion
rejected += 1
continue
#print chi2_map
logger.info('MCMC estimating is done.')
acceptance_rate = float(steps) / (steps + rejected)
chi2_map[::,4] = np.log10(chi2_map[::,4])
cur.close()
return chi2_map
# Apply the offset.
offset %= window_size
arranged_x = x[offset:]
# Reshape based on the size of the window.
arranged_x = arranged_x[:(arranged_x.size // window_size) *
window_size].reshape(-1, window_size)
# Derive mean values of each bin.
down_x = arranged_x.mean(axis=1)
# Return down-sampled (binned) results.
return down_x
logger = Logger().getLogger()
logger.info('Connect to DB')
conn = psycopg2.connect(host='localhost', port='5432', database='taos-ii',
user='******')
cur = conn.cursor()
base_sampling = 120
sampling = [10, 20, 30, 40]
obss = ['width', 'depth', 'height']
tels = [1, 2, 3]
#'''
# Drop the table if exists.
logger.info('Drop the existing table')
cur.execute('DROP TABLE IF EXISTS synthetic4')
# Create a SQL string for observables for different sampling in Hz.
# Sampling rates.
curr_samplings = [40, 30, 20, 10]
curr_samplings = [10]
# For different SNRs.
SNRs = [np.inf, 30, 10, 5, 3]
SNRs = [30, 10, 5, 3]
# The number of sources to estimate parameters.
n_random_source = 300
for curr_sampling in curr_samplings:
b_bin_steps = base_sampling / curr_sampling
for m in range(len(SNRs)):
start_time = time.time()
logger.info('%s Hz and %s SNRs..' %
(str(curr_sampling), str(SNRs[m])))
# Set the same seed for each SNR,
# so that the same sources are used.
np.random.seed(10)
true_id = []
pred_id = []
true = []
predicted = []
predicted_error = []
predicted_error_by_search_table = []
std_error = []
p_value = []
n_samples = []
for i in range(n_random_source):
import os
import sys
import psycopg2
import psycopg2.extras
import numpy as np
import pylab as pl
import matplotlib.gridspec as gridspec
from occultation.utils.logger import Logger
logger = Logger().getLogger()
logger.info('Connect to DB')
conn = psycopg2.connect(host='localhost', port='5432',
database='taos-ii', user='******')
logger.info('Select from DB')
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
base_sampling = 120
sampling = [20] # one of [10, 20, 30, 40]. ONLY ONE.
obss = ['width', 'depth', 'height']
tels = [1, 2, 3]
# Generate column names.
obs_string = []
for i in range(len(sampling)):
b_bin_steps = base_sampling / sampling[i]
for j in range(b_bin_steps):
for tel in tels:
for obs in obss:
return chi2_map, ids
if __name__ == '__main__':
logger = Logger().getLogger()
np.random.seed(1)
# Connect to DB
conn = psycopg2.connect(database='taos-ii', user='******')
true_id = []
pred_id = []
true = []
predicted = []
for i in range(1000):
logger.info('%dth search..' % (i + 1))
# Select one random event (three light curves).
results, c_id = select_random(conn)
true_id.append(c_id)
# Add noise to the selected event.
#Also, make the light curves longer than 200 data points.
results['flux_a'] = generate_event_lc(results['flux_a'])
results['flux_b'] = generate_event_lc(results['flux_b'])
results['flux_c'] = generate_event_lc(results['flux_c'])
# 20 Hz time sampling. Centered at zero.
a_time = np.arange(len(results['flux_a'])) / 20.
a_time -= np.mean(a_time)
b_time = np.arange(len(results['flux_b'])) / 20.
b_time -= np.mean(b_time)
c_time = np.arange(len(results['flux_c'])) / 20.