def __init__(self, path, reference_list_path='reference_list.txt',
out_db='mc_results.db'):
'''loads data and makes db'''
# get all results*.pik files
files = glob.glob(os.path.join(path, 'results*.pik'))
# make db for quick access
if not os.path.exists(out_db):
reference = np.loadtxt(reference_list_path)
# create new dbs
self.db = numpy_sql(out_db)
self.conn = self.db.cursor()
for f in files:
# get result number
num = ''.join(re.findall('\d',f))
self.conn.execute('''CREATE TABLE s%s (Real_SFH real, Real_age real, Real_Z real, chains array, log_post array)'''%num)
results = pik.load(open(f))
# put in db
for res in results:
if len(results[res]) < 1:
continue
samp = results[res][0]
row = (results[res][2][0], results[res][2][1], results[res][2][2],
adapt_array(samp.flatchain),adapt_array(samp.flatlnprobability))
self.conn.execute('INSERT INTO s%s VALUES (?,?,?,?,?)'%num, row)
self.conn.execute('CREATE UNIQUE INDEX i%s ON s%s (Real_SFH, Real_age, Real_Z)'%(num, num))
self.db.commit()
else:
self.db = numpy_sql(out_db)
self.conn = self.db.cursor()
# get tables
self.tables = []
for i in self.conn.execute('select * from sqlite_master').fetchall():
if i[0] == 'table':
self.tables.append(i[1])
def __init__(self, data, db_name='burst_dtau_10.db', have_dust=False,
have_losvd=False):
self.has_dust = have_dust
self.has_losvd = have_losvd
self.db_name = db_name
#check if data is right type
self.data = {}
#get mean data values to 1
self.norm = {}
self.norm_prior = {}
for i in data:
self.norm[i] = 1./data[i][:,1].mean()
self.norm_prior[i] = np.log10(self.norm[i])
self.data[i] = data[i].copy()
self.data[i][:,1] *= self.norm[i]
if self.data[i].shape[1] == 3:
# Propagate the uncertany
self.data[i][:,2] *= self.norm[i]
self.db = util.numpy_sql(db_name)
self._table_name = self.db.execute('select * from sqlite_master').fetchall()[0][1]
# Tell which models are avalible and how many galaxies to fit
self.models = {'burst': data.keys()}
# Get param range (tau, age, metal)
self.param_range = []
for column in ['tau', 'age', 'metalicity']:
self.param_range.append(np.sort(np.ravel(self.db.execute(
'Select DISTINCT %s FROM %s'%(column,self._table_name)).fetchall())))
self._hull = None
# make resolution
self._define_resolu()
def random_exp_gal(db_path='/home/thuso/Phd/experements/hierarical/LRG_Stack/exp_dtau_10.db'):
'''gets num_gal of exponetal model spectra without any background model'''
db = util.numpy_sql(db_path)
param = get_points(db)
data = tri_lin_interp(db, param, make_param_range(db))
# Make noise
return data, param
def random_exp_gal(
db_path='/home/thuso/Phd/experements/hierarical/LRG_Stack/exp_dtau_10.db'
):
'''gets num_gal of exponetal model spectra without any background model'''
db = util.numpy_sql(db_path)
param = get_points(db)
data = tri_lin_interp(db, param, make_param_range(db))
# Make noise
return data, param
def random_burst_gal(db_path='/home/thuso/Phd/experements/hierarical/LRG_Stack/burst_dtau_10.db'):
'''gets num_gal of burst model spectra without any background model'''
db = util.numpy_sql(db_path)
param = get_points(db)
data = tri_lin_interp(db, param[['tau','age','metalicity']], make_param_range(db))
# Make noise
# Change wavelength range
data = data[nu.where(nu.logical_and(data[:,0]>=3200, data[:,0] <= 9500))]
return data, param
def make_csp_lib(csp_type, csp_num=10, save_path='.'):
'''Makes a csp library. does n steps of tau in GYR to 0 -5 GYR.
With multicore processing
'''
# get burst function
assert csp_type.lower() in ['burst', 'exp'], "csp_type must be 'burst' or 'exp'"
if csp_type == 'burst':
model = make_burst
else:
model = make_exp
#pool = Pool()
tau = nu.linspace(0, 5, csp_num)
models = map(model, tau)
# save spectra to database
save_name = os.path.join(save_path, '%s_dtau_%d.db'%(csp_type,csp_num))
conn = numpy_sql(save_name)
c = conn.cursor()
#create table
c.execute('''CREATE TABLE %s (imf text, model text, tau real, age real,
metalicity real ,
spec array)'''%csp_type)
for csp in models:
for meta_gal in csp:
for age in nu.log10(1+meta_gal.ages):
if age == 0:
age = nu.log10(1*10**5)
if nu.isclose(10**(age-9), 20):
data = nu.vstack((meta_gal.ls, meta_gal.get_sed(20))).T
else:
data = nu.vstack((meta_gal.ls,
meta_gal.get_sed(10**(age-9)))).T
# reverse
data = data[-1:0:-1]
if 'length' in meta_gal.meta_data:
length = float(meta_gal.meta_data['length'])
elif 'tau' in meta_gal.meta_data:
length = float(meta_gal.meta_data['tau'])
else:
# if SSP
length = 0.
insert = (meta_gal.meta_data['imf'],meta_gal.meta_data['model']
, length, age,
nu.log10(float(meta_gal.meta_data['met'])), data,)
c.execute('INSERT INTO %s VALUES (?,?,?,?,?,?)'%csp_type, insert)
# save
conn.commit()
conn.close()
def random_burst_gal(
db_path='/home/thuso/Phd/experements/hierarical/LRG_Stack/burst_dtau_10.db'
):
'''gets num_gal of burst model spectra without any background model'''
db = util.numpy_sql(db_path)
param = get_points(db)
data = tri_lin_interp(db, param[['tau', 'age', 'metalicity']],
make_param_range(db))
# Make noise
# Change wavelength range
data = data[nu.where(nu.logical_and(data[:, 0] >= 3200,
data[:, 0] <= 9500))]
return data, param
def get_data(db_path):
'''retrives data from db.
returns matrix of params (ssp, n param), spec (ssp, wavelength)'''
db = util.numpy_sql(db_path)
# get table name
table_name = db.execute('select * from sqlite_master').fetchall()[0][1]
# fetch all
spec, param = [] ,[]
for imf, model, tau, age, metal, buf_spec in db.execute('SELECT * From %s'%table_name):
spec.append(util.convert_array(buf_spec)[:,1])
param.append([tau, age, metal])
param = np.array(param)
spec = np.array(spec)
return param, spec, util.convert_array(buf_spec)[:,0]
def get_data(db_path):
'''retrives data from db.
returns matrix of params (ssp, n param), spec (ssp, wavelength)'''
db = util.numpy_sql(db_path)
# get table name
table_name = db.execute('select * from sqlite_master').fetchall()[0][1]
# fetch all
spec, param = [], []
for imf, model, tau, age, metal, buf_spec in db.execute(
'SELECT * From %s' % table_name):
spec.append(util.convert_array(buf_spec)[:, 1])
param.append([tau, age, metal])
param = np.array(param)
spec = np.array(spec)
return param, spec, util.convert_array(buf_spec)[:, 0]
def __init__(self,
data,
db_name='burst_dtau_10.db',
have_dust=False,
have_losvd=False):
self.has_dust = have_dust
self.has_losvd = have_losvd
self.db_name = db_name
#check if data is right type
self.data = {}
#get mean data values to 1
self.norm = {}
self.norm_prior = {}
for i in data:
self.norm[i] = 1. / data[i][:, 1].mean()
self.norm_prior[i] = np.log10(self.norm[i])
self.data[i] = data[i].copy()
self.data[i][:, 1] *= self.norm[i]
if self.data[i].shape[1] == 3:
# Propagate the uncertany
self.data[i][:, 2] *= self.norm[i]
self.db = util.numpy_sql(db_name)
self._table_name = self.db.execute(
'select * from sqlite_master').fetchall()[0][1]
# Tell which models are avalible and how many galaxies to fit
self.models = {'burst': data.keys()}
# Get param range (tau, age, metal)
self.param_range = []
for column in ['tau', 'age', 'metalicity']:
self.param_range.append(
np.sort(
np.ravel(
self.db.execute(
'Select DISTINCT %s FROM %s' %
(column, self._table_name)).fetchall())))
self._hull = None
# make resolution
self._define_resolu()
def lik(self, param, bins, return_model=False, normalize=True):
'''Calculates log likelyhood for burst model'''
for gal in param[bins]:
#ipdb.set_trace()
# get interp spectra
#check if points are in range
columns = ['tau', 'age', 'metalicity']
if self.is_in_hull(param[bins][gal][columns]):
# check if db works
try:
self.db.execute('select * from sqlite_master').fetchall()[0][1]
except: #ProgrammingError:
#make new database connection
self.db = util.numpy_sql(self.db_name)
spec = tri_lin_interp(self.db,
param[bins][gal][columns], self.param_range)
else:
if return_model:
yield -np.inf, gal, []
else:
yield -np.inf, gal
continue
if normalize:
model = {'wave':spec[:,0],0: spec[:,1]}
else:
model = {'wave':spec[:,0],
0: spec[:,1] * 10**param[bins][gal]['normalization'].iat[0]}
# Redshift
model['wave'] = ag.redshift(model['wave'],
param[bins][gal]['redshift'])
# Dust
if self.has_dust:
columns = ['$T_{bc}$','$T_{ism}$']
model = ag.dust(param[bins][gal][columns].iloc[0],
model)
# LOSVD
if self.has_losvd:
# wave range for convolution
wave_range = [self.data[gal][:,0].min(),
self.data[gal][:,0].max()]
# check if resolution has been calculated
columns = ['$\\sigma$','$V$','$h_3$','$h_4$']
send_param = param[bins][gal][columns].iloc[0]
model = ag.LOSVD(model, send_param,
wave_range, self.resolu[gal])
#match data wavelengths with model
try:
model = ag.data_match(self.data[gal], model)
except:
model = ag.data_match(self.data[gal], model, rebin=False)
#calculate map for normalization
norm = ag.normalize(self.data[gal], model[0])
self.norm_prior[gal] = np.log10(norm)
if normalize:
model[0] *= norm
# Calc liklihood
if self.data[gal].shape[1] >= 3:
# Has Uncertanty
out_lik = stats_dist.norm.logpdf(
self.data[gal][:,1], model[0], self.data[gal][:,2])
else:
#no uncertanty or bad entry
out_lik = stats_dist.norm.logpdf(
self.data[gal][:,1], model[0])
if return_model:
yield out_lik.sum(), gal, model
else:
yield out_lik.sum(), gal
def lik(self, param, bins, return_model=False, normalize=True):
'''Calculates log likelyhood for burst model'''
for gal in param[bins]:
#ipdb.set_trace()
# get interp spectra
#check if points are in range
columns = ['tau', 'age', 'metalicity']
if self.is_in_hull(param[bins][gal][columns]):
# check if db works
try:
self.db.execute(
'select * from sqlite_master').fetchall()[0][1]
except: #ProgrammingError:
#make new database connection
self.db = util.numpy_sql(self.db_name)
spec = tri_lin_interp(self.db, param[bins][gal][columns],
self.param_range)
else:
if return_model:
yield -np.inf, gal, []
else:
yield -np.inf, gal
continue
if normalize:
model = {'wave': spec[:, 0], 0: spec[:, 1]}
else:
model = {
'wave': spec[:, 0],
0:
spec[:, 1] * 10**param[bins][gal]['normalization'].iat[0]
}
# Redshift
model['wave'] = ag.redshift(model['wave'],
param[bins][gal]['redshift'])
# Dust
if self.has_dust:
columns = ['$T_{bc}$', '$T_{ism}$']
model = ag.dust(param[bins][gal][columns].iloc[0], model)
# LOSVD
if self.has_losvd:
# wave range for convolution
wave_range = [
self.data[gal][:, 0].min(), self.data[gal][:, 0].max()
]
# check if resolution has been calculated
columns = ['$\\sigma$', '$V$', '$h_3$', '$h_4$']
send_param = param[bins][gal][columns].iloc[0]
model = ag.LOSVD(model, send_param, wave_range,
self.resolu[gal])
#match data wavelengths with model
try:
model = ag.data_match(self.data[gal], model)
except:
model = ag.data_match(self.data[gal], model, rebin=False)
#calculate map for normalization
norm = ag.normalize(self.data[gal], model[0])
self.norm_prior[gal] = np.log10(norm)
if normalize:
model[0] *= norm
# Calc liklihood
if self.data[gal].shape[1] >= 3:
# Has Uncertanty
out_lik = stats_dist.norm.logpdf(self.data[gal][:, 1],
model[0], self.data[gal][:,
2])
else:
#no uncertanty or bad entry
out_lik = stats_dist.norm.logpdf(self.data[gal][:, 1],
model[0])
if return_model:
yield out_lik.sum(), gal, model
else:
yield out_lik.sum(), gal