def repair_bad_variance_spectra(db_file):
con = sq3.connect(db_file)
cur = con.cursor()
sql_input = "SELECT * from Spectra inner join Events ON Spectra.SN = Events.SN"
print 'querying'
bad_ivars = []
SN_Array = composite.grab(sql_input,
multi_epoch=True,
make_corr=False,
selection='max_coverage',
grab_all=True)
for SN in SN_Array:
if len(np.where(np.isnan(SN.ivar))[0] == True) == 5500:
bad_ivars.append(SN)
for SN in bad_ivars:
print SN.source
if SN.source == 'bsnip':
path = '../data/spectra/bsnip/' + SN.filename
elif SN.source == 'cfa':
path = '../data/spectra/cfa/' + SN.filename.split(
'-')[0] + '/' + SN.filename
elif SN.source == 'other':
path = '../data/spectra/other/' + SN.filename
elif SN.source == 'uv':
path = '../data/spectra/uv/' + SN.filename
spectrum = np.loadtxt(path)
newdata, snr = prep.compprep(spectrum,
SN.name,
SN.redshift,
SN.source,
use_old_error=False,
testing=False)
try:
interped = msg.packb(newdata)
cur.execute("UPDATE Spectra SET snr = ? where filename = ?",
(snr.value, SN.filename))
cur.execute(
"UPDATE Spectra SET Interpolated_Spectra = ? where filename = ?",
(buffer(interped), SN.filename))
print "Added: ", SN.filename
except Exception, e:
print "Interp failed: ", SN.filename
def main():
data_file = '../data/spectra/swift_uvspec/swift_uv_log.txt'
files = glob.glob("..\data\spectra\swift_uvspec\*.flm")
con = sq3.connect('../data/kaepora_v1.db')
with open(data_file) as data:
data_dict = {}
for line in data.readlines()[1:]:
data_dict[line.split()[4]] = line.split()[0:4]
for spec_file in files:
with open(spec_file) as spec:
spectrum = np.loadtxt(spec)
source = 'swift_uv'
print spec_file
sn_data = data_dict[spec_file.split('\\')[4]]
if sn_data[0][0:2].lower() == 'sn':
sn_name = sn_data[0][2:]
else:
sn_name = sn_name = sn_data[0]
print sn_name
redshift = float(sn_data[1])
phase = float(sn_data[2])
Dm15 = sn_data[3]
if Dm15 == '-99':
Dm15 = None
else:
Dm15 = float(Dm15)
interp_spec, sig_noise = prep.compprep(spectrum, sn_name, redshift, source)
interped = msg.packb(interp_spec)
name = spec_file.split('\\')[4]
min_wave = spectrum[0][0]
max_wave = spectrum[-1][0]
mjd = None
ref = None
con.execute("""INSERT INTO Spectra(Filename, SN, Source,
Phase, MinWave, MaxWave, snr, Interpolated_Spectra,
MJD, Ref)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(name, sn_name, source, phase, min_wave, max_wave,
sig_noise, buffer(interped), mjd, ref))
con.commit()
if name in ref_dict:
ref = ref_dict[name]
waves = spectra[:, 0]
min_wave = waves[0]
max_wave = waves[len(spectra) - 1]
spec = msg.packb(spectra)
if sn_name in NED_red_dict:
redshift = NED_red_dict[sn_name]
try:
interp_spec, sig_noise = prep.compprep(spectra,
sn_name,
redshift,
source,
testing=False)
except Exception, e:
#raise
print e
print "Interp failed"
bad_interp.append(name)
bad_files.append(name)
interp_spec, sig_noise = None, None
print 'before', phase
if sn_name in sndict:
if sndict[sn_name][4] == '9.99':
Dm15 = None
else:
spectrum = np.loadtxt(spec)
source = 'maguire'
filename = sp.split('\\')[-1]
sn_obj_data = obj_dict[filename.split('_')[0]]
sn_spec_data = spec_dict[filename.split('_')[0]]
if filename.split('_')[0].lower() == 'sn':
sn_name = filename.split('_')[0][2:]
else:
sn_name = filename.split('_')[0]
print sn_name
redshift = float(sn_spec_data[4])
mjd_max = float(sn_obj_data[0])
mjd_spec = float(sn_spec_data[3])
phase = (mjd_spec - mjd_max) / (1. + redshift)
interp_spec, sig_noise = prep.compprep(spectrum, sn_name,
redshift, source)
interped = msg.packb(interp_spec)
name = spec_file.split('\\')[3]
min_wave = spectrum[0][0]
max_wave = spectrum[-1][0]
m_b = None
bm_vm = None
vel = None
morph = None
carbon = None
gasrich = None
hubble_residual = None
mjd = None
ref = 'Maguire et al. (2012)'
c = 299792.458
source = 'bsnip'
data = bsnip_vals[sn_name.lower()]
redshift = data[0]/c
phase = None
Dm15 = None
m_b = None
bm_vm = None
waves = spectra[:, 0]
min_wave = waves[0]
max_wave = waves[len(spectra) - 1]
spec = msg.packb(spectra)
try:
interp_spec, sig_noise = prep.compprep(spectra, sn_name, redshift, source)
except:
raise
print "Interp failed"
bad_interp.append(name)
bad_files.append(name)
interp_spec, sig_noise = None, None
if sn_name in negout:
carbon = 0
elif sn_name in posout:
carbon = 1
else:
carbon = None
if sn_name in morph_dict:
def main():
data_file = '../data/spectra/hst_foley/hst_metadata_spec.txt'
files = glob.glob("../data/spectra/hst_foley/*.flm")
con = sq3.connect('../data/kaepora_v1.db')
cur = con.cursor()
cur.execute("DELETE FROM Spectra where source = 'foley_hst'")
with open(data_file) as data:
data_dict = {}
for line in data.readlines():
data_dict[line.split()[0]] = line.split()
for spec_file in files:
with open(spec_file) as spec:
spectrum = np.loadtxt(spec)
source = 'foley_hst'
sn_data = data_dict[spec_file.split('\\')[1]]
if sn_data[0][0:2].lower() == 'sn':
sn_name = sn_data[0].split('-')[0][2:]
else:
sn_name = sn_data[0].split(
'-')[0] + '-' + sn_data[0].split('-')[1]
redshift = float(sn_data[2])
phase = float(sn_data[1])
Dm15 = None
interp_spec, sig_noise = prep.compprep(spectrum,
sn_name,
redshift,
source,
testing=False)
interped = msg.packb(interp_spec)
name = spec_file.split('\\')[1]
min_wave = spectrum[0][0]
max_wave = spectrum[-1][0]
sig_noise = sig_noise
print name, sn_name, phase, redshift, sig_noise
# plt.plot(interp_spec[0,:],interp_spec[1,:])
# plt.show()
# plt.plot(interp_spec[0,:],interp_spec[2,:])
# plt.show()
m_b = None
bm_vm = None
vel = None
morph = None
carbon = None
gasrich = None
hubble_residual = None
mjd = None
ref = sn_data[3]
# sql_input = "SELECT * from Supernovae inner join Photometry ON Supernovae.SN = Photometry.SN where filename=" + "'" + name + "'"
# cur.execute(sql_input)
# in_database = False
# for row in cur:
# in_database = row[0]
# if in_database:
# con.execute("""REPLACE INTO Supernovae(Filename, SN, Source,
# Phase, MinWave, MaxWave,
# snr, Interpolated_Spectra,
# MJD, Ref)
# VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
# (name, sn_name, source, phase, min_wave, max_wave,
# # sig_noise, buffer(interped), mjd, ref))
# else:
con.execute(
"""INSERT INTO Spectra(Filename, SN, Source,
Phase, MinWave, MaxWave,
snr, Interpolated_Spectra,
MJD, Ref)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(name, sn_name, source, phase, min_wave, max_wave,
sig_noise, buffer(interped), mjd, ref))
#with open(data_file_meta) as data_meta:
# data_dict = {}
# for line in data_meta.readlines():
# data_dict[line.split()[0]] = line.split()
# for sn in data_dict.keys():
# dm15 = float(data_dict[sn][1])
# dm15_err = float(data_dict[sn][2])
# v_at_max = float(data_dict[sn][3])
# av = float(data_dict[sn][5])*3.1
# ned_host = data_dict[sn][7]
# print sn, dm15, dm15_err, v_at_max, av, ned_host
# cur.execute("UPDATE Photometry SET dm15_source = ? where SN = ?", (dm15, sn))
# cur.execute("UPDATE Photometry SET e_dm15 = ? where SN = ?", (dm15_err, sn))
# cur.execute("UPDATE Photometry SET v_at_max = ? where SN = ?", (v_at_max, sn))
# cur.execute("UPDATE Photometry SET av_mlcs31 = ? where SN = ?", (av, sn))
# cur.execute("UPDATE Photometry SET NED_host = ? where SN = ?", (ned_host, sn))
con.commit()
def main():
# raise TypeError #uncomment when building db
#build necessary dictionaries
sndict, date_dict = read_cfa_info(
'../data/spectra/cfa/cfasnIa_param.dat',
'../data/spectra/cfa/cfasnIa_mjdspec.dat')
bsnip_vals = read_bsnip_data('obj_info_table.txt')
short_bsnip_dict = create_short_bsnip_dict(bsnip_vals)
rsd = build_redshift_dict(short_bsnip_dict, sndict)
NED_red_dict = build_NED_redshift_dict(
'../data/info_files/NED_redshift_info.txt')
ref_dict = build_bsnip_ref_dict()
ts = time.clock()
#con = sq3.connect('SNe.db')
# con = sq3.connect('SNe_11.db')
# con = sq3.connect('../data/SNIaDB_Spec_v20_phot_v10.db')
con = sq3.connect('../data/kaepora_v1.db')
#make sure no prior table in db to avoid doubling/multiple copies of same data
con.execute("""DROP TABLE IF EXISTS Spectra""")
con.execute("""CREATE TABLE IF NOT EXISTS Spectra (Filename
TEXT PRIMARY KEY, SN Text, Source Text,
Phase REAL, MinWave REAL, MaxWave REAL, snr REAL,
Interpolated_Spectra BLOB, MJD REAL, Ref Text)""")
#read all bsnip to find corrected
corr_list = []
allspec = []
for paths, subdirs, files in os.walk('../data/spectra/bsnip'):
for name in files:
if name.endswith('.flm'):
allspec.append(name)
if 'corrected' in name:
corr_list.append(name)
clean1 = [re.sub('-corrected.flm', '', x) for x in corr_list]
clean2 = [re.sub('.flm', '', x) for x in allspec]
#find files that have both corrected and raw
res = set(clean1).intersection(set(clean2))
#have_both contains files to ignore
have_both = []
for s in res:
s += '.flm'
have_both.append(s)
#change this depending on where script is
root = '../data/spectra'
bad_files = []
bad_interp = []
shiftless = []
#ignore bad files
spectra_ignore = [
'SN06mr_061113_g01_BAA_IM.dat', 'SN07af_070310_g01_BAA_IM.dat',
'SN07ai_070310_g01_BAA_IM.dat', 'SN07af_070310_g01_BAA_IM.dat',
'SN07ai_070312_g01_BAA_IM.dat', 'sn1996ai-19960620.17-fast.flm',
'sn1994T-19940612.25-mmt.flm', 'sn1994T-19940613.19-fast.flm',
'sn2000dn-20001006.25-fast.flm', 'sn2006bt-20060502.33-fast.flm',
'sn2006bt-20060503.33-fast.flm', 'sn1994Q-19940612.32-mmt.flm',
'sn2006H-20060206.21-fast.flm', 'sn2002dj-20020614.17-fast.flm',
'sn1999ek-19991030.47-fast.flm', 'sn2003cq-20030408.26-fast.flm',
'sn1996ai-19960620.17-fast.flm', 'SN05hc_051018_r01_NTT_EM.dat',
'2002er_20020901_3213_9175_00.dat', '2005cf_20050601_3243_9720_00.dat',
'sn1994S-19940612.26-mmt.flm', 'sn1995ac-19950929.27-fast.flm',
'sn2006lf-20061028.51-fast.flm', 'sn1994M-19940612.22-mmt.flm',
'2002er_20020901_3213_9175_00.dat', '2002bo_20020328_3099_8801_00.dat',
'2005cf_20050603_3721_8786_00.dat', 'sn1996ai-19960620-uo.flm',
'SN07N_070131_b01_NTT_EM.dat',
'sn2007sr-20071228.39-fixedbg-swift.flm',
'sn2006ob-20061201.107-ui.flm', 'sn2005A-20050107.25-fast.flm',
'sn1986g-19860529.301-iue.flm', 'sn1994t-19940715-ui.flm'
]
print "Adding information to table"
count = 1
for path, subdirs, files in os.walk(root):
for name in files:
#ignore bad spectra and uncorrected where corrected exist and save swift/hst for later
if name in spectra_ignore or name in have_both or 'swift_uvspec' in path or 'hst_foley' in path:
continue
#reset all fields to prevent carrying over from last loop iteration
Dm15 = None
m_b = None
bm_vm = None
phase = None
redshift = None
source = None
mjd = None
ref = None
f = os.path.join(path, name)
if f.endswith('.flm') or f.endswith('.dat'):
if 'cfasnIa' in f:
continue
try:
if 'csp' in path:
spectra, info = read_csp(f)
sn_name = find_SN(name, 'csp', info)
else:
spectra = read_cfa_or_bsnip_or_uv(f)
if 'other' in path:
sn_name = find_SN(name, 'other')
else:
sn_name = find_SN(name)
except:
bad_files.append(f)
continue
print count, sn_name
count += 1
#other source
if 'other' in f:
print f
source = 'other'
if 'snifs' in f:
with open(f) as otherfile:
lines = (line for line in f
if line.startswith('#'))
for line in lines:
if line.split()[1] == 'MJD':
mjd = float(line.split()[3])
if line.split()[1] == 'TMAX':
phase = float(line.split()[3])
if mjd is None:
if len(name.split('_')) > 1:
year, month, day = float(
name.split('_')[1][0:4]), float(
name.split('_')[1][4:6]), float(
name.split('_')[1][6:])
else:
year, month, day = float(
name.split('-')[1][0:4]), float(
name.split('-')[1][4:6]), float(
name.split('-')[1][6:])
mjd = jdcal.gcal2jd(year, month, day)[1]
if sn_name in rsd:
redshift = rsd[sn_name]
#cfa source
elif 'cfa' in f:
if 'sn2011' not in name:
sn_cfa = sndict[sn_name]
else:
snd = None
sn_cfa = [None] * 14
#csp source
if 'csp' in f:
print f
source = 'csp'
redshift = float(info[2])
phase = (float(info[4]) - float(info[3])) / (1. + redshift)
mjd = float(info[4]) - 2400000.5
ref = '2013ApJ...773...53F'
#uv source
elif 'uv' in path:
print f
print name
source = 'uv'
year = float(name.split('-')[1][0:4])
month = float(name.split('-')[1][4:6])
day = float(name.split('-')[1][6:])
mjd = jdcal.gcal2jd(year, month, day)[1]
if sn_name in rsd:
redshift = rsd[sn_name]
#cfa spectra
elif 'cfa' in f:
print f
source = 'cfa'
redshift = float(sn_cfa[0])
#if statements assign values to fields if they exist, else none is assigned
if sn_cfa[1] == '99999.9':
phase = None
else:
#try/except catches and fixes sn2011 errors
try:
phase = (float(date_dict[name]) -
float(sn_cfa[1])) / (1. + redshift)
mjd = float(date_dict[name])
except:
phase = None
mjd = None
if sn_cfa[4] == '9.99':
Dm15 = None
else:
Dm15 = sn_cfa[4]
if sn_cfa[7] == '-99.99':
m_b = None
else:
m_b = sn_cfa[7]
if sn_cfa[11] == '-9.99':
bm_vm = None
else:
bm_vm = sn_cfa[11]
ref = '2012AJ....143..126B'
#bsnip spectra
elif 'bsnip' in f:
source = 'bsnip'
"""
certain files have words in their name that cause the
split to be misalligned from the typical splits.
"""
if is_number(name.split('-')[1][:8]):
longdate = name.split('-')[1][:8]
else:
longdate = name.split('-')[2][:8]
data = bsnip_vals[sn_name.lower() + '-' + longdate]
print f
if math.isnan(data[1]):
#skippy shitty redshiftless spectra
shiftless.append(name)
continue
redshift = data[1] / c
#if phase exists, add to db
if not math.isnan(data[2]):
phase = data[2]
mjd = data[3]
if type(mjd) != float:
mjd = float(mjd[0:9])
if name in ref_dict:
ref = ref_dict[name]
waves = spectra[:, 0]
min_wave = waves[0]
max_wave = waves[len(spectra) - 1]
spec = msg.packb(spectra)
if sn_name in NED_red_dict:
redshift = NED_red_dict[sn_name]
try:
interp_spec, sig_noise = prep.compprep(spectra,
sn_name,
redshift,
source,
testing=False)
except Exception, e:
#raise
print e
print "Interp failed"
bad_interp.append(name)
bad_files.append(name)
interp_spec, sig_noise = None, None
if sn_name in sndict:
if sndict[sn_name][4] == '9.99':
Dm15 = None
else:
Dm15 = sndict[sn_name][4]
if sndict[sn_name][7] == '-99.99':
m_b = None
else:
m_b = sndict[sn_name][7]
if sndict[sn_name][11] == '-9.99':
bm_vm = None
else:
bm_vm = sndict[sn_name][11]
if mjd is not None and phase is None and sndict[sn_name][
1] != '99999.9' and redshift is not None:
phase = (mjd - float(sndict[sn_name][1])) / (1. +
redshift)
#add redshift to redshift dictionary if not already present
if name not in rsd and redshift is not None:
rsd[name] = redshift
interped = msg.packb(interp_spec)
##version 15
con.execute(
"""INSERT INTO Spectra(Filename, SN, Source,
Phase, MinWave, MaxWave,
snr, Interpolated_Spectra,
MJD, Ref)
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(name, sn_name, source, phase, min_wave, max_wave,
sig_noise, buffer(interped), mjd, ref))