def grab_meta():
""" Grab GGG meta Table
Returns
-------
"""
# This table has units in it!
meta = Table.read(os.getenv('RAW_IGMSPEC')+'/GGG/GGG_catalog.fits.gz')
nqso = len(meta)
# Turn off RA/DEC units
for key in ['RA', 'DEC']:
meta[key].unit = None
meta.rename_column('RA', 'RA_GROUP')
meta.rename_column('DEC', 'DEC_GROUP')
#
# Add zem
meta['zem_GROUP'] = meta['z_gmos']
meta['sig_zem'] = meta['zerror_gmos']
meta['flag_zem'] = [str('GGG')]*nqso
meta.add_column(Column([2000.]*nqso, name='EPOCH'))
#
meta['STYPE'] = [str('QSO')]*nqso
# Double up for the two gratings
ggg_meta = vstack([meta,meta])
# Check
assert chk_meta(ggg_meta, chk_cat_only=True)
# Return
return ggg_meta
def grab_meta():
""" Grab KODIAQ meta Table
Returns
-------
"""
hstz2_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/HST_z2/hst_z2.ascii', format='ascii')
nspec = len(hstz2_meta)
# RA/DEC, DATE
ra = []
dec = []
for row in hstz2_meta:
# Fix DEC
# Get RA/DEC
coord = ltu.radec_to_coord((row['ra'],row['dec']))
ra.append(coord.ra.value)
dec.append(coord.dec.value)
hstz2_meta.add_column(Column(ra, name='RA_GROUP'))
hstz2_meta.add_column(Column(dec, name='DEC_GROUP'))
# z
hstz2_meta.rename_column('zem', 'zem_GROUP')
hstz2_meta['sig_zem'] = [0.]*nspec
hstz2_meta['flag_zem'] = [str('SDSS_PIPE')]*nspec
hstz2_meta['STYPE'] = [str('QSO')]*nspec
#
hstz2_meta.rename_column('obsdate','DATE-OBS')
hstz2_meta.rename_column('tel','TELESCOPE')
hstz2_meta.rename_column('inst','INSTR')
hstz2_meta.rename_column('grating','DISPERSER')
hstz2_meta.rename_column('resolution','R')
# Check
assert chk_meta(hstz2_meta, chk_cat_only=True)
# Return
return hstz2_meta
def grab_meta():
""" Grab GGG meta Table
Returns
-------
"""
# This table has units in it!
meta = Table.read(os.getenv('RAW_IGMSPEC')+'/ESI_z6/overview_data_igmspec.txt', format='ascii', delimiter='\t')
nqso = len(meta)
# Rename
meta.rename_column('RAdeg', 'RA_GROUP')
meta.rename_column('DECdeg', 'DEC_GROUP')
meta.rename_column('z', 'zem_GROUP')
meta.rename_column('instrument', 'INSTR')
meta.rename_column('telescope', 'TELESCOPE')
meta.rename_column('date', 'DATE-OBS')
#
# Add zem
meta['sig_zem'] = 0.
meta['flag_zem'] = str('ESI_z6')
meta.add_column(Column([2000.]*nqso, name='EPOCH'))
#
meta['STYPE'] = str('QSO')
meta['DISPERSER'] = str('ECH')
# Check
assert chk_meta(meta, chk_cat_only=True)
# Return
return meta
def grab_meta():
""" Grab KODIAQ meta Table
Returns
-------
"""
hstz2_meta = Table.read(os.getenv('RAW_IGMSPEC') + '/HST_z2/hst_z2.ascii',
format='ascii')
nspec = len(hstz2_meta)
# RA/DEC, DATE
ra = []
dec = []
for row in hstz2_meta:
# Fix DEC
# Get RA/DEC
coord = ltu.radec_to_coord((row['ra'], row['dec']))
ra.append(coord.ra.value)
dec.append(coord.dec.value)
hstz2_meta.add_column(Column(ra, name='RA_GROUP'))
hstz2_meta.add_column(Column(dec, name='DEC_GROUP'))
# z
hstz2_meta.rename_column('zem', 'zem_GROUP')
hstz2_meta['sig_zem'] = [0.] * nspec
hstz2_meta['flag_zem'] = [str('SDSS_PIPE')] * nspec
hstz2_meta['STYPE'] = [str('QSO')] * nspec
#
hstz2_meta.rename_column('obsdate', 'DATE-OBS')
hstz2_meta.rename_column('tel', 'TELESCOPE')
hstz2_meta.rename_column('inst', 'INSTR')
hstz2_meta.rename_column('grating', 'DISPERSER')
hstz2_meta.rename_column('resolution', 'R')
# Check
assert chk_meta(hstz2_meta, chk_cat_only=True)
# Return
return hstz2_meta
def grab_meta():
""" Grab GGG meta Table
Returns
-------
"""
# This table has units in it!
meta = Table.read(os.getenv('RAW_IGMSPEC') + '/GGG/GGG_catalog.fits.gz')
nqso = len(meta)
# Turn off RA/DEC units
for key in ['RA', 'DEC']:
meta[key].unit = None
meta.rename_column('RA', 'RA_GROUP')
meta.rename_column('DEC', 'DEC_GROUP')
#
# Add zem
meta['zem_GROUP'] = meta['z_gmos']
meta['sig_zem'] = meta['zerror_gmos']
meta['flag_zem'] = [str('GGG')] * nqso
meta.add_column(Column([2000.] * nqso, name='EPOCH'))
#
meta['STYPE'] = [str('QSO')] * nqso
# Double up for the two gratings
ggg_meta = vstack([meta, meta])
# Check
assert chk_meta(ggg_meta, chk_cat_only=True)
# Return
return ggg_meta
def grab_meta():
""" Grab GGG meta Table
Returns
-------
"""
# This table has units in it!
meta = Table.read(os.getenv('RAW_IGMSPEC') +
'/ESI_z6/overview_data_igmspec.txt',
format='ascii',
delimiter='\t')
nqso = len(meta)
# Rename
meta.rename_column('RAdeg', 'RA_GROUP')
meta.rename_column('DECdeg', 'DEC_GROUP')
meta.rename_column('z', 'zem_GROUP')
meta.rename_column('instrument', 'INSTR')
meta.rename_column('telescope', 'TELESCOPE')
meta.rename_column('date', 'DATE-OBS')
#
# Add zem
meta['sig_zem'] = 0.
meta['flag_zem'] = str('ESI_z6')
meta.add_column(Column([2000.] * nqso, name='EPOCH'))
#
meta['STYPE'] = str('QSO')
meta['DISPERSER'] = str('ECH')
# Check
assert chk_meta(meta, chk_cat_only=True)
# Return
return meta
def grab_meta():
""" Grab High-z ESI meta Table
Returns
-------
"""
#
esidla_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/HighzESIDLA/ascii_highz_rafelski.list', format='ascii')
nspec = len(esidla_meta)
# DATE
datearr = [day.split('/') for day in list(esidla_meta['ObsDate'])]
ndate = ['20'+str(day[2])+'-'+str(day[0])+'-'+str(day[1]) for day in datearr]
t = Time(ndate, out_subfmt='date') # Fixes to YYYY-MM-DD
esidla_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add zem
esidla_meta['sig_zem'] = [0.]*nspec
esidla_meta['flag_zem'] = [str('SDSS')]*nspec
#
esidla_meta.add_column(Column([2000.]*nspec, name='EPOCH'))
esidla_meta.add_column(Column(['KeckII']*nspec, name='TELESCOPE'))
esidla_meta.add_column(Column(['ESI']*nspec, name='INSTR'))
esidla_meta.add_column(Column(['ECH']*nspec, name='DISPERSER'))
# Rename
esidla_meta.rename_column('RA', 'RA_GROUP')
esidla_meta.rename_column('DEC', 'DEC_GROUP')
esidla_meta.rename_column('zem', 'zem_GROUP')
esidla_meta['STYPE'] = str('QSO')
# Sort
esidla_meta.sort('RA_GROUP')
# Check
assert chk_meta(esidla_meta, chk_cat_only=True)
#
return esidla_meta
def grab_meta():
""" Grab UVES Dall'Aglio meta table
Returns
-------
"""
#
uvesdall_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/UVES_Dall/uves_dall_summ.dat', format='ascii')
nspec = len(uvesdall_meta)
# DATE
#datearr = [day.split('/') for day in list(uvesdall_meta['ObsDate'])]
#ndate = ['20'+str(day[2])+'-'+str(day[0])+'-'+str(day[1]) for day in datearr]
t = Time(uvesdall_meta['OBS-DATE'], out_subfmt='date') # Fixes to YYYY-MM-DD
uvesdall_meta.add_column(Column(t.iso, name='DATE-OBS'))
# RA/DEC
coord = SkyCoord(ra=uvesdall_meta['RA'], dec=uvesdall_meta['DEC'], unit=(u.hour,u.deg))
rad = [icoord.ra.value for icoord in coord]
decd = [icoord.dec.value for icoord in coord]
uvesdall_meta.rename_column('RA', 'RA_STR')
uvesdall_meta.rename_column('DEC', 'DEC_STR')
uvesdall_meta['RA_GROUP'] = rad
uvesdall_meta['DEC_GROUP'] = decd
# Add zem
igmsp = IgmSpec()
ztbl = Table(igmsp.hdf['quasars'].value)
zem, zsource = zem_from_radec(rad, decd, ztbl)
badz = np.where(zem < 0.1)[0]
for ibadz in badz:
if uvesdall_meta['NAME'][ibadz] == 'HE2243-6031':
zem[ibadz] = 3.005
zsource[ibadz] = 'FOP13' # Fumagalli+13
elif uvesdall_meta['NAME'][ibadz] == 'HE1341-1020':
zem[ibadz] = 2.137
zsource[ibadz] = 'Dall08' # Dall'Aglio+08
elif uvesdall_meta['NAME'][ibadz] == 'Q0002-422':
zem[ibadz] = 2.769
zsource[ibadz] = 'Dall08' # Dall'Aglio+08
elif uvesdall_meta['NAME'][ibadz] == 'PKS2000-330':
zem[ibadz] = 3.786
zsource[ibadz] = 'Dall08' # Dall'Aglio+08
else:
raise ValueError("Should not be here")
uvesdall_meta['zem_GROUP'] = zem
uvesdall_meta['sig_zem'] = [0.]*nspec
uvesdall_meta['flag_zem'] = zsource
#
uvesdall_meta.add_column(Column([2000.]*nspec, name='EPOCH'))
uvesdall_meta.add_column(Column(['VLT']*nspec, name='TELESCOPE'))
uvesdall_meta.add_column(Column(['UVES']*nspec, name='INSTR'))
uvesdall_meta.add_column(Column(['BOTH']*nspec, name='DISPERSER'))
uvesdall_meta.add_column(Column([45000.]*nspec, name='R'))
uvesdall_meta['STYPE'] = str('QSO')
# Sort
uvesdall_meta.sort('RA_GROUP')
# Check
assert chk_meta(uvesdall_meta, chk_cat_only=True)
return uvesdall_meta
def grab_meta():
""" Grab COS-Dwarfs meta table
Returns
-------
"""
from time import strptime
cosdwarfs = COSDwarfs()
cosdwarfs.load_sys(tfile=cosdwarfs.cdir +
'/cos-dwarfs_systems.v1.1.tar.gz',
chk_lowz=False)
visit_file = os.getenv(
'RAW_IGMSPEC') + '/COS-Dwarfs/HST_Observing_Dates.dat'
cd_visits = Table.read(visit_file, format='ascii')
# Coord
lst = [getattr(cgm_abs.igm_sys, 'coord') for cgm_abs in cosdwarfs.cgm_abs]
ra = [coord.ra.value for coord in lst]
dec = [coord.dec.value for coord in lst]
# Short names
shrt_names = [name.split('_')[0] for name in cosdwarfs.name]
cdwarfs_meta = Table()
cdwarfs_meta['RA'] = ra
cdwarfs_meta['DEC'] = dec
# RA/DEC, DATE
datet = []
for kk, row in enumerate(cdwarfs_meta):
#
name = shrt_names[kk]
mtv = np.where(cd_visits['QSO'] == name)[0]
if len(mtv) != 1:
pdb.set_trace()
else:
chv = cd_visits['Start_UT'][mtv].data[0]
icmma = chv.find(',')
datet.append('{:s}-{:02d}-{:02d}'.format(
chv[icmma + 1:icmma + 5],
strptime(chv[:3], '%b').tm_mon, int(chv[3:icmma])))
cdwarfs_meta.add_column(Column(datet, name='DATE-OBS'))
# Others
cdwarfs_meta.add_column(
Column(['G130M/G160M'] * len(cdwarfs_meta), name='DISPERSER'))
cdwarfs_meta.add_column(Column([20000.] * len(cdwarfs_meta), name='R'))
cdwarfs_meta.add_column(Column([2000.] * len(cdwarfs_meta), name='EPOCH'))
cdwarfs_meta['INSTR'] = 'COS' # Deals with padding
cdwarfs_meta['TELESCOPE'] = 'HST'
cdwarfs_meta['zem_GROUP'] = cosdwarfs.zem
cdwarfs_meta['sig_zem'] = 0. # Need to add
cdwarfs_meta['flag_zem'] = 'SDSS'
cdwarfs_meta['STYPE'] = str('QSO')
# Rename
cdwarfs_meta.rename_column('RA', 'RA_GROUP')
cdwarfs_meta.rename_column('DEC', 'DEC_GROUP')
# Check
assert chk_meta(cdwarfs_meta, chk_cat_only=True)
# Done
return cdwarfs_meta
def grab_meta():
""" Grab COS-Dwarfs meta table
Returns
-------
"""
from time import strptime
cosdwarfs = COSDwarfs()
cosdwarfs.load_sys(tfile=cosdwarfs.cdir+'/cos-dwarfs_systems.v1.1.tar.gz', chk_lowz=False)
visit_file = os.getenv('RAW_IGMSPEC')+'/COS-Dwarfs/HST_Observing_Dates.dat'
cd_visits = Table.read(visit_file,format='ascii')
# Coord
lst = [getattr(cgm_abs.igm_sys, 'coord') for cgm_abs in cosdwarfs.cgm_abs]
ra = [coord.ra.value for coord in lst]
dec = [coord.dec.value for coord in lst]
# Short names
shrt_names = [name.split('_')[0] for name in cosdwarfs.name]
cdwarfs_meta = Table()
cdwarfs_meta['RA'] = ra
cdwarfs_meta['DEC'] = dec
# RA/DEC, DATE
datet = []
for kk,row in enumerate(cdwarfs_meta):
#
name = shrt_names[kk]
mtv = np.where(cd_visits['QSO'] == name)[0]
if len(mtv) != 1:
pdb.set_trace()
else:
chv = cd_visits['Start_UT'][mtv].data[0]
icmma = chv.find(',')
datet.append('{:s}-{:02d}-{:02d}'.format(
chv[icmma+1:icmma+5], strptime(chv[:3],'%b').tm_mon,
int(chv[3:icmma])))
cdwarfs_meta.add_column(Column(datet, name='DATE-OBS'))
# Others
cdwarfs_meta.add_column(Column(['G130M/G160M']*len(cdwarfs_meta), name='DISPERSER'))
cdwarfs_meta.add_column(Column([20000.]*len(cdwarfs_meta), name='R'))
cdwarfs_meta.add_column(Column([2000.]*len(cdwarfs_meta), name='EPOCH'))
cdwarfs_meta['INSTR'] = 'COS' # Deals with padding
cdwarfs_meta['TELESCOPE'] = 'HST'
cdwarfs_meta['zem_GROUP'] = cosdwarfs.zem
cdwarfs_meta['sig_zem'] = 0. # Need to add
cdwarfs_meta['flag_zem'] = 'SDSS'
cdwarfs_meta['STYPE'] = str('QSO')
# Rename
cdwarfs_meta.rename_column('RA', 'RA_GROUP')
cdwarfs_meta.rename_column('DEC', 'DEC_GROUP')
# Check
assert chk_meta(cdwarfs_meta, chk_cat_only=True)
# Done
return cdwarfs_meta
def grab_meta(hdf, old=False):
""" Grab SDSS meta Table
Returns
-------
meta
"""
from specdb.zem.utils import zem_from_radec
#sdss_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/SDSS/SDSS_DR7_qso.fits.gz')
sdss_meta = Table.read(os.getenv('RAW_IGMSPEC') + '/SDSS/dr7qso.fit.gz')
nspec = len(sdss_meta)
# DATE
#t = Time(list(sdss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
t = Time(list(sdss_meta['SMJD'].data), format='mjd',
out_subfmt='date') # Fixes to YYYY-MM-DD
sdss_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add a few columns
sdss_meta.add_column(Column([2000.] * nspec, name='EPOCH'))
sdss_meta.add_column(Column([2000.] * nspec, name='R'))
sdss_meta.add_column(Column(['SDSS'] * nspec, name='INSTR'))
sdss_meta.add_column(Column(['BOTH'] * nspec, name='GRATING'))
sdss_meta.add_column(Column(['SDSS 2.5-M'] * nspec, name='TELESCOPE'))
# Rename
if old:
# Some of these were corrected by QPQ
sdss_meta.rename_column('RAOBJ', 'RA')
sdss_meta.rename_column('DECOBJ', 'DEC')
sdss_meta.rename_column('Z_ERR', 'sig_zem')
pdb.set_trace()
else:
sdss_meta.rename_column('z', 'zem_GROUP')
sdss_meta['sig_zem'] = 0.
sdss_meta['flag_zem'] = str(' ')
# Fix zem
zem, zsource = zem_from_radec(sdss_meta['RA'],
sdss_meta['DEC'],
hdf['quasars'].value,
toler=1.0 * u.arcsec)
gdz = zem > 0.
sdss_meta['zem_GROUP'][gdz] = zem[gdz]
sdss_meta['flag_zem'] = zsource
sdss_meta['flag_zem'][~gdz] = str('SDSS-DR7')
# Sort
sdss_meta.sort('RA')
# Rename
sdss_meta.rename_column('RA', 'RA_GROUP')
sdss_meta.rename_column('DEC', 'DEC_GROUP')
# Add
sdss_meta['STYPE'] = [str('QSO')] * nspec
# Check
assert chk_meta(sdss_meta, chk_cat_only=True)
# Return
return sdss_meta
def grab_meta():
""" Grab BOSS meta Table
Returns
-------
boss_meta : Table
"""
#http://www.sdss.org/dr12/algorithms/boss-dr12-quasar-catalog/
boss_dr12 = Table.read(os.getenv('RAW_IGMSPEC') + '/BOSS/DR12Q.fits.gz')
boss_dr12['CAT'] = ['DR12Q'] * len(boss_dr12)
gd = np.any([boss_dr12['Z_PIPE'] > 0., boss_dr12['Z_PCA'] > 0.],
axis=0) # CUTS Z_VI
boss_dr12 = boss_dr12[gd]
#
boss_sup = Table.read(os.getenv('RAW_IGMSPEC') + '/BOSS/DR12Q_sup.fits.gz')
boss_sup['CAT'] = ['SUPGD'] * len(boss_sup)
boss_supbad = Table.read(
os.getenv('RAW_IGMSPEC') + '/BOSS/DR12Q_supbad.fits.gz')
boss_supbad['CAT'] = ['SUPBD'] * len(boss_supbad)
# Collate
boss_meta = vstack([boss_dr12, boss_sup, boss_supbad], join_type='outer')
#
nboss = len(boss_meta)
# DATE-OBS
t = Time(list(boss_meta['MJD'].data), format='mjd',
out_subfmt='date') # Fixes to YYYY-MM-DD
boss_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add columns
boss_meta.add_column(Column(['BOSS'] * nboss, name='INSTR'))
boss_meta.add_column(Column(['BOTH'] * nboss, name='GRATING'))
#http://www.sdss.org/instruments/boss_spectrograph/
boss_meta.add_column(Column([2100.] * nboss, name='R')) # RESOLUTION
boss_meta.add_column(Column(['SDSS 2.5-M'] * nboss, name='TELESCOPE'))
# Redshift logic
boss_meta['zem_GROUP'] = boss_meta['Z_PCA']
boss_meta['sig_zem'] = boss_meta['ERR_ZPCA']
boss_meta['flag_zem'] = [str('BOSS_PCA ')] * nboss
# Fix bad redshifts
bad_pca = boss_meta['Z_PCA'] < 0.
boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca]
boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca]
boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE')
# Rename RA/DEC
boss_meta.rename_column('RA', 'RA_GROUP')
boss_meta.rename_column('DEC', 'DEC_GROUP')
# STYPE
boss_meta['STYPE'] = [str('QSO')] * len(boss_meta)
# Check
assert chk_meta(boss_meta, chk_cat_only=True)
# Return
return boss_meta
def grab_meta():
""" Grab BOSS meta Table
Returns
-------
boss_meta : Table
"""
#http://www.sdss.org/dr12/algorithms/boss-dr12-quasar-catalog/
boss_dr12 = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS/DR12Q.fits.gz')
boss_dr12['CAT'] = ['DR12Q']*len(boss_dr12)
gd = np.any([boss_dr12['Z_PIPE'] > 0., boss_dr12['Z_PCA'] > 0.],axis=0) # CUTS Z_VI
boss_dr12 = boss_dr12[gd]
#
boss_sup = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS/DR12Q_sup.fits.gz')
boss_sup['CAT'] = ['SUPGD']*len(boss_sup)
boss_supbad = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS/DR12Q_supbad.fits.gz')
boss_supbad['CAT'] = ['SUPBD']*len(boss_supbad)
# Collate
boss_meta = vstack([boss_dr12, boss_sup, boss_supbad], join_type='outer')
#
nboss = len(boss_meta)
# DATE-OBS
t = Time(list(boss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
boss_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add columns
boss_meta.add_column(Column(['BOSS']*nboss, name='INSTR'))
boss_meta.add_column(Column(['BOTH']*nboss, name='GRATING'))
#http://www.sdss.org/instruments/boss_spectrograph/
boss_meta.add_column(Column([2100.]*nboss, name='R')) # RESOLUTION
boss_meta.add_column(Column(['SDSS 2.5-M']*nboss, name='TELESCOPE'))
# Redshift logic
boss_meta['zem_GROUP'] = boss_meta['Z_PCA']
boss_meta['sig_zem'] = boss_meta['ERR_ZPCA']
boss_meta['flag_zem'] = [str('BOSS_PCA ')]*nboss
# Fix bad redshifts
bad_pca = boss_meta['Z_PCA'] < 0.
boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca]
boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca]
boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE')
# Rename RA/DEC
boss_meta.rename_column('RA', 'RA_GROUP')
boss_meta.rename_column('DEC', 'DEC_GROUP')
# STYPE
boss_meta['STYPE'] = [str('QSO')]*len(boss_meta)
# Check
assert chk_meta(boss_meta, chk_cat_only=True)
# Return
return boss_meta
def grab_meta(hdf, old=False):
""" Grab SDSS meta Table
Returns
-------
meta
"""
from specdb.zem.utils import zem_from_radec
#sdss_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/SDSS/SDSS_DR7_qso.fits.gz')
sdss_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/SDSS/dr7qso.fit.gz')
nspec = len(sdss_meta)
# DATE
#t = Time(list(sdss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
t = Time(list(sdss_meta['SMJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
sdss_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add a few columns
sdss_meta.add_column(Column([2000.]*nspec, name='EPOCH'))
sdss_meta.add_column(Column([2000.]*nspec, name='R'))
sdss_meta.add_column(Column(['SDSS']*nspec, name='INSTR'))
sdss_meta.add_column(Column(['BOTH']*nspec, name='GRATING'))
sdss_meta.add_column(Column(['SDSS 2.5-M']*nspec, name='TELESCOPE'))
# Rename
if old:
# Some of these were corrected by QPQ
sdss_meta.rename_column('RAOBJ', 'RA')
sdss_meta.rename_column('DECOBJ', 'DEC')
sdss_meta.rename_column('Z_ERR', 'sig_zem')
pdb.set_trace()
else:
sdss_meta.rename_column('z', 'zem_GROUP')
sdss_meta['sig_zem'] = 0.
sdss_meta['flag_zem'] = str(' ')
# Fix zem
zem, zsource = zem_from_radec(sdss_meta['RA'], sdss_meta['DEC'], hdf['quasars'].value, toler=1.0*u.arcsec)
gdz = zem > 0.
sdss_meta['zem_GROUP'][gdz] = zem[gdz]
sdss_meta['flag_zem'] = zsource
sdss_meta['flag_zem'][~gdz] = str('SDSS-DR7')
# Sort
sdss_meta.sort('RA')
# Rename
sdss_meta.rename_column('RA', 'RA_GROUP')
sdss_meta.rename_column('DEC', 'DEC_GROUP')
# Add
sdss_meta['STYPE'] = [str('QSO')]*nspec
# Check
assert chk_meta(sdss_meta, chk_cat_only=True)
# Return
return sdss_meta
def grab_meta():
""" Generates the meta data needed for the IGMSpec build
Returns
-------
meta : Table
"""
# Read
hdlls_meta = Table.read(
os.getenv('RAW_IGMSPEC') + '/HD-LLS_DR1/HD-LLS_DR1.fits')
# Rename
hdlls_meta.rename_column('RA', 'RA_GROUP')
hdlls_meta.rename_column('DEC', 'DEC_GROUP')
hdlls_meta.rename_column('Z_QSO', 'zem_GROUP')
# Kludgy Table judo
hdlls_full = hdlls_meta[0:1]
spec_files = []
# Loop me to bid the full survey catalog
for kk, row in enumerate(hdlls_meta):
for spec_file in row['SPEC_FILES']:
if spec_file == 'NULL':
continue
# Add to full table
hdlls_full.add_row(row)
spec_files.append(spec_file)
# Build
hdlls_full = hdlls_full[1:]
hdlls_full.remove_column('SPEC_FILES')
hdlls_full.add_column(
Column(np.array(spec_files).astype(str), name='SPEC_FILE'))
# Cut on unique SPEC_FILEs
uni, uni_idx = np.unique(np.array(spec_files).astype(str),
return_index=True)
# REMOVE ONE FILE (A DUPLICATE) BY HAND
mt = uni != 'HD-LLS_J130756.73+042215.5_MIKE.fits'
uni_idx = uni_idx[mt]
#
hdlls_full = hdlls_full[uni_idx]
#
nspec = len(hdlls_full)
hdlls_full['sig_zem'] = [0.] * nspec
hdlls_full['flag_zem'] = [str('UNKN')] * nspec
hdlls_full['STYPE'] = [str('QSO')] * nspec
assert chk_meta(hdlls_full, chk_cat_only=True)
# Return
return hdlls_full
def grab_meta():
""" Generates the meta data needed for the IGMSpec build
Returns
-------
meta : Table
spec_files : list
List of spec_file names
"""
# Load DLA
from pyigm.surveys.dlasurvey import DLASurvey
hdla100 = DLASurvey.neeleman13_tree()
# Cut down to unique QSOs
spec_files = []
names = []
ra = []
dec = []
coords = hdla100.coord
cnt = 0
for coord in coords:
# Load
names.append('J{:s}{:s}'.format(
coord.ra.to_string(unit=u.hour, sep='', pad=True, precision=2),
coord.dec.to_string(sep='', pad=True, precision=1)))
# RA/DEC
ra.append(coord.ra.value)
dec.append(coord.dec.value)
# SPEC_FILE
fname = hdla100._abs_sys[cnt]._datdict['hi res file'].split('/')[-1]
spec_files.append(fname)
cnt += 1
uni, uni_idx = np.unique(names, return_index=True)
nqso = len(uni_idx)
#
meta = Table()
meta['RA_GROUP'] = np.array(ra)[uni_idx]
meta['DEC_GROUP'] = np.array(dec)[uni_idx]
meta['zem_GROUP'] = hdla100.zem[uni_idx]
meta['sig_zem'] = [0.] * nqso
meta['flag_zem'] = [str('UNKN')] * nqso
meta['STYPE'] = [str('QSO')] * nqso
meta['SPEC_FILE'] = np.array(spec_files)[uni_idx]
# Check
assert chk_meta(meta, chk_cat_only=True)
return meta
def grab_meta():
""" Generates the meta data needed for the IGMSpec build
Returns
-------
meta : Table
"""
# Read
hdlls_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/HD-LLS_DR1/HD-LLS_DR1.fits')
# Rename
hdlls_meta.rename_column('RA', 'RA_GROUP')
hdlls_meta.rename_column('DEC', 'DEC_GROUP')
hdlls_meta.rename_column('Z_QSO', 'zem_GROUP')
# Kludgy Table judo
hdlls_full = hdlls_meta[0:1]
spec_files = []
# Loop me to bid the full survey catalog
for kk,row in enumerate(hdlls_meta):
for spec_file in row['SPEC_FILES']:
if spec_file == 'NULL':
continue
# Add to full table
hdlls_full.add_row(row)
spec_files.append(spec_file)
# Build
hdlls_full = hdlls_full[1:]
hdlls_full.remove_column('SPEC_FILES')
hdlls_full.add_column(Column(np.array(spec_files).astype(str),name='SPEC_FILE'))
# Cut on unique SPEC_FILEs
uni, uni_idx = np.unique(np.array(spec_files).astype(str), return_index=True)
# REMOVE ONE FILE (A DUPLICATE) BY HAND
mt = uni != 'HD-LLS_J130756.73+042215.5_MIKE.fits'
uni_idx = uni_idx[mt]
#
hdlls_full = hdlls_full[uni_idx]
#
nspec = len(hdlls_full)
hdlls_full['sig_zem'] = [0.]*nspec
hdlls_full['flag_zem'] = [str('UNKN')]*nspec
hdlls_full['STYPE'] = [str('QSO')]*nspec
assert chk_meta(hdlls_full, chk_cat_only=True)
# Return
return hdlls_full
def grab_meta():
""" Generates the meta data needed for the IGMSpec build
Returns
-------
meta : Table
spec_files : list
List of spec_file names
"""
# Load DLA
from pyigm.surveys.dlasurvey import DLASurvey
hdla100 = DLASurvey.neeleman13_tree()
# Cut down to unique QSOs
spec_files = []
names = []
ra = []
dec = []
coords = hdla100.coord
cnt = 0
for coord in coords:
# Load
names.append('J{:s}{:s}'.format(coord.ra.to_string(unit=u.hour, sep='', pad=True, precision=2),
coord.dec.to_string(sep='', pad=True, precision=1)))
# RA/DEC
ra.append(coord.ra.value)
dec.append(coord.dec.value)
# SPEC_FILE
fname = hdla100._abs_sys[cnt]._datdict['hi res file'].split('/')[-1]
spec_files.append(fname)
cnt += 1
uni, uni_idx = np.unique(names, return_index=True)
nqso = len(uni_idx)
#
meta = Table()
meta['RA_GROUP'] = np.array(ra)[uni_idx]
meta['DEC_GROUP'] = np.array(dec)[uni_idx]
meta['zem_GROUP'] = hdla100.zem[uni_idx]
meta['sig_zem'] = [0.]*nqso
meta['flag_zem'] = [str('UNKN')]*nqso
meta['STYPE'] = [str('QSO')]*nqso
meta['SPEC_FILE'] = np.array(spec_files)[uni_idx]
# Check
assert chk_meta(meta, chk_cat_only=True)
return meta
def grab_meta():
""" Grab KODIAQ meta Table
Returns
-------
"""
kodiaq_file = igms_path + '/data/meta/KODIAQ_DR1_summary.ascii'
kodiaq_meta = Table.read(kodiaq_file, format='ascii', comment='#')
nspec = len(kodiaq_meta)
# Verify DR1
for row in kodiaq_meta:
assert row['kodrelease'] == 1
# RA/DEC, DATE
ra = []
dec = []
dateobs = []
for row in kodiaq_meta:
# Fix DEC
# Get RA/DEC
coord = ltu.radec_to_coord((row['sRA'], row['sDEC']))
ra.append(coord.ra.value)
dec.append(coord.dec.value)
# DATE
dvals = row['pi_date'].split('_')
tymd = str('{:s}-{:s}-{:02d}'.format(dvals[-1], dvals[1][0:3],
int(dvals[2])))
tval = datetime.datetime.strptime(tymd, '%Y-%b-%d')
dateobs.append(datetime.datetime.strftime(tval, '%Y-%m-%d'))
kodiaq_meta.add_column(Column(ra, name='RA_GROUP'))
kodiaq_meta.add_column(Column(dec, name='DEC_GROUP'))
kodiaq_meta.add_column(Column(dateobs, name='DATE-OBS'))
#
kodiaq_meta.add_column(Column(['HIRES'] * nspec, name='INSTR'))
kodiaq_meta.add_column(Column(['Keck-I'] * nspec, name='TELESCOPE'))
kodiaq_meta['STYPE'] = [str('QSO')] * nspec
# z
kodiaq_meta.rename_column('zem', 'zem_GROUP')
kodiaq_meta['sig_zem'] = [0.] * nspec
kodiaq_meta['flag_zem'] = [str('SIMBAD')] * nspec
#
assert chk_meta(kodiaq_meta, chk_cat_only=True)
return kodiaq_meta
def grab_meta():
""" Grab HST/FUSE Cooksey meta Table
Returns
-------
"""
hstc_file = os.getenv('RAW_IGMSPEC') + '/HST_Cooksey/HSTQSO_pre-SM4.lst'
hstc_meta = Table.read(hstc_file, format='ascii')
# Cutting those without proper header (for now)
#badf = ['PKS2005-489lif1a.fits', 'NGC7469lif2a.fits', 'NGC7469sic2b.fits',
# 'NGC7469lif2b.fits', 'NGC7469sic2a.fits',
# 'AKN564lif1a.fits', 'AKN564lif1b.fits', 'AKN564lif2a.fits', 'AKN564lif2b.fits',
# 'AKN564sic1a.fits', 'AKN564sic1b.fits', 'AKN564sic2a.fits', 'AKN564sic2b.fits']
badf = []
gdm = np.array([True] * len(hstc_meta))
for ibadf in badf:
mt = np.where(hstc_meta['SPEC_FILE'] == ibadf)[0]
gdm[mt] = False
for jj, row in enumerate(hstc_meta): # Skip continua
if '_c.fits' in row['SPEC_FILE']:
gdm[jj] = False
if '_E.fits' in row['SPEC_FILE']:
gdm[jj] = False
#if row['INSTR'] == 'GHRS':
# gdm[jj] = False
hstc_meta = hstc_meta[gdm]
gdf = hstc_meta['INSTR'] == 'FUSE'
#hstc_meta = hstc_meta[gdf]
hstc_meta['TELESCOPE'] = 'FUSE'
hstc_meta[~gdf]['TELESCOPE'] = 'HST'
#
hstc_meta.add_column(Column([2000.] * len(hstc_meta), name='EPOCH'))
hstc_meta['sig_zem'] = 0.
hstc_meta['flag_zem'] = str('UNKWN')
hstc_meta['STYPE'] = str('QSO')
# RENAME
hstc_meta.rename_column('RA', 'RA_GROUP')
hstc_meta.rename_column('DEC', 'DEC_GROUP')
hstc_meta.rename_column('zem', 'zem_GROUP')
# Check
assert chk_meta(hstc_meta, chk_cat_only=True)
return hstc_meta
def grab_meta():
""" Grab KODIAQ meta Table
Returns
-------
"""
kodiaq_file = igms_path+'/data/meta/KODIAQ_DR1_summary.ascii'
kodiaq_meta = Table.read(kodiaq_file, format='ascii', comment='#')
nspec = len(kodiaq_meta)
# Verify DR1
for row in kodiaq_meta:
assert row['kodrelease'] == 1
# RA/DEC, DATE
ra = []
dec = []
dateobs = []
for row in kodiaq_meta:
# Fix DEC
# Get RA/DEC
coord = ltu.radec_to_coord((row['sRA'],row['sDEC']))
ra.append(coord.ra.value)
dec.append(coord.dec.value)
# DATE
dvals = row['pi_date'].split('_')
tymd = str('{:s}-{:s}-{:02d}'.format(dvals[-1],dvals[1][0:3],int(dvals[2])))
tval = datetime.datetime.strptime(tymd, '%Y-%b-%d')
dateobs.append(datetime.datetime.strftime(tval,'%Y-%m-%d'))
kodiaq_meta.add_column(Column(ra, name='RA_GROUP'))
kodiaq_meta.add_column(Column(dec, name='DEC_GROUP'))
kodiaq_meta.add_column(Column(dateobs, name='DATE-OBS'))
#
kodiaq_meta.add_column(Column(['HIRES']*nspec, name='INSTR'))
kodiaq_meta.add_column(Column(['Keck-I']*nspec, name='TELESCOPE'))
kodiaq_meta['STYPE'] = [str('QSO')]*nspec
# z
kodiaq_meta.rename_column('zem', 'zem_GROUP')
kodiaq_meta['sig_zem'] = [0.]*nspec
kodiaq_meta['flag_zem'] = [str('SIMBAD')]*nspec
#
assert chk_meta(kodiaq_meta, chk_cat_only=True)
return kodiaq_meta
def grab_meta():
""" Grab KODIAQ meta Table
Returns
-------
"""
kodiaq_file = os.getenv('RAW_IGMSPEC')+'/KODIAQ2/KODIAQ_DR2_summary.ascii'
kodiaq_meta = Table.read(kodiaq_file, format='ascii', comment='#')
# Verify DR1
dr2 = kodiaq_meta['kodrelease'] == 2
kodiaq_meta = kodiaq_meta[dr2]
# RA/DEC, DATE
ra = []
dec = []
dateobs = []
for row in kodiaq_meta:
# Fix DEC
# Get RA/DEC
coord = ltu.radec_to_coord((row['sRA'],row['sDEC']))
ra.append(coord.ra.value)
dec.append(coord.dec.value)
# DATE
dvals = row['pi_date'].split('_')
tymd = str('{:s}-{:s}-{:02d}'.format(dvals[-1],dvals[1][0:3],int(dvals[2])))
tval = datetime.datetime.strptime(tymd, '%Y-%b-%d')
dateobs.append(datetime.datetime.strftime(tval,'%Y-%m-%d'))
kodiaq_meta.add_column(Column(ra, name='RA_GROUP'))
kodiaq_meta.add_column(Column(dec, name='DEC_GROUP'))
kodiaq_meta.add_column(Column(dateobs, name='DATE-OBS'))
#
kodiaq_meta['INSTR'] = 'HIRES'
kodiaq_meta['TELESCOPE'] = 'Keck-I'
kodiaq_meta['STYPE'] = str('QSO')
# z
kodiaq_meta.rename_column('zem', 'zem_GROUP')
kodiaq_meta['sig_zem'] = 0.
kodiaq_meta['flag_zem'] = str('SDSS-SIMBAD')
#
assert chk_meta(kodiaq_meta, chk_cat_only=True)
return kodiaq_meta
def grab_meta():
""" Grab High-z ESI meta Table
Returns
-------
"""
#
esidla_meta = Table.read(os.getenv('RAW_IGMSPEC') +
'/HighzESIDLA/ascii_highz_rafelski.list',
format='ascii')
nspec = len(esidla_meta)
# DATE
datearr = [day.split('/') for day in list(esidla_meta['ObsDate'])]
ndate = [
'20' + str(day[2]) + '-' + str(day[0]) + '-' + str(day[1])
for day in datearr
]
t = Time(ndate, out_subfmt='date') # Fixes to YYYY-MM-DD
esidla_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add zem
esidla_meta['sig_zem'] = [0.] * nspec
esidla_meta['flag_zem'] = [str('SDSS')] * nspec
#
esidla_meta.add_column(Column([2000.] * nspec, name='EPOCH'))
esidla_meta.add_column(Column(['KeckII'] * nspec, name='TELESCOPE'))
esidla_meta.add_column(Column(['ESI'] * nspec, name='INSTR'))
esidla_meta.add_column(Column(['ECH'] * nspec, name='DISPERSER'))
# Rename
esidla_meta.rename_column('RA', 'RA_GROUP')
esidla_meta.rename_column('DEC', 'DEC_GROUP')
esidla_meta.rename_column('zem', 'zem_GROUP')
esidla_meta['STYPE'] = str('QSO')
# Sort
esidla_meta.sort('RA_GROUP')
# Check
assert chk_meta(esidla_meta, chk_cat_only=True)
#
return esidla_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append ESI data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
esidla_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
# Checks
if sname != 'ESI_DLA':
raise IOError("Expecting ESI_DLA!!")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 50000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
maxpix = 0
for jj, row in enumerate(meta):
#
specfile = os.getenv(
'RAW_IGMSPEC') + '/HighzESIDLA/{:s}a_xF.fits'.format(row['Name'])
print("ESI_DLA: Reading {:s}".format(specfile))
spec = lsio.readspec(specfile)
# Parse name
fname = specfile.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Continuum
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.to('AA').value
#data['co'][0][:npix] = spec.co.value
# Meta
head = spec.header
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
try:
Rlist.append(Rdicts['ESI'][head['SLMSKNAM']])
except KeyError:
if row['Slit'] == 0.75:
Rlist.append(Rdicts['ESI']['0.75_arcsec'])
elif row['Slit'] == 0.5:
Rlist.append(Rdicts['ESI']['0.50_arcsec'])
else:
pdb.set_trace()
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2012ApJ...755...89R',
bib='rafelski+12'),
dict(url='http://adsabs.harvard.edu/abs/2014ApJ...782L..29R',
bib='rafelski+14'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf,
sname,
hdla100_meta,
debug=False,
chk_meta_only=False,
mk_test_file=False):
""" Append HDLA100 data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hdlls_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
# Checks
if sname != 'HDLA100':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
#if mk_test_file:
# hdla100_full = hdlls_full[0:3]
max_npix = 192000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
nspec = len(hdla100_meta)
spec_set.resize((nspec, ))
Rlist = []
wvminlist = []
wvmaxlist = []
dateobslist = []
npixlist = []
gratinglist = []
# Loop
for jj, row in enumerate(hdla100_meta):
kk = jj
# Extract
f = os.getenv('RAW_IGMSPEC') + '/HDLA100/' + row['SPEC_FILE']
spec = lsio.readspec(f)
# Parse name
fname = f.split('/')[-1]
# npix
head = spec.header
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
try:
Rlist.append(set_resolution(head))
except ValueError:
raise ValueError("Header is required for {:s}".format(fname))
else:
if '/' in head['DATE-OBS']:
spl = head['DATE-OBS'].split('/')
t = Time(datetime.datetime(
int(spl[2]) + 1900, int(spl[1]), int(spl[0])),
format='datetime')
else:
t = Time(head['DATE-OBS'], format='isot', out_subfmt='date')
dateobslist.append(t.iso)
# Grating
try:
gratinglist.append(head['XDISPERS'])
except KeyError:
try:
yr = t.value.year
except AttributeError:
yr = int(t.value[0:4])
if yr <= 1997:
gratinglist.append('RED')
else:
pdb.set_trace()
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[kk] = data
# Add columns
nmeta = len(hdla100_meta)
hdla100_meta.add_column(Column([2000.] * nmeta, name='EPOCH'))
hdla100_meta.add_column(Column(npixlist, name='NPIX'))
hdla100_meta.add_column(
Column([str(date) for date in dateobslist], name='DATE-OBS'))
hdla100_meta.add_column(Column(wvminlist, name='WV_MIN'))
hdla100_meta.add_column(Column(wvmaxlist, name='WV_MAX'))
hdla100_meta.add_column(Column(Rlist, name='R'))
hdla100_meta.add_column(
Column(np.arange(nmeta, dtype=int), name='GROUP_ID'))
hdla100_meta.add_column(Column(gratinglist, name='DISPERSER'))
hdla100_meta['INSTR'] = ['HIRES'] * nspec
hdla100_meta['TELESCOPE'] = ['Keck-I'] * nspec
#hdla100_meta.rename_column('Z_QSO', 'zem')
# Add HDLLS meta to hdf5
if chk_meta(hdla100_meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = hdla100_meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2013ApJ...769...54N',
bib='neeleman+13'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append KODIAQ data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
kodiaq_grp = hdf.create_group(sname)
# Load up
# Checks
if sname != 'KODIAQ_DR2':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 60000 # Just needs to be large enough
# Init
data = init_data(max_npix, include_co=False)
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
# Lists
Rlist = []
wvminlist = []
wvmaxlist = []
gratinglist = []
npixlist = []
speclist = []
# Loop
path = os.getenv('RAW_IGMSPEC')+'/KODIAQ2/Data/'
maxpix = 0
for jj,row in enumerate(meta):
# Generate full file
full_file = path+row['qso']+'/'+row['pi_date']+'/'+row['spec_prefix']+'_f.fits'
# Extract
print("KODIAQ: Reading {:s}".format(full_file))
hduf = fits.open(full_file)
head = hduf[0].header
spec = lsio.readspec(full_file)
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
if 'XDISPERS' in head.keys():
if head['XDISPERS'].strip() == 'UV':
gratinglist.append('BLUE')
else:
gratinglist.append('RED')
else: # Original, earl data
gratinglist.append('RED')
npixlist.append(npix)
try:
Rlist.append(set_resolution(head))
except ValueError:
pdb.set_trace()
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column([2000.]*nspec, name='EPOCH'))
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(gratinglist, name='DISPERSER'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2017AJ....154..114O',
bib='kodiaq')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def grab_meta():
""" Grab COS-Halos meta table
Returns
-------
"""
from time import strptime
from specdb.zem.utils import zem_from_radec
from specdb.specdb import IgmSpec
from specdb.defs import get_res_dicts
Rdicts = get_res_dicts()
igmsp = IgmSpec(db_file=os.getenv('SPECDB')+'/IGMspec_DB_v01.hdf5', skip_test=True)
summ_file = os.getenv('RAW_IGMSPEC')+'/COS-Halos/cos_halos_obs.ascii'
chalos_meta = Table.read(summ_file, format='ascii')
# RA/DEC, DATE
# Visits from this page: http://www.stsci.edu/cgi-bin/get-visit-status?id=11598&markupFormat=html
visit_file = os.getenv('RAW_IGMSPEC')+'/COS-Halos/cos_halos_visits.ascii'
ch_visits = Table.read(visit_file,format='ascii')
ra = []
dec = []
datet = []
for row in chalos_meta:
coord = ltu.radec_to_coord(row['QSO'])
ra.append(coord.ra.value)
dec.append(coord.dec.value)
#
visit = row['Visit']
mtv = np.where(ch_visits['Visit'] == visit)[0]
if len(mtv) != 1:
pdb.set_trace()
else:
chv = ch_visits['Start_UT'][mtv].data[0]
icmma = chv.find(',')
datet.append('{:s}-{:02d}-{:02d}'.format(
chv[icmma+1:icmma+5], strptime(chv[:3],'%b').tm_mon,
int(chv[3:icmma])))
chalos_meta.add_column(Column(ra, name='RA'))
chalos_meta.add_column(Column(dec, name='DEC'))
chalos_meta.add_column(Column(datet, name='DATE-OBS'))
# Others
chalos_meta.add_column(Column([' ']*len(chalos_meta), name='TELESCOPE')) # Padding
chalos_meta.add_column(Column([' ']*len(chalos_meta), name='INSTR')) # Padding for HIRES
chalos_meta.add_column(Column(['G130M/G160M']*len(chalos_meta), name='DISPERSER'))
chalos_meta.add_column(Column([20000.]*len(chalos_meta), name='R'))
chalos_meta.add_column(Column([2000.]*len(chalos_meta), name='EPOCH'))
chalos_meta['INSTR'] = 'COS' # Deals with padding
chalos_meta['TELESCOPE'] = 'HST'
# Myers for zem
zem, zsource = zem_from_radec(chalos_meta['RA'], chalos_meta['DEC'], Table(igmsp.hdf['quasars'].value))
badz = zem <= 0.
if np.sum(badz) > 0:
raise ValueError("Bad zem in COS-Halos")
chalos_meta['zem'] = zem
chalos_meta['sig_zem'] = 0. # Need to add
chalos_meta['flag_zem'] = zsource
# HIRES
hires_files = glob.glob(os.getenv('RAW_IGMSPEC')+'/COS-Halos/HIRES/J*f.fits.gz')
hires_tab = chalos_meta[0:0]
subnm = np.array([row['QSO'][4:9] for row in chalos_meta])
signs = np.array([row['QSO'][14] for row in chalos_meta])
for ifile in hires_files:
print(ifile)
fname = ifile.split('/')[-1]
mt = np.where((subnm == fname[0:5]) & (signs == fname[5]))[0]
if len(mt) != 1:
pdb.set_trace()
# Add row
hires_tab.add_row(chalos_meta[mt[0]])
hires_tab[-1]['INSTR'] = 'HIRES'
hires_tab[-1]['TELESCOPE'] = 'Keck I'
hires_tab[-1]['DISPERSER'] = 'Red'
hires_tab[-1]['R'] = Rdicts['HIRES']['C1']
# Combine
chalos_meta = vstack([chalos_meta, hires_tab])
chalos_meta['STYPE'] = str('QSO')
# Rename
chalos_meta.rename_column('RA', 'RA_GROUP')
chalos_meta.rename_column('DEC', 'DEC_GROUP')
chalos_meta.rename_column('zem', 'zem_GROUP')
# Check
assert chk_meta(chalos_meta, chk_cat_only=True)
# Done
return chalos_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append XQ-100 data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
xq100_grp = hdf.create_group(sname)
if len(meta) != 300:
pdb.set_trace()
# Checks
if sname != 'XQ-100':
raise IOError("Expecting XQ-100!!")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 20000 # Just needs to be large enough
data = init_data(max_npix, include_co=True)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
gratinglist = []
telelist = []
instrlist = []
# Loop
maxpix = 0
for jj,row in enumerate(meta):
#
print("XQ-100: Reading {:s}".format(row['SPEC_FILE']))
spec = lsio.readspec(row['SPEC_FILE'])
# Parse name
fname = row['SPEC_FILE'].split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Continuum
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.to('AA').value
data['co'][0][:npix] = spec.co.value
# Meta
head = spec.header
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
telelist.append(head['TELESCOP'])
instrlist.append(head['INSTRUME'])
gratinglist.append(head['DISPELEM'])
npixlist.append(npix)
if gratinglist[-1] == 'NIR': # From Lopez+16
Rlist.append(4350.)
elif gratinglist[-1] == 'VIS':
Rlist.append(7450.)
elif gratinglist[-1] == 'UVB':
Rlist.append(5300.)
else:
pdb.set_trace()
raise ValueError("UH OH")
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(gratinglist, name='DISPERSER'))
meta.add_column(Column(telelist, name='TELESCOPE'))
meta.add_column(Column(instrlist, name='INSTR'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2016arXiv160708776L',
bib='lopez+16')]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf,
sname,
meta,
debug=False,
chk_meta_only=False,
sdss_hdf=None,
**kwargs):
""" Add SDSS data to the DB
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
if sdss_hdf is not None:
print("Using previously generated {:s} dataset...".format(sname))
sdss_hdf.copy(sname, hdf)
return
sdss_grp = hdf.create_group(sname)
# Load up
# Checks
if sname != 'SDSS_DR7':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 4000 # Just needs to be large enough
data = init_data(max_npix, include_co=True)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
# Read Zhu continua, wave file
cfile = os.getenv(
'RAW_IGMSPEC') + '/SDSS/ALLQSO_SPEC_106_continuum_nointerp.fits'
zhu_conti = Table.read(cfile)
wvfile = cfile.replace('continuum', 'wave')
zhu_wave = Table.read(wvfile)
#
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
maxpix = 0
for jj, row in enumerate(meta):
full_file = get_specfil(row)
if not os.path.isfile(full_file):
full_file = get_specfil(row, dr7=True)
# Extract
#print("SDSS: Reading {:s}".format(full_file))
# Parse name
fname = full_file.split('/')[-1]
# Generate full file
spec = lsio.readspec(full_file)
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Some fiddling about
for key in ['wave', 'flux', 'sig', 'co']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Continuum
mtc = (zhu_conti['PLATE'] == row['PLATE']) & (zhu_conti['FIBER']
== row['FIBER'])
mtw = (zhu_wave['PLATE'] == row['PLATE']) & (zhu_wave['FIBER']
== row['FIBER'])
if np.sum(mtc) == 1:
imin = np.argmin(
np.abs(zhu_wave['WAVE'][0][:, np.where(mtw)[1]] -
spec.wavelength[0].value))
data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][
imin:npix + imin, np.where(mtc)[1]].flatten()
elif np.sum(mtc) > 1:
print(
"Multiple continua for plate={:d}, row={:d}. Taking the first"
.format(row['PLATE'], row['FIBER']))
imin = np.argmin(
np.abs(zhu_wave['WAVE'][0][:, np.where(mtw)[1][0]] -
spec.wavelength[0].value))
data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][
imin:npix + imin, np.where(mtc)[1][0]].flatten()
elif np.sum(mtc) == 0:
print("No SDSS continuum for plate={:d}, row={:d}".format(
row['PLATE'], row['FIBER']))
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
#xdb.xplot(data['wave'][0], data['flux'][0], data['co'][0])
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2010AJ....139.2360S',
bib='sdss_qso_dr7'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf,
sname,
meta,
debug=False,
chk_meta_only=False,
mk_test_file=False):
""" Append HST/FUSE data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
Rdicts = defs.get_res_dicts()
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hstc_grp = hdf.create_group(sname)
# Checks
if sname != 'UVpSM4':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 40000 # Just needs to be large enough
# Init
data = init_data(max_npix, include_co=True)
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
gratinglist = []
datelist = []
badf = []
badstis = []
badghrs = []
# Loop
path = os.getenv('RAW_IGMSPEC') + '/HST_Cooksey/'
maxpix = 0
for jj, row in enumerate(meta):
# Generate full file
full_file = path + '{:s}/{:s}/{:s}'.format(row['QSO'], row['INSTR'],
row['SPEC_FILE'])
# Extract
if row['INSTR'] == 'FUSE':
hext = 1
else:
hext = 0
print("HST_Cooksey: Reading {:s}".format(full_file))
try:
spec = lsio.readspec(full_file, head_exten=hext, masking='edges')
except: # BAD HEADER
hdu = fits.open(full_file)
head1 = hdu[1].header
hdu[1].verify('fix')
tbl = Table(hdu[1].data)
spec = lsio.readspec(tbl, masking='edges')
spec.meta['headers'][spec.select] = head1
# Continuum
cfile = full_file.replace('.fits', '_c.fits')
if os.path.isfile(cfile):
# Watch that mask!
gdp = ~spec.data['flux'][spec.select].mask
spec.data['co'][spec.select][gdp] = (
fits.open(cfile)[0].data)[gdp]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Some fiddling about
for key in ['wave', 'flux', 'sig', 'co']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
if spec.co_is_set:
try:
data['co'][0][:npix] = spec.co.value
except ValueError:
pdb.set_trace()
# Meta
datet = None
if row['INSTR'] == 'FUSE':
if 'HISTORY' in spec.header.keys():
ncards = len(spec.header['HISTORY'])
flg_H = True
else:
flg_H = False
hdu = fits.open(full_file)
head0 = hdu[0].header
ncards = len(head0)
# Is this a good one?
if 'APER_ACT' in head0:
pass
else: # Need to fight harder for the header
# Look for untrim
untrim = full_file + '.untrim'
if not os.path.isfile(untrim):
pdb.set_trace()
# Read
hduu = fits.open(untrim)
if 'PKS2005' in untrim: # One extra kludge..
head0 = hduu[1].header
flg_H = True
ncards = len(head0['HISTORY'])
else:
head0 = hduu[0].header
ncards = len(head0)
spec.meta['headers'][spec.select] = head0
# Read from history
for ss in range(ncards):
if flg_H:
try:
card = Header.fromstring(spec.header['HISTORY'][ss])
except:
pdb.set_trace()
try:
ckey = list(card.keys())[0]
except IndexError:
continue
else:
card0 = card[0]
else:
ckey, card0 = list(spec.header.keys())[ss], spec.header[ss]
# Parse
if ckey == 'APERTURE':
aper = card0
elif ckey == 'DETECTOR':
det = card0
elif ckey == 'APER_ACT': # Extracted aperture
ext_ap = card0
elif ckey == 'DATE': # Extracted aperture
datet = card0
gratinglist.append(ext_ap + det)
elif row['INSTR'] == 'STIS':
try:
datet = spec.header['DATE']
except KeyError: # handful of kludged coadds
if 'HISTORY' not in spec.header.keys():
# Grab from the other extension, e.g. PKS0405
hdu = fits.open(full_file)
head1 = hdu[1].header
spec.meta['headers'][0] = head1
for ihist in spec.header['HISTORY']:
if 'TDATEOBS' in ihist:
idash = ihist.find('-')
datet = ihist[idash - 4:idash + 6]
# Grating from name
i0 = full_file.rfind('_')
i1 = full_file.rfind('.fits')
gratinglist.append(full_file[i0 + 1:i1])
if datet is None:
pdb.set_trace()
else:
gratinglist.append(spec.header['OPT_ELEM'])
elif row['INSTR'] == 'GHRS':
# Date
try:
tmp = spec.header['DATE-OBS']
except KeyError:
# Pull header from parallel file
iM = full_file.find('M_1')
if iM <= 0:
iM = full_file.find('L_1')
ofile = full_file[:iM + 1] + '_F.fits'
if not os.path.isfile(ofile):
if 'NGC4151' in ofile: # Kludge
ofile = ofile.replace('G160M', 'G160Mmd')
elif 'PKS2155-304_GHRS_G140L' in ofile: # Kludge
ofile = ofile.replace('G140L', 'G140Llo')
elif 'PKS2155-304_GHRS_G160M' in ofile: # Kludge
ofile = ofile.replace('G160M', 'G160Mmd')
else:
pdb.set_trace()
hdu = fits.open(ofile)
head0 = hdu[0].header
spec.meta['headers'][spec.select] = head0
# Reformat
prs = tmp.split('/')
if prs[2][0] == '9':
yr = '19' + prs[2]
else:
yr = '20' + prs[2]
datet = yr + '-' + prs[1] + '-{:02d}'.format(int(prs[0]))
# Grating
gratinglist.append(spec.header['GRATING'])
else:
pdb.set_trace()
if datet is None:
try:
datet = spec.header['DATE-OBS']
except KeyError:
print("Missing Header for file: {:s}".format(full_file))
badf.append(full_file)
datet = '9999-9-9'
t = Time(datet, format='isot',
out_subfmt='date') # Fixes to YYYY-MM-DD
datelist.append(t.iso)
try:
Rlist.append(Rdicts[row['INSTR']][gratinglist[-1]])
except KeyError:
print(gratinglist[-1])
pdb.set_trace()
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
#
if (len(badstis)) > 0:
raise ValueError("Somehow have a bad STIS header..")
if len(badf) > 0:
print("We still have bad FUSE headers")
pdb.set_trace()
if len(badghrs) > 0:
print("We still have bad GHRS headers")
pdb.set_trace()
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(gratinglist, name='DISPERSER'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(datelist, name='DATE-OBS'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
pdb.set_trace()
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2010ApJ...708..868C',
bib='cooksey10')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, sdss_hdf=None, **kwargs):
""" Add SDSS data to the DB
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
if sdss_hdf is not None:
print("Using previously generated {:s} dataset...".format(sname))
sdss_hdf.copy(sname, hdf)
return
sdss_grp = hdf.create_group(sname)
# Load up
# Checks
if sname != 'SDSS_DR7':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 4000 # Just needs to be large enough
data = init_data(max_npix, include_co=True)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
# Read Zhu continua, wave file
cfile = os.getenv('RAW_IGMSPEC')+'/SDSS/ALLQSO_SPEC_106_continuum_nointerp.fits'
zhu_conti = Table.read(cfile)
wvfile = cfile.replace('continuum','wave')
zhu_wave = Table.read(wvfile)
#
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
maxpix = 0
for jj,row in enumerate(meta):
full_file = get_specfil(row)
if not os.path.isfile(full_file):
full_file = get_specfil(row, dr7=True)
# Extract
#print("SDSS: Reading {:s}".format(full_file))
# Parse name
fname = full_file.split('/')[-1]
# Generate full file
spec = lsio.readspec(full_file)
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Some fiddling about
for key in ['wave','flux','sig','co']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Continuum
mtc = (zhu_conti['PLATE'] == row['PLATE']) & (zhu_conti['FIBER']==row['FIBER'])
mtw = (zhu_wave['PLATE'] == row['PLATE']) & (zhu_wave['FIBER']==row['FIBER'])
if np.sum(mtc) == 1:
imin = np.argmin(np.abs(zhu_wave['WAVE'][0][:,np.where(mtw)[1]]-spec.wavelength[0].value))
data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][imin:npix+imin,np.where(mtc)[1]].flatten()
elif np.sum(mtc) > 1:
print("Multiple continua for plate={:d}, row={:d}. Taking the first".format(row['PLATE'], row['FIBER']))
imin = np.argmin(np.abs(zhu_wave['WAVE'][0][:,np.where(mtw)[1][0]]-spec.wavelength[0].value))
data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][imin:npix+imin,np.where(mtc)[1][0]].flatten()
elif np.sum(mtc) == 0:
print("No SDSS continuum for plate={:d}, row={:d}".format(row['PLATE'], row['FIBER']))
#from xastropy.xutils import xdebug as xdb
#xdb.set_trace()
#xdb.xplot(data['wave'][0], data['flux'][0], data['co'][0])
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2010AJ....139.2360S',
bib='sdss_qso_dr7'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def ingest_spectra(hdf, sname, meta, max_npix=10000, chk_meta_only=False,
debug=False, xspec=None, scale=1.,
refs=None, verbose=False, badf=None, set_idkey=None,
grab_conti=False, **kwargs):
""" Ingest the spectra
Parameters
----------
hdf : hdf5 pointer
sname : str
Name of dataset
meta : Table
max_npix : int, optional
Maximum length of the spectra
chk_meta_only : bool, optional
Only check meta file; will not write
refs : list, optional
list of dicts with reference info
badf : list, optional
List of bad spectra [use only if you know what you are doing!]
grab_conti : bool, optional
Grab continua. They should exist but do not have to
set_idkey : str, optional
Only required if you are not performing the full script
xspec : list or XSpectrum1D object, optional
Take spectra from this object instead of reading from files
scale : float, optional
Scale the spectra by this factor. Useful for fluxing
**kwargs : optional
Passed to readspec()
Returns
-------
"""
if set_idkey is not None:
set_sv_idkey(set_idkey)
# Add Survey
print("Adding {:s} group to DB".format(sname))
grp = hdf.create_group(sname)
# Spectra
nspec = len(meta)
dtypes=[(str('wave'), 'float64', (max_npix)),
(str('flux'), 'float32', (max_npix)),
(str('sig'), 'float32', (max_npix))]
dkeys = ['wave','flux','sig']
if grab_conti:
dtypes += [(str('co'), 'float32', (max_npix))]
dkeys += ['co']
data = np.ma.empty((1,), dtype=dtypes)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
wvminlist = []
wvmaxlist = []
npixlist = []
# Loop
for jj,member in enumerate(meta['SPEC_FILE']):
# Extract
f = member
# Parse name
fname = f.split('/')[-1]
if verbose:
print(fname)
# Read
if xspec is None:
if badf is not None:
for ibadf in badf:
if ibadf in f:
spec = dumb_spec()
else:
try:
spec = lsio.readspec(f, **kwargs)
except ValueError: # Probably a continuum problem
pdb.set_trace()
else:
spec = lsio.readspec(f, **kwargs)
else:
spec = xspec[jj]
if debug:
pdb.set_trace()
# Meta
#head = spec.header
if spec.masking == 'none':
npix = spec.wavelength.value.size
else:
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d} vs {:d})".format(
npix, max_npix))
# Some fiddling about
for key in dkeys:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value * scale
data['sig'][0][:npix] = spec.sig.value * scale
data['wave'][0][:npix] = spec.wavelength.value
if grab_conti:
if spec.co_is_set:
data['co'][0][:npix] = spec.co.value
# Meta
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
# Set
spec_set[jj] = data
# Add columns
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
# Add HDLLS meta to hdf5
if spbu.chk_meta(meta):#, skip_igmid=True):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
pdb.set_trace()
raise ValueError("meta file failed")
# References
if refs is not None:
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, hdla100_meta, debug=False, chk_meta_only=False,
mk_test_file=False):
""" Append HDLA100 data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hdlls_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
# Checks
if sname != 'HDLA100':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
#if mk_test_file:
# hdla100_full = hdlls_full[0:3]
max_npix = 192000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
nspec = len(hdla100_meta)
spec_set.resize((nspec,))
Rlist = []
wvminlist = []
wvmaxlist = []
dateobslist = []
npixlist = []
gratinglist = []
# Loop
for jj,row in enumerate(hdla100_meta):
kk = jj
# Extract
f = os.getenv('RAW_IGMSPEC')+'/HDLA100/'+row['SPEC_FILE']
spec = lsio.readspec(f)
# Parse name
fname = f.split('/')[-1]
# npix
head = spec.header
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
try:
Rlist.append(set_resolution(head))
except ValueError:
raise ValueError("Header is required for {:s}".format(fname))
else:
if '/' in head['DATE-OBS']:
spl = head['DATE-OBS'].split('/')
t = Time(datetime.datetime(int(spl[2])+1900, int(spl[1]), int(spl[0])), format='datetime')
else:
t = Time(head['DATE-OBS'], format='isot', out_subfmt='date')
dateobslist.append(t.iso)
# Grating
try:
gratinglist.append(head['XDISPERS'])
except KeyError:
try:
yr = t.value.year
except AttributeError:
yr = int(t.value[0:4])
if yr <= 1997:
gratinglist.append('RED')
else:
pdb.set_trace()
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[kk] = data
# Add columns
nmeta = len(hdla100_meta)
hdla100_meta.add_column(Column([2000.]*nmeta, name='EPOCH'))
hdla100_meta.add_column(Column(npixlist, name='NPIX'))
hdla100_meta.add_column(Column([str(date) for date in dateobslist], name='DATE-OBS'))
hdla100_meta.add_column(Column(wvminlist, name='WV_MIN'))
hdla100_meta.add_column(Column(wvmaxlist, name='WV_MAX'))
hdla100_meta.add_column(Column(Rlist, name='R'))
hdla100_meta.add_column(Column(np.arange(nmeta,dtype=int),name='GROUP_ID'))
hdla100_meta.add_column(Column(gratinglist, name='DISPERSER'))
hdla100_meta['INSTR'] = ['HIRES']*nspec
hdla100_meta['TELESCOPE'] = ['Keck-I']*nspec
#hdla100_meta.rename_column('Z_QSO', 'zem')
# Add HDLLS meta to hdf5
if chk_meta(hdla100_meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = hdla100_meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2013ApJ...769...54N',
bib='neeleman+13'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Add 2QZ data to the DB
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
tqz_grp = hdf.create_group(sname)
# Checks
if sname != '2QZ':
raise IOError("Not expecting this survey..")
# Add zem
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 4000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
maxpix = 0
for jj, row in enumerate(meta):
full_file = get_specfil(row)
# Parse name
fname = full_file.split('/')[-1]
# Read
hdu = fits.open(full_file)
head0 = hdu[0].header
wave = lsio.setwave(head0)
flux = hdu[0].data
var = hdu[2].data
sig = np.zeros_like(flux)
gd = var > 0.
if np.sum(gd) == 0:
print("{:s} has a bad var array. Not including".format(fname))
pdb.set_trace()
continue
sig[gd] = np.sqrt(var[gd])
# npix
spec = XSpectrum1D.from_tuple((wave, flux, sig))
npix = spec.npix
spec.meta['headers'][0] = head0
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
#
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2004MNRAS.349.1397C',
bib='2QZ')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False,
mk_test_file=False):
""" Append HSTQSO data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hstz2_grp = hdf.create_group(sname)
# Checks
if sname != 'HSTQSO':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 80000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
wvminlist = []
wvmaxlist = []
npixlist = []
Rlist = []
# Loop
#path = os.getenv('RAW_IGMSPEC')+'/KODIAQ_data_20150421/'
path = os.getenv('RAW_IGMSPEC')+'/HSTQSO/'
maxpix = 0
for jj,row in enumerate(meta):
# Generate full file
full_file = path+row['SPEC_FILE']+'.gz'
# Extract
print("HSTQSO: Reading {:s}".format(full_file))
hduf = fits.open(full_file)
head0 = hduf[0].header
spec = lsio.readspec(full_file, masking='edges')
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
if 'FOS-L' in fname:
Rlist.append(300.)
elif 'FOS-H' in fname:
Rlist.append(14000.)
elif 'STIS' in fname:
if row['DISPERSER'] == 'G230L':
Rlist.append(700.)
elif row['DISPERSER'] == 'G140L':
Rlist.append(1200.)
else:
raise ValueError("Bad STIS grating")
elif 'hsla' in fname: # COS
Rlist.append(18000.)
row['DATE-OBS'] = hduf[1].data['DATEOBS'][0][0]
else:
pdb.set_trace()
raise ValueError("Missing instrument!")
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column([2000.]*nspec, name='EPOCH'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2011ApJ...736...42R',
bib='ribuado11'),
dict(url='http://adsabs.harvard.edu/abs/2016ApJ...818..113N',
bib='neeleman16'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def grab_meta():
""" Grab GGG meta Table
Catalog -- http://www.2dfquasar.org/Spec_Cat/catalogue.html
Returns
-------
meta
"""
catfil = os.getenv('RAW_IGMSPEC')+'/2QZ/2QZ393524355693.out'
tdf_meta = Table.read(catfil, format='ascii')
# Rename columns
clms = ['Name', 'RAh00', 'RAm00', 'RAs00', 'DECd00', 'DECm00', 'DECs00',
'ID','cat_name', 'Sector', 'RAh50', 'RAm50', 'RAs50', 'DECd50', 'DECm50', 'DECs50',
'UKST', 'XAPM','YAPM','RA50','DEC50','bj','u-b','b-r','Nobs',
'z1','q1','ID1','date1','fld1','fiber1','SN1',
'z2','q2','ID2','date2','fld2','fiber2','SN2',
'zprev','rflux','Xray','EBV','comm1','comm2']
for ii in range(1,46):
tdf_meta.rename_column('col{:d}'.format(ii), clms[ii-1])
# Cut down to QSOs and take only 1 spectrum
ispec = []
zspec = []
datespec = []
for row in tdf_meta:
if 'QSO' in row['ID1']:
ispec.append(1)
zspec.append(row['z1'])
sdate = str(row['date1'])
datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4],sdate[4:6], sdate[6:8]))
elif 'QSO' in row['ID2']:
ispec.append(2)
zspec.append(row['z2'])
sdate = str(row['date2'])
datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4],sdate[4:6], sdate[6:8]))
else:
ispec.append(0)
zspec.append(-1.)
datespec.append('')
tdf_meta['ispec'] = ispec
tdf_meta['zem_GROUP'] = zspec
tdf_meta['DATE'] = datespec
cut = tdf_meta['ispec'] > 0
tdf_meta = tdf_meta[cut]
nspec = len(tdf_meta)
# DATE
t = Time(list(tdf_meta['DATE'].data), format='iso', out_subfmt='date') # Fixes to YYYY-MM-DD
tdf_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add a few columns
tdf_meta.add_column(Column([2000.]*nspec, name='EPOCH'))
# Resolution
# 2df 8.6A FWHM
# R=580 at 5000A
# 6df??
tdf_meta.add_column(Column([580.]*nspec, name='R'))
#
tdf_meta.add_column(Column([str('2dF')]*nspec, name='INSTR'))
tdf_meta.add_column(Column([str('300B')]*nspec, name='DISPERSER'))
tdf_meta.add_column(Column([str('UKST')]*nspec, name='TELESCOPE'))
# Rename
rad = (tdf_meta['RAh00']*3600 + tdf_meta['RAm00']*60 + tdf_meta['RAs00'])*360./86400.
decd = np.abs(tdf_meta['DECd00']) + tdf_meta['DECm00']/60 + tdf_meta['DECs00']/3600.
# Yup the following is necessary
neg = [False]*len(rad)
for jj,row in enumerate(tdf_meta):
if '-' in row['Name']:
neg[jj] = True
#if '-00' in row['Name']:
# print('jj={:d}'.format(jj))
neg = np.array(neg)
decd[neg] = -1.*decd[neg]
tdf_meta['RA_GROUP'] = rad
tdf_meta['DEC_GROUP'] = decd
tdf_meta['sig_zem'] = [0.]*nspec
tdf_meta['flag_zem'] = str('2QZ')
tdf_meta['STYPE'] = str('QSO')
# Require a spectrum exist
gdm = np.array([True]*len(tdf_meta))
for jj,row in enumerate(tdf_meta):
full_file = get_specfil(row)
if not os.path.isfile(full_file):
print("{:s} has no spectrum. Not including".format(full_file))
gdm[jj] = False
continue
tdf_meta = tdf_meta[gdm]
# Sort
tdf_meta.sort('RA_GROUP')
# Check
assert chk_meta(tdf_meta, chk_cat_only=True)
# Return
return tdf_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append UVES_Dall data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
uvesdall_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
# Checks
if sname != 'UVES_Dall':
raise IOError("Expecting UVES_Dall!!")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 150000 # Just needs to be large enough
data = init_data(max_npix, include_co=True)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
wvminlist, wvmaxlist, npixlist, speclist = [], [], [], []
# Loop
maxpix = 0
for jj,row in enumerate(meta):
# Read
specfile = os.getenv('RAW_IGMSPEC')+'/UVES_Dall/{:s}_flux.dat'.format(row['NAME'])
print("UVES_Dall: Reading {:s}".format(specfile))
spec = Table.read(specfile,format='ascii.fast_no_header',guess=False)#, data_start=1)
# Parse name
fname = specfile.split('/')[-1]
# npix
npix = len(spec['col1'])
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Continuum
# Some fiddling about
for key in ['wave','flux','sig','co']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec['col2']
data['sig'][0][:npix] = spec['col3']
data['wave'][0][:npix] = spec['col1']
data['co'][0][:npix] = spec['col4']
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2008A%26A...491..465D',
bib='dallaglio+08'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf,
sname,
meta,
debug=False,
chk_meta_only=False,
mk_test_file=False):
""" Append COS-Halos data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
chalos_grp = hdf.create_group(sname)
# Load up
# Checks
if sname != 'COS-Halos':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 160000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
path = os.getenv('RAW_IGMSPEC') + '/COS-Halos/'
maxpix = 0
for jj, row in enumerate(meta):
# Generate full file
coord = ltu.radec_to_coord((row['RA_GROUP'], row['DEC_GROUP']))
if row['INSTR'].strip() == 'COS':
full_file = path + '/J{:s}{:s}_nbin3_coadd.fits.gz'.format(
coord.ra.to_string(unit=u.hour, sep='', pad=True)[0:4],
coord.dec.to_string(sep='', pad=True, alwayssign=True)[0:5])
else: # HIRES
full_file = path + '/HIRES/J{:s}{:s}_f.fits.gz'.format(
coord.ra.to_string(unit=u.hour, sep='', pad=True)[0:4],
coord.dec.to_string(sep='', pad=True, alwayssign=True)[0:5])
# Extract
print("COS-Halos: Reading {:s}".format(full_file))
spec = lsio.readspec(full_file)
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2013ApJ...777...59T',
bib='tumlinson+13'),
dict(url='http://adsabs.harvard.edu/abs/2013ApJS..204...17W',
bib='werk+13')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append GGG data to the h5 file
Parameters
----------
hdf : hdf5 pointer
sname : str
Survey name
meta : Table
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
_ = hdf.create_group(sname)
# Load up
if sname != 'ESI_z6':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 30000 # Just needs to be large enough
# Init
data = init_data(max_npix, include_co=True)
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
gratinglist = []
telelist = []
dateobslist = []
instrlist = []
# Loop
maxpix = 0
for jj, row in enumerate(meta):
# Generate full file
full_file, spec = read_spec(row)
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
data['co'][0][:npix] = spec.co.value
# Meta
#head = spec.header
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
#telelist.append(head['OBSERVAT'])
#instrlist.append(head['INSTRUME'])
#tval = Time(head['DATE'], format='isot', out_subfmt='date')
#dateobslist.append(tval.iso)
npixlist.append(npix)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
#meta.add_column(Column(telelist, name='TELESCOPE'))
#meta.add_column(Column(instrlist, name='INSTR'))
#meta.add_column(Column(dateobslist, name='DATE-OBS'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
#meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2014MNRAS.445.1745W',
bib='worseck+14')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def grab_meta():
""" Grab COS-Halos meta table
Returns
-------
"""
from time import strptime
from specdb.zem.utils import zem_from_radec
from specdb.specdb import IgmSpec
from specdb.defs import get_res_dicts
Rdicts = get_res_dicts()
igmsp = IgmSpec(db_file=os.getenv('SPECDB') + '/IGMspec_DB_v01.hdf5',
skip_test=True)
summ_file = os.getenv('RAW_IGMSPEC') + '/COS-Halos/cos_halos_obs.ascii'
chalos_meta = Table.read(summ_file, format='ascii')
# RA/DEC, DATE
# Visits from this page: http://www.stsci.edu/cgi-bin/get-visit-status?id=11598&markupFormat=html
visit_file = os.getenv('RAW_IGMSPEC') + '/COS-Halos/cos_halos_visits.ascii'
ch_visits = Table.read(visit_file, format='ascii')
ra = []
dec = []
datet = []
for row in chalos_meta:
coord = ltu.radec_to_coord(row['QSO'])
ra.append(coord.ra.value)
dec.append(coord.dec.value)
#
visit = row['Visit']
mtv = np.where(ch_visits['Visit'] == visit)[0]
if len(mtv) != 1:
pdb.set_trace()
else:
chv = ch_visits['Start_UT'][mtv].data[0]
icmma = chv.find(',')
datet.append('{:s}-{:02d}-{:02d}'.format(
chv[icmma + 1:icmma + 5],
strptime(chv[:3], '%b').tm_mon, int(chv[3:icmma])))
chalos_meta.add_column(Column(ra, name='RA'))
chalos_meta.add_column(Column(dec, name='DEC'))
chalos_meta.add_column(Column(datet, name='DATE-OBS'))
# Others
chalos_meta.add_column(
Column([' '] * len(chalos_meta), name='TELESCOPE')) # Padding
chalos_meta.add_column(Column([' '] * len(chalos_meta),
name='INSTR')) # Padding for HIRES
chalos_meta.add_column(
Column(['G130M/G160M'] * len(chalos_meta), name='DISPERSER'))
chalos_meta.add_column(Column([20000.] * len(chalos_meta), name='R'))
chalos_meta.add_column(Column([2000.] * len(chalos_meta), name='EPOCH'))
chalos_meta['INSTR'] = 'COS' # Deals with padding
chalos_meta['TELESCOPE'] = 'HST'
# Myers for zem
zem, zsource = zem_from_radec(chalos_meta['RA'], chalos_meta['DEC'],
Table(igmsp.hdf['quasars'].value))
badz = zem <= 0.
if np.sum(badz) > 0:
raise ValueError("Bad zem in COS-Halos")
chalos_meta['zem'] = zem
chalos_meta['sig_zem'] = 0. # Need to add
chalos_meta['flag_zem'] = zsource
# HIRES
hires_files = glob.glob(
os.getenv('RAW_IGMSPEC') + '/COS-Halos/HIRES/J*f.fits.gz')
hires_tab = chalos_meta[0:0]
subnm = np.array([row['QSO'][4:9] for row in chalos_meta])
signs = np.array([row['QSO'][14] for row in chalos_meta])
for ifile in hires_files:
print(ifile)
fname = ifile.split('/')[-1]
mt = np.where((subnm == fname[0:5]) & (signs == fname[5]))[0]
if len(mt) != 1:
pdb.set_trace()
# Add row
hires_tab.add_row(chalos_meta[mt[0]])
hires_tab[-1]['INSTR'] = 'HIRES'
hires_tab[-1]['TELESCOPE'] = 'Keck I'
hires_tab[-1]['DISPERSER'] = 'Red'
hires_tab[-1]['R'] = Rdicts['HIRES']['C1']
# Combine
chalos_meta = vstack([chalos_meta, hires_tab])
chalos_meta['STYPE'] = str('QSO')
# Rename
chalos_meta.rename_column('RA', 'RA_GROUP')
chalos_meta.rename_column('DEC', 'DEC_GROUP')
chalos_meta.rename_column('zem', 'zem_GROUP')
# Check
assert chk_meta(chalos_meta, chk_cat_only=True)
# Done
return chalos_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append ESI data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
esidla_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
# Checks
if sname != 'ESI_DLA':
raise IOError("Expecting ESI_DLA!!")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 50000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
maxpix = 0
for jj,row in enumerate(meta):
#
specfile = os.getenv('RAW_IGMSPEC')+'/HighzESIDLA/{:s}a_xF.fits'.format(
row['Name'])
print("ESI_DLA: Reading {:s}".format(specfile))
spec = lsio.readspec(specfile)
# Parse name
fname = specfile.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Continuum
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.to('AA').value
#data['co'][0][:npix] = spec.co.value
# Meta
head = spec.header
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
try:
Rlist.append(Rdicts['ESI'][head['SLMSKNAM']])
except KeyError:
if row['Slit'] == 0.75:
Rlist.append(Rdicts['ESI']['0.75_arcsec'])
elif row['Slit'] == 0.5:
Rlist.append(Rdicts['ESI']['0.50_arcsec'])
else:
pdb.set_trace()
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2012ApJ...755...89R',
bib='rafelski+12'),
dict(url='http://adsabs.harvard.edu/abs/2014ApJ...782L..29R',
bib='rafelski+14'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Add 2QZ data to the DB
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
tqz_grp = hdf.create_group(sname)
# Checks
if sname != '2QZ':
raise IOError("Not expecting this survey..")
# Add zem
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 4000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
maxpix = 0
for jj,row in enumerate(meta):
full_file = get_specfil(row)
# Parse name
fname = full_file.split('/')[-1]
# Read
hdu = fits.open(full_file)
head0 = hdu[0].header
wave = lsio.setwave(head0)
flux = hdu[0].data
var = hdu[2].data
sig = np.zeros_like(flux)
gd = var > 0.
if np.sum(gd) == 0:
print("{:s} has a bad var array. Not including".format(fname))
pdb.set_trace()
continue
sig[gd] = np.sqrt(var[gd])
# npix
spec = XSpectrum1D.from_tuple((wave,flux,sig))
npix = spec.npix
spec.meta['headers'][0] = head0
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
#
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2004MNRAS.349.1397C',
bib='2QZ')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def grab_meta():
""" Grab GGG meta Table
Catalog -- http://www.2dfquasar.org/Spec_Cat/catalogue.html
Returns
-------
meta
"""
catfil = os.getenv('RAW_IGMSPEC') + '/2QZ/2QZ393524355693.out'
tdf_meta = Table.read(catfil, format='ascii')
# Rename columns
clms = [
'Name', 'RAh00', 'RAm00', 'RAs00', 'DECd00', 'DECm00', 'DECs00', 'ID',
'cat_name', 'Sector', 'RAh50', 'RAm50', 'RAs50', 'DECd50', 'DECm50',
'DECs50', 'UKST', 'XAPM', 'YAPM', 'RA50', 'DEC50', 'bj', 'u-b', 'b-r',
'Nobs', 'z1', 'q1', 'ID1', 'date1', 'fld1', 'fiber1', 'SN1', 'z2',
'q2', 'ID2', 'date2', 'fld2', 'fiber2', 'SN2', 'zprev', 'rflux',
'Xray', 'EBV', 'comm1', 'comm2'
]
for ii in range(1, 46):
tdf_meta.rename_column('col{:d}'.format(ii), clms[ii - 1])
# Cut down to QSOs and take only 1 spectrum
ispec = []
zspec = []
datespec = []
for row in tdf_meta:
if 'QSO' in row['ID1']:
ispec.append(1)
zspec.append(row['z1'])
sdate = str(row['date1'])
datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4], sdate[4:6],
sdate[6:8]))
elif 'QSO' in row['ID2']:
ispec.append(2)
zspec.append(row['z2'])
sdate = str(row['date2'])
datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4], sdate[4:6],
sdate[6:8]))
else:
ispec.append(0)
zspec.append(-1.)
datespec.append('')
tdf_meta['ispec'] = ispec
tdf_meta['zem_GROUP'] = zspec
tdf_meta['DATE'] = datespec
cut = tdf_meta['ispec'] > 0
tdf_meta = tdf_meta[cut]
nspec = len(tdf_meta)
# DATE
t = Time(list(tdf_meta['DATE'].data), format='iso',
out_subfmt='date') # Fixes to YYYY-MM-DD
tdf_meta.add_column(Column(t.iso, name='DATE-OBS'))
# Add a few columns
tdf_meta.add_column(Column([2000.] * nspec, name='EPOCH'))
# Resolution
# 2df 8.6A FWHM
# R=580 at 5000A
# 6df??
tdf_meta.add_column(Column([580.] * nspec, name='R'))
#
tdf_meta.add_column(Column([str('2dF')] * nspec, name='INSTR'))
tdf_meta.add_column(Column([str('300B')] * nspec, name='DISPERSER'))
tdf_meta.add_column(Column([str('UKST')] * nspec, name='TELESCOPE'))
# Rename
rad = (tdf_meta['RAh00'] * 3600 + tdf_meta['RAm00'] * 60 +
tdf_meta['RAs00']) * 360. / 86400.
decd = np.abs(tdf_meta['DECd00']
) + tdf_meta['DECm00'] / 60 + tdf_meta['DECs00'] / 3600.
# Yup the following is necessary
neg = [False] * len(rad)
for jj, row in enumerate(tdf_meta):
if '-' in row['Name']:
neg[jj] = True
#if '-00' in row['Name']:
# print('jj={:d}'.format(jj))
neg = np.array(neg)
decd[neg] = -1. * decd[neg]
tdf_meta['RA_GROUP'] = rad
tdf_meta['DEC_GROUP'] = decd
tdf_meta['sig_zem'] = [0.] * nspec
tdf_meta['flag_zem'] = str('2QZ')
tdf_meta['STYPE'] = str('QSO')
# Require a spectrum exist
gdm = np.array([True] * len(tdf_meta))
for jj, row in enumerate(tdf_meta):
full_file = get_specfil(row)
if not os.path.isfile(full_file):
print("{:s} has no spectrum. Not including".format(full_file))
gdm[jj] = False
continue
tdf_meta = tdf_meta[gdm]
# Sort
tdf_meta.sort('RA_GROUP')
# Check
assert chk_meta(tdf_meta, chk_cat_only=True)
# Return
return tdf_meta
def grab_meta(test=False):
""" Grab BOSS meta Table
Returns
-------
boss_meta : Table
"""
# Paris et al.
qsos_dr14 = Table.read(
os.getenv('RAW_IGMSPEC') + '/BOSS_DR14/DR14Q_v4_4.fits.gz')
# Add original ID to coordinate with spectra and remove it later
qsos_dr14['ORIG_ID'] = np.arange(len(qsos_dr14))
# Cut on BOSS (do not include DR7 files)
boss_dr14 = qsos_dr14[qsos_dr14['SPECTRO'] == 'BOSS']
# Cut out bad ones
bad_meta = Table.read(
resource_filename('igmspec', 'ingest/files/bad_dr14.fits'))
keep = np.array([True] * len(boss_dr14))
for row in bad_meta:
bad_qso = np.where((boss_dr14['PLATE'] == row['PLATE'])
& (boss_dr14['FIBERID'] == row['FIBERID']))[0]
keep[bad_qso] = False
boss_dr14 = boss_dr14[keep]
if test:
boss_dr14 = boss_dr14[:100]
'''
# Cut out Plate 7840
plate7840 = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate7840_matched.ascii'), format='ascii')
not_7840 = boss_dr14['PLATE'] != 7840
for ii in range(len(plate7840)):
isplit = plate7840['sfile'][ii].split('-')
idx = np.where((boss_dr14['PLATE'] == 7840) & (boss_dr14['FIBERID'] == int(isplit[-1][0:4])))[0]
assert len(idx) == 1
not_7840[idx[0]] = True
boss_dr14 = boss_dr14[not_7840]
# Cut out others
for plate in [7879, 3678, 7513, 4869, 7306]:
ptbl = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate{:d}_matched.ascii'.format(plate)),
format='ascii.csv')
not_in = boss_dr14['PLATE'] != plate
for ii in range(len(ptbl)):
idx = np.where((boss_dr14['PLATE'] == ptbl['plate'][ii]) & (boss_dr14['FIBERID'] == ptbl['fiber'][ii]))[0]
not_in[idx[0]] = True
boss_dr14 = boss_dr14[not_in]
'''
# Proceed
boss_dr14['CAT'] = str('DR14')
# Cut on redshift?
#gd = np.any([boss_dr14['Z_PIPE'] > 0., boss_dr14['Z_PCA'] > 0.],axis=0) # CUTS Z_VI
#boss_dr12 = boss_dr12[gd]
#
#
nboss = len(boss_dr14)
# DATE-OBS
t = Time(list(boss_dr14['MJD'].data), format='mjd',
out_subfmt='date') # Fixes to YYYY-MM-DD
boss_dr14.add_column(Column(t.iso, name='DATE-OBS'))
# Add columns
boss_dr14.add_column(Column(['BOSS'] * nboss, name='INSTR'))
boss_dr14.add_column(Column(['BOTH'] * nboss, name='DISPERSER'))
#http://www.sdss.org/instruments/boss_spectrograph/
boss_dr14.add_column(Column([2100.] * nboss, name='R')) # RESOLUTION
boss_dr14.add_column(Column(['SDSS 2.5-M'] * nboss, name='TELESCOPE'))
# Redshift logic
boss_dr14['zem_GROUP'] = boss_dr14['Z']
boss_dr14['sig_zem'] = boss_dr14['Z_PIPE_ERR']
boss_dr14['flag_zem'] = boss_dr14['SOURCE_Z']
'''
# Fix bad redshifts
bad_pca = boss_meta['Z_PCA'] < 0.
boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca]
boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca]
boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE')
'''
# Rename RA/DEC
boss_dr14.rename_column('RA', 'RA_GROUP')
boss_dr14.rename_column('DEC', 'DEC_GROUP')
# STYPE
boss_dr14['STYPE'] = str('QSO')
# Check
assert chk_meta(boss_dr14, chk_cat_only=True)
# Return
return boss_dr14
def hdf5_adddata(hdf,
sname,
meta,
debug=False,
chk_meta_only=False,
mk_test_file=False):
""" Append HD-LLS data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hdlls_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
mike_meta = grab_meta_mike()
mike_coord = SkyCoord(ra=mike_meta['RA_GROUP'],
dec=mike_meta['DEC_GROUP'],
unit='deg')
# Checks
if sname != 'HD-LLS_DR1':
raise IOError("Not expecting this survey..")
full_coord = SkyCoord(ra=meta['RA_GROUP'],
dec=meta['DEC_GROUP'],
unit='deg')
# Build spectra (and parse for meta)
if mk_test_file:
meta = meta[0:3]
nspec = len(meta)
max_npix = 210000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
full_idx = np.zeros(len(meta), dtype=int)
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
Rlist = []
wvminlist = []
wvmaxlist = []
dateobslist = []
npixlist = []
instrlist = []
gratinglist = []
telelist = []
# Loop
members = glob.glob(os.getenv('RAW_IGMSPEC') + '/{:s}/*fits'.format(sname))
kk = -1
for jj, member in enumerate(members):
if 'HD-LLS_DR1.fits' in member:
continue
kk += 1
# Extract
f = member
hdu = fits.open(f)
# Parse name
fname = f.split('/')[-1]
mt = np.where(meta['SPEC_FILE'] == fname)[0]
if mk_test_file and (jj >= 3):
continue
if len(mt) != 1:
pdb.set_trace()
raise ValueError("HD-LLS: No match to spectral file?!")
else:
print('loading {:s}'.format(fname))
full_idx[kk] = mt[0]
# npix
head = hdu[0].header
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
# Double check
if kk == 0:
assert hdu[1].name == 'ERROR'
assert hdu[2].name == 'WAVELENGTH'
# Write
spec = lsio.readspec(f) # Handles dummy pixels in ESI
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
#data['flux'][0][:npix] = hdu[0].data
#data['sig'][0][:npix] = hdu[1].data
#data['wave'][0][:npix] = hdu[2].data
# Meta
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if 'HIRES' in fname:
instrlist.append('HIRES')
telelist.append('Keck-I')
gratinglist.append('BOTH')
try:
Rlist.append(set_resolution(head))
except ValueError:
# A few by hand (pulled from Table 1)
if 'J073149' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2006-01-04', '%Y-%m-%d')
elif 'J081435' in fname:
Rlist.append(Rdicts['HIRES']['C1'])
tval = datetime.datetime.strptime('2006-12-26',
'%Y-%m-%d') # 2008 too
elif 'J095309' in fname:
Rlist.append(Rdicts['HIRES']['C1'])
tval = datetime.datetime.strptime('2005-03-18', '%Y-%m-%d')
elif 'J113418' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2006-01-05', '%Y-%m-%d')
elif 'J135706' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2007-04-28', '%Y-%m-%d')
elif 'J155556.9' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2005-04-15', '%Y-%m-%d')
elif 'J212329' in fname:
Rlist.append(Rdicts['HIRES']['E3'])
tval = datetime.datetime.strptime('2006-08-20', '%Y-%m-%d')
else:
pdb.set_trace()
else:
tval = datetime.datetime.strptime(head['DATE-OBS'], '%Y-%m-%d')
dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d'))
elif 'ESI' in fname:
instrlist.append('ESI')
telelist.append('Keck-II')
gratinglist.append('ECH')
try:
Rlist.append(set_resolution(head))
except ValueError:
print("Using R=6,000 for ESI")
Rlist.append(6000.)
try:
tval = datetime.datetime.strptime(head['DATE'], '%Y-%m-%d')
except KeyError:
if ('J223438.5' in fname) or ('J231543' in fname):
tval = datetime.datetime.strptime('2004-09-11', '%Y-%m-%d')
else:
pdb.set_trace()
dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d'))
elif 'MIKE' in fname: # APPROXIMATE
if 'MIKEr' in fname:
instrlist.append('MIKEr')
gratinglist.append('RED')
elif 'MIKEb' in fname:
instrlist.append('MIKEb')
gratinglist.append('BLUE')
else:
instrlist.append('MIKE')
gratinglist.append('BOTH')
telelist.append('Magellan')
sep = full_coord[mt[0]].separation(mike_coord)
imin = np.argmin(sep)
if sep[imin] > 1. * u.arcsec:
pdb.set_trace()
raise ValueError("Bad separation in MIKE")
# R and Date
Rlist.append(25000. / mike_meta['Slit'][imin])
tval = datetime.datetime.strptime(mike_meta['DATE-OBS'][imin],
'%Y-%b-%d')
dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d'))
elif 'MAGE' in fname: # APPROXIMATE
instrlist.append('MagE')
if 'Clay' in head['TELESCOP']:
telelist.append('Magellan/Clay')
else:
telelist.append('Magellan/Baade')
gratinglist.append('N/A')
Rlist.append(set_resolution(head))
dateobslist.append(head['DATE-OBS'])
else: # MagE
raise ValueError("UH OH")
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[kk] = data
# Add columns
meta = meta[full_idx]
nmeta = len(meta)
meta.add_column(Column([2000.] * nmeta, name='EPOCH'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(
Column([str(date) for date in dateobslist], name='DATE-OBS'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nmeta, dtype=int), name='GROUP_ID'))
meta.add_column(Column(gratinglist, name='GRATING'))
meta.add_column(Column(instrlist, name='INSTR'))
meta.add_column(Column(telelist, name='TELESCOPE'))
# v02
meta.rename_column('GRATING', 'DISPERSER')
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2015ApJS..221....2P',
bib='prochaska+15'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False,
mk_test_file=False):
""" Append HD-LLS data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
from specdb import defs
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hdlls_grp = hdf.create_group(sname)
# Load up
Rdicts = defs.get_res_dicts()
mike_meta = grab_meta_mike()
mike_coord = SkyCoord(ra=mike_meta['RA_GROUP'], dec=mike_meta['DEC_GROUP'], unit='deg')
# Checks
if sname != 'HD-LLS_DR1':
raise IOError("Not expecting this survey..")
full_coord = SkyCoord(ra=meta['RA_GROUP'], dec=meta['DEC_GROUP'], unit='deg')
# Build spectra (and parse for meta)
if mk_test_file:
meta = meta[0:3]
nspec = len(meta)
max_npix = 210000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
full_idx = np.zeros(len(meta), dtype=int)
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
Rlist = []
wvminlist = []
wvmaxlist = []
dateobslist = []
npixlist = []
instrlist = []
gratinglist = []
telelist = []
# Loop
members = glob.glob(os.getenv('RAW_IGMSPEC')+'/{:s}/*fits'.format(sname))
kk = -1
for jj,member in enumerate(members):
if 'HD-LLS_DR1.fits' in member:
continue
kk += 1
# Extract
f = member
hdu = fits.open(f)
# Parse name
fname = f.split('/')[-1]
mt = np.where(meta['SPEC_FILE'] == fname)[0]
if mk_test_file and (jj>=3):
continue
if len(mt) != 1:
pdb.set_trace()
raise ValueError("HD-LLS: No match to spectral file?!")
else:
print('loading {:s}'.format(fname))
full_idx[kk] = mt[0]
# npix
head = hdu[0].header
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
# Double check
if kk == 0:
assert hdu[1].name == 'ERROR'
assert hdu[2].name == 'WAVELENGTH'
# Write
spec = lsio.readspec(f) # Handles dummy pixels in ESI
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
#data['flux'][0][:npix] = hdu[0].data
#data['sig'][0][:npix] = hdu[1].data
#data['wave'][0][:npix] = hdu[2].data
# Meta
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if 'HIRES' in fname:
instrlist.append('HIRES')
telelist.append('Keck-I')
gratinglist.append('BOTH')
try:
Rlist.append(set_resolution(head))
except ValueError:
# A few by hand (pulled from Table 1)
if 'J073149' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2006-01-04', '%Y-%m-%d')
elif 'J081435' in fname:
Rlist.append(Rdicts['HIRES']['C1'])
tval = datetime.datetime.strptime('2006-12-26', '%Y-%m-%d') # 2008 too
elif 'J095309' in fname:
Rlist.append(Rdicts['HIRES']['C1'])
tval = datetime.datetime.strptime('2005-03-18', '%Y-%m-%d')
elif 'J113418' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2006-01-05', '%Y-%m-%d')
elif 'J135706' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2007-04-28', '%Y-%m-%d')
elif 'J155556.9' in fname:
Rlist.append(Rdicts['HIRES']['C5'])
tval = datetime.datetime.strptime('2005-04-15', '%Y-%m-%d')
elif 'J212329' in fname:
Rlist.append(Rdicts['HIRES']['E3'])
tval = datetime.datetime.strptime('2006-08-20', '%Y-%m-%d')
else:
pdb.set_trace()
else:
tval = datetime.datetime.strptime(head['DATE-OBS'], '%Y-%m-%d')
dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d'))
elif 'ESI' in fname:
instrlist.append('ESI')
telelist.append('Keck-II')
gratinglist.append('ECH')
try:
Rlist.append(set_resolution(head))
except ValueError:
print("Using R=6,000 for ESI")
Rlist.append(6000.)
try:
tval = datetime.datetime.strptime(head['DATE'], '%Y-%m-%d')
except KeyError:
if ('J223438.5' in fname) or ('J231543' in fname):
tval = datetime.datetime.strptime('2004-09-11', '%Y-%m-%d')
else:
pdb.set_trace()
dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d'))
elif 'MIKE' in fname: # APPROXIMATE
if 'MIKEr' in fname:
instrlist.append('MIKEr')
gratinglist.append('RED')
elif 'MIKEb' in fname:
instrlist.append('MIKEb')
gratinglist.append('BLUE')
else:
instrlist.append('MIKE')
gratinglist.append('BOTH')
telelist.append('Magellan')
sep = full_coord[mt[0]].separation(mike_coord)
imin = np.argmin(sep)
if sep[imin] > 1.*u.arcsec:
pdb.set_trace()
raise ValueError("Bad separation in MIKE")
# R and Date
Rlist.append(25000. / mike_meta['Slit'][imin])
tval = datetime.datetime.strptime(mike_meta['DATE-OBS'][imin], '%Y-%b-%d')
dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d'))
elif 'MAGE' in fname: # APPROXIMATE
instrlist.append('MagE')
if 'Clay' in head['TELESCOP']:
telelist.append('Magellan/Clay')
else:
telelist.append('Magellan/Baade')
gratinglist.append('N/A')
Rlist.append(set_resolution(head))
dateobslist.append(head['DATE-OBS'])
else: # MagE
raise ValueError("UH OH")
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[kk] = data
# Add columns
meta = meta[full_idx]
nmeta = len(meta)
meta.add_column(Column([2000.]*nmeta, name='EPOCH'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column([str(date) for date in dateobslist], name='DATE-OBS'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nmeta,dtype=int),name='GROUP_ID'))
meta.add_column(Column(gratinglist, name='GRATING'))
meta.add_column(Column(instrlist, name='INSTR'))
meta.add_column(Column(telelist, name='TELESCOPE'))
# v02
meta.rename_column('GRATING', 'DISPERSER')
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2015ApJS..221....2P',
bib='prochaska+15'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, boss_hdf=None, **kwargs):
""" Add BOSS data to the DB
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
boss_hdf : str, optional
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
if boss_hdf is not None:
print("Using previously generated {:s} dataset...".format(sname))
boss_hdf.copy(sname, hdf)
return
boss_grp = hdf.create_group(sname)
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 4650 # Just needs to be large enough
data = init_data(max_npix, include_co=True)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
wvminlist = []
wvmaxlist = []
speclist = []
npixlist = []
# Loop
maxpix = 0
for jj,row in enumerate(meta):
# Generate full file
full_file = get_specfil(row)
if full_file == 'None':
continue
# Read
spec = lsio.readspec(full_file)
# npix
npix = spec.npix
# Kludge for higest redshift systems
if npix < 10:
full_file = get_specfil(row, hiz=True)
try:
spec = lsio.readspec(full_file)
except:
print("Missing: {:s}".format(full_file))
npix = spec.npix
elif npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Parse name
fname = full_file.split('/')[-1]
# Fill
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# GZ Continuum -- packed in with spectrum, generated by my IDL script
try:
co = spec.co.value
except AttributeError:
co = np.zeros_like(spec.flux.value)
# KG Continuum
KG_file = get_specfil(row, KG=True)
if os.path.isfile(KG_file) and (npix>1): # Latter is for junk in GZ file. Needs fixing
hduKG = fits.open(KG_file)
KGtbl = hduKG[1].data
wvKG = 10.**KGtbl['LOGLAM']
try:
assert (wvKG[0]-spec.wavelength[0].value) < 1e-5
except:
pdb.set_trace()
gdpix = np.where(wvKG < (1+row['zem_GROUP'])*1200.)[0]
co[gdpix] = KGtbl['CONT'][gdpix]
data['co'][0][:npix] = co
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
meta.add_column(Column([2000.]*len(meta), name='EPOCH'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
pdb.set_trace()
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2015ApJS..219...12A',
bib='boss_qso_dr12'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False,
mk_test_file=False):
""" Append COS-Halos data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
chalos_grp = hdf.create_group(sname)
# Load up
# Checks
if sname != 'COS-Halos':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 160000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
# Loop
path = os.getenv('RAW_IGMSPEC')+'/COS-Halos/'
maxpix = 0
for jj,row in enumerate(meta):
# Generate full file
coord = ltu.radec_to_coord((row['RA_GROUP'],row['DEC_GROUP']))
if row['INSTR'].strip() == 'COS':
full_file = path+'/J{:s}{:s}_nbin3_coadd.fits.gz'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True)[0:4],
coord.dec.to_string(sep='',pad=True,alwayssign=True)[0:5])
else: # HIRES
full_file = path+'/HIRES/J{:s}{:s}_f.fits.gz'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True)[0:4], coord.dec.to_string(sep='',pad=True,alwayssign=True)[0:5])
# Extract
print("COS-Halos: Reading {:s}".format(full_file))
spec = lsio.readspec(full_file)
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec,dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2013ApJ...777...59T',
bib='tumlinson+13'),
dict(url='http://adsabs.harvard.edu/abs/2013ApJS..204...17W',
bib='werk+13')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf,
sname,
meta,
debug=False,
chk_meta_only=False,
boss_hdf=None,
**kwargs):
""" Add BOSS data to the DB
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
boss_hdf : str, optional
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
if boss_hdf is not None:
print("Using previously generated {:s} dataset...".format(sname))
boss_hdf.copy(sname, hdf)
return
boss_grp = hdf.create_group(sname)
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 4660 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
wvminlist = []
wvmaxlist = []
speclist = []
npixlist = []
# Loop
maxpix = 0
bad_spec = np.array([False] * len(meta))
for jj in range(len(meta)):
# Generate full file
full_file = get_specfil(jj, meta)
if full_file == 'None':
continue
# Read
try:
spec = lsio.readspec(full_file, masking='edges')
except:
print("Failed on full_file: {:s}, {:d}".format(full_file, jj))
bad_spec[jj] = True
continue
# npix
npix = spec.npix
if npix < 10:
print("Not enough pixels in file: {:s}, {:d}".format(
full_file, jj))
bad_spec[jj] = True
continue
'''
# Kludge for higest redshift systems
if npix < 10:
full_file = get_specfil(jj, meta, hiz=True)
try:
spec = lsio.readspec(full_file)
except:
print("Missing: {:s}".format(full_file))
npix = spec.npix
elif npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
'''
maxpix = max(npix, maxpix)
# Parse name
fname = full_file.split('/')[-1]
# Fill
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
'''
# GZ Continuum -- packed in with spectrum, generated by my IDL script
try:
co = spec.co.value
except AttributeError:
co = np.zeros_like(spec.flux.value)
# KG Continuum
KG_file = get_specfil(row, KG=True)
if os.path.isfile(KG_file) and (npix>1): # Latter is for junk in GZ file. Needs fixing
hduKG = fits.open(KG_file)
KGtbl = hduKG[1].data
wvKG = 10.**KGtbl['LOGLAM']
try:
assert (wvKG[0]-spec.wavelength[0].value) < 1e-5
except:
pdb.set_trace()
gdpix = np.where(wvKG < (1+row['zem_GROUP'])*1200.)[0]
co[gdpix] = KGtbl['CONT'][gdpix]
data['co'][0][:npix] = co
'''
# Meta
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
# Deal with null spec -- Should only be done once, saved and then ready to go
if np.any(bad_spec):
bad_meta = meta[bad_spec]
pdb.set_trace()
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
meta.add_column(Column([2000.] * len(meta), name='EPOCH'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
pdb.set_trace()
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2015ApJS..219...12A',
bib='boss_qso_dr12'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def grab_meta(test=False):
""" Grab BOSS meta Table
Returns
-------
boss_meta : Table
"""
# Paris et al.
qsos_dr14 = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS_DR14/DR14Q_v4_4.fits.gz')
# Add original ID to coordinate with spectra and remove it later
qsos_dr14['ORIG_ID'] = np.arange(len(qsos_dr14))
# Cut on BOSS (do not include DR7 files)
boss_dr14 = qsos_dr14[qsos_dr14['SPECTRO'] == 'BOSS']
# Cut out bad ones
bad_meta = Table.read(resource_filename('igmspec', 'ingest/files/bad_dr14.fits'))
keep = np.array([True]*len(boss_dr14))
for row in bad_meta:
bad_qso = np.where((boss_dr14['PLATE'] == row['PLATE']) & (
boss_dr14['FIBERID'] == row['FIBERID']))[0]
keep[bad_qso] = False
boss_dr14 = boss_dr14[keep]
if test:
boss_dr14 = boss_dr14[:100]
'''
# Cut out Plate 7840
plate7840 = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate7840_matched.ascii'), format='ascii')
not_7840 = boss_dr14['PLATE'] != 7840
for ii in range(len(plate7840)):
isplit = plate7840['sfile'][ii].split('-')
idx = np.where((boss_dr14['PLATE'] == 7840) & (boss_dr14['FIBERID'] == int(isplit[-1][0:4])))[0]
assert len(idx) == 1
not_7840[idx[0]] = True
boss_dr14 = boss_dr14[not_7840]
# Cut out others
for plate in [7879, 3678, 7513, 4869, 7306]:
ptbl = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate{:d}_matched.ascii'.format(plate)),
format='ascii.csv')
not_in = boss_dr14['PLATE'] != plate
for ii in range(len(ptbl)):
idx = np.where((boss_dr14['PLATE'] == ptbl['plate'][ii]) & (boss_dr14['FIBERID'] == ptbl['fiber'][ii]))[0]
not_in[idx[0]] = True
boss_dr14 = boss_dr14[not_in]
'''
# Proceed
boss_dr14['CAT'] = str('DR14')
# Cut on redshift?
#gd = np.any([boss_dr14['Z_PIPE'] > 0., boss_dr14['Z_PCA'] > 0.],axis=0) # CUTS Z_VI
#boss_dr12 = boss_dr12[gd]
#
#
nboss = len(boss_dr14)
# DATE-OBS
t = Time(list(boss_dr14['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
boss_dr14.add_column(Column(t.iso, name='DATE-OBS'))
# Add columns
boss_dr14.add_column(Column(['BOSS']*nboss, name='INSTR'))
boss_dr14.add_column(Column(['BOTH']*nboss, name='DISPERSER'))
#http://www.sdss.org/instruments/boss_spectrograph/
boss_dr14.add_column(Column([2100.]*nboss, name='R')) # RESOLUTION
boss_dr14.add_column(Column(['SDSS 2.5-M']*nboss, name='TELESCOPE'))
# Redshift logic
boss_dr14['zem_GROUP'] = boss_dr14['Z']
boss_dr14['sig_zem'] = boss_dr14['Z_PIPE_ERR']
boss_dr14['flag_zem'] = boss_dr14['SOURCE_Z']
'''
# Fix bad redshifts
bad_pca = boss_meta['Z_PCA'] < 0.
boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca]
boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca]
boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE')
'''
# Rename RA/DEC
boss_dr14.rename_column('RA', 'RA_GROUP')
boss_dr14.rename_column('DEC', 'DEC_GROUP')
# STYPE
boss_dr14['STYPE'] = str('QSO')
# Check
assert chk_meta(boss_dr14, chk_cat_only=True)
# Return
return boss_dr14
def grab_meta():
""" Grab XQ-100 meta Table
Returns
-------
"""
from specdb.specdb import IgmSpec
igmsp = IgmSpec()
#
xq100_table = Table.read(
os.getenv('RAW_IGMSPEC') + '/XQ-100/XQ100_v1_2.fits.gz')
nqso = len(xq100_table)
# ESO meta
eso_tbl = Table.read(os.getenv('RAW_IGMSPEC') +
'/XQ-100/metadata_eso_XQ100.csv',
format='ascii.csv')
ar_files = eso_tbl['ARCFILE'].data
# Spectral files
spec_files = glob.glob(os.getenv('RAW_IGMSPEC') + '/XQ-100/ADP.*')
# Dummy column
xq100_coords = SkyCoord(ra=xq100_table['RA'],
dec=xq100_table['DEC'],
unit='deg')
matches = []
sv_spec_files = []
sv_orig_files = []
sv_rescale_files = []
for spec_file in spec_files:
if 'ADP.2016-07-15T08:22:40.682.fits' in spec_file:
print("XQ-100: Skipping summary file")
continue
# ESO file
ssfile = spec_file[spec_file.rfind('/') + 1:-5]
eso_mt = np.where(ar_files == ssfile)[0]
try:
ofile = eso_tbl['ORIGFILE'][eso_mt][0]
except IndexError:
print("XQ-100: File {:s} not really in XQ100!".format(spec_file))
continue
if ('_1' in ofile) or ('_2' in ofile) or ('_3' in ofile) or ('_4'
in ofile):
print("XQ-100: Skipping additional file: {:s}".format(ofile))
continue
# Match
hdu = fits.open(spec_file)
head0 = hdu[0].header
if head0['DISPELEM'] == 'UVB,VIS,NIR':
print("XQ-100: Skipping merged spectrum file")
if 'rescale' not in ofile:
print('no rescale')
pdb.set_trace()
continue
try:
coord = SkyCoord(ra=head0['RA'], dec=head0['DEC'], unit='deg')
except KeyError:
pdb.set_trace()
sep = coord.separation(xq100_coords)
imt = np.argmin(sep)
if sep[imt] > 0.1 * u.arcsec:
pdb.set_trace()
raise ValueError("Bad offset")
# Save
matches.append(imt)
sv_spec_files.append(spec_file)
sv_orig_files.append(ofile)
# Finish up
xq100_meta = xq100_table[np.array(matches)]
nspec = len(xq100_meta)
# Add spec_files
xq100_meta['SPEC_FILE'] = sv_spec_files
xq100_meta['ORIG_FILE'] = sv_orig_files
# Add zem
xq100_meta['zem_GROUP'] = xq100_meta['Z_QSO']
xq100_meta['sig_zem'] = xq100_meta['ERR_ZQSO']
xq100_meta['flag_zem'] = [str('XQ-100')] * nspec
# Rename
xq100_meta.rename_column('RA', 'RA_GROUP')
xq100_meta.rename_column('DEC', 'DEC_GROUP')
# Match to Myers
myers = Table(igmsp.hdf['quasars'].value)
myers_coord = SkyCoord(ra=myers['RA'], dec=myers['DEC'], unit='deg')
xq100_coord = SkyCoord(ra=xq100_meta['RA_GROUP'],
dec=xq100_meta['DEC_GROUP'],
unit='deg')
idx, d2d, _ = match_coordinates_sky(xq100_coord,
myers_coord,
nthneighbor=1)
xq100_meta['RA_GROUP'] = myers_coord.ra.value[idx]
xq100_meta['DEC_GROUP'] = myers_coord.dec.value[idx]
# One bad one (Taking RA/DEC from Simbad)
bad_c = d2d.to('arcsec') > 20 * u.arcsec
xq100_meta['RA_GROUP'][bad_c] = 215.2823
xq100_meta['DEC_GROUP'][bad_c] = -6.73232
# DATE-OBS
meanmjd = []
for row in xq100_meta:
gdm = row['MJD_OBS'] > 0.
meanmjd.append(np.mean(row['MJD_OBS'][gdm]))
t = Time(meanmjd, format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
xq100_meta.add_column(Column(t.iso, name='DATE-OBS'))
#
xq100_meta.add_column(Column([2000.] * nspec, name='EPOCH'))
xq100_meta['STYPE'] = str('QSO')
# Sort
xq100_meta.sort('RA_GROUP')
# Check
assert chk_meta(xq100_meta, chk_cat_only=True)
#
return xq100_meta
def grab_meta():
""" Grab XQ-100 meta Table
Returns
-------
"""
from specdb.specdb import IgmSpec
igmsp = IgmSpec()
#
xq100_table = Table.read(os.getenv('RAW_IGMSPEC')+'/XQ-100/XQ100_v1_2.fits.gz')
nqso = len(xq100_table)
# ESO meta
eso_tbl = Table.read(os.getenv('RAW_IGMSPEC')+'/XQ-100/metadata_eso_XQ100.csv', format='ascii.csv')
ar_files = eso_tbl['ARCFILE'].data
# Spectral files
spec_files = glob.glob(os.getenv('RAW_IGMSPEC')+'/XQ-100/ADP.*')
# Dummy column
xq100_coords = SkyCoord(ra=xq100_table['RA'], dec=xq100_table['DEC'], unit='deg')
matches = []
sv_spec_files = []
sv_orig_files = []
sv_rescale_files = []
for spec_file in spec_files:
if 'ADP.2016-07-15T08:22:40.682.fits' in spec_file:
print("XQ-100: Skipping summary file")
continue
# ESO file
ssfile = spec_file[spec_file.rfind('/')+1:-5]
eso_mt = np.where(ar_files == ssfile)[0]
try:
ofile = eso_tbl['ORIGFILE'][eso_mt][0]
except IndexError:
print("XQ-100: File {:s} not really in XQ100!".format(spec_file))
continue
if ('_1' in ofile) or ('_2' in ofile) or ('_3' in ofile) or ('_4' in ofile):
print("XQ-100: Skipping additional file: {:s}".format(ofile))
continue
# Match
hdu = fits.open(spec_file)
head0 = hdu[0].header
if head0['DISPELEM'] == 'UVB,VIS,NIR':
print("XQ-100: Skipping merged spectrum file")
if 'rescale' not in ofile:
print('no rescale')
pdb.set_trace()
continue
try:
coord = SkyCoord(ra=head0['RA'], dec=head0['DEC'], unit='deg')
except KeyError:
pdb.set_trace()
sep = coord.separation(xq100_coords)
imt = np.argmin(sep)
if sep[imt] > 0.1*u.arcsec:
pdb.set_trace()
raise ValueError("Bad offset")
# Save
matches.append(imt)
sv_spec_files.append(spec_file)
sv_orig_files.append(ofile)
# Finish up
xq100_meta = xq100_table[np.array(matches)]
nspec = len(xq100_meta)
# Add spec_files
xq100_meta['SPEC_FILE'] = sv_spec_files
xq100_meta['ORIG_FILE'] = sv_orig_files
# Add zem
xq100_meta['zem_GROUP'] = xq100_meta['Z_QSO']
xq100_meta['sig_zem'] = xq100_meta['ERR_ZQSO']
xq100_meta['flag_zem'] = [str('XQ-100')]*nspec
# Rename
xq100_meta.rename_column('RA','RA_GROUP')
xq100_meta.rename_column('DEC','DEC_GROUP')
# Match to Myers
myers = Table(igmsp.hdf['quasars'].value)
myers_coord = SkyCoord(ra=myers['RA'], dec=myers['DEC'], unit='deg')
xq100_coord = SkyCoord(ra=xq100_meta['RA_GROUP'], dec=xq100_meta['DEC_GROUP'], unit='deg')
idx, d2d, _ = match_coordinates_sky(xq100_coord, myers_coord, nthneighbor=1)
xq100_meta['RA_GROUP'] = myers_coord.ra.value[idx]
xq100_meta['DEC_GROUP'] = myers_coord.dec.value[idx]
# One bad one (Taking RA/DEC from Simbad)
bad_c = d2d.to('arcsec') > 20*u.arcsec
xq100_meta['RA_GROUP'][bad_c] = 215.2823
xq100_meta['DEC_GROUP'][bad_c] = -6.73232
# DATE-OBS
meanmjd = []
for row in xq100_meta:
gdm = row['MJD_OBS'] > 0.
meanmjd.append(np.mean(row['MJD_OBS'][gdm]))
t = Time(meanmjd, format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD
xq100_meta.add_column(Column(t.iso, name='DATE-OBS'))
#
xq100_meta.add_column(Column([2000.]*nspec, name='EPOCH'))
xq100_meta['STYPE'] = str('QSO')
# Sort
xq100_meta.sort('RA_GROUP')
# Check
assert chk_meta(xq100_meta, chk_cat_only=True)
#
return xq100_meta
def hdf5_adddata(hdf,
sname,
meta,
debug=False,
chk_meta_only=False,
mk_test_file=False):
""" Append HSTQSO data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
mk_test_file : bool, optional
Generate the debug test file for Travis??
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
hstz2_grp = hdf.create_group(sname)
# Checks
if sname != 'HSTQSO':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 80000 # Just needs to be large enough
data = init_data(max_npix, include_co=False)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
wvminlist = []
wvmaxlist = []
npixlist = []
Rlist = []
# Loop
#path = os.getenv('RAW_IGMSPEC')+'/KODIAQ_data_20150421/'
path = os.getenv('RAW_IGMSPEC') + '/HSTQSO/'
maxpix = 0
for jj, row in enumerate(meta):
# Generate full file
full_file = path + row['SPEC_FILE'] + '.gz'
# Extract
print("HSTQSO: Reading {:s}".format(full_file))
hduf = fits.open(full_file)
head0 = hduf[0].header
spec = lsio.readspec(full_file, masking='edges')
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
if 'FOS-L' in fname:
Rlist.append(300.)
elif 'FOS-H' in fname:
Rlist.append(14000.)
elif 'STIS' in fname:
if row['DISPERSER'] == 'G230L':
Rlist.append(700.)
elif row['DISPERSER'] == 'G140L':
Rlist.append(1200.)
else:
raise ValueError("Bad STIS grating")
elif 'hsla' in fname: # COS
Rlist.append(18000.)
row['DATE-OBS'] = hduf[1].data['DATEOBS'][0][0]
else:
pdb.set_trace()
raise ValueError("Missing instrument!")
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
npixlist.append(npix)
if chk_meta_only:
continue
# Only way to set the dataset correctly
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column([2000.] * nspec, name='EPOCH'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2011ApJ...736...42R',
bib='ribuado11'),
dict(url='http://adsabs.harvard.edu/abs/2016ApJ...818..113N',
bib='neeleman16'),
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append XQ-100 data to the h5 file
Parameters
----------
hdf : hdf5 pointer
IDs : ndarray
int array of IGM_ID values in mainDB
sname : str
Survey name
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
xq100_grp = hdf.create_group(sname)
if len(meta) != 300:
pdb.set_trace()
# Checks
if sname != 'XQ-100':
raise IOError("Expecting XQ-100!!")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 20000 # Just needs to be large enough
data = init_data(max_npix, include_co=True)
# Init
spec_set = hdf[sname].create_dataset('spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
gratinglist = []
telelist = []
instrlist = []
# Loop
maxpix = 0
for jj, row in enumerate(meta):
#
print("XQ-100: Reading {:s}".format(row['SPEC_FILE']))
spec = lsio.readspec(row['SPEC_FILE'])
# Parse name
fname = row['SPEC_FILE'].split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError(
"Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix, maxpix)
# Continuum
# Some fiddling about
for key in ['wave', 'flux', 'sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.to('AA').value
data['co'][0][:npix] = spec.co.value
# Meta
head = spec.header
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
telelist.append(head['TELESCOP'])
instrlist.append(head['INSTRUME'])
gratinglist.append(head['DISPELEM'])
npixlist.append(npix)
if gratinglist[-1] == 'NIR': # From Lopez+16
Rlist.append(4350.)
elif gratinglist[-1] == 'VIS':
Rlist.append(7450.)
elif gratinglist[-1] == 'UVB':
Rlist.append(5300.)
else:
pdb.set_trace()
raise ValueError("UH OH")
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(gratinglist, name='DISPERSER'))
meta.add_column(Column(telelist, name='TELESCOPE'))
meta.add_column(Column(instrlist, name='INSTR'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [
dict(url='http://adsabs.harvard.edu/abs/2016arXiv160708776L',
bib='lopez+16')
]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
def grab_meta():
""" Grab HSTQSO meta Table
Returns
-------
"""
summ_file = os.getenv('RAW_IGMSPEC') + '/HSTQSO/hstqso.lst'
hstqso_meta = Table.read(summ_file, format='ascii')
spec_files = [str(ii) for ii in hstqso_meta['SPEC_FILE'].data]
nspec = len(hstqso_meta)
# RA/DEC
radec_file = os.getenv('RAW_IGMSPEC') + '/HSTQSO/all_qso_table.txt'
radec = Table.read(radec_file, format='ascii')
# DATE-OBS
date_files = glob.glob(os.getenv('RAW_IGMSPEC') + '/HSTQSO/date_obs*')
for ss, date_file in enumerate(date_files):
if ss == 0:
tab_date = Table.read(date_file, format='ascii')
else:
tab_date = vstack(
[tab_date, Table.read(date_file, format='ascii')])
# RA/DEC, DATE
hstqso_meta.add_column(Column(['2000-01-01'] * nspec, name='DATE-OBS'))
for jj, row in enumerate(hstqso_meta):
if row['INST'] == 'COS':
spec_files[jj] = str(row['QSO_ALT_NAME'] + '_hsla.fits')
continue
# DATE
spec = row['SPEC_FILE'].split('.')[0]
mt1 = np.where(tab_date['SPEC'] == spec)[0]
if len(mt1) == 0:
print("NO DATE MATCH for {:s}!".format(spec))
pdb.set_trace()
else:
mt1 = mt1[0] # TAKING THE FIRST ONE
joe_date = tab_date['DATE-OBS'][mt1].split('-')
hstqso_meta[jj]['DATE-OBS'] = '{:s}-{:02d}-{:02d}'.format(
joe_date[0], int(joe_date[1]), int(joe_date[2]))
if int(joe_date[1]) > 12:
pdb.set_trace()
# RA/DEC
if row['INST'] != 'FOS':
continue
mt = np.where(radec['File_ID'] == row['QSO_ALT_NAME'])[0]
if len(mt) == 0:
mt = np.where(radec['File_ID'] == row['QSO_NAME'])[0]
if len(mt) == 0:
print("NO RA/DEC MATCH!")
pdb.set_trace()
else:
mt = mt[0]
else:
mt = mt[0]
hstqso_meta[jj]['RA'] = radec['RA'][mt]
hstqso_meta[jj]['DEC'] = radec['DEC'][mt]
# Deal with Dups (mainly bad FOS coords)
coord = SkyCoord(ra=hstqso_meta['RA'], dec=hstqso_meta['DEC'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(coord, coord, nthneighbor=2)
dups = np.where(d2d < 2.0 * u.arcsec)[0] # Closest lens is ~2"
flag_dup = np.array([False] * len(hstqso_meta))
for idup in dups:
if flag_dup[idup]:
continue
dcoord = SkyCoord(ra=hstqso_meta['RA'][idup],
dec=hstqso_meta['DEC'][idup],
unit='deg')
sep = dcoord.separation(coord)
isep = np.where(sep < 2.0 * u.arcsec)[0]
# Search for COS first
icos = np.where(hstqso_meta['INST'][isep] == 'COS')[0]
if len(icos) > 0:
hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[icos[0]]]
hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[icos[0]]]
flag_dup[isep] = True
else: # STIS
istis = np.where(hstqso_meta['INST'][isep] == 'STIS')[0]
if len(istis) > 0:
hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[istis[0]]]
hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[istis[0]]]
flag_dup[isep] = True
else: # FOS only -- taking first value
hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[0]]
hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[0]]
# REPLACE
hstqso_meta.rename_column('SPEC_FILE', 'ORIG_SPEC_FILE')
hstqso_meta['SPEC_FILE'] = spec_files
# RENAME
hstqso_meta.rename_column('GRATE', 'DISPERSER')
hstqso_meta.rename_column('QSO_ZEM', 'zem_GROUP')
hstqso_meta.rename_column('INST', 'INSTR')
hstqso_meta['STYPE'] = str('QSO')
hstqso_meta.rename_column('RA', 'RA_GROUP')
hstqso_meta.rename_column('DEC', 'DEC_GROUP')
# ADD
hstqso_meta.add_column(Column(['HST'] * nspec, name='TELESCOPE'))
hstqso_meta['sig_zem'] = 0.
hstqso_meta['flag_zem'] = str('UNKWN')
# Check
assert chk_meta(hstqso_meta, chk_cat_only=True)
# Return
return hstqso_meta
def grab_meta():
""" Grab HSTQSO meta Table
Returns
-------
"""
summ_file = os.getenv('RAW_IGMSPEC')+'/HSTQSO/hstqso.lst'
hstqso_meta = Table.read(summ_file, format='ascii')
spec_files = [str(ii) for ii in hstqso_meta['SPEC_FILE'].data]
nspec = len(hstqso_meta)
# RA/DEC
radec_file = os.getenv('RAW_IGMSPEC')+'/HSTQSO/all_qso_table.txt'
radec = Table.read(radec_file, format='ascii')
# DATE-OBS
date_files = glob.glob(os.getenv('RAW_IGMSPEC')+'/HSTQSO/date_obs*')
for ss,date_file in enumerate(date_files):
if ss == 0:
tab_date = Table.read(date_file, format='ascii')
else:
tab_date = vstack([tab_date, Table.read(date_file, format='ascii')])
# RA/DEC, DATE
hstqso_meta.add_column(Column(['2000-01-01']*nspec, name='DATE-OBS'))
for jj,row in enumerate(hstqso_meta):
if row['INST'] == 'COS':
spec_files[jj] = str(row['QSO_ALT_NAME']+'_hsla.fits')
continue
# DATE
spec = row['SPEC_FILE'].split('.')[0]
mt1 = np.where(tab_date['SPEC'] == spec)[0]
if len(mt1) == 0:
print("NO DATE MATCH for {:s}!".format(spec))
pdb.set_trace()
else:
mt1 = mt1[0] # TAKING THE FIRST ONE
joe_date = tab_date['DATE-OBS'][mt1].split('-')
hstqso_meta[jj]['DATE-OBS'] = '{:s}-{:02d}-{:02d}'.format(joe_date[0], int(joe_date[1]), int(joe_date[2]))
if int(joe_date[1]) > 12:
pdb.set_trace()
# RA/DEC
if row['INST'] != 'FOS':
continue
mt = np.where(radec['File_ID'] == row['QSO_ALT_NAME'])[0]
if len(mt) == 0:
mt = np.where(radec['File_ID'] == row['QSO_NAME'])[0]
if len(mt) == 0:
print("NO RA/DEC MATCH!")
pdb.set_trace()
else:
mt = mt[0]
else:
mt = mt[0]
hstqso_meta[jj]['RA'] = radec['RA'][mt]
hstqso_meta[jj]['DEC'] = radec['DEC'][mt]
# Deal with Dups (mainly bad FOS coords)
coord = SkyCoord(ra=hstqso_meta['RA'], dec=hstqso_meta['DEC'], unit='deg')
idx, d2d, d3d = match_coordinates_sky(coord, coord, nthneighbor=2)
dups = np.where(d2d < 2.0*u.arcsec)[0] # Closest lens is ~2"
flag_dup = np.array([False]*len(hstqso_meta))
for idup in dups:
if flag_dup[idup]:
continue
dcoord = SkyCoord(ra=hstqso_meta['RA'][idup], dec=hstqso_meta['DEC'][idup], unit='deg')
sep = dcoord.separation(coord)
isep = np.where(sep < 2.0*u.arcsec)[0]
# Search for COS first
icos = np.where(hstqso_meta['INST'][isep] == 'COS')[0]
if len(icos) > 0:
hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[icos[0]]]
hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[icos[0]]]
flag_dup[isep] = True
else: # STIS
istis = np.where(hstqso_meta['INST'][isep] == 'STIS')[0]
if len(istis) > 0:
hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[istis[0]]]
hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[istis[0]]]
flag_dup[isep] = True
else: # FOS only -- taking first value
hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[0]]
hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[0]]
# REPLACE
hstqso_meta.rename_column('SPEC_FILE', 'ORIG_SPEC_FILE')
hstqso_meta['SPEC_FILE'] = spec_files
# RENAME
hstqso_meta.rename_column('GRATE', 'DISPERSER')
hstqso_meta.rename_column('QSO_ZEM', 'zem_GROUP')
hstqso_meta.rename_column('INST', 'INSTR')
hstqso_meta['STYPE'] = str('QSO')
hstqso_meta.rename_column('RA', 'RA_GROUP')
hstqso_meta.rename_column('DEC', 'DEC_GROUP')
# ADD
hstqso_meta.add_column(Column(['HST']*nspec, name='TELESCOPE'))
hstqso_meta['sig_zem'] = 0.
hstqso_meta['flag_zem'] = str('UNKWN')
# Check
assert chk_meta(hstqso_meta, chk_cat_only=True)
# Return
return hstqso_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False):
""" Append GGG data to the h5 file
Parameters
----------
hdf : hdf5 pointer
sname : str
Survey name
meta : Table
chk_meta_only : bool, optional
Only check meta file; will not write
Returns
-------
"""
# Add Survey
print("Adding {:s} survey to DB".format(sname))
ggg_grp = hdf.create_group(sname)
# Load up
if sname != 'GGG':
raise IOError("Not expecting this survey..")
# Build spectra (and parse for meta)
nspec = len(meta)
max_npix = 1600 # Just needs to be large enough
# Init
data = init_data(max_npix, include_co=False)
spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True,
maxshape=(None,), compression='gzip')
spec_set.resize((nspec,))
Rlist = []
wvminlist = []
wvmaxlist = []
npixlist = []
speclist = []
gratinglist = []
telelist = []
dateobslist = []
instrlist = []
# Loop
path = os.getenv('RAW_IGMSPEC')+'/GGG/'
maxpix = 0
for jj,row in enumerate(meta):
# Generate full file
if jj >= nspec//2:
full_file = path+row['name']+'_R400.fits.gz'
gratinglist.append('R400')
else:
full_file = path+row['name']+'_B600.fits.gz'
gratinglist.append('B600')
# Extract
print("GGG: Reading {:s}".format(full_file))
spec = lsio.readspec(full_file)
# Parse name
fname = full_file.split('/')[-1]
# npix
npix = spec.npix
if npix > max_npix:
raise ValueError("Not enough pixels in the data... ({:d})".format(npix))
else:
maxpix = max(npix,maxpix)
# Some fiddling about
for key in ['wave','flux','sig']:
data[key] = 0. # Important to init (for compression too)
data['flux'][0][:npix] = spec.flux.value
data['sig'][0][:npix] = spec.sig.value
data['wave'][0][:npix] = spec.wavelength.value
# Meta
head = spec.header
speclist.append(str(fname))
wvminlist.append(np.min(data['wave'][0][:npix]))
wvmaxlist.append(np.max(data['wave'][0][:npix]))
telelist.append(head['OBSERVAT'])
instrlist.append(head['INSTRUME'])
tval = Time(head['DATE'], format='isot', out_subfmt='date')
dateobslist.append(tval.iso)
npixlist.append(npix)
if 'R400' in fname:
Rlist.append(833.)
else:
Rlist.append(940.)
# Only way to set the dataset correctly
if chk_meta_only:
continue
spec_set[jj] = data
#
print("Max pix = {:d}".format(maxpix))
# Add columns
meta.add_column(Column(speclist, name='SPEC_FILE'))
meta.add_column(Column(gratinglist, name='GRATING'))
meta.add_column(Column(telelist, name='TELESCOPE'))
meta.add_column(Column(instrlist, name='INSTR'))
meta.add_column(Column(dateobslist, name='DATE-OBS'))
meta.add_column(Column(npixlist, name='NPIX'))
meta.add_column(Column(wvminlist, name='WV_MIN'))
meta.add_column(Column(wvmaxlist, name='WV_MAX'))
meta.add_column(Column(Rlist, name='R'))
meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID'))
# Add HDLLS meta to hdf5
if chk_meta(meta):
if chk_meta_only:
pdb.set_trace()
hdf[sname]['meta'] = meta
else:
raise ValueError("meta file failed")
# References
refs = [dict(url='http://adsabs.harvard.edu/abs/2014MNRAS.445.1745W',
bib='worseck+14')]
jrefs = ltu.jsonify(refs)
hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs)
#
return
