def grab_meta(): """ Grab GGG meta Table Returns ------- """ # This table has units in it! meta = Table.read(os.getenv('RAW_IGMSPEC')+'/GGG/GGG_catalog.fits.gz') nqso = len(meta) # Turn off RA/DEC units for key in ['RA', 'DEC']: meta[key].unit = None meta.rename_column('RA', 'RA_GROUP') meta.rename_column('DEC', 'DEC_GROUP') # # Add zem meta['zem_GROUP'] = meta['z_gmos'] meta['sig_zem'] = meta['zerror_gmos'] meta['flag_zem'] = [str('GGG')]*nqso meta.add_column(Column([2000.]*nqso, name='EPOCH')) # meta['STYPE'] = [str('QSO')]*nqso # Double up for the two gratings ggg_meta = vstack([meta,meta]) # Check assert chk_meta(ggg_meta, chk_cat_only=True) # Return return ggg_meta
def grab_meta(): """ Grab KODIAQ meta Table Returns ------- """ hstz2_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/HST_z2/hst_z2.ascii', format='ascii') nspec = len(hstz2_meta) # RA/DEC, DATE ra = [] dec = [] for row in hstz2_meta: # Fix DEC # Get RA/DEC coord = ltu.radec_to_coord((row['ra'],row['dec'])) ra.append(coord.ra.value) dec.append(coord.dec.value) hstz2_meta.add_column(Column(ra, name='RA_GROUP')) hstz2_meta.add_column(Column(dec, name='DEC_GROUP')) # z hstz2_meta.rename_column('zem', 'zem_GROUP') hstz2_meta['sig_zem'] = [0.]*nspec hstz2_meta['flag_zem'] = [str('SDSS_PIPE')]*nspec hstz2_meta['STYPE'] = [str('QSO')]*nspec # hstz2_meta.rename_column('obsdate','DATE-OBS') hstz2_meta.rename_column('tel','TELESCOPE') hstz2_meta.rename_column('inst','INSTR') hstz2_meta.rename_column('grating','DISPERSER') hstz2_meta.rename_column('resolution','R') # Check assert chk_meta(hstz2_meta, chk_cat_only=True) # Return return hstz2_meta
def grab_meta(): """ Grab GGG meta Table Returns ------- """ # This table has units in it! meta = Table.read(os.getenv('RAW_IGMSPEC')+'/ESI_z6/overview_data_igmspec.txt', format='ascii', delimiter='\t') nqso = len(meta) # Rename meta.rename_column('RAdeg', 'RA_GROUP') meta.rename_column('DECdeg', 'DEC_GROUP') meta.rename_column('z', 'zem_GROUP') meta.rename_column('instrument', 'INSTR') meta.rename_column('telescope', 'TELESCOPE') meta.rename_column('date', 'DATE-OBS') # # Add zem meta['sig_zem'] = 0. meta['flag_zem'] = str('ESI_z6') meta.add_column(Column([2000.]*nqso, name='EPOCH')) # meta['STYPE'] = str('QSO') meta['DISPERSER'] = str('ECH') # Check assert chk_meta(meta, chk_cat_only=True) # Return return meta
def grab_meta(): """ Grab KODIAQ meta Table Returns ------- """ hstz2_meta = Table.read(os.getenv('RAW_IGMSPEC') + '/HST_z2/hst_z2.ascii', format='ascii') nspec = len(hstz2_meta) # RA/DEC, DATE ra = [] dec = [] for row in hstz2_meta: # Fix DEC # Get RA/DEC coord = ltu.radec_to_coord((row['ra'], row['dec'])) ra.append(coord.ra.value) dec.append(coord.dec.value) hstz2_meta.add_column(Column(ra, name='RA_GROUP')) hstz2_meta.add_column(Column(dec, name='DEC_GROUP')) # z hstz2_meta.rename_column('zem', 'zem_GROUP') hstz2_meta['sig_zem'] = [0.] * nspec hstz2_meta['flag_zem'] = [str('SDSS_PIPE')] * nspec hstz2_meta['STYPE'] = [str('QSO')] * nspec # hstz2_meta.rename_column('obsdate', 'DATE-OBS') hstz2_meta.rename_column('tel', 'TELESCOPE') hstz2_meta.rename_column('inst', 'INSTR') hstz2_meta.rename_column('grating', 'DISPERSER') hstz2_meta.rename_column('resolution', 'R') # Check assert chk_meta(hstz2_meta, chk_cat_only=True) # Return return hstz2_meta
def grab_meta(): """ Grab GGG meta Table Returns ------- """ # This table has units in it! meta = Table.read(os.getenv('RAW_IGMSPEC') + '/GGG/GGG_catalog.fits.gz') nqso = len(meta) # Turn off RA/DEC units for key in ['RA', 'DEC']: meta[key].unit = None meta.rename_column('RA', 'RA_GROUP') meta.rename_column('DEC', 'DEC_GROUP') # # Add zem meta['zem_GROUP'] = meta['z_gmos'] meta['sig_zem'] = meta['zerror_gmos'] meta['flag_zem'] = [str('GGG')] * nqso meta.add_column(Column([2000.] * nqso, name='EPOCH')) # meta['STYPE'] = [str('QSO')] * nqso # Double up for the two gratings ggg_meta = vstack([meta, meta]) # Check assert chk_meta(ggg_meta, chk_cat_only=True) # Return return ggg_meta
def grab_meta(): """ Grab GGG meta Table Returns ------- """ # This table has units in it! meta = Table.read(os.getenv('RAW_IGMSPEC') + '/ESI_z6/overview_data_igmspec.txt', format='ascii', delimiter='\t') nqso = len(meta) # Rename meta.rename_column('RAdeg', 'RA_GROUP') meta.rename_column('DECdeg', 'DEC_GROUP') meta.rename_column('z', 'zem_GROUP') meta.rename_column('instrument', 'INSTR') meta.rename_column('telescope', 'TELESCOPE') meta.rename_column('date', 'DATE-OBS') # # Add zem meta['sig_zem'] = 0. meta['flag_zem'] = str('ESI_z6') meta.add_column(Column([2000.] * nqso, name='EPOCH')) # meta['STYPE'] = str('QSO') meta['DISPERSER'] = str('ECH') # Check assert chk_meta(meta, chk_cat_only=True) # Return return meta
def grab_meta(): """ Grab High-z ESI meta Table Returns ------- """ # esidla_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/HighzESIDLA/ascii_highz_rafelski.list', format='ascii') nspec = len(esidla_meta) # DATE datearr = [day.split('/') for day in list(esidla_meta['ObsDate'])] ndate = ['20'+str(day[2])+'-'+str(day[0])+'-'+str(day[1]) for day in datearr] t = Time(ndate, out_subfmt='date') # Fixes to YYYY-MM-DD esidla_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add zem esidla_meta['sig_zem'] = [0.]*nspec esidla_meta['flag_zem'] = [str('SDSS')]*nspec # esidla_meta.add_column(Column([2000.]*nspec, name='EPOCH')) esidla_meta.add_column(Column(['KeckII']*nspec, name='TELESCOPE')) esidla_meta.add_column(Column(['ESI']*nspec, name='INSTR')) esidla_meta.add_column(Column(['ECH']*nspec, name='DISPERSER')) # Rename esidla_meta.rename_column('RA', 'RA_GROUP') esidla_meta.rename_column('DEC', 'DEC_GROUP') esidla_meta.rename_column('zem', 'zem_GROUP') esidla_meta['STYPE'] = str('QSO') # Sort esidla_meta.sort('RA_GROUP') # Check assert chk_meta(esidla_meta, chk_cat_only=True) # return esidla_meta
def grab_meta(): """ Grab UVES Dall'Aglio meta table Returns ------- """ # uvesdall_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/UVES_Dall/uves_dall_summ.dat', format='ascii') nspec = len(uvesdall_meta) # DATE #datearr = [day.split('/') for day in list(uvesdall_meta['ObsDate'])] #ndate = ['20'+str(day[2])+'-'+str(day[0])+'-'+str(day[1]) for day in datearr] t = Time(uvesdall_meta['OBS-DATE'], out_subfmt='date') # Fixes to YYYY-MM-DD uvesdall_meta.add_column(Column(t.iso, name='DATE-OBS')) # RA/DEC coord = SkyCoord(ra=uvesdall_meta['RA'], dec=uvesdall_meta['DEC'], unit=(u.hour,u.deg)) rad = [icoord.ra.value for icoord in coord] decd = [icoord.dec.value for icoord in coord] uvesdall_meta.rename_column('RA', 'RA_STR') uvesdall_meta.rename_column('DEC', 'DEC_STR') uvesdall_meta['RA_GROUP'] = rad uvesdall_meta['DEC_GROUP'] = decd # Add zem igmsp = IgmSpec() ztbl = Table(igmsp.hdf['quasars'].value) zem, zsource = zem_from_radec(rad, decd, ztbl) badz = np.where(zem < 0.1)[0] for ibadz in badz: if uvesdall_meta['NAME'][ibadz] == 'HE2243-6031': zem[ibadz] = 3.005 zsource[ibadz] = 'FOP13' # Fumagalli+13 elif uvesdall_meta['NAME'][ibadz] == 'HE1341-1020': zem[ibadz] = 2.137 zsource[ibadz] = 'Dall08' # Dall'Aglio+08 elif uvesdall_meta['NAME'][ibadz] == 'Q0002-422': zem[ibadz] = 2.769 zsource[ibadz] = 'Dall08' # Dall'Aglio+08 elif uvesdall_meta['NAME'][ibadz] == 'PKS2000-330': zem[ibadz] = 3.786 zsource[ibadz] = 'Dall08' # Dall'Aglio+08 else: raise ValueError("Should not be here") uvesdall_meta['zem_GROUP'] = zem uvesdall_meta['sig_zem'] = [0.]*nspec uvesdall_meta['flag_zem'] = zsource # uvesdall_meta.add_column(Column([2000.]*nspec, name='EPOCH')) uvesdall_meta.add_column(Column(['VLT']*nspec, name='TELESCOPE')) uvesdall_meta.add_column(Column(['UVES']*nspec, name='INSTR')) uvesdall_meta.add_column(Column(['BOTH']*nspec, name='DISPERSER')) uvesdall_meta.add_column(Column([45000.]*nspec, name='R')) uvesdall_meta['STYPE'] = str('QSO') # Sort uvesdall_meta.sort('RA_GROUP') # Check assert chk_meta(uvesdall_meta, chk_cat_only=True) return uvesdall_meta
def grab_meta(): """ Grab COS-Dwarfs meta table Returns ------- """ from time import strptime cosdwarfs = COSDwarfs() cosdwarfs.load_sys(tfile=cosdwarfs.cdir + '/cos-dwarfs_systems.v1.1.tar.gz', chk_lowz=False) visit_file = os.getenv( 'RAW_IGMSPEC') + '/COS-Dwarfs/HST_Observing_Dates.dat' cd_visits = Table.read(visit_file, format='ascii') # Coord lst = [getattr(cgm_abs.igm_sys, 'coord') for cgm_abs in cosdwarfs.cgm_abs] ra = [coord.ra.value for coord in lst] dec = [coord.dec.value for coord in lst] # Short names shrt_names = [name.split('_')[0] for name in cosdwarfs.name] cdwarfs_meta = Table() cdwarfs_meta['RA'] = ra cdwarfs_meta['DEC'] = dec # RA/DEC, DATE datet = [] for kk, row in enumerate(cdwarfs_meta): # name = shrt_names[kk] mtv = np.where(cd_visits['QSO'] == name)[0] if len(mtv) != 1: pdb.set_trace() else: chv = cd_visits['Start_UT'][mtv].data[0] icmma = chv.find(',') datet.append('{:s}-{:02d}-{:02d}'.format( chv[icmma + 1:icmma + 5], strptime(chv[:3], '%b').tm_mon, int(chv[3:icmma]))) cdwarfs_meta.add_column(Column(datet, name='DATE-OBS')) # Others cdwarfs_meta.add_column( Column(['G130M/G160M'] * len(cdwarfs_meta), name='DISPERSER')) cdwarfs_meta.add_column(Column([20000.] * len(cdwarfs_meta), name='R')) cdwarfs_meta.add_column(Column([2000.] * len(cdwarfs_meta), name='EPOCH')) cdwarfs_meta['INSTR'] = 'COS' # Deals with padding cdwarfs_meta['TELESCOPE'] = 'HST' cdwarfs_meta['zem_GROUP'] = cosdwarfs.zem cdwarfs_meta['sig_zem'] = 0. # Need to add cdwarfs_meta['flag_zem'] = 'SDSS' cdwarfs_meta['STYPE'] = str('QSO') # Rename cdwarfs_meta.rename_column('RA', 'RA_GROUP') cdwarfs_meta.rename_column('DEC', 'DEC_GROUP') # Check assert chk_meta(cdwarfs_meta, chk_cat_only=True) # Done return cdwarfs_meta
def grab_meta(): """ Grab COS-Dwarfs meta table Returns ------- """ from time import strptime cosdwarfs = COSDwarfs() cosdwarfs.load_sys(tfile=cosdwarfs.cdir+'/cos-dwarfs_systems.v1.1.tar.gz', chk_lowz=False) visit_file = os.getenv('RAW_IGMSPEC')+'/COS-Dwarfs/HST_Observing_Dates.dat' cd_visits = Table.read(visit_file,format='ascii') # Coord lst = [getattr(cgm_abs.igm_sys, 'coord') for cgm_abs in cosdwarfs.cgm_abs] ra = [coord.ra.value for coord in lst] dec = [coord.dec.value for coord in lst] # Short names shrt_names = [name.split('_')[0] for name in cosdwarfs.name] cdwarfs_meta = Table() cdwarfs_meta['RA'] = ra cdwarfs_meta['DEC'] = dec # RA/DEC, DATE datet = [] for kk,row in enumerate(cdwarfs_meta): # name = shrt_names[kk] mtv = np.where(cd_visits['QSO'] == name)[0] if len(mtv) != 1: pdb.set_trace() else: chv = cd_visits['Start_UT'][mtv].data[0] icmma = chv.find(',') datet.append('{:s}-{:02d}-{:02d}'.format( chv[icmma+1:icmma+5], strptime(chv[:3],'%b').tm_mon, int(chv[3:icmma]))) cdwarfs_meta.add_column(Column(datet, name='DATE-OBS')) # Others cdwarfs_meta.add_column(Column(['G130M/G160M']*len(cdwarfs_meta), name='DISPERSER')) cdwarfs_meta.add_column(Column([20000.]*len(cdwarfs_meta), name='R')) cdwarfs_meta.add_column(Column([2000.]*len(cdwarfs_meta), name='EPOCH')) cdwarfs_meta['INSTR'] = 'COS' # Deals with padding cdwarfs_meta['TELESCOPE'] = 'HST' cdwarfs_meta['zem_GROUP'] = cosdwarfs.zem cdwarfs_meta['sig_zem'] = 0. # Need to add cdwarfs_meta['flag_zem'] = 'SDSS' cdwarfs_meta['STYPE'] = str('QSO') # Rename cdwarfs_meta.rename_column('RA', 'RA_GROUP') cdwarfs_meta.rename_column('DEC', 'DEC_GROUP') # Check assert chk_meta(cdwarfs_meta, chk_cat_only=True) # Done return cdwarfs_meta
def grab_meta(hdf, old=False): """ Grab SDSS meta Table Returns ------- meta """ from specdb.zem.utils import zem_from_radec #sdss_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/SDSS/SDSS_DR7_qso.fits.gz') sdss_meta = Table.read(os.getenv('RAW_IGMSPEC') + '/SDSS/dr7qso.fit.gz') nspec = len(sdss_meta) # DATE #t = Time(list(sdss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD t = Time(list(sdss_meta['SMJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD sdss_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add a few columns sdss_meta.add_column(Column([2000.] * nspec, name='EPOCH')) sdss_meta.add_column(Column([2000.] * nspec, name='R')) sdss_meta.add_column(Column(['SDSS'] * nspec, name='INSTR')) sdss_meta.add_column(Column(['BOTH'] * nspec, name='GRATING')) sdss_meta.add_column(Column(['SDSS 2.5-M'] * nspec, name='TELESCOPE')) # Rename if old: # Some of these were corrected by QPQ sdss_meta.rename_column('RAOBJ', 'RA') sdss_meta.rename_column('DECOBJ', 'DEC') sdss_meta.rename_column('Z_ERR', 'sig_zem') pdb.set_trace() else: sdss_meta.rename_column('z', 'zem_GROUP') sdss_meta['sig_zem'] = 0. sdss_meta['flag_zem'] = str(' ') # Fix zem zem, zsource = zem_from_radec(sdss_meta['RA'], sdss_meta['DEC'], hdf['quasars'].value, toler=1.0 * u.arcsec) gdz = zem > 0. sdss_meta['zem_GROUP'][gdz] = zem[gdz] sdss_meta['flag_zem'] = zsource sdss_meta['flag_zem'][~gdz] = str('SDSS-DR7') # Sort sdss_meta.sort('RA') # Rename sdss_meta.rename_column('RA', 'RA_GROUP') sdss_meta.rename_column('DEC', 'DEC_GROUP') # Add sdss_meta['STYPE'] = [str('QSO')] * nspec # Check assert chk_meta(sdss_meta, chk_cat_only=True) # Return return sdss_meta
def grab_meta(): """ Grab BOSS meta Table Returns ------- boss_meta : Table """ #http://www.sdss.org/dr12/algorithms/boss-dr12-quasar-catalog/ boss_dr12 = Table.read(os.getenv('RAW_IGMSPEC') + '/BOSS/DR12Q.fits.gz') boss_dr12['CAT'] = ['DR12Q'] * len(boss_dr12) gd = np.any([boss_dr12['Z_PIPE'] > 0., boss_dr12['Z_PCA'] > 0.], axis=0) # CUTS Z_VI boss_dr12 = boss_dr12[gd] # boss_sup = Table.read(os.getenv('RAW_IGMSPEC') + '/BOSS/DR12Q_sup.fits.gz') boss_sup['CAT'] = ['SUPGD'] * len(boss_sup) boss_supbad = Table.read( os.getenv('RAW_IGMSPEC') + '/BOSS/DR12Q_supbad.fits.gz') boss_supbad['CAT'] = ['SUPBD'] * len(boss_supbad) # Collate boss_meta = vstack([boss_dr12, boss_sup, boss_supbad], join_type='outer') # nboss = len(boss_meta) # DATE-OBS t = Time(list(boss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD boss_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add columns boss_meta.add_column(Column(['BOSS'] * nboss, name='INSTR')) boss_meta.add_column(Column(['BOTH'] * nboss, name='GRATING')) #http://www.sdss.org/instruments/boss_spectrograph/ boss_meta.add_column(Column([2100.] * nboss, name='R')) # RESOLUTION boss_meta.add_column(Column(['SDSS 2.5-M'] * nboss, name='TELESCOPE')) # Redshift logic boss_meta['zem_GROUP'] = boss_meta['Z_PCA'] boss_meta['sig_zem'] = boss_meta['ERR_ZPCA'] boss_meta['flag_zem'] = [str('BOSS_PCA ')] * nboss # Fix bad redshifts bad_pca = boss_meta['Z_PCA'] < 0. boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca] boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca] boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE') # Rename RA/DEC boss_meta.rename_column('RA', 'RA_GROUP') boss_meta.rename_column('DEC', 'DEC_GROUP') # STYPE boss_meta['STYPE'] = [str('QSO')] * len(boss_meta) # Check assert chk_meta(boss_meta, chk_cat_only=True) # Return return boss_meta
def grab_meta(): """ Grab BOSS meta Table Returns ------- boss_meta : Table """ #http://www.sdss.org/dr12/algorithms/boss-dr12-quasar-catalog/ boss_dr12 = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS/DR12Q.fits.gz') boss_dr12['CAT'] = ['DR12Q']*len(boss_dr12) gd = np.any([boss_dr12['Z_PIPE'] > 0., boss_dr12['Z_PCA'] > 0.],axis=0) # CUTS Z_VI boss_dr12 = boss_dr12[gd] # boss_sup = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS/DR12Q_sup.fits.gz') boss_sup['CAT'] = ['SUPGD']*len(boss_sup) boss_supbad = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS/DR12Q_supbad.fits.gz') boss_supbad['CAT'] = ['SUPBD']*len(boss_supbad) # Collate boss_meta = vstack([boss_dr12, boss_sup, boss_supbad], join_type='outer') # nboss = len(boss_meta) # DATE-OBS t = Time(list(boss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD boss_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add columns boss_meta.add_column(Column(['BOSS']*nboss, name='INSTR')) boss_meta.add_column(Column(['BOTH']*nboss, name='GRATING')) #http://www.sdss.org/instruments/boss_spectrograph/ boss_meta.add_column(Column([2100.]*nboss, name='R')) # RESOLUTION boss_meta.add_column(Column(['SDSS 2.5-M']*nboss, name='TELESCOPE')) # Redshift logic boss_meta['zem_GROUP'] = boss_meta['Z_PCA'] boss_meta['sig_zem'] = boss_meta['ERR_ZPCA'] boss_meta['flag_zem'] = [str('BOSS_PCA ')]*nboss # Fix bad redshifts bad_pca = boss_meta['Z_PCA'] < 0. boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca] boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca] boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE') # Rename RA/DEC boss_meta.rename_column('RA', 'RA_GROUP') boss_meta.rename_column('DEC', 'DEC_GROUP') # STYPE boss_meta['STYPE'] = [str('QSO')]*len(boss_meta) # Check assert chk_meta(boss_meta, chk_cat_only=True) # Return return boss_meta
def grab_meta(hdf, old=False): """ Grab SDSS meta Table Returns ------- meta """ from specdb.zem.utils import zem_from_radec #sdss_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/SDSS/SDSS_DR7_qso.fits.gz') sdss_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/SDSS/dr7qso.fit.gz') nspec = len(sdss_meta) # DATE #t = Time(list(sdss_meta['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD t = Time(list(sdss_meta['SMJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD sdss_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add a few columns sdss_meta.add_column(Column([2000.]*nspec, name='EPOCH')) sdss_meta.add_column(Column([2000.]*nspec, name='R')) sdss_meta.add_column(Column(['SDSS']*nspec, name='INSTR')) sdss_meta.add_column(Column(['BOTH']*nspec, name='GRATING')) sdss_meta.add_column(Column(['SDSS 2.5-M']*nspec, name='TELESCOPE')) # Rename if old: # Some of these were corrected by QPQ sdss_meta.rename_column('RAOBJ', 'RA') sdss_meta.rename_column('DECOBJ', 'DEC') sdss_meta.rename_column('Z_ERR', 'sig_zem') pdb.set_trace() else: sdss_meta.rename_column('z', 'zem_GROUP') sdss_meta['sig_zem'] = 0. sdss_meta['flag_zem'] = str(' ') # Fix zem zem, zsource = zem_from_radec(sdss_meta['RA'], sdss_meta['DEC'], hdf['quasars'].value, toler=1.0*u.arcsec) gdz = zem > 0. sdss_meta['zem_GROUP'][gdz] = zem[gdz] sdss_meta['flag_zem'] = zsource sdss_meta['flag_zem'][~gdz] = str('SDSS-DR7') # Sort sdss_meta.sort('RA') # Rename sdss_meta.rename_column('RA', 'RA_GROUP') sdss_meta.rename_column('DEC', 'DEC_GROUP') # Add sdss_meta['STYPE'] = [str('QSO')]*nspec # Check assert chk_meta(sdss_meta, chk_cat_only=True) # Return return sdss_meta
def grab_meta(): """ Generates the meta data needed for the IGMSpec build Returns ------- meta : Table """ # Read hdlls_meta = Table.read( os.getenv('RAW_IGMSPEC') + '/HD-LLS_DR1/HD-LLS_DR1.fits') # Rename hdlls_meta.rename_column('RA', 'RA_GROUP') hdlls_meta.rename_column('DEC', 'DEC_GROUP') hdlls_meta.rename_column('Z_QSO', 'zem_GROUP') # Kludgy Table judo hdlls_full = hdlls_meta[0:1] spec_files = [] # Loop me to bid the full survey catalog for kk, row in enumerate(hdlls_meta): for spec_file in row['SPEC_FILES']: if spec_file == 'NULL': continue # Add to full table hdlls_full.add_row(row) spec_files.append(spec_file) # Build hdlls_full = hdlls_full[1:] hdlls_full.remove_column('SPEC_FILES') hdlls_full.add_column( Column(np.array(spec_files).astype(str), name='SPEC_FILE')) # Cut on unique SPEC_FILEs uni, uni_idx = np.unique(np.array(spec_files).astype(str), return_index=True) # REMOVE ONE FILE (A DUPLICATE) BY HAND mt = uni != 'HD-LLS_J130756.73+042215.5_MIKE.fits' uni_idx = uni_idx[mt] # hdlls_full = hdlls_full[uni_idx] # nspec = len(hdlls_full) hdlls_full['sig_zem'] = [0.] * nspec hdlls_full['flag_zem'] = [str('UNKN')] * nspec hdlls_full['STYPE'] = [str('QSO')] * nspec assert chk_meta(hdlls_full, chk_cat_only=True) # Return return hdlls_full
def grab_meta(): """ Generates the meta data needed for the IGMSpec build Returns ------- meta : Table spec_files : list List of spec_file names """ # Load DLA from pyigm.surveys.dlasurvey import DLASurvey hdla100 = DLASurvey.neeleman13_tree() # Cut down to unique QSOs spec_files = [] names = [] ra = [] dec = [] coords = hdla100.coord cnt = 0 for coord in coords: # Load names.append('J{:s}{:s}'.format( coord.ra.to_string(unit=u.hour, sep='', pad=True, precision=2), coord.dec.to_string(sep='', pad=True, precision=1))) # RA/DEC ra.append(coord.ra.value) dec.append(coord.dec.value) # SPEC_FILE fname = hdla100._abs_sys[cnt]._datdict['hi res file'].split('/')[-1] spec_files.append(fname) cnt += 1 uni, uni_idx = np.unique(names, return_index=True) nqso = len(uni_idx) # meta = Table() meta['RA_GROUP'] = np.array(ra)[uni_idx] meta['DEC_GROUP'] = np.array(dec)[uni_idx] meta['zem_GROUP'] = hdla100.zem[uni_idx] meta['sig_zem'] = [0.] * nqso meta['flag_zem'] = [str('UNKN')] * nqso meta['STYPE'] = [str('QSO')] * nqso meta['SPEC_FILE'] = np.array(spec_files)[uni_idx] # Check assert chk_meta(meta, chk_cat_only=True) return meta
def grab_meta(): """ Generates the meta data needed for the IGMSpec build Returns ------- meta : Table """ # Read hdlls_meta = Table.read(os.getenv('RAW_IGMSPEC')+'/HD-LLS_DR1/HD-LLS_DR1.fits') # Rename hdlls_meta.rename_column('RA', 'RA_GROUP') hdlls_meta.rename_column('DEC', 'DEC_GROUP') hdlls_meta.rename_column('Z_QSO', 'zem_GROUP') # Kludgy Table judo hdlls_full = hdlls_meta[0:1] spec_files = [] # Loop me to bid the full survey catalog for kk,row in enumerate(hdlls_meta): for spec_file in row['SPEC_FILES']: if spec_file == 'NULL': continue # Add to full table hdlls_full.add_row(row) spec_files.append(spec_file) # Build hdlls_full = hdlls_full[1:] hdlls_full.remove_column('SPEC_FILES') hdlls_full.add_column(Column(np.array(spec_files).astype(str),name='SPEC_FILE')) # Cut on unique SPEC_FILEs uni, uni_idx = np.unique(np.array(spec_files).astype(str), return_index=True) # REMOVE ONE FILE (A DUPLICATE) BY HAND mt = uni != 'HD-LLS_J130756.73+042215.5_MIKE.fits' uni_idx = uni_idx[mt] # hdlls_full = hdlls_full[uni_idx] # nspec = len(hdlls_full) hdlls_full['sig_zem'] = [0.]*nspec hdlls_full['flag_zem'] = [str('UNKN')]*nspec hdlls_full['STYPE'] = [str('QSO')]*nspec assert chk_meta(hdlls_full, chk_cat_only=True) # Return return hdlls_full
def grab_meta(): """ Generates the meta data needed for the IGMSpec build Returns ------- meta : Table spec_files : list List of spec_file names """ # Load DLA from pyigm.surveys.dlasurvey import DLASurvey hdla100 = DLASurvey.neeleman13_tree() # Cut down to unique QSOs spec_files = [] names = [] ra = [] dec = [] coords = hdla100.coord cnt = 0 for coord in coords: # Load names.append('J{:s}{:s}'.format(coord.ra.to_string(unit=u.hour, sep='', pad=True, precision=2), coord.dec.to_string(sep='', pad=True, precision=1))) # RA/DEC ra.append(coord.ra.value) dec.append(coord.dec.value) # SPEC_FILE fname = hdla100._abs_sys[cnt]._datdict['hi res file'].split('/')[-1] spec_files.append(fname) cnt += 1 uni, uni_idx = np.unique(names, return_index=True) nqso = len(uni_idx) # meta = Table() meta['RA_GROUP'] = np.array(ra)[uni_idx] meta['DEC_GROUP'] = np.array(dec)[uni_idx] meta['zem_GROUP'] = hdla100.zem[uni_idx] meta['sig_zem'] = [0.]*nqso meta['flag_zem'] = [str('UNKN')]*nqso meta['STYPE'] = [str('QSO')]*nqso meta['SPEC_FILE'] = np.array(spec_files)[uni_idx] # Check assert chk_meta(meta, chk_cat_only=True) return meta
def grab_meta(): """ Grab KODIAQ meta Table Returns ------- """ kodiaq_file = igms_path + '/data/meta/KODIAQ_DR1_summary.ascii' kodiaq_meta = Table.read(kodiaq_file, format='ascii', comment='#') nspec = len(kodiaq_meta) # Verify DR1 for row in kodiaq_meta: assert row['kodrelease'] == 1 # RA/DEC, DATE ra = [] dec = [] dateobs = [] for row in kodiaq_meta: # Fix DEC # Get RA/DEC coord = ltu.radec_to_coord((row['sRA'], row['sDEC'])) ra.append(coord.ra.value) dec.append(coord.dec.value) # DATE dvals = row['pi_date'].split('_') tymd = str('{:s}-{:s}-{:02d}'.format(dvals[-1], dvals[1][0:3], int(dvals[2]))) tval = datetime.datetime.strptime(tymd, '%Y-%b-%d') dateobs.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) kodiaq_meta.add_column(Column(ra, name='RA_GROUP')) kodiaq_meta.add_column(Column(dec, name='DEC_GROUP')) kodiaq_meta.add_column(Column(dateobs, name='DATE-OBS')) # kodiaq_meta.add_column(Column(['HIRES'] * nspec, name='INSTR')) kodiaq_meta.add_column(Column(['Keck-I'] * nspec, name='TELESCOPE')) kodiaq_meta['STYPE'] = [str('QSO')] * nspec # z kodiaq_meta.rename_column('zem', 'zem_GROUP') kodiaq_meta['sig_zem'] = [0.] * nspec kodiaq_meta['flag_zem'] = [str('SIMBAD')] * nspec # assert chk_meta(kodiaq_meta, chk_cat_only=True) return kodiaq_meta
def grab_meta(): """ Grab HST/FUSE Cooksey meta Table Returns ------- """ hstc_file = os.getenv('RAW_IGMSPEC') + '/HST_Cooksey/HSTQSO_pre-SM4.lst' hstc_meta = Table.read(hstc_file, format='ascii') # Cutting those without proper header (for now) #badf = ['PKS2005-489lif1a.fits', 'NGC7469lif2a.fits', 'NGC7469sic2b.fits', # 'NGC7469lif2b.fits', 'NGC7469sic2a.fits', # 'AKN564lif1a.fits', 'AKN564lif1b.fits', 'AKN564lif2a.fits', 'AKN564lif2b.fits', # 'AKN564sic1a.fits', 'AKN564sic1b.fits', 'AKN564sic2a.fits', 'AKN564sic2b.fits'] badf = [] gdm = np.array([True] * len(hstc_meta)) for ibadf in badf: mt = np.where(hstc_meta['SPEC_FILE'] == ibadf)[0] gdm[mt] = False for jj, row in enumerate(hstc_meta): # Skip continua if '_c.fits' in row['SPEC_FILE']: gdm[jj] = False if '_E.fits' in row['SPEC_FILE']: gdm[jj] = False #if row['INSTR'] == 'GHRS': # gdm[jj] = False hstc_meta = hstc_meta[gdm] gdf = hstc_meta['INSTR'] == 'FUSE' #hstc_meta = hstc_meta[gdf] hstc_meta['TELESCOPE'] = 'FUSE' hstc_meta[~gdf]['TELESCOPE'] = 'HST' # hstc_meta.add_column(Column([2000.] * len(hstc_meta), name='EPOCH')) hstc_meta['sig_zem'] = 0. hstc_meta['flag_zem'] = str('UNKWN') hstc_meta['STYPE'] = str('QSO') # RENAME hstc_meta.rename_column('RA', 'RA_GROUP') hstc_meta.rename_column('DEC', 'DEC_GROUP') hstc_meta.rename_column('zem', 'zem_GROUP') # Check assert chk_meta(hstc_meta, chk_cat_only=True) return hstc_meta
def grab_meta(): """ Grab KODIAQ meta Table Returns ------- """ kodiaq_file = igms_path+'/data/meta/KODIAQ_DR1_summary.ascii' kodiaq_meta = Table.read(kodiaq_file, format='ascii', comment='#') nspec = len(kodiaq_meta) # Verify DR1 for row in kodiaq_meta: assert row['kodrelease'] == 1 # RA/DEC, DATE ra = [] dec = [] dateobs = [] for row in kodiaq_meta: # Fix DEC # Get RA/DEC coord = ltu.radec_to_coord((row['sRA'],row['sDEC'])) ra.append(coord.ra.value) dec.append(coord.dec.value) # DATE dvals = row['pi_date'].split('_') tymd = str('{:s}-{:s}-{:02d}'.format(dvals[-1],dvals[1][0:3],int(dvals[2]))) tval = datetime.datetime.strptime(tymd, '%Y-%b-%d') dateobs.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) kodiaq_meta.add_column(Column(ra, name='RA_GROUP')) kodiaq_meta.add_column(Column(dec, name='DEC_GROUP')) kodiaq_meta.add_column(Column(dateobs, name='DATE-OBS')) # kodiaq_meta.add_column(Column(['HIRES']*nspec, name='INSTR')) kodiaq_meta.add_column(Column(['Keck-I']*nspec, name='TELESCOPE')) kodiaq_meta['STYPE'] = [str('QSO')]*nspec # z kodiaq_meta.rename_column('zem', 'zem_GROUP') kodiaq_meta['sig_zem'] = [0.]*nspec kodiaq_meta['flag_zem'] = [str('SIMBAD')]*nspec # assert chk_meta(kodiaq_meta, chk_cat_only=True) return kodiaq_meta
def grab_meta(): """ Grab KODIAQ meta Table Returns ------- """ kodiaq_file = os.getenv('RAW_IGMSPEC')+'/KODIAQ2/KODIAQ_DR2_summary.ascii' kodiaq_meta = Table.read(kodiaq_file, format='ascii', comment='#') # Verify DR1 dr2 = kodiaq_meta['kodrelease'] == 2 kodiaq_meta = kodiaq_meta[dr2] # RA/DEC, DATE ra = [] dec = [] dateobs = [] for row in kodiaq_meta: # Fix DEC # Get RA/DEC coord = ltu.radec_to_coord((row['sRA'],row['sDEC'])) ra.append(coord.ra.value) dec.append(coord.dec.value) # DATE dvals = row['pi_date'].split('_') tymd = str('{:s}-{:s}-{:02d}'.format(dvals[-1],dvals[1][0:3],int(dvals[2]))) tval = datetime.datetime.strptime(tymd, '%Y-%b-%d') dateobs.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) kodiaq_meta.add_column(Column(ra, name='RA_GROUP')) kodiaq_meta.add_column(Column(dec, name='DEC_GROUP')) kodiaq_meta.add_column(Column(dateobs, name='DATE-OBS')) # kodiaq_meta['INSTR'] = 'HIRES' kodiaq_meta['TELESCOPE'] = 'Keck-I' kodiaq_meta['STYPE'] = str('QSO') # z kodiaq_meta.rename_column('zem', 'zem_GROUP') kodiaq_meta['sig_zem'] = 0. kodiaq_meta['flag_zem'] = str('SDSS-SIMBAD') # assert chk_meta(kodiaq_meta, chk_cat_only=True) return kodiaq_meta
def grab_meta(): """ Grab High-z ESI meta Table Returns ------- """ # esidla_meta = Table.read(os.getenv('RAW_IGMSPEC') + '/HighzESIDLA/ascii_highz_rafelski.list', format='ascii') nspec = len(esidla_meta) # DATE datearr = [day.split('/') for day in list(esidla_meta['ObsDate'])] ndate = [ '20' + str(day[2]) + '-' + str(day[0]) + '-' + str(day[1]) for day in datearr ] t = Time(ndate, out_subfmt='date') # Fixes to YYYY-MM-DD esidla_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add zem esidla_meta['sig_zem'] = [0.] * nspec esidla_meta['flag_zem'] = [str('SDSS')] * nspec # esidla_meta.add_column(Column([2000.] * nspec, name='EPOCH')) esidla_meta.add_column(Column(['KeckII'] * nspec, name='TELESCOPE')) esidla_meta.add_column(Column(['ESI'] * nspec, name='INSTR')) esidla_meta.add_column(Column(['ECH'] * nspec, name='DISPERSER')) # Rename esidla_meta.rename_column('RA', 'RA_GROUP') esidla_meta.rename_column('DEC', 'DEC_GROUP') esidla_meta.rename_column('zem', 'zem_GROUP') esidla_meta['STYPE'] = str('QSO') # Sort esidla_meta.sort('RA_GROUP') # Check assert chk_meta(esidla_meta, chk_cat_only=True) # return esidla_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append ESI data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) esidla_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'ESI_DLA': raise IOError("Expecting ESI_DLA!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 50000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): # specfile = os.getenv( 'RAW_IGMSPEC') + '/HighzESIDLA/{:s}a_xF.fits'.format(row['Name']) print("ESI_DLA: Reading {:s}".format(specfile)) spec = lsio.readspec(specfile) # Parse name fname = specfile.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Continuum # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value #data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(Rdicts['ESI'][head['SLMSKNAM']]) except KeyError: if row['Slit'] == 0.75: Rlist.append(Rdicts['ESI']['0.75_arcsec']) elif row['Slit'] == 0.5: Rlist.append(Rdicts['ESI']['0.50_arcsec']) else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2012ApJ...755...89R', bib='rafelski+12'), dict(url='http://adsabs.harvard.edu/abs/2014ApJ...782L..29R', bib='rafelski+14'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, hdla100_meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HDLA100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'HDLA100': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) #if mk_test_file: # hdla100_full = hdlls_full[0:3] max_npix = 192000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') nspec = len(hdla100_meta) spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] gratinglist = [] # Loop for jj, row in enumerate(hdla100_meta): kk = jj # Extract f = os.getenv('RAW_IGMSPEC') + '/HDLA100/' + row['SPEC_FILE'] spec = lsio.readspec(f) # Parse name fname = f.split('/')[-1] # npix head = spec.header npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(set_resolution(head)) except ValueError: raise ValueError("Header is required for {:s}".format(fname)) else: if '/' in head['DATE-OBS']: spl = head['DATE-OBS'].split('/') t = Time(datetime.datetime( int(spl[2]) + 1900, int(spl[1]), int(spl[0])), format='datetime') else: t = Time(head['DATE-OBS'], format='isot', out_subfmt='date') dateobslist.append(t.iso) # Grating try: gratinglist.append(head['XDISPERS']) except KeyError: try: yr = t.value.year except AttributeError: yr = int(t.value[0:4]) if yr <= 1997: gratinglist.append('RED') else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns nmeta = len(hdla100_meta) hdla100_meta.add_column(Column([2000.] * nmeta, name='EPOCH')) hdla100_meta.add_column(Column(npixlist, name='NPIX')) hdla100_meta.add_column( Column([str(date) for date in dateobslist], name='DATE-OBS')) hdla100_meta.add_column(Column(wvminlist, name='WV_MIN')) hdla100_meta.add_column(Column(wvmaxlist, name='WV_MAX')) hdla100_meta.add_column(Column(Rlist, name='R')) hdla100_meta.add_column( Column(np.arange(nmeta, dtype=int), name='GROUP_ID')) hdla100_meta.add_column(Column(gratinglist, name='DISPERSER')) hdla100_meta['INSTR'] = ['HIRES'] * nspec hdla100_meta['TELESCOPE'] = ['Keck-I'] * nspec #hdla100_meta.rename_column('Z_QSO', 'zem') # Add HDLLS meta to hdf5 if chk_meta(hdla100_meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = hdla100_meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2013ApJ...769...54N', bib='neeleman+13'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append KODIAQ data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) kodiaq_grp = hdf.create_group(sname) # Load up # Checks if sname != 'KODIAQ_DR2': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 60000 # Just needs to be large enough # Init data = init_data(max_npix, include_co=False) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) # Lists Rlist = [] wvminlist = [] wvmaxlist = [] gratinglist = [] npixlist = [] speclist = [] # Loop path = os.getenv('RAW_IGMSPEC')+'/KODIAQ2/Data/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file full_file = path+row['qso']+'/'+row['pi_date']+'/'+row['spec_prefix']+'_f.fits' # Extract print("KODIAQ: Reading {:s}".format(full_file)) hduf = fits.open(full_file) head = hduf[0].header spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) if 'XDISPERS' in head.keys(): if head['XDISPERS'].strip() == 'UV': gratinglist.append('BLUE') else: gratinglist.append('RED') else: # Original, earl data gratinglist.append('RED') npixlist.append(npix) try: Rlist.append(set_resolution(head)) except ValueError: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column([2000.]*nspec, name='EPOCH')) meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2017AJ....154..114O', bib='kodiaq') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def grab_meta(): """ Grab COS-Halos meta table Returns ------- """ from time import strptime from specdb.zem.utils import zem_from_radec from specdb.specdb import IgmSpec from specdb.defs import get_res_dicts Rdicts = get_res_dicts() igmsp = IgmSpec(db_file=os.getenv('SPECDB')+'/IGMspec_DB_v01.hdf5', skip_test=True) summ_file = os.getenv('RAW_IGMSPEC')+'/COS-Halos/cos_halos_obs.ascii' chalos_meta = Table.read(summ_file, format='ascii') # RA/DEC, DATE # Visits from this page: http://www.stsci.edu/cgi-bin/get-visit-status?id=11598&markupFormat=html visit_file = os.getenv('RAW_IGMSPEC')+'/COS-Halos/cos_halos_visits.ascii' ch_visits = Table.read(visit_file,format='ascii') ra = [] dec = [] datet = [] for row in chalos_meta: coord = ltu.radec_to_coord(row['QSO']) ra.append(coord.ra.value) dec.append(coord.dec.value) # visit = row['Visit'] mtv = np.where(ch_visits['Visit'] == visit)[0] if len(mtv) != 1: pdb.set_trace() else: chv = ch_visits['Start_UT'][mtv].data[0] icmma = chv.find(',') datet.append('{:s}-{:02d}-{:02d}'.format( chv[icmma+1:icmma+5], strptime(chv[:3],'%b').tm_mon, int(chv[3:icmma]))) chalos_meta.add_column(Column(ra, name='RA')) chalos_meta.add_column(Column(dec, name='DEC')) chalos_meta.add_column(Column(datet, name='DATE-OBS')) # Others chalos_meta.add_column(Column([' ']*len(chalos_meta), name='TELESCOPE')) # Padding chalos_meta.add_column(Column([' ']*len(chalos_meta), name='INSTR')) # Padding for HIRES chalos_meta.add_column(Column(['G130M/G160M']*len(chalos_meta), name='DISPERSER')) chalos_meta.add_column(Column([20000.]*len(chalos_meta), name='R')) chalos_meta.add_column(Column([2000.]*len(chalos_meta), name='EPOCH')) chalos_meta['INSTR'] = 'COS' # Deals with padding chalos_meta['TELESCOPE'] = 'HST' # Myers for zem zem, zsource = zem_from_radec(chalos_meta['RA'], chalos_meta['DEC'], Table(igmsp.hdf['quasars'].value)) badz = zem <= 0. if np.sum(badz) > 0: raise ValueError("Bad zem in COS-Halos") chalos_meta['zem'] = zem chalos_meta['sig_zem'] = 0. # Need to add chalos_meta['flag_zem'] = zsource # HIRES hires_files = glob.glob(os.getenv('RAW_IGMSPEC')+'/COS-Halos/HIRES/J*f.fits.gz') hires_tab = chalos_meta[0:0] subnm = np.array([row['QSO'][4:9] for row in chalos_meta]) signs = np.array([row['QSO'][14] for row in chalos_meta]) for ifile in hires_files: print(ifile) fname = ifile.split('/')[-1] mt = np.where((subnm == fname[0:5]) & (signs == fname[5]))[0] if len(mt) != 1: pdb.set_trace() # Add row hires_tab.add_row(chalos_meta[mt[0]]) hires_tab[-1]['INSTR'] = 'HIRES' hires_tab[-1]['TELESCOPE'] = 'Keck I' hires_tab[-1]['DISPERSER'] = 'Red' hires_tab[-1]['R'] = Rdicts['HIRES']['C1'] # Combine chalos_meta = vstack([chalos_meta, hires_tab]) chalos_meta['STYPE'] = str('QSO') # Rename chalos_meta.rename_column('RA', 'RA_GROUP') chalos_meta.rename_column('DEC', 'DEC_GROUP') chalos_meta.rename_column('zem', 'zem_GROUP') # Check assert chk_meta(chalos_meta, chk_cat_only=True) # Done return chalos_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append XQ-100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) xq100_grp = hdf.create_group(sname) if len(meta) != 300: pdb.set_trace() # Checks if sname != 'XQ-100': raise IOError("Expecting XQ-100!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 20000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] instrlist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): # print("XQ-100: Reading {:s}".format(row['SPEC_FILE'])) spec = lsio.readspec(row['SPEC_FILE']) # Parse name fname = row['SPEC_FILE'].split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Continuum # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) telelist.append(head['TELESCOP']) instrlist.append(head['INSTRUME']) gratinglist.append(head['DISPELEM']) npixlist.append(npix) if gratinglist[-1] == 'NIR': # From Lopez+16 Rlist.append(4350.) elif gratinglist[-1] == 'VIS': Rlist.append(7450.) elif gratinglist[-1] == 'UVB': Rlist.append(5300.) else: pdb.set_trace() raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(telelist, name='TELESCOPE')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2016arXiv160708776L', bib='lopez+16')] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, sdss_hdf=None, **kwargs): """ Add SDSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if sdss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) sdss_hdf.copy(sname, hdf) return sdss_grp = hdf.create_group(sname) # Load up # Checks if sname != 'SDSS_DR7': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) # Read Zhu continua, wave file cfile = os.getenv( 'RAW_IGMSPEC') + '/SDSS/ALLQSO_SPEC_106_continuum_nointerp.fits' zhu_conti = Table.read(cfile) wvfile = cfile.replace('continuum', 'wave') zhu_wave = Table.read(wvfile) # wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): full_file = get_specfil(row) if not os.path.isfile(full_file): full_file = get_specfil(row, dr7=True) # Extract #print("SDSS: Reading {:s}".format(full_file)) # Parse name fname = full_file.split('/')[-1] # Generate full file spec = lsio.readspec(full_file) # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig', 'co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Continuum mtc = (zhu_conti['PLATE'] == row['PLATE']) & (zhu_conti['FIBER'] == row['FIBER']) mtw = (zhu_wave['PLATE'] == row['PLATE']) & (zhu_wave['FIBER'] == row['FIBER']) if np.sum(mtc) == 1: imin = np.argmin( np.abs(zhu_wave['WAVE'][0][:, np.where(mtw)[1]] - spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][ imin:npix + imin, np.where(mtc)[1]].flatten() elif np.sum(mtc) > 1: print( "Multiple continua for plate={:d}, row={:d}. Taking the first" .format(row['PLATE'], row['FIBER'])) imin = np.argmin( np.abs(zhu_wave['WAVE'][0][:, np.where(mtw)[1][0]] - spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][ imin:npix + imin, np.where(mtc)[1][0]].flatten() elif np.sum(mtc) == 0: print("No SDSS continuum for plate={:d}, row={:d}".format( row['PLATE'], row['FIBER'])) #from xastropy.xutils import xdebug as xdb #xdb.set_trace() #xdb.xplot(data['wave'][0], data['flux'][0], data['co'][0]) # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2010AJ....139.2360S', bib='sdss_qso_dr7'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HST/FUSE data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ Rdicts = defs.get_res_dicts() # Add Survey print("Adding {:s} survey to DB".format(sname)) hstc_grp = hdf.create_group(sname) # Checks if sname != 'UVpSM4': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 40000 # Just needs to be large enough # Init data = init_data(max_npix, include_co=True) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] gratinglist = [] datelist = [] badf = [] badstis = [] badghrs = [] # Loop path = os.getenv('RAW_IGMSPEC') + '/HST_Cooksey/' maxpix = 0 for jj, row in enumerate(meta): # Generate full file full_file = path + '{:s}/{:s}/{:s}'.format(row['QSO'], row['INSTR'], row['SPEC_FILE']) # Extract if row['INSTR'] == 'FUSE': hext = 1 else: hext = 0 print("HST_Cooksey: Reading {:s}".format(full_file)) try: spec = lsio.readspec(full_file, head_exten=hext, masking='edges') except: # BAD HEADER hdu = fits.open(full_file) head1 = hdu[1].header hdu[1].verify('fix') tbl = Table(hdu[1].data) spec = lsio.readspec(tbl, masking='edges') spec.meta['headers'][spec.select] = head1 # Continuum cfile = full_file.replace('.fits', '_c.fits') if os.path.isfile(cfile): # Watch that mask! gdp = ~spec.data['flux'][spec.select].mask spec.data['co'][spec.select][gdp] = ( fits.open(cfile)[0].data)[gdp] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig', 'co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value if spec.co_is_set: try: data['co'][0][:npix] = spec.co.value except ValueError: pdb.set_trace() # Meta datet = None if row['INSTR'] == 'FUSE': if 'HISTORY' in spec.header.keys(): ncards = len(spec.header['HISTORY']) flg_H = True else: flg_H = False hdu = fits.open(full_file) head0 = hdu[0].header ncards = len(head0) # Is this a good one? if 'APER_ACT' in head0: pass else: # Need to fight harder for the header # Look for untrim untrim = full_file + '.untrim' if not os.path.isfile(untrim): pdb.set_trace() # Read hduu = fits.open(untrim) if 'PKS2005' in untrim: # One extra kludge.. head0 = hduu[1].header flg_H = True ncards = len(head0['HISTORY']) else: head0 = hduu[0].header ncards = len(head0) spec.meta['headers'][spec.select] = head0 # Read from history for ss in range(ncards): if flg_H: try: card = Header.fromstring(spec.header['HISTORY'][ss]) except: pdb.set_trace() try: ckey = list(card.keys())[0] except IndexError: continue else: card0 = card[0] else: ckey, card0 = list(spec.header.keys())[ss], spec.header[ss] # Parse if ckey == 'APERTURE': aper = card0 elif ckey == 'DETECTOR': det = card0 elif ckey == 'APER_ACT': # Extracted aperture ext_ap = card0 elif ckey == 'DATE': # Extracted aperture datet = card0 gratinglist.append(ext_ap + det) elif row['INSTR'] == 'STIS': try: datet = spec.header['DATE'] except KeyError: # handful of kludged coadds if 'HISTORY' not in spec.header.keys(): # Grab from the other extension, e.g. PKS0405 hdu = fits.open(full_file) head1 = hdu[1].header spec.meta['headers'][0] = head1 for ihist in spec.header['HISTORY']: if 'TDATEOBS' in ihist: idash = ihist.find('-') datet = ihist[idash - 4:idash + 6] # Grating from name i0 = full_file.rfind('_') i1 = full_file.rfind('.fits') gratinglist.append(full_file[i0 + 1:i1]) if datet is None: pdb.set_trace() else: gratinglist.append(spec.header['OPT_ELEM']) elif row['INSTR'] == 'GHRS': # Date try: tmp = spec.header['DATE-OBS'] except KeyError: # Pull header from parallel file iM = full_file.find('M_1') if iM <= 0: iM = full_file.find('L_1') ofile = full_file[:iM + 1] + '_F.fits' if not os.path.isfile(ofile): if 'NGC4151' in ofile: # Kludge ofile = ofile.replace('G160M', 'G160Mmd') elif 'PKS2155-304_GHRS_G140L' in ofile: # Kludge ofile = ofile.replace('G140L', 'G140Llo') elif 'PKS2155-304_GHRS_G160M' in ofile: # Kludge ofile = ofile.replace('G160M', 'G160Mmd') else: pdb.set_trace() hdu = fits.open(ofile) head0 = hdu[0].header spec.meta['headers'][spec.select] = head0 # Reformat prs = tmp.split('/') if prs[2][0] == '9': yr = '19' + prs[2] else: yr = '20' + prs[2] datet = yr + '-' + prs[1] + '-{:02d}'.format(int(prs[0])) # Grating gratinglist.append(spec.header['GRATING']) else: pdb.set_trace() if datet is None: try: datet = spec.header['DATE-OBS'] except KeyError: print("Missing Header for file: {:s}".format(full_file)) badf.append(full_file) datet = '9999-9-9' t = Time(datet, format='isot', out_subfmt='date') # Fixes to YYYY-MM-DD datelist.append(t.iso) try: Rlist.append(Rdicts[row['INSTR']][gratinglist[-1]]) except KeyError: print(gratinglist[-1]) pdb.set_trace() wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # if (len(badstis)) > 0: raise ValueError("Somehow have a bad STIS header..") if len(badf) > 0: print("We still have bad FUSE headers") pdb.set_trace() if len(badghrs) > 0: print("We still have bad GHRS headers") pdb.set_trace() print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(datelist, name='DATE-OBS')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2010ApJ...708..868C', bib='cooksey10') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, sdss_hdf=None, **kwargs): """ Add SDSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if sdss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) sdss_hdf.copy(sname, hdf) return sdss_grp = hdf.create_group(sname) # Load up # Checks if sname != 'SDSS_DR7': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) # Read Zhu continua, wave file cfile = os.getenv('RAW_IGMSPEC')+'/SDSS/ALLQSO_SPEC_106_continuum_nointerp.fits' zhu_conti = Table.read(cfile) wvfile = cfile.replace('continuum','wave') zhu_wave = Table.read(wvfile) # wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): full_file = get_specfil(row) if not os.path.isfile(full_file): full_file = get_specfil(row, dr7=True) # Extract #print("SDSS: Reading {:s}".format(full_file)) # Parse name fname = full_file.split('/')[-1] # Generate full file spec = lsio.readspec(full_file) # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig','co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Continuum mtc = (zhu_conti['PLATE'] == row['PLATE']) & (zhu_conti['FIBER']==row['FIBER']) mtw = (zhu_wave['PLATE'] == row['PLATE']) & (zhu_wave['FIBER']==row['FIBER']) if np.sum(mtc) == 1: imin = np.argmin(np.abs(zhu_wave['WAVE'][0][:,np.where(mtw)[1]]-spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][imin:npix+imin,np.where(mtc)[1]].flatten() elif np.sum(mtc) > 1: print("Multiple continua for plate={:d}, row={:d}. Taking the first".format(row['PLATE'], row['FIBER'])) imin = np.argmin(np.abs(zhu_wave['WAVE'][0][:,np.where(mtw)[1][0]]-spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][imin:npix+imin,np.where(mtc)[1][0]].flatten() elif np.sum(mtc) == 0: print("No SDSS continuum for plate={:d}, row={:d}".format(row['PLATE'], row['FIBER'])) #from xastropy.xutils import xdebug as xdb #xdb.set_trace() #xdb.xplot(data['wave'][0], data['flux'][0], data['co'][0]) # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2010AJ....139.2360S', bib='sdss_qso_dr7'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def ingest_spectra(hdf, sname, meta, max_npix=10000, chk_meta_only=False, debug=False, xspec=None, scale=1., refs=None, verbose=False, badf=None, set_idkey=None, grab_conti=False, **kwargs): """ Ingest the spectra Parameters ---------- hdf : hdf5 pointer sname : str Name of dataset meta : Table max_npix : int, optional Maximum length of the spectra chk_meta_only : bool, optional Only check meta file; will not write refs : list, optional list of dicts with reference info badf : list, optional List of bad spectra [use only if you know what you are doing!] grab_conti : bool, optional Grab continua. They should exist but do not have to set_idkey : str, optional Only required if you are not performing the full script xspec : list or XSpectrum1D object, optional Take spectra from this object instead of reading from files scale : float, optional Scale the spectra by this factor. Useful for fluxing **kwargs : optional Passed to readspec() Returns ------- """ if set_idkey is not None: set_sv_idkey(set_idkey) # Add Survey print("Adding {:s} group to DB".format(sname)) grp = hdf.create_group(sname) # Spectra nspec = len(meta) dtypes=[(str('wave'), 'float64', (max_npix)), (str('flux'), 'float32', (max_npix)), (str('sig'), 'float32', (max_npix))] dkeys = ['wave','flux','sig'] if grab_conti: dtypes += [(str('co'), 'float32', (max_npix))] dkeys += ['co'] data = np.ma.empty((1,), dtype=dtypes) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] # Loop for jj,member in enumerate(meta['SPEC_FILE']): # Extract f = member # Parse name fname = f.split('/')[-1] if verbose: print(fname) # Read if xspec is None: if badf is not None: for ibadf in badf: if ibadf in f: spec = dumb_spec() else: try: spec = lsio.readspec(f, **kwargs) except ValueError: # Probably a continuum problem pdb.set_trace() else: spec = lsio.readspec(f, **kwargs) else: spec = xspec[jj] if debug: pdb.set_trace() # Meta #head = spec.header if spec.masking == 'none': npix = spec.wavelength.value.size else: npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d} vs {:d})".format( npix, max_npix)) # Some fiddling about for key in dkeys: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value * scale data['sig'][0][:npix] = spec.sig.value * scale data['wave'][0][:npix] = spec.wavelength.value if grab_conti: if spec.co_is_set: data['co'][0][:npix] = spec.co.value # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Set spec_set[jj] = data # Add columns meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) # Add HDLLS meta to hdf5 if spbu.chk_meta(meta):#, skip_igmid=True): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References if refs is not None: jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, hdla100_meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HDLA100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'HDLA100': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) #if mk_test_file: # hdla100_full = hdlls_full[0:3] max_npix = 192000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') nspec = len(hdla100_meta) spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] gratinglist = [] # Loop for jj,row in enumerate(hdla100_meta): kk = jj # Extract f = os.getenv('RAW_IGMSPEC')+'/HDLA100/'+row['SPEC_FILE'] spec = lsio.readspec(f) # Parse name fname = f.split('/')[-1] # npix head = spec.header npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(set_resolution(head)) except ValueError: raise ValueError("Header is required for {:s}".format(fname)) else: if '/' in head['DATE-OBS']: spl = head['DATE-OBS'].split('/') t = Time(datetime.datetime(int(spl[2])+1900, int(spl[1]), int(spl[0])), format='datetime') else: t = Time(head['DATE-OBS'], format='isot', out_subfmt='date') dateobslist.append(t.iso) # Grating try: gratinglist.append(head['XDISPERS']) except KeyError: try: yr = t.value.year except AttributeError: yr = int(t.value[0:4]) if yr <= 1997: gratinglist.append('RED') else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns nmeta = len(hdla100_meta) hdla100_meta.add_column(Column([2000.]*nmeta, name='EPOCH')) hdla100_meta.add_column(Column(npixlist, name='NPIX')) hdla100_meta.add_column(Column([str(date) for date in dateobslist], name='DATE-OBS')) hdla100_meta.add_column(Column(wvminlist, name='WV_MIN')) hdla100_meta.add_column(Column(wvmaxlist, name='WV_MAX')) hdla100_meta.add_column(Column(Rlist, name='R')) hdla100_meta.add_column(Column(np.arange(nmeta,dtype=int),name='GROUP_ID')) hdla100_meta.add_column(Column(gratinglist, name='DISPERSER')) hdla100_meta['INSTR'] = ['HIRES']*nspec hdla100_meta['TELESCOPE'] = ['Keck-I']*nspec #hdla100_meta.rename_column('Z_QSO', 'zem') # Add HDLLS meta to hdf5 if chk_meta(hdla100_meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = hdla100_meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2013ApJ...769...54N', bib='neeleman+13'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Add 2QZ data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) tqz_grp = hdf.create_group(sname) # Checks if sname != '2QZ': raise IOError("Not expecting this survey..") # Add zem # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): full_file = get_specfil(row) # Parse name fname = full_file.split('/')[-1] # Read hdu = fits.open(full_file) head0 = hdu[0].header wave = lsio.setwave(head0) flux = hdu[0].data var = hdu[2].data sig = np.zeros_like(flux) gd = var > 0. if np.sum(gd) == 0: print("{:s} has a bad var array. Not including".format(fname)) pdb.set_trace() continue sig[gd] = np.sqrt(var[gd]) # npix spec = XSpectrum1D.from_tuple((wave, flux, sig)) npix = spec.npix spec.meta['headers'][0] = head0 if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2004MNRAS.349.1397C', bib='2QZ') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HSTQSO data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) hstz2_grp = hdf.create_group(sname) # Checks if sname != 'HSTQSO': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 80000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] Rlist = [] # Loop #path = os.getenv('RAW_IGMSPEC')+'/KODIAQ_data_20150421/' path = os.getenv('RAW_IGMSPEC')+'/HSTQSO/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file full_file = path+row['SPEC_FILE']+'.gz' # Extract print("HSTQSO: Reading {:s}".format(full_file)) hduf = fits.open(full_file) head0 = hduf[0].header spec = lsio.readspec(full_file, masking='edges') # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta if 'FOS-L' in fname: Rlist.append(300.) elif 'FOS-H' in fname: Rlist.append(14000.) elif 'STIS' in fname: if row['DISPERSER'] == 'G230L': Rlist.append(700.) elif row['DISPERSER'] == 'G140L': Rlist.append(1200.) else: raise ValueError("Bad STIS grating") elif 'hsla' in fname: # COS Rlist.append(18000.) row['DATE-OBS'] = hduf[1].data['DATEOBS'][0][0] else: pdb.set_trace() raise ValueError("Missing instrument!") wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column([2000.]*nspec, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2011ApJ...736...42R', bib='ribuado11'), dict(url='http://adsabs.harvard.edu/abs/2016ApJ...818..113N', bib='neeleman16'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def grab_meta(): """ Grab GGG meta Table Catalog -- http://www.2dfquasar.org/Spec_Cat/catalogue.html Returns ------- meta """ catfil = os.getenv('RAW_IGMSPEC')+'/2QZ/2QZ393524355693.out' tdf_meta = Table.read(catfil, format='ascii') # Rename columns clms = ['Name', 'RAh00', 'RAm00', 'RAs00', 'DECd00', 'DECm00', 'DECs00', 'ID','cat_name', 'Sector', 'RAh50', 'RAm50', 'RAs50', 'DECd50', 'DECm50', 'DECs50', 'UKST', 'XAPM','YAPM','RA50','DEC50','bj','u-b','b-r','Nobs', 'z1','q1','ID1','date1','fld1','fiber1','SN1', 'z2','q2','ID2','date2','fld2','fiber2','SN2', 'zprev','rflux','Xray','EBV','comm1','comm2'] for ii in range(1,46): tdf_meta.rename_column('col{:d}'.format(ii), clms[ii-1]) # Cut down to QSOs and take only 1 spectrum ispec = [] zspec = [] datespec = [] for row in tdf_meta: if 'QSO' in row['ID1']: ispec.append(1) zspec.append(row['z1']) sdate = str(row['date1']) datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4],sdate[4:6], sdate[6:8])) elif 'QSO' in row['ID2']: ispec.append(2) zspec.append(row['z2']) sdate = str(row['date2']) datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4],sdate[4:6], sdate[6:8])) else: ispec.append(0) zspec.append(-1.) datespec.append('') tdf_meta['ispec'] = ispec tdf_meta['zem_GROUP'] = zspec tdf_meta['DATE'] = datespec cut = tdf_meta['ispec'] > 0 tdf_meta = tdf_meta[cut] nspec = len(tdf_meta) # DATE t = Time(list(tdf_meta['DATE'].data), format='iso', out_subfmt='date') # Fixes to YYYY-MM-DD tdf_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add a few columns tdf_meta.add_column(Column([2000.]*nspec, name='EPOCH')) # Resolution # 2df 8.6A FWHM # R=580 at 5000A # 6df?? tdf_meta.add_column(Column([580.]*nspec, name='R')) # tdf_meta.add_column(Column([str('2dF')]*nspec, name='INSTR')) tdf_meta.add_column(Column([str('300B')]*nspec, name='DISPERSER')) tdf_meta.add_column(Column([str('UKST')]*nspec, name='TELESCOPE')) # Rename rad = (tdf_meta['RAh00']*3600 + tdf_meta['RAm00']*60 + tdf_meta['RAs00'])*360./86400. decd = np.abs(tdf_meta['DECd00']) + tdf_meta['DECm00']/60 + tdf_meta['DECs00']/3600. # Yup the following is necessary neg = [False]*len(rad) for jj,row in enumerate(tdf_meta): if '-' in row['Name']: neg[jj] = True #if '-00' in row['Name']: # print('jj={:d}'.format(jj)) neg = np.array(neg) decd[neg] = -1.*decd[neg] tdf_meta['RA_GROUP'] = rad tdf_meta['DEC_GROUP'] = decd tdf_meta['sig_zem'] = [0.]*nspec tdf_meta['flag_zem'] = str('2QZ') tdf_meta['STYPE'] = str('QSO') # Require a spectrum exist gdm = np.array([True]*len(tdf_meta)) for jj,row in enumerate(tdf_meta): full_file = get_specfil(row) if not os.path.isfile(full_file): print("{:s} has no spectrum. Not including".format(full_file)) gdm[jj] = False continue tdf_meta = tdf_meta[gdm] # Sort tdf_meta.sort('RA_GROUP') # Check assert chk_meta(tdf_meta, chk_cat_only=True) # Return return tdf_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append UVES_Dall data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) uvesdall_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'UVES_Dall': raise IOError("Expecting UVES_Dall!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 150000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist, wvmaxlist, npixlist, speclist = [], [], [], [] # Loop maxpix = 0 for jj,row in enumerate(meta): # Read specfile = os.getenv('RAW_IGMSPEC')+'/UVES_Dall/{:s}_flux.dat'.format(row['NAME']) print("UVES_Dall: Reading {:s}".format(specfile)) spec = Table.read(specfile,format='ascii.fast_no_header',guess=False)#, data_start=1) # Parse name fname = specfile.split('/')[-1] # npix npix = len(spec['col1']) if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Continuum # Some fiddling about for key in ['wave','flux','sig','co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec['col2'] data['sig'][0][:npix] = spec['col3'] data['wave'][0][:npix] = spec['col1'] data['co'][0][:npix] = spec['col4'] # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2008A%26A...491..465D', bib='dallaglio+08'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append COS-Halos data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) chalos_grp = hdf.create_group(sname) # Load up # Checks if sname != 'COS-Halos': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 160000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop path = os.getenv('RAW_IGMSPEC') + '/COS-Halos/' maxpix = 0 for jj, row in enumerate(meta): # Generate full file coord = ltu.radec_to_coord((row['RA_GROUP'], row['DEC_GROUP'])) if row['INSTR'].strip() == 'COS': full_file = path + '/J{:s}{:s}_nbin3_coadd.fits.gz'.format( coord.ra.to_string(unit=u.hour, sep='', pad=True)[0:4], coord.dec.to_string(sep='', pad=True, alwayssign=True)[0:5]) else: # HIRES full_file = path + '/HIRES/J{:s}{:s}_f.fits.gz'.format( coord.ra.to_string(unit=u.hour, sep='', pad=True)[0:4], coord.dec.to_string(sep='', pad=True, alwayssign=True)[0:5]) # Extract print("COS-Halos: Reading {:s}".format(full_file)) spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2013ApJ...777...59T', bib='tumlinson+13'), dict(url='http://adsabs.harvard.edu/abs/2013ApJS..204...17W', bib='werk+13') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append GGG data to the h5 file Parameters ---------- hdf : hdf5 pointer sname : str Survey name meta : Table chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) _ = hdf.create_group(sname) # Load up if sname != 'ESI_z6': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 30000 # Just needs to be large enough # Init data = init_data(max_npix, include_co=True) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] dateobslist = [] instrlist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): # Generate full file full_file, spec = read_spec(row) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value data['co'][0][:npix] = spec.co.value # Meta #head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) #telelist.append(head['OBSERVAT']) #instrlist.append(head['INSTRUME']) #tval = Time(head['DATE'], format='isot', out_subfmt='date') #dateobslist.append(tval.iso) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) #meta.add_column(Column(telelist, name='TELESCOPE')) #meta.add_column(Column(instrlist, name='INSTR')) #meta.add_column(Column(dateobslist, name='DATE-OBS')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) #meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2014MNRAS.445.1745W', bib='worseck+14') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def grab_meta(): """ Grab COS-Halos meta table Returns ------- """ from time import strptime from specdb.zem.utils import zem_from_radec from specdb.specdb import IgmSpec from specdb.defs import get_res_dicts Rdicts = get_res_dicts() igmsp = IgmSpec(db_file=os.getenv('SPECDB') + '/IGMspec_DB_v01.hdf5', skip_test=True) summ_file = os.getenv('RAW_IGMSPEC') + '/COS-Halos/cos_halos_obs.ascii' chalos_meta = Table.read(summ_file, format='ascii') # RA/DEC, DATE # Visits from this page: http://www.stsci.edu/cgi-bin/get-visit-status?id=11598&markupFormat=html visit_file = os.getenv('RAW_IGMSPEC') + '/COS-Halos/cos_halos_visits.ascii' ch_visits = Table.read(visit_file, format='ascii') ra = [] dec = [] datet = [] for row in chalos_meta: coord = ltu.radec_to_coord(row['QSO']) ra.append(coord.ra.value) dec.append(coord.dec.value) # visit = row['Visit'] mtv = np.where(ch_visits['Visit'] == visit)[0] if len(mtv) != 1: pdb.set_trace() else: chv = ch_visits['Start_UT'][mtv].data[0] icmma = chv.find(',') datet.append('{:s}-{:02d}-{:02d}'.format( chv[icmma + 1:icmma + 5], strptime(chv[:3], '%b').tm_mon, int(chv[3:icmma]))) chalos_meta.add_column(Column(ra, name='RA')) chalos_meta.add_column(Column(dec, name='DEC')) chalos_meta.add_column(Column(datet, name='DATE-OBS')) # Others chalos_meta.add_column( Column([' '] * len(chalos_meta), name='TELESCOPE')) # Padding chalos_meta.add_column(Column([' '] * len(chalos_meta), name='INSTR')) # Padding for HIRES chalos_meta.add_column( Column(['G130M/G160M'] * len(chalos_meta), name='DISPERSER')) chalos_meta.add_column(Column([20000.] * len(chalos_meta), name='R')) chalos_meta.add_column(Column([2000.] * len(chalos_meta), name='EPOCH')) chalos_meta['INSTR'] = 'COS' # Deals with padding chalos_meta['TELESCOPE'] = 'HST' # Myers for zem zem, zsource = zem_from_radec(chalos_meta['RA'], chalos_meta['DEC'], Table(igmsp.hdf['quasars'].value)) badz = zem <= 0. if np.sum(badz) > 0: raise ValueError("Bad zem in COS-Halos") chalos_meta['zem'] = zem chalos_meta['sig_zem'] = 0. # Need to add chalos_meta['flag_zem'] = zsource # HIRES hires_files = glob.glob( os.getenv('RAW_IGMSPEC') + '/COS-Halos/HIRES/J*f.fits.gz') hires_tab = chalos_meta[0:0] subnm = np.array([row['QSO'][4:9] for row in chalos_meta]) signs = np.array([row['QSO'][14] for row in chalos_meta]) for ifile in hires_files: print(ifile) fname = ifile.split('/')[-1] mt = np.where((subnm == fname[0:5]) & (signs == fname[5]))[0] if len(mt) != 1: pdb.set_trace() # Add row hires_tab.add_row(chalos_meta[mt[0]]) hires_tab[-1]['INSTR'] = 'HIRES' hires_tab[-1]['TELESCOPE'] = 'Keck I' hires_tab[-1]['DISPERSER'] = 'Red' hires_tab[-1]['R'] = Rdicts['HIRES']['C1'] # Combine chalos_meta = vstack([chalos_meta, hires_tab]) chalos_meta['STYPE'] = str('QSO') # Rename chalos_meta.rename_column('RA', 'RA_GROUP') chalos_meta.rename_column('DEC', 'DEC_GROUP') chalos_meta.rename_column('zem', 'zem_GROUP') # Check assert chk_meta(chalos_meta, chk_cat_only=True) # Done return chalos_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append ESI data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) esidla_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'ESI_DLA': raise IOError("Expecting ESI_DLA!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 50000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): # specfile = os.getenv('RAW_IGMSPEC')+'/HighzESIDLA/{:s}a_xF.fits'.format( row['Name']) print("ESI_DLA: Reading {:s}".format(specfile)) spec = lsio.readspec(specfile) # Parse name fname = specfile.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Continuum # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value #data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(Rdicts['ESI'][head['SLMSKNAM']]) except KeyError: if row['Slit'] == 0.75: Rlist.append(Rdicts['ESI']['0.75_arcsec']) elif row['Slit'] == 0.5: Rlist.append(Rdicts['ESI']['0.50_arcsec']) else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2012ApJ...755...89R', bib='rafelski+12'), dict(url='http://adsabs.harvard.edu/abs/2014ApJ...782L..29R', bib='rafelski+14'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Add 2QZ data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) tqz_grp = hdf.create_group(sname) # Checks if sname != '2QZ': raise IOError("Not expecting this survey..") # Add zem # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): full_file = get_specfil(row) # Parse name fname = full_file.split('/')[-1] # Read hdu = fits.open(full_file) head0 = hdu[0].header wave = lsio.setwave(head0) flux = hdu[0].data var = hdu[2].data sig = np.zeros_like(flux) gd = var > 0. if np.sum(gd) == 0: print("{:s} has a bad var array. Not including".format(fname)) pdb.set_trace() continue sig[gd] = np.sqrt(var[gd]) # npix spec = XSpectrum1D.from_tuple((wave,flux,sig)) npix = spec.npix spec.meta['headers'][0] = head0 if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # # References refs = [dict(url='http://adsabs.harvard.edu/abs/2004MNRAS.349.1397C', bib='2QZ') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def grab_meta(): """ Grab GGG meta Table Catalog -- http://www.2dfquasar.org/Spec_Cat/catalogue.html Returns ------- meta """ catfil = os.getenv('RAW_IGMSPEC') + '/2QZ/2QZ393524355693.out' tdf_meta = Table.read(catfil, format='ascii') # Rename columns clms = [ 'Name', 'RAh00', 'RAm00', 'RAs00', 'DECd00', 'DECm00', 'DECs00', 'ID', 'cat_name', 'Sector', 'RAh50', 'RAm50', 'RAs50', 'DECd50', 'DECm50', 'DECs50', 'UKST', 'XAPM', 'YAPM', 'RA50', 'DEC50', 'bj', 'u-b', 'b-r', 'Nobs', 'z1', 'q1', 'ID1', 'date1', 'fld1', 'fiber1', 'SN1', 'z2', 'q2', 'ID2', 'date2', 'fld2', 'fiber2', 'SN2', 'zprev', 'rflux', 'Xray', 'EBV', 'comm1', 'comm2' ] for ii in range(1, 46): tdf_meta.rename_column('col{:d}'.format(ii), clms[ii - 1]) # Cut down to QSOs and take only 1 spectrum ispec = [] zspec = [] datespec = [] for row in tdf_meta: if 'QSO' in row['ID1']: ispec.append(1) zspec.append(row['z1']) sdate = str(row['date1']) datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4], sdate[4:6], sdate[6:8])) elif 'QSO' in row['ID2']: ispec.append(2) zspec.append(row['z2']) sdate = str(row['date2']) datespec.append('{:s}-{:s}-{:s}'.format(sdate[0:4], sdate[4:6], sdate[6:8])) else: ispec.append(0) zspec.append(-1.) datespec.append('') tdf_meta['ispec'] = ispec tdf_meta['zem_GROUP'] = zspec tdf_meta['DATE'] = datespec cut = tdf_meta['ispec'] > 0 tdf_meta = tdf_meta[cut] nspec = len(tdf_meta) # DATE t = Time(list(tdf_meta['DATE'].data), format='iso', out_subfmt='date') # Fixes to YYYY-MM-DD tdf_meta.add_column(Column(t.iso, name='DATE-OBS')) # Add a few columns tdf_meta.add_column(Column([2000.] * nspec, name='EPOCH')) # Resolution # 2df 8.6A FWHM # R=580 at 5000A # 6df?? tdf_meta.add_column(Column([580.] * nspec, name='R')) # tdf_meta.add_column(Column([str('2dF')] * nspec, name='INSTR')) tdf_meta.add_column(Column([str('300B')] * nspec, name='DISPERSER')) tdf_meta.add_column(Column([str('UKST')] * nspec, name='TELESCOPE')) # Rename rad = (tdf_meta['RAh00'] * 3600 + tdf_meta['RAm00'] * 60 + tdf_meta['RAs00']) * 360. / 86400. decd = np.abs(tdf_meta['DECd00'] ) + tdf_meta['DECm00'] / 60 + tdf_meta['DECs00'] / 3600. # Yup the following is necessary neg = [False] * len(rad) for jj, row in enumerate(tdf_meta): if '-' in row['Name']: neg[jj] = True #if '-00' in row['Name']: # print('jj={:d}'.format(jj)) neg = np.array(neg) decd[neg] = -1. * decd[neg] tdf_meta['RA_GROUP'] = rad tdf_meta['DEC_GROUP'] = decd tdf_meta['sig_zem'] = [0.] * nspec tdf_meta['flag_zem'] = str('2QZ') tdf_meta['STYPE'] = str('QSO') # Require a spectrum exist gdm = np.array([True] * len(tdf_meta)) for jj, row in enumerate(tdf_meta): full_file = get_specfil(row) if not os.path.isfile(full_file): print("{:s} has no spectrum. Not including".format(full_file)) gdm[jj] = False continue tdf_meta = tdf_meta[gdm] # Sort tdf_meta.sort('RA_GROUP') # Check assert chk_meta(tdf_meta, chk_cat_only=True) # Return return tdf_meta
def grab_meta(test=False): """ Grab BOSS meta Table Returns ------- boss_meta : Table """ # Paris et al. qsos_dr14 = Table.read( os.getenv('RAW_IGMSPEC') + '/BOSS_DR14/DR14Q_v4_4.fits.gz') # Add original ID to coordinate with spectra and remove it later qsos_dr14['ORIG_ID'] = np.arange(len(qsos_dr14)) # Cut on BOSS (do not include DR7 files) boss_dr14 = qsos_dr14[qsos_dr14['SPECTRO'] == 'BOSS'] # Cut out bad ones bad_meta = Table.read( resource_filename('igmspec', 'ingest/files/bad_dr14.fits')) keep = np.array([True] * len(boss_dr14)) for row in bad_meta: bad_qso = np.where((boss_dr14['PLATE'] == row['PLATE']) & (boss_dr14['FIBERID'] == row['FIBERID']))[0] keep[bad_qso] = False boss_dr14 = boss_dr14[keep] if test: boss_dr14 = boss_dr14[:100] ''' # Cut out Plate 7840 plate7840 = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate7840_matched.ascii'), format='ascii') not_7840 = boss_dr14['PLATE'] != 7840 for ii in range(len(plate7840)): isplit = plate7840['sfile'][ii].split('-') idx = np.where((boss_dr14['PLATE'] == 7840) & (boss_dr14['FIBERID'] == int(isplit[-1][0:4])))[0] assert len(idx) == 1 not_7840[idx[0]] = True boss_dr14 = boss_dr14[not_7840] # Cut out others for plate in [7879, 3678, 7513, 4869, 7306]: ptbl = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate{:d}_matched.ascii'.format(plate)), format='ascii.csv') not_in = boss_dr14['PLATE'] != plate for ii in range(len(ptbl)): idx = np.where((boss_dr14['PLATE'] == ptbl['plate'][ii]) & (boss_dr14['FIBERID'] == ptbl['fiber'][ii]))[0] not_in[idx[0]] = True boss_dr14 = boss_dr14[not_in] ''' # Proceed boss_dr14['CAT'] = str('DR14') # Cut on redshift? #gd = np.any([boss_dr14['Z_PIPE'] > 0., boss_dr14['Z_PCA'] > 0.],axis=0) # CUTS Z_VI #boss_dr12 = boss_dr12[gd] # # nboss = len(boss_dr14) # DATE-OBS t = Time(list(boss_dr14['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD boss_dr14.add_column(Column(t.iso, name='DATE-OBS')) # Add columns boss_dr14.add_column(Column(['BOSS'] * nboss, name='INSTR')) boss_dr14.add_column(Column(['BOTH'] * nboss, name='DISPERSER')) #http://www.sdss.org/instruments/boss_spectrograph/ boss_dr14.add_column(Column([2100.] * nboss, name='R')) # RESOLUTION boss_dr14.add_column(Column(['SDSS 2.5-M'] * nboss, name='TELESCOPE')) # Redshift logic boss_dr14['zem_GROUP'] = boss_dr14['Z'] boss_dr14['sig_zem'] = boss_dr14['Z_PIPE_ERR'] boss_dr14['flag_zem'] = boss_dr14['SOURCE_Z'] ''' # Fix bad redshifts bad_pca = boss_meta['Z_PCA'] < 0. boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca] boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca] boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE') ''' # Rename RA/DEC boss_dr14.rename_column('RA', 'RA_GROUP') boss_dr14.rename_column('DEC', 'DEC_GROUP') # STYPE boss_dr14['STYPE'] = str('QSO') # Check assert chk_meta(boss_dr14, chk_cat_only=True) # Return return boss_dr14
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HD-LLS data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() mike_meta = grab_meta_mike() mike_coord = SkyCoord(ra=mike_meta['RA_GROUP'], dec=mike_meta['DEC_GROUP'], unit='deg') # Checks if sname != 'HD-LLS_DR1': raise IOError("Not expecting this survey..") full_coord = SkyCoord(ra=meta['RA_GROUP'], dec=meta['DEC_GROUP'], unit='deg') # Build spectra (and parse for meta) if mk_test_file: meta = meta[0:3] nspec = len(meta) max_npix = 210000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init full_idx = np.zeros(len(meta), dtype=int) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] instrlist = [] gratinglist = [] telelist = [] # Loop members = glob.glob(os.getenv('RAW_IGMSPEC') + '/{:s}/*fits'.format(sname)) kk = -1 for jj, member in enumerate(members): if 'HD-LLS_DR1.fits' in member: continue kk += 1 # Extract f = member hdu = fits.open(f) # Parse name fname = f.split('/')[-1] mt = np.where(meta['SPEC_FILE'] == fname)[0] if mk_test_file and (jj >= 3): continue if len(mt) != 1: pdb.set_trace() raise ValueError("HD-LLS: No match to spectral file?!") else: print('loading {:s}'.format(fname)) full_idx[kk] = mt[0] # npix head = hdu[0].header # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) # Double check if kk == 0: assert hdu[1].name == 'ERROR' assert hdu[2].name == 'WAVELENGTH' # Write spec = lsio.readspec(f) # Handles dummy pixels in ESI npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value #data['flux'][0][:npix] = hdu[0].data #data['sig'][0][:npix] = hdu[1].data #data['wave'][0][:npix] = hdu[2].data # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if 'HIRES' in fname: instrlist.append('HIRES') telelist.append('Keck-I') gratinglist.append('BOTH') try: Rlist.append(set_resolution(head)) except ValueError: # A few by hand (pulled from Table 1) if 'J073149' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-04', '%Y-%m-%d') elif 'J081435' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2006-12-26', '%Y-%m-%d') # 2008 too elif 'J095309' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2005-03-18', '%Y-%m-%d') elif 'J113418' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-05', '%Y-%m-%d') elif 'J135706' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2007-04-28', '%Y-%m-%d') elif 'J155556.9' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2005-04-15', '%Y-%m-%d') elif 'J212329' in fname: Rlist.append(Rdicts['HIRES']['E3']) tval = datetime.datetime.strptime('2006-08-20', '%Y-%m-%d') else: pdb.set_trace() else: tval = datetime.datetime.strptime(head['DATE-OBS'], '%Y-%m-%d') dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) elif 'ESI' in fname: instrlist.append('ESI') telelist.append('Keck-II') gratinglist.append('ECH') try: Rlist.append(set_resolution(head)) except ValueError: print("Using R=6,000 for ESI") Rlist.append(6000.) try: tval = datetime.datetime.strptime(head['DATE'], '%Y-%m-%d') except KeyError: if ('J223438.5' in fname) or ('J231543' in fname): tval = datetime.datetime.strptime('2004-09-11', '%Y-%m-%d') else: pdb.set_trace() dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) elif 'MIKE' in fname: # APPROXIMATE if 'MIKEr' in fname: instrlist.append('MIKEr') gratinglist.append('RED') elif 'MIKEb' in fname: instrlist.append('MIKEb') gratinglist.append('BLUE') else: instrlist.append('MIKE') gratinglist.append('BOTH') telelist.append('Magellan') sep = full_coord[mt[0]].separation(mike_coord) imin = np.argmin(sep) if sep[imin] > 1. * u.arcsec: pdb.set_trace() raise ValueError("Bad separation in MIKE") # R and Date Rlist.append(25000. / mike_meta['Slit'][imin]) tval = datetime.datetime.strptime(mike_meta['DATE-OBS'][imin], '%Y-%b-%d') dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) elif 'MAGE' in fname: # APPROXIMATE instrlist.append('MagE') if 'Clay' in head['TELESCOP']: telelist.append('Magellan/Clay') else: telelist.append('Magellan/Baade') gratinglist.append('N/A') Rlist.append(set_resolution(head)) dateobslist.append(head['DATE-OBS']) else: # MagE raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns meta = meta[full_idx] nmeta = len(meta) meta.add_column(Column([2000.] * nmeta, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column( Column([str(date) for date in dateobslist], name='DATE-OBS')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nmeta, dtype=int), name='GROUP_ID')) meta.add_column(Column(gratinglist, name='GRATING')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(telelist, name='TELESCOPE')) # v02 meta.rename_column('GRATING', 'DISPERSER') # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2015ApJS..221....2P', bib='prochaska+15'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HD-LLS data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() mike_meta = grab_meta_mike() mike_coord = SkyCoord(ra=mike_meta['RA_GROUP'], dec=mike_meta['DEC_GROUP'], unit='deg') # Checks if sname != 'HD-LLS_DR1': raise IOError("Not expecting this survey..") full_coord = SkyCoord(ra=meta['RA_GROUP'], dec=meta['DEC_GROUP'], unit='deg') # Build spectra (and parse for meta) if mk_test_file: meta = meta[0:3] nspec = len(meta) max_npix = 210000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init full_idx = np.zeros(len(meta), dtype=int) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] instrlist = [] gratinglist = [] telelist = [] # Loop members = glob.glob(os.getenv('RAW_IGMSPEC')+'/{:s}/*fits'.format(sname)) kk = -1 for jj,member in enumerate(members): if 'HD-LLS_DR1.fits' in member: continue kk += 1 # Extract f = member hdu = fits.open(f) # Parse name fname = f.split('/')[-1] mt = np.where(meta['SPEC_FILE'] == fname)[0] if mk_test_file and (jj>=3): continue if len(mt) != 1: pdb.set_trace() raise ValueError("HD-LLS: No match to spectral file?!") else: print('loading {:s}'.format(fname)) full_idx[kk] = mt[0] # npix head = hdu[0].header # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) # Double check if kk == 0: assert hdu[1].name == 'ERROR' assert hdu[2].name == 'WAVELENGTH' # Write spec = lsio.readspec(f) # Handles dummy pixels in ESI npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value #data['flux'][0][:npix] = hdu[0].data #data['sig'][0][:npix] = hdu[1].data #data['wave'][0][:npix] = hdu[2].data # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if 'HIRES' in fname: instrlist.append('HIRES') telelist.append('Keck-I') gratinglist.append('BOTH') try: Rlist.append(set_resolution(head)) except ValueError: # A few by hand (pulled from Table 1) if 'J073149' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-04', '%Y-%m-%d') elif 'J081435' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2006-12-26', '%Y-%m-%d') # 2008 too elif 'J095309' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2005-03-18', '%Y-%m-%d') elif 'J113418' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-05', '%Y-%m-%d') elif 'J135706' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2007-04-28', '%Y-%m-%d') elif 'J155556.9' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2005-04-15', '%Y-%m-%d') elif 'J212329' in fname: Rlist.append(Rdicts['HIRES']['E3']) tval = datetime.datetime.strptime('2006-08-20', '%Y-%m-%d') else: pdb.set_trace() else: tval = datetime.datetime.strptime(head['DATE-OBS'], '%Y-%m-%d') dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) elif 'ESI' in fname: instrlist.append('ESI') telelist.append('Keck-II') gratinglist.append('ECH') try: Rlist.append(set_resolution(head)) except ValueError: print("Using R=6,000 for ESI") Rlist.append(6000.) try: tval = datetime.datetime.strptime(head['DATE'], '%Y-%m-%d') except KeyError: if ('J223438.5' in fname) or ('J231543' in fname): tval = datetime.datetime.strptime('2004-09-11', '%Y-%m-%d') else: pdb.set_trace() dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) elif 'MIKE' in fname: # APPROXIMATE if 'MIKEr' in fname: instrlist.append('MIKEr') gratinglist.append('RED') elif 'MIKEb' in fname: instrlist.append('MIKEb') gratinglist.append('BLUE') else: instrlist.append('MIKE') gratinglist.append('BOTH') telelist.append('Magellan') sep = full_coord[mt[0]].separation(mike_coord) imin = np.argmin(sep) if sep[imin] > 1.*u.arcsec: pdb.set_trace() raise ValueError("Bad separation in MIKE") # R and Date Rlist.append(25000. / mike_meta['Slit'][imin]) tval = datetime.datetime.strptime(mike_meta['DATE-OBS'][imin], '%Y-%b-%d') dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) elif 'MAGE' in fname: # APPROXIMATE instrlist.append('MagE') if 'Clay' in head['TELESCOP']: telelist.append('Magellan/Clay') else: telelist.append('Magellan/Baade') gratinglist.append('N/A') Rlist.append(set_resolution(head)) dateobslist.append(head['DATE-OBS']) else: # MagE raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns meta = meta[full_idx] nmeta = len(meta) meta.add_column(Column([2000.]*nmeta, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column([str(date) for date in dateobslist], name='DATE-OBS')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nmeta,dtype=int),name='GROUP_ID')) meta.add_column(Column(gratinglist, name='GRATING')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(telelist, name='TELESCOPE')) # v02 meta.rename_column('GRATING', 'DISPERSER') # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2015ApJS..221....2P', bib='prochaska+15'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, boss_hdf=None, **kwargs): """ Add BOSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write boss_hdf : str, optional Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if boss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) boss_hdf.copy(sname, hdf) return boss_grp = hdf.create_group(sname) # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4650 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] speclist = [] npixlist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): # Generate full file full_file = get_specfil(row) if full_file == 'None': continue # Read spec = lsio.readspec(full_file) # npix npix = spec.npix # Kludge for higest redshift systems if npix < 10: full_file = get_specfil(row, hiz=True) try: spec = lsio.readspec(full_file) except: print("Missing: {:s}".format(full_file)) npix = spec.npix elif npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Parse name fname = full_file.split('/')[-1] # Fill for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # GZ Continuum -- packed in with spectrum, generated by my IDL script try: co = spec.co.value except AttributeError: co = np.zeros_like(spec.flux.value) # KG Continuum KG_file = get_specfil(row, KG=True) if os.path.isfile(KG_file) and (npix>1): # Latter is for junk in GZ file. Needs fixing hduKG = fits.open(KG_file) KGtbl = hduKG[1].data wvKG = 10.**KGtbl['LOGLAM'] try: assert (wvKG[0]-spec.wavelength[0].value) < 1e-5 except: pdb.set_trace() gdpix = np.where(wvKG < (1+row['zem_GROUP'])*1200.)[0] co[gdpix] = KGtbl['CONT'][gdpix] data['co'][0][:npix] = co # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) meta.add_column(Column([2000.]*len(meta), name='EPOCH')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2015ApJS..219...12A', bib='boss_qso_dr12'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append COS-Halos data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) chalos_grp = hdf.create_group(sname) # Load up # Checks if sname != 'COS-Halos': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 160000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop path = os.getenv('RAW_IGMSPEC')+'/COS-Halos/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file coord = ltu.radec_to_coord((row['RA_GROUP'],row['DEC_GROUP'])) if row['INSTR'].strip() == 'COS': full_file = path+'/J{:s}{:s}_nbin3_coadd.fits.gz'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True)[0:4], coord.dec.to_string(sep='',pad=True,alwayssign=True)[0:5]) else: # HIRES full_file = path+'/HIRES/J{:s}{:s}_f.fits.gz'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True)[0:4], coord.dec.to_string(sep='',pad=True,alwayssign=True)[0:5]) # Extract print("COS-Halos: Reading {:s}".format(full_file)) spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2013ApJ...777...59T', bib='tumlinson+13'), dict(url='http://adsabs.harvard.edu/abs/2013ApJS..204...17W', bib='werk+13') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, boss_hdf=None, **kwargs): """ Add BOSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write boss_hdf : str, optional Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if boss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) boss_hdf.copy(sname, hdf) return boss_grp = hdf.create_group(sname) # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4660 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] speclist = [] npixlist = [] # Loop maxpix = 0 bad_spec = np.array([False] * len(meta)) for jj in range(len(meta)): # Generate full file full_file = get_specfil(jj, meta) if full_file == 'None': continue # Read try: spec = lsio.readspec(full_file, masking='edges') except: print("Failed on full_file: {:s}, {:d}".format(full_file, jj)) bad_spec[jj] = True continue # npix npix = spec.npix if npix < 10: print("Not enough pixels in file: {:s}, {:d}".format( full_file, jj)) bad_spec[jj] = True continue ''' # Kludge for higest redshift systems if npix < 10: full_file = get_specfil(jj, meta, hiz=True) try: spec = lsio.readspec(full_file) except: print("Missing: {:s}".format(full_file)) npix = spec.npix elif npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) ''' maxpix = max(npix, maxpix) # Parse name fname = full_file.split('/')[-1] # Fill for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value ''' # GZ Continuum -- packed in with spectrum, generated by my IDL script try: co = spec.co.value except AttributeError: co = np.zeros_like(spec.flux.value) # KG Continuum KG_file = get_specfil(row, KG=True) if os.path.isfile(KG_file) and (npix>1): # Latter is for junk in GZ file. Needs fixing hduKG = fits.open(KG_file) KGtbl = hduKG[1].data wvKG = 10.**KGtbl['LOGLAM'] try: assert (wvKG[0]-spec.wavelength[0].value) < 1e-5 except: pdb.set_trace() gdpix = np.where(wvKG < (1+row['zem_GROUP'])*1200.)[0] co[gdpix] = KGtbl['CONT'][gdpix] data['co'][0][:npix] = co ''' # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # Deal with null spec -- Should only be done once, saved and then ready to go if np.any(bad_spec): bad_meta = meta[bad_spec] pdb.set_trace() print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) meta.add_column(Column([2000.] * len(meta), name='EPOCH')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2015ApJS..219...12A', bib='boss_qso_dr12'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def grab_meta(test=False): """ Grab BOSS meta Table Returns ------- boss_meta : Table """ # Paris et al. qsos_dr14 = Table.read(os.getenv('RAW_IGMSPEC')+'/BOSS_DR14/DR14Q_v4_4.fits.gz') # Add original ID to coordinate with spectra and remove it later qsos_dr14['ORIG_ID'] = np.arange(len(qsos_dr14)) # Cut on BOSS (do not include DR7 files) boss_dr14 = qsos_dr14[qsos_dr14['SPECTRO'] == 'BOSS'] # Cut out bad ones bad_meta = Table.read(resource_filename('igmspec', 'ingest/files/bad_dr14.fits')) keep = np.array([True]*len(boss_dr14)) for row in bad_meta: bad_qso = np.where((boss_dr14['PLATE'] == row['PLATE']) & ( boss_dr14['FIBERID'] == row['FIBERID']))[0] keep[bad_qso] = False boss_dr14 = boss_dr14[keep] if test: boss_dr14 = boss_dr14[:100] ''' # Cut out Plate 7840 plate7840 = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate7840_matched.ascii'), format='ascii') not_7840 = boss_dr14['PLATE'] != 7840 for ii in range(len(plate7840)): isplit = plate7840['sfile'][ii].split('-') idx = np.where((boss_dr14['PLATE'] == 7840) & (boss_dr14['FIBERID'] == int(isplit[-1][0:4])))[0] assert len(idx) == 1 not_7840[idx[0]] = True boss_dr14 = boss_dr14[not_7840] # Cut out others for plate in [7879, 3678, 7513, 4869, 7306]: ptbl = Table.read(resource_filename('igmspec', 'ingest/files/bossdr14_plate{:d}_matched.ascii'.format(plate)), format='ascii.csv') not_in = boss_dr14['PLATE'] != plate for ii in range(len(ptbl)): idx = np.where((boss_dr14['PLATE'] == ptbl['plate'][ii]) & (boss_dr14['FIBERID'] == ptbl['fiber'][ii]))[0] not_in[idx[0]] = True boss_dr14 = boss_dr14[not_in] ''' # Proceed boss_dr14['CAT'] = str('DR14') # Cut on redshift? #gd = np.any([boss_dr14['Z_PIPE'] > 0., boss_dr14['Z_PCA'] > 0.],axis=0) # CUTS Z_VI #boss_dr12 = boss_dr12[gd] # # nboss = len(boss_dr14) # DATE-OBS t = Time(list(boss_dr14['MJD'].data), format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD boss_dr14.add_column(Column(t.iso, name='DATE-OBS')) # Add columns boss_dr14.add_column(Column(['BOSS']*nboss, name='INSTR')) boss_dr14.add_column(Column(['BOTH']*nboss, name='DISPERSER')) #http://www.sdss.org/instruments/boss_spectrograph/ boss_dr14.add_column(Column([2100.]*nboss, name='R')) # RESOLUTION boss_dr14.add_column(Column(['SDSS 2.5-M']*nboss, name='TELESCOPE')) # Redshift logic boss_dr14['zem_GROUP'] = boss_dr14['Z'] boss_dr14['sig_zem'] = boss_dr14['Z_PIPE_ERR'] boss_dr14['flag_zem'] = boss_dr14['SOURCE_Z'] ''' # Fix bad redshifts bad_pca = boss_meta['Z_PCA'] < 0. boss_meta['zem_GROUP'][bad_pca] = boss_meta['Z_PIPE'][bad_pca] boss_meta['sig_zem'][bad_pca] = boss_meta['ERR_ZPIPE'][bad_pca] boss_meta['flag_zem'][bad_pca] = str('BOSS_PIPE') ''' # Rename RA/DEC boss_dr14.rename_column('RA', 'RA_GROUP') boss_dr14.rename_column('DEC', 'DEC_GROUP') # STYPE boss_dr14['STYPE'] = str('QSO') # Check assert chk_meta(boss_dr14, chk_cat_only=True) # Return return boss_dr14
def grab_meta(): """ Grab XQ-100 meta Table Returns ------- """ from specdb.specdb import IgmSpec igmsp = IgmSpec() # xq100_table = Table.read( os.getenv('RAW_IGMSPEC') + '/XQ-100/XQ100_v1_2.fits.gz') nqso = len(xq100_table) # ESO meta eso_tbl = Table.read(os.getenv('RAW_IGMSPEC') + '/XQ-100/metadata_eso_XQ100.csv', format='ascii.csv') ar_files = eso_tbl['ARCFILE'].data # Spectral files spec_files = glob.glob(os.getenv('RAW_IGMSPEC') + '/XQ-100/ADP.*') # Dummy column xq100_coords = SkyCoord(ra=xq100_table['RA'], dec=xq100_table['DEC'], unit='deg') matches = [] sv_spec_files = [] sv_orig_files = [] sv_rescale_files = [] for spec_file in spec_files: if 'ADP.2016-07-15T08:22:40.682.fits' in spec_file: print("XQ-100: Skipping summary file") continue # ESO file ssfile = spec_file[spec_file.rfind('/') + 1:-5] eso_mt = np.where(ar_files == ssfile)[0] try: ofile = eso_tbl['ORIGFILE'][eso_mt][0] except IndexError: print("XQ-100: File {:s} not really in XQ100!".format(spec_file)) continue if ('_1' in ofile) or ('_2' in ofile) or ('_3' in ofile) or ('_4' in ofile): print("XQ-100: Skipping additional file: {:s}".format(ofile)) continue # Match hdu = fits.open(spec_file) head0 = hdu[0].header if head0['DISPELEM'] == 'UVB,VIS,NIR': print("XQ-100: Skipping merged spectrum file") if 'rescale' not in ofile: print('no rescale') pdb.set_trace() continue try: coord = SkyCoord(ra=head0['RA'], dec=head0['DEC'], unit='deg') except KeyError: pdb.set_trace() sep = coord.separation(xq100_coords) imt = np.argmin(sep) if sep[imt] > 0.1 * u.arcsec: pdb.set_trace() raise ValueError("Bad offset") # Save matches.append(imt) sv_spec_files.append(spec_file) sv_orig_files.append(ofile) # Finish up xq100_meta = xq100_table[np.array(matches)] nspec = len(xq100_meta) # Add spec_files xq100_meta['SPEC_FILE'] = sv_spec_files xq100_meta['ORIG_FILE'] = sv_orig_files # Add zem xq100_meta['zem_GROUP'] = xq100_meta['Z_QSO'] xq100_meta['sig_zem'] = xq100_meta['ERR_ZQSO'] xq100_meta['flag_zem'] = [str('XQ-100')] * nspec # Rename xq100_meta.rename_column('RA', 'RA_GROUP') xq100_meta.rename_column('DEC', 'DEC_GROUP') # Match to Myers myers = Table(igmsp.hdf['quasars'].value) myers_coord = SkyCoord(ra=myers['RA'], dec=myers['DEC'], unit='deg') xq100_coord = SkyCoord(ra=xq100_meta['RA_GROUP'], dec=xq100_meta['DEC_GROUP'], unit='deg') idx, d2d, _ = match_coordinates_sky(xq100_coord, myers_coord, nthneighbor=1) xq100_meta['RA_GROUP'] = myers_coord.ra.value[idx] xq100_meta['DEC_GROUP'] = myers_coord.dec.value[idx] # One bad one (Taking RA/DEC from Simbad) bad_c = d2d.to('arcsec') > 20 * u.arcsec xq100_meta['RA_GROUP'][bad_c] = 215.2823 xq100_meta['DEC_GROUP'][bad_c] = -6.73232 # DATE-OBS meanmjd = [] for row in xq100_meta: gdm = row['MJD_OBS'] > 0. meanmjd.append(np.mean(row['MJD_OBS'][gdm])) t = Time(meanmjd, format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD xq100_meta.add_column(Column(t.iso, name='DATE-OBS')) # xq100_meta.add_column(Column([2000.] * nspec, name='EPOCH')) xq100_meta['STYPE'] = str('QSO') # Sort xq100_meta.sort('RA_GROUP') # Check assert chk_meta(xq100_meta, chk_cat_only=True) # return xq100_meta
def grab_meta(): """ Grab XQ-100 meta Table Returns ------- """ from specdb.specdb import IgmSpec igmsp = IgmSpec() # xq100_table = Table.read(os.getenv('RAW_IGMSPEC')+'/XQ-100/XQ100_v1_2.fits.gz') nqso = len(xq100_table) # ESO meta eso_tbl = Table.read(os.getenv('RAW_IGMSPEC')+'/XQ-100/metadata_eso_XQ100.csv', format='ascii.csv') ar_files = eso_tbl['ARCFILE'].data # Spectral files spec_files = glob.glob(os.getenv('RAW_IGMSPEC')+'/XQ-100/ADP.*') # Dummy column xq100_coords = SkyCoord(ra=xq100_table['RA'], dec=xq100_table['DEC'], unit='deg') matches = [] sv_spec_files = [] sv_orig_files = [] sv_rescale_files = [] for spec_file in spec_files: if 'ADP.2016-07-15T08:22:40.682.fits' in spec_file: print("XQ-100: Skipping summary file") continue # ESO file ssfile = spec_file[spec_file.rfind('/')+1:-5] eso_mt = np.where(ar_files == ssfile)[0] try: ofile = eso_tbl['ORIGFILE'][eso_mt][0] except IndexError: print("XQ-100: File {:s} not really in XQ100!".format(spec_file)) continue if ('_1' in ofile) or ('_2' in ofile) or ('_3' in ofile) or ('_4' in ofile): print("XQ-100: Skipping additional file: {:s}".format(ofile)) continue # Match hdu = fits.open(spec_file) head0 = hdu[0].header if head0['DISPELEM'] == 'UVB,VIS,NIR': print("XQ-100: Skipping merged spectrum file") if 'rescale' not in ofile: print('no rescale') pdb.set_trace() continue try: coord = SkyCoord(ra=head0['RA'], dec=head0['DEC'], unit='deg') except KeyError: pdb.set_trace() sep = coord.separation(xq100_coords) imt = np.argmin(sep) if sep[imt] > 0.1*u.arcsec: pdb.set_trace() raise ValueError("Bad offset") # Save matches.append(imt) sv_spec_files.append(spec_file) sv_orig_files.append(ofile) # Finish up xq100_meta = xq100_table[np.array(matches)] nspec = len(xq100_meta) # Add spec_files xq100_meta['SPEC_FILE'] = sv_spec_files xq100_meta['ORIG_FILE'] = sv_orig_files # Add zem xq100_meta['zem_GROUP'] = xq100_meta['Z_QSO'] xq100_meta['sig_zem'] = xq100_meta['ERR_ZQSO'] xq100_meta['flag_zem'] = [str('XQ-100')]*nspec # Rename xq100_meta.rename_column('RA','RA_GROUP') xq100_meta.rename_column('DEC','DEC_GROUP') # Match to Myers myers = Table(igmsp.hdf['quasars'].value) myers_coord = SkyCoord(ra=myers['RA'], dec=myers['DEC'], unit='deg') xq100_coord = SkyCoord(ra=xq100_meta['RA_GROUP'], dec=xq100_meta['DEC_GROUP'], unit='deg') idx, d2d, _ = match_coordinates_sky(xq100_coord, myers_coord, nthneighbor=1) xq100_meta['RA_GROUP'] = myers_coord.ra.value[idx] xq100_meta['DEC_GROUP'] = myers_coord.dec.value[idx] # One bad one (Taking RA/DEC from Simbad) bad_c = d2d.to('arcsec') > 20*u.arcsec xq100_meta['RA_GROUP'][bad_c] = 215.2823 xq100_meta['DEC_GROUP'][bad_c] = -6.73232 # DATE-OBS meanmjd = [] for row in xq100_meta: gdm = row['MJD_OBS'] > 0. meanmjd.append(np.mean(row['MJD_OBS'][gdm])) t = Time(meanmjd, format='mjd', out_subfmt='date') # Fixes to YYYY-MM-DD xq100_meta.add_column(Column(t.iso, name='DATE-OBS')) # xq100_meta.add_column(Column([2000.]*nspec, name='EPOCH')) xq100_meta['STYPE'] = str('QSO') # Sort xq100_meta.sort('RA_GROUP') # Check assert chk_meta(xq100_meta, chk_cat_only=True) # return xq100_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HSTQSO data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) hstz2_grp = hdf.create_group(sname) # Checks if sname != 'HSTQSO': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 80000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] Rlist = [] # Loop #path = os.getenv('RAW_IGMSPEC')+'/KODIAQ_data_20150421/' path = os.getenv('RAW_IGMSPEC') + '/HSTQSO/' maxpix = 0 for jj, row in enumerate(meta): # Generate full file full_file = path + row['SPEC_FILE'] + '.gz' # Extract print("HSTQSO: Reading {:s}".format(full_file)) hduf = fits.open(full_file) head0 = hduf[0].header spec = lsio.readspec(full_file, masking='edges') # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta if 'FOS-L' in fname: Rlist.append(300.) elif 'FOS-H' in fname: Rlist.append(14000.) elif 'STIS' in fname: if row['DISPERSER'] == 'G230L': Rlist.append(700.) elif row['DISPERSER'] == 'G140L': Rlist.append(1200.) else: raise ValueError("Bad STIS grating") elif 'hsla' in fname: # COS Rlist.append(18000.) row['DATE-OBS'] = hduf[1].data['DATEOBS'][0][0] else: pdb.set_trace() raise ValueError("Missing instrument!") wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column([2000.] * nspec, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2011ApJ...736...42R', bib='ribuado11'), dict(url='http://adsabs.harvard.edu/abs/2016ApJ...818..113N', bib='neeleman16'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append XQ-100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) xq100_grp = hdf.create_group(sname) if len(meta) != 300: pdb.set_trace() # Checks if sname != 'XQ-100': raise IOError("Expecting XQ-100!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 20000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] instrlist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): # print("XQ-100: Reading {:s}".format(row['SPEC_FILE'])) spec = lsio.readspec(row['SPEC_FILE']) # Parse name fname = row['SPEC_FILE'].split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Continuum # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) telelist.append(head['TELESCOP']) instrlist.append(head['INSTRUME']) gratinglist.append(head['DISPELEM']) npixlist.append(npix) if gratinglist[-1] == 'NIR': # From Lopez+16 Rlist.append(4350.) elif gratinglist[-1] == 'VIS': Rlist.append(7450.) elif gratinglist[-1] == 'UVB': Rlist.append(5300.) else: pdb.set_trace() raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(telelist, name='TELESCOPE')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2016arXiv160708776L', bib='lopez+16') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def grab_meta(): """ Grab HSTQSO meta Table Returns ------- """ summ_file = os.getenv('RAW_IGMSPEC') + '/HSTQSO/hstqso.lst' hstqso_meta = Table.read(summ_file, format='ascii') spec_files = [str(ii) for ii in hstqso_meta['SPEC_FILE'].data] nspec = len(hstqso_meta) # RA/DEC radec_file = os.getenv('RAW_IGMSPEC') + '/HSTQSO/all_qso_table.txt' radec = Table.read(radec_file, format='ascii') # DATE-OBS date_files = glob.glob(os.getenv('RAW_IGMSPEC') + '/HSTQSO/date_obs*') for ss, date_file in enumerate(date_files): if ss == 0: tab_date = Table.read(date_file, format='ascii') else: tab_date = vstack( [tab_date, Table.read(date_file, format='ascii')]) # RA/DEC, DATE hstqso_meta.add_column(Column(['2000-01-01'] * nspec, name='DATE-OBS')) for jj, row in enumerate(hstqso_meta): if row['INST'] == 'COS': spec_files[jj] = str(row['QSO_ALT_NAME'] + '_hsla.fits') continue # DATE spec = row['SPEC_FILE'].split('.')[0] mt1 = np.where(tab_date['SPEC'] == spec)[0] if len(mt1) == 0: print("NO DATE MATCH for {:s}!".format(spec)) pdb.set_trace() else: mt1 = mt1[0] # TAKING THE FIRST ONE joe_date = tab_date['DATE-OBS'][mt1].split('-') hstqso_meta[jj]['DATE-OBS'] = '{:s}-{:02d}-{:02d}'.format( joe_date[0], int(joe_date[1]), int(joe_date[2])) if int(joe_date[1]) > 12: pdb.set_trace() # RA/DEC if row['INST'] != 'FOS': continue mt = np.where(radec['File_ID'] == row['QSO_ALT_NAME'])[0] if len(mt) == 0: mt = np.where(radec['File_ID'] == row['QSO_NAME'])[0] if len(mt) == 0: print("NO RA/DEC MATCH!") pdb.set_trace() else: mt = mt[0] else: mt = mt[0] hstqso_meta[jj]['RA'] = radec['RA'][mt] hstqso_meta[jj]['DEC'] = radec['DEC'][mt] # Deal with Dups (mainly bad FOS coords) coord = SkyCoord(ra=hstqso_meta['RA'], dec=hstqso_meta['DEC'], unit='deg') idx, d2d, d3d = match_coordinates_sky(coord, coord, nthneighbor=2) dups = np.where(d2d < 2.0 * u.arcsec)[0] # Closest lens is ~2" flag_dup = np.array([False] * len(hstqso_meta)) for idup in dups: if flag_dup[idup]: continue dcoord = SkyCoord(ra=hstqso_meta['RA'][idup], dec=hstqso_meta['DEC'][idup], unit='deg') sep = dcoord.separation(coord) isep = np.where(sep < 2.0 * u.arcsec)[0] # Search for COS first icos = np.where(hstqso_meta['INST'][isep] == 'COS')[0] if len(icos) > 0: hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[icos[0]]] hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[icos[0]]] flag_dup[isep] = True else: # STIS istis = np.where(hstqso_meta['INST'][isep] == 'STIS')[0] if len(istis) > 0: hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[istis[0]]] hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[istis[0]]] flag_dup[isep] = True else: # FOS only -- taking first value hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[0]] hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[0]] # REPLACE hstqso_meta.rename_column('SPEC_FILE', 'ORIG_SPEC_FILE') hstqso_meta['SPEC_FILE'] = spec_files # RENAME hstqso_meta.rename_column('GRATE', 'DISPERSER') hstqso_meta.rename_column('QSO_ZEM', 'zem_GROUP') hstqso_meta.rename_column('INST', 'INSTR') hstqso_meta['STYPE'] = str('QSO') hstqso_meta.rename_column('RA', 'RA_GROUP') hstqso_meta.rename_column('DEC', 'DEC_GROUP') # ADD hstqso_meta.add_column(Column(['HST'] * nspec, name='TELESCOPE')) hstqso_meta['sig_zem'] = 0. hstqso_meta['flag_zem'] = str('UNKWN') # Check assert chk_meta(hstqso_meta, chk_cat_only=True) # Return return hstqso_meta
def grab_meta(): """ Grab HSTQSO meta Table Returns ------- """ summ_file = os.getenv('RAW_IGMSPEC')+'/HSTQSO/hstqso.lst' hstqso_meta = Table.read(summ_file, format='ascii') spec_files = [str(ii) for ii in hstqso_meta['SPEC_FILE'].data] nspec = len(hstqso_meta) # RA/DEC radec_file = os.getenv('RAW_IGMSPEC')+'/HSTQSO/all_qso_table.txt' radec = Table.read(radec_file, format='ascii') # DATE-OBS date_files = glob.glob(os.getenv('RAW_IGMSPEC')+'/HSTQSO/date_obs*') for ss,date_file in enumerate(date_files): if ss == 0: tab_date = Table.read(date_file, format='ascii') else: tab_date = vstack([tab_date, Table.read(date_file, format='ascii')]) # RA/DEC, DATE hstqso_meta.add_column(Column(['2000-01-01']*nspec, name='DATE-OBS')) for jj,row in enumerate(hstqso_meta): if row['INST'] == 'COS': spec_files[jj] = str(row['QSO_ALT_NAME']+'_hsla.fits') continue # DATE spec = row['SPEC_FILE'].split('.')[0] mt1 = np.where(tab_date['SPEC'] == spec)[0] if len(mt1) == 0: print("NO DATE MATCH for {:s}!".format(spec)) pdb.set_trace() else: mt1 = mt1[0] # TAKING THE FIRST ONE joe_date = tab_date['DATE-OBS'][mt1].split('-') hstqso_meta[jj]['DATE-OBS'] = '{:s}-{:02d}-{:02d}'.format(joe_date[0], int(joe_date[1]), int(joe_date[2])) if int(joe_date[1]) > 12: pdb.set_trace() # RA/DEC if row['INST'] != 'FOS': continue mt = np.where(radec['File_ID'] == row['QSO_ALT_NAME'])[0] if len(mt) == 0: mt = np.where(radec['File_ID'] == row['QSO_NAME'])[0] if len(mt) == 0: print("NO RA/DEC MATCH!") pdb.set_trace() else: mt = mt[0] else: mt = mt[0] hstqso_meta[jj]['RA'] = radec['RA'][mt] hstqso_meta[jj]['DEC'] = radec['DEC'][mt] # Deal with Dups (mainly bad FOS coords) coord = SkyCoord(ra=hstqso_meta['RA'], dec=hstqso_meta['DEC'], unit='deg') idx, d2d, d3d = match_coordinates_sky(coord, coord, nthneighbor=2) dups = np.where(d2d < 2.0*u.arcsec)[0] # Closest lens is ~2" flag_dup = np.array([False]*len(hstqso_meta)) for idup in dups: if flag_dup[idup]: continue dcoord = SkyCoord(ra=hstqso_meta['RA'][idup], dec=hstqso_meta['DEC'][idup], unit='deg') sep = dcoord.separation(coord) isep = np.where(sep < 2.0*u.arcsec)[0] # Search for COS first icos = np.where(hstqso_meta['INST'][isep] == 'COS')[0] if len(icos) > 0: hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[icos[0]]] hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[icos[0]]] flag_dup[isep] = True else: # STIS istis = np.where(hstqso_meta['INST'][isep] == 'STIS')[0] if len(istis) > 0: hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[istis[0]]] hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[istis[0]]] flag_dup[isep] = True else: # FOS only -- taking first value hstqso_meta['RA'][isep] = hstqso_meta['RA'][isep[0]] hstqso_meta['DEC'][isep] = hstqso_meta['DEC'][isep[0]] # REPLACE hstqso_meta.rename_column('SPEC_FILE', 'ORIG_SPEC_FILE') hstqso_meta['SPEC_FILE'] = spec_files # RENAME hstqso_meta.rename_column('GRATE', 'DISPERSER') hstqso_meta.rename_column('QSO_ZEM', 'zem_GROUP') hstqso_meta.rename_column('INST', 'INSTR') hstqso_meta['STYPE'] = str('QSO') hstqso_meta.rename_column('RA', 'RA_GROUP') hstqso_meta.rename_column('DEC', 'DEC_GROUP') # ADD hstqso_meta.add_column(Column(['HST']*nspec, name='TELESCOPE')) hstqso_meta['sig_zem'] = 0. hstqso_meta['flag_zem'] = str('UNKWN') # Check assert chk_meta(hstqso_meta, chk_cat_only=True) # Return return hstqso_meta
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append GGG data to the h5 file Parameters ---------- hdf : hdf5 pointer sname : str Survey name meta : Table chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) ggg_grp = hdf.create_group(sname) # Load up if sname != 'GGG': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 1600 # Just needs to be large enough # Init data = init_data(max_npix, include_co=False) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] dateobslist = [] instrlist = [] # Loop path = os.getenv('RAW_IGMSPEC')+'/GGG/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file if jj >= nspec//2: full_file = path+row['name']+'_R400.fits.gz' gratinglist.append('R400') else: full_file = path+row['name']+'_B600.fits.gz' gratinglist.append('B600') # Extract print("GGG: Reading {:s}".format(full_file)) spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) telelist.append(head['OBSERVAT']) instrlist.append(head['INSTRUME']) tval = Time(head['DATE'], format='isot', out_subfmt='date') dateobslist.append(tval.iso) npixlist.append(npix) if 'R400' in fname: Rlist.append(833.) else: Rlist.append(940.) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(gratinglist, name='GRATING')) meta.add_column(Column(telelist, name='TELESCOPE')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(dateobslist, name='DATE-OBS')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2014MNRAS.445.1745W', bib='worseck+14')] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return