def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append XQ-100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) xq100_grp = hdf.create_group(sname) if len(meta) != 300: pdb.set_trace() # Checks if sname != 'XQ-100': raise IOError("Expecting XQ-100!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 20000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] instrlist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): # print("XQ-100: Reading {:s}".format(row['SPEC_FILE'])) spec = lsio.readspec(row['SPEC_FILE']) # Parse name fname = row['SPEC_FILE'].split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Continuum # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) telelist.append(head['TELESCOP']) instrlist.append(head['INSTRUME']) gratinglist.append(head['DISPELEM']) npixlist.append(npix) if gratinglist[-1] == 'NIR': # From Lopez+16 Rlist.append(4350.) elif gratinglist[-1] == 'VIS': Rlist.append(7450.) elif gratinglist[-1] == 'UVB': Rlist.append(5300.) else: pdb.set_trace() raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(telelist, name='TELESCOPE')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2016arXiv160708776L', bib='lopez+16') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, hdla100_meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HDLA100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'HDLA100': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) #if mk_test_file: # hdla100_full = hdlls_full[0:3] max_npix = 192000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') nspec = len(hdla100_meta) spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] gratinglist = [] # Loop for jj,row in enumerate(hdla100_meta): kk = jj # Extract f = os.getenv('RAW_IGMSPEC')+'/HDLA100/'+row['SPEC_FILE'] spec = lsio.readspec(f) # Parse name fname = f.split('/')[-1] # npix head = spec.header npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(set_resolution(head)) except ValueError: raise ValueError("Header is required for {:s}".format(fname)) else: if '/' in head['DATE-OBS']: spl = head['DATE-OBS'].split('/') t = Time(datetime.datetime(int(spl[2])+1900, int(spl[1]), int(spl[0])), format='datetime') else: t = Time(head['DATE-OBS'], format='isot', out_subfmt='date') dateobslist.append(t.iso) # Grating try: gratinglist.append(head['XDISPERS']) except KeyError: try: yr = t.value.year except AttributeError: yr = int(t.value[0:4]) if yr <= 1997: gratinglist.append('RED') else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns nmeta = len(hdla100_meta) hdla100_meta.add_column(Column([2000.]*nmeta, name='EPOCH')) hdla100_meta.add_column(Column(npixlist, name='NPIX')) hdla100_meta.add_column(Column([str(date) for date in dateobslist], name='DATE-OBS')) hdla100_meta.add_column(Column(wvminlist, name='WV_MIN')) hdla100_meta.add_column(Column(wvmaxlist, name='WV_MAX')) hdla100_meta.add_column(Column(Rlist, name='R')) hdla100_meta.add_column(Column(np.arange(nmeta,dtype=int),name='GROUP_ID')) hdla100_meta.add_column(Column(gratinglist, name='DISPERSER')) hdla100_meta['INSTR'] = ['HIRES']*nspec hdla100_meta['TELESCOPE'] = ['Keck-I']*nspec #hdla100_meta.rename_column('Z_QSO', 'zem') # Add HDLLS meta to hdf5 if chk_meta(hdla100_meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = hdla100_meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2013ApJ...769...54N', bib='neeleman+13'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, sdss_hdf=None, **kwargs): """ Add SDSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if sdss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) sdss_hdf.copy(sname, hdf) return sdss_grp = hdf.create_group(sname) # Load up # Checks if sname != 'SDSS_DR7': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) # Read Zhu continua, wave file cfile = os.getenv('RAW_IGMSPEC')+'/SDSS/ALLQSO_SPEC_106_continuum_nointerp.fits' zhu_conti = Table.read(cfile) wvfile = cfile.replace('continuum','wave') zhu_wave = Table.read(wvfile) # wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): full_file = get_specfil(row) if not os.path.isfile(full_file): full_file = get_specfil(row, dr7=True) # Extract #print("SDSS: Reading {:s}".format(full_file)) # Parse name fname = full_file.split('/')[-1] # Generate full file spec = lsio.readspec(full_file) # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig','co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Continuum mtc = (zhu_conti['PLATE'] == row['PLATE']) & (zhu_conti['FIBER']==row['FIBER']) mtw = (zhu_wave['PLATE'] == row['PLATE']) & (zhu_wave['FIBER']==row['FIBER']) if np.sum(mtc) == 1: imin = np.argmin(np.abs(zhu_wave['WAVE'][0][:,np.where(mtw)[1]]-spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][imin:npix+imin,np.where(mtc)[1]].flatten() elif np.sum(mtc) > 1: print("Multiple continua for plate={:d}, row={:d}. Taking the first".format(row['PLATE'], row['FIBER'])) imin = np.argmin(np.abs(zhu_wave['WAVE'][0][:,np.where(mtw)[1][0]]-spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][imin:npix+imin,np.where(mtc)[1][0]].flatten() elif np.sum(mtc) == 0: print("No SDSS continuum for plate={:d}, row={:d}".format(row['PLATE'], row['FIBER'])) #from xastropy.xutils import xdebug as xdb #xdb.set_trace() #xdb.xplot(data['wave'][0], data['flux'][0], data['co'][0]) # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2010AJ....139.2360S', bib='sdss_qso_dr7'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_writter(path, reso, lowl, highl, outputname): ''' :param path: path to the spectra file :param lowl float: lower limit for request redshift :param highl float: higher limit for request redshift :param reso: delta v in km/s to rebin the data :param outputname: the name for the output file :return: hdf5file ''' outfil = str(outputname) hdf = h5py.File(outfil, 'w') # gdict = {} # creating group for z in np.arange(lowl, highl, 0.05): # hdf.create_group('z'+str(z)+'-'+str(z+0.05)) print(z) spectest1 = [] spec2 = [] spectest1, spec2 = reading_data(path, reso, z, z + 0.05) print(np.round(z, 2), len(spectest1.z)) # hdf_append = h5py.File('tmp'+str(z)+'.hdf', 'w') # hdf.create_group('z' + str(z) + '-' + str(z + 0.05)) f = hdf.create_group('z' + str(np.round(z, 2)) + '-' + str(np.round(z + 0.05, 2))) npix = len(spec2.wavelength) data = sdb_u.init_data(npix) nspec = len(spectest1.z) print(npix) #creat dataset spec_set = hdf['z' + str(np.round(z, 2)) + '-' + str(np.round(z + 0.05, 2))].create_dataset( 'spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) for ii in range(nspec): data['flux'][0][:npix] = spec2[ii].flux # Should be flux values data['sig'][0][:npix] = spec2[ii].sig # SHould be sigma values # print (spec[ii].sig) data['wave'][0][:npix] = spec2[ ii].wavelength # Should be wavelength values # Fill spec_set[ii] = data # hdf.copy('z'+str(z)+'-'+str(z+0.05), hdf_append['z'+str(z)+'-'+str(z+0.05)]) # making meta data # hdfnew = h5py.File('z2.8_specdb_test1.hdf', 'w') # group = 'z2.8-2.85' # _ = hdfnew.create_group(group) group = 'z' + str(np.round(z, 2)) + '-' + str(np.round(z + 0.05, 2)) id_key = 'DESI_ID' maindb, tkeys = spbu.start_maindb(id_key) meta = Table() meta['zem_GROUP'] = spectest1.z meta['RA_GROUP'] = spectest1.ra meta['DEC_GROUP'] = spectest1.dec meta['EPOCH'] = 2000. meta['sig_zem'] = 0. meta['flag_zem'] = np.string_('DESI') meta['STYPE'] = np.string_('QSO') # Observation meta['SPEC_FILE'] = np.array(spectest1.filename, dtype=float) meta['DATE-OBS'] = spectest1.date # meta['GROUP_ID'] = np.arange(len(meta)).astype(int) # Spectrograph meta['R'] = 3000. meta['TELESCOPE'] = np.string_('KPNO-4m') meta['DISPERSER'] = np.string_('ALL') meta['INSTR'] = np.string_('DESI') meta['WV_MIN'] = 3800. # Should be the right value meta['WV_MAX'] = 9900. # Should be the right value meta['NPIX'] = 8000 # Should be the right value meta['PLATE'] = np.array(np.tile(1, len(spectest1.id)), dtype=int) meta['FIBERID'] = spectest1.id meta['MOCK_ID'] = spectest1.id flag_g = spbu.add_to_group_dict(group, gdict) maindb = spbu.add_ids(maindb, meta, flag_g, tkeys, 'DESI_ID', first=(flag_g == flag_g)) hdf[group]['meta'] = meta zpri = spb_defs.z_priority() print(flag_g) spbu.write_hdf(hdf, str('DESI_v05'), maindb, zpri, gdict, str('v0.1'), Publisher='jding') hdf.close()
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, boss_hdf=None, **kwargs): """ Add BOSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write boss_hdf : str, optional Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if boss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) boss_hdf.copy(sname, hdf) return boss_grp = hdf.create_group(sname) # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4650 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] speclist = [] npixlist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): # Generate full file full_file = get_specfil(row) if full_file == 'None': continue # Read spec = lsio.readspec(full_file) # npix npix = spec.npix # Kludge for higest redshift systems if npix < 10: full_file = get_specfil(row, hiz=True) try: spec = lsio.readspec(full_file) except: print("Missing: {:s}".format(full_file)) npix = spec.npix elif npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Parse name fname = full_file.split('/')[-1] # Fill for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # GZ Continuum -- packed in with spectrum, generated by my IDL script try: co = spec.co.value except AttributeError: co = np.zeros_like(spec.flux.value) # KG Continuum KG_file = get_specfil(row, KG=True) if os.path.isfile(KG_file) and (npix>1): # Latter is for junk in GZ file. Needs fixing hduKG = fits.open(KG_file) KGtbl = hduKG[1].data wvKG = 10.**KGtbl['LOGLAM'] try: assert (wvKG[0]-spec.wavelength[0].value) < 1e-5 except: pdb.set_trace() gdpix = np.where(wvKG < (1+row['zem_GROUP'])*1200.)[0] co[gdpix] = KGtbl['CONT'][gdpix] data['co'][0][:npix] = co # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) meta.add_column(Column([2000.]*len(meta), name='EPOCH')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2015ApJS..219...12A', bib='boss_qso_dr12'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, sdss_hdf=None, **kwargs): """ Add SDSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if sdss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) sdss_hdf.copy(sname, hdf) return sdss_grp = hdf.create_group(sname) # Load up # Checks if sname != 'SDSS_DR7': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) # Read Zhu continua, wave file cfile = os.getenv( 'RAW_IGMSPEC') + '/SDSS/ALLQSO_SPEC_106_continuum_nointerp.fits' zhu_conti = Table.read(cfile) wvfile = cfile.replace('continuum', 'wave') zhu_wave = Table.read(wvfile) # wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): full_file = get_specfil(row) if not os.path.isfile(full_file): full_file = get_specfil(row, dr7=True) # Extract #print("SDSS: Reading {:s}".format(full_file)) # Parse name fname = full_file.split('/')[-1] # Generate full file spec = lsio.readspec(full_file) # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig', 'co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Continuum mtc = (zhu_conti['PLATE'] == row['PLATE']) & (zhu_conti['FIBER'] == row['FIBER']) mtw = (zhu_wave['PLATE'] == row['PLATE']) & (zhu_wave['FIBER'] == row['FIBER']) if np.sum(mtc) == 1: imin = np.argmin( np.abs(zhu_wave['WAVE'][0][:, np.where(mtw)[1]] - spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][ imin:npix + imin, np.where(mtc)[1]].flatten() elif np.sum(mtc) > 1: print( "Multiple continua for plate={:d}, row={:d}. Taking the first" .format(row['PLATE'], row['FIBER'])) imin = np.argmin( np.abs(zhu_wave['WAVE'][0][:, np.where(mtw)[1][0]] - spec.wavelength[0].value)) data['co'][0][:npix] = zhu_conti['CONTINUUM'][0][ imin:npix + imin, np.where(mtc)[1][0]].flatten() elif np.sum(mtc) == 0: print("No SDSS continuum for plate={:d}, row={:d}".format( row['PLATE'], row['FIBER'])) #from xastropy.xutils import xdebug as xdb #xdb.set_trace() #xdb.xplot(data['wave'][0], data['flux'][0], data['co'][0]) # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2010AJ....139.2360S', bib='sdss_qso_dr7'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append ESI data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) esidla_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'ESI_DLA': raise IOError("Expecting ESI_DLA!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 50000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): # specfile = os.getenv('RAW_IGMSPEC')+'/HighzESIDLA/{:s}a_xF.fits'.format( row['Name']) print("ESI_DLA: Reading {:s}".format(specfile)) spec = lsio.readspec(specfile) # Parse name fname = specfile.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Continuum # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value #data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(Rdicts['ESI'][head['SLMSKNAM']]) except KeyError: if row['Slit'] == 0.75: Rlist.append(Rdicts['ESI']['0.75_arcsec']) elif row['Slit'] == 0.5: Rlist.append(Rdicts['ESI']['0.50_arcsec']) else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2012ApJ...755...89R', bib='rafelski+12'), dict(url='http://adsabs.harvard.edu/abs/2014ApJ...782L..29R', bib='rafelski+14'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, boss_hdf=None, **kwargs): """ Add BOSS data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write boss_hdf : str, optional Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) if boss_hdf is not None: print("Using previously generated {:s} dataset...".format(sname)) boss_hdf.copy(sname, hdf) return boss_grp = hdf.create_group(sname) # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4660 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] speclist = [] npixlist = [] # Loop maxpix = 0 bad_spec = np.array([False] * len(meta)) for jj in range(len(meta)): # Generate full file full_file = get_specfil(jj, meta) if full_file == 'None': continue # Read try: spec = lsio.readspec(full_file, masking='edges') except: print("Failed on full_file: {:s}, {:d}".format(full_file, jj)) bad_spec[jj] = True continue # npix npix = spec.npix if npix < 10: print("Not enough pixels in file: {:s}, {:d}".format( full_file, jj)) bad_spec[jj] = True continue ''' # Kludge for higest redshift systems if npix < 10: full_file = get_specfil(jj, meta, hiz=True) try: spec = lsio.readspec(full_file) except: print("Missing: {:s}".format(full_file)) npix = spec.npix elif npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) ''' maxpix = max(npix, maxpix) # Parse name fname = full_file.split('/')[-1] # Fill for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value ''' # GZ Continuum -- packed in with spectrum, generated by my IDL script try: co = spec.co.value except AttributeError: co = np.zeros_like(spec.flux.value) # KG Continuum KG_file = get_specfil(row, KG=True) if os.path.isfile(KG_file) and (npix>1): # Latter is for junk in GZ file. Needs fixing hduKG = fits.open(KG_file) KGtbl = hduKG[1].data wvKG = 10.**KGtbl['LOGLAM'] try: assert (wvKG[0]-spec.wavelength[0].value) < 1e-5 except: pdb.set_trace() gdpix = np.where(wvKG < (1+row['zem_GROUP'])*1200.)[0] co[gdpix] = KGtbl['CONT'][gdpix] data['co'][0][:npix] = co ''' # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # Deal with null spec -- Should only be done once, saved and then ready to go if np.any(bad_spec): bad_meta = meta[bad_spec] pdb.set_trace() print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) meta.add_column(Column([2000.] * len(meta), name='EPOCH')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2015ApJS..219...12A', bib='boss_qso_dr12'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Add 2QZ data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) tqz_grp = hdf.create_group(sname) # Checks if sname != '2QZ': raise IOError("Not expecting this survey..") # Add zem # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): full_file = get_specfil(row) # Parse name fname = full_file.split('/')[-1] # Read hdu = fits.open(full_file) head0 = hdu[0].header wave = lsio.setwave(head0) flux = hdu[0].data var = hdu[2].data sig = np.zeros_like(flux) gd = var > 0. if np.sum(gd) == 0: print("{:s} has a bad var array. Not including".format(fname)) pdb.set_trace() continue sig[gd] = np.sqrt(var[gd]) # npix spec = XSpectrum1D.from_tuple((wave,flux,sig)) npix = spec.npix spec.meta['headers'][0] = head0 if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # # References refs = [dict(url='http://adsabs.harvard.edu/abs/2004MNRAS.349.1397C', bib='2QZ') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HD-LLS data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() mike_meta = grab_meta_mike() mike_coord = SkyCoord(ra=mike_meta['RA_GROUP'], dec=mike_meta['DEC_GROUP'], unit='deg') # Checks if sname != 'HD-LLS_DR1': raise IOError("Not expecting this survey..") full_coord = SkyCoord(ra=meta['RA_GROUP'], dec=meta['DEC_GROUP'], unit='deg') # Build spectra (and parse for meta) if mk_test_file: meta = meta[0:3] nspec = len(meta) max_npix = 210000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init full_idx = np.zeros(len(meta), dtype=int) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] instrlist = [] gratinglist = [] telelist = [] # Loop members = glob.glob(os.getenv('RAW_IGMSPEC') + '/{:s}/*fits'.format(sname)) kk = -1 for jj, member in enumerate(members): if 'HD-LLS_DR1.fits' in member: continue kk += 1 # Extract f = member hdu = fits.open(f) # Parse name fname = f.split('/')[-1] mt = np.where(meta['SPEC_FILE'] == fname)[0] if mk_test_file and (jj >= 3): continue if len(mt) != 1: pdb.set_trace() raise ValueError("HD-LLS: No match to spectral file?!") else: print('loading {:s}'.format(fname)) full_idx[kk] = mt[0] # npix head = hdu[0].header # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) # Double check if kk == 0: assert hdu[1].name == 'ERROR' assert hdu[2].name == 'WAVELENGTH' # Write spec = lsio.readspec(f) # Handles dummy pixels in ESI npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value #data['flux'][0][:npix] = hdu[0].data #data['sig'][0][:npix] = hdu[1].data #data['wave'][0][:npix] = hdu[2].data # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if 'HIRES' in fname: instrlist.append('HIRES') telelist.append('Keck-I') gratinglist.append('BOTH') try: Rlist.append(set_resolution(head)) except ValueError: # A few by hand (pulled from Table 1) if 'J073149' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-04', '%Y-%m-%d') elif 'J081435' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2006-12-26', '%Y-%m-%d') # 2008 too elif 'J095309' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2005-03-18', '%Y-%m-%d') elif 'J113418' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-05', '%Y-%m-%d') elif 'J135706' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2007-04-28', '%Y-%m-%d') elif 'J155556.9' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2005-04-15', '%Y-%m-%d') elif 'J212329' in fname: Rlist.append(Rdicts['HIRES']['E3']) tval = datetime.datetime.strptime('2006-08-20', '%Y-%m-%d') else: pdb.set_trace() else: tval = datetime.datetime.strptime(head['DATE-OBS'], '%Y-%m-%d') dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) elif 'ESI' in fname: instrlist.append('ESI') telelist.append('Keck-II') gratinglist.append('ECH') try: Rlist.append(set_resolution(head)) except ValueError: print("Using R=6,000 for ESI") Rlist.append(6000.) try: tval = datetime.datetime.strptime(head['DATE'], '%Y-%m-%d') except KeyError: if ('J223438.5' in fname) or ('J231543' in fname): tval = datetime.datetime.strptime('2004-09-11', '%Y-%m-%d') else: pdb.set_trace() dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) elif 'MIKE' in fname: # APPROXIMATE if 'MIKEr' in fname: instrlist.append('MIKEr') gratinglist.append('RED') elif 'MIKEb' in fname: instrlist.append('MIKEb') gratinglist.append('BLUE') else: instrlist.append('MIKE') gratinglist.append('BOTH') telelist.append('Magellan') sep = full_coord[mt[0]].separation(mike_coord) imin = np.argmin(sep) if sep[imin] > 1. * u.arcsec: pdb.set_trace() raise ValueError("Bad separation in MIKE") # R and Date Rlist.append(25000. / mike_meta['Slit'][imin]) tval = datetime.datetime.strptime(mike_meta['DATE-OBS'][imin], '%Y-%b-%d') dateobslist.append(datetime.datetime.strftime(tval, '%Y-%m-%d')) elif 'MAGE' in fname: # APPROXIMATE instrlist.append('MagE') if 'Clay' in head['TELESCOP']: telelist.append('Magellan/Clay') else: telelist.append('Magellan/Baade') gratinglist.append('N/A') Rlist.append(set_resolution(head)) dateobslist.append(head['DATE-OBS']) else: # MagE raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns meta = meta[full_idx] nmeta = len(meta) meta.add_column(Column([2000.] * nmeta, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column( Column([str(date) for date in dateobslist], name='DATE-OBS')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nmeta, dtype=int), name='GROUP_ID')) meta.add_column(Column(gratinglist, name='GRATING')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(telelist, name='TELESCOPE')) # v02 meta.rename_column('GRATING', 'DISPERSER') # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2015ApJS..221....2P', bib='prochaska+15'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HSTQSO data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) hstz2_grp = hdf.create_group(sname) # Checks if sname != 'HSTQSO': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 80000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] Rlist = [] # Loop #path = os.getenv('RAW_IGMSPEC')+'/KODIAQ_data_20150421/' path = os.getenv('RAW_IGMSPEC') + '/HSTQSO/' maxpix = 0 for jj, row in enumerate(meta): # Generate full file full_file = path + row['SPEC_FILE'] + '.gz' # Extract print("HSTQSO: Reading {:s}".format(full_file)) hduf = fits.open(full_file) head0 = hduf[0].header spec = lsio.readspec(full_file, masking='edges') # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta if 'FOS-L' in fname: Rlist.append(300.) elif 'FOS-H' in fname: Rlist.append(14000.) elif 'STIS' in fname: if row['DISPERSER'] == 'G230L': Rlist.append(700.) elif row['DISPERSER'] == 'G140L': Rlist.append(1200.) else: raise ValueError("Bad STIS grating") elif 'hsla' in fname: # COS Rlist.append(18000.) row['DATE-OBS'] = hduf[1].data['DATEOBS'][0][0] else: pdb.set_trace() raise ValueError("Missing instrument!") wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column([2000.] * nspec, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2011ApJ...736...42R', bib='ribuado11'), dict(url='http://adsabs.harvard.edu/abs/2016ApJ...818..113N', bib='neeleman16'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append ESI data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) esidla_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'ESI_DLA': raise IOError("Expecting ESI_DLA!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 50000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): # specfile = os.getenv( 'RAW_IGMSPEC') + '/HighzESIDLA/{:s}a_xF.fits'.format(row['Name']) print("ESI_DLA: Reading {:s}".format(specfile)) spec = lsio.readspec(specfile) # Parse name fname = specfile.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Continuum # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value #data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(Rdicts['ESI'][head['SLMSKNAM']]) except KeyError: if row['Slit'] == 0.75: Rlist.append(Rdicts['ESI']['0.75_arcsec']) elif row['Slit'] == 0.5: Rlist.append(Rdicts['ESI']['0.50_arcsec']) else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2012ApJ...755...89R', bib='rafelski+12'), dict(url='http://adsabs.harvard.edu/abs/2014ApJ...782L..29R', bib='rafelski+14'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, musodla_meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append MUSoDLA data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb.build.utils import chk_meta # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Checks if sname != 'MUSoDLA': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) max_npix = 230000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') nspec = len(musodla_meta) spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] # Loop for jj, row in enumerate(musodla_meta): kk = jj # Extract f = os.getenv('RAW_IGMSPEC') + '/MUSoDLA/data/' + row['SPEC_FILE'] try: spec = lsio.readspec(f, masking='edges') except: pdb.set_trace() # Parse name fname = f.split('/')[-1] # npix head = spec.header npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value if 'MagE' in f: if fname in ['J2122-0014_MagE.ascii', 'J0011+1446_MagE.ascii']: data['sig'][0][:npix] = spec.sig.value # Special cases.. else: data['sig'][0][:npix] = 1. / np.sqrt(spec.sig.value) # IVAR else: data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns nmeta = len(musodla_meta) musodla_meta.add_column(Column([2000.] * nmeta, name='EPOCH')) musodla_meta.add_column(Column(npixlist, name='NPIX')) musodla_meta.add_column(Column(wvminlist, name='WV_MIN')) musodla_meta.add_column(Column(wvmaxlist, name='WV_MAX')) musodla_meta.add_column( Column(np.arange(nmeta, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(musodla_meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = musodla_meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2013MNRAS.435..482J', bib='regina+13'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HD-LLS data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() mike_meta = grab_meta_mike() mike_coord = SkyCoord(ra=mike_meta['RA_GROUP'], dec=mike_meta['DEC_GROUP'], unit='deg') # Checks if sname != 'HD-LLS_DR1': raise IOError("Not expecting this survey..") full_coord = SkyCoord(ra=meta['RA_GROUP'], dec=meta['DEC_GROUP'], unit='deg') # Build spectra (and parse for meta) if mk_test_file: meta = meta[0:3] nspec = len(meta) max_npix = 210000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init full_idx = np.zeros(len(meta), dtype=int) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] instrlist = [] gratinglist = [] telelist = [] # Loop members = glob.glob(os.getenv('RAW_IGMSPEC')+'/{:s}/*fits'.format(sname)) kk = -1 for jj,member in enumerate(members): if 'HD-LLS_DR1.fits' in member: continue kk += 1 # Extract f = member hdu = fits.open(f) # Parse name fname = f.split('/')[-1] mt = np.where(meta['SPEC_FILE'] == fname)[0] if mk_test_file and (jj>=3): continue if len(mt) != 1: pdb.set_trace() raise ValueError("HD-LLS: No match to spectral file?!") else: print('loading {:s}'.format(fname)) full_idx[kk] = mt[0] # npix head = hdu[0].header # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) # Double check if kk == 0: assert hdu[1].name == 'ERROR' assert hdu[2].name == 'WAVELENGTH' # Write spec = lsio.readspec(f) # Handles dummy pixels in ESI npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value #data['flux'][0][:npix] = hdu[0].data #data['sig'][0][:npix] = hdu[1].data #data['wave'][0][:npix] = hdu[2].data # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if 'HIRES' in fname: instrlist.append('HIRES') telelist.append('Keck-I') gratinglist.append('BOTH') try: Rlist.append(set_resolution(head)) except ValueError: # A few by hand (pulled from Table 1) if 'J073149' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-04', '%Y-%m-%d') elif 'J081435' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2006-12-26', '%Y-%m-%d') # 2008 too elif 'J095309' in fname: Rlist.append(Rdicts['HIRES']['C1']) tval = datetime.datetime.strptime('2005-03-18', '%Y-%m-%d') elif 'J113418' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2006-01-05', '%Y-%m-%d') elif 'J135706' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2007-04-28', '%Y-%m-%d') elif 'J155556.9' in fname: Rlist.append(Rdicts['HIRES']['C5']) tval = datetime.datetime.strptime('2005-04-15', '%Y-%m-%d') elif 'J212329' in fname: Rlist.append(Rdicts['HIRES']['E3']) tval = datetime.datetime.strptime('2006-08-20', '%Y-%m-%d') else: pdb.set_trace() else: tval = datetime.datetime.strptime(head['DATE-OBS'], '%Y-%m-%d') dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) elif 'ESI' in fname: instrlist.append('ESI') telelist.append('Keck-II') gratinglist.append('ECH') try: Rlist.append(set_resolution(head)) except ValueError: print("Using R=6,000 for ESI") Rlist.append(6000.) try: tval = datetime.datetime.strptime(head['DATE'], '%Y-%m-%d') except KeyError: if ('J223438.5' in fname) or ('J231543' in fname): tval = datetime.datetime.strptime('2004-09-11', '%Y-%m-%d') else: pdb.set_trace() dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) elif 'MIKE' in fname: # APPROXIMATE if 'MIKEr' in fname: instrlist.append('MIKEr') gratinglist.append('RED') elif 'MIKEb' in fname: instrlist.append('MIKEb') gratinglist.append('BLUE') else: instrlist.append('MIKE') gratinglist.append('BOTH') telelist.append('Magellan') sep = full_coord[mt[0]].separation(mike_coord) imin = np.argmin(sep) if sep[imin] > 1.*u.arcsec: pdb.set_trace() raise ValueError("Bad separation in MIKE") # R and Date Rlist.append(25000. / mike_meta['Slit'][imin]) tval = datetime.datetime.strptime(mike_meta['DATE-OBS'][imin], '%Y-%b-%d') dateobslist.append(datetime.datetime.strftime(tval,'%Y-%m-%d')) elif 'MAGE' in fname: # APPROXIMATE instrlist.append('MagE') if 'Clay' in head['TELESCOP']: telelist.append('Magellan/Clay') else: telelist.append('Magellan/Baade') gratinglist.append('N/A') Rlist.append(set_resolution(head)) dateobslist.append(head['DATE-OBS']) else: # MagE raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns meta = meta[full_idx] nmeta = len(meta) meta.add_column(Column([2000.]*nmeta, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column([str(date) for date in dateobslist], name='DATE-OBS')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nmeta,dtype=int),name='GROUP_ID')) meta.add_column(Column(gratinglist, name='GRATING')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(telelist, name='TELESCOPE')) # v02 meta.rename_column('GRATING', 'DISPERSER') # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2015ApJS..221....2P', bib='prochaska+15'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append KODIAQ data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) kodiaq_grp = hdf.create_group(sname) # Load up # Checks if sname != 'KODIAQ_DR2': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 60000 # Just needs to be large enough # Init data = init_data(max_npix, include_co=False) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) # Lists Rlist = [] wvminlist = [] wvmaxlist = [] gratinglist = [] npixlist = [] speclist = [] # Loop path = os.getenv('RAW_IGMSPEC')+'/KODIAQ2/Data/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file full_file = path+row['qso']+'/'+row['pi_date']+'/'+row['spec_prefix']+'_f.fits' # Extract print("KODIAQ: Reading {:s}".format(full_file)) hduf = fits.open(full_file) head = hduf[0].header spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) if 'XDISPERS' in head.keys(): if head['XDISPERS'].strip() == 'UV': gratinglist.append('BLUE') else: gratinglist.append('RED') else: # Original, earl data gratinglist.append('RED') npixlist.append(npix) try: Rlist.append(set_resolution(head)) except ValueError: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column([2000.]*nspec, name='EPOCH')) meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2017AJ....154..114O', bib='kodiaq') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, hdla100_meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HDLA100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) hdlls_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'HDLA100': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) #if mk_test_file: # hdla100_full = hdlls_full[0:3] max_npix = 192000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') nspec = len(hdla100_meta) spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] dateobslist = [] npixlist = [] gratinglist = [] # Loop for jj, row in enumerate(hdla100_meta): kk = jj # Extract f = os.getenv('RAW_IGMSPEC') + '/HDLA100/' + row['SPEC_FILE'] spec = lsio.readspec(f) # Parse name fname = f.split('/')[-1] # npix head = spec.header npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) try: Rlist.append(set_resolution(head)) except ValueError: raise ValueError("Header is required for {:s}".format(fname)) else: if '/' in head['DATE-OBS']: spl = head['DATE-OBS'].split('/') t = Time(datetime.datetime( int(spl[2]) + 1900, int(spl[1]), int(spl[0])), format='datetime') else: t = Time(head['DATE-OBS'], format='isot', out_subfmt='date') dateobslist.append(t.iso) # Grating try: gratinglist.append(head['XDISPERS']) except KeyError: try: yr = t.value.year except AttributeError: yr = int(t.value[0:4]) if yr <= 1997: gratinglist.append('RED') else: pdb.set_trace() # Only way to set the dataset correctly if chk_meta_only: continue spec_set[kk] = data # Add columns nmeta = len(hdla100_meta) hdla100_meta.add_column(Column([2000.] * nmeta, name='EPOCH')) hdla100_meta.add_column(Column(npixlist, name='NPIX')) hdla100_meta.add_column( Column([str(date) for date in dateobslist], name='DATE-OBS')) hdla100_meta.add_column(Column(wvminlist, name='WV_MIN')) hdla100_meta.add_column(Column(wvmaxlist, name='WV_MAX')) hdla100_meta.add_column(Column(Rlist, name='R')) hdla100_meta.add_column( Column(np.arange(nmeta, dtype=int), name='GROUP_ID')) hdla100_meta.add_column(Column(gratinglist, name='DISPERSER')) hdla100_meta['INSTR'] = ['HIRES'] * nspec hdla100_meta['TELESCOPE'] = ['Keck-I'] * nspec #hdla100_meta.rename_column('Z_QSO', 'zem') # Add HDLLS meta to hdf5 if chk_meta(hdla100_meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = hdla100_meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2013ApJ...769...54N', bib='neeleman+13'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append COS-Halos data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) chalos_grp = hdf.create_group(sname) # Load up # Checks if sname != 'COS-Halos': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 160000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop path = os.getenv('RAW_IGMSPEC') + '/COS-Halos/' maxpix = 0 for jj, row in enumerate(meta): # Generate full file coord = ltu.radec_to_coord((row['RA_GROUP'], row['DEC_GROUP'])) if row['INSTR'].strip() == 'COS': full_file = path + '/J{:s}{:s}_nbin3_coadd.fits.gz'.format( coord.ra.to_string(unit=u.hour, sep='', pad=True)[0:4], coord.dec.to_string(sep='', pad=True, alwayssign=True)[0:5]) else: # HIRES full_file = path + '/HIRES/J{:s}{:s}_f.fits.gz'.format( coord.ra.to_string(unit=u.hour, sep='', pad=True)[0:4], coord.dec.to_string(sep='', pad=True, alwayssign=True)[0:5]) # Extract print("COS-Halos: Reading {:s}".format(full_file)) spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2013ApJ...777...59T', bib='tumlinson+13'), dict(url='http://adsabs.harvard.edu/abs/2013ApJS..204...17W', bib='werk+13') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Add 2QZ data to the DB Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) tqz_grp = hdf.create_group(sname) # Checks if sname != '2QZ': raise IOError("Not expecting this survey..") # Add zem # Build spectra (and parse for meta) nspec = len(meta) max_npix = 4000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): full_file = get_specfil(row) # Parse name fname = full_file.split('/')[-1] # Read hdu = fits.open(full_file) head0 = hdu[0].header wave = lsio.setwave(head0) flux = hdu[0].data var = hdu[2].data sig = np.zeros_like(flux) gd = var > 0. if np.sum(gd) == 0: print("{:s} has a bad var array. Not including".format(fname)) pdb.set_trace() continue sig[gd] = np.sqrt(var[gd]) # npix spec = XSpectrum1D.from_tuple((wave, flux, sig)) npix = spec.npix spec.meta['headers'][0] = head0 if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2004MNRAS.349.1397C', bib='2QZ') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append GGG data to the h5 file Parameters ---------- hdf : hdf5 pointer sname : str Survey name meta : Table chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) _ = hdf.create_group(sname) # Load up if sname != 'ESI_z6': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 30000 # Just needs to be large enough # Init data = init_data(max_npix, include_co=True) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] dateobslist = [] instrlist = [] # Loop maxpix = 0 for jj, row in enumerate(meta): # Generate full file full_file, spec = read_spec(row) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value data['co'][0][:npix] = spec.co.value # Meta #head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) #telelist.append(head['OBSERVAT']) #instrlist.append(head['INSTRUME']) #tval = Time(head['DATE'], format='isot', out_subfmt='date') #dateobslist.append(tval.iso) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) #meta.add_column(Column(telelist, name='TELESCOPE')) #meta.add_column(Column(instrlist, name='INSTR')) #meta.add_column(Column(dateobslist, name='DATE-OBS')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) #meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2014MNRAS.445.1745W', bib='worseck+14') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HST/FUSE data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ Rdicts = defs.get_res_dicts() # Add Survey print("Adding {:s} survey to DB".format(sname)) hstc_grp = hdf.create_group(sname) # Checks if sname != 'UVpSM4': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 40000 # Just needs to be large enough # Init data = init_data(max_npix, include_co=True) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] gratinglist = [] datelist = [] badf = [] badstis = [] badghrs = [] # Loop path = os.getenv('RAW_IGMSPEC') + '/HST_Cooksey/' maxpix = 0 for jj, row in enumerate(meta): # Generate full file full_file = path + '{:s}/{:s}/{:s}'.format(row['QSO'], row['INSTR'], row['SPEC_FILE']) # Extract if row['INSTR'] == 'FUSE': hext = 1 else: hext = 0 print("HST_Cooksey: Reading {:s}".format(full_file)) try: spec = lsio.readspec(full_file, head_exten=hext, masking='edges') except: # BAD HEADER hdu = fits.open(full_file) head1 = hdu[1].header hdu[1].verify('fix') tbl = Table(hdu[1].data) spec = lsio.readspec(tbl, masking='edges') spec.meta['headers'][spec.select] = head1 # Continuum cfile = full_file.replace('.fits', '_c.fits') if os.path.isfile(cfile): # Watch that mask! gdp = ~spec.data['flux'][spec.select].mask spec.data['co'][spec.select][gdp] = ( fits.open(cfile)[0].data)[gdp] # npix npix = spec.npix if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Some fiddling about for key in ['wave', 'flux', 'sig', 'co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value if spec.co_is_set: try: data['co'][0][:npix] = spec.co.value except ValueError: pdb.set_trace() # Meta datet = None if row['INSTR'] == 'FUSE': if 'HISTORY' in spec.header.keys(): ncards = len(spec.header['HISTORY']) flg_H = True else: flg_H = False hdu = fits.open(full_file) head0 = hdu[0].header ncards = len(head0) # Is this a good one? if 'APER_ACT' in head0: pass else: # Need to fight harder for the header # Look for untrim untrim = full_file + '.untrim' if not os.path.isfile(untrim): pdb.set_trace() # Read hduu = fits.open(untrim) if 'PKS2005' in untrim: # One extra kludge.. head0 = hduu[1].header flg_H = True ncards = len(head0['HISTORY']) else: head0 = hduu[0].header ncards = len(head0) spec.meta['headers'][spec.select] = head0 # Read from history for ss in range(ncards): if flg_H: try: card = Header.fromstring(spec.header['HISTORY'][ss]) except: pdb.set_trace() try: ckey = list(card.keys())[0] except IndexError: continue else: card0 = card[0] else: ckey, card0 = list(spec.header.keys())[ss], spec.header[ss] # Parse if ckey == 'APERTURE': aper = card0 elif ckey == 'DETECTOR': det = card0 elif ckey == 'APER_ACT': # Extracted aperture ext_ap = card0 elif ckey == 'DATE': # Extracted aperture datet = card0 gratinglist.append(ext_ap + det) elif row['INSTR'] == 'STIS': try: datet = spec.header['DATE'] except KeyError: # handful of kludged coadds if 'HISTORY' not in spec.header.keys(): # Grab from the other extension, e.g. PKS0405 hdu = fits.open(full_file) head1 = hdu[1].header spec.meta['headers'][0] = head1 for ihist in spec.header['HISTORY']: if 'TDATEOBS' in ihist: idash = ihist.find('-') datet = ihist[idash - 4:idash + 6] # Grating from name i0 = full_file.rfind('_') i1 = full_file.rfind('.fits') gratinglist.append(full_file[i0 + 1:i1]) if datet is None: pdb.set_trace() else: gratinglist.append(spec.header['OPT_ELEM']) elif row['INSTR'] == 'GHRS': # Date try: tmp = spec.header['DATE-OBS'] except KeyError: # Pull header from parallel file iM = full_file.find('M_1') if iM <= 0: iM = full_file.find('L_1') ofile = full_file[:iM + 1] + '_F.fits' if not os.path.isfile(ofile): if 'NGC4151' in ofile: # Kludge ofile = ofile.replace('G160M', 'G160Mmd') elif 'PKS2155-304_GHRS_G140L' in ofile: # Kludge ofile = ofile.replace('G140L', 'G140Llo') elif 'PKS2155-304_GHRS_G160M' in ofile: # Kludge ofile = ofile.replace('G160M', 'G160Mmd') else: pdb.set_trace() hdu = fits.open(ofile) head0 = hdu[0].header spec.meta['headers'][spec.select] = head0 # Reformat prs = tmp.split('/') if prs[2][0] == '9': yr = '19' + prs[2] else: yr = '20' + prs[2] datet = yr + '-' + prs[1] + '-{:02d}'.format(int(prs[0])) # Grating gratinglist.append(spec.header['GRATING']) else: pdb.set_trace() if datet is None: try: datet = spec.header['DATE-OBS'] except KeyError: print("Missing Header for file: {:s}".format(full_file)) badf.append(full_file) datet = '9999-9-9' t = Time(datet, format='isot', out_subfmt='date') # Fixes to YYYY-MM-DD datelist.append(t.iso) try: Rlist.append(Rdicts[row['INSTR']][gratinglist[-1]]) except KeyError: print(gratinglist[-1]) pdb.set_trace() wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # if (len(badstis)) > 0: raise ValueError("Somehow have a bad STIS header..") if len(badf) > 0: print("We still have bad FUSE headers") pdb.set_trace() if len(badghrs) > 0: print("We still have bad GHRS headers") pdb.set_trace() print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(datelist, name='DATE-OBS')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: pdb.set_trace() raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2010ApJ...708..868C', bib='cooksey10') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append COS-Halos data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) chalos_grp = hdf.create_group(sname) # Load up # Checks if sname != 'COS-Halos': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 160000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] # Loop path = os.getenv('RAW_IGMSPEC')+'/COS-Halos/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file coord = ltu.radec_to_coord((row['RA_GROUP'],row['DEC_GROUP'])) if row['INSTR'].strip() == 'COS': full_file = path+'/J{:s}{:s}_nbin3_coadd.fits.gz'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True)[0:4], coord.dec.to_string(sep='',pad=True,alwayssign=True)[0:5]) else: # HIRES full_file = path+'/HIRES/J{:s}{:s}_f.fits.gz'.format(coord.ra.to_string(unit=u.hour,sep='',pad=True)[0:4], coord.dec.to_string(sep='',pad=True,alwayssign=True)[0:5]) # Extract print("COS-Halos: Reading {:s}".format(full_file)) spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2013ApJ...777...59T', bib='tumlinson+13'), dict(url='http://adsabs.harvard.edu/abs/2013ApJS..204...17W', bib='werk+13') ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append UVES_Dall data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) uvesdall_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'UVES_Dall': raise IOError("Expecting UVES_Dall!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 150000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist, wvmaxlist, npixlist, speclist = [], [], [], [] # Loop maxpix = 0 for jj,row in enumerate(meta): # Read specfile = os.getenv('RAW_IGMSPEC')+'/UVES_Dall/{:s}_flux.dat'.format(row['NAME']) print("UVES_Dall: Reading {:s}".format(specfile)) spec = Table.read(specfile,format='ascii.fast_no_header',guess=False)#, data_start=1) # Parse name fname = specfile.split('/')[-1] # npix npix = len(spec['col1']) if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Continuum # Some fiddling about for key in ['wave','flux','sig','co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec['col2'] data['sig'][0][:npix] = spec['col3'] data['wave'][0][:npix] = spec['col1'] data['co'][0][:npix] = spec['col4'] # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2008A%26A...491..465D', bib='dallaglio+08'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append XQ-100 data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) xq100_grp = hdf.create_group(sname) if len(meta) != 300: pdb.set_trace() # Checks if sname != 'XQ-100': raise IOError("Expecting XQ-100!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 20000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] instrlist = [] # Loop maxpix = 0 for jj,row in enumerate(meta): # print("XQ-100: Reading {:s}".format(row['SPEC_FILE'])) spec = lsio.readspec(row['SPEC_FILE']) # Parse name fname = row['SPEC_FILE'].split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Continuum # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.to('AA').value data['co'][0][:npix] = spec.co.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) telelist.append(head['TELESCOP']) instrlist.append(head['INSTRUME']) gratinglist.append(head['DISPELEM']) npixlist.append(npix) if gratinglist[-1] == 'NIR': # From Lopez+16 Rlist.append(4350.) elif gratinglist[-1] == 'VIS': Rlist.append(7450.) elif gratinglist[-1] == 'UVB': Rlist.append(5300.) else: pdb.set_trace() raise ValueError("UH OH") # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(gratinglist, name='DISPERSER')) meta.add_column(Column(telelist, name='TELESCOPE')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2016arXiv160708776L', bib='lopez+16')] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append GGG data to the h5 file Parameters ---------- hdf : hdf5 pointer sname : str Survey name meta : Table chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) ggg_grp = hdf.create_group(sname) # Load up if sname != 'GGG': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 1600 # Just needs to be large enough # Init data = init_data(max_npix, include_co=False) spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) Rlist = [] wvminlist = [] wvmaxlist = [] npixlist = [] speclist = [] gratinglist = [] telelist = [] dateobslist = [] instrlist = [] # Loop path = os.getenv('RAW_IGMSPEC')+'/GGG/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file if jj >= nspec//2: full_file = path+row['name']+'_R400.fits.gz' gratinglist.append('R400') else: full_file = path+row['name']+'_B600.fits.gz' gratinglist.append('B600') # Extract print("GGG: Reading {:s}".format(full_file)) spec = lsio.readspec(full_file) # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta head = spec.header speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) telelist.append(head['OBSERVAT']) instrlist.append(head['INSTRUME']) tval = Time(head['DATE'], format='isot', out_subfmt='date') dateobslist.append(tval.iso) npixlist.append(npix) if 'R400' in fname: Rlist.append(833.) else: Rlist.append(940.) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(gratinglist, name='GRATING')) meta.add_column(Column(telelist, name='TELESCOPE')) meta.add_column(Column(instrlist, name='INSTR')) meta.add_column(Column(dateobslist, name='DATE-OBS')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2014MNRAS.445.1745W', bib='worseck+14')] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False, mk_test_file=False): """ Append HSTQSO data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write mk_test_file : bool, optional Generate the debug test file for Travis?? Returns ------- """ # Add Survey print("Adding {:s} survey to DB".format(sname)) hstz2_grp = hdf.create_group(sname) # Checks if sname != 'HSTQSO': raise IOError("Not expecting this survey..") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 80000 # Just needs to be large enough data = init_data(max_npix, include_co=False) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None,), compression='gzip') spec_set.resize((nspec,)) wvminlist = [] wvmaxlist = [] npixlist = [] Rlist = [] # Loop #path = os.getenv('RAW_IGMSPEC')+'/KODIAQ_data_20150421/' path = os.getenv('RAW_IGMSPEC')+'/HSTQSO/' maxpix = 0 for jj,row in enumerate(meta): # Generate full file full_file = path+row['SPEC_FILE']+'.gz' # Extract print("HSTQSO: Reading {:s}".format(full_file)) hduf = fits.open(full_file) head0 = hduf[0].header spec = lsio.readspec(full_file, masking='edges') # Parse name fname = full_file.split('/')[-1] # npix npix = spec.npix if npix > max_npix: raise ValueError("Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix,maxpix) # Some fiddling about for key in ['wave','flux','sig']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec.flux.value data['sig'][0][:npix] = spec.sig.value data['wave'][0][:npix] = spec.wavelength.value # Meta if 'FOS-L' in fname: Rlist.append(300.) elif 'FOS-H' in fname: Rlist.append(14000.) elif 'STIS' in fname: if row['DISPERSER'] == 'G230L': Rlist.append(700.) elif row['DISPERSER'] == 'G140L': Rlist.append(1200.) else: raise ValueError("Bad STIS grating") elif 'hsla' in fname: # COS Rlist.append(18000.) row['DATE-OBS'] = hduf[1].data['DATEOBS'][0][0] else: pdb.set_trace() raise ValueError("Missing instrument!") wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) if chk_meta_only: continue # Only way to set the dataset correctly spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column([2000.]*nspec, name='EPOCH')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(Rlist, name='R')) meta.add_column(Column(np.arange(nspec,dtype=int),name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [dict(url='http://adsabs.harvard.edu/abs/2011ApJ...736...42R', bib='ribuado11'), dict(url='http://adsabs.harvard.edu/abs/2016ApJ...818..113N', bib='neeleman16'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return
def hdf5_adddata(hdf, sname, meta, debug=False, chk_meta_only=False): """ Append UVES_Dall data to the h5 file Parameters ---------- hdf : hdf5 pointer IDs : ndarray int array of IGM_ID values in mainDB sname : str Survey name chk_meta_only : bool, optional Only check meta file; will not write Returns ------- """ from specdb import defs # Add Survey print("Adding {:s} survey to DB".format(sname)) uvesdall_grp = hdf.create_group(sname) # Load up Rdicts = defs.get_res_dicts() # Checks if sname != 'UVES_Dall': raise IOError("Expecting UVES_Dall!!") # Build spectra (and parse for meta) nspec = len(meta) max_npix = 150000 # Just needs to be large enough data = init_data(max_npix, include_co=True) # Init spec_set = hdf[sname].create_dataset('spec', data=data, chunks=True, maxshape=(None, ), compression='gzip') spec_set.resize((nspec, )) wvminlist, wvmaxlist, npixlist, speclist = [], [], [], [] # Loop maxpix = 0 for jj, row in enumerate(meta): # Read specfile = os.getenv( 'RAW_IGMSPEC') + '/UVES_Dall/{:s}_flux.dat'.format(row['NAME']) print("UVES_Dall: Reading {:s}".format(specfile)) spec = Table.read(specfile, format='ascii.fast_no_header', guess=False) #, data_start=1) # Parse name fname = specfile.split('/')[-1] # npix npix = len(spec['col1']) if npix > max_npix: raise ValueError( "Not enough pixels in the data... ({:d})".format(npix)) else: maxpix = max(npix, maxpix) # Continuum # Some fiddling about for key in ['wave', 'flux', 'sig', 'co']: data[key] = 0. # Important to init (for compression too) data['flux'][0][:npix] = spec['col2'] data['sig'][0][:npix] = spec['col3'] data['wave'][0][:npix] = spec['col1'] data['co'][0][:npix] = spec['col4'] # Meta speclist.append(str(fname)) wvminlist.append(np.min(data['wave'][0][:npix])) wvmaxlist.append(np.max(data['wave'][0][:npix])) npixlist.append(npix) # Only way to set the dataset correctly if chk_meta_only: continue spec_set[jj] = data # print("Max pix = {:d}".format(maxpix)) # Add columns meta.add_column(Column(speclist, name='SPEC_FILE')) meta.add_column(Column(npixlist, name='NPIX')) meta.add_column(Column(wvminlist, name='WV_MIN')) meta.add_column(Column(wvmaxlist, name='WV_MAX')) meta.add_column(Column(np.arange(nspec, dtype=int), name='GROUP_ID')) # Add HDLLS meta to hdf5 if chk_meta(meta): if chk_meta_only: pdb.set_trace() hdf[sname]['meta'] = meta else: raise ValueError("meta file failed") # References refs = [ dict(url='http://adsabs.harvard.edu/abs/2008A%26A...491..465D', bib='dallaglio+08'), ] jrefs = ltu.jsonify(refs) hdf[sname]['meta'].attrs['Refs'] = json.dumps(jrefs) # return