def test_ingest(): # Begin id_key = 'TEST_ID' maindb, tkeys = spbu.start_maindb(id_key) # ztbl = Table.read(os.path.join(os.path.dirname(__file__), 'files', 'ztbl_E.fits')) data_dir = os.path.join(os.path.dirname(__file__), 'files') ffiles,_ = pbuild.grab_files(data_dir) meta = pbuild.mk_meta(ffiles, ztbl, fname=True, skip_badz=True, mdict=dict(INSTR='HIRES')) # Group and IDs gdict = {} flag_g = spbu.add_to_group_dict('COS', gdict) maindb = pbuild.add_ids(maindb, meta, flag_g, tkeys, id_key, first=(flag_g==1)) # hdf = h5py.File('tmp.hdf5','w') pbuild.ingest_spectra(hdf, 'test', meta) hdf.close() # Read tmp = h5py.File('tmp.hdf5','r') # Test assert 'meta' in tmp['test'].keys() assert isinstance(tmp['test/spec'].value, np.ndarray)
def generate_by_refs(input_refs, outfile, version): """ Build a specDB file according to the input references Args: input_refs (list): List of references from which to build the specDB outfile (str): Output filename version (str): Version number """ # Not elegant but it works all_folders = glob.glob(db_path+'/*/*') all_refs = [os.path.basename(ifolder) for ifolder in all_folders] # z_tbl allz_tbl = Table() # Loop in input refs all_spec_files = [] refs_list = [] for ref in input_refs: idx = all_refs.index(ref) # Redshift tables z_tbl = load_z_tables(all_folders[idx]) allz_tbl = vstack([allz_tbl, z_tbl]) # Grab the list of spectra specs = glob.glob(os.path.join(all_folders[idx], 'J*_spec.fits')) if len(specs) == 0: continue # Save all_spec_files += specs refs_list += [ref]*len(specs) # Get it started # HDF5 file hdf = h5py.File(outfile, 'w') # Defs zpri = defs.z_priority() # Main DB Table id_key = 'FRB_ID' maindb, tkeys = spbu.start_maindb(id_key) tkeys += ['ZQ'] gdict = {} # Loop on Instruments #pair_groups = ['MUSE'] pair_groups = [] badf = None for instr in all_instruments: print("Working on {}".format(instr)) fits_files, irefs = grab_files(all_spec_files, refs_list, instr) if len(fits_files) == 0: continue # Option dicts mwargs = {} mwargs['toler'] = 1.0 * units.arcsec # Require an skipz = False swargs = {} # Meta parse_head, mdict, fname = None, None, True if instr == 'SDSS': mdict = dict(DISPERSER='BOTH', R=2000., TELESCOPE='SDSS 2.5-M', INSTR='SDSS') parse_head = {'DATE-OBS': 'MJD'} maxpix = 4000 scale = 1e-17 elif instr == 'FORS2': mdict = dict(TELESCOPE='VLT', INSTR='FORS2') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'R': True} maxpix = 2050 scale = 1e-17 elif instr == 'MUSE': mdict = dict(TELESCOPE='VLT', R=2000.) parse_head = {'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'} maxpix = 4000 scale = 1e-20 elif instr == 'KCWI': mdict = dict(TELESCOPE='Keck-2') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME', 'R': True} maxpix = 4000 scale = 1e-17 elif instr == 'MagE': parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'TELESCOPE': 'TELESCOP', 'INSTR': 'INSTRUME', 'DISPERSER': 'DISPNAME'} maxpix = 18000 scale = 1e-17 elif instr == 'GMOS-S': mdict = dict(TELESCOPE='Gemini-S', INSTR='GMOS-S') parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME'} maxpix = 3500 scale = 1e-17 elif instr == 'LRISb': mdict = dict(TELESCOPE='Keck-1') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'} maxpix = 2050 # 2x binning elif instr == 'GMOS-N': mdict = dict(TELESCOPE='Gemini-N', INSTR='GMOS-N') parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME'} maxpix = 3500 scale = 1e-17 elif instr == 'LRISr': mdict = dict(TELESCOPE='Keck-1') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'} maxpix = 2050 scale = 1e-17 elif instr == 'DEIMOS': mdict = dict(TELESCOPE='Keck-2') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'} maxpix = 9000 scale = 1e-17 elif instr == 'Goodman': mdict = dict(TELESCOPE='SOAR', INSTR='Goodman') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'R': True} maxpix = 2048 scale = 1e-17 elif instr == 'XSHOOTER': mdict = dict(TELESCOPE='VLT') parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'} maxpix = 33000 scale = 1e-17 else: embed(header='172') # Meta full_meta = pbuild.mk_meta(fits_files, allz_tbl, mdict=mdict, fname=fname, verbose=True, parse_head=parse_head, skip_badz=skipz, stype='GAL', chkz=True, **mwargs) full_meta['Ref'] = irefs # Survey flag flag_g = spbu.add_to_group_dict(instr, gdict, skip_for_debug=True) # IDs #if 'MUSE' in instr: # embed(header='278 of build specdb') maindb = spbu.add_ids(maindb, full_meta, flag_g, tkeys, id_key, first=(flag_g==1), mtch_toler=1.*units.arcsec, close_pairs=(instr in pair_groups)) # Ingest -- pbuild.ingest_spectra(hdf, instr, full_meta, max_npix=maxpix, verbose=False, badf=badf, grab_conti=False, scale=scale, **swargs) # Write spbu.write_hdf(hdf, str('FRB'), maindb, zpri, gdict, version, Publisher=str('JXP')) print("Wrote {:s} DB file".format(outfile)) print("You probably need to move it into SPECDB")