def test_meta():
ztbl = Table.read(os.path.join(os.path.dirname(__file__), 'files', 'ztbl_E.fits'))
data_dir = os.path.join(os.path.dirname(__file__), 'files')
ffiles,_ = pbuild.grab_files(data_dir)
meta = pbuild.mk_meta(ffiles, ztbl, fname=True, skip_badz=True, mdict=dict(INSTR='HIRES'))
#
np.testing.assert_allclose(meta['zem_GROUP'].data, (2.39499998093, 2.59719920158))
def test_ingest():
# Begin
id_key = 'TEST_ID'
maindb, tkeys = spbu.start_maindb(id_key)
#
ztbl = Table.read(os.path.join(os.path.dirname(__file__), 'files', 'ztbl_E.fits'))
data_dir = os.path.join(os.path.dirname(__file__), 'files')
ffiles,_ = pbuild.grab_files(data_dir)
meta = pbuild.mk_meta(ffiles, ztbl, fname=True, skip_badz=True, mdict=dict(INSTR='HIRES'))
# Group and IDs
gdict = {}
flag_g = spbu.add_to_group_dict('COS', gdict)
maindb = pbuild.add_ids(maindb, meta, flag_g, tkeys, id_key, first=(flag_g==1))
#
hdf = h5py.File('tmp.hdf5','w')
pbuild.ingest_spectra(hdf, 'test', meta)
hdf.close()
# Read
tmp = h5py.File('tmp.hdf5','r')
# Test
assert 'meta' in tmp['test'].keys()
assert isinstance(tmp['test/spec'].value, np.ndarray)
def generate_by_refs(input_refs, outfile, version):
"""
Build a specDB file according to the input references
Args:
input_refs (list):
List of references from which to build the specDB
outfile (str):
Output filename
version (str):
Version number
"""
# Not elegant but it works
all_folders = glob.glob(db_path+'/*/*')
all_refs = [os.path.basename(ifolder) for ifolder in all_folders]
# z_tbl
allz_tbl = Table()
# Loop in input refs
all_spec_files = []
refs_list = []
for ref in input_refs:
idx = all_refs.index(ref)
# Redshift tables
z_tbl = load_z_tables(all_folders[idx])
allz_tbl = vstack([allz_tbl, z_tbl])
# Grab the list of spectra
specs = glob.glob(os.path.join(all_folders[idx], 'J*_spec.fits'))
if len(specs) == 0:
continue
# Save
all_spec_files += specs
refs_list += [ref]*len(specs)
# Get it started
# HDF5 file
hdf = h5py.File(outfile, 'w')
# Defs
zpri = defs.z_priority()
# Main DB Table
id_key = 'FRB_ID'
maindb, tkeys = spbu.start_maindb(id_key)
tkeys += ['ZQ']
gdict = {}
# Loop on Instruments
#pair_groups = ['MUSE']
pair_groups = []
badf = None
for instr in all_instruments:
print("Working on {}".format(instr))
fits_files, irefs = grab_files(all_spec_files, refs_list, instr)
if len(fits_files) == 0:
continue
# Option dicts
mwargs = {}
mwargs['toler'] = 1.0 * units.arcsec # Require an
skipz = False
swargs = {}
# Meta
parse_head, mdict, fname = None, None, True
if instr == 'SDSS':
mdict = dict(DISPERSER='BOTH', R=2000., TELESCOPE='SDSS 2.5-M', INSTR='SDSS')
parse_head = {'DATE-OBS': 'MJD'}
maxpix = 4000
scale = 1e-17
elif instr == 'FORS2':
mdict = dict(TELESCOPE='VLT', INSTR='FORS2')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'R': True}
maxpix = 2050
scale = 1e-17
elif instr == 'MUSE':
mdict = dict(TELESCOPE='VLT', R=2000.)
parse_head = {'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 4000
scale = 1e-20
elif instr == 'KCWI':
mdict = dict(TELESCOPE='Keck-2')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME', 'R': True}
maxpix = 4000
scale = 1e-17
elif instr == 'MagE':
parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'TELESCOPE': 'TELESCOP',
'INSTR': 'INSTRUME', 'DISPERSER': 'DISPNAME'}
maxpix = 18000
scale = 1e-17
elif instr == 'GMOS-S':
mdict = dict(TELESCOPE='Gemini-S', INSTR='GMOS-S')
parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME'}
maxpix = 3500
scale = 1e-17
elif instr == 'LRISb':
mdict = dict(TELESCOPE='Keck-1')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 2050 # 2x binning
elif instr == 'GMOS-N':
mdict = dict(TELESCOPE='Gemini-N', INSTR='GMOS-N')
parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS',
'DISPERSER': 'DISPNAME'}
maxpix = 3500
scale = 1e-17
elif instr == 'LRISr':
mdict = dict(TELESCOPE='Keck-1')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 2050
scale = 1e-17
elif instr == 'DEIMOS':
mdict = dict(TELESCOPE='Keck-2')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME',
'INSTR': 'INSTRUME'}
maxpix = 9000
scale = 1e-17
elif instr == 'Goodman':
mdict = dict(TELESCOPE='SOAR', INSTR='Goodman')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME',
'R': True}
maxpix = 2048
scale = 1e-17
elif instr == 'XSHOOTER':
mdict = dict(TELESCOPE='VLT')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 33000
scale = 1e-17
else:
embed(header='172')
# Meta
full_meta = pbuild.mk_meta(fits_files, allz_tbl, mdict=mdict, fname=fname,
verbose=True, parse_head=parse_head, skip_badz=skipz,
stype='GAL',
chkz=True, **mwargs)
full_meta['Ref'] = irefs
# Survey flag
flag_g = spbu.add_to_group_dict(instr, gdict, skip_for_debug=True)
# IDs
#if 'MUSE' in instr:
# embed(header='278 of build specdb')
maindb = spbu.add_ids(maindb, full_meta, flag_g, tkeys, id_key,
first=(flag_g==1),
mtch_toler=1.*units.arcsec,
close_pairs=(instr in pair_groups))
# Ingest --
pbuild.ingest_spectra(hdf, instr, full_meta, max_npix=maxpix, verbose=False,
badf=badf, grab_conti=False, scale=scale, **swargs)
# Write
spbu.write_hdf(hdf, str('FRB'), maindb, zpri, gdict, version, Publisher=str('JXP'))
print("Wrote {:s} DB file".format(outfile))
print("You probably need to move it into SPECDB")