def ver01(test=False, clobber=False, publisher='J.X. Prochaska', **kwargs):
""" Build version 1.0
Parameters
----------
test : bool, optional
Run test only
Returns
-------
"""
pdb.set_trace() # THIS VERSION IS NOW FROZEN
raise IOError("THIS VERSION IS NOW FROZEN")
version = 'v01'
# HDF5 file
outfil = igmspec.__path__[0] + '/../DB/IGMspec_DB_{:s}.hdf5'.format(
version)
# Chk clobber
if os.path.isfile(outfil):
if clobber:
warnings.warn("Overwriting previous DB file {:s}".format(outfil))
else:
warnings.warn(
"Not overwiting previous DB file. Use clobber=True to do so")
return
# Begin
hdf = h5py.File(outfil, 'w')
''' Myers QSOs '''
myers.orig_add_to_hdf(hdf)
# Main DB Table
idkey = 'IGM_ID'
maindb, tkeys = sdbbu.start_maindb(idkey)
# Group dict
group_dict = {}
# Organize for main loop
groups = get_build_groups(version)
pair_groups = ['SDSS_DR7']
meta_only = False
# Loop over the groups
for gname in groups:
# Meta
if gname == 'SDSS_DR7':
meta = groups[gname].grab_meta(hdf)
else:
meta = groups[gname].grab_meta()
# Survey flag
flag_g = sdbbu.add_to_group_dict(gname, group_dict)
# IDs
maindb = sdbbu.add_ids(maindb,
meta,
flag_g,
tkeys,
idkey,
first=(flag_g == 1),
close_pairs=(gname in pair_groups))
# Spectra
if not meta_only:
groups[gname].hdf5_adddata(hdf, gname, meta, idkey)
# Check for duplicates -- There is 1 pair in SDSS (i.e. 2 duplicates)
if not sdbbu.chk_for_duplicates(maindb, dup_lim=2):
raise ValueError("Failed duplicates")
# Check for junk
zpri = defs.z_priority()
# Finish
sdbbu.write_hdf(hdf,
str('igmspec'),
maindb,
zpri,
group_dict,
version,
Publisher=str(publisher))
print("Wrote {:s} DB file".format(outfil))
print("Update DB info in specdb.defs.dbase_info !!")
def hdf5_writter(path, reso, lowl, highl, outputname):
'''
:param path: path to the spectra file
:param lowl float: lower limit for request redshift
:param highl float: higher limit for request redshift
:param reso: delta v in km/s to rebin the data
:param outputname: the name for the output file
:return: hdf5file
'''
outfil = str(outputname)
hdf = h5py.File(outfil, 'w')
#
gdict = {}
# creating group
for z in np.arange(lowl, highl, 0.05):
# hdf.create_group('z'+str(z)+'-'+str(z+0.05))
print(z)
spectest1 = []
spec2 = []
spectest1, spec2 = reading_data(path, reso, z, z + 0.05)
print(np.round(z, 2), len(spectest1.z))
# hdf_append = h5py.File('tmp'+str(z)+'.hdf', 'w')
# hdf.create_group('z' + str(z) + '-' + str(z + 0.05))
f = hdf.create_group('z' + str(np.round(z, 2)) + '-' +
str(np.round(z + 0.05, 2)))
npix = len(spec2.wavelength)
data = sdb_u.init_data(npix)
nspec = len(spectest1.z)
print(npix)
#creat dataset
spec_set = hdf['z' + str(np.round(z, 2)) + '-' +
str(np.round(z + 0.05, 2))].create_dataset(
'spec',
data=data,
chunks=True,
maxshape=(None, ),
compression='gzip')
spec_set.resize((nspec, ))
for ii in range(nspec):
data['flux'][0][:npix] = spec2[ii].flux # Should be flux values
data['sig'][0][:npix] = spec2[ii].sig # SHould be sigma values
# print (spec[ii].sig)
data['wave'][0][:npix] = spec2[
ii].wavelength # Should be wavelength values
# Fill
spec_set[ii] = data
# hdf.copy('z'+str(z)+'-'+str(z+0.05), hdf_append['z'+str(z)+'-'+str(z+0.05)])
# making meta data
# hdfnew = h5py.File('z2.8_specdb_test1.hdf', 'w')
# group = 'z2.8-2.85'
# _ = hdfnew.create_group(group)
group = 'z' + str(np.round(z, 2)) + '-' + str(np.round(z + 0.05, 2))
id_key = 'DESI_ID'
maindb, tkeys = spbu.start_maindb(id_key)
meta = Table()
meta['zem_GROUP'] = spectest1.z
meta['RA_GROUP'] = spectest1.ra
meta['DEC_GROUP'] = spectest1.dec
meta['EPOCH'] = 2000.
meta['sig_zem'] = 0.
meta['flag_zem'] = np.string_('DESI')
meta['STYPE'] = np.string_('QSO')
# Observation
meta['SPEC_FILE'] = np.array(spectest1.filename, dtype=float)
meta['DATE-OBS'] = spectest1.date
#
meta['GROUP_ID'] = np.arange(len(meta)).astype(int)
# Spectrograph
meta['R'] = 3000.
meta['TELESCOPE'] = np.string_('KPNO-4m')
meta['DISPERSER'] = np.string_('ALL')
meta['INSTR'] = np.string_('DESI')
meta['WV_MIN'] = 3800. # Should be the right value
meta['WV_MAX'] = 9900. # Should be the right value
meta['NPIX'] = 8000 # Should be the right value
meta['PLATE'] = np.array(np.tile(1, len(spectest1.id)), dtype=int)
meta['FIBERID'] = spectest1.id
meta['MOCK_ID'] = spectest1.id
flag_g = spbu.add_to_group_dict(group, gdict)
maindb = spbu.add_ids(maindb,
meta,
flag_g,
tkeys,
'DESI_ID',
first=(flag_g == flag_g))
hdf[group]['meta'] = meta
zpri = spb_defs.z_priority()
print(flag_g)
spbu.write_hdf(hdf,
str('DESI_v05'),
maindb,
zpri,
gdict,
str('v0.1'),
Publisher='jding')
hdf.close()
def mk_db(dbname, tree, outfil, iztbl, version='v00', id_key='PRIV_ID',
publisher='Unknown', **kwargs):
""" Generate the DB
Parameters
----------
dbname : str
Name for the database
tree : str
Path to top level of the tree of FITS files
Typically, each branch in the tree corresponds to a single instrument
outfil : str
Output file name for the hdf5 file
iztbl : Table or str
If Table, see meta() docs for details on its format
If str, it must be 'igmspec' and the user must have that DB downloaded
version : str, optional
Version code
Returns
-------
"""
from specdb import defs
# ztbl
if isinstance(iztbl, str):
if iztbl == 'igmspec':
from specdb.specdb import IgmSpec
igmsp = IgmSpec()
ztbl = Table(igmsp.idb.hdf['quasars'][...])
elif isinstance(iztbl, Table):
ztbl = iztbl
else:
raise IOError("Bad type for ztbl")
# Find the branches
branches = glob.glob(tree+'/*')
branches.sort()
# HDF5 file
hdf = h5py.File(outfil,'w')
# Defs
zpri = defs.z_priority()
gdict = {}
# Main DB Table
maindb, tkeys = spbu.start_maindb(id_key)
# MAIN LOOP
for ss,branch in enumerate(branches):
# Skip files
if not os.path.isdir(branch):
continue
print('Working on branch: {:s}'.format(branch))
# Files
fits_files, out_tup = grab_files(branch)
meta_file, mtbl_file, ssa_file = out_tup
# Meta
maxpix, phead, mdict, stype = 10000, None, None, 'QSO'
if meta_file is not None:
# Load
meta_dict = ltu.loadjson(meta_file)
# Maxpix
if 'maxpix' in meta_dict.keys():
maxpix = meta_dict['maxpix']
# STYPE
if 'stype' in meta_dict.keys():
stype = meta_dict['stype']
# Parse header
if 'parse_head' in meta_dict.keys():
phead = meta_dict['parse_head']
if 'meta_dict' in meta_dict.keys():
mdict = meta_dict['meta_dict']
full_meta = mk_meta(fits_files, ztbl, mtbl_file=mtbl_file,
parse_head=phead, mdict=mdict, **kwargs)
# Update group dict
group_name = branch.split('/')[-1]
flag_g = spbu.add_to_group_dict(group_name, gdict)
# IDs
maindb = add_ids(maindb, full_meta, flag_g, tkeys, 'PRIV_ID', first=(flag_g==1))
# Ingest
ingest_spectra(hdf, group_name, full_meta, max_npix=maxpix, **kwargs)
# SSA
if ssa_file is not None:
user_ssa = ltu.loadjson(ssa_file)
ssa_dict = default_fields(user_ssa['Title'], flux=user_ssa['flux'], fxcalib=user_ssa['fxcalib'])
hdf[group_name]['meta'].attrs['SSA'] = json.dumps(ltu.jsonify(ssa_dict))
# Check stacking
if not spbu.chk_vstack(hdf):
print("Meta data will not stack using specdb.utils.clean_vstack")
print("Proceed to write at your own risk..")
pdb.set_trace()
# Write
write_hdf(hdf, str(dbname), maindb, zpri, gdict, version,
Publisher=publisher)
print("Wrote {:s} DB file".format(outfil))
def generate_by_refs(input_refs, outfile, version):
"""
Build a specDB file according to the input references
Args:
input_refs (list):
List of references from which to build the specDB
outfile (str):
Output filename
version (str):
Version number
"""
# Not elegant but it works
all_folders = glob.glob(db_path+'/*/*')
all_refs = [os.path.basename(ifolder) for ifolder in all_folders]
# z_tbl
allz_tbl = Table()
# Loop in input refs
all_spec_files = []
refs_list = []
for ref in input_refs:
idx = all_refs.index(ref)
# Redshift tables
z_tbl = load_z_tables(all_folders[idx])
allz_tbl = vstack([allz_tbl, z_tbl])
# Grab the list of spectra
specs = glob.glob(os.path.join(all_folders[idx], 'J*_spec.fits'))
if len(specs) == 0:
continue
# Save
all_spec_files += specs
refs_list += [ref]*len(specs)
# Get it started
# HDF5 file
hdf = h5py.File(outfile, 'w')
# Defs
zpri = defs.z_priority()
# Main DB Table
id_key = 'FRB_ID'
maindb, tkeys = spbu.start_maindb(id_key)
tkeys += ['ZQ']
gdict = {}
# Loop on Instruments
#pair_groups = ['MUSE']
pair_groups = []
badf = None
for instr in all_instruments:
print("Working on {}".format(instr))
fits_files, irefs = grab_files(all_spec_files, refs_list, instr)
if len(fits_files) == 0:
continue
# Option dicts
mwargs = {}
mwargs['toler'] = 1.0 * units.arcsec # Require an
skipz = False
swargs = {}
# Meta
parse_head, mdict, fname = None, None, True
if instr == 'SDSS':
mdict = dict(DISPERSER='BOTH', R=2000., TELESCOPE='SDSS 2.5-M', INSTR='SDSS')
parse_head = {'DATE-OBS': 'MJD'}
maxpix = 4000
scale = 1e-17
elif instr == 'FORS2':
mdict = dict(TELESCOPE='VLT', INSTR='FORS2')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'R': True}
maxpix = 2050
scale = 1e-17
elif instr == 'MUSE':
mdict = dict(TELESCOPE='VLT', R=2000.)
parse_head = {'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 4000
scale = 1e-20
elif instr == 'KCWI':
mdict = dict(TELESCOPE='Keck-2')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME', 'R': True}
maxpix = 4000
scale = 1e-17
elif instr == 'MagE':
parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'TELESCOPE': 'TELESCOP',
'INSTR': 'INSTRUME', 'DISPERSER': 'DISPNAME'}
maxpix = 18000
scale = 1e-17
elif instr == 'GMOS-S':
mdict = dict(TELESCOPE='Gemini-S', INSTR='GMOS-S')
parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS', 'DISPERSER': 'DISPNAME'}
maxpix = 3500
scale = 1e-17
elif instr == 'LRISb':
mdict = dict(TELESCOPE='Keck-1')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 2050 # 2x binning
elif instr == 'GMOS-N':
mdict = dict(TELESCOPE='Gemini-N', INSTR='GMOS-N')
parse_head = {'R': True, 'DATE-OBS': 'MJD-OBS',
'DISPERSER': 'DISPNAME'}
maxpix = 3500
scale = 1e-17
elif instr == 'LRISr':
mdict = dict(TELESCOPE='Keck-1')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 2050
scale = 1e-17
elif instr == 'DEIMOS':
mdict = dict(TELESCOPE='Keck-2')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME',
'INSTR': 'INSTRUME'}
maxpix = 9000
scale = 1e-17
elif instr == 'Goodman':
mdict = dict(TELESCOPE='SOAR', INSTR='Goodman')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME',
'R': True}
maxpix = 2048
scale = 1e-17
elif instr == 'XSHOOTER':
mdict = dict(TELESCOPE='VLT')
parse_head = {'DATE-OBS': 'MJD', 'DISPERSER': 'DISPNAME', 'INSTR': 'INSTRUME'}
maxpix = 33000
scale = 1e-17
else:
embed(header='172')
# Meta
full_meta = pbuild.mk_meta(fits_files, allz_tbl, mdict=mdict, fname=fname,
verbose=True, parse_head=parse_head, skip_badz=skipz,
stype='GAL',
chkz=True, **mwargs)
full_meta['Ref'] = irefs
# Survey flag
flag_g = spbu.add_to_group_dict(instr, gdict, skip_for_debug=True)
# IDs
#if 'MUSE' in instr:
# embed(header='278 of build specdb')
maindb = spbu.add_ids(maindb, full_meta, flag_g, tkeys, id_key,
first=(flag_g==1),
mtch_toler=1.*units.arcsec,
close_pairs=(instr in pair_groups))
# Ingest --
pbuild.ingest_spectra(hdf, instr, full_meta, max_npix=maxpix, verbose=False,
badf=badf, grab_conti=False, scale=scale, **swargs)
# Write
spbu.write_hdf(hdf, str('FRB'), maindb, zpri, gdict, version, Publisher=str('JXP'))
print("Wrote {:s} DB file".format(outfile))
print("You probably need to move it into SPECDB")
def ver01(test=False, clobber=False, publisher='J.X. Prochaska', **kwargs):
""" Build version 1.0
Parameters
----------
test : bool, optional
Run test only
Returns
-------
"""
pdb.set_trace() # THIS VERSION IS NOW FROZEN
raise IOError("THIS VERSION IS NOW FROZEN")
version = 'v01'
# HDF5 file
outfil = igmspec.__path__[0]+'/../DB/IGMspec_DB_{:s}.hdf5'.format(version)
# Chk clobber
if os.path.isfile(outfil):
if clobber:
warnings.warn("Overwriting previous DB file {:s}".format(outfil))
else:
warnings.warn("Not overwiting previous DB file. Use clobber=True to do so")
return
# Begin
hdf = h5py.File(outfil,'w')
''' Myers QSOs '''
myers.orig_add_to_hdf(hdf)
# Main DB Table
idkey = 'IGM_ID'
maindb, tkeys = sdbbu.start_maindb(idkey)
# Group dict
group_dict = {}
# Organize for main loop
groups = get_build_groups(version)
pair_groups = ['SDSS_DR7']
meta_only = False
# Loop over the groups
for gname in groups:
# Meta
if gname == 'SDSS_DR7':
meta = groups[gname].grab_meta(hdf)
else:
meta = groups[gname].grab_meta()
# Survey flag
flag_g = sdbbu.add_to_group_dict(gname, group_dict)
# IDs
maindb = sdbbu.add_ids(maindb, meta, flag_g, tkeys, idkey,
first=(flag_g==1), close_pairs=(gname in pair_groups))
# Spectra
if not meta_only:
groups[gname].hdf5_adddata(hdf, gname, meta, idkey)
# Check for duplicates -- There is 1 pair in SDSS (i.e. 2 duplicates)
if not sdbbu.chk_for_duplicates(maindb, dup_lim=2):
raise ValueError("Failed duplicates")
# Check for junk
zpri = defs.z_priority()
# Finish
sdbbu.write_hdf(hdf, str('igmspec'), maindb, zpri,
group_dict, version, Publisher=str(publisher))
print("Wrote {:s} DB file".format(outfil))
print("Update DB info in specdb.defs.dbase_info !!")