def get_2pt1_extinction_fix(pad=True):
"""
This is to fix the E(B-V)=0.02
flat extinction applied to 2.1
Curve from Karl
/work/00115/gebhardt/fluxcor/extinction
Paramaters
----------
pad bool
This is to pad curve to 3470 and 5400 to match
wave_rect
"""
config = HDRconfig()
karl_data = np.loadtxt( config.extinction_fix)
if pad:
karl_wave = np.concatenate(([3450.],
karl_data[:, 0],
[5550.]))
karl_curve = np.concatenate(([karl_data[0, 1]],
karl_data[:, 1],
[karl_data[-1, 1]]))
else:
karl_wave = karl_data[:, 0]
karl_curve = karl_data[:, 1]
correction = interpolate.interp1d( karl_wave, karl_curve)
return correction
def test_path_exists(attribute, release):
"""
Test that some of the
important paths in config exist.
"""
conf = HDRconfig(survey=release)
assert exists(getattr(conf, attribute))
def update_hdr_config(self):
self.survey = self.survey_widget.value.lower()
self.hetdex_api_config = HDRconfig( survey=self.survey)
self.FibIndex = FiberIndex(self.survey)
self.ampflag_table = Table.read(self.hetdex_api_config.badamp)
# update survey class and shot list
self.survey_class = Survey(self.survey)
def dustmaps_setup():
# this is how I intially set up dustmaps
# Need to put maps in config.dustmaps
from dustmaps.config import config as dustmaps_config
import dustmaps
config = HDRconfig()
dustmaps_config['data_dir'] = config.dustmaps
dustmaps.sfd.fetch() #don't need to do this
def test_simulation_interpolator(release, sncut):
"""
Compare the interpolator to the file directly
and ensure they are the same
"""
conf = HDRconfig(survey=release)
fdir = conf.flim_sim_completeness
sinterp = SimulationInterpolator(fdir, wl_collapse=False, cmax=None)
ffile = fdir + "/sn{:2.1f}.use".format(sncut)
waves, f50s, compl_curves, fluxes = read_karl_file(ffile)
for wl, f50, compl_curve in zip(waves, f50s, compl_curves):
model = sinterp(fluxes, f50, wl, sncut)
assert model == pytest.approx(compl_curve, rel=1e-2, abs=1e-2)
def __init__(self, survey=LATEST_HDR_NAME, loadall=False):
"""
Initialize the Fiber class for a given data release
Parameters
----------
survey : string
Data release you would like to load, i.e., 'hdr1','HDR2'
This is case insensitive.
Returns
-------
FiberIndex class object
"""
self.survey = survey
if self.survey == "hdr1":
print("Sorry there is no FiberIndex for hdr1")
return None
global config
config = HDRconfig(survey=survey.lower())
self.filename = config.fiberindexh5
self.hdfile = tb.open_file(self.filename, mode="r")
if loadall:
colnames = self.hdfile.root.FiberIndex.colnames
for name in colnames:
if isinstance(
getattr(self.hdfile.root.FiberIndex.cols, name)[0], np.bytes_
):
setattr(
self,
name,
getattr(self.hdfile.root.FiberIndex.cols, name)[:].astype(str),
)
else:
setattr(
self, name, getattr(self.hdfile.root.FiberIndex.cols, name)[:]
)
self.coords = SkyCoord(
self.ra[:] * u.degree, self.dec[:] * u.degree, frame="icrs"
)
def return_sensitivity_hdf_path(datevobs, release="hdr2.1",
return_mask_fn = False):
"""
Return the full file path
to a HDF5 container of sensitivity
cubes
Parameters
----------
datevobs : str
the date of the observation
release : str (optional)
the name of the release,
default: hdr2
Returns
-------
path_to_hdf5 : str
the absolute path to
the file
Raises
------
NoFluxLimsAvailable :
Returned when flux limits
not found for that shot
"""
config = HDRconfig(release)
flim_dir = config.flim_dir
mask_dir = config.flimmask
path_to_hdf5s = join(flim_dir, "{:s}_sensitivity_cube.h5".format(datevobs))
if isfile(path_to_hdf5s):
if return_mask_fn:
mask_fn = join(mask_dir, "{:s}_mask.h5".format(datevobs))
return path_to_hdf5s, mask_fn
else:
return path_to_hdf5s
else:
raise NoFluxLimsAvailable(
"Cannot find flux limits for that shot!"
" Tried here {:s}".format(path_to_hdf5s)
)
def open_shot_file(shotid, survey=LATEST_HDR_NAME):
"""
Open the H5 file for a shot. This is a global function that allows you
to open an H5 file based on its shotid and data release.
Parameters
----------
shotid
either in the form of integer shotid (eg. 20180123009) or
string datevobs (eg. '20180123v009')
survey : string
Data release you would like to load, i.e., 'HDR1', 'hdr2',
This is case insensitive.
Example
-------
>>> from hetdex_api.shot import open_shot_file
>>> fileh = open_shot_file(20180123009)
or
>>> fileh = open_shot_file('20180123v009')
"""
global config
config = HDRconfig(survey=survey.lower())
if re.search("v", str(shotid)):
file = op.join(config.data_dir, str(shotid) + ".h5")
else:
file = op.join(config.data_dir,
str(shotid)[0:8] + "v" + str(shotid)[8:11] + ".h5")
fileh = tb.open_file(file, "r")
return fileh
def main(argv=None):
''' Main Function '''
# Call initial parser from init_utils
parser = ap.ArgumentParser(description="""Create HDF5 Astrometry file.""",
add_help=True)
parser.add_argument("-d", "--date",
help='''Date, e.g., 20170321, YYYYMMDD''',
type=str, default=None)
parser.add_argument("-o", "--observation",
help='''Observation number, "00000007" or "7"''',
type=str, default=None)
parser.add_argument("-r", "--rootdir",
help='''Root Directory for Shifts''',
type=str, default='/scratch/00115/gebhardt/vdrp/shifts')
parser.add_argument('-of', '--outfilename', type=str,
help='''Relative or absolute path for output HDF5
file.''', default=None)
parser.add_argument('-a', '--append',
help='''Appending to existing file.''',
action="count", default=0)
parser.add_argument("-tp", "--tpdir",
help='''Directory for Throughput Info''',
type=str,
default='/scratch/00115/gebhardt/detect')
parser.add_argument("-detdir", "--detectdir",
help='''Directory for Detect Info''',
type=str,
default='/scratch/03946/hetdex/detect')
parser.add_argument("-survey", "--survey",
help="""{hdr1, hdr2, hdr2.1, hdr3}""",
type=str, default="hdr3")
args = parser.parse_args(argv)
args.log = setup_logging()
# Creates a new file if the "--append" option is not set or the file
# does not already exist.
does_exist = False
if op.exists(args.outfilename) and args.append:
args.log.info('Appending astrometry to %s' % args.outfilename)
fileh = tb.open_file(args.outfilename, 'a')
does_exist = True
try:
fileh.remove_node(fileh.root.Astrometry, recursive=True)
except:
pass
else:
args.log.info('Creating new file for astrometry %s' % args.outfilename)
fileh = tb.open_file(args.outfilename, 'w')
groupAstrometry = fileh.create_group(fileh.root, 'Astrometry', 'Astrometry Info')
groupCoadd = fileh.create_group(groupAstrometry, 'CoaddImages', 'Coadd Images')
groupDithall = fileh.create_group(groupAstrometry, 'Dithall', 'Fiber Astrometry Info')
groupOffsets = fileh.create_group(groupAstrometry, 'PositionOffsets',
'Offset in star matches')
groupMatches = fileh.create_group(groupAstrometry, 'CatalogMatches', 'Match Catalog Info')
tableQA = fileh.create_table(groupAstrometry, 'QA', QualityAssessment,
'Quality Assessment')
tableNV = fileh.create_table(groupAstrometry,'NominalVals', NominalVals,
'Nominal Values')
datevshot = str(args.date) + 'v' + str(args.observation).zfill(3)
shotid = int(str(args.date) + str(args.observation).zfill(3))
#check if shotid is in badlist
config = HDRconfig(args.survey)
badshots = np.loadtxt(config.badshot, dtype=int)
badshotflag = False
if shotid in badshots:
badshotflag = True
# store shuffle.cfg and DATEvOBS.log files
fileshuffle = op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3),
'shuffle.cfg')
try:
f = open(fileshuffle, 'r')
shuffle = fileh.create_array(groupAstrometry, 'ShuffleCfg', f.read().encode())
shuffle.set_attr('filename','shuffle.cfg')
f.close()
except:
args.log.warning('Could not include %s' % fileshuffle)
# store fplane table
filefplane = op.join(args.tpdir, str(args.date) + "v" + str(args.observation).zfill(3),
'coords','fplane.txt')
try:
f = ascii.read(filefplane, names=['ifuslot', 'fpx', 'fpy', 'specid',
'specslot', 'ifuid', 'ifurot', 'platesc'])
fplanetable = fileh.create_table(groupAstrometry, 'fplane', f.as_array())
fplanetable.set_attr('filename', 'fplane.txt')
except:
args.log.warning('Could not include %s' % filefplane)
# store catalogs of stars used in astrometric fit
file_stars = op.join(args.tpdir, str(args.date) + 'v' + str(args.observation).zfill(3),
str(args.date) + 'v' + str(args.observation).zfill(3) + '.ifu')
try:
f_stars = ascii.read(file_stars, names=['ignore', 'star_ID', 'ra_cat', 'dec_cat',
'u', 'g', 'r', 'i', 'z'])
starstable = fileh.create_table(groupAstrometry, 'StarCatalog', f_stars.as_array())
starstable.set_attr('filename', 'DATEvOBS.ifu')
if any(f_stars['z'] > 0):
starstable.set_attr('catalog', 'SDSS')
else:
starstable.set_attr('catalog', 'GAIA')
except:
args.log.warning('Could not include %s' % file_stars)
pngfiles = glob.glob(op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3), '*.png'))
pngnum = 1
for pngfile in pngfiles:
plt_image = plt.imread(pngfile)
pngim = fileh.create_array(groupCoadd, 'png_exp' + str(pngnum).zfill(2), plt_image)
pngim.attrs['CLASS'] = 'IMAGE'
pngim.attrs['filename'] = pngfile
pngnum += 1
fileallmch = op.join(args.tpdir, str(args.date) + 'v' + str(args.observation).zfill(3),
'coords', 'all.mch')
try:
allmch = ascii.read(fileallmch)
except:
args.log.warning('Could not include %s' % fileallmch)
filenorm = op.join(args.detectdir, 'norm.all')
try:
norm = Table.read(filenorm, format='ascii.no_header')
except:
args.log.warning('Could not include %s' % filenorm)
# index over dithers to gather diher specific info
for idx, expn in enumerate(['exp01', 'exp02', 'exp03']):
radecfile = op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3),
'radec2_' + expn + '.dat')
rowNV = tableNV.row
try:
radec = ascii.read(radecfile)
rowNV['expnum'] = int(expn[3:5])
rowNV['ra'] = radec['col1']
rowNV['dec'] = radec['col2']
rowNV['parangle'] = radec['col3']
except:
args.log.warning('Could not include %s' % radecfile)
try:
sel_datevobs = norm['col1'] == str(args.date) + 'v' + str(args.observation).zfill(3)
if idx == 0:
rowNV['relflux_virus'] = norm['col2'][sel_datevobs]
elif idx == 1:
rowNV['relflux_virus'] = norm['col3'][sel_datevobs]
elif idx == 2:
rowNV['relflux_virus'] = norm['col4'][sel_datevobs]
except Exception:
args.log.warning('Could not include norm.all')
try:
rowNV['x_dither_pos'] = allmch['col3'][idx]
rowNV['y_dither_pos'] = allmch['col4'][idx]
rowNV['als_filename'] = allmch['col1'][idx]
except:
args.log.warning('Could not include %s' % fileallmch)
rowNV.append()
fitsfile = op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3),
str(args.date) + 'v' + str(args.observation).zfill(3)
+ 'fp_' + expn + '.fits')
if op.exists(fitsfile):
F = fits.open(fitsfile)
fitsim = fileh.create_array(groupCoadd, expn, F[0].data)
fitsim.attrs['CLASS'] = 'IMAGE'
fitsim.attrs['IMAGE_MINMAXRANGE'] = (-1.5, 100)
fitsim.attrs['HEADER'] = F[0].header
fitsim.attrs['filename'] = 'DATEvOBSfp_exp??.fits'
F.close()
matchpdf = op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3),
'match_' + expn + '.pdf')
matchpng = 'match_pngs/match_'+ str(args.date) + 'v' + str(args.observation).zfill(3) + '_' + expn + '.png'
if op.exists(matchpdf):
os.system('convert ' + matchpdf + ' ' + matchpng)
plt_matchim = plt.imread(matchpng)
matchim = fileh.create_array(groupCoadd, 'match_' + expn, plt_matchim)
matchim.attrs['CLASS'] = 'IMAGE'
matchim.attrs['filename'] = matchpdf
else:
args.log.warning('Count not include %s' % matchpdf)
# populate offset info for catalog matches
file_getoff = op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3),
'getoff_' + expn + '.out')
try:
f_getoff = ascii.read(file_getoff, names=['xoffset', 'yoffset', 'ra_dex',
'dec_dex', 'ra_cat', 'dec_cat',
'ifuslot'])
getoffinfo = fileh.create_table(groupOffsets, expn, f_getoff.as_array())
getoffinfo.set_attr('filename', 'getoff_exp??.out')
except:
args.log.warning('Could not include %s' % file_getoff)
# populate fiber astrometry data
file_dith = op.join(args.rootdir, str(args.date) + 'v' + str(args.observation).zfill(3),
'dith_' + expn + '.all')
try:
f_dith = ascii.read(file_dith)
dithtab = fileh.create_table(groupDithall, expn, Dithall)
for f_dith_row in f_dith:
dithrow = dithtab.row
dithrow['ra'] = f_dith_row['ra']
dithrow['dec'] = f_dith_row['dec']
dithrow['ifuslot'] = f_dith_row['ifuslot']
dithrow['XS'] = f_dith_row['XS']
dithrow['YS'] = f_dith_row['YS']
dithrow['xfplane'] = f_dith_row['xfplane']
dithrow['yfplane'] = f_dith_row['yfplane']
dithrow['multifits'] = f_dith_row['multifits']
dithrow['timestamp'] = f_dith_row['timestamp']
dithrow['exposure'] = f_dith_row['exposure']
dithrow.append()
dithtab.set_attr('filename', file_dith)
dithtab.flush()
except:
args.log.warning('Could not include %s' % file_dith)
# populate median and rms in offsets for quality assessment purposes
file_getoff2 = op.join(args.rootdir, str(args.date) + 'v'
+ str(args.observation).zfill(3), 'getoff2_' + expn + '.out')
try:
f_getoff2 = ascii.read(file_getoff2)
row = tableQA.row
row['expnum'] = int(expn[3:5])
row['xoffset'] = f_getoff2['col1']
row['yoffset'] = f_getoff2['col2']
row['xrms'] = f_getoff2['col3']
row['yrms'] = f_getoff2['col4']
row['nstars'] = f_getoff2['col5']
row.append()
except:
args.log.warning('Could not include %s' % file_getoff2)
file_xy = op.join(args.rootdir, str(args.date) + 'v'
+ str(args.observation).zfill(3), 'xy_' + expn + '.dat')
try:
xy_table = ascii.read(file_xy)
tableXY = fileh.create_table(groupMatches, expn, xy_table.as_array())
tableXY.set_attr('filename','xy_exp??.dat')
except:
args.log.warning('Could not include %s' % file_xy)
tableQA.set_attr('filename', 'getoff2_exp??.out')
tableNV.set_attr('dither_file', 'all.mch')
tableNV.set_attr('norm_file', 'norm.dat')
tableNV.set_attr('radec_file', 'radec2_exp??.dat')
tableQA.flush()
tableNV.flush()
tableQA = fileh.root.Astrometry.QA
tableNV = fileh.root.Astrometry.NominalVals
try:
radecfinalfile = op.join(args.tpdir,
str(args.date) + 'v' + str(args.observation).zfill(3),
'coords', 'radec2.dat')
radectab = ascii.read(radecfinalfile, names=['ra','dec','pa'])
except:
if badshotflag:
args.log.warning('Could not open %s' % radecfinalfile)
else:
args.log.error('Could not open %s' % radecfinalfile)
shottable = fileh.root.Shot
for shot in shottable:
if op.exists(radecfinalfile):
shot['ra'] = radectab['ra'][0]
shot['dec'] = radectab['dec'][0]
shot['pa'] = radectab['pa'][0]
else:
if badshotflag:
args.log.warning('Could not open %s' % radecfinalfile)
else:
args.log.error('Could not open %s' % radecfinalfile)
try:
shot['xoffset'] = tableQA.cols.xoffset[:]
shot['yoffset'] = tableQA.cols.yoffset[:]
shot['xrms'] = tableQA.cols.xrms[:]
shot['yrms'] = tableQA.cols.yrms[:]
shot['nstars_fit'] = tableQA.cols.nstars[:]
except:
if badshotflag:
args.log.warning('Could not include astrometry shot info for %s' % datevshot)
else:
args.log.error('Could not include astrometry shot info for %s' % datevshot)
try:
shot['xditherpos'] = tableNV.cols.x_dither_pos[:]
shot['yditherpos'] = tableNV.cols.y_dither_pos[:]
except:
args.log.warning('Could not include astrometry shot info for %s' % datevshot)
try:
shot['relflux_virus'] = tableNV.cols.relflux_virus[:]
except:
if badshotflag:
args.log.warning('Could not include relflux_virus info for %s' % datevshot)
else:
args.log.error('Could not include relflux_virus info for %s' % datevshot)
shot.update()
fileh.close()
def __init__(self,
sigmas,
header,
wavelengths,
alphas,
aper_corr=1.0,
nsigma=1.0,
flim_model="hdr2pt1pt1",
mask=None,
cache_sim_interp=True):
if type(mask) != type(None):
mask = logical_not(mask)
mask3d = repeat(mask[newaxis, :, :], sigmas.shape[0], axis=0)
self.sigmas = maskedarray(sigmas / nsigma,
mask=mask3d,
fill_value=999.0)
else:
self.sigmas = maskedarray(sigmas / nsigma, fill_value=999.0)
self.nsigma = nsigma
# Decide between Fleming function or interpolation
if (flim_model == "hdr1") or (flim_model == "hdr2pt1"):
self.sinterp = None
elif flim_model_cache.cached_model == flim_model and cache_sim_interp:
self.sinterp = flim_model_cache.cached_sim_interp
else:
conf = HDRconfig()
fdir = conf.flim_sim_completeness
self.sinterp = flim_models.SimulationInterpolator(fdir)
# cache the model to save reinitialising the object
if cache_sim_interp:
flim_model_cache.cached_sim_interp = self.sinterp
flim_model_cache.cached_model = flim_model
# Fix issue with header
if not "CD3_3" in header:
header["CD3_3"] = header["CDELT3"]
header["CD3_1"] = 0.0
header["CD3_2"] = 0.0
header["CD2_3"] = 0.0
header["CD1_3"] = 0.0
self.wcs = WCS(header)
self.header = header
# Deal with aperture corrections
if aper_corr:
self.aper_corr = aper_corr
elif "APCOR" in self.header:
self.aper_corr = self.header["APCOR"]
elif "APCOR0" in self.header:
self.aper_corr = self.header["APCOR0"]
else:
self.aper_corr = 1.0
self.sigmas = self.sigmas * self.aper_corr
self.alphas = array(alphas)
self.wavelengths = wavelengths
# Depends if alphas depend on wavelength or
# is specified per cube cell
if len(self.alphas.shape) == 3:
self.alpha_is_cube = True
else:
self.alpha_is_cube = False
self.alpha_func = interp1d(wavelengths,
alphas,
fill_value="extrapolate")
# Select flux limit model
try:
self.f50_from_noise = getattr(
flim_models, "{:s}_f50_from_noise".format(flim_model))
except AttributeError as e:
print("Chosen flux limit model not found!")
raise e
mag_sdss_g = tb.Float32Col()
mag_sdss_g_err = tb.Float32Col()
plae_sdss_g = tb.Float32Col()
plae_sdss_g_max = tb.Float32Col()
plae_sdss_g_min = tb.Float32Col()
#ELiXer combined (rules, inv variance, weights and Bayes) classification info
combined_plae = tb.Float32Col()
combined_plae_err = tb.Float32Col()
plae_classification = tb.Float32Col()
combined_continuum = tb.Float32Col()
combined_continuum_err = tb.Float32Col()
detecth5 = sys.argv[1]
config = HDRconfig('hdr2.1')
filedet = tb.open_file(detecth5, 'a')
fileelix = tb.open_file(config.elixerh5, 'r')
detectid = filedet.root.Detections.cols.detectid[:]
elix_table = fileelix.root.Detections
try:
filedet.remove_node(filedet.root.Elixer, recursive=True)
except:
pass
tableElixer = filedet.create_table(filedet.root, "Elixer", Elixer,
def main(argv=None):
""" Main Function """
# Call initial parser from init_utils
parser = ap.ArgumentParser(description="""Create HDF5 Astrometry file.""",
add_help=True)
parser.add_argument(
"-sdir",
"--shotdir",
help="""Directory for shot H5 files to ingest""",
type=str,
default="/scratch/03946/hetdex/hdr3/reduction/data",
)
parser.add_argument(
"-sl",
"--shotlist",
help="""Text file of DATE OBS list""",
type=str,
default="hdr3.shotlist",
)
parser.add_argument(
"-of",
"--outfilename",
type=str,
help="""Relative or absolute path for output HDF5
file.""",
default=None,
)
parser.add_argument("-survey", "--survey", type=str, default="hdr3")
parser.add_argument(
"-m",
"--month",
type=int,
default=None,
help="""Create FiberIndex for a single month""",
)
parser.add_argument(
"--merge",
"-merge",
help="""Boolean trigger to merge all 2*.fits files in cwd""",
default=False,
required=False,
action="store_true",
)
args = parser.parse_args(argv)
args.log = setup_logging()
fileh = tb.open_file(args.outfilename,
mode="w",
title=args.survey.upper() + " Fiber Index file ")
shotlist = Table.read(args.shotlist,
format="ascii.no_header",
names=["date", "obs"])
tableFibers = fileh.create_table(
fileh.root,
"FiberIndex",
VIRUSFiberIndex,
"Survey Fiber Coord Info",
expectedrows=300000000,
)
if args.merge:
files = glob.glob("mfi*h5")
for file in files:
args.log.info("Appending detect H5 file: %s" % file)
fileh_i = tb.open_file(file, "r")
tableFibers_i = fileh_i.root.FiberIndex.read()
tableFibers.append(tableFibers_i)
tableFibers.cols.healpix.create_csindex()
tableFibers.cols.ra.create_csindex()
tableFibers.cols.shotid.create_csindex()
tableFibers.flush()
fileh.close()
args.log.info("Completed {}".format(args.outfilename))
sys.exit()
# set up HEALPIX options
Nside = 2**15
hp.max_pixrad(Nside, degrees=True) * 3600 # in unit of arcsec
config = HDRconfig(survey=args.survey)
badshot = np.loadtxt(config.badshot, dtype=int)
if args.month is not None:
args.log.info("Working on month {}".format(args.month))
# if working on a single month downselect
shotlist["month"] = np.array(shotlist["date"] / 100, dtype=int)
sel_month = shotlist["month"] == args.month
shotlist = shotlist[sel_month]
for shotrow in shotlist:
datevshot = str(shotrow["date"]) + "v" + str(shotrow["obs"]).zfill(3)
shotid = int(str(shotrow["date"]) + str(shotrow["obs"]).zfill(3))
date = shotrow["date"]
try:
args.log.info("Ingesting %s" % datevshot)
file_obs = tb.open_file(op.join(args.shotdir, datevshot + ".h5"),
"r")
tableFibers_i = file_obs.root.Data.FiberIndex
for row_i in tableFibers_i:
row_main = tableFibers.row
for col in tableFibers_i.colnames:
row_main[col] = row_i[col]
fiberid = row_i["fiber_id"]
try:
row_main["healpix"] = hp.ang2pix(Nside,
row_i["ra"],
row_i["dec"],
lonlat=True)
except:
row_main["healpix"] = 0
row_main["shotid"] = shotid
row_main["date"] = date
row_main["datevobs"] = datevshot
row_main["specid"] = fiberid[20:23]
row_main["ifuslot"] = fiberid[24:27]
row_main["ifuid"] = fiberid[28:31]
row_main["amp"] = fiberid[32:34]
row_main.append()
file_obs.close()
except:
if shotid in badshot:
pass
else:
args.log.error("could not ingest %s" % datevshot)
tableFibers.cols.healpix.create_csindex()
tableFibers.cols.ra.create_csindex()
tableFibers.cols.shotid.create_csindex()
tableFibers.flush()
fileh.close()
args.log.info("Completed {}".format(args.outfilename))
def return_astropy_table(Source_dict,
fiberweights=False,
return_fiber_info=False):
"""Returns an astropy table fom a source dictionary"""
id_arr = []
shotid_arr = []
wave_arr = []
spec_arr = []
spec_err_arr = []
weights_arr = []
fiber_weights_arr = []
fiber_info_arr = []
gal_flag_arr = []
meteor_flag_arr = []
amp_flag_arr = []
flag_arr = []
config = HDRconfig()
bad_amps_table = Table.read(config.badamp)
galaxy_cat = Table.read(config.rc3cat, format="ascii")
# loop over every ID/observation combo:
for ID in Source_dict.keys():
for shotid in Source_dict[ID].keys():
wave_rect = 2.0 * np.arange(1036) + 3470.0
spec = Source_dict[ID][shotid][0]
spec_err = Source_dict[ID][shotid][1]
weights = Source_dict[ID][shotid][2]
if fiberweights:
fiber_weights = Source_dict[ID][shotid][3]
# get fiber info to make masks
try:
fiber_info = Source_dict[ID][shotid][4]
except:
fiber_info = None
if Source_dict[ID][shotid][5] is None:
amp_flag = True
gal_flag = True
meteor_flag = True
flag = True
else:
meteor_flag, gal_flag, amp_flag, flag = Source_dict[ID][shotid][5]
sel = np.isfinite(spec)
if np.sum(sel) > 0:
id_arr.append(ID)
shotid_arr.append(shotid)
wave_arr.append(wave_rect)
spec_arr.append(spec)
spec_err_arr.append(spec_err)
weights_arr.append(weights)
if fiberweights:
fiber_weights_arr.append(fiber_weights)
fiber_info_arr.append(fiber_info)
flag_arr.append(flag)
amp_flag_arr.append(amp_flag)
meteor_flag_arr.append(meteor_flag)
gal_flag_arr.append(gal_flag)
output = Table()
fluxden_u = 1e-17 * u.erg * u.s ** (-1) * u.cm ** (-2) * u.AA ** (-1)
output.add_column(Column(id_arr), name="ID")
output.add_column(Column(shotid_arr), name="shotid")
output.add_column(Column(wave_arr, unit=u.AA, name="wavelength"))
output.add_column(Column(spec_arr, unit=fluxden_u, name="spec"))
output.add_column(Column(spec_err_arr, unit=fluxden_u, name="spec_err"))
output.add_column(Column(weights_arr), name="weights")
output.add_column(Column(flag_arr, name='flag', dtype=int))
output.add_column(Column(gal_flag_arr, name='gal_flag', dtype=int))
output.add_column(Column(amp_flag_arr, name='amp_flag', dtype=int))
output.add_column(Column(meteor_flag_arr, name='meteor_flag', dtype=int))
if fiberweights:
output.add_column(Column(fiber_weights_arr), name="fiber_weights")
if return_fiber_info:
output.add_column(Column(fiber_info_arr, name="fiber_info"))
return output
"""
Scripts to deal with extinction.
This includes HDR2.1 flat extinction fix
"""
import numpy as np
from scipy import interpolate
import extinction
from hetdex_api.config import HDRconfig
LATEST_HDR_NAME = HDRconfig.LATEST_HDR_NAME
config = HDRconfig(LATEST_HDR_NAME)
from dustmaps.config import config as dustmaps_config
if dustmaps_config['data_dir'] is None:
print("Populating dustmaps config with {}".format(config.dustmaps))
dustmaps_config['data_dir'] = config.dustmaps
from dustmaps.sfd import SFDQuery
def dustmaps_setup():
# this is how I intially set up dustmaps
# Need to put maps in config.dustmaps
from dustmaps.config import config as dustmaps_config
import dustmaps
config = HDRconfig()
dustmaps_config['data_dir'] = config.dustmaps
dustmaps.sfd.fetch() #don't need to do this
import tables
from hetdex_tools.get_spec import get_spectra
try:
from hetdex_api.config import HDRconfig
LATEST_HDR_NAME = HDRconfig.LATEST_HDR_NAME
except Exception as e:
print("Warning! Cannot find or import HDRconfig from hetdex_api!!", e)
LATEST_HDR_NAME = "hdr2.1"
HDR_NAME_DICT = {10: "hdr1", 20: "hdr2", 21: "hdr2.1"}
try: # using HDRconfig
HETDEX_API_CONFIG = HDRconfig(survey=LATEST_HDR_NAME)
HDR_BASEPATH = HETDEX_API_CONFIG.hdr_dir[LATEST_HDR_NAME]
HETDEX_DETECT_HDF5_FN = HETDEX_API_CONFIG.detecth5
HETDEX_DETECT_HDF5_HANDLE = None
CONT_H5_FN = HETDEX_API_CONFIG.contsourceh5
CONT_H5_HANDLE = None
HETDEX_ELIXER_HDF5 = HETDEX_API_CONFIG.elixerh5
ELIXER_H5 = None
elix_dir = None
except Exception as e:
HETDEX_API_CONFIG = None
# needed only if detection observered wavelength is not supplied
try:
HETDEX_DETECT_HDF5_FN = "/work/03946/hetdex/hdr2.1/detect/detect_hdr2.1.h5"
HETDEX_ELIXER_HDF5 = "/work/03946/hetdex/hdr2.1/elixer.h5"
def __init__(self, survey=LATEST_HDR_NAME):
"""
Initialize the Survey class for a given data release
Parameters
----------
survey : string
Data release you would like to load, i.e., 'hdr1','HDR2'
This is case insensitive.
Returns
-------
Survey class object with the following attributes
"""
global config
config = HDRconfig(survey=survey.lower())
self.filename = config.surveyh5
self.hdfile = tb.open_file(self.filename, mode="r")
colnames = self.hdfile.root.Survey.colnames
for name in colnames:
if name == "ra_flag":
setattr(
self,
name,
getattr(self.hdfile.root.Survey.cols, name)[:].astype(str),
)
elif isinstance(getattr(self.hdfile.root.Survey.cols, name)[0], np.bytes_):
setattr(
self,
name,
getattr(self.hdfile.root.Survey.cols, name)[:].astype(str),
)
else:
setattr(self, name, getattr(self.hdfile.root.Survey.cols, name)[:])
# set the SkyCoords
self.coords = SkyCoord(self.ra * u.degree, self.dec * u.degree, frame="icrs")
# append flux limits
if survey == "hdr2.1":
flim = Table.read(
config.flim_avg,
format="ascii",
names=["datevobs", "col2", "fluxlimit_4540"],
)
fluxlimit = []
for datevobs in self.datevobs:
sel = flim["datevobs"] == datevobs
if np.sum(sel) == 1:
fluxlimit.extend(flim["fluxlimit_4540"][sel])
elif np.sum(sel) > 1:
print("Found two fluxlimits for ", datevobs)
fluxlimit.extend(flim["fluxlimit_4540"][sel][0])
else:
fluxlimit.append(np.nan)
self.fluxlimit_4540 = np.array(fluxlimit)
def __init__(
self,
survey=LATEST_HDR_NAME,
catalog_type="lines",
curated_version=None,
loadtable=True,
):
"""
Initialize the detection catalog class for a given data release
Input
-----
survey : string
Data release you would like to load, i.e., 'hdr2','HDR1'
This is case insensitive.
catalog_type : string
Catalog to laod up. Either 'lines' or 'continuum'. Default is
'lines'.
load_table : bool
Boolean flag to load all detection table info upon initialization.
For example, if you just want to grab a spectrum this isn't needed.
"""
survey_options = ["hdr1", "hdr2", "hdr2.1"]
catalog_type_options = ["lines", "continuum", "broad"]
if survey.lower() not in survey_options:
print("survey not in survey options")
print(survey_options)
return None
if catalog_type.lower() not in catalog_type_options:
print("catalog_type not in catalog_type options")
print(catalog_type_options)
return None
# store to class
if curated_version is not None:
self.version = curated_version
self.loadtable = False
self.survey = "hdr" + curated_version[0:3]
else:
self.version = None
self.survey = survey
self.loadtable = loadtable
global config
config = HDRconfig(survey=self.survey)
if catalog_type == "lines":
self.filename = config.detecth5
elif catalog_type == "continuum":
self.filename = config.contsourceh5
elif catalog_type == "broad":
try:
self.filename = config.detectbroadh5
except:
print("Could not locate broad line catalog")
self.hdfile = tb.open_file(self.filename, mode="r")
# store to class
if curated_version is not None:
self.version = curated_version
self.loadtable = False
self.survey = "hdr" + curated_version[0:3]
else:
self.survey = survey
self.loadtable = loadtable
if self.version is not None:
try:
catfile = op.join(
config.detect_dir, "catalogs", "detect_hdr" + self.version + ".fits"
)
det_table = Table.read(catfile)
for col in det_table.colnames:
if isinstance(det_table[col][0], str):
setattr(self, col, np.array(det_table[col]).astype(str))
else:
setattr(self, col, np.array(det_table[col]))
self.vis_class = -1 * np.ones(np.size(self.detectid))
except:
print("Could not open curated catalog version: " + self.version)
return None
elif self.loadtable:
colnames = self.hdfile.root.Detections.colnames
for name in colnames:
if isinstance(
getattr(self.hdfile.root.Detections.cols, name)[0], np.bytes_
):
setattr(
self,
name,
getattr(self.hdfile.root.Detections.cols, name)[:].astype(str),
)
else:
setattr(
self, name, getattr(self.hdfile.root.Detections.cols, name)[:]
)
if self.survey == "hdr2.1":
# Fix fluxes and continuum values for aperture corrections
wave = self.hdfile.root.Detections.cols.wave[:]
apcor = self.hdfile.root.Spectra.cols.apcor[:]
wave_spec = self.hdfile.root.Spectra.cols.wave1d[:]
apcor_array = np.ones_like(wave)
for idx in np.arange(0, np.size(wave)):
sel_apcor = np.where(wave_spec[idx, :] > wave[idx])[0][0]
apcor_array[idx]=apcor[idx, sel_apcor]
self.flux /= apcor_array
self.flux_err /= apcor_array
self.continuum /= apcor_array
self.continuum_err /= apcor_array
self.apcor = apcor_array
# add in the elixer probabilties and associated info:
if self.survey == "hdr1" and catalog_type == "lines":
self.hdfile_elix = tb.open_file(config.elixerh5, mode="r")
colnames2 = self.hdfile_elix.root.Classifications.colnames
for name2 in colnames2:
if name2 == "detectid":
setattr(
self,
"detectid_elix",
self.hdfile_elix.root.Classifications.cols.detectid[:],
)
else:
if isinstance(
getattr(self.hdfile_elix.root.Classifications.cols, name2)[
0
],
np.bytes_,
):
setattr(
self,
name2,
getattr(
self.hdfile_elix.root.Classifications.cols, name2
)[:].astype(str),
)
else:
setattr(
self,
name2,
getattr(
self.hdfile_elix.root.Classifications.cols, name2
)[:],
)
else:
# add elixer info if node exists
try:
colnames = self.hdfile.root.Elixer.colnames
for name in colnames:
if name == "detectid":
continue
if isinstance(
getattr(self.hdfile.root.Elixer.cols, name)[0], np.bytes_
):
setattr(
self,
name,
getattr(self.hdfile.root.Elixer.cols, name)[:].astype(
str
),
)
else:
setattr(
self,
name,
getattr(self.hdfile.root.Elixer.cols, name)[:],
)
self.gmag = self.mag_sdss_g
self.gmag_err = self.mag_sdss_g
except:
print("No Elixer table found")
# also assign a field and some QA identifiers
self.field = np.chararray(np.size(self.detectid), 12, unicode=True)
self.fwhm = np.zeros(np.size(self.detectid))
if self.survey == "hdr1":
self.fluxlimit_4550 = np.zeros(np.size(self.detectid))
else:
self.fluxlimit_4540 = np.zeros(np.size(self.detectid))
self.throughput = np.zeros(np.size(self.detectid))
self.n_ifu = np.zeros(np.size(self.detectid), dtype=int)
S = Survey(self.survey)
for index, shot in enumerate(S.shotid):
ix = np.where(self.shotid == shot)
self.field[ix] = S.field[index].astype(str)
# NOTE: python2 to python3 strings now unicode
if self.survey == "hdr1":
self.fwhm[ix] = S.fwhm_moffat[index]
self.fluxlimit_4550[ix] = S.fluxlimit_4550[index]
else:
self.fwhm[ix] = S.fwhm_virus[index]
try:
self.fluxlimit_4540[ix] = S.fluxlimit_4540[index]
except:
pass
self.throughput[ix] = S.response_4540[index]
self.n_ifu[ix] = S.n_ifu[index]
# assign a vis_class field for future classification
# -2 = ignore (bad detectid, shot)
# -1 = no assignemnt
# 0 = artifact
# 1 = OII emitter
# 2 = LAE emitter
# 3 = star
# 4 = nearby galaxies (HBeta, OIII usually)
# 5 = other line
# close the survey HDF5 file
S.close()
self.vis_class = -1 * np.ones(np.size(self.detectid))
if self.survey == "hdr1":
self.add_hetdex_gmag(loadpickle=True, picklefile=config.gmags)
if self.survey == "hdr1":
if PYTHON_MAJOR_VERSION < 3:
self.plae_poii_hetdex_gmag = np.array(
pickle.load(open(config.plae_poii_hetdex_gmag, "rb"))
)
else:
self.plae_poii_hetdex_gmag = np.array(
pickle.load(
open(config.plae_poii_hetdex_gmag, "rb"), encoding="bytes"
)
)
else:
# just get coordinates and detectid
self.detectid = self.hdfile.root.Detections.cols.detectid[:]
self.ra = self.hdfile.root.Detections.cols.ra[:]
self.dec = self.hdfile.root.Detections.cols.dec[:]
# set the SkyCoords
self.coords = SkyCoord(self.ra * u.degree, self.dec * u.degree, frame="icrs")
def main(argv=None):
''' Main Function '''
# Call initial parser from init_utils
parser = ap.ArgumentParser(description="""Create HDF5 Astrometry file.""",
add_help=True)
parser.add_argument("-d",
"--date",
help='''Date, e.g., 20170321, YYYYMMDD''',
type=str,
default=None)
parser.add_argument("-o",
"--observation",
help='''Observation number, "00000007" or "7"''',
type=str,
default=None)
parser.add_argument("-r",
"--rootdir",
help='''Root Directory for Shifts''',
type=str,
default='/data/00115/gebhardt/skysub/skys/')
parser.add_argument('-of',
'--outfilename',
type=str,
help='''Relative or absolute path for output HDF5
file.''',
default=None)
parser.add_argument('-a',
'--append',
help='''Appending to existing file.''',
action="count",
default=0)
parser.add_argument("-survey",
"--survey",
help="""{hdr1, hdr2, hdr2.1}""",
type=str,
default="hdr2.1")
args = parser.parse_args(argv)
args.log = setup_logging()
if op.exists(args.outfilename) and args.append:
args.log.info('Appending fullskymodel to %s' % args.outfilename)
fileh = tb.open_file(args.outfilename, 'a')
try:
fileh.remove_node(fileh.root.FullSkyModel, recursive=True)
except:
pass
else:
args.log.info('Creating new file for FullSkyModel %s' %
args.outfilename)
fileh = tb.open_file(args.outfilename, 'w')
groupFullSkyModel = fileh.create_group(fileh.root, 'FullSkyModel',
'FullSkyModel')
datevshot = str(args.date) + 'v' + str(args.observation).zfill(3)
shotid = int(str(args.date) + str(args.observation).zfill(3))
#check if shotid is in badlist
config = HDRconfig(args.survey)
badshots = np.loadtxt(config.badshot, dtype=int)
badshotflag = False
if shotid in badshots:
badshotflag = True
# store shuffle.cfg and DATEvOBS.log files
for expn in ['exp01', 'exp02', 'exp03']:
skyfile = op.join(
args.rootdir, 'd' + str(args.date) + 's' +
str(args.observation).zfill(3) + expn + 'sky.dat')
try:
sky_array = np.loadtxt(skyfile)
if np.size(sky_array) > 0:
fileh.create_array(groupFullSkyModel, expn, sky_array)
else:
if badshotflag:
args.log.warning('File empty %s' % skyfile)
else:
args.log.error('File empty %s' % skyfile)
except:
if badshotflag:
args.log.warning('Could not include %s' % skyfile)
else:
args.log.error('Could not include %s' % skyfile)
fileh.close()
def __init__(
self,
coords=None,
detectid=None,
survey="hdr2.1",
aperture=3.0 * u.arcsec,
cutout_size=5.0 * u.arcmin,
zoom=3,
):
self.survey = survey.lower()
self.detectid = detectid
self.aperture = aperture
self.cutout_size = cutout_size
self.zoom = zoom
config = HDRconfig(survey=survey)
self.fileh5dets = tb.open_file(config.detecth5, "r")
self.catlib = catalogs.CatalogLibrary()
if coords:
self.coords = coords
self.detectid = 1000000000
elif detectid:
self.detectid = detectid
self.update_det_coords()
else:
self.coords = SkyCoord(191.663132 * u.deg,
50.712696 * u.deg,
frame="icrs")
self.detectid = 2101848640
# initialize the image widget from astrowidgets
self.imw = ImageWidget(image_width=600, image_height=600)
self.survey_widget = widgets.Dropdown(
options=["HDR1", "HDR2", "HDR2.1"],
value=self.survey.upper(),
layout=Layout(width="10%"),
)
self.detectbox = widgets.BoundedIntText(
value=self.detectid,
min=1000000000,
max=3000000000,
step=1,
description="DetectID:",
disabled=False,
)
self.im_ra = widgets.FloatText(
value=self.coords.ra.value,
description="RA (deg):",
layout=Layout(width="20%"),
)
self.im_dec = widgets.FloatText(
value=self.coords.dec.value,
description="DEC (deg):",
layout=Layout(width="20%"),
)
self.pan_to_coords = widgets.Button(description="Pan to coords",
disabled=False,
button_style="success")
self.marking_button = widgets.Button(description="Mark Sources",
button_style="success")
self.reset_marking_button = widgets.Button(description="Reset",
button_style="success")
self.extract_button = widgets.Button(description="Extract Object",
button_style="success")
self.marker_table_output = widgets.Output(
layout={"border": "1px solid black"})
# self.spec_output = widgets.Output(layout={'border': '1px solid black'})
self.spec_output = widgets.Tab(description="Extracted Spectra:",
layout={"border": "1px solid black"})
self.textimpath = widgets.Text(description="Source: ",
value="",
layout=Layout(width="90%"))
self.topbox = widgets.HBox([
self.survey_widget,
self.detectbox,
self.im_ra,
self.im_dec,
self.pan_to_coords,
])
self.leftbox = widgets.VBox([self.imw, self.textimpath],
layout=Layout(width="800px"))
self.rightbox = widgets.VBox(
[
widgets.HBox([
self.marking_button,
self.reset_marking_button,
self.extract_button,
]),
self.marker_table_output,
self.spec_output,
],
layout=Layout(width="800px"),
)
self.bottombox = widgets.Output(layout={"border": "1px solid black"})
self.load_image()
self.all_box = widgets.VBox([
self.topbox,
widgets.HBox([self.leftbox, self.rightbox]),
#self.spec_output,
self.bottombox
])
display(self.all_box)
self.detectbox.observe(self.on_det_change)
self.pan_to_coords.on_click(self.pan_to_coords_click)
self.marking_button.on_click(self.marking_on_click)
self.reset_marking_button.on_click(self.reset_marking_on_click)
self.extract_button.on_click(self.extract_on_click)
self.survey_widget.observe(self.on_survey_change)
import sys
import os
import os.path as op
import subprocess
import numpy as np
import pandas as pd
from astropy.table import Table, Column, vstack, join, hstack
from hetdex_api.config import HDRconfig
survey = "hdr2.1"
config = HDRconfig(survey)
update = True
create = False
if create:
# I start with Karl's calibration summary file
#! cp /data/00115/gebhardt/skysub/analysis/all.hdr2.1 all.hdr2.1
#! awk '{printf "%20s %14s %8.3f %5i %8.3f %8.3f %8.3f %8.3f %4i \n", $1, $2, $3, $4, $5, $6, $7, $8, $9}' all.hdr2.1 > all.hdr2.1_emc
#! sed -i s/'s'/''/ all.hdr2.1_emc
#! sed -i s/'d'/''/ all.hdr2.1_emc
#! sed -i s/'exp0'/' '/ all.hdr2.1_emc
amp_stats = Table.read(
"all.hdr2.1_emc",
format="ascii.no_header",
fast_reader=False,
def main(argv=None):
""" Main Function """
# Call initial parser from init_utils
parser = ap.ArgumentParser(
description="""Create HDF5 Astrometry file.""", add_help=True
)
parser.add_argument(
"-sdir",
"--shotdir",
help="""Directory for shot H5 files to ingest""",
type=str,
default="/data/05350/ecooper/hdr2.1/reduction/data",
)
parser.add_argument(
"-sl",
"--shotlist",
help="""Text file of DATE OBS list""",
type=str,
default="hdr2.1.shotlist",
)
parser.add_argument(
"-of",
"--outfilename",
type=str,
help="""Relative or absolute path for output HDF5
file.""",
default=None,
)
parser.add_argument("-survey", "--survey", type=str, default="hdr2.1")
args = parser.parse_args(argv)
args.log = setup_logging()
fileh = tb.open_file(
args.outfilename, mode="w", title=args.survey.upper() + " Fiber Index file "
)
shotlist = Table.read(
args.shotlist, format="ascii.no_header", names=["date", "obs"]
)
tableFibers = fileh.create_table(
fileh.root,
"FiberIndex",
VIRUSFiberIndex,
"Survey Fiber Coord Info",
expectedrows=140369264,
)
# set up HEALPIX options
Nside = 2 ** 15
hp.max_pixrad(Nside, degrees=True) * 3600 # in unit of arcsec
config = HDRconfig(survey=args.survey)
badshot = np.loadtxt(config.badshot, dtype=int)
for shotrow in shotlist:
datevshot = str(shotrow["date"]) + "v" + str(shotrow["obs"]).zfill(3)
shotid = int(str(shotrow["date"]) + str(shotrow["obs"]).zfill(3))
date = shotrow["date"]
try:
args.log.info("Ingesting %s" % datevshot)
file_obs = tb.open_file(op.join(args.shotdir, datevshot + ".h5"), "r")
tableFibers_i = file_obs.root.Data.FiberIndex
for row_i in tableFibers_i:
row_main = tableFibers.row
for col in tableFibers_i.colnames:
row_main[col] = row_i[col]
fiberid = row_i["fiber_id"]
try:
row_main["healpix"] = hp.ang2pix(
Nside, row_i["ra"], row_i["dec"], lonlat=True)
except:
row_main["healpix"] = 0
row_main["shotid"] = shotid
row_main["date"] = date
row_main["datevobs"] = datevshot
row_main["specid"] = fiberid[20:23]
row_main["ifuslot"] = fiberid[24:27]
row_main["ifuid"] = fiberid[28:31]
row_main["amp"] = fiberid[32:34]
row_main.append()
file_obs.close()
except:
if shotid in badshot:
pass
else:
args.log.error("could not ingest %s" % datevshot)
tableFibers.cols.healpix.create_csindex()
tableFibers.cols.ra.create_csindex()
tableFibers.flush()
fileh.close()
def return_flux_limit_model(flim_model, cache_sim_interp=True, verbose=False):
"""
Return the noise -> 50% completeness
scaling and a function for the
completeness curves
"""
# old models for legacy support
if flim_model in ["hdr1", "hdr2pt1"]:
if verbose:
print("Using flim model: {:s}".format(flim_model))
return return_flux_limit_model_old(flim_model)
models = {
"one_sigma_nearest_pixel":
ModelInfo("curves_v1", [1.0],
None,
False,
False,
snlow=0.999999,
snhigh=1.000001),
"one_sigma_interpolate":
ModelInfo("curves_v1", [1.0],
None,
True,
False,
snlow=0.999999,
snhigh=1.000001),
"hdr2pt1pt1":
ModelInfo(
"curves_v1",
[2.76096687e-03, 2.09732448e-02, 7.21681512e-02, 3.36040017e+00],
None, False, False),
"hdr2pt1pt3":
ModelInfo(
"curves_v1",
[6.90111625e-04, 5.99169372e-02, 2.92352510e-01, 1.74348070e+00],
None, False, False),
"v1":
ModelInfo("curves_v1", [
-8.80650683e-02, 2.03488098e+00, -1.73733048e+01, 6.56038443e+01,
-8.84158092e+01
], None, False, True),
"v1.1":
ModelInfo("curves_v1", [
-8.80650683e-02, 2.03488098e+00, -1.73733048e+01, 6.56038443e+01,
-8.84158092e+01
], None, True, True),
"v2":
ModelInfo("curves_v1", [1.0, 0.0], [
-1.59395767e-14, 3.10388106e-10, -2.26855051e-06, 7.38507004e-03,
-8.06953973e+00
],
False,
True,
lw_scaling=linewidth_f50_scaling_v1)
}
default = "v2"
if not flim_model:
flim_model = default
model = models[flim_model]
if verbose:
print("Using flim model: {:s}".format(flim_model))
if flim_model_cache.cached_model == flim_model and cache_sim_interp:
sinterp = flim_model_cache.cached_sim_interp
else:
conf = HDRconfig()
fdir = conf.flim_sim_completeness
fdir = join(fdir, model.snfile_version)
sinterp = SimulationInterpolator(fdir,
model.dont_interp_to_zero,
snmode=False,
verbose=verbose)
# save model in cache
if cache_sim_interp:
flim_model_cache.cached_model = flim_model
flim_model_cache.cached_sim_interp = sinterp
def f50_from_noise(noise, lambda_, sncut, linewidth=None):
"""
Return the 50% completeness
flux given noise and S/N cut.
Parameters
----------
noise : float
the noise from the
sensitivity cubes
sncut : float
the signal to noise
cut to assume
linewidth : float
the linewidth in A,
only used with supported
models
Returns
-------
f50s : array
the fluxes at 50%
completeness
"""
try:
if sncut < model.snlow or sncut > model.snhigh:
print("WARNING: model {:s} not calibrated for this S/N range".
format(flim_model))
except ValueError:
if any(sncut < 4.5) or any(ncut > 7.5):
print("WARNING: model {:s} not calibrated for this S/N range".
format(flim_model))
bad = noise > 998
if model.wavepoly:
noise = noise * polyval(model.wavepoly, lambda_)
if type(linewidth) != type(None):
if type(model.lw_scaling) != type(None):
lw_scale = model.lw_scaling(linewidth, sncut)
else:
raise NoLineWidthModel(
"Linewidth dependence not available for this flim model")
noise = noise * lw_scale
snmult = polyval(model.snpoly, sncut)
f50 = snmult * noise
# keep bad values unscaled
try:
f50[bad] = 999
except TypeError:
if bad:
f50 = 999
return f50
return f50_from_noise, sinterp, model.interp_sigmas
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.table import Table
from astrowidgets import ImageWidget
import ipywidgets as widgets
from ipywidgets import Layout
from hetdex_api.config import HDRconfig
from hetdex_api.shot import Fibers, get_image2D_amp, open_shot_file
from hetdex_api.survey import Survey, FiberIndex
try: # using HDRconfig
LATEST_HDR_NAME = HDRconfig.LATEST_HDR_NAME
CONFIG_HDR2 = HDRconfig('hdr2.1')
CONFIG_HDR3 = HDRconfig('hdr3')
except Exception as e:
print("Warning! Cannot find or import HDRconfig from hetdex_api!!", e)
LATEST_HDR_NAME = "hdr2.1"
OPEN_DET_FILE = None
DET_HANDLE = None
class AmpWidget:
def __init__(
self,
survey=LATEST_HDR_NAME,
coords=None,
radius=3.0,
def __init__(self,
survey=LATEST_HDR_NAME,
load_fiber_table=False,
loadall=False):
"""
Initialize the Fiber class for a given data release
Parameters
----------
survey : string
Data release you would like to load, i.e., 'hdr1','HDR2'
This is case insensitive.
load_fiber_table : bool
Option to read in all fibers. This takes about a minute
and will use a lot of memory.
Returns
-------
FiberIndex class object
"""
self.survey = survey
if self.survey == "hdr1":
print("Sorry there is no FiberIndex for hdr1")
return None
global config
config = HDRconfig(survey=survey.lower())
self.filename = config.fiberindexh5
self.hdfile = tb.open_file(self.filename, mode="r")
self.fiber_table = None
try:
self.fibermaskh5 = tb.open_file(config.fibermaskh5, 'r')
except:
print('Could not find fiber mask file in {}'.format(
config.fibermaskh5))
self.fibermaskh5 = None
if load_fiber_table:
self.fiber_table = Table(self.hdfile.root.FiberIndex.read())
self.coords = SkyCoord(
self.fiber_table["ra"] * u.degree,
self.fiber_table["dec"] * u.degree,
frame="icrs",
)
# add masking info if found
if self.fibermaskh5 is not None:
self.mask_table = Table(self.fibermaskh5.root.Flags.read())
self.fiber_table = hstack([self.fiber_table, self.mask_table])
for row in self.fiber_table:
if row['fiber_id_1'] == row['fiber_id_2']:
continue
else:
print(
'Something is wrong. Mismatcheded fiber:{} and {}'.
format(row['fiber_id_1'], row['fiber_id_2']))
self.fiber_table.rename_column('fiber_id_1', 'fiber_id')
self.fiber_table.remove_column('fiber_id_2')
def main(argv=None):
""" Main Function """
# Call initial parser from init_utils
parser = ap.ArgumentParser(description="""Create HDF5 file.""",
add_help=True)
parser.add_argument(
"-d",
"--date",
help="""Date, e.g., 20170321, YYYYMMDD""",
type=str,
default=None,
)
parser.add_argument(
"-o",
"--observation",
help='''Observation number, "00000007" or "7"''',
type=str,
default=None,
)
parser.add_argument(
"-r",
"--rootdir",
help="""Root Directory for Reductions""",
type=str,
default="/data/05350/ecooper/",
)
parser.add_argument(
"-of",
"--outfilename",
type=str,
help="""Relative or absolute path for output HDF5
file.""",
default=None,
)
parser.add_argument(
"-a",
"--append",
help="""Appending to existing file.""",
action="count",
default=0,
)
parser.add_argument(
"-dp",
"--detect_path",
help="""Path to detections""",
type=str,
# default="/work/00115/gebhardt/maverick/detect",
default="/data/00115/gebhardt/detect")
parser.add_argument("-survey",
"--survey",
help="""{hdr1, hdr2, hdr2.1}""",
type=str,
default="hdr2.1")
parser.add_argument("-tar",
"--tar",
help="""Flag to open tarred multifits""",
action="store_true")
parser.add_argument("-tp", "--tmppath", type=str, default=os.getcwd())
args = parser.parse_args(argv)
args.log = setup_logging()
global hdr_survey
hdr_survey = args.survey
if args.survey != hdr_survey:
args.log.warning("Hard coded hdr_survey does not match input survey.")
sys.exit()
config = HDRconfig(args.survey)
badshots = np.loadtxt(config.badshot, dtype=int)
# Get the daterange over which reduced files will be collected
files = get_files(args)
datestr = "%sv%03d" % (args.date, int(args.observation))
filepath = "%s/%s/dithall.use" % (args.detect_path, datestr)
shotid = int(str(args.date) + str(args.observation).zfill(3))
badshotflag = False
if shotid in badshots:
badshotflag = True
args.log.warning('Shot is in badshot list. Ingesting anyways')
try:
T = Table.read(filepath, format="ascii")
except:
T = None
if badshotflag:
args.log.info("Could not open the dithall file from %s" % filepath)
else:
args.log.error("Could not open the dithall file from %s" %
filepath)
# Creates a new file if the "--append" option is not set or the file
# does not already exist.
does_exist = False
if op.exists(args.outfilename) and args.append:
# for now this is used to update the shot table
fileh = tb.open_file(args.outfilename, "a")
fileh.remove_node(fileh.root, "Shot")
fileh.create_table(fileh.root, "Shot", VIRUSShot, "Shot Info")
#does_exist = True
else:
outfile = op.join(args.tmppath, args.outfilename)
fileh = tb.open_file(outfile, "w")
group = fileh.create_group(fileh.root, "Data",
"VIRUS Fiber Data and Metadata")
fileh.create_table(group, "Fibers", VIRUSFiber, "Fiber Info")
fileh.create_table(fileh.root, "Shot", VIRUSShot, "Shot Info")
fileh.create_table(group, "Images", VIRUSImage, "Image Info")
fileh.create_table(group, "FiberIndex", VIRUSFiberIndex,
"Fiber Coord Info")
# Grab the fiber table and amplifier table for writing
fibtable = fileh.root.Data.Fibers
shottable = fileh.root.Shot
imagetable = fileh.root.Data.Images
fibindextable = fileh.root.Data.FiberIndex
if does_exist:
cnt = shottable[-1]["obsind"]
else:
cnt = 1
if args.tar == True:
shot = shottable.row
n_ifu = {}
for file_i in files:
tar = tarfile.open(name=file_i, mode="r")
members = tar.getmembers()
fn = tar.extractfile(members[0])
filename = fn.name
idx = filename.find("exp")
expn = filename[idx:idx + 5]
n_ifu[expn] = int(len(members) / 4)
success = append_shot_to_table(shot, shottable, fn, cnt)
if args.append:
continue
for member in members:
fn = tar.extractfile(member)
args.log.info("Working on %s" % member.name)
fib = fibtable.row
im = imagetable.row
fibindex = fibindextable.row
success = append_fibers_to_table(fibindex, fib, im, fn, cnt, T,
args)
if success:
fibtable.flush()
imagetable.flush()
fibindextable.flush()
shot["n_ifu"] = n_ifu["exp01"]
else:
shot = shottable.row
try:
filename = files[0]
idx = filename.find("exp")
expn = filename[idx:idx + 5]
f_exp01 = re.sub(expn, "exp01", filename)
f_exp02 = re.sub(expn, "exp02", filename)
f_exp03 = re.sub(expn, "exp03", filename)
success = append_shot_to_table(shot, shottable, f_exp01, cnt)
success = append_shot_to_table(shot, shottable, f_exp02, cnt)
success = append_shot_to_table(shot, shottable, f_exp03, cnt)
n_ifu = int(len(files) / 12)
shot["n_ifu"] = n_ifu
except:
if badshotflag:
args.log.info('No multifits files for %s' % datestr)
else:
args.log.error('No multifits files for %s' % datestr)
if args.append:
args.log.info('Appending shot table only.')
else:
for fn in files:
args.log.info("Working on %s" % fn)
fib = fibtable.row
im = imagetable.row
fibindex = fibindextable.row
success = append_fibers_to_table(fibindex, fib, im, fn, cnt, T,
args)
if success:
fibtable.flush()
imagetable.flush()
fibindextable.flush()
shot.append()
# create completely sorted index on the specid to make queries against that column much faster
# specid chosen as the old multi*fits naming started with specid and it is fixed vs ifuslot and ifuid
# for any given shot
if args.append:
pass
else:
fibtable.cols.ra.create_csindex()
fibindextable.cols.ra.create_csindex()
imagetable.cols.multiframe.create_csindex()
fibindextable.cols.multiframe.create_csindex()
fibtable.cols.multiframe.create_csindex()
fibtable.flush()
fibindextable.flush()
imagetable.flush()
shottable.flush()
fileh.close()
# remove all temporary multifits
if args.tar:
datestr = "d%ss%03d" % (args.date, int(args.observation))
datepath = op.join(args.tmppath, datestr)
shutil.rmtree(datepath, ignore_errors=True)
outfile = op.join(args.tmppath, args.outfilename)
try:
shutil.move(outfile, args.outfilename)
except:
os.remove(args.outfilename)
shututil.move(outfile, args.outfilename)
"/work/03946/hetdex/hdr2.1/detect/image_db",
"/work/03261/polonius/hdr2.1/detect/image_db"
],
}
#
# add paths from hetdex_api to search (place in first position)
#
for v in DICT_DB_PATHS.keys():
try:
release_number = v/10.0
if v % 10 == 0:
release_string = "hdr{:d}".format(int(release_number))
else:
release_string = "hdr{:2.1f}".format(release_number)
DICT_DB_PATHS[v].insert(0,op.join(HDRconfig(survey=release_string).elix_dir))
except:# Exception as e:
#print(e)
continue
def get_elixer_report_db_path(detectid,report_type="report"):
"""
Return the top (first found) path to database file based on the detectid (assumes the HDR version is part of the
prefix, i.e. HDR1 files are 1000*, HDR2 are 2000*, and so on)
:param detectid:
:param report_type: choose one of "report" (normal ELiXer report image) [default]
"nei" (ELiXer neighborhood image)
"mini" (ELiXer mini-report image for phone app)
:return: None or database filename
"""
def main(argv=None):
""" Main Function """
# Call initial parser from init_utils
parser = ap.ArgumentParser(
description="""Append HDF5 Calibration info table.""", add_help=True)
parser.add_argument(
"-d",
"--date",
help="""Date, e.g., 20170321, YYYYMMDD""",
type=str,
default=None,
)
parser.add_argument(
"-o",
"--observation",
help='''Observation number, "00000007" or "7"''',
type=str,
default=None,
)
parser.add_argument(
"-tp",
"--tpdir",
help="""Directory for Throughput Info""",
type=str,
default="/scratch/00115/gebhardt/detect",
)
parser.add_argument(
"-detdir",
"--detectdir",
help="""Directory for Detect Info""",
type=str,
default="/scratch/03946/hetdex/detect",
)
parser.add_argument(
"-of",
"--outfilename",
type=str,
help="""Relative or absolute path for output HDF5file.""",
default=None,
)
parser.add_argument(
"-a",
"--append",
help="""Appending to existing shot HDF5 file.""",
action="count",
default=0,
)
parser.add_argument("-survey",
"--survey",
help="""{hdr1, hdr2, hdr2.1, hdr3}""",
type=str,
default="hdr3")
args = parser.parse_args(argv)
args.log = setup_logging()
# Creates a new file if the "--append" option is not set or the file
# does not already exist.
does_exist = False
if op.exists(args.outfilename) and args.append:
fileh = tb.open_file(args.outfilename, "a")
args.log.info("Appending calibration info to %s" % args.outfilename)
does_exist = True
else:
fileh = tb.open_file(args.outfilename, "w")
args.log.info("Writingcalibration info to %s" % args.outfilename)
shotid = int(str(args.date) + str(args.observation).zfill(3))
#check if shotid is in badlist
config = HDRconfig(args.survey)
badshots = np.loadtxt(config.badshot, dtype=int)
badshotflag = False
if shotid in badshots:
badshotflag = True
try:
fileh.remove_node(fileh.root.Calibration, recursive=True)
except:
args.log.info("Creating new Calibration group")
group = fileh.create_group(fileh.root, "Calibration",
"HETDEX Calibration Info")
groupThroughput = fileh.create_group(group, "Throughput",
"Throughput Curve")
datevshot = str(args.date) + "v" + str(args.observation.zfill(3))
tpfile = op.join(args.detectdir, "tp", datevshot + "sedtp_f.dat")
try:
tp_data = ascii.read(
tpfile,
names=[
"wavelength",
"throughput",
"tp_low",
"tp_high",
"rat_poly",
"tp_gband",
],
)
tp_4540 = tp_data["throughput"][np.where(
tp_data["wavelength"] == 4540.0)][0]
tp_array = fileh.create_table(groupThroughput, "throughput",
tp_data.as_array())
tp_array.set_attr("filename", tpfile)
except:
args.log.warning("Could not include %s" % tpfile)
tppngfile = op.join(
args.tpdir,
str(args.date) + "v" + str(args.observation.zfill(3)),
"res",
str(args.date) + "v" + str(args.observation.zfill(3)) + "sedtpa.png",
)
try:
pngimarr = plt.imread(tppngfile)
pngim = fileh.create_array(groupThroughput, "tp_png", pngimarr)
pngim.attrs["CLASS"] = "IMAGE"
except:
args.log.warning("Could not include %s" % tppngfile)
# add virus FWHM and response_4540 to the Shot table
shottable = fileh.root.Shot
fwhm_file = op.join(args.detectdir, "fwhm.all")
try:
fwhm_tab = Table.read(fwhm_file, format='ascii.no_header')
sel_datevobs = fwhm_tab['col1'] == str(args.date) + "v" + str(
args.observation.zfill(3))
for row in shottable:
row["fwhm_virus"] = float(fwhm_tab['col2'][sel_datevobs])
row["fwhm_virus_err"] = float(fwhm_tab['col3'][sel_datevobs])
row["nstars_fit_fwhm"] = int(fwhm_tab['col4'][sel_datevobs])
row["response_4540"] = tp_4540
row.update()
except:
if badshotflag:
args.log.warning(
"Could not include cal info in shot table for %s" % datevshot)
else:
args.log.error("Could not include cal info in shot table for %s" %
datevshot)
fileh.close()
def __init__(self,
datevshot,
release=None,
flim_model=None,
rad=3.5,
ffsky=False,
wavenpix=3,
d25scale=3.0,
verbose=False,
sclean_bad=True,
log_level="WARNING"):
self.conf = HDRconfig()
self.extractor = Extract()
self.shotid = int(datevshot.replace("v", ""))
self.date = datevshot[:8]
self.rad = rad
self.ffsky = ffsky
self.wavenpix = wavenpix
self.sclean_bad = sclean_bad
logger = logging.getLogger(name="ShotSensitivity")
logger.setLevel(log_level)
if verbose:
raise DeprecationWarning(
"Using verbose is deprecated, set log_level instead")
logger.setLevel("DEBUG")
logger.info("shotid: {:d}".format(self.shotid))
if not release:
self.release = self.conf.LATEST_HDR_NAME
else:
self.release = release
logger.info("Data release: {:s}".format(self.release))
self.survey = Survey(survey=self.release)
# Set up flux limit model
self.f50_from_noise, self.sinterp, interp_sigmas \
= return_flux_limit_model(flim_model,
cache_sim_interp=False,
verbose=verbose)
# Generate astrometry for this shot
survey_sel = (self.survey.shotid == self.shotid)
self.shot_pa = self.survey.pa[survey_sel][0]
self.shot_ra = self.survey.ra[survey_sel][0]
self.shot_dec = self.survey.dec[survey_sel][0]
rot = 360.0 - (self.shot_pa + 90.)
self.tp = TangentPlane(self.shot_ra, self.shot_dec, rot)
#Set up masking
logger.info("Using d25scale {:f}".format(d25scale))
self.setup_mask(d25scale)
# Set up spectral extraction
if release == "hdr1":
fwhm = self.survey.fwhm_moffat[survey_sel][0]
else:
fwhm = self.survey.fwhm_virus[survey_sel][0]
logger.info("Using Moffat PSF with FWHM {:f}".format(fwhm))
self.moffat = self.extractor.moffat_psf(fwhm, 3. * rad, 0.25)
self.extractor.load_shot(self.shotid, fibers=True, survey=self.release)
# Set up the focal plane astrometry
fplane_table = self.extractor.shoth5.root.Astrometry.fplane
# Bit of a hack to avoid changing pyhetdex
with NamedTemporaryFile(mode='w') as tpf:
for row in fplane_table.iterrows():
tpf.write(
"{:03d} {:8.5f} {:8.5f} {:03d} {:03d} {:03d} {:8.5f} {:8.5f}\n"
.format(row['ifuslot'], row['fpx'], row['fpy'],
row['specid'], row['specslot'], row['ifuid'],
row['ifurot'], row['platesc']))
tpf.seek(0)
self.fplane = FPlane(tpf.name)
from __future__ import print_function
import numpy as np
from astropy.table import Table, join
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy import wcs
from regions import EllipseSkyRegion, EllipsePixelRegion
from hetdex_api.config import HDRconfig
from hetdex_api.survey import FiberIndex
config = HDRconfig()
def amp_flag_from_coords(coords,
FibIndex,
bad_amps_table,
radius=3. * u.arcsec,
shotid=None):
"""
Returns a boolean flag whether the amp has been flagged usable
Parameters
----------
coords
an astropy.coordinates SkyCoord object
FibIndex
def main(argv=None):
''' Main Function '''
# Call initial parser from init_utils
parser = ap.ArgumentParser(description="""Create HDF5 file.""",
add_help=True)
parser.add_argument("-d", "--date",
help='''Date, e.g., 20170321, YYYYMMDD''',
type=str, default=None)
parser.add_argument("-o", "--observation",
help='''Observation number, "00000007" or "7"''',
type=str, default=None)
parser.add_argument("-r", "--rootdir",
help='''Root Directory for Reductions''',
type=str, default='/data/00115/gebhardt/calfits/')
parser.add_argument('-of', '--outfilename', type=str,
help='''Relative or absolute path for output HDF5
file.''', default=None)
parser.add_argument("-survey", "--survey", help='''{hdr1, hdr2, hdr2.1}''',
type=str, default='hdr2.1')
args = parser.parse_args(argv)
args.log = setup_logging()
calfiles = get_cal_files(args)
datestr = '%sv%03d' % (args.date, int(args.observation))
shotid = int(str(args.date) + str(args.observation).zfill(3))
#check if shotid is in badlist
config = HDRconfig(args.survey)
badshots = np.loadtxt(config.badshot, dtype=int)
badshotflag = False
if shotid in badshots:
badshotflag = True
if len(calfiles) == 0:
if badshotflag:
args.log.warning("No calfits file to append for %s" % datestr)
else:
args.log.error("No calfits file to append for %s" % datestr)
sys.exit('Exiting cal append script for %s' % datestr)
if op.exists(args.outfilename):
fileh = tb.open_file(args.outfilename, 'a')
else:
args.log.error('Problem opening : ' + args.outfilename)
sys.exit('Exiting Script')
args.log.info('Appending calibrated fiber arrays to ' + args.outfilename)
fibtable = fileh.root.Data.Fibers
for calfile in calfiles:
multi = calfile[49:60]
try:
cal_table = get_cal_table(calfile)
except:
continue
args.log.error('Could not ingest calfile: %s' % calfile)
args.log.info('Working on IFU ' + multi )
for amp_i in ['LL','LU','RL','RU']:
multiframe_i = 'multi_'+ multi + '_' + amp_i
for fibrow in fibtable.where('multiframe == multiframe_i'):
idx = (cal_table['expnum'] == fibrow['expnum']) * (cal_table['multiframe'] == fibrow['multiframe'].decode()) * (cal_table['fibidx'] == fibrow['fibidx'])
if np.sum(idx) >= 1:
fibrow['calfib'] = cal_table['calfib'][idx]
fibrow['calfibe'] = cal_table['calfibe'][idx]
fibrow['calfib_counts'] = cal_table['calfib_counts'][idx]
fibrow['calfibe_counts'] = cal_table['calfibe_counts'][idx]
fibrow['spec_fullsky_sub'] = cal_table['spec_fullsky_sub'][idx]
fibrow.update()
#else:
# args.log.warning("No fiber match for %s" % fibrow['fiber_id'])
args.log.info('Flushing and closing H5 file')
fibtable.flush()
fileh.close()