def adjust_alignment(self,
name_ima_reference,
list_expo=None,
line=None,
filter_name="CousinsR"):
"""Adjust the alignment using a background image
"""
# Selecting the table with the right iexpo
found_expo, list_expo, group_list_expo, align_table = self._select_list_expo(
"OBJECT", "ALL", "processed", list_expo)
if not found_expo:
if self.verbose:
upipe.print_warning(
"No exposure recovered for the fine alignment")
return
self.groupexpo = []
suffix = "_{0}".format(filter_name)
if line is not None:
suffix += "_{0}".format(line)
for gtable in align_table.groups:
mytpl, mymjd = self._get_tpl_meanmjd(gtable)
list_group_expo = gtable['iexpo'].data
list_names_muse = [
joinpath(
self._get_fullpath_expo("OBJECT", "processed"),
'IMAGE_FOV{0}_{1:04d}_{2}.fits'.format(
suffix, iexpo, mytpl)) for iexpo in list_group_expo
]
self.groupexpo.append(
AlignMusePointing(name_ima_reference,
list_names_muse,
flag="mytpl"))
def _select_list_expo(self, expotype, tpl, stage, list_expo=None):
"""Select the expo numbers which exists for a certain expotype
"""
# First selecting the files via the grouped table
tpl_table = self.select_tpl_files(expotype=expotype,
tpl=tpl,
stage=stage)
# Selecting the table with the right iexpo
if list_expo is None:
list_expo = tpl_table['iexpo'].data
# First we isolate the unique values of iexpo from the table
list_expo = np.unique(list_expo)
# Then we select those who exist in the table
# And don't forget to group the table by tpls
group_table = tpl_table[np.isin(tpl_table['iexpo'],
list_expo)].group_by('tpls')
group_list_expo = []
for gtable in group_table:
group_list_expo.append(gtable['iexpo'].data)
found_expo = True
if len(group_table) == 0:
found_expo = False
if self.verbose:
upipe.print_warning(
"No {0} recovered from the {1} astropy file "
"Table - Aborting".format(expotype, stage))
return found_expo, list_expo, group_list_expo, group_table
def __init__(self,
nifu=-1,
first_cpu=0,
ncpu=24,
list_cpu=[],
likwid=default_likwid,
fakemode=True,
domerge=True,
nocache=True,
nochecksum=True):
"""Initialisation of PipeRecipes
"""
# Fake mode
self.fakemode = fakemode
if self.verbose:
if fakemode: upipe.print_warning("WARNING: running in FAKE mode")
else: upipe.print_warning("WARNING: running actual recipes")
if nocache: self.nocache = "nocache"
else: self.nocache = ""
# Addressing CPU by number (cpu0=start, cpu1=end)
self.first_cpu = first_cpu
self.ncpu = ncpu
self.likwid = likwid
self.nifu = nifu
self._set_cpu(first_cpu, ncpu, list_cpu)
self._domerge = domerge
self.nochecksum = nochecksum
def run_twilight(self,
sof_filename='twilight',
tpl="ALL",
update=None,
illum=True):
"""Reducing the files and creating the TWILIGHT CUBE.
Will run the esorex muse_twilight command on all TWILIGHT
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
tpl_gtable = self.select_tpl_files(expotype='TWILIGHT', tpl=tpl)
if len(tpl_gtable) == 0:
if self.verbose:
upipe.print_warning(
"No TWILIGHT recovered from the astropy file Table - Aborting"
)
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the LSF including
# the list of files to be processed for one MASTER Flat
self._add_calib_to_sofdict("BADPIX_TABLE", reset=True)
self._add_calib_to_sofdict("VIGNETTING_MASK")
for gtable in tpl_gtable.groups:
# extract the tpl (string) and mean mjd (float)
tpl, mean_mjd = self._get_tpl_meanmjd(gtable)
self._add_geometry_to_sofdict(tpl)
# Provide the list of files to the dictionary
self._sofdict['SKYFLAT'] = add_listpath(self.paths.rawfiles,
list(gtable['filename']))
# Finding the best tpl for BIAS, FLAT, ILLUM, TRACE, WAVE
if illum:
self._add_tplraw_to_sofdict(mean_mjd, "ILLUM")
self._add_list_tplmaster_to_sofdict(
mean_mjd, ['BIAS', 'FLAT', 'TRACE', 'WAVE'])
# Writing the sof file
self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True)
# Names and folder of final Master Wave
dir_twilight = self._get_fullpath_expo('TWILIGHT', "master")
name_twilight = deepcopy(dic_files_products['TWILIGHT'])
self.recipe_twilight(self.current_sof, dir_twilight, name_twilight,
tpl)
# Write the MASTER TWILIGHT Table and save it
self.save_expo_table('TWILIGHT', tpl_gtable, "master", update=update)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def run_flat(self, sof_filename='flat', tpl="ALL", update=None):
"""Reducing the Flat files and creating a Master Flat
Will run the esorex muse_flat command on all Flats
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
tpl_gtable = self.select_tpl_files(expotype='FLAT', tpl=tpl)
if len(tpl_gtable) == 0:
if self.verbose:
upipe.print_warning(
"No FLAT recovered from the astropy Table - Aborting")
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the FLATs including
# the list of files to be processed for one MASTER Flat
# Adding the bad pixel table
self._add_calib_to_sofdict("BADPIX_TABLE", reset=True)
for gtable in tpl_gtable.groups:
# Provide the list of files to the dictionary
self._sofdict['FLAT'] = add_listpath(self.paths.rawfiles,
list(gtable['filename']))
# extract the tpl (string) and mean mjd (float)
tpl, mean_mjd = self._get_tpl_meanmjd(gtable)
# Adding the best tpc MASTER_BIAS
self._add_tplmaster_to_sofdict(mean_mjd, 'BIAS')
# Writing the sof file
self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True)
# Name of final Master Flat and Trace Table
dir_flat = self._get_fullpath_expo('FLAT', "master")
name_flat = self._get_suffix_product('FLAT')
dir_trace = self._get_fullpath_expo('TRACE', "master")
name_tracetable = self._get_suffix_product('TRACE')
# Run the recipe
self.recipe_flat(self.current_sof, dir_flat, name_flat, dir_trace,
name_tracetable, tpl)
# Write the MASTER FLAT Table and save it
self.save_expo_table('FLAT', tpl_gtable, "master", update=update)
self.save_expo_table('TRACE', tpl_gtable, "master", update=update)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def run_standard(self, sof_filename='standard', tpl="ALL", update=None):
"""Reducing the STD files after they have been obtained
Running the muse_standard routine
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
std_table = self.select_tpl_files("STD", tpl=tpl, stage="processed")
if len(std_table) == 0:
if self.verbose:
upipe.print_warning(
"No processed STD recovered from the astropy file Table - Aborting"
)
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the STD Sof
for i in range(len(std_table)):
mytpl = std_table['tpls'][i]
iexpo = np.int(std_table['iexpo'][i])
if tpl != "ALL" and tpl != mytpl:
continue
# Now starting with the standard recipe
self._add_calib_to_sofdict("EXTINCT_TABLE", reset=True)
self._add_calib_to_sofdict("STD_FLUX_TABLE")
self._sofdict['PIXTABLE_STD'] = [
joinpath(
self._get_fullpath_expo("STD", "processed"),
'PIXTABLE_STD_{0}_{1:04d}-{2:02d}.fits'.format(
mytpl, iexpo, j + 1)) for j in range(24)
]
self.write_sof(sof_filename=sof_filename + "_" + mytpl, new=True)
name_std = deepcopy(dic_files_products['STD'])
dir_std = self._get_fullpath_expo('STD', "master")
self.recipe_std(self.current_sof, dir_std, name_std, mytpl)
# Write the MASTER files Table and save it
self.save_expo_table("STD",
std_table,
"master",
aggregate=False,
update=update)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def open_offset_table(self, name_table=None):
"""Read offset table from fits file
"""
if name_table is None:
if not hasattr(self, name_table):
upipe.print_error("No FITS table name provided, Aborting Open")
return None, Table()
if not os.path.isfile(name_table):
upipe.print_warning("FITS Table does not exist yet"
"({0})".format(name_table))
return False, Table()
return True, Table.read(name_table)
def _get_list_muse_images(self):
# test if 1 or several images
if isinstance(self.name_muse_images, str):
self.list_muse_images = [self.name_muse_images]
elif isinstance(self.name_muse_images, list):
self.list_muse_images = self.name_muse_images
else:
upipe.print_warning("Name of images is not a string or a list, "
"please check input name_muse_images")
self.nimages = 0
return
# Number of images to deal with
self.nimages = len(self.list_muse_images)
def __init__(self,
dirname="Config/",
rc_filename=None,
cal_filename=None,
verbose=True,
**kwargs):
"""Define the default parameters (folders/calibration files)
and name suffixes for the MUSE data reduction
"""
self.verbose = verbose
# Will first test if there is an rc_file provided
# If not, it will look for a default rc_filename, the name of which is provided
# above. If not, the hardcoded default will be used.
# First adding the suffix to the dictionaries
# attributing the dictionaries
self._dic_folders = dic_folders
self._dic_input_folders = dic_input_folders
if rc_filename is None:
if not os.path.isfile(default_rc_filename):
upipe.print_warning(
("No filename or {default_rc} file "
"to initialise from. We will use the default hardcoded "
"in the init_musepipe.py module").format(
default_rc=default_rc_filename))
self.init_default_param(dic_user_folders)
else:
self.read_param_file(default_rc_filename, dic_user_folders)
self.rcfile = "default_values"
else:
rcfile = joinpath(dirname, rc_filename)
self.read_param_file(rcfile, dic_user_folders)
self.rcfile = rcfile
# Initialisation of fixed attributes for the structure
self.init_default_param(dic_folders)
self.init_default_param(dic_input_folders)
# Same happens with the calibration files.
# If filename is provided, will use that, otherwise use the hard coded values.
if cal_filename is None:
self.init_default_param(dic_calib_tables)
self.calfile = "default_values"
else:
calfile = joinpath(dirname, cal_filename)
self.read_param_file(calfile, dic_calib_tables)
self.calfile = calfile
def run_lsf(self, sof_filename='lsf', tpl="ALL", update=None):
"""Reducing the LSF files and creating the LSF PROFILE
Will run the esorex muse_lsf command on all Flats
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
tpl_gtable = self.select_tpl_files(expotype='WAVE', tpl=tpl)
if len(tpl_gtable) == 0:
if self.verbose:
upipe.print_warning(
"No WAVE recovered from the astropy file Table - Aborting")
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the LSF including
# the list of files to be processed for one MASTER Flat
# Adding Badpix table and Line Catalog
self._add_calib_to_sofdict("BADPIX_TABLE", reset=True)
self._add_calib_to_sofdict("LINE_CATALOG")
for gtable in tpl_gtable.groups:
# Provide the list of files to the dictionary
self._sofdict['ARC'] = add_listpath(self.paths.rawfiles,
list(gtable['filename']))
# extract the tpl (string) and mean mjd (float)
tpl, mean_mjd = self._get_tpl_meanmjd(gtable)
# Finding the best tpl for BIAS, TRACE, WAVE
self._add_list_tplmaster_to_sofdict(mean_mjd,
['BIAS', 'TRACE', 'WAVE'])
# Writing the sof file
self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True)
# Name of final Master Wave
dir_lsf = self._get_fullpath_expo('LSF', "master")
name_lsf = self._get_suffix_product('LSF')
# Run the recipe
self.recipe_lsf(self.current_sof, dir_lsf, name_lsf, tpl)
# Write the MASTER LSF PROFILE Table and save it
self.save_expo_table('LSF', tpl_gtable, "master", update=update)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def save_expo_table(self,
expotype,
tpl_gtable,
stage="master",
fits_tablename=None,
aggregate=True,
suffix="",
overwrite=None,
update=None):
"""Save the Expo (Master or not) Table corresponding to the expotype
"""
self._set_option_astropy_table(overwrite, update)
if fits_tablename is None:
fits_tablename = self._get_fitstablename_expo(
expotype, stage, suffix)
attr_expo = self._get_attr_expo(expotype)
full_tablename = joinpath(self.paths.astro_tables, fits_tablename)
if aggregate:
table_to_save = tpl_gtable.groups.aggregate(np.mean)['tpls', 'mjd',
'tplnexp']
else:
table_to_save = copy.copy(tpl_gtable)
# If the file already exists
if os.path.isfile(full_tablename):
# Check if we update
if self._update_astropy_table:
# Reading the existing table
upipe.print_warning(
"Reading the existing Astropy table {0}".format(
fits_tablename))
existing_table = Table.read(full_tablename, format="fits")
# first try to see if they are compatible by using vstack
try:
stack_temptable = vstack([existing_table, table_to_save],
join_type='exact')
upipe.print_warning(
"Updating the existing Astropy table {0}".format(
fits_tablename))
table_to_save = apy.table.unique(stack_temptable,
keep='first')
except TableMergeError:
upipe.print_warning(
"Astropy Table cannot be joined to the existing one")
return
# Check if we want to overwrite or add the line in
elif not self._overwrite_astropy_table:
upipe.print_warning(
"Astropy Table {0} already exists, "
" use overwrite_astropy_table to overwrite it".format(
fits_tablename))
return
table_to_save.write(full_tablename, format="fits", overwrite=True)
setattr(self._dic_tables[stage], attr_expo, table_to_save)
def read_astropy_table(self, expotype=None, stage="master"):
"""Read an existing Masterfile data table to start the pipeline
"""
# Read the astropy table
name_table = self._get_fitstablename_expo(expotype, stage)
if not os.path.isfile(name_table):
upipe.print_warning(
"Astropy table {0} does not exist - setting up an empty one".
format(name_table))
return Table([[], [], []], names=['tpls', 'mjd', 'tplnexp'])
else:
if self.verbose:
upipe.print_info(
"Reading Astropy fits Table {0}".format(name_table))
return Table.read(name_table, format="fits")
def run_bias(self, sof_filename='bias', tpl="ALL", update=None):
"""Reducing the Bias files and creating a Master Bias
Will run the esorex muse_bias command on all Biases
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
tpl_gtable = self.select_tpl_files(expotype='BIAS', tpl=tpl)
if len(tpl_gtable) == 0:
if self.verbose:
upipe.print_warning(
"No BIAS recovered from the astropy Table - Aborting")
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the BIASES including
# the list of files to be processed for one MASTER BIAS
# Adding the bad pixel table
self._add_calib_to_sofdict("BADPIX_TABLE", reset=True)
for gtable in tpl_gtable.groups:
# Provide the list of files to the dictionary
self._sofdict['BIAS'] = add_listpath(self.paths.rawfiles,
list(gtable['filename']))
# extract the tpl (string)
tpl = gtable['tpls'][0]
# Writing the sof file
self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True)
# Name of final Master Bias
name_bias = self._get_suffix_product('BIAS')
dir_bias = self._get_fullpath_expo('BIAS', "master")
# Run the recipe
self.recipe_bias(self.current_sof, dir_bias, name_bias, tpl)
# Write the MASTER BIAS Table and save it
self.save_expo_table('BIAS', tpl_gtable, "master", update=update)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def save_fits_offset_table(self,
name_output_table=None,
overwrite=False,
suffix=""):
"""Save the Offsets into a fits Table
"""
if name_out_table is None: name_out_table = self.name_input_table
self.suffix = suffix
name_output_table = name_output_table.replace(
".fits", "{0}.fits".format(self.suffix))
exist_table, fits_table = self.open_offset_table(self.name_out_table)
if exist_table is None:
upipe.print_error("Save is aborted")
return
# Check if RA_OFFSET is there
if 'RA_OFFSET' in fits_table.columns:
# if yes, then check if the ORIG column is there
if 'RA_OFFSET_ORIG' not in fits_table.columns:
fits_table['RA_OFFSET_ORIG'] = fits_table['RA_OFFSET']
fits_table['DEC_OFFSET_ORIG'] = fits_table['DEC_OFFSET']
# if not, just continue
# as it means the ORIG columns were already done
# Saving the final values
fits_table['RA_OFFSET'] = self.total_off_arcsec[:, 0] / 3600.
fits_table['DEC_OFFSET'] = self.total_off_arcsec[:, 1] / 3600.
fits_table['RA_CROSS_OFFSET'] = self.cross_off_arcsec[:, 0] / 3600.
fits_table['DEC_CROSS_OFFSET'] = self.cross_off_arcsec[:, 1] / 3600.
# Writing it up
if exist_table and not overwrite:
upipe.print_warning(
"Table already exists, but overwrite is set to False")
upipe.print_warning(
"If you wish to overwrite the table {0}, "
"please set overwrite to True".format(name_output_table))
return
fits_table.write(self.name_output_table, overwrite=overwrite)
self.name_output_table = name_output_table
def init_guess_offset(self,
firstguess="crosscorr",
name_input_table="OFFSET_LIST.fits"):
"""Initialise first guess, either from cross-correlation (default)
or from an Offset FITS Table
"""
# Implement the guess
if firstguess not in ["crosscorr", "fits"]:
firstguess = "crosscorr"
upipe.print_warning("Keyword 'firstguess' not recognised")
upipe.print_warning(
"Using Cross-Correlation as a first guess of the alignment")
if firstguess == "crosscorr":
self.init_off_arcsec = self.cross_off_arcsec
self.init_off_pixel = self.cross_off_pixel
elif firstguess == "fits":
self.name_input_table = name_input_table
exist_table, self.offset_table = self.open_offset_table(
self.name_input_table)
if exist_table is not True:
upipe.print_warning(
"Fits initialisation table not found, setting init value to 0"
)
self.init_off_pixel *= 0.
self.init_off_arcsec *= 0.
else:
self.init_off_arcsec = np.vstack(
(self.offset_table['RA_OFFSET'],
self.offset_table['DEC_OFFSET'])).T * 3600.
for i in range(self.nimages):
self.init_off_pixel[nima] = arcsec_to_pixel(
self.list_muse_hdu[nima], self.init_off_arcsec[nima])
def select_tpl_files(self, expotype=None, tpl="ALL", stage="raw"):
"""Selecting a subset of files from a certain type
"""
if expotype not in musepipe.listexpo_types.keys():
upipe.print_info(
"ERROR: input {0} is not in the list of possible values".
format(expotype))
return
MUSE_subtable = self._get_table_expo(expotype, stage)
if len(MUSE_subtable) == 0:
if self.verbose:
upipe.print_warning(
"Empty file table of type {0}".format(expotype))
upipe.print_warning(
"Returning an empty Table from the tpl -astropy- selection"
)
return MUSE_subtable
group_table = MUSE_subtable.group_by('tpls')
if tpl == "ALL":
return group_table
else:
return group_table.groups[group_table.groups.keys['tpls'] == tpl]
def __init__(self,
name_reference,
name_muse_images,
median_window=10,
subim_window=10,
dynamic_range=10,
border=50,
hdu_ext=[0, 1],
**kwargs):
"""Initialise the AlignMuseImages class
Keywords
--------
median_window: int
Size of window used in median filter to extract features in
cross-correlation. Should be an odd integer
subim_window: int
Size of window for fitting peak of cross-correlation function
dynamic_range: float
Apply an arctan transform to data to suppress values more than
DynamicRange times the median of the image
border: int
Ignore pixels this close to the border in the cross correlation
xoffset: float
Offset in arcseconds to be added to the cross-correlation offset (X)
yoffset: float
Offset in arcseconds to be added to the cross-correlation offset (Y)
run: boolean
If True will use extra_xyoffset and plot the result
"""
# Some input variables for the cross-correlation
self.use_montage = kwargs.pop("use_montage", True)
self.verbose = kwargs.pop("verbose", True)
self.plot = kwargs.pop("plot", True)
self.border = border
self.subim_window = subim_window
self.median_window = median_window
self.dynamic_range = dynamic_range
self.name_reference = name_reference
self.name_muse_images = name_muse_images
self.name_musehdr = kwargs.pop("name_musehdr", "muse")
self.name_offmusehdr = kwargs.pop("name_offmusehdr", "offsetmuse")
self.name_refhdr = kwargs.pop("name_refhdr", "reference.hdr")
self.flag = kwargs.pop("flag", "")
self._get_list_muse_images(**kwargs)
if self.nimages == 0:
upipe.print_warning("0 MUSE images detected as input")
return
self.list_offmuse_hdu = [None] * self.nimages
self.list_proj_refhdu = [None] * self.nimages
self.cross_off_pixel = np.zeros((self.nimages, 2), dtype=np.float32)
self.extra_off_pixel = np.zeros_like(self.cross_off_pixel)
self.init_off_pixel = np.zeros_like(self.cross_off_pixel)
self.total_off_pixel = np.zeros_like(self.cross_off_pixel)
self.cross_off_arcsec = np.zeros_like(self.cross_off_pixel)
self.extra_off_arcsec = np.zeros_like(self.cross_off_pixel)
self.init_off_arcsec = np.zeros_like(self.cross_off_pixel)
self.total_off_arcsec = np.zeros_like(self.cross_off_pixel)
# Cross normalisation for the images
# This contains the 2 parameters for a linear transformation
self.ima_polynorm = np.zeros((self.nimages, 2), dtype=np.float32)
# Which extension to be used for the ref and muse images
self.hdu_ext = hdu_ext
# Open the Ref and MUSE image
self.open_hdu()
# find the cross correlation peaks for each image
self.find_ncross_peak()
# Initialise the offset
firstguess = kwargs.pop("firstguess", "crosscorr")
self.init_guess_offset(firstguess, **kwargs)
self.total_off_arcsec = self.init_off_arcsec + self.extra_off_arcsec
self.total_off_pixel = self.init_off_pixel + self.extra_off_pixel
def run_scipost(self,
sof_filename='scipost',
expotype="OBJECT",
tpl="ALL",
stage="processed",
list_expo=None,
lambdaminmax=[4000., 10000.],
suffix="",
AC_suffix="_AC",
**kwargs):
"""Scipost treatment of the objects
Will run the esorex muse_scipost routine
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
list_expo: list of integers providing the exposure numbers
"""
# Backward compatibility
old_naming_convention = kwargs.pop("old_naming_convention", False)
# Selecting the table with the right iexpo
found_expo, list_expo, group_list_expo, scipost_table = self._select_list_expo(
expotype, tpl, stage, list_expo)
if not found_expo:
return
if len(list_expo) == 1:
suffix += "_{0:04d}".format(list_expo[0])
# Lambda min and max?
[lambdamin, lambdamax] = lambdaminmax
# Save options
save = kwargs.pop("save", "cube,skymodel,individual")
# Filters
filter_list = kwargs.pop("filter_list", "white,Cousins_R")
filter_for_alignment = kwargs.pop("filter_for_alignment", "Cousins_R")
offset_list = kwargs.pop("offset_list", "True")
autocalib = kwargs.pop("autocalib", "none")
rvcorr = kwargs.pop("rvcorr", "bary")
default_suffix_skycontinuum = ""
suffix_skycontinuum = kwargs.pop("suffix_skycont",
default_suffix_skycontinuum)
if rvcorr != "bary":
upipe.print_warning(
"Scipost will use '{0}' as barycentric "
"correction [Options are bary, helio, geo, none]".format(
rvcorr))
if suffix_skycontinuum != default_suffix_skycontinuum:
upipe.print_warning(
"Scipost will use '{0}' as suffix "
"for the SKY_CONTINUUM files".format(suffix_skycontinuum))
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for scipost
# Selecting either one or all of the exposures
for gtable in scipost_table.groups:
# extract the tpl (string) and mean mjd (float)
tpl, mean_mjd = self._get_tpl_meanmjd(gtable)
# Now starting with the standard recipe
self._add_calib_to_sofdict("EXTINCT_TABLE", reset=True)
self._add_calib_to_sofdict("SKY_LINES")
self._add_calib_to_sofdict("FILTER_LIST")
if autocalib == "user":
self._add_skycalib_to_sofdict("AUTOCAL_FACTORS",
mean_mjd,
'SKY',
"processed",
suffix=AC_suffix)
self._add_astrometry_to_sofdict(tpl)
self._add_skycalib_to_sofdict("STD_RESPONSE", mean_mjd, 'STD')
self._add_skycalib_to_sofdict("STD_TELLURIC", mean_mjd, 'STD')
self._add_skycalib_to_sofdict("SKY_CONTINUUM",
mean_mjd,
'SKY',
"processed",
perexpo=True,
suffix=suffix_skycontinuum)
self._add_tplmaster_to_sofdict(mean_mjd, 'LSF')
if offset_list:
self._sofdict['OFFSET_LIST'] = [
joinpath(
self._get_fullpath_expo(expotype, "processed"),
'{0}_{1}_{2}.fits'.format(
dic_files_products['ALIGN'][0],
filter_for_alignment, tpl))
]
# Selecting only exposures to be treated
# We need to force 'OBJECT' to make sure scipost will deal with the exposure
# even if it is e.g., a SKY
pixtable_name = self._get_suffix_product('OBJECT')
pixtable_name_thisone = self._get_suffix_product(expotype)
self._sofdict[pixtable_name] = []
list_group_expo = gtable['iexpo'].data
for iexpo in list_group_expo:
if old_naming_convention:
self._sofdict[pixtable_name] += [
joinpath(
self._get_fullpath_expo(expotype, "processed"),
'{0}_{1:04d}-{2:02d}.fits'.format(
pixtable_name_thisone, iexpo, j + 1))
for j in range(24)
]
else:
self._sofdict[pixtable_name] += [
joinpath(
self._get_fullpath_expo(expotype, "processed"),
'{0}_{1}_{2:04d}-{3:02d}.fits'.format(
pixtable_name_thisone, tpl, iexpo, j + 1))
for j in range(24)
]
self.write_sof(sof_filename="{0}_{1}{2}_{3}".format(
sof_filename, expotype, suffix, tpl),
new=True)
# products
dir_products = self._get_fullpath_expo(expotype, "processed")
name_products, suffix_products, suffix_finalnames = self._get_scipost_products(
save, list_group_expo, filter_list)
self.recipe_scipost(self.current_sof,
tpl,
expotype,
dir_products,
name_products,
suffix_products,
suffix_finalnames,
lambdamin=lambdamin,
lambdamax=lambdamax,
save=save,
list_expo=list_group_expo,
suffix=suffix,
filter_list=filter_list,
autocalib=autocalib,
rvcorr=rvcorr,
**kwargs)
# Write the MASTER files Table and save it
self.save_expo_table(
expotype,
scipost_table,
"reduced",
"IMAGES_FOV_{0}{1}_{2}_list_table.fits".format(
expotype, suffix, tpl),
aggregate=False,
update=True)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def run_sky(self,
sof_filename='sky',
tpl="ALL",
fraction=0.8,
update=None):
"""Reducing the SKY after they have been scibasic reduced
Will run the esorex muse_create_sky routine
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
sky_table = self._get_table_expo("SKY", "processed")
if len(sky_table) == 0:
if self.verbose:
upipe.print_warning(
"No SKY recovered from the astropy file Table - Aborting")
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the LSF including
# the list of files to be processed for one MASTER Flat
for i in range(len(sky_table)):
mytpl = sky_table['tpls'][i]
mymjd = sky_table['mjd'][i]
iexpo = np.int(sky_table['iexpo'][i])
if tpl != "ALL" and tpl != mytpl:
continue
# Now starting with the standard recipe
self._add_calib_to_sofdict("EXTINCT_TABLE", reset=True)
self._add_calib_to_sofdict("SKY_LINES")
self._add_skycalib_to_sofdict("STD_RESPONSE", mymjd, 'STD')
self._add_skycalib_to_sofdict("STD_TELLURIC", mymjd, 'STD')
self._add_tplmaster_to_sofdict(mymjd, 'LSF')
self._sofdict['PIXTABLE_SKY'] = [
joinpath(
self._get_fullpath_expo("SKY", "processed"),
'PIXTABLE_SKY_{0}_{1:04d}-{2:02d}.fits'.format(
mytpl, iexpo, j + 1)) for j in range(24)
]
self.write_sof(sof_filename=sof_filename +
"{0:02d}".format(iexpo) + "_" + mytpl,
new=True)
dir_sky = self._get_fullpath_expo('SKY', "processed")
name_sky = deepcopy(dic_files_products['SKY'])
self.recipe_sky(self.current_sof, dir_sky, name_sky, mytpl, iexpo,
fraction)
# Write the MASTER files Table and save it
self.save_expo_table("SKY",
sky_table,
"processed",
aggregate=False,
update=update)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def init_raw_table(self, reset=False):
""" Create a fits table with all the information from
the Raw files
Also create an astropy table with the same info
"""
if self.verbose:
upipe.print_info("Creating the astropy fits raw data table")
if reset or not hasattr(self, "Tables"):
self._reset_tables()
# Testing if raw table exists
name_table = self._get_fitstablename_expo('RAWFILES', "raw")
# ---- File exists - we READ it ------------------- #
overwrite = True
if os.path.isfile(name_table):
if self._overwrite_astropy_table:
upipe.print_warning("The raw-files table will be overwritten")
else:
upipe.print_warning("The raw files table already exists")
upipe.print_warning(
"If you wish to overwrite it, "
" please turn on the 'overwrite_astropy_table' option to 'True'"
)
upipe.print_warning(
"In the meantime, the existing table will be read and used"
)
self.Tables.Rawfiles = self.read_astropy_table(
'RAWFILES', "raw")
overwrite = False
# ---- File does not exist - we create it ---------- #
if overwrite:
# Check the raw folder
self.goto_folder(self.paths.rawfiles)
# Get the list of files from the Raw data folder
files = os.listdir(".")
smalldic = {"FILENAME": ['filename', '', str, '100A']}
fulldic = listexpo_files.copy()
fulldic.update(smalldic)
# Init the lists
MUSE_infodic = {}
for key in fulldic.keys():
MUSE_infodic[key] = []
# Looping over the files
for f in files:
# Excluding the files without MUSE and fits.fz
if ('MUSE' in f) and ('.fits.fz') in f:
MUSE_infodic['FILENAME'].append(f)
header = pyfits.getheader(f, 0)
for k in listexpo_files.keys():
[namecol, keyword, func, form] = listexpo_files[k]
MUSE_infodic[k].append(func(header[keyword]))
# Transforming into numpy arrayimport pymusepipe
for k in fulldic.keys():
MUSE_infodic[k] = np.array(MUSE_infodic[k])
# Getting a sorted array with indices
idxsort = np.argsort(MUSE_infodic['FILENAME'])
# Creating the astropy table
self.Tables.Rawfiles = Table([MUSE_infodic['FILENAME'][idxsort]],
names=['filename'],
meta={'name': 'raw file table'})
# Creating the columns
for k in fulldic.keys():
[namecol, keyword, func, form] = fulldic[k]
self.Tables.Rawfiles[namecol] = MUSE_infodic[k][idxsort]
# Writing up the table
self.Tables.Rawfiles.write(name_table,
format="fits",
overwrite=self._overwrite_astropy_table)
# Going back to the original folder
self.goto_prevfolder()
# Sorting the types ====================================
self.sort_raw_tables()
def update(self, **kwargs) :
## 1 required attribute
if not hasattr(self, 'subtitle') :
if 'subtitle' in kwargs :
upipe.print_warning("Overiding subtitle")
self.subtitle = kwargs.get('subtitle', "")
def run_scibasic(self,
sof_filename='scibasic',
expotype="OBJECT",
tpl="ALL",
illum=True):
"""Reducing the files of a certain category and creating the PIXTABLES
Will run the esorex muse_scibasic
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
tpl_gtable = self.select_tpl_files(expotype=expotype,
tpl=tpl,
stage="raw")
if len(tpl_gtable) == 0:
if self.verbose:
upipe.print_warning(
"No {0} recovered from the astropy file Table - Aborting".
format(expotype))
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the LSF including
# the list of files to be processed for one MASTER Flat
for gtable in tpl_gtable.groups:
self._add_calib_to_sofdict("BADPIX_TABLE", reset=True)
self._add_calib_to_sofdict("LINE_CATALOG")
# extract the tpl (string) and mean mjd (float)
tpl, mean_mjd = self._get_tpl_meanmjd(gtable)
self._add_geometry_to_sofdict(tpl)
# Provide the list of files to the dictionary
self._sofdict[expotype] = add_listpath(self.paths.rawfiles,
list(gtable['filename']))
# Number of objects
Nexpo = len(self._sofdict[expotype])
if self.verbose:
upipe.print_info(
"Number of expo is {Nexpo} for {expotype}".format(
Nexpo=Nexpo, expotype=expotype))
# Finding the best tpl for BIAS
if illum:
self._add_tplraw_to_sofdict(mean_mjd, "ILLUM")
self._add_list_tplmaster_to_sofdict(
mean_mjd, ['BIAS', 'FLAT', 'TRACE', 'WAVE', 'TWILIGHT'])
# Writing the sof file
self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True)
# Run the recipe to reduce the standard (muse_scibasic)
dir_products = self._get_fullpath_expo(expotype, "processed")
suffix = self._get_suffix_product(expotype)
name_products = []
list_expo = np.arange(Nexpo).astype(np.int) + 1
for iexpo in list_expo:
name_products += [
'{0:04d}-{1:02d}.fits'.format(iexpo, j + 1)
for j in range(24)
]
self.recipe_scibasic(self.current_sof, tpl, expotype, dir_products,
name_products, suffix)
# Write the Processed files Table and save it
gtable['iexpo'] = list_expo
self.save_expo_table(expotype,
gtable,
"processed",
aggregate=False,
update=True)
# Go back to original folder
self.goto_prevfolder(logfile=True)
