def goto_prevfolder(self, logfile=False):
"""Go back to previous folder
"""
upipe.print_info("Going back to the original folder {0}".format(
self.paths._prev_folder))
self.goto_folder(self.paths._prev_folder,
logfile=logfile,
verbose=False)
def init_default_param(self, dic_param):
"""Initialise the parameters as defined in the input dictionary
Hardcoded in init_musepipe.py
"""
for key in dic_param.keys():
if self.verbose:
upipe.print_info(
"Default initialisation of attribute {0}".format(key))
setattr(self, key, dic_param[key])
def _set_cpu(self, first_cpu=0, ncpu=24, list_cpu=None):
"""Setting the cpu format for calling the esorex command
"""
if (list_cpu is None) | (len(list_cpu) < 1):
self.list_cpu = "{0}-{1}".format(first_cpu, first_cpu + ncpu - 1)
else:
self.list_cpu = "{0}".format(list_cpu[0])
for i in range(1, len(list_cpu)):
self.list_cpu += ":{0}".format(list_cpu[i])
if self.verbose:
upipe.print_info("LIST_CPU: {0}".format(self.list_cpu))
def read_astropy_table(self, expotype=None, stage="master"):
"""Read an existing Masterfile data table to start the pipeline
"""
# Read the astropy table
name_table = self._get_fitstablename_expo(expotype, stage)
if not os.path.isfile(name_table):
upipe.print_warning(
"Astropy table {0} does not exist - setting up an empty one".
format(name_table))
return Table([[], [], []], names=['tpls', 'mjd', 'tplnexp'])
else:
if self.verbose:
upipe.print_info(
"Reading Astropy fits Table {0}".format(name_table))
return Table.read(name_table, format="fits")
def goto_folder(self, newpath, logfile=False, verbose=True):
"""Changing directory and keeping memory of the old working one
"""
try:
prev_folder = os.getcwd()
newpath = os.path.normpath(newpath)
os.chdir(newpath)
if verbose:
upipe.print_info("Going to folder {0}".format(newpath))
if logfile:
upipe.append_file(joinpath(self.paths.data, self.logfile),
"cd {0}\n".format(newpath))
self.paths._prev_folder = prev_folder
except OSError:
if not os.path.isdir(newpath):
raise
def __init__(self, galaxyname=None, list_pointings=[1]) :
if galaxyname not in MUSEPIPE_sample.keys() :
upipe.print_error("ERROR: no Galaxy named {gal} in the defined sample".format(gal=galaxyname))
return
# Galaxy name
upipe.print_info("Initialising Target {name}".format(name=galaxyname))
self.targetname = galaxyname
# Info of the pointings and extracting the observing run for each pointing
self.info_pointings = MUSEPIPE_sample[galaxyname]
if any([_ not in self.info_pointings.keys() for _ in list_pointings]) :
upipe.print_error("ERROR: no pointing {0} for the Galaxy".format(list_pointings))
return
self.list_pointings = list_pointings
self.observing_run = [self.info_pointing[_] for _ in self.list_pointings]
def print_offset(self):
"""Print out the offset
"""
upipe.print_info("#---- Offset recorded so far ----#")
upipe.print_info("Total in ARCSEC")
for nima in range(self.nimages):
upipe.print_info(" Image {0:02d}: "
"{1:8.4f} {2:8.4f}".format(
nima, self.total_off_arcsec[nima][0],
self.total_off_arcsec[nima][1]))
upipe.print_info("Total in PIXEL")
for nima in range(self.nimages):
upipe.print_info(" Image {0:02d}: "
"{1:8.4f} {2:8.4f}".format(
nima, self.total_off_pixel[nima][0],
self.total_off_pixel[nima][1]))
def write_sof(self, sof_filename, new=False, verbose=None):
"""Feeding an sof file with input filenames from a dictionary
"""
# Removing the extension of the file if already set
if not sof_filename.lower().endswith(".sof"):
sof_filename = sof_filename + ".sof"
sof = joinpath(self.paths.sof, sof_filename)
# If new file, start from scratch (overwrite)
if new:
sof_file = open(sof, "w+")
if verbose:
upipe.print_info("Writing in file {0}".format(sof))
# if not new, then append
else:
sof_file = open(sof, "a")
if verbose:
upipe.print_info("Appending in file {0}".format(sof))
# Use dictionary to write up the lines
for key in self._sofdict.keys():
for item in self._sofdict[key]:
text_to_write = "{0} {1}\n".format(item, key)
sof_file.write(text_to_write)
if verbose:
upipe.print_info(text_to_write)
sof_file.close()
# Returning the current sof as relative path
self.current_sof = upipe.normpath(os.path.relpath(sof))
def select_tpl_files(self, expotype=None, tpl="ALL", stage="raw"):
"""Selecting a subset of files from a certain type
"""
if expotype not in musepipe.listexpo_types.keys():
upipe.print_info(
"ERROR: input {0} is not in the list of possible values".
format(expotype))
return
MUSE_subtable = self._get_table_expo(expotype, stage)
if len(MUSE_subtable) == 0:
if self.verbose:
upipe.print_warning(
"Empty file table of type {0}".format(expotype))
upipe.print_warning(
"Returning an empty Table from the tpl -astropy- selection"
)
return MUSE_subtable
group_table = MUSE_subtable.group_by('tpls')
if tpl == "ALL":
return group_table
else:
return group_table.groups[group_table.groups.keys['tpls'] == tpl]
def run_pipeline(self, list_pointings=[1], fakemode=False,
suffix_logfile="logfile", suffix_rcfile="rcfile", suffix_calfile="calfile"):
"""Run the pipeline for all pointings in the list
"""
if any([_ not in self.info_pointings.keys() for _ in self.list_pointings]) :
upipe.print_error("ERROR: some pointing are not in "
"the available list ({0})".format(self.list_pointings))
return
# Setting up the suffixes for the files
self.suffix_logfile = suffix_logfile
self.suffix_calfile = suffix_calfile
self.suffix_rcfile = suffix_rcfile
# Loop on the pointings
self.pipelines = []
upipe.print_info("---- Starting the Data Reduction ----")
self.history = []
for pointing in list_pointings:
# Setting up the names of the output files
logfile = self._get_logfile_name(self.suffix_logfile, pointing)
calfile = self._get_calfile_name(self.suffix_calfile, pointing)
rcfile = self._get_rcfile_name(self.suffix_rcfile, pointing)
python_command = ("mypipe = musepipe.MusePipe(galaxyname={0}, "
"pointing={1}, rc_filename={2}, cal_filename={3}, "
"outlog=None, logfile={4}, fakemode={5}, "
"nocache=False)".format(galaxyname, pointing, rcfile, calfile,
logfile, fakemode))
upipe.print_info("====== START - POINTING {0:2d} ======".format(pointing))
upipe.print_info(python_command)
upipe.print_info("====== END - POINTING {0:2d} ======".format(pointing))
self.history.append(python_command)
mypipe = MusePipe(galaxyname=galaxyname, pointing=pointing,
rc_filename=rcfile, cal_filename=calfile, outlog=None,
logfile=logfile, fakemode=fakemode, nocache=False)
self.pipelines.append(mypipe)
mypipe.run_all_recipes()
def print_offset_fromfits(self, name_table=None):
"""Print out the offset
"""
exist_table, fits_table = self.open_offset_table(name_table)
if exist_table is None:
return
if ('RA_OFFSET'
not in fits_table.columns) or ('DEC_OFFSET'
not in fits_table.columns):
upipe.print_error(
"Table does not contain 'RA/DEC_OFFSET' columns, Aborting")
return
upipe.print_info("Offset recorded in OFFSET_LIST Table")
upipe.print_info("Total in ARCSEC")
for i in self.nimages:
upipe.print_info("Image {0}: "
"{1:8.4f} {1:8.4f}".format(
fits_table['RA_OFFSET'] * 3600,
fits_table['DEC_OFFSET'] * 3600.))
def read_param_file(self, filename, dic_param):
"""Reading an input parameter initialisation file
"""
# Testing existence of filename
if not os.path.isfile(filename):
upipe.print_info(
("ERROR: input parameter {inputname} cannot be found. "
"We will use the default hardcoded in the "
"init_musepipe.py module").format(inputname=filename))
return
# If it exists, open and read it
f_param = open(filename)
lines = f_param.readlines()
# Dummy dictionary to see which items are not initialised
dummy_dic_param = copy.copy(dic_param)
for line in lines:
if line[0] in ["#", "%"]: continue
sline = line.split()
if sline[0] in dic_param.keys():
if self.verbose:
upipe.print_info("Initialisation of attribute {0}".format(
sline[0]))
setattr(self, sline[0], sline[1])
# Here we drop the item which was initialised
val = dummy_dic_param.pop(sline[0])
else:
continue
# Set of non initialised folders
not_initialised_param = dummy_dic_param.keys()
# Listing them as warning and using the hardcoded default
for key in not_initialised_param:
upipe.print_info(("WARNING: parameter {param} not initialised "
"We will use the default hardcoded value from "
"init_musepipe.py").format(param=key))
setattr(self, key, dic_param[key])
def compare(self,
muse_hdu=None,
ref_hdu=None,
factor=1.0,
start_nfig=1,
nlevels=7,
levels=None,
muse_smooth=1.,
ref_smooth=1.,
samecontour=True,
nima=0,
showcontours=True,
showcuts=True,
difference=True,
normalise=True,
median_filter=True,
ncuts=5,
showdiff=True,
percentage=10.):
"""Plot the contours of the projected reference and MUSE image
"""
# Getting the data
if muse_hdu is None:
muse_hdu = self.list_offmuse_hdu[nima]
if ref_hdu is None:
ref_hdu = self.list_proj_refhdu[nima]
# Filtering the data if median_filter is True
if median_filter:
musedata = self._filtermed_image(muse_hdu.data)
refdata = self._filtermed_image(ref_hdu.data)
else:
musedata = muse_hdu.data
refdata = ref_hdu.data
# If normalising, using the median ratio fit
if normalise:
polypar = get_image_norm_poly(musedata, refdata, chunk_size=15)
if self.verbose:
upipe.print_info("Renormalising the MUSE data as NewMUSE = "
"{1:8.4f} * ({0:8.4f} + MUSE)".format(
polypar.beta[0], polypar.beta[1]))
musedata = (polypar.beta[0] + musedata) * polypar.beta[1]
self.ima_polynorm[nima] = polypar.beta
# Getting the range of relevant fluxes
lowlevel_muse, highlevel_muse = self._get_flux_range(musedata)
lowlevel_ref, highlevel_ref = self._get_flux_range(refdata)
if self.verbose:
print("Low / High level MUSE flux: "
"{0:8.4f} {1:8.4f}".format(lowlevel_muse, highlevel_muse))
print("Low / High level REF flux: "
"{0:8.4f} {1:8.4f}".format(lowlevel_ref, highlevel_ref))
# Smoothing out the result in case it is needed
if muse_smooth > 0:
musedata = nd.filters.gaussian_filter(musedata, muse_smooth)
if ref_smooth > 0:
refdata = nd.filters.gaussian_filter(refdata, ref_smooth)
# Get the WCS
musewcs = wcs.WCS(muse_hdu.header)
refwcs = wcs.WCS(ref_hdu.header)
# Preparing the figure
current_fig = start_nfig
self.list_figures = []
# Starting the plotting
if showcontours:
fig, ax = open_new_wcs_figure(current_fig, musewcs)
if levels is not None:
mylevels = levels
samecontour = True
else:
# First contours - MUSE
levels_muse = np.linspace(np.log10(lowlevel_muse),
np.log10(highlevel_muse), nlevels)
levels_ref = np.linspace(np.log10(lowlevel_ref),
np.log10(highlevel_ref), nlevels)
mylevels = levels_muse
cmuseset = ax.contour(np.log10(musedata),
mylevels,
colors='k',
origin='lower',
linestyles='solid')
# Second contours - Ref
if samecontour:
crefset = ax.contour(np.log10(refdata),
levels=cmuseset.levels,
colors='r',
origin='lower',
alpha=0.5,
linestyles='solid')
else:
crefset = ax.contour(np.log10(refdata),
levels=levels_ref,
colors='r',
origin='lower',
alpha=0.5,
linestyles='solid')
ax.set_aspect('equal')
h1, _ = cmuseset.legend_elements()
h2, _ = crefset.legend_elements()
ax.legend([h1[0], h2[0]], ['MUSE', 'REF'])
if nima is not None:
plt.title("Image #{0:02d}".format(nima))
self.list_figures.append(current_fig)
current_fig += 1
if showcuts:
fig, ax = open_new_wcs_figure(current_fig)
diffima = (refdata - musedata) * 200. / (lowlevel_muse +
highlevel_muse)
chunk_x = musedata.shape[0] // (ncuts + 1)
chunk_y = musedata.shape[1] // (ncuts + 1)
ax.plot(diffima[np.arange(ncuts) * chunk_x, :].T, 'k-')
ax.plot(diffima[:, np.arange(ncuts) * chunk_x], 'r-')
ax.set_ylim(-20, 20)
ax.set_xlabel("[pixels]", fontsize=20)
ax.set_ylabel("[%]", fontsize=20)
self.list_figures.append(current_fig)
current_fig += 1
if showdiff:
fig, ax = open_new_wcs_figure(current_fig, musewcs)
ratio = 100. * (refdata - musedata) / (musedata + 1.e-12)
im = ax.imshow(ratio, vmin=-percentage, vmax=percentage)
cbar = fig.colorbar(im)
self.list_figures.append(current_fig)
current_fig += 1
def wrapped(*myargs, **mykwargs):
upipe.print_info(f.__name__)
return f(*myargs, **mykwargs)
def run_scibasic(self,
sof_filename='scibasic',
expotype="OBJECT",
tpl="ALL",
illum=True):
"""Reducing the files of a certain category and creating the PIXTABLES
Will run the esorex muse_scibasic
Parameters
----------
sof_filename: string (without the file extension)
Name of the SOF file which will contain the Bias frames
tpl: ALL by default or a special tpl time
"""
# First selecting the files via the grouped table
tpl_gtable = self.select_tpl_files(expotype=expotype,
tpl=tpl,
stage="raw")
if len(tpl_gtable) == 0:
if self.verbose:
upipe.print_warning(
"No {0} recovered from the astropy file Table - Aborting".
format(expotype))
return
# Go to the data folder
self.goto_folder(self.paths.data, logfile=True)
# Create the dictionary for the LSF including
# the list of files to be processed for one MASTER Flat
for gtable in tpl_gtable.groups:
self._add_calib_to_sofdict("BADPIX_TABLE", reset=True)
self._add_calib_to_sofdict("LINE_CATALOG")
# extract the tpl (string) and mean mjd (float)
tpl, mean_mjd = self._get_tpl_meanmjd(gtable)
self._add_geometry_to_sofdict(tpl)
# Provide the list of files to the dictionary
self._sofdict[expotype] = add_listpath(self.paths.rawfiles,
list(gtable['filename']))
# Number of objects
Nexpo = len(self._sofdict[expotype])
if self.verbose:
upipe.print_info(
"Number of expo is {Nexpo} for {expotype}".format(
Nexpo=Nexpo, expotype=expotype))
# Finding the best tpl for BIAS
if illum:
self._add_tplraw_to_sofdict(mean_mjd, "ILLUM")
self._add_list_tplmaster_to_sofdict(
mean_mjd, ['BIAS', 'FLAT', 'TRACE', 'WAVE', 'TWILIGHT'])
# Writing the sof file
self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True)
# Run the recipe to reduce the standard (muse_scibasic)
dir_products = self._get_fullpath_expo(expotype, "processed")
suffix = self._get_suffix_product(expotype)
name_products = []
list_expo = np.arange(Nexpo).astype(np.int) + 1
for iexpo in list_expo:
name_products += [
'{0:04d}-{1:02d}.fits'.format(iexpo, j + 1)
for j in range(24)
]
self.recipe_scibasic(self.current_sof, tpl, expotype, dir_products,
name_products, suffix)
# Write the Processed files Table and save it
gtable['iexpo'] = list_expo
self.save_expo_table(expotype,
gtable,
"processed",
aggregate=False,
update=True)
# Go back to original folder
self.goto_prevfolder(logfile=True)
def init_raw_table(self, reset=False):
""" Create a fits table with all the information from
the Raw files
Also create an astropy table with the same info
"""
if self.verbose:
upipe.print_info("Creating the astropy fits raw data table")
if reset or not hasattr(self, "Tables"):
self._reset_tables()
# Testing if raw table exists
name_table = self._get_fitstablename_expo('RAWFILES', "raw")
# ---- File exists - we READ it ------------------- #
overwrite = True
if os.path.isfile(name_table):
if self._overwrite_astropy_table:
upipe.print_warning("The raw-files table will be overwritten")
else:
upipe.print_warning("The raw files table already exists")
upipe.print_warning(
"If you wish to overwrite it, "
" please turn on the 'overwrite_astropy_table' option to 'True'"
)
upipe.print_warning(
"In the meantime, the existing table will be read and used"
)
self.Tables.Rawfiles = self.read_astropy_table(
'RAWFILES', "raw")
overwrite = False
# ---- File does not exist - we create it ---------- #
if overwrite:
# Check the raw folder
self.goto_folder(self.paths.rawfiles)
# Get the list of files from the Raw data folder
files = os.listdir(".")
smalldic = {"FILENAME": ['filename', '', str, '100A']}
fulldic = listexpo_files.copy()
fulldic.update(smalldic)
# Init the lists
MUSE_infodic = {}
for key in fulldic.keys():
MUSE_infodic[key] = []
# Looping over the files
for f in files:
# Excluding the files without MUSE and fits.fz
if ('MUSE' in f) and ('.fits.fz') in f:
MUSE_infodic['FILENAME'].append(f)
header = pyfits.getheader(f, 0)
for k in listexpo_files.keys():
[namecol, keyword, func, form] = listexpo_files[k]
MUSE_infodic[k].append(func(header[keyword]))
# Transforming into numpy arrayimport pymusepipe
for k in fulldic.keys():
MUSE_infodic[k] = np.array(MUSE_infodic[k])
# Getting a sorted array with indices
idxsort = np.argsort(MUSE_infodic['FILENAME'])
# Creating the astropy table
self.Tables.Rawfiles = Table([MUSE_infodic['FILENAME'][idxsort]],
names=['filename'],
meta={'name': 'raw file table'})
# Creating the columns
for k in fulldic.keys():
[namecol, keyword, func, form] = fulldic[k]
self.Tables.Rawfiles[namecol] = MUSE_infodic[k][idxsort]
# Writing up the table
self.Tables.Rawfiles.write(name_table,
format="fits",
overwrite=self._overwrite_astropy_table)
# Going back to the original folder
self.goto_prevfolder()
# Sorting the types ====================================
self.sort_raw_tables()
def __init__(self,
galaxyname=None,
pointing=0,
objectlist=[],
rc_filename=None,
cal_filename=None,
outlog=None,
logfile="MusePipe.log",
reset_log=False,
verbose=True,
musemode="WFM-NOAO-N",
checkmode=True,
strong_checkmode=False,
**kwargs):
"""Initialise the file parameters to be used during the run
Input
-----
galaxyname: string (e.g., 'NGC1208'). default is None.
objectlist= list of objects (string=filenames) to process. Default is empty
rc_filename: filename to initialise folders
cal_filename: filename to initiale FIXED calibration MUSE files
outlog: string, output directory for the log files
verbose: boolean. Give more information as output (default is True)
musemode: string (default is WFM_N) String to define the mode to be considered
checkmode: boolean (default is True) Check the mode or not when reducing
strong_checkmode: boolean (default is False) Enforce the checkmode for all if True,
or exclude DARK/BIAS if False
vsystemic: float (default is 0), indicating the systemic velocity of the galaxy [in km/s]
Other possible entries
----------------------
overwrite_astropy_table: boolean (default is False). Overwrite the astropy table even when
it exists.
warnings: strong ('ignore'by default. If set to ignore, will ignore the Astropy Warnings.
time_geo_table: boolean (default is True). Use the time dependent geo_table
time_astro_table: boolean (default is True). Use the time dependent astro_table
"""
# Verbose option
self.verbose = verbose
# Warnings for astropy
self.warnings = kwargs.pop("warnings", 'ignore')
if self.warnings == 'ignore':
warnings.simplefilter('ignore', category=AstropyWarning)
# Overwriting option for the astropy table
self._overwrite_astropy_table = kwargs.pop("overwrite_astropy_table",
False)
# Updating the astropy table
self._update_astropy_table = kwargs.pop("update_astropy_table", False)
# Use time dependent geo_table
self._time_geo_table = kwargs.pop("time_geo_table", True)
self._time_astro_table = kwargs.pop("time_astro_table", True)
# super(MusePipe, self).__init__(**kwargs)
# Setting the default attibutes #####################
self.galaxyname = galaxyname
self.pointing = pointing
self.vsystemic = np.float(kwargs.pop("vsystemic", 0.))
# Setting other default attributes
if outlog is None:
outlog = "log_{timestamp}".format(
timestamp=upipe.create_time_name())
upipe.print_info("The Log folder will be {log}".format(outlog))
self.outlog = outlog
self.logfile = joinpath(self.outlog, logfile)
# Further reduction options =====================================
# Mode of the observations
self.musemode = musemode
# Checking if mode is correct
self.checkmode = checkmode
# Checking if mode is correct also for BIAS & DARK
self.strong_checkmode = strong_checkmode
# Set of objects to reduced
self.objectlist = objectlist
# End of parameter settings #########################
# Init of the subclasses
PipePrep.__init__(self)
PipeRecipes.__init__(self, **kwargs)
# =========================================================== #
# Setting up the folders and names for the data reduction
# Can be initialised by either an rc_file,
# or a default rc_file or harcoded defaults.
self.my_params = InitMuseParameters(rc_filename=rc_filename,
cal_filename=cal_filename)
# Setting up the relative path for the data, using Galaxy Name + Pointing
self.my_params.data = "{0}/P{1:02d}/".format(self.galaxyname,
self.pointing)
# Create full path folder
self.set_fullpath_names()
# Go to the data directory
# and Recording the folder where we start
self.paths.orig = os.getcwd()
# Making the output folders in a safe mode
if self.verbose:
upipe.print_info("Creating directory structure")
self.goto_folder(self.paths.data)
# ==============================================
# Creating the extra pipeline folder structure
for folder in self.my_params._dic_input_folders.keys():
upipe.safely_create_folder(
self.my_params._dic_input_folders[folder], verbose=verbose)
# ==============================================
# Creating the folder structure itself if needed
for folder in self.my_params._dic_folders.keys():
upipe.safely_create_folder(self.my_params._dic_folders[folder],
verbose=verbose)
# ==============================================
# Init the Master exposure flag dictionary
self.Master = {}
for mastertype in dic_listMaster.keys():
upipe.safely_create_folder(self._get_path_expo(
mastertype, "master"),
verbose=self.verbose)
self.Master[mastertype] = False
# Init the Object folder
for objecttype in dic_listObject.keys():
upipe.safely_create_folder(self._get_path_expo(
objecttype, "processed"),
verbose=self.verbose)
self._dic_listMasterObject = {**dic_listMaster, **dic_listObject}
# ==============================================
# Going back to initial working directory
self.goto_prevfolder()
# ===========================================================
# Now creating the raw table, and attribute containing the
# astropy dataset probing the rawfiles folder
# When creating the table, if the table already exists
# it will read the old one, except if an overwrite_astropy_table
# is set to True.
self.init_raw_table()
self.read_all_astro_tables()
