def madcombine(self,
folder_cubes=None,
outcube_name="dummy.fits",
mad=True):
"""Combine the CubeMosaic and write it out.
Args:
folder_cubes (str): name of the folder for the cube [None]
outcube_name (str): name of the outcube
mad (bool): using mad or not [True]
Creates:
A new cube, combination of all input cubes listes in CubeMosaic
"""
# Combine
upipe.print_info("Starting the combine of "
"all {} input cubes".format(self.ncubes))
cube, expmap, statpix, rejmap = self.pycombine(mad=mad)
# Saving
if folder_cubes is not None:
self.folder_cubes = folder_cubes
if not self._check_folder():
return
full_cube_name = joinpath(self.folder_cubes, outcube_name)
upipe.print_info("Writing the new Cube {}".format(full_cube_name))
cube.write(full_cube_name)
def read(self):
"""Reading the data in the file
"""
if self.filter_ascii_file is None:
try:
upipe.print_info("Using the fits file {0} as input".format(
self.filter_fits_file))
filter_data = pyfits.getdata(self.filter_fits_file,
extname=self.filter_name)
self.wave = filter_data['lambda']
self.throughput = filter_data['throughput']
except:
upipe.print_error(
"Problem opening the filter fits file {0}".format(
self.filter_fits_file))
upipe.print_error(
"Did not manage to get the filter {0} throughput".format(
self.filter_name))
self.wave = np.zeros(0)
self.throughput = np.zeros(0)
else:
upipe.print_info("Using the ascii file {0} as input".format(
self.filter_ascii_file))
self.wave, self.throughput = np.loadtxt(self.filter_ascii_file,
unpack=True)
def init_default_param(self, dic_param):
"""Initialise the parameters as defined in the input dictionary
Hardcoded in init_musepipe.py
"""
for key in dic_param:
upipe.print_info(
"Default initialisation of attribute {0}".format(key),
pipe=self)
setattr(self, key, dic_param[key])
def rotate_pixtables_target(self,
targetname=None,
list_pointings=None,
folder_offset_table=None,
offset_table_name=None,
fakemode=False,
**kwargs):
"""Rotate all pixel table of a certain targetname and pointings
"""
# General print out
upipe.print_info("---- Starting the PIXTABLE ROTATION "
"for Target={0} ----".format(targetname))
# Initialise the pipe if needed
if not self.pipes[targetname]._initialised \
or "first_recipe" in kwargs or "last_recipe" in kwargs:
self.set_pipe_target(targetname=targetname,
list_pointings=list_pointings,
**kwargs)
# Check if pointings are valid
list_pointings = self._check_pointings(targetname, list_pointings)
if len(list_pointings) == 0:
return
# add_targetname = kwargs.pop("add_targetname", self.add_targetname)
prefix = kwargs.pop("prefix", "")
# if add_targetname:
# prefix = "{}_{}".format(targetname, prefix)
if folder_offset_table is None:
folder_offset_table = self.pipes[targetname][
list_pointings[0]].paths.alignment
offset_table = Table.read(
joinpath(folder_offset_table, offset_table_name))
offset_table.sort(["POINTING_OBS", "IEXPO_OBS"])
# Loop on the pointings
for row in offset_table:
iexpo = row['IEXPO_OBS']
pointing = row['POINTING_OBS']
tpls = row['TPL_START']
angle = row['ROTANGLE']
upipe.print_info("Rotation ={0} Deg for Pointing={1:02d}, "
"TPLS={2} - Expo {3:02d}".format(
angle, pointing, tpls, iexpo))
folder_expos = self._get_path_files(targetname, pointing)
name_suffix = "{0}_{1:04d}".format(tpls, iexpo)
rotate_pixtables(folder=folder_expos,
name_suffix=name_suffix,
list_ifu=None,
angle=angle,
fakemode=fakemode,
prefix=prefix,
**kwargs)
def reduce_all_targets(self, **kwargs):
"""Reduce all targets already initialised
Input
-----
first_recipe: int or str
One of the recipe to start with
last_recipe: int or str
One of the recipe to end with
"""
for target in self.targets:
upipe.print_info(
"=== Start Reduction of Target {name} ===".format(name=target))
self.reduce_target(targetname=target, **kwargs)
upipe.print_info(
"=== End Reduction of Target {name} ===".format(name=target))
def get_filter_image(self,
filter_name=None,
own_filter_file=None,
filter_folder="",
dic_extra_filters=None):
"""Get an image given by a filter. If the filter belongs to
the filter list, then use that, otherwise use the given file
"""
try:
upipe.print_info("Reading MUSE filter {0}".format(filter_name))
refimage = self.get_band_image(filter_name)
except ValueError:
# initialise the filter file
upipe.print_info(
"Reading private reference filter {0}".format(filter_name))
if dic_extra_filters is not None:
if filter_name in dic_extra_filters:
filter_file = dic_extra_filters[filter_name]
else:
upipe.print_error(
"[mpdaf_pipe / get_filter_image] "
"Filter name not in private dictionary - Aborting")
return
else:
if own_filter_file is None:
upipe.print_error(
"[mpdaf_pipe / get_filter_image] "
"No extra filter dictionary and "
"the private filter file is not set - Aborting")
return
else:
filter_file = own_filter_file
# Now reading the filter data
path_filter = joinpath(filter_folder, filter_file)
filter_wave, filter_sensitivity = np.loadtxt(path_filter,
unpack=True)
refimage = self.bandpass_image(filter_wave,
filter_sensitivity,
interpolation='linear')
key = 'HIERARCH ESO DRS MUSE FILTER NAME'
refimage.primary_header[key] = (filter_name, 'filter name used')
add_mpdaf_method_keywords(refimage.primary_header,
"cube.bandpass_image", ['name'],
[filter_name], ['filter name used'])
return refimage
def build_list(self, folder_cubes=None, prefix_cubes=None, **kwargs):
"""Building the list of cubes to process
Args:
folder_cubes (str): folder for the cubes
prefix_cubes (str): prefix to be used
"""
self.list_suffix = kwargs.pop("list_suffix", self.list_suffix)
# Get the folder if needed
if folder_cubes is not None:
self.folder_cubes = folder_cubes
if not self._check_folder():
return
# get the prefix if provided
if prefix_cubes is not None:
self.prefix_cubes = prefix_cubes
# get the list of cubes and return if 0 found
list_cubes = glob.glob("{0}{1}*.fits".format(self.folder_cubes,
self.prefix_cubes))
# if the list of suffix is empty, just use all cubes
if len(self.list_suffix) == 0:
self.list_cubes = list_cubes
else:
# Filtering out the ones that don't have any of the suffixes
self.list_cubes = []
for l in list_cubes:
if any([suff in l for suff in self.list_suffix]):
self.list_cubes.append(l)
if self.verbose:
for i, name in enumerate(self.list_cubes):
upipe.print_info("Cube {0:03d}: {1}".format(i + 1, name))
self.ncubes = len(self.list_cubes)
if self.ncubes == 0:
upipe.print_error("Found 0 cubes in this folder with suffix"
" {}: please change suffix".format(
self.prefix_cubes))
else:
upipe.print_info("Found {} cubes in this folder".format(
self.ncubes))
def save_normalised(self, norm_factor=1.0, prefix="norm", overwrite=False):
"""Normalises a sky continuum spectrum and save it
within a new fits file
Input
-----
norm_factor: float
Scale factor to multiply the input continuum
prefix: str
Prefix for the new continuum fits name. Default
is 'norm', so that the new file is 'norm_oldname.fits'
overwrite: bool
If True, existing file will be overwritten.
Default is False.
"""
if prefix == "":
upipe.print_error(
"[mpdaf_pipe / save_normalised] The new and old sky "
"continuum fits files will share the same name")
upipe.print_error("This is not recommended - Aborting")
return
folder_spec, filename = os.path.split(self.filename)
newfilename = "{0}{1}".format(prefix, filename)
norm_filename = joinpath(folder_spec, newfilename)
# Opening the fits file
skycont = pyfits.open(self.filename)
# getting the data
dcont = skycont['CONTINUUM'].data
# Create new continuum
# ------------------------------
new_cont = dcont['flux'] * norm_factor
skycont['CONTINUUM'].data['flux'] = new_cont
# Writing to the new file
skycont.writeto(norm_filename, overwrite=overwrite)
upipe.print_info(
'Normalised Factor used = {0:8.4f}'.format(norm_factor))
upipe.print_info('Normalised Sky Continuum {} has been created'.format(
norm_filename))
def run_target_recipe(self,
recipe_name,
targetname=None,
list_pointings=None,
**kwargs):
"""Run just one recipe on target
Input
-----
recipe_name: str
targetname: str
Name of the target
list_pointings: list
Pointing numbers. Default is None (meaning all pointings
indicated in the dictonary will be reduced)
"""
# General print out
upipe.print_info("---- Starting the Recipe {0} for Target={1} "
"----".format(recipe_name, targetname))
kwargs_recipe = {}
for key, default in zip(['fraction', 'skymethod', 'illum'],
[0.8, "model", True]):
_ = kwargs.pop(key, default)
kwargs_recipe[key] = default
# some parameters which depend on the pointings for this recipe
kwargs_per_pointing = kwargs.pop("kwargs_per_pointing", {})
# Initialise the pipe if needed
self.set_pipe_target(targetname=targetname,
list_pointings=list_pointings,
first_recipe=recipe_name,
last_recipe=recipe_name,
**kwargs)
# Check if pointings are valid
list_pointings = self._check_pointings(targetname, list_pointings)
if len(list_pointings) == 0:
return
# Loop on the pointings
for pointing in list_pointings:
upipe.print_info("====== START - POINTING {0:2d} "
"======".format(pointing))
param_recipes = {}
if pointing in kwargs_per_pointing:
param_recipes[recipe_name] = kwargs_per_pointing[pointing]
# Initialise raw tables if not already done (takes some time)
if not self.pipes[targetname][pointing]._raw_table_initialised:
self.pipes[targetname][pointing].init_raw_table(overwrite=True)
if self.__phangs:
self.pipes[targetname][pointing].run_phangs_recipes(
param_recipes=param_recipes, **kwargs_recipe)
else:
self.pipes[targetname][pointing].run_recipes(
param_recipes=param_recipes, **kwargs_recipe)
upipe.print_info(
"====== END - POINTING {0:2d} ======".format(pointing))
def write_sof(self, sof_filename, new=False, verbose=None) :
"""Feeding an sof file with input filenames from a dictionary
"""
# Removing the extension of the file if already set
if not sof_filename.lower().endswith(".sof") :
sof_filename = sof_filename + ".sof"
sof = joinpath(self.paths.sof, sof_filename)
# If new file, start from scratch (overwrite)
if new :
sof_file = open(sof, "w+")
if verbose :
upipe.print_info("Writing in file {0}".format(sof))
# if not new, then append
else :
sof_file = open(sof, "a")
if verbose :
upipe.print_info("Appending in file {0}".format(sof))
# Use dictionary to write up the lines
for key in self._sofdict:
for item in self._sofdict[key] :
text_to_write = "{0} {1}\n".format(item, key)
sof_file.write(text_to_write)
if verbose :
upipe.print_info(text_to_write)
sof_file.close()
# Returning the current sof as relative path
self.current_sof = upipe.normpath(os.path.relpath(sof))
def read_param_file(self, filename, dic_param):
"""Reading an input parameter initialisation file
"""
# Testing existence of filename
if not os.path.isfile(filename):
upipe.print_error(
("Input parameter {inputname} cannot be found. "
"We will use the default hardcoded in the "
"init_musepipe.py module").format(inputname=filename))
return
# If it exists, open and read it
f_param = open(filename)
lines = f_param.readlines()
# Dummy dictionary to see which items are not initialised
noninit_dic_param = copy.copy(dic_param)
for line in lines:
if line[0] in ["#", "%"]: continue
sline = re.split(r'(\s+)', line)
keyword_name = sline[0]
keyword = ("".join(sline[2:])).rstrip()
if keyword_name in dic_param:
upipe.print_info(
"Initialisation of attribute {0}".format(keyword_name),
pipe=self)
setattr(self, keyword_name, keyword)
# Here we drop the item which was initialised
val = noninit_dic_param.pop(keyword_name)
else:
continue
# Listing them as warning and using the hardcoded default
for key in noninit_dic_param:
upipe.print_warning(
("Parameter {param} not initialised "
"We will use the default hardcoded value from "
"init_musepipe.py").format(param=key))
setattr(self, key, dic_param[key])
def extract_onespectral_cube(self,
wave1=default_wave_wcs,
outcube_name=None,
**kwargs):
"""Create a single pixel cube extracted from this one
Input
----
wave1: float
Value of the wavelength to extract. In Angstroems.
outcube_name: str
Name of the output cube
prefix: str
If outcube_name is None (default), use that prefix to append
in front of the input cube name (same folder)
Write
-----
A new cube with only 2 lambda. To be used as a WCS reference for
masks.
"""
# Find the wavelength
k1 = self.wave.pixel([wave1], nearest=True)[0]
# extracting the cube
subcube = self[k1:k1 + 2, :, :]
cube_folder, cube_name = os.path.split(self.filename)
if outcube_name is None:
prefix = kwargs.pop("prefix", "l{0:.0f}_".format(wave1))
outcube_name = "{0}{1}".format(prefix, cube_name)
upipe.print_info("Writing up single wave-cube {0}\n"
"in folder {1}".format(outcube_name, cube_folder))
subcube.write(joinpath(cube_folder, outcube_name))
return cube_folder, outcube_name
def reduce_target(self, targetname=None, list_pointings=None, **kwargs):
"""Reduce one target for a list of pointings
Input
-----
targetname: str
Name of the target
list_pointings: list
Pointing numbers. Default is None (meaning all pointings
indicated in the dictonary will be reduced)
first_recipe: str or int [1]
last_recipe: str or int [max of all recipes]
Name or number of the first and last recipes to process
"""
# General print out
upipe.print_info(
"---- Starting the Data Reduction for Target={0} ----".format(
targetname))
kwargs_recipe = {}
for key, default in zip(['fraction', 'skymethod', 'illum'],
[0.8, "model", True]):
item = kwargs.pop(key, default)
kwargs_recipe[key] = default
# Initialise the pipe if needed
if not self.pipes[targetname]._initialised \
or "first_recipe" in kwargs or "last_recipe" in kwargs:
self.set_pipe_target(targetname=targetname,
list_pointings=list_pointings,
**kwargs)
# Check if pointings are valid
list_pointings = self._check_pointings(targetname, list_pointings)
if len(list_pointings) == 0:
return
# Loop on the pointings
for pointing in list_pointings:
upipe.print_info(
"====== START - POINTING {0:2d} ======".format(pointing))
# Initialise raw tables if not already done (takes some time)
if not self.pipes[targetname][pointing]._raw_table_initialised:
self.pipes[targetname][pointing].init_raw_table(overwrite=True)
if self.__phangs:
self.pipes[targetname][pointing].run_phangs_recipes(
**kwargs_recipe)
else:
self.pipes[targetname][pointing].run_recipes(**kwargs_recipe)
upipe.print_info(
"====== END - POINTING {0:2d} ======".format(pointing))
def set_pipe_target(self, targetname=None, list_pointings=None, **kwargs):
"""Create the musepipe instance for that target and list of pointings
Input
-----
targetname: str
Name of the target
list_pointings: list
Pointing numbers. Default is None (meaning all pointings
indicated in the dictonary will be reduced)
config_args: dic
Dictionary including extra configuration parameters to pass
to MusePipe. This allows to define a global configuration.
If self.__phangs is set to True, this is overwritten with the default
PHANGS configuration parameters as provided in config_pipe.py.
"""
verbose = kwargs.pop("verbose", self.verbose)
# Check if targetname is valid
if not self._check_targetname(targetname):
return
# Galaxy name
upipe.print_info(
"=== Initialising MusePipe for Target {name} ===".format(
name=targetname))
# Check if pointings are valid
list_pointings = self._check_pointings(targetname, list_pointings)
if len(list_pointings) == 0:
return
# Get the filename and extension of log file
log_filename, log_fileext = os.path.splitext(
kwargs.pop("log_filename",
"{0}_{1}.log".format(targetname, version_pack)))
# Reading extra arguments from config dictionary
if self.__phangs:
config_args = PHANGS_reduc_config
# Set overwrite to False to keep existing tables
config_args['overwrite_astropy_table'] = False
else:
config_args = kwargs.pop("config_args", None)
first_recipe = kwargs.pop("first_recipe", 1)
last_recipe = kwargs.pop("last_recipe", None)
# Over-writing the arguments in kwargs from config dictionary
if config_args is not None:
for attr in config_args:
if attr not in kwargs:
kwargs[attr] = config_args[attr]
# extracting the kwargs
list_kwargs = ', '.join(
['{}={!r}'.format(k, v) for k, v in kwargs.items()])
# Config files
rc_filename = self.targets[targetname].rc_filename
cal_filename = self.targets[targetname].cal_filename
folder_config = self.targets[targetname].folder_config
# Loop on the pointings
for pointing in list_pointings:
upipe.print_info(
"Initialise Pipe for Target = {0:10s} / Pointing {1:02d} ".
format(targetname, pointing))
# New log file name with pointing included
log_filename_pointing = "{0}_P{1:02d}{2}".format(
log_filename, pointing, log_fileext)
# Setting up the names of the output files
python_command = ("mypipe = musepipe.MusePipe(targetname='{0}', "
"pointing={1}, folder_config='{2}', "
"rc_filename='{3}', "
"cal_filename='{4}', "
"log_filename='{5}', verbose={6}, "
"{7})".format(targetname, pointing,
folder_config, rc_filename,
cal_filename,
log_filename_pointing, verbose,
list_kwargs))
# Creating the musepipe instance, using the shortcut
self.pipes[targetname][pointing] = MusePipe(
targetname=targetname,
pointing=pointing,
folder_config=folder_config,
rc_filename=rc_filename,
cal_filename=cal_filename,
log_filename=log_filename_pointing,
first_recipe=first_recipe,
last_recipe=last_recipe,
init_raw_table=False,
verbose=verbose,
**kwargs)
# Saving the command
self.pipes[targetname][pointing].history = python_command
# Setting back verbose to True to make sure we have a full account
self.pipes[targetname][pointing].verbose = True
upipe.print_info(python_command, pipe=self)
upipe.print_info("End of Pipe initialisation")
self.pipes[targetname]._initialised = True
def mask_pixtable(self, mask_name=None, **kwargs):
"""Use the Image Mask and create a new Pixtable
Input
-----
mask_name: str
Name of the mask to be used (FITS file)
use_folder: bool
If True, use the same folder as the Pixtable
Otherwise just write where you stand
suffix_out: str
Suffix for the name of the output Pixtable
If provided, will overwrite the one in self.suffix_out
"""
# Open the PixTable
upipe.print_info("Opening the Pixtable {0}".format(self.pixtable_name))
pixtable = PixTable(self.pixtable_name)
# Use the Image mask and create a pixtable mask
if mask_name is not None:
self.mask_name = mask_name
else:
if not hasattr(self, "mask_name"):
upipe.print_error("Please provide a mask name (FITS file)")
return
upipe.print_info("Creating a column Mask from file {0}".format(
self.mask_name))
mask_col = pixtable.mask_column(self.mask_name)
# extract the right data using the pixtable mask
upipe.print_info("Extracting the Mask")
newpixtable = pixtable.extract_from_mask(mask_col.maskcol)
# Rewrite a new pixtable
self.suffix_out = kwargs.pop("suffix_out", self.suffix_out)
use_folder = kwargs.pop("use_folder", True)
if use_folder:
self.newpixtable_name = joinpath(
self.pixtable_folder, "{0}{1}".format(self.suffix_out,
self.pixtable_name))
else:
self.newpixtable_name = "{0}{1}".format(self.suffix_out,
self.pixtable_name)
upipe.print_info("Writing the new PixTable in {0}".format(
self.newpixtable_name))
newpixtable.write(self.newpixtable_name)
# Now transfer the flat field if it exists
ext_name = 'PIXTABLE_FLAT_FIELD'
try:
# Test if Extension exists by reading header
# If it exists then do nothing
test_data = pyfits.getheader(self.newpixtable_name, ext_name)
upipe.print_warning(
"Flat field extension already exists in masked PixTable - all good"
)
# If it does not exist test if it exists in the original PixTable
except KeyError:
try:
# Read data and header
ff_ext_data = pyfits.getdata(self.pixtable_name, ext_name)
ff_ext_h = pyfits.getheader(self.pixtable_name, ext_name)
upipe.print_warning(
"Flat field extension will be transferred from PixTable")
# Append it to the new pixtable
pyfits.append(self.newpixtable_name, ff_ext_data, ff_ext_h)
except KeyError:
upipe.print_warning(
"No Flat field extension to transfer - all good")
except:
pass
except:
pass
# Patch to fix the extension names of the PixTable
# We have to put a number of extension in lowercase to make sure
# the MUSE recipes understand them
descl = ['xpos', 'ypos', 'lambda', 'data', 'dq', 'stat', 'origin']
for d in descl:
try:
pyfits.setval(self.newpixtable_name,
keyword='EXTNAME',
value=d,
extname=d.upper())
upipe.print_warning(
"Rewriting extension name {0} as lowercase".format(
d.upper()))
except:
upipe.print_warning(
"Extension {0} not present - patch ignored".format(
d.upper()))
def convolve_cube(self,
target_fwhm,
target_nmoffat=None,
input_function='moffat',
target_function="gaussian",
outcube_name=None,
factor_fwhm=3,
fft=True):
"""Convolve the cube for a target function 'gaussian' or 'moffat'
Args:
target_fwhm (float): target FWHM in arcsecond
target_nmoffat: target n if Moffat function
input_function (str): 'gaussian' or 'moffat' ['moffat']
target_function (str): 'gaussian' or 'moffat' ['gaussian']
factor_fwhm (float): number of FWHM for size of Kernel
fft (bool): use FFT to convolve or not [False]
Creates:
Convolved cube
"""
# Separate folder and name of file
cube_folder, cube_name = os.path.split(self.filename)
# Creating the outcube filename
if outcube_name is None:
outcube_name = "conv{0}_{1:.2f}{2}".format(
target_function.lower()[0], target_fwhm, cube_name)
upipe.print_info("The new cube will be named: {}".format(outcube_name))
# Getting the shape of the Kernel
scale_spaxel = self.get_step(unit_wcs=u.arcsec)[1]
nspaxel = np.int(factor_fwhm * target_fwhm / scale_spaxel)
# Make nspaxel odd to have a PSF centred at the centre of the frame
if nspaxel % 2 == 0: nspaxel += 1
shape = [self.shape[0], nspaxel, nspaxel]
# Computing the kernel
kernel3d = cube_kernel(shape,
self.wave.coord(),
self.psf_fwhm0,
target_fwhm,
input_function,
target_function,
lambda0=self.psf_l0,
input_nmoffat=self.psf_nmoffat,
target_nmoffat=target_nmoffat,
b=self.psf_b,
scale=scale_spaxel,
compute_kernel='pypher')
if fft:
upipe.print_info("Starting the FFT convolution")
conv_cube = self.fftconvolve(other=kernel3d)
else:
upipe.print_info("Starting the convolution")
conv_cube = self.convolve(other=kernel3d)
# Write the output
upipe.print_info("Writing up the derived cube")
conv_cube.write(joinpath(cube_folder, outcube_name))
# just provide the output name by folder+name
return cube_folder, outcube_name
