def _check_pointings(self, targetname, list_pointings):
"""Check if pointing is in the list of pointings
Returns the list of pointings if ok. If not, return an empty list
Input
-----
targetname: str
name of the target
list_pointings: list
List of integer (pointings).
Returns
-------
list_pointings: list
Empty if input list of pointings is not fully in defined list.
"""
# Info of the pointings and extracting the observing run for each pointing
target_pointings = self.targets[targetname].list_pointings
# if no list_pointings we just do them all
if list_pointings == None:
list_pointings = target_pointings
else:
if any([_ not in target_pointings for _ in list_pointings]):
upipe.print_error(
"ERROR: no pointing {0} for the given target".format(
list_pointings))
return []
return list_pointings
def read(self):
"""Reading the data in the file
"""
if self.filter_ascii_file is None:
try:
upipe.print_info("Using the fits file {0} as input".format(
self.filter_fits_file))
filter_data = pyfits.getdata(self.filter_fits_file,
extname=self.filter_name)
self.wave = filter_data['lambda']
self.throughput = filter_data['throughput']
except:
upipe.print_error(
"Problem opening the filter fits file {0}".format(
self.filter_fits_file))
upipe.print_error(
"Did not manage to get the filter {0} throughput".format(
self.filter_name))
self.wave = np.zeros(0)
self.throughput = np.zeros(0)
else:
upipe.print_info("Using the ascii file {0} as input".format(
self.filter_ascii_file))
self.wave, self.throughput = np.loadtxt(self.filter_ascii_file,
unpack=True)
def __init__(self, pixtable_name, image_name, suffix_out="tmask"):
"""Class made to associate a PixTable and an Image
Used to select all pixels from a pixtable outside the trail
defined by the trail mask of an image
Then flushes the pixtable into a new one.
Input
-----
pixtable_name: str
Name of the Pixel Table
image_name: str
Name of Image which should include a mask
suffix_out: str
Suffix to be used for the new PixTable
"""
self.suffix_out = suffix_out
if not os.path.isfile(pixtable_name):
upipe.print_error("Input PixTable does not exist")
return
if not os.path.isfile(image_name):
upipe.print_error("Input Image does not exist")
return
self.image_name = image_name
self.pixtable_name = pixtable_name
self.pixtable_folder, self.pixtable_name = os.path.split(pixtable_name)
self.image_folder, self.image_name = os.path.split(self.image_name)
self.image = MuseImage(filename=image_name)
def __init__(self,
TargetDic,
rc_filename=None,
cal_filename=None,
folder_config="",
first_recipe=1,
**kwargs):
"""Using a given dictionary to initialise the sample
That dictionary should include the names of the targets
as keys and the subfolder plus pointings to consider
Input
-----
TargetDic: dic
Dictionary of targets. Keys are target names.
Values for each target name should be a list of 2 parameters.
- The first one is the name of the subfolder (e.g. 'P101')
- The second one is the list of pointings, itself a dictionary
with a 0 or 1 for each pointing number depending on whether
this should be included in the reduction or not.
Results can be seen in self.dic_targets dictionary.
rc_filename: str
Default to None
cal_filename: str
Default to None
PHANGS: bool
Default to False. If True, will use default configuration dictionary
from config_pipe.
"""
self.sample = TargetDic
self.targetnames = list(TargetDic.keys())
self.first_recipe = first_recipe
self.__phangs = kwargs.pop("PHANGS", False)
self.verbose = kwargs.pop("verbose", False)
# Reading configuration filenames
if rc_filename is None or cal_filename is None:
upipe.print_error(
"rc_filename and/or cal_filename is None. Please define both.")
return
self.cal_filename = cal_filename
self.rc_filename = rc_filename
self.folder_config = folder_config
# Initialisation of rc and cal files
self._init_calib_files()
# Initialisation of targets
self.init_pipes = kwargs.pop("init_pipes", True)
self.add_targetname = kwargs.pop("add_targetname", True)
self._init_targets()
def _add_tplraw_to_sofdict(self, mean_mjd, expotype, reset=False):
""" Add item to dictionary for the sof writing
"""
if reset: self._sofdict.clear()
# Finding the best tpl for this raw file type
expo_table = self._get_table_expo(expotype, "raw")
index, this_tpl = self._select_closest_mjd(mean_mjd, expo_table)
if index >= 0:
self._sofdict[expotype] = [upipe.normpath(joinpath(self.paths.rawfiles,
expo_table['filename'][index]))]
else:
upipe.print_error("Failed to find a raw exposure of type {} "
"in this table".format(expotype))
def get_sky_spectrum(specname):
"""Read sky spectrum from MUSE data reduction
"""
if not os.path.isfile(specname):
upipe.print_error("{0} not found".format(specname))
return None
sky = pyfits.getdata(specname)
crval = sky['lambda'][0]
cdelt = sky['lambda'][1] - crval
wavein = WaveCoord(cdelt=cdelt, crval=crval, cunit=u.angstrom)
spec = Spectrum(wave=wavein, data=sky['data'], var=sky['stat'])
return spec
def _add_tplmaster_to_sofdict(self, mean_mjd, expotype, reset=False):
""" Add item to dictionary for the sof writing
"""
if reset: self._sofdict.clear()
# Finding the best tpl for this master
index, this_tpl = self._select_closest_mjd(mean_mjd, self._get_table_expo(expotype))
if self._debug:
upipe.print_debug("Index = {0}, Tpl = {1}".format(index, this_tpl))
if index >= 0:
dir_master = self._get_fullpath_expo(expotype)
self._sofdict[get_suffix_product(expotype)] = [upipe.normpath(joinpath(dir_master,
get_suffix_product(expotype) + "_" + this_tpl + ".fits"))]
else:
upipe.print_error("Failed to find a master exposure of type {} "
"in this table".format(expotype))
def read(self):
"""Read sky continuum spectrum from MUSE data reduction
"""
if not os.path.isfile(self.filename):
upipe.print_error("{0} not found".format(self.filename))
crval = 0.0
data = np.zeros(0)
cdelt = 1.0
else:
sky = pyfits.getdata(self.filename)
crval = sky['lambda'][0]
cdelt = sky['lambda'][1] - crval
data = sky['flux']
wavein = WaveCoord(cdelt=cdelt, crval=crval, cunit=u.angstrom)
self.spec = Spectrum(wave=wavein, data=data)
def update_calib_file(filename, subfolder="", folder_config=""):
"""Update the rcfile with a new root
Input
-----
filename: str
Name of the input filename
folder_config: str
Default is "". Name of folder for filename
subfolder: str
Name of subfolder to add in the path
"""
full_filename = joinpath(folder_config, filename)
if full_filename is None:
upipe.print_error("ERROR: input filename is None")
return ""
# Testing existence of filename
if not os.path.isfile(full_filename):
upipe.print_error(
"ERROR: input filename {inputname} cannot be found. ".format(
inputname=full_filename))
return ""
# If it exists, open and read it
old_rc = open(full_filename)
lines = old_rc.readlines()
# Create new file
new_filename = insert_suffix(full_filename, subfolder)
new_rc = open(new_filename, 'w')
# loop on lines
for line in lines:
sline = line.split()
if sline[0] != "root":
new_rc.write(line)
continue
if not os.path.isdir(sline[1]):
upipe.print_warning(
"{} not an existing folder (from rcfile)".format(sline[1]))
newline = line.replace(sline[1], joinpath(sline[1], subfolder))
new_rc.write(newline)
new_rc.close()
old_rc.close()
return new_filename
def build_list(self, folder_cubes=None, prefix_cubes=None, **kwargs):
"""Building the list of cubes to process
Args:
folder_cubes (str): folder for the cubes
prefix_cubes (str): prefix to be used
"""
self.list_suffix = kwargs.pop("list_suffix", self.list_suffix)
# Get the folder if needed
if folder_cubes is not None:
self.folder_cubes = folder_cubes
if not self._check_folder():
return
# get the prefix if provided
if prefix_cubes is not None:
self.prefix_cubes = prefix_cubes
# get the list of cubes and return if 0 found
list_cubes = glob.glob("{0}{1}*.fits".format(self.folder_cubes,
self.prefix_cubes))
# if the list of suffix is empty, just use all cubes
if len(self.list_suffix) == 0:
self.list_cubes = list_cubes
else:
# Filtering out the ones that don't have any of the suffixes
self.list_cubes = []
for l in list_cubes:
if any([suff in l for suff in self.list_suffix]):
self.list_cubes.append(l)
if self.verbose:
for i, name in enumerate(self.list_cubes):
upipe.print_info("Cube {0:03d}: {1}".format(i + 1, name))
self.ncubes = len(self.list_cubes)
if self.ncubes == 0:
upipe.print_error("Found 0 cubes in this folder with suffix"
" {}: please change suffix".format(
self.prefix_cubes))
else:
upipe.print_info("Found {} cubes in this folder".format(
self.ncubes))
def get_filter_image(self,
filter_name=None,
own_filter_file=None,
filter_folder="",
dic_extra_filters=None):
"""Get an image given by a filter. If the filter belongs to
the filter list, then use that, otherwise use the given file
"""
try:
upipe.print_info("Reading MUSE filter {0}".format(filter_name))
refimage = self.get_band_image(filter_name)
except ValueError:
# initialise the filter file
upipe.print_info(
"Reading private reference filter {0}".format(filter_name))
if dic_extra_filters is not None:
if filter_name in dic_extra_filters:
filter_file = dic_extra_filters[filter_name]
else:
upipe.print_error(
"[mpdaf_pipe / get_filter_image] "
"Filter name not in private dictionary - Aborting")
return
else:
if own_filter_file is None:
upipe.print_error(
"[mpdaf_pipe / get_filter_image] "
"No extra filter dictionary and "
"the private filter file is not set - Aborting")
return
else:
filter_file = own_filter_file
# Now reading the filter data
path_filter = joinpath(filter_folder, filter_file)
filter_wave, filter_sensitivity = np.loadtxt(path_filter,
unpack=True)
refimage = self.bandpass_image(filter_wave,
filter_sensitivity,
interpolation='linear')
key = 'HIERARCH ESO DRS MUSE FILTER NAME'
refimage.primary_header[key] = (filter_name, 'filter name used')
add_mpdaf_method_keywords(refimage.primary_header,
"cube.bandpass_image", ['name'],
[filter_name], ['filter name used'])
return refimage
def _get_calib_filenames(self, targetname=None):
"""Get calibration file names
Input
----
targetname: str
Returns
-------
folder_name: str
rcname: str
calname: str
"""
# using targetname or not
if targetname is None:
name_rc = self.rc_filename
name_cal = self.cal_filename
else:
name_rc = insert_suffix(self.rc_filename,
self.targets[targetname].subfolder)
name_cal = insert_suffix(self.cal_filename,
self.targets[targetname].subfolder)
folder_config = self.folder_config
# Checking the folders
folder_rc, rc_filename_target = os.path.split(
joinpath(folder_config, name_rc))
folder_cal, cal_filename_target = os.path.split(
joinpath(folder_config, name_cal))
if rc_filename_target == "" or cal_filename_target == "":
upipe.print_error("Missing a calibration name file")
return
if folder_rc == folder_cal:
folder_config = folder_rc
else:
rc_filename_target = joinpath(folder_rc, rc_filename_target)
cal_filename_target = joinpath(folder_cal, cal_filename_target)
folder_config = ""
return folder_config, rc_filename_target, cal_filename_target
def save_normalised(self, norm_factor=1.0, prefix="norm", overwrite=False):
"""Normalises a sky continuum spectrum and save it
within a new fits file
Input
-----
norm_factor: float
Scale factor to multiply the input continuum
prefix: str
Prefix for the new continuum fits name. Default
is 'norm', so that the new file is 'norm_oldname.fits'
overwrite: bool
If True, existing file will be overwritten.
Default is False.
"""
if prefix == "":
upipe.print_error(
"[mpdaf_pipe / save_normalised] The new and old sky "
"continuum fits files will share the same name")
upipe.print_error("This is not recommended - Aborting")
return
folder_spec, filename = os.path.split(self.filename)
newfilename = "{0}{1}".format(prefix, filename)
norm_filename = joinpath(folder_spec, newfilename)
# Opening the fits file
skycont = pyfits.open(self.filename)
# getting the data
dcont = skycont['CONTINUUM'].data
# Create new continuum
# ------------------------------
new_cont = dcont['flux'] * norm_factor
skycont['CONTINUUM'].data['flux'] = new_cont
# Writing to the new file
skycont.writeto(norm_filename, overwrite=overwrite)
upipe.print_info(
'Normalised Factor used = {0:8.4f}'.format(norm_factor))
upipe.print_info('Normalised Sky Continuum {} has been created'.format(
norm_filename))
def _check_targetname(self, targetname):
"""Check if targetname is in list
Input
-----
targetname: str
Returns
-------
status: bool
True if yes, False if no.
"""
if targetname not in self.targetnames:
upipe.print_error(
"ERROR: no Target named {name} in the defined sample".format(
name=targetname))
return False
else:
return True
def read_param_file(self, filename, dic_param):
"""Reading an input parameter initialisation file
"""
# Testing existence of filename
if not os.path.isfile(filename):
upipe.print_error(
("Input parameter {inputname} cannot be found. "
"We will use the default hardcoded in the "
"init_musepipe.py module").format(inputname=filename))
return
# If it exists, open and read it
f_param = open(filename)
lines = f_param.readlines()
# Dummy dictionary to see which items are not initialised
noninit_dic_param = copy.copy(dic_param)
for line in lines:
if line[0] in ["#", "%"]: continue
sline = re.split(r'(\s+)', line)
keyword_name = sline[0]
keyword = ("".join(sline[2:])).rstrip()
if keyword_name in dic_param:
upipe.print_info(
"Initialisation of attribute {0}".format(keyword_name),
pipe=self)
setattr(self, keyword_name, keyword)
# Here we drop the item which was initialised
val = noninit_dic_param.pop(keyword_name)
else:
continue
# Listing them as warning and using the hardcoded default
for key in noninit_dic_param:
upipe.print_warning(
("Parameter {param} not initialised "
"We will use the default hardcoded value from "
"init_musepipe.py").format(param=key))
setattr(self, key, dic_param[key])
def get_normfactor(self, background, filter_name="Cousins_R"):
"""Get the normalisation factor given a background value
Takes the background value and the sky continuuum spectrum
and convert this to the scaling Ks needed for this sky continuum
The principle relies on having the background measured as:
MUSE_calib = ((MUSE - Sky_cont) + Background) * Norm
as measured from the alignment procedure.
Since we want:
MUSE_calib = ((MUSE - Ks * Sky_cont) + 0) * Norm
This means that: Ks * Sky_cont = Sky_cont - Background
==> Ks = 1 - Background / Sky_cont
So we integrate the Sky_cont to get the corresponding S value
and then provide Ks as 1-B/S
Input
-----
background: float
Value of the background to consider
filter_name: str
Name of the filter to consider
"""
if not hasattr(self, filter_name):
upipe.print_error(
"No integration value for filter {0}".format(filter_name))
norm = 1.
else:
muse_filter = getattr(self, filter_name)
if muse_filter.flux_cont != 0.:
norm = 1. - background / muse_filter.flux_cont
else:
norm = 1.
# Saving the norm value for that filter
muse_filter.norm = norm
muse_filter.background = background
def mask_pixtable(self, mask_name=None, **kwargs):
"""Use the Image Mask and create a new Pixtable
Input
-----
mask_name: str
Name of the mask to be used (FITS file)
use_folder: bool
If True, use the same folder as the Pixtable
Otherwise just write where you stand
suffix_out: str
Suffix for the name of the output Pixtable
If provided, will overwrite the one in self.suffix_out
"""
# Open the PixTable
upipe.print_info("Opening the Pixtable {0}".format(self.pixtable_name))
pixtable = PixTable(self.pixtable_name)
# Use the Image mask and create a pixtable mask
if mask_name is not None:
self.mask_name = mask_name
else:
if not hasattr(self, "mask_name"):
upipe.print_error("Please provide a mask name (FITS file)")
return
upipe.print_info("Creating a column Mask from file {0}".format(
self.mask_name))
mask_col = pixtable.mask_column(self.mask_name)
# extract the right data using the pixtable mask
upipe.print_info("Extracting the Mask")
newpixtable = pixtable.extract_from_mask(mask_col.maskcol)
# Rewrite a new pixtable
self.suffix_out = kwargs.pop("suffix_out", self.suffix_out)
use_folder = kwargs.pop("use_folder", True)
if use_folder:
self.newpixtable_name = joinpath(
self.pixtable_folder, "{0}{1}".format(self.suffix_out,
self.pixtable_name))
else:
self.newpixtable_name = "{0}{1}".format(self.suffix_out,
self.pixtable_name)
upipe.print_info("Writing the new PixTable in {0}".format(
self.newpixtable_name))
newpixtable.write(self.newpixtable_name)
# Now transfer the flat field if it exists
ext_name = 'PIXTABLE_FLAT_FIELD'
try:
# Test if Extension exists by reading header
# If it exists then do nothing
test_data = pyfits.getheader(self.newpixtable_name, ext_name)
upipe.print_warning(
"Flat field extension already exists in masked PixTable - all good"
)
# If it does not exist test if it exists in the original PixTable
except KeyError:
try:
# Read data and header
ff_ext_data = pyfits.getdata(self.pixtable_name, ext_name)
ff_ext_h = pyfits.getheader(self.pixtable_name, ext_name)
upipe.print_warning(
"Flat field extension will be transferred from PixTable")
# Append it to the new pixtable
pyfits.append(self.newpixtable_name, ff_ext_data, ff_ext_h)
except KeyError:
upipe.print_warning(
"No Flat field extension to transfer - all good")
except:
pass
except:
pass
# Patch to fix the extension names of the PixTable
# We have to put a number of extension in lowercase to make sure
# the MUSE recipes understand them
descl = ['xpos', 'ypos', 'lambda', 'data', 'dq', 'stat', 'origin']
for d in descl:
try:
pyfits.setval(self.newpixtable_name,
keyword='EXTNAME',
value=d,
extname=d.upper())
upipe.print_warning(
"Rewriting extension name {0} as lowercase".format(
d.upper()))
except:
upipe.print_warning(
"Extension {0} not present - patch ignored".format(
d.upper()))
def _check_folder(self):
if not os.path.isdir(self.folder_cubes):
upipe.print_error("Cube Folder {} does not exists \n"
"- Aborting".format(self.folder_cubes))
return False
return True
