def madcombine(self, folder_cubes=None, outcube_name="dummy.fits", mad=True): """Combine the CubeMosaic and write it out. Args: folder_cubes (str): name of the folder for the cube [None] outcube_name (str): name of the outcube mad (bool): using mad or not [True] Creates: A new cube, combination of all input cubes listes in CubeMosaic """ # Combine upipe.print_info("Starting the combine of " "all {} input cubes".format(self.ncubes)) cube, expmap, statpix, rejmap = self.pycombine(mad=mad) # Saving if folder_cubes is not None: self.folder_cubes = folder_cubes if not self._check_folder(): return full_cube_name = joinpath(self.folder_cubes, outcube_name) upipe.print_info("Writing the new Cube {}".format(full_cube_name)) cube.write(full_cube_name)
def read(self): """Reading the data in the file """ if self.filter_ascii_file is None: try: upipe.print_info("Using the fits file {0} as input".format( self.filter_fits_file)) filter_data = pyfits.getdata(self.filter_fits_file, extname=self.filter_name) self.wave = filter_data['lambda'] self.throughput = filter_data['throughput'] except: upipe.print_error( "Problem opening the filter fits file {0}".format( self.filter_fits_file)) upipe.print_error( "Did not manage to get the filter {0} throughput".format( self.filter_name)) self.wave = np.zeros(0) self.throughput = np.zeros(0) else: upipe.print_info("Using the ascii file {0} as input".format( self.filter_ascii_file)) self.wave, self.throughput = np.loadtxt(self.filter_ascii_file, unpack=True)
def init_default_param(self, dic_param): """Initialise the parameters as defined in the input dictionary Hardcoded in init_musepipe.py """ for key in dic_param: upipe.print_info( "Default initialisation of attribute {0}".format(key), pipe=self) setattr(self, key, dic_param[key])
def rotate_pixtables_target(self, targetname=None, list_pointings=None, folder_offset_table=None, offset_table_name=None, fakemode=False, **kwargs): """Rotate all pixel table of a certain targetname and pointings """ # General print out upipe.print_info("---- Starting the PIXTABLE ROTATION " "for Target={0} ----".format(targetname)) # Initialise the pipe if needed if not self.pipes[targetname]._initialised \ or "first_recipe" in kwargs or "last_recipe" in kwargs: self.set_pipe_target(targetname=targetname, list_pointings=list_pointings, **kwargs) # Check if pointings are valid list_pointings = self._check_pointings(targetname, list_pointings) if len(list_pointings) == 0: return # add_targetname = kwargs.pop("add_targetname", self.add_targetname) prefix = kwargs.pop("prefix", "") # if add_targetname: # prefix = "{}_{}".format(targetname, prefix) if folder_offset_table is None: folder_offset_table = self.pipes[targetname][ list_pointings[0]].paths.alignment offset_table = Table.read( joinpath(folder_offset_table, offset_table_name)) offset_table.sort(["POINTING_OBS", "IEXPO_OBS"]) # Loop on the pointings for row in offset_table: iexpo = row['IEXPO_OBS'] pointing = row['POINTING_OBS'] tpls = row['TPL_START'] angle = row['ROTANGLE'] upipe.print_info("Rotation ={0} Deg for Pointing={1:02d}, " "TPLS={2} - Expo {3:02d}".format( angle, pointing, tpls, iexpo)) folder_expos = self._get_path_files(targetname, pointing) name_suffix = "{0}_{1:04d}".format(tpls, iexpo) rotate_pixtables(folder=folder_expos, name_suffix=name_suffix, list_ifu=None, angle=angle, fakemode=fakemode, prefix=prefix, **kwargs)
def reduce_all_targets(self, **kwargs): """Reduce all targets already initialised Input ----- first_recipe: int or str One of the recipe to start with last_recipe: int or str One of the recipe to end with """ for target in self.targets: upipe.print_info( "=== Start Reduction of Target {name} ===".format(name=target)) self.reduce_target(targetname=target, **kwargs) upipe.print_info( "=== End Reduction of Target {name} ===".format(name=target))
def get_filter_image(self, filter_name=None, own_filter_file=None, filter_folder="", dic_extra_filters=None): """Get an image given by a filter. If the filter belongs to the filter list, then use that, otherwise use the given file """ try: upipe.print_info("Reading MUSE filter {0}".format(filter_name)) refimage = self.get_band_image(filter_name) except ValueError: # initialise the filter file upipe.print_info( "Reading private reference filter {0}".format(filter_name)) if dic_extra_filters is not None: if filter_name in dic_extra_filters: filter_file = dic_extra_filters[filter_name] else: upipe.print_error( "[mpdaf_pipe / get_filter_image] " "Filter name not in private dictionary - Aborting") return else: if own_filter_file is None: upipe.print_error( "[mpdaf_pipe / get_filter_image] " "No extra filter dictionary and " "the private filter file is not set - Aborting") return else: filter_file = own_filter_file # Now reading the filter data path_filter = joinpath(filter_folder, filter_file) filter_wave, filter_sensitivity = np.loadtxt(path_filter, unpack=True) refimage = self.bandpass_image(filter_wave, filter_sensitivity, interpolation='linear') key = 'HIERARCH ESO DRS MUSE FILTER NAME' refimage.primary_header[key] = (filter_name, 'filter name used') add_mpdaf_method_keywords(refimage.primary_header, "cube.bandpass_image", ['name'], [filter_name], ['filter name used']) return refimage
def build_list(self, folder_cubes=None, prefix_cubes=None, **kwargs): """Building the list of cubes to process Args: folder_cubes (str): folder for the cubes prefix_cubes (str): prefix to be used """ self.list_suffix = kwargs.pop("list_suffix", self.list_suffix) # Get the folder if needed if folder_cubes is not None: self.folder_cubes = folder_cubes if not self._check_folder(): return # get the prefix if provided if prefix_cubes is not None: self.prefix_cubes = prefix_cubes # get the list of cubes and return if 0 found list_cubes = glob.glob("{0}{1}*.fits".format(self.folder_cubes, self.prefix_cubes)) # if the list of suffix is empty, just use all cubes if len(self.list_suffix) == 0: self.list_cubes = list_cubes else: # Filtering out the ones that don't have any of the suffixes self.list_cubes = [] for l in list_cubes: if any([suff in l for suff in self.list_suffix]): self.list_cubes.append(l) if self.verbose: for i, name in enumerate(self.list_cubes): upipe.print_info("Cube {0:03d}: {1}".format(i + 1, name)) self.ncubes = len(self.list_cubes) if self.ncubes == 0: upipe.print_error("Found 0 cubes in this folder with suffix" " {}: please change suffix".format( self.prefix_cubes)) else: upipe.print_info("Found {} cubes in this folder".format( self.ncubes))
def save_normalised(self, norm_factor=1.0, prefix="norm", overwrite=False): """Normalises a sky continuum spectrum and save it within a new fits file Input ----- norm_factor: float Scale factor to multiply the input continuum prefix: str Prefix for the new continuum fits name. Default is 'norm', so that the new file is 'norm_oldname.fits' overwrite: bool If True, existing file will be overwritten. Default is False. """ if prefix == "": upipe.print_error( "[mpdaf_pipe / save_normalised] The new and old sky " "continuum fits files will share the same name") upipe.print_error("This is not recommended - Aborting") return folder_spec, filename = os.path.split(self.filename) newfilename = "{0}{1}".format(prefix, filename) norm_filename = joinpath(folder_spec, newfilename) # Opening the fits file skycont = pyfits.open(self.filename) # getting the data dcont = skycont['CONTINUUM'].data # Create new continuum # ------------------------------ new_cont = dcont['flux'] * norm_factor skycont['CONTINUUM'].data['flux'] = new_cont # Writing to the new file skycont.writeto(norm_filename, overwrite=overwrite) upipe.print_info( 'Normalised Factor used = {0:8.4f}'.format(norm_factor)) upipe.print_info('Normalised Sky Continuum {} has been created'.format( norm_filename))
def run_target_recipe(self, recipe_name, targetname=None, list_pointings=None, **kwargs): """Run just one recipe on target Input ----- recipe_name: str targetname: str Name of the target list_pointings: list Pointing numbers. Default is None (meaning all pointings indicated in the dictonary will be reduced) """ # General print out upipe.print_info("---- Starting the Recipe {0} for Target={1} " "----".format(recipe_name, targetname)) kwargs_recipe = {} for key, default in zip(['fraction', 'skymethod', 'illum'], [0.8, "model", True]): _ = kwargs.pop(key, default) kwargs_recipe[key] = default # some parameters which depend on the pointings for this recipe kwargs_per_pointing = kwargs.pop("kwargs_per_pointing", {}) # Initialise the pipe if needed self.set_pipe_target(targetname=targetname, list_pointings=list_pointings, first_recipe=recipe_name, last_recipe=recipe_name, **kwargs) # Check if pointings are valid list_pointings = self._check_pointings(targetname, list_pointings) if len(list_pointings) == 0: return # Loop on the pointings for pointing in list_pointings: upipe.print_info("====== START - POINTING {0:2d} " "======".format(pointing)) param_recipes = {} if pointing in kwargs_per_pointing: param_recipes[recipe_name] = kwargs_per_pointing[pointing] # Initialise raw tables if not already done (takes some time) if not self.pipes[targetname][pointing]._raw_table_initialised: self.pipes[targetname][pointing].init_raw_table(overwrite=True) if self.__phangs: self.pipes[targetname][pointing].run_phangs_recipes( param_recipes=param_recipes, **kwargs_recipe) else: self.pipes[targetname][pointing].run_recipes( param_recipes=param_recipes, **kwargs_recipe) upipe.print_info( "====== END - POINTING {0:2d} ======".format(pointing))
def write_sof(self, sof_filename, new=False, verbose=None) : """Feeding an sof file with input filenames from a dictionary """ # Removing the extension of the file if already set if not sof_filename.lower().endswith(".sof") : sof_filename = sof_filename + ".sof" sof = joinpath(self.paths.sof, sof_filename) # If new file, start from scratch (overwrite) if new : sof_file = open(sof, "w+") if verbose : upipe.print_info("Writing in file {0}".format(sof)) # if not new, then append else : sof_file = open(sof, "a") if verbose : upipe.print_info("Appending in file {0}".format(sof)) # Use dictionary to write up the lines for key in self._sofdict: for item in self._sofdict[key] : text_to_write = "{0} {1}\n".format(item, key) sof_file.write(text_to_write) if verbose : upipe.print_info(text_to_write) sof_file.close() # Returning the current sof as relative path self.current_sof = upipe.normpath(os.path.relpath(sof))
def read_param_file(self, filename, dic_param): """Reading an input parameter initialisation file """ # Testing existence of filename if not os.path.isfile(filename): upipe.print_error( ("Input parameter {inputname} cannot be found. " "We will use the default hardcoded in the " "init_musepipe.py module").format(inputname=filename)) return # If it exists, open and read it f_param = open(filename) lines = f_param.readlines() # Dummy dictionary to see which items are not initialised noninit_dic_param = copy.copy(dic_param) for line in lines: if line[0] in ["#", "%"]: continue sline = re.split(r'(\s+)', line) keyword_name = sline[0] keyword = ("".join(sline[2:])).rstrip() if keyword_name in dic_param: upipe.print_info( "Initialisation of attribute {0}".format(keyword_name), pipe=self) setattr(self, keyword_name, keyword) # Here we drop the item which was initialised val = noninit_dic_param.pop(keyword_name) else: continue # Listing them as warning and using the hardcoded default for key in noninit_dic_param: upipe.print_warning( ("Parameter {param} not initialised " "We will use the default hardcoded value from " "init_musepipe.py").format(param=key)) setattr(self, key, dic_param[key])
def extract_onespectral_cube(self, wave1=default_wave_wcs, outcube_name=None, **kwargs): """Create a single pixel cube extracted from this one Input ---- wave1: float Value of the wavelength to extract. In Angstroems. outcube_name: str Name of the output cube prefix: str If outcube_name is None (default), use that prefix to append in front of the input cube name (same folder) Write ----- A new cube with only 2 lambda. To be used as a WCS reference for masks. """ # Find the wavelength k1 = self.wave.pixel([wave1], nearest=True)[0] # extracting the cube subcube = self[k1:k1 + 2, :, :] cube_folder, cube_name = os.path.split(self.filename) if outcube_name is None: prefix = kwargs.pop("prefix", "l{0:.0f}_".format(wave1)) outcube_name = "{0}{1}".format(prefix, cube_name) upipe.print_info("Writing up single wave-cube {0}\n" "in folder {1}".format(outcube_name, cube_folder)) subcube.write(joinpath(cube_folder, outcube_name)) return cube_folder, outcube_name
def reduce_target(self, targetname=None, list_pointings=None, **kwargs): """Reduce one target for a list of pointings Input ----- targetname: str Name of the target list_pointings: list Pointing numbers. Default is None (meaning all pointings indicated in the dictonary will be reduced) first_recipe: str or int [1] last_recipe: str or int [max of all recipes] Name or number of the first and last recipes to process """ # General print out upipe.print_info( "---- Starting the Data Reduction for Target={0} ----".format( targetname)) kwargs_recipe = {} for key, default in zip(['fraction', 'skymethod', 'illum'], [0.8, "model", True]): item = kwargs.pop(key, default) kwargs_recipe[key] = default # Initialise the pipe if needed if not self.pipes[targetname]._initialised \ or "first_recipe" in kwargs or "last_recipe" in kwargs: self.set_pipe_target(targetname=targetname, list_pointings=list_pointings, **kwargs) # Check if pointings are valid list_pointings = self._check_pointings(targetname, list_pointings) if len(list_pointings) == 0: return # Loop on the pointings for pointing in list_pointings: upipe.print_info( "====== START - POINTING {0:2d} ======".format(pointing)) # Initialise raw tables if not already done (takes some time) if not self.pipes[targetname][pointing]._raw_table_initialised: self.pipes[targetname][pointing].init_raw_table(overwrite=True) if self.__phangs: self.pipes[targetname][pointing].run_phangs_recipes( **kwargs_recipe) else: self.pipes[targetname][pointing].run_recipes(**kwargs_recipe) upipe.print_info( "====== END - POINTING {0:2d} ======".format(pointing))
def set_pipe_target(self, targetname=None, list_pointings=None, **kwargs): """Create the musepipe instance for that target and list of pointings Input ----- targetname: str Name of the target list_pointings: list Pointing numbers. Default is None (meaning all pointings indicated in the dictonary will be reduced) config_args: dic Dictionary including extra configuration parameters to pass to MusePipe. This allows to define a global configuration. If self.__phangs is set to True, this is overwritten with the default PHANGS configuration parameters as provided in config_pipe.py. """ verbose = kwargs.pop("verbose", self.verbose) # Check if targetname is valid if not self._check_targetname(targetname): return # Galaxy name upipe.print_info( "=== Initialising MusePipe for Target {name} ===".format( name=targetname)) # Check if pointings are valid list_pointings = self._check_pointings(targetname, list_pointings) if len(list_pointings) == 0: return # Get the filename and extension of log file log_filename, log_fileext = os.path.splitext( kwargs.pop("log_filename", "{0}_{1}.log".format(targetname, version_pack))) # Reading extra arguments from config dictionary if self.__phangs: config_args = PHANGS_reduc_config # Set overwrite to False to keep existing tables config_args['overwrite_astropy_table'] = False else: config_args = kwargs.pop("config_args", None) first_recipe = kwargs.pop("first_recipe", 1) last_recipe = kwargs.pop("last_recipe", None) # Over-writing the arguments in kwargs from config dictionary if config_args is not None: for attr in config_args: if attr not in kwargs: kwargs[attr] = config_args[attr] # extracting the kwargs list_kwargs = ', '.join( ['{}={!r}'.format(k, v) for k, v in kwargs.items()]) # Config files rc_filename = self.targets[targetname].rc_filename cal_filename = self.targets[targetname].cal_filename folder_config = self.targets[targetname].folder_config # Loop on the pointings for pointing in list_pointings: upipe.print_info( "Initialise Pipe for Target = {0:10s} / Pointing {1:02d} ". format(targetname, pointing)) # New log file name with pointing included log_filename_pointing = "{0}_P{1:02d}{2}".format( log_filename, pointing, log_fileext) # Setting up the names of the output files python_command = ("mypipe = musepipe.MusePipe(targetname='{0}', " "pointing={1}, folder_config='{2}', " "rc_filename='{3}', " "cal_filename='{4}', " "log_filename='{5}', verbose={6}, " "{7})".format(targetname, pointing, folder_config, rc_filename, cal_filename, log_filename_pointing, verbose, list_kwargs)) # Creating the musepipe instance, using the shortcut self.pipes[targetname][pointing] = MusePipe( targetname=targetname, pointing=pointing, folder_config=folder_config, rc_filename=rc_filename, cal_filename=cal_filename, log_filename=log_filename_pointing, first_recipe=first_recipe, last_recipe=last_recipe, init_raw_table=False, verbose=verbose, **kwargs) # Saving the command self.pipes[targetname][pointing].history = python_command # Setting back verbose to True to make sure we have a full account self.pipes[targetname][pointing].verbose = True upipe.print_info(python_command, pipe=self) upipe.print_info("End of Pipe initialisation") self.pipes[targetname]._initialised = True
def mask_pixtable(self, mask_name=None, **kwargs): """Use the Image Mask and create a new Pixtable Input ----- mask_name: str Name of the mask to be used (FITS file) use_folder: bool If True, use the same folder as the Pixtable Otherwise just write where you stand suffix_out: str Suffix for the name of the output Pixtable If provided, will overwrite the one in self.suffix_out """ # Open the PixTable upipe.print_info("Opening the Pixtable {0}".format(self.pixtable_name)) pixtable = PixTable(self.pixtable_name) # Use the Image mask and create a pixtable mask if mask_name is not None: self.mask_name = mask_name else: if not hasattr(self, "mask_name"): upipe.print_error("Please provide a mask name (FITS file)") return upipe.print_info("Creating a column Mask from file {0}".format( self.mask_name)) mask_col = pixtable.mask_column(self.mask_name) # extract the right data using the pixtable mask upipe.print_info("Extracting the Mask") newpixtable = pixtable.extract_from_mask(mask_col.maskcol) # Rewrite a new pixtable self.suffix_out = kwargs.pop("suffix_out", self.suffix_out) use_folder = kwargs.pop("use_folder", True) if use_folder: self.newpixtable_name = joinpath( self.pixtable_folder, "{0}{1}".format(self.suffix_out, self.pixtable_name)) else: self.newpixtable_name = "{0}{1}".format(self.suffix_out, self.pixtable_name) upipe.print_info("Writing the new PixTable in {0}".format( self.newpixtable_name)) newpixtable.write(self.newpixtable_name) # Now transfer the flat field if it exists ext_name = 'PIXTABLE_FLAT_FIELD' try: # Test if Extension exists by reading header # If it exists then do nothing test_data = pyfits.getheader(self.newpixtable_name, ext_name) upipe.print_warning( "Flat field extension already exists in masked PixTable - all good" ) # If it does not exist test if it exists in the original PixTable except KeyError: try: # Read data and header ff_ext_data = pyfits.getdata(self.pixtable_name, ext_name) ff_ext_h = pyfits.getheader(self.pixtable_name, ext_name) upipe.print_warning( "Flat field extension will be transferred from PixTable") # Append it to the new pixtable pyfits.append(self.newpixtable_name, ff_ext_data, ff_ext_h) except KeyError: upipe.print_warning( "No Flat field extension to transfer - all good") except: pass except: pass # Patch to fix the extension names of the PixTable # We have to put a number of extension in lowercase to make sure # the MUSE recipes understand them descl = ['xpos', 'ypos', 'lambda', 'data', 'dq', 'stat', 'origin'] for d in descl: try: pyfits.setval(self.newpixtable_name, keyword='EXTNAME', value=d, extname=d.upper()) upipe.print_warning( "Rewriting extension name {0} as lowercase".format( d.upper())) except: upipe.print_warning( "Extension {0} not present - patch ignored".format( d.upper()))
def convolve_cube(self, target_fwhm, target_nmoffat=None, input_function='moffat', target_function="gaussian", outcube_name=None, factor_fwhm=3, fft=True): """Convolve the cube for a target function 'gaussian' or 'moffat' Args: target_fwhm (float): target FWHM in arcsecond target_nmoffat: target n if Moffat function input_function (str): 'gaussian' or 'moffat' ['moffat'] target_function (str): 'gaussian' or 'moffat' ['gaussian'] factor_fwhm (float): number of FWHM for size of Kernel fft (bool): use FFT to convolve or not [False] Creates: Convolved cube """ # Separate folder and name of file cube_folder, cube_name = os.path.split(self.filename) # Creating the outcube filename if outcube_name is None: outcube_name = "conv{0}_{1:.2f}{2}".format( target_function.lower()[0], target_fwhm, cube_name) upipe.print_info("The new cube will be named: {}".format(outcube_name)) # Getting the shape of the Kernel scale_spaxel = self.get_step(unit_wcs=u.arcsec)[1] nspaxel = np.int(factor_fwhm * target_fwhm / scale_spaxel) # Make nspaxel odd to have a PSF centred at the centre of the frame if nspaxel % 2 == 0: nspaxel += 1 shape = [self.shape[0], nspaxel, nspaxel] # Computing the kernel kernel3d = cube_kernel(shape, self.wave.coord(), self.psf_fwhm0, target_fwhm, input_function, target_function, lambda0=self.psf_l0, input_nmoffat=self.psf_nmoffat, target_nmoffat=target_nmoffat, b=self.psf_b, scale=scale_spaxel, compute_kernel='pypher') if fft: upipe.print_info("Starting the FFT convolution") conv_cube = self.fftconvolve(other=kernel3d) else: upipe.print_info("Starting the convolution") conv_cube = self.convolve(other=kernel3d) # Write the output upipe.print_info("Writing up the derived cube") conv_cube.write(joinpath(cube_folder, outcube_name)) # just provide the output name by folder+name return cube_folder, outcube_name