def _check_pointings(self, targetname, list_pointings): """Check if pointing is in the list of pointings Returns the list of pointings if ok. If not, return an empty list Input ----- targetname: str name of the target list_pointings: list List of integer (pointings). Returns ------- list_pointings: list Empty if input list of pointings is not fully in defined list. """ # Info of the pointings and extracting the observing run for each pointing target_pointings = self.targets[targetname].list_pointings # if no list_pointings we just do them all if list_pointings == None: list_pointings = target_pointings else: if any([_ not in target_pointings for _ in list_pointings]): upipe.print_error( "ERROR: no pointing {0} for the given target".format( list_pointings)) return [] return list_pointings
def read(self): """Reading the data in the file """ if self.filter_ascii_file is None: try: upipe.print_info("Using the fits file {0} as input".format( self.filter_fits_file)) filter_data = pyfits.getdata(self.filter_fits_file, extname=self.filter_name) self.wave = filter_data['lambda'] self.throughput = filter_data['throughput'] except: upipe.print_error( "Problem opening the filter fits file {0}".format( self.filter_fits_file)) upipe.print_error( "Did not manage to get the filter {0} throughput".format( self.filter_name)) self.wave = np.zeros(0) self.throughput = np.zeros(0) else: upipe.print_info("Using the ascii file {0} as input".format( self.filter_ascii_file)) self.wave, self.throughput = np.loadtxt(self.filter_ascii_file, unpack=True)
def __init__(self, pixtable_name, image_name, suffix_out="tmask"): """Class made to associate a PixTable and an Image Used to select all pixels from a pixtable outside the trail defined by the trail mask of an image Then flushes the pixtable into a new one. Input ----- pixtable_name: str Name of the Pixel Table image_name: str Name of Image which should include a mask suffix_out: str Suffix to be used for the new PixTable """ self.suffix_out = suffix_out if not os.path.isfile(pixtable_name): upipe.print_error("Input PixTable does not exist") return if not os.path.isfile(image_name): upipe.print_error("Input Image does not exist") return self.image_name = image_name self.pixtable_name = pixtable_name self.pixtable_folder, self.pixtable_name = os.path.split(pixtable_name) self.image_folder, self.image_name = os.path.split(self.image_name) self.image = MuseImage(filename=image_name)
def __init__(self, TargetDic, rc_filename=None, cal_filename=None, folder_config="", first_recipe=1, **kwargs): """Using a given dictionary to initialise the sample That dictionary should include the names of the targets as keys and the subfolder plus pointings to consider Input ----- TargetDic: dic Dictionary of targets. Keys are target names. Values for each target name should be a list of 2 parameters. - The first one is the name of the subfolder (e.g. 'P101') - The second one is the list of pointings, itself a dictionary with a 0 or 1 for each pointing number depending on whether this should be included in the reduction or not. Results can be seen in self.dic_targets dictionary. rc_filename: str Default to None cal_filename: str Default to None PHANGS: bool Default to False. If True, will use default configuration dictionary from config_pipe. """ self.sample = TargetDic self.targetnames = list(TargetDic.keys()) self.first_recipe = first_recipe self.__phangs = kwargs.pop("PHANGS", False) self.verbose = kwargs.pop("verbose", False) # Reading configuration filenames if rc_filename is None or cal_filename is None: upipe.print_error( "rc_filename and/or cal_filename is None. Please define both.") return self.cal_filename = cal_filename self.rc_filename = rc_filename self.folder_config = folder_config # Initialisation of rc and cal files self._init_calib_files() # Initialisation of targets self.init_pipes = kwargs.pop("init_pipes", True) self.add_targetname = kwargs.pop("add_targetname", True) self._init_targets()
def _add_tplraw_to_sofdict(self, mean_mjd, expotype, reset=False): """ Add item to dictionary for the sof writing """ if reset: self._sofdict.clear() # Finding the best tpl for this raw file type expo_table = self._get_table_expo(expotype, "raw") index, this_tpl = self._select_closest_mjd(mean_mjd, expo_table) if index >= 0: self._sofdict[expotype] = [upipe.normpath(joinpath(self.paths.rawfiles, expo_table['filename'][index]))] else: upipe.print_error("Failed to find a raw exposure of type {} " "in this table".format(expotype))
def get_sky_spectrum(specname): """Read sky spectrum from MUSE data reduction """ if not os.path.isfile(specname): upipe.print_error("{0} not found".format(specname)) return None sky = pyfits.getdata(specname) crval = sky['lambda'][0] cdelt = sky['lambda'][1] - crval wavein = WaveCoord(cdelt=cdelt, crval=crval, cunit=u.angstrom) spec = Spectrum(wave=wavein, data=sky['data'], var=sky['stat']) return spec
def _add_tplmaster_to_sofdict(self, mean_mjd, expotype, reset=False): """ Add item to dictionary for the sof writing """ if reset: self._sofdict.clear() # Finding the best tpl for this master index, this_tpl = self._select_closest_mjd(mean_mjd, self._get_table_expo(expotype)) if self._debug: upipe.print_debug("Index = {0}, Tpl = {1}".format(index, this_tpl)) if index >= 0: dir_master = self._get_fullpath_expo(expotype) self._sofdict[get_suffix_product(expotype)] = [upipe.normpath(joinpath(dir_master, get_suffix_product(expotype) + "_" + this_tpl + ".fits"))] else: upipe.print_error("Failed to find a master exposure of type {} " "in this table".format(expotype))
def read(self): """Read sky continuum spectrum from MUSE data reduction """ if not os.path.isfile(self.filename): upipe.print_error("{0} not found".format(self.filename)) crval = 0.0 data = np.zeros(0) cdelt = 1.0 else: sky = pyfits.getdata(self.filename) crval = sky['lambda'][0] cdelt = sky['lambda'][1] - crval data = sky['flux'] wavein = WaveCoord(cdelt=cdelt, crval=crval, cunit=u.angstrom) self.spec = Spectrum(wave=wavein, data=data)
def update_calib_file(filename, subfolder="", folder_config=""): """Update the rcfile with a new root Input ----- filename: str Name of the input filename folder_config: str Default is "". Name of folder for filename subfolder: str Name of subfolder to add in the path """ full_filename = joinpath(folder_config, filename) if full_filename is None: upipe.print_error("ERROR: input filename is None") return "" # Testing existence of filename if not os.path.isfile(full_filename): upipe.print_error( "ERROR: input filename {inputname} cannot be found. ".format( inputname=full_filename)) return "" # If it exists, open and read it old_rc = open(full_filename) lines = old_rc.readlines() # Create new file new_filename = insert_suffix(full_filename, subfolder) new_rc = open(new_filename, 'w') # loop on lines for line in lines: sline = line.split() if sline[0] != "root": new_rc.write(line) continue if not os.path.isdir(sline[1]): upipe.print_warning( "{} not an existing folder (from rcfile)".format(sline[1])) newline = line.replace(sline[1], joinpath(sline[1], subfolder)) new_rc.write(newline) new_rc.close() old_rc.close() return new_filename
def build_list(self, folder_cubes=None, prefix_cubes=None, **kwargs): """Building the list of cubes to process Args: folder_cubes (str): folder for the cubes prefix_cubes (str): prefix to be used """ self.list_suffix = kwargs.pop("list_suffix", self.list_suffix) # Get the folder if needed if folder_cubes is not None: self.folder_cubes = folder_cubes if not self._check_folder(): return # get the prefix if provided if prefix_cubes is not None: self.prefix_cubes = prefix_cubes # get the list of cubes and return if 0 found list_cubes = glob.glob("{0}{1}*.fits".format(self.folder_cubes, self.prefix_cubes)) # if the list of suffix is empty, just use all cubes if len(self.list_suffix) == 0: self.list_cubes = list_cubes else: # Filtering out the ones that don't have any of the suffixes self.list_cubes = [] for l in list_cubes: if any([suff in l for suff in self.list_suffix]): self.list_cubes.append(l) if self.verbose: for i, name in enumerate(self.list_cubes): upipe.print_info("Cube {0:03d}: {1}".format(i + 1, name)) self.ncubes = len(self.list_cubes) if self.ncubes == 0: upipe.print_error("Found 0 cubes in this folder with suffix" " {}: please change suffix".format( self.prefix_cubes)) else: upipe.print_info("Found {} cubes in this folder".format( self.ncubes))
def get_filter_image(self, filter_name=None, own_filter_file=None, filter_folder="", dic_extra_filters=None): """Get an image given by a filter. If the filter belongs to the filter list, then use that, otherwise use the given file """ try: upipe.print_info("Reading MUSE filter {0}".format(filter_name)) refimage = self.get_band_image(filter_name) except ValueError: # initialise the filter file upipe.print_info( "Reading private reference filter {0}".format(filter_name)) if dic_extra_filters is not None: if filter_name in dic_extra_filters: filter_file = dic_extra_filters[filter_name] else: upipe.print_error( "[mpdaf_pipe / get_filter_image] " "Filter name not in private dictionary - Aborting") return else: if own_filter_file is None: upipe.print_error( "[mpdaf_pipe / get_filter_image] " "No extra filter dictionary and " "the private filter file is not set - Aborting") return else: filter_file = own_filter_file # Now reading the filter data path_filter = joinpath(filter_folder, filter_file) filter_wave, filter_sensitivity = np.loadtxt(path_filter, unpack=True) refimage = self.bandpass_image(filter_wave, filter_sensitivity, interpolation='linear') key = 'HIERARCH ESO DRS MUSE FILTER NAME' refimage.primary_header[key] = (filter_name, 'filter name used') add_mpdaf_method_keywords(refimage.primary_header, "cube.bandpass_image", ['name'], [filter_name], ['filter name used']) return refimage
def _get_calib_filenames(self, targetname=None): """Get calibration file names Input ---- targetname: str Returns ------- folder_name: str rcname: str calname: str """ # using targetname or not if targetname is None: name_rc = self.rc_filename name_cal = self.cal_filename else: name_rc = insert_suffix(self.rc_filename, self.targets[targetname].subfolder) name_cal = insert_suffix(self.cal_filename, self.targets[targetname].subfolder) folder_config = self.folder_config # Checking the folders folder_rc, rc_filename_target = os.path.split( joinpath(folder_config, name_rc)) folder_cal, cal_filename_target = os.path.split( joinpath(folder_config, name_cal)) if rc_filename_target == "" or cal_filename_target == "": upipe.print_error("Missing a calibration name file") return if folder_rc == folder_cal: folder_config = folder_rc else: rc_filename_target = joinpath(folder_rc, rc_filename_target) cal_filename_target = joinpath(folder_cal, cal_filename_target) folder_config = "" return folder_config, rc_filename_target, cal_filename_target
def save_normalised(self, norm_factor=1.0, prefix="norm", overwrite=False): """Normalises a sky continuum spectrum and save it within a new fits file Input ----- norm_factor: float Scale factor to multiply the input continuum prefix: str Prefix for the new continuum fits name. Default is 'norm', so that the new file is 'norm_oldname.fits' overwrite: bool If True, existing file will be overwritten. Default is False. """ if prefix == "": upipe.print_error( "[mpdaf_pipe / save_normalised] The new and old sky " "continuum fits files will share the same name") upipe.print_error("This is not recommended - Aborting") return folder_spec, filename = os.path.split(self.filename) newfilename = "{0}{1}".format(prefix, filename) norm_filename = joinpath(folder_spec, newfilename) # Opening the fits file skycont = pyfits.open(self.filename) # getting the data dcont = skycont['CONTINUUM'].data # Create new continuum # ------------------------------ new_cont = dcont['flux'] * norm_factor skycont['CONTINUUM'].data['flux'] = new_cont # Writing to the new file skycont.writeto(norm_filename, overwrite=overwrite) upipe.print_info( 'Normalised Factor used = {0:8.4f}'.format(norm_factor)) upipe.print_info('Normalised Sky Continuum {} has been created'.format( norm_filename))
def _check_targetname(self, targetname): """Check if targetname is in list Input ----- targetname: str Returns ------- status: bool True if yes, False if no. """ if targetname not in self.targetnames: upipe.print_error( "ERROR: no Target named {name} in the defined sample".format( name=targetname)) return False else: return True
def read_param_file(self, filename, dic_param): """Reading an input parameter initialisation file """ # Testing existence of filename if not os.path.isfile(filename): upipe.print_error( ("Input parameter {inputname} cannot be found. " "We will use the default hardcoded in the " "init_musepipe.py module").format(inputname=filename)) return # If it exists, open and read it f_param = open(filename) lines = f_param.readlines() # Dummy dictionary to see which items are not initialised noninit_dic_param = copy.copy(dic_param) for line in lines: if line[0] in ["#", "%"]: continue sline = re.split(r'(\s+)', line) keyword_name = sline[0] keyword = ("".join(sline[2:])).rstrip() if keyword_name in dic_param: upipe.print_info( "Initialisation of attribute {0}".format(keyword_name), pipe=self) setattr(self, keyword_name, keyword) # Here we drop the item which was initialised val = noninit_dic_param.pop(keyword_name) else: continue # Listing them as warning and using the hardcoded default for key in noninit_dic_param: upipe.print_warning( ("Parameter {param} not initialised " "We will use the default hardcoded value from " "init_musepipe.py").format(param=key)) setattr(self, key, dic_param[key])
def get_normfactor(self, background, filter_name="Cousins_R"): """Get the normalisation factor given a background value Takes the background value and the sky continuuum spectrum and convert this to the scaling Ks needed for this sky continuum The principle relies on having the background measured as: MUSE_calib = ((MUSE - Sky_cont) + Background) * Norm as measured from the alignment procedure. Since we want: MUSE_calib = ((MUSE - Ks * Sky_cont) + 0) * Norm This means that: Ks * Sky_cont = Sky_cont - Background ==> Ks = 1 - Background / Sky_cont So we integrate the Sky_cont to get the corresponding S value and then provide Ks as 1-B/S Input ----- background: float Value of the background to consider filter_name: str Name of the filter to consider """ if not hasattr(self, filter_name): upipe.print_error( "No integration value for filter {0}".format(filter_name)) norm = 1. else: muse_filter = getattr(self, filter_name) if muse_filter.flux_cont != 0.: norm = 1. - background / muse_filter.flux_cont else: norm = 1. # Saving the norm value for that filter muse_filter.norm = norm muse_filter.background = background
def mask_pixtable(self, mask_name=None, **kwargs): """Use the Image Mask and create a new Pixtable Input ----- mask_name: str Name of the mask to be used (FITS file) use_folder: bool If True, use the same folder as the Pixtable Otherwise just write where you stand suffix_out: str Suffix for the name of the output Pixtable If provided, will overwrite the one in self.suffix_out """ # Open the PixTable upipe.print_info("Opening the Pixtable {0}".format(self.pixtable_name)) pixtable = PixTable(self.pixtable_name) # Use the Image mask and create a pixtable mask if mask_name is not None: self.mask_name = mask_name else: if not hasattr(self, "mask_name"): upipe.print_error("Please provide a mask name (FITS file)") return upipe.print_info("Creating a column Mask from file {0}".format( self.mask_name)) mask_col = pixtable.mask_column(self.mask_name) # extract the right data using the pixtable mask upipe.print_info("Extracting the Mask") newpixtable = pixtable.extract_from_mask(mask_col.maskcol) # Rewrite a new pixtable self.suffix_out = kwargs.pop("suffix_out", self.suffix_out) use_folder = kwargs.pop("use_folder", True) if use_folder: self.newpixtable_name = joinpath( self.pixtable_folder, "{0}{1}".format(self.suffix_out, self.pixtable_name)) else: self.newpixtable_name = "{0}{1}".format(self.suffix_out, self.pixtable_name) upipe.print_info("Writing the new PixTable in {0}".format( self.newpixtable_name)) newpixtable.write(self.newpixtable_name) # Now transfer the flat field if it exists ext_name = 'PIXTABLE_FLAT_FIELD' try: # Test if Extension exists by reading header # If it exists then do nothing test_data = pyfits.getheader(self.newpixtable_name, ext_name) upipe.print_warning( "Flat field extension already exists in masked PixTable - all good" ) # If it does not exist test if it exists in the original PixTable except KeyError: try: # Read data and header ff_ext_data = pyfits.getdata(self.pixtable_name, ext_name) ff_ext_h = pyfits.getheader(self.pixtable_name, ext_name) upipe.print_warning( "Flat field extension will be transferred from PixTable") # Append it to the new pixtable pyfits.append(self.newpixtable_name, ff_ext_data, ff_ext_h) except KeyError: upipe.print_warning( "No Flat field extension to transfer - all good") except: pass except: pass # Patch to fix the extension names of the PixTable # We have to put a number of extension in lowercase to make sure # the MUSE recipes understand them descl = ['xpos', 'ypos', 'lambda', 'data', 'dq', 'stat', 'origin'] for d in descl: try: pyfits.setval(self.newpixtable_name, keyword='EXTNAME', value=d, extname=d.upper()) upipe.print_warning( "Rewriting extension name {0} as lowercase".format( d.upper())) except: upipe.print_warning( "Extension {0} not present - patch ignored".format( d.upper()))
def _check_folder(self): if not os.path.isdir(self.folder_cubes): upipe.print_error("Cube Folder {} does not exists \n" "- Aborting".format(self.folder_cubes)) return False return True