def adjust_alignment(self, name_ima_reference, list_expo=None, line=None, filter_name="CousinsR"): """Adjust the alignment using a background image """ # Selecting the table with the right iexpo found_expo, list_expo, group_list_expo, align_table = self._select_list_expo( "OBJECT", "ALL", "processed", list_expo) if not found_expo: if self.verbose: upipe.print_warning( "No exposure recovered for the fine alignment") return self.groupexpo = [] suffix = "_{0}".format(filter_name) if line is not None: suffix += "_{0}".format(line) for gtable in align_table.groups: mytpl, mymjd = self._get_tpl_meanmjd(gtable) list_group_expo = gtable['iexpo'].data list_names_muse = [ joinpath( self._get_fullpath_expo("OBJECT", "processed"), 'IMAGE_FOV{0}_{1:04d}_{2}.fits'.format( suffix, iexpo, mytpl)) for iexpo in list_group_expo ] self.groupexpo.append( AlignMusePointing(name_ima_reference, list_names_muse, flag="mytpl"))
def _select_list_expo(self, expotype, tpl, stage, list_expo=None): """Select the expo numbers which exists for a certain expotype """ # First selecting the files via the grouped table tpl_table = self.select_tpl_files(expotype=expotype, tpl=tpl, stage=stage) # Selecting the table with the right iexpo if list_expo is None: list_expo = tpl_table['iexpo'].data # First we isolate the unique values of iexpo from the table list_expo = np.unique(list_expo) # Then we select those who exist in the table # And don't forget to group the table by tpls group_table = tpl_table[np.isin(tpl_table['iexpo'], list_expo)].group_by('tpls') group_list_expo = [] for gtable in group_table: group_list_expo.append(gtable['iexpo'].data) found_expo = True if len(group_table) == 0: found_expo = False if self.verbose: upipe.print_warning( "No {0} recovered from the {1} astropy file " "Table - Aborting".format(expotype, stage)) return found_expo, list_expo, group_list_expo, group_table
def __init__(self, nifu=-1, first_cpu=0, ncpu=24, list_cpu=[], likwid=default_likwid, fakemode=True, domerge=True, nocache=True, nochecksum=True): """Initialisation of PipeRecipes """ # Fake mode self.fakemode = fakemode if self.verbose: if fakemode: upipe.print_warning("WARNING: running in FAKE mode") else: upipe.print_warning("WARNING: running actual recipes") if nocache: self.nocache = "nocache" else: self.nocache = "" # Addressing CPU by number (cpu0=start, cpu1=end) self.first_cpu = first_cpu self.ncpu = ncpu self.likwid = likwid self.nifu = nifu self._set_cpu(first_cpu, ncpu, list_cpu) self._domerge = domerge self.nochecksum = nochecksum
def run_twilight(self, sof_filename='twilight', tpl="ALL", update=None, illum=True): """Reducing the files and creating the TWILIGHT CUBE. Will run the esorex muse_twilight command on all TWILIGHT Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table tpl_gtable = self.select_tpl_files(expotype='TWILIGHT', tpl=tpl) if len(tpl_gtable) == 0: if self.verbose: upipe.print_warning( "No TWILIGHT recovered from the astropy file Table - Aborting" ) return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the LSF including # the list of files to be processed for one MASTER Flat self._add_calib_to_sofdict("BADPIX_TABLE", reset=True) self._add_calib_to_sofdict("VIGNETTING_MASK") for gtable in tpl_gtable.groups: # extract the tpl (string) and mean mjd (float) tpl, mean_mjd = self._get_tpl_meanmjd(gtable) self._add_geometry_to_sofdict(tpl) # Provide the list of files to the dictionary self._sofdict['SKYFLAT'] = add_listpath(self.paths.rawfiles, list(gtable['filename'])) # Finding the best tpl for BIAS, FLAT, ILLUM, TRACE, WAVE if illum: self._add_tplraw_to_sofdict(mean_mjd, "ILLUM") self._add_list_tplmaster_to_sofdict( mean_mjd, ['BIAS', 'FLAT', 'TRACE', 'WAVE']) # Writing the sof file self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True) # Names and folder of final Master Wave dir_twilight = self._get_fullpath_expo('TWILIGHT', "master") name_twilight = deepcopy(dic_files_products['TWILIGHT']) self.recipe_twilight(self.current_sof, dir_twilight, name_twilight, tpl) # Write the MASTER TWILIGHT Table and save it self.save_expo_table('TWILIGHT', tpl_gtable, "master", update=update) # Go back to original folder self.goto_prevfolder(logfile=True)
def run_flat(self, sof_filename='flat', tpl="ALL", update=None): """Reducing the Flat files and creating a Master Flat Will run the esorex muse_flat command on all Flats Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table tpl_gtable = self.select_tpl_files(expotype='FLAT', tpl=tpl) if len(tpl_gtable) == 0: if self.verbose: upipe.print_warning( "No FLAT recovered from the astropy Table - Aborting") return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the FLATs including # the list of files to be processed for one MASTER Flat # Adding the bad pixel table self._add_calib_to_sofdict("BADPIX_TABLE", reset=True) for gtable in tpl_gtable.groups: # Provide the list of files to the dictionary self._sofdict['FLAT'] = add_listpath(self.paths.rawfiles, list(gtable['filename'])) # extract the tpl (string) and mean mjd (float) tpl, mean_mjd = self._get_tpl_meanmjd(gtable) # Adding the best tpc MASTER_BIAS self._add_tplmaster_to_sofdict(mean_mjd, 'BIAS') # Writing the sof file self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True) # Name of final Master Flat and Trace Table dir_flat = self._get_fullpath_expo('FLAT', "master") name_flat = self._get_suffix_product('FLAT') dir_trace = self._get_fullpath_expo('TRACE', "master") name_tracetable = self._get_suffix_product('TRACE') # Run the recipe self.recipe_flat(self.current_sof, dir_flat, name_flat, dir_trace, name_tracetable, tpl) # Write the MASTER FLAT Table and save it self.save_expo_table('FLAT', tpl_gtable, "master", update=update) self.save_expo_table('TRACE', tpl_gtable, "master", update=update) # Go back to original folder self.goto_prevfolder(logfile=True)
def run_standard(self, sof_filename='standard', tpl="ALL", update=None): """Reducing the STD files after they have been obtained Running the muse_standard routine Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table std_table = self.select_tpl_files("STD", tpl=tpl, stage="processed") if len(std_table) == 0: if self.verbose: upipe.print_warning( "No processed STD recovered from the astropy file Table - Aborting" ) return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the STD Sof for i in range(len(std_table)): mytpl = std_table['tpls'][i] iexpo = np.int(std_table['iexpo'][i]) if tpl != "ALL" and tpl != mytpl: continue # Now starting with the standard recipe self._add_calib_to_sofdict("EXTINCT_TABLE", reset=True) self._add_calib_to_sofdict("STD_FLUX_TABLE") self._sofdict['PIXTABLE_STD'] = [ joinpath( self._get_fullpath_expo("STD", "processed"), 'PIXTABLE_STD_{0}_{1:04d}-{2:02d}.fits'.format( mytpl, iexpo, j + 1)) for j in range(24) ] self.write_sof(sof_filename=sof_filename + "_" + mytpl, new=True) name_std = deepcopy(dic_files_products['STD']) dir_std = self._get_fullpath_expo('STD', "master") self.recipe_std(self.current_sof, dir_std, name_std, mytpl) # Write the MASTER files Table and save it self.save_expo_table("STD", std_table, "master", aggregate=False, update=update) # Go back to original folder self.goto_prevfolder(logfile=True)
def open_offset_table(self, name_table=None): """Read offset table from fits file """ if name_table is None: if not hasattr(self, name_table): upipe.print_error("No FITS table name provided, Aborting Open") return None, Table() if not os.path.isfile(name_table): upipe.print_warning("FITS Table does not exist yet" "({0})".format(name_table)) return False, Table() return True, Table.read(name_table)
def _get_list_muse_images(self): # test if 1 or several images if isinstance(self.name_muse_images, str): self.list_muse_images = [self.name_muse_images] elif isinstance(self.name_muse_images, list): self.list_muse_images = self.name_muse_images else: upipe.print_warning("Name of images is not a string or a list, " "please check input name_muse_images") self.nimages = 0 return # Number of images to deal with self.nimages = len(self.list_muse_images)
def __init__(self, dirname="Config/", rc_filename=None, cal_filename=None, verbose=True, **kwargs): """Define the default parameters (folders/calibration files) and name suffixes for the MUSE data reduction """ self.verbose = verbose # Will first test if there is an rc_file provided # If not, it will look for a default rc_filename, the name of which is provided # above. If not, the hardcoded default will be used. # First adding the suffix to the dictionaries # attributing the dictionaries self._dic_folders = dic_folders self._dic_input_folders = dic_input_folders if rc_filename is None: if not os.path.isfile(default_rc_filename): upipe.print_warning( ("No filename or {default_rc} file " "to initialise from. We will use the default hardcoded " "in the init_musepipe.py module").format( default_rc=default_rc_filename)) self.init_default_param(dic_user_folders) else: self.read_param_file(default_rc_filename, dic_user_folders) self.rcfile = "default_values" else: rcfile = joinpath(dirname, rc_filename) self.read_param_file(rcfile, dic_user_folders) self.rcfile = rcfile # Initialisation of fixed attributes for the structure self.init_default_param(dic_folders) self.init_default_param(dic_input_folders) # Same happens with the calibration files. # If filename is provided, will use that, otherwise use the hard coded values. if cal_filename is None: self.init_default_param(dic_calib_tables) self.calfile = "default_values" else: calfile = joinpath(dirname, cal_filename) self.read_param_file(calfile, dic_calib_tables) self.calfile = calfile
def run_lsf(self, sof_filename='lsf', tpl="ALL", update=None): """Reducing the LSF files and creating the LSF PROFILE Will run the esorex muse_lsf command on all Flats Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table tpl_gtable = self.select_tpl_files(expotype='WAVE', tpl=tpl) if len(tpl_gtable) == 0: if self.verbose: upipe.print_warning( "No WAVE recovered from the astropy file Table - Aborting") return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the LSF including # the list of files to be processed for one MASTER Flat # Adding Badpix table and Line Catalog self._add_calib_to_sofdict("BADPIX_TABLE", reset=True) self._add_calib_to_sofdict("LINE_CATALOG") for gtable in tpl_gtable.groups: # Provide the list of files to the dictionary self._sofdict['ARC'] = add_listpath(self.paths.rawfiles, list(gtable['filename'])) # extract the tpl (string) and mean mjd (float) tpl, mean_mjd = self._get_tpl_meanmjd(gtable) # Finding the best tpl for BIAS, TRACE, WAVE self._add_list_tplmaster_to_sofdict(mean_mjd, ['BIAS', 'TRACE', 'WAVE']) # Writing the sof file self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True) # Name of final Master Wave dir_lsf = self._get_fullpath_expo('LSF', "master") name_lsf = self._get_suffix_product('LSF') # Run the recipe self.recipe_lsf(self.current_sof, dir_lsf, name_lsf, tpl) # Write the MASTER LSF PROFILE Table and save it self.save_expo_table('LSF', tpl_gtable, "master", update=update) # Go back to original folder self.goto_prevfolder(logfile=True)
def save_expo_table(self, expotype, tpl_gtable, stage="master", fits_tablename=None, aggregate=True, suffix="", overwrite=None, update=None): """Save the Expo (Master or not) Table corresponding to the expotype """ self._set_option_astropy_table(overwrite, update) if fits_tablename is None: fits_tablename = self._get_fitstablename_expo( expotype, stage, suffix) attr_expo = self._get_attr_expo(expotype) full_tablename = joinpath(self.paths.astro_tables, fits_tablename) if aggregate: table_to_save = tpl_gtable.groups.aggregate(np.mean)['tpls', 'mjd', 'tplnexp'] else: table_to_save = copy.copy(tpl_gtable) # If the file already exists if os.path.isfile(full_tablename): # Check if we update if self._update_astropy_table: # Reading the existing table upipe.print_warning( "Reading the existing Astropy table {0}".format( fits_tablename)) existing_table = Table.read(full_tablename, format="fits") # first try to see if they are compatible by using vstack try: stack_temptable = vstack([existing_table, table_to_save], join_type='exact') upipe.print_warning( "Updating the existing Astropy table {0}".format( fits_tablename)) table_to_save = apy.table.unique(stack_temptable, keep='first') except TableMergeError: upipe.print_warning( "Astropy Table cannot be joined to the existing one") return # Check if we want to overwrite or add the line in elif not self._overwrite_astropy_table: upipe.print_warning( "Astropy Table {0} already exists, " " use overwrite_astropy_table to overwrite it".format( fits_tablename)) return table_to_save.write(full_tablename, format="fits", overwrite=True) setattr(self._dic_tables[stage], attr_expo, table_to_save)
def read_astropy_table(self, expotype=None, stage="master"): """Read an existing Masterfile data table to start the pipeline """ # Read the astropy table name_table = self._get_fitstablename_expo(expotype, stage) if not os.path.isfile(name_table): upipe.print_warning( "Astropy table {0} does not exist - setting up an empty one". format(name_table)) return Table([[], [], []], names=['tpls', 'mjd', 'tplnexp']) else: if self.verbose: upipe.print_info( "Reading Astropy fits Table {0}".format(name_table)) return Table.read(name_table, format="fits")
def run_bias(self, sof_filename='bias', tpl="ALL", update=None): """Reducing the Bias files and creating a Master Bias Will run the esorex muse_bias command on all Biases Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table tpl_gtable = self.select_tpl_files(expotype='BIAS', tpl=tpl) if len(tpl_gtable) == 0: if self.verbose: upipe.print_warning( "No BIAS recovered from the astropy Table - Aborting") return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the BIASES including # the list of files to be processed for one MASTER BIAS # Adding the bad pixel table self._add_calib_to_sofdict("BADPIX_TABLE", reset=True) for gtable in tpl_gtable.groups: # Provide the list of files to the dictionary self._sofdict['BIAS'] = add_listpath(self.paths.rawfiles, list(gtable['filename'])) # extract the tpl (string) tpl = gtable['tpls'][0] # Writing the sof file self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True) # Name of final Master Bias name_bias = self._get_suffix_product('BIAS') dir_bias = self._get_fullpath_expo('BIAS', "master") # Run the recipe self.recipe_bias(self.current_sof, dir_bias, name_bias, tpl) # Write the MASTER BIAS Table and save it self.save_expo_table('BIAS', tpl_gtable, "master", update=update) # Go back to original folder self.goto_prevfolder(logfile=True)
def save_fits_offset_table(self, name_output_table=None, overwrite=False, suffix=""): """Save the Offsets into a fits Table """ if name_out_table is None: name_out_table = self.name_input_table self.suffix = suffix name_output_table = name_output_table.replace( ".fits", "{0}.fits".format(self.suffix)) exist_table, fits_table = self.open_offset_table(self.name_out_table) if exist_table is None: upipe.print_error("Save is aborted") return # Check if RA_OFFSET is there if 'RA_OFFSET' in fits_table.columns: # if yes, then check if the ORIG column is there if 'RA_OFFSET_ORIG' not in fits_table.columns: fits_table['RA_OFFSET_ORIG'] = fits_table['RA_OFFSET'] fits_table['DEC_OFFSET_ORIG'] = fits_table['DEC_OFFSET'] # if not, just continue # as it means the ORIG columns were already done # Saving the final values fits_table['RA_OFFSET'] = self.total_off_arcsec[:, 0] / 3600. fits_table['DEC_OFFSET'] = self.total_off_arcsec[:, 1] / 3600. fits_table['RA_CROSS_OFFSET'] = self.cross_off_arcsec[:, 0] / 3600. fits_table['DEC_CROSS_OFFSET'] = self.cross_off_arcsec[:, 1] / 3600. # Writing it up if exist_table and not overwrite: upipe.print_warning( "Table already exists, but overwrite is set to False") upipe.print_warning( "If you wish to overwrite the table {0}, " "please set overwrite to True".format(name_output_table)) return fits_table.write(self.name_output_table, overwrite=overwrite) self.name_output_table = name_output_table
def init_guess_offset(self, firstguess="crosscorr", name_input_table="OFFSET_LIST.fits"): """Initialise first guess, either from cross-correlation (default) or from an Offset FITS Table """ # Implement the guess if firstguess not in ["crosscorr", "fits"]: firstguess = "crosscorr" upipe.print_warning("Keyword 'firstguess' not recognised") upipe.print_warning( "Using Cross-Correlation as a first guess of the alignment") if firstguess == "crosscorr": self.init_off_arcsec = self.cross_off_arcsec self.init_off_pixel = self.cross_off_pixel elif firstguess == "fits": self.name_input_table = name_input_table exist_table, self.offset_table = self.open_offset_table( self.name_input_table) if exist_table is not True: upipe.print_warning( "Fits initialisation table not found, setting init value to 0" ) self.init_off_pixel *= 0. self.init_off_arcsec *= 0. else: self.init_off_arcsec = np.vstack( (self.offset_table['RA_OFFSET'], self.offset_table['DEC_OFFSET'])).T * 3600. for i in range(self.nimages): self.init_off_pixel[nima] = arcsec_to_pixel( self.list_muse_hdu[nima], self.init_off_arcsec[nima])
def select_tpl_files(self, expotype=None, tpl="ALL", stage="raw"): """Selecting a subset of files from a certain type """ if expotype not in musepipe.listexpo_types.keys(): upipe.print_info( "ERROR: input {0} is not in the list of possible values". format(expotype)) return MUSE_subtable = self._get_table_expo(expotype, stage) if len(MUSE_subtable) == 0: if self.verbose: upipe.print_warning( "Empty file table of type {0}".format(expotype)) upipe.print_warning( "Returning an empty Table from the tpl -astropy- selection" ) return MUSE_subtable group_table = MUSE_subtable.group_by('tpls') if tpl == "ALL": return group_table else: return group_table.groups[group_table.groups.keys['tpls'] == tpl]
def __init__(self, name_reference, name_muse_images, median_window=10, subim_window=10, dynamic_range=10, border=50, hdu_ext=[0, 1], **kwargs): """Initialise the AlignMuseImages class Keywords -------- median_window: int Size of window used in median filter to extract features in cross-correlation. Should be an odd integer subim_window: int Size of window for fitting peak of cross-correlation function dynamic_range: float Apply an arctan transform to data to suppress values more than DynamicRange times the median of the image border: int Ignore pixels this close to the border in the cross correlation xoffset: float Offset in arcseconds to be added to the cross-correlation offset (X) yoffset: float Offset in arcseconds to be added to the cross-correlation offset (Y) run: boolean If True will use extra_xyoffset and plot the result """ # Some input variables for the cross-correlation self.use_montage = kwargs.pop("use_montage", True) self.verbose = kwargs.pop("verbose", True) self.plot = kwargs.pop("plot", True) self.border = border self.subim_window = subim_window self.median_window = median_window self.dynamic_range = dynamic_range self.name_reference = name_reference self.name_muse_images = name_muse_images self.name_musehdr = kwargs.pop("name_musehdr", "muse") self.name_offmusehdr = kwargs.pop("name_offmusehdr", "offsetmuse") self.name_refhdr = kwargs.pop("name_refhdr", "reference.hdr") self.flag = kwargs.pop("flag", "") self._get_list_muse_images(**kwargs) if self.nimages == 0: upipe.print_warning("0 MUSE images detected as input") return self.list_offmuse_hdu = [None] * self.nimages self.list_proj_refhdu = [None] * self.nimages self.cross_off_pixel = np.zeros((self.nimages, 2), dtype=np.float32) self.extra_off_pixel = np.zeros_like(self.cross_off_pixel) self.init_off_pixel = np.zeros_like(self.cross_off_pixel) self.total_off_pixel = np.zeros_like(self.cross_off_pixel) self.cross_off_arcsec = np.zeros_like(self.cross_off_pixel) self.extra_off_arcsec = np.zeros_like(self.cross_off_pixel) self.init_off_arcsec = np.zeros_like(self.cross_off_pixel) self.total_off_arcsec = np.zeros_like(self.cross_off_pixel) # Cross normalisation for the images # This contains the 2 parameters for a linear transformation self.ima_polynorm = np.zeros((self.nimages, 2), dtype=np.float32) # Which extension to be used for the ref and muse images self.hdu_ext = hdu_ext # Open the Ref and MUSE image self.open_hdu() # find the cross correlation peaks for each image self.find_ncross_peak() # Initialise the offset firstguess = kwargs.pop("firstguess", "crosscorr") self.init_guess_offset(firstguess, **kwargs) self.total_off_arcsec = self.init_off_arcsec + self.extra_off_arcsec self.total_off_pixel = self.init_off_pixel + self.extra_off_pixel
def run_scipost(self, sof_filename='scipost', expotype="OBJECT", tpl="ALL", stage="processed", list_expo=None, lambdaminmax=[4000., 10000.], suffix="", AC_suffix="_AC", **kwargs): """Scipost treatment of the objects Will run the esorex muse_scipost routine Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time list_expo: list of integers providing the exposure numbers """ # Backward compatibility old_naming_convention = kwargs.pop("old_naming_convention", False) # Selecting the table with the right iexpo found_expo, list_expo, group_list_expo, scipost_table = self._select_list_expo( expotype, tpl, stage, list_expo) if not found_expo: return if len(list_expo) == 1: suffix += "_{0:04d}".format(list_expo[0]) # Lambda min and max? [lambdamin, lambdamax] = lambdaminmax # Save options save = kwargs.pop("save", "cube,skymodel,individual") # Filters filter_list = kwargs.pop("filter_list", "white,Cousins_R") filter_for_alignment = kwargs.pop("filter_for_alignment", "Cousins_R") offset_list = kwargs.pop("offset_list", "True") autocalib = kwargs.pop("autocalib", "none") rvcorr = kwargs.pop("rvcorr", "bary") default_suffix_skycontinuum = "" suffix_skycontinuum = kwargs.pop("suffix_skycont", default_suffix_skycontinuum) if rvcorr != "bary": upipe.print_warning( "Scipost will use '{0}' as barycentric " "correction [Options are bary, helio, geo, none]".format( rvcorr)) if suffix_skycontinuum != default_suffix_skycontinuum: upipe.print_warning( "Scipost will use '{0}' as suffix " "for the SKY_CONTINUUM files".format(suffix_skycontinuum)) # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for scipost # Selecting either one or all of the exposures for gtable in scipost_table.groups: # extract the tpl (string) and mean mjd (float) tpl, mean_mjd = self._get_tpl_meanmjd(gtable) # Now starting with the standard recipe self._add_calib_to_sofdict("EXTINCT_TABLE", reset=True) self._add_calib_to_sofdict("SKY_LINES") self._add_calib_to_sofdict("FILTER_LIST") if autocalib == "user": self._add_skycalib_to_sofdict("AUTOCAL_FACTORS", mean_mjd, 'SKY', "processed", suffix=AC_suffix) self._add_astrometry_to_sofdict(tpl) self._add_skycalib_to_sofdict("STD_RESPONSE", mean_mjd, 'STD') self._add_skycalib_to_sofdict("STD_TELLURIC", mean_mjd, 'STD') self._add_skycalib_to_sofdict("SKY_CONTINUUM", mean_mjd, 'SKY', "processed", perexpo=True, suffix=suffix_skycontinuum) self._add_tplmaster_to_sofdict(mean_mjd, 'LSF') if offset_list: self._sofdict['OFFSET_LIST'] = [ joinpath( self._get_fullpath_expo(expotype, "processed"), '{0}_{1}_{2}.fits'.format( dic_files_products['ALIGN'][0], filter_for_alignment, tpl)) ] # Selecting only exposures to be treated # We need to force 'OBJECT' to make sure scipost will deal with the exposure # even if it is e.g., a SKY pixtable_name = self._get_suffix_product('OBJECT') pixtable_name_thisone = self._get_suffix_product(expotype) self._sofdict[pixtable_name] = [] list_group_expo = gtable['iexpo'].data for iexpo in list_group_expo: if old_naming_convention: self._sofdict[pixtable_name] += [ joinpath( self._get_fullpath_expo(expotype, "processed"), '{0}_{1:04d}-{2:02d}.fits'.format( pixtable_name_thisone, iexpo, j + 1)) for j in range(24) ] else: self._sofdict[pixtable_name] += [ joinpath( self._get_fullpath_expo(expotype, "processed"), '{0}_{1}_{2:04d}-{3:02d}.fits'.format( pixtable_name_thisone, tpl, iexpo, j + 1)) for j in range(24) ] self.write_sof(sof_filename="{0}_{1}{2}_{3}".format( sof_filename, expotype, suffix, tpl), new=True) # products dir_products = self._get_fullpath_expo(expotype, "processed") name_products, suffix_products, suffix_finalnames = self._get_scipost_products( save, list_group_expo, filter_list) self.recipe_scipost(self.current_sof, tpl, expotype, dir_products, name_products, suffix_products, suffix_finalnames, lambdamin=lambdamin, lambdamax=lambdamax, save=save, list_expo=list_group_expo, suffix=suffix, filter_list=filter_list, autocalib=autocalib, rvcorr=rvcorr, **kwargs) # Write the MASTER files Table and save it self.save_expo_table( expotype, scipost_table, "reduced", "IMAGES_FOV_{0}{1}_{2}_list_table.fits".format( expotype, suffix, tpl), aggregate=False, update=True) # Go back to original folder self.goto_prevfolder(logfile=True)
def run_sky(self, sof_filename='sky', tpl="ALL", fraction=0.8, update=None): """Reducing the SKY after they have been scibasic reduced Will run the esorex muse_create_sky routine Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table sky_table = self._get_table_expo("SKY", "processed") if len(sky_table) == 0: if self.verbose: upipe.print_warning( "No SKY recovered from the astropy file Table - Aborting") return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the LSF including # the list of files to be processed for one MASTER Flat for i in range(len(sky_table)): mytpl = sky_table['tpls'][i] mymjd = sky_table['mjd'][i] iexpo = np.int(sky_table['iexpo'][i]) if tpl != "ALL" and tpl != mytpl: continue # Now starting with the standard recipe self._add_calib_to_sofdict("EXTINCT_TABLE", reset=True) self._add_calib_to_sofdict("SKY_LINES") self._add_skycalib_to_sofdict("STD_RESPONSE", mymjd, 'STD') self._add_skycalib_to_sofdict("STD_TELLURIC", mymjd, 'STD') self._add_tplmaster_to_sofdict(mymjd, 'LSF') self._sofdict['PIXTABLE_SKY'] = [ joinpath( self._get_fullpath_expo("SKY", "processed"), 'PIXTABLE_SKY_{0}_{1:04d}-{2:02d}.fits'.format( mytpl, iexpo, j + 1)) for j in range(24) ] self.write_sof(sof_filename=sof_filename + "{0:02d}".format(iexpo) + "_" + mytpl, new=True) dir_sky = self._get_fullpath_expo('SKY', "processed") name_sky = deepcopy(dic_files_products['SKY']) self.recipe_sky(self.current_sof, dir_sky, name_sky, mytpl, iexpo, fraction) # Write the MASTER files Table and save it self.save_expo_table("SKY", sky_table, "processed", aggregate=False, update=update) # Go back to original folder self.goto_prevfolder(logfile=True)
def init_raw_table(self, reset=False): """ Create a fits table with all the information from the Raw files Also create an astropy table with the same info """ if self.verbose: upipe.print_info("Creating the astropy fits raw data table") if reset or not hasattr(self, "Tables"): self._reset_tables() # Testing if raw table exists name_table = self._get_fitstablename_expo('RAWFILES', "raw") # ---- File exists - we READ it ------------------- # overwrite = True if os.path.isfile(name_table): if self._overwrite_astropy_table: upipe.print_warning("The raw-files table will be overwritten") else: upipe.print_warning("The raw files table already exists") upipe.print_warning( "If you wish to overwrite it, " " please turn on the 'overwrite_astropy_table' option to 'True'" ) upipe.print_warning( "In the meantime, the existing table will be read and used" ) self.Tables.Rawfiles = self.read_astropy_table( 'RAWFILES', "raw") overwrite = False # ---- File does not exist - we create it ---------- # if overwrite: # Check the raw folder self.goto_folder(self.paths.rawfiles) # Get the list of files from the Raw data folder files = os.listdir(".") smalldic = {"FILENAME": ['filename', '', str, '100A']} fulldic = listexpo_files.copy() fulldic.update(smalldic) # Init the lists MUSE_infodic = {} for key in fulldic.keys(): MUSE_infodic[key] = [] # Looping over the files for f in files: # Excluding the files without MUSE and fits.fz if ('MUSE' in f) and ('.fits.fz') in f: MUSE_infodic['FILENAME'].append(f) header = pyfits.getheader(f, 0) for k in listexpo_files.keys(): [namecol, keyword, func, form] = listexpo_files[k] MUSE_infodic[k].append(func(header[keyword])) # Transforming into numpy arrayimport pymusepipe for k in fulldic.keys(): MUSE_infodic[k] = np.array(MUSE_infodic[k]) # Getting a sorted array with indices idxsort = np.argsort(MUSE_infodic['FILENAME']) # Creating the astropy table self.Tables.Rawfiles = Table([MUSE_infodic['FILENAME'][idxsort]], names=['filename'], meta={'name': 'raw file table'}) # Creating the columns for k in fulldic.keys(): [namecol, keyword, func, form] = fulldic[k] self.Tables.Rawfiles[namecol] = MUSE_infodic[k][idxsort] # Writing up the table self.Tables.Rawfiles.write(name_table, format="fits", overwrite=self._overwrite_astropy_table) # Going back to the original folder self.goto_prevfolder() # Sorting the types ==================================== self.sort_raw_tables()
def update(self, **kwargs) : ## 1 required attribute if not hasattr(self, 'subtitle') : if 'subtitle' in kwargs : upipe.print_warning("Overiding subtitle") self.subtitle = kwargs.get('subtitle', "")
def run_scibasic(self, sof_filename='scibasic', expotype="OBJECT", tpl="ALL", illum=True): """Reducing the files of a certain category and creating the PIXTABLES Will run the esorex muse_scibasic Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table tpl_gtable = self.select_tpl_files(expotype=expotype, tpl=tpl, stage="raw") if len(tpl_gtable) == 0: if self.verbose: upipe.print_warning( "No {0} recovered from the astropy file Table - Aborting". format(expotype)) return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the LSF including # the list of files to be processed for one MASTER Flat for gtable in tpl_gtable.groups: self._add_calib_to_sofdict("BADPIX_TABLE", reset=True) self._add_calib_to_sofdict("LINE_CATALOG") # extract the tpl (string) and mean mjd (float) tpl, mean_mjd = self._get_tpl_meanmjd(gtable) self._add_geometry_to_sofdict(tpl) # Provide the list of files to the dictionary self._sofdict[expotype] = add_listpath(self.paths.rawfiles, list(gtable['filename'])) # Number of objects Nexpo = len(self._sofdict[expotype]) if self.verbose: upipe.print_info( "Number of expo is {Nexpo} for {expotype}".format( Nexpo=Nexpo, expotype=expotype)) # Finding the best tpl for BIAS if illum: self._add_tplraw_to_sofdict(mean_mjd, "ILLUM") self._add_list_tplmaster_to_sofdict( mean_mjd, ['BIAS', 'FLAT', 'TRACE', 'WAVE', 'TWILIGHT']) # Writing the sof file self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True) # Run the recipe to reduce the standard (muse_scibasic) dir_products = self._get_fullpath_expo(expotype, "processed") suffix = self._get_suffix_product(expotype) name_products = [] list_expo = np.arange(Nexpo).astype(np.int) + 1 for iexpo in list_expo: name_products += [ '{0:04d}-{1:02d}.fits'.format(iexpo, j + 1) for j in range(24) ] self.recipe_scibasic(self.current_sof, tpl, expotype, dir_products, name_products, suffix) # Write the Processed files Table and save it gtable['iexpo'] = list_expo self.save_expo_table(expotype, gtable, "processed", aggregate=False, update=True) # Go back to original folder self.goto_prevfolder(logfile=True)