def goto_prevfolder(self, logfile=False): """Go back to previous folder """ upipe.print_info("Going back to the original folder {0}".format( self.paths._prev_folder)) self.goto_folder(self.paths._prev_folder, logfile=logfile, verbose=False)
def init_default_param(self, dic_param): """Initialise the parameters as defined in the input dictionary Hardcoded in init_musepipe.py """ for key in dic_param.keys(): if self.verbose: upipe.print_info( "Default initialisation of attribute {0}".format(key)) setattr(self, key, dic_param[key])
def _set_cpu(self, first_cpu=0, ncpu=24, list_cpu=None): """Setting the cpu format for calling the esorex command """ if (list_cpu is None) | (len(list_cpu) < 1): self.list_cpu = "{0}-{1}".format(first_cpu, first_cpu + ncpu - 1) else: self.list_cpu = "{0}".format(list_cpu[0]) for i in range(1, len(list_cpu)): self.list_cpu += ":{0}".format(list_cpu[i]) if self.verbose: upipe.print_info("LIST_CPU: {0}".format(self.list_cpu))
def read_astropy_table(self, expotype=None, stage="master"): """Read an existing Masterfile data table to start the pipeline """ # Read the astropy table name_table = self._get_fitstablename_expo(expotype, stage) if not os.path.isfile(name_table): upipe.print_warning( "Astropy table {0} does not exist - setting up an empty one". format(name_table)) return Table([[], [], []], names=['tpls', 'mjd', 'tplnexp']) else: if self.verbose: upipe.print_info( "Reading Astropy fits Table {0}".format(name_table)) return Table.read(name_table, format="fits")
def goto_folder(self, newpath, logfile=False, verbose=True): """Changing directory and keeping memory of the old working one """ try: prev_folder = os.getcwd() newpath = os.path.normpath(newpath) os.chdir(newpath) if verbose: upipe.print_info("Going to folder {0}".format(newpath)) if logfile: upipe.append_file(joinpath(self.paths.data, self.logfile), "cd {0}\n".format(newpath)) self.paths._prev_folder = prev_folder except OSError: if not os.path.isdir(newpath): raise
def __init__(self, galaxyname=None, list_pointings=[1]) : if galaxyname not in MUSEPIPE_sample.keys() : upipe.print_error("ERROR: no Galaxy named {gal} in the defined sample".format(gal=galaxyname)) return # Galaxy name upipe.print_info("Initialising Target {name}".format(name=galaxyname)) self.targetname = galaxyname # Info of the pointings and extracting the observing run for each pointing self.info_pointings = MUSEPIPE_sample[galaxyname] if any([_ not in self.info_pointings.keys() for _ in list_pointings]) : upipe.print_error("ERROR: no pointing {0} for the Galaxy".format(list_pointings)) return self.list_pointings = list_pointings self.observing_run = [self.info_pointing[_] for _ in self.list_pointings]
def print_offset(self): """Print out the offset """ upipe.print_info("#---- Offset recorded so far ----#") upipe.print_info("Total in ARCSEC") for nima in range(self.nimages): upipe.print_info(" Image {0:02d}: " "{1:8.4f} {2:8.4f}".format( nima, self.total_off_arcsec[nima][0], self.total_off_arcsec[nima][1])) upipe.print_info("Total in PIXEL") for nima in range(self.nimages): upipe.print_info(" Image {0:02d}: " "{1:8.4f} {2:8.4f}".format( nima, self.total_off_pixel[nima][0], self.total_off_pixel[nima][1]))
def write_sof(self, sof_filename, new=False, verbose=None): """Feeding an sof file with input filenames from a dictionary """ # Removing the extension of the file if already set if not sof_filename.lower().endswith(".sof"): sof_filename = sof_filename + ".sof" sof = joinpath(self.paths.sof, sof_filename) # If new file, start from scratch (overwrite) if new: sof_file = open(sof, "w+") if verbose: upipe.print_info("Writing in file {0}".format(sof)) # if not new, then append else: sof_file = open(sof, "a") if verbose: upipe.print_info("Appending in file {0}".format(sof)) # Use dictionary to write up the lines for key in self._sofdict.keys(): for item in self._sofdict[key]: text_to_write = "{0} {1}\n".format(item, key) sof_file.write(text_to_write) if verbose: upipe.print_info(text_to_write) sof_file.close() # Returning the current sof as relative path self.current_sof = upipe.normpath(os.path.relpath(sof))
def select_tpl_files(self, expotype=None, tpl="ALL", stage="raw"): """Selecting a subset of files from a certain type """ if expotype not in musepipe.listexpo_types.keys(): upipe.print_info( "ERROR: input {0} is not in the list of possible values". format(expotype)) return MUSE_subtable = self._get_table_expo(expotype, stage) if len(MUSE_subtable) == 0: if self.verbose: upipe.print_warning( "Empty file table of type {0}".format(expotype)) upipe.print_warning( "Returning an empty Table from the tpl -astropy- selection" ) return MUSE_subtable group_table = MUSE_subtable.group_by('tpls') if tpl == "ALL": return group_table else: return group_table.groups[group_table.groups.keys['tpls'] == tpl]
def run_pipeline(self, list_pointings=[1], fakemode=False, suffix_logfile="logfile", suffix_rcfile="rcfile", suffix_calfile="calfile"): """Run the pipeline for all pointings in the list """ if any([_ not in self.info_pointings.keys() for _ in self.list_pointings]) : upipe.print_error("ERROR: some pointing are not in " "the available list ({0})".format(self.list_pointings)) return # Setting up the suffixes for the files self.suffix_logfile = suffix_logfile self.suffix_calfile = suffix_calfile self.suffix_rcfile = suffix_rcfile # Loop on the pointings self.pipelines = [] upipe.print_info("---- Starting the Data Reduction ----") self.history = [] for pointing in list_pointings: # Setting up the names of the output files logfile = self._get_logfile_name(self.suffix_logfile, pointing) calfile = self._get_calfile_name(self.suffix_calfile, pointing) rcfile = self._get_rcfile_name(self.suffix_rcfile, pointing) python_command = ("mypipe = musepipe.MusePipe(galaxyname={0}, " "pointing={1}, rc_filename={2}, cal_filename={3}, " "outlog=None, logfile={4}, fakemode={5}, " "nocache=False)".format(galaxyname, pointing, rcfile, calfile, logfile, fakemode)) upipe.print_info("====== START - POINTING {0:2d} ======".format(pointing)) upipe.print_info(python_command) upipe.print_info("====== END - POINTING {0:2d} ======".format(pointing)) self.history.append(python_command) mypipe = MusePipe(galaxyname=galaxyname, pointing=pointing, rc_filename=rcfile, cal_filename=calfile, outlog=None, logfile=logfile, fakemode=fakemode, nocache=False) self.pipelines.append(mypipe) mypipe.run_all_recipes()
def print_offset_fromfits(self, name_table=None): """Print out the offset """ exist_table, fits_table = self.open_offset_table(name_table) if exist_table is None: return if ('RA_OFFSET' not in fits_table.columns) or ('DEC_OFFSET' not in fits_table.columns): upipe.print_error( "Table does not contain 'RA/DEC_OFFSET' columns, Aborting") return upipe.print_info("Offset recorded in OFFSET_LIST Table") upipe.print_info("Total in ARCSEC") for i in self.nimages: upipe.print_info("Image {0}: " "{1:8.4f} {1:8.4f}".format( fits_table['RA_OFFSET'] * 3600, fits_table['DEC_OFFSET'] * 3600.))
def read_param_file(self, filename, dic_param): """Reading an input parameter initialisation file """ # Testing existence of filename if not os.path.isfile(filename): upipe.print_info( ("ERROR: input parameter {inputname} cannot be found. " "We will use the default hardcoded in the " "init_musepipe.py module").format(inputname=filename)) return # If it exists, open and read it f_param = open(filename) lines = f_param.readlines() # Dummy dictionary to see which items are not initialised dummy_dic_param = copy.copy(dic_param) for line in lines: if line[0] in ["#", "%"]: continue sline = line.split() if sline[0] in dic_param.keys(): if self.verbose: upipe.print_info("Initialisation of attribute {0}".format( sline[0])) setattr(self, sline[0], sline[1]) # Here we drop the item which was initialised val = dummy_dic_param.pop(sline[0]) else: continue # Set of non initialised folders not_initialised_param = dummy_dic_param.keys() # Listing them as warning and using the hardcoded default for key in not_initialised_param: upipe.print_info(("WARNING: parameter {param} not initialised " "We will use the default hardcoded value from " "init_musepipe.py").format(param=key)) setattr(self, key, dic_param[key])
def compare(self, muse_hdu=None, ref_hdu=None, factor=1.0, start_nfig=1, nlevels=7, levels=None, muse_smooth=1., ref_smooth=1., samecontour=True, nima=0, showcontours=True, showcuts=True, difference=True, normalise=True, median_filter=True, ncuts=5, showdiff=True, percentage=10.): """Plot the contours of the projected reference and MUSE image """ # Getting the data if muse_hdu is None: muse_hdu = self.list_offmuse_hdu[nima] if ref_hdu is None: ref_hdu = self.list_proj_refhdu[nima] # Filtering the data if median_filter is True if median_filter: musedata = self._filtermed_image(muse_hdu.data) refdata = self._filtermed_image(ref_hdu.data) else: musedata = muse_hdu.data refdata = ref_hdu.data # If normalising, using the median ratio fit if normalise: polypar = get_image_norm_poly(musedata, refdata, chunk_size=15) if self.verbose: upipe.print_info("Renormalising the MUSE data as NewMUSE = " "{1:8.4f} * ({0:8.4f} + MUSE)".format( polypar.beta[0], polypar.beta[1])) musedata = (polypar.beta[0] + musedata) * polypar.beta[1] self.ima_polynorm[nima] = polypar.beta # Getting the range of relevant fluxes lowlevel_muse, highlevel_muse = self._get_flux_range(musedata) lowlevel_ref, highlevel_ref = self._get_flux_range(refdata) if self.verbose: print("Low / High level MUSE flux: " "{0:8.4f} {1:8.4f}".format(lowlevel_muse, highlevel_muse)) print("Low / High level REF flux: " "{0:8.4f} {1:8.4f}".format(lowlevel_ref, highlevel_ref)) # Smoothing out the result in case it is needed if muse_smooth > 0: musedata = nd.filters.gaussian_filter(musedata, muse_smooth) if ref_smooth > 0: refdata = nd.filters.gaussian_filter(refdata, ref_smooth) # Get the WCS musewcs = wcs.WCS(muse_hdu.header) refwcs = wcs.WCS(ref_hdu.header) # Preparing the figure current_fig = start_nfig self.list_figures = [] # Starting the plotting if showcontours: fig, ax = open_new_wcs_figure(current_fig, musewcs) if levels is not None: mylevels = levels samecontour = True else: # First contours - MUSE levels_muse = np.linspace(np.log10(lowlevel_muse), np.log10(highlevel_muse), nlevels) levels_ref = np.linspace(np.log10(lowlevel_ref), np.log10(highlevel_ref), nlevels) mylevels = levels_muse cmuseset = ax.contour(np.log10(musedata), mylevels, colors='k', origin='lower', linestyles='solid') # Second contours - Ref if samecontour: crefset = ax.contour(np.log10(refdata), levels=cmuseset.levels, colors='r', origin='lower', alpha=0.5, linestyles='solid') else: crefset = ax.contour(np.log10(refdata), levels=levels_ref, colors='r', origin='lower', alpha=0.5, linestyles='solid') ax.set_aspect('equal') h1, _ = cmuseset.legend_elements() h2, _ = crefset.legend_elements() ax.legend([h1[0], h2[0]], ['MUSE', 'REF']) if nima is not None: plt.title("Image #{0:02d}".format(nima)) self.list_figures.append(current_fig) current_fig += 1 if showcuts: fig, ax = open_new_wcs_figure(current_fig) diffima = (refdata - musedata) * 200. / (lowlevel_muse + highlevel_muse) chunk_x = musedata.shape[0] // (ncuts + 1) chunk_y = musedata.shape[1] // (ncuts + 1) ax.plot(diffima[np.arange(ncuts) * chunk_x, :].T, 'k-') ax.plot(diffima[:, np.arange(ncuts) * chunk_x], 'r-') ax.set_ylim(-20, 20) ax.set_xlabel("[pixels]", fontsize=20) ax.set_ylabel("[%]", fontsize=20) self.list_figures.append(current_fig) current_fig += 1 if showdiff: fig, ax = open_new_wcs_figure(current_fig, musewcs) ratio = 100. * (refdata - musedata) / (musedata + 1.e-12) im = ax.imshow(ratio, vmin=-percentage, vmax=percentage) cbar = fig.colorbar(im) self.list_figures.append(current_fig) current_fig += 1
def wrapped(*myargs, **mykwargs): upipe.print_info(f.__name__) return f(*myargs, **mykwargs)
def run_scibasic(self, sof_filename='scibasic', expotype="OBJECT", tpl="ALL", illum=True): """Reducing the files of a certain category and creating the PIXTABLES Will run the esorex muse_scibasic Parameters ---------- sof_filename: string (without the file extension) Name of the SOF file which will contain the Bias frames tpl: ALL by default or a special tpl time """ # First selecting the files via the grouped table tpl_gtable = self.select_tpl_files(expotype=expotype, tpl=tpl, stage="raw") if len(tpl_gtable) == 0: if self.verbose: upipe.print_warning( "No {0} recovered from the astropy file Table - Aborting". format(expotype)) return # Go to the data folder self.goto_folder(self.paths.data, logfile=True) # Create the dictionary for the LSF including # the list of files to be processed for one MASTER Flat for gtable in tpl_gtable.groups: self._add_calib_to_sofdict("BADPIX_TABLE", reset=True) self._add_calib_to_sofdict("LINE_CATALOG") # extract the tpl (string) and mean mjd (float) tpl, mean_mjd = self._get_tpl_meanmjd(gtable) self._add_geometry_to_sofdict(tpl) # Provide the list of files to the dictionary self._sofdict[expotype] = add_listpath(self.paths.rawfiles, list(gtable['filename'])) # Number of objects Nexpo = len(self._sofdict[expotype]) if self.verbose: upipe.print_info( "Number of expo is {Nexpo} for {expotype}".format( Nexpo=Nexpo, expotype=expotype)) # Finding the best tpl for BIAS if illum: self._add_tplraw_to_sofdict(mean_mjd, "ILLUM") self._add_list_tplmaster_to_sofdict( mean_mjd, ['BIAS', 'FLAT', 'TRACE', 'WAVE', 'TWILIGHT']) # Writing the sof file self.write_sof(sof_filename=sof_filename + "_" + tpl, new=True) # Run the recipe to reduce the standard (muse_scibasic) dir_products = self._get_fullpath_expo(expotype, "processed") suffix = self._get_suffix_product(expotype) name_products = [] list_expo = np.arange(Nexpo).astype(np.int) + 1 for iexpo in list_expo: name_products += [ '{0:04d}-{1:02d}.fits'.format(iexpo, j + 1) for j in range(24) ] self.recipe_scibasic(self.current_sof, tpl, expotype, dir_products, name_products, suffix) # Write the Processed files Table and save it gtable['iexpo'] = list_expo self.save_expo_table(expotype, gtable, "processed", aggregate=False, update=True) # Go back to original folder self.goto_prevfolder(logfile=True)
def init_raw_table(self, reset=False): """ Create a fits table with all the information from the Raw files Also create an astropy table with the same info """ if self.verbose: upipe.print_info("Creating the astropy fits raw data table") if reset or not hasattr(self, "Tables"): self._reset_tables() # Testing if raw table exists name_table = self._get_fitstablename_expo('RAWFILES', "raw") # ---- File exists - we READ it ------------------- # overwrite = True if os.path.isfile(name_table): if self._overwrite_astropy_table: upipe.print_warning("The raw-files table will be overwritten") else: upipe.print_warning("The raw files table already exists") upipe.print_warning( "If you wish to overwrite it, " " please turn on the 'overwrite_astropy_table' option to 'True'" ) upipe.print_warning( "In the meantime, the existing table will be read and used" ) self.Tables.Rawfiles = self.read_astropy_table( 'RAWFILES', "raw") overwrite = False # ---- File does not exist - we create it ---------- # if overwrite: # Check the raw folder self.goto_folder(self.paths.rawfiles) # Get the list of files from the Raw data folder files = os.listdir(".") smalldic = {"FILENAME": ['filename', '', str, '100A']} fulldic = listexpo_files.copy() fulldic.update(smalldic) # Init the lists MUSE_infodic = {} for key in fulldic.keys(): MUSE_infodic[key] = [] # Looping over the files for f in files: # Excluding the files without MUSE and fits.fz if ('MUSE' in f) and ('.fits.fz') in f: MUSE_infodic['FILENAME'].append(f) header = pyfits.getheader(f, 0) for k in listexpo_files.keys(): [namecol, keyword, func, form] = listexpo_files[k] MUSE_infodic[k].append(func(header[keyword])) # Transforming into numpy arrayimport pymusepipe for k in fulldic.keys(): MUSE_infodic[k] = np.array(MUSE_infodic[k]) # Getting a sorted array with indices idxsort = np.argsort(MUSE_infodic['FILENAME']) # Creating the astropy table self.Tables.Rawfiles = Table([MUSE_infodic['FILENAME'][idxsort]], names=['filename'], meta={'name': 'raw file table'}) # Creating the columns for k in fulldic.keys(): [namecol, keyword, func, form] = fulldic[k] self.Tables.Rawfiles[namecol] = MUSE_infodic[k][idxsort] # Writing up the table self.Tables.Rawfiles.write(name_table, format="fits", overwrite=self._overwrite_astropy_table) # Going back to the original folder self.goto_prevfolder() # Sorting the types ==================================== self.sort_raw_tables()
def __init__(self, galaxyname=None, pointing=0, objectlist=[], rc_filename=None, cal_filename=None, outlog=None, logfile="MusePipe.log", reset_log=False, verbose=True, musemode="WFM-NOAO-N", checkmode=True, strong_checkmode=False, **kwargs): """Initialise the file parameters to be used during the run Input ----- galaxyname: string (e.g., 'NGC1208'). default is None. objectlist= list of objects (string=filenames) to process. Default is empty rc_filename: filename to initialise folders cal_filename: filename to initiale FIXED calibration MUSE files outlog: string, output directory for the log files verbose: boolean. Give more information as output (default is True) musemode: string (default is WFM_N) String to define the mode to be considered checkmode: boolean (default is True) Check the mode or not when reducing strong_checkmode: boolean (default is False) Enforce the checkmode for all if True, or exclude DARK/BIAS if False vsystemic: float (default is 0), indicating the systemic velocity of the galaxy [in km/s] Other possible entries ---------------------- overwrite_astropy_table: boolean (default is False). Overwrite the astropy table even when it exists. warnings: strong ('ignore'by default. If set to ignore, will ignore the Astropy Warnings. time_geo_table: boolean (default is True). Use the time dependent geo_table time_astro_table: boolean (default is True). Use the time dependent astro_table """ # Verbose option self.verbose = verbose # Warnings for astropy self.warnings = kwargs.pop("warnings", 'ignore') if self.warnings == 'ignore': warnings.simplefilter('ignore', category=AstropyWarning) # Overwriting option for the astropy table self._overwrite_astropy_table = kwargs.pop("overwrite_astropy_table", False) # Updating the astropy table self._update_astropy_table = kwargs.pop("update_astropy_table", False) # Use time dependent geo_table self._time_geo_table = kwargs.pop("time_geo_table", True) self._time_astro_table = kwargs.pop("time_astro_table", True) # super(MusePipe, self).__init__(**kwargs) # Setting the default attibutes ##################### self.galaxyname = galaxyname self.pointing = pointing self.vsystemic = np.float(kwargs.pop("vsystemic", 0.)) # Setting other default attributes if outlog is None: outlog = "log_{timestamp}".format( timestamp=upipe.create_time_name()) upipe.print_info("The Log folder will be {log}".format(outlog)) self.outlog = outlog self.logfile = joinpath(self.outlog, logfile) # Further reduction options ===================================== # Mode of the observations self.musemode = musemode # Checking if mode is correct self.checkmode = checkmode # Checking if mode is correct also for BIAS & DARK self.strong_checkmode = strong_checkmode # Set of objects to reduced self.objectlist = objectlist # End of parameter settings ######################### # Init of the subclasses PipePrep.__init__(self) PipeRecipes.__init__(self, **kwargs) # =========================================================== # # Setting up the folders and names for the data reduction # Can be initialised by either an rc_file, # or a default rc_file or harcoded defaults. self.my_params = InitMuseParameters(rc_filename=rc_filename, cal_filename=cal_filename) # Setting up the relative path for the data, using Galaxy Name + Pointing self.my_params.data = "{0}/P{1:02d}/".format(self.galaxyname, self.pointing) # Create full path folder self.set_fullpath_names() # Go to the data directory # and Recording the folder where we start self.paths.orig = os.getcwd() # Making the output folders in a safe mode if self.verbose: upipe.print_info("Creating directory structure") self.goto_folder(self.paths.data) # ============================================== # Creating the extra pipeline folder structure for folder in self.my_params._dic_input_folders.keys(): upipe.safely_create_folder( self.my_params._dic_input_folders[folder], verbose=verbose) # ============================================== # Creating the folder structure itself if needed for folder in self.my_params._dic_folders.keys(): upipe.safely_create_folder(self.my_params._dic_folders[folder], verbose=verbose) # ============================================== # Init the Master exposure flag dictionary self.Master = {} for mastertype in dic_listMaster.keys(): upipe.safely_create_folder(self._get_path_expo( mastertype, "master"), verbose=self.verbose) self.Master[mastertype] = False # Init the Object folder for objecttype in dic_listObject.keys(): upipe.safely_create_folder(self._get_path_expo( objecttype, "processed"), verbose=self.verbose) self._dic_listMasterObject = {**dic_listMaster, **dic_listObject} # ============================================== # Going back to initial working directory self.goto_prevfolder() # =========================================================== # Now creating the raw table, and attribute containing the # astropy dataset probing the rawfiles folder # When creating the table, if the table already exists # it will read the old one, except if an overwrite_astropy_table # is set to True. self.init_raw_table() self.read_all_astro_tables()