def input_catl_dir_path(self, catl_kind='master', check_exist=True, create_dir=False): """ Path to the file being analyzed. Parameters: ------------ catl_kind : {'master', 'random', 'redmapper'}, `str` Option for which kind of catalogue is being analyzed. Options: - `master` : Catalogue of objects. - `random` : Catalogue of random catalogues - `redmapper` : Catalogue of RedMapper Clusters check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns ------------ input_catl_dir : `str` Path to the file being analyzed. """ # Check input parameters # `check_exist` if not (isinstance(check_exist, bool)): msg = '`check_exist` ({0}) must be of `boolean` type!'.format( type(check_exist)) raise TypeError(msg) # # `create_dir` if not (isinstance(create_dir, bool)): msg = '`create_dir` ({0}) must be of `boolean` type!'.format( type(create_dir)) raise TypeError(msg) # # `catl_kind` if not (catl_kind in ['master', 'redmapper', 'random']): msg = '`catl_kind` ({1}) is not a valid input variable!' msg = msg.format(catl_kind) raise ValueError(msg) # Path to directory input_catl_dir = os.path.join(self.proj_dict['ext_dir'], catl_kind) # Creating directory if create_dir: cfutils.Path_Folder(input_catl_dir) # Check for its existence if check_exist: if not (os.path.exists(input_catl_dir)): msg = '`input_catl_dir` ({0}) was not found!'.format( input_catl_dir) raise FileNotFoundError(msg) return input_catl_dir
def catl_filtered_dir(self, catl_kind='master', check_exist=True, create_dir=False): """ Path to the `filtered` file for `master`, after having applied all of the magnitude and redshift cuts. Parameters: ------------ catl_kind : {'master'}, `str` Option for which kind of catalogue is being analyzed. Options: - `master` : Catalogue of objects. check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns ------------ filtered_dir path : `str` Path to the `filtered` version of `master` file. """ # Check input parameters # `check_exist` if not (isinstance(check_exist, bool)): msg = '`check_exist` ({0}) must be of `boolean` type!'.format( type(check_exist)) raise TypeError(msg) # # `create_dir` if not (isinstance(create_dir, bool)): msg = '`create_dir` ({0}) must be of `boolean` type!'.format( type(create_dir)) raise TypeError(msg) # # `catl_kind` if not (catl_kind in ['master', 'redmapper', 'random']): msg = '`catl_kind` ({1}) is not a valid input variable!' msg = msg.format(catl_kind) raise ValueError(msg) # Path to directory filtered_dir = os.path.join(self.proj_dict['int_dir'], 'filtered', catl_kind) # Creating directory if create_dir: cfutils.Path_Folder(filtered_dir) # Check for its existence if check_exist: if not (os.path.exists(filtered_dir)): msg = '`filtered_dir` ({0}) was not found!'.format( filtered_dir) raise FileNotFoundError(msg) return filtered_dir
def cosmo_outdir(self, check_exist=False, create_dir=False): """ Output directory for cosmology-related files. Parameters ----------- check_exist : `bool`, optional If `True`, it checks for whether or not the directory exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- cosmo_dir : `str` Path to the directory that contains cosmologica-related files. """ # Figure directory - Output cosmo_dir = os.path.join(self.proj_dict['raw_dir'], self.cosmo_choice, 'cosmo_dir') # Creating directory if create_dir: cfutils.Path_Folder(cosmo_dir) # Check for its existence if check_exist: if not (os.path.exists(cosmo_dir)): msg = '`cosmo_dir` ({0}) was not found!'.format(cosmo_dir) raise FileNotFoundError(msg) return cosmo_dir
def tar_outdir(self, check_exist=False, create_dir=False): """ Output directory for the TAR file produced in this project. Parameters ----------- check_exist : `bool`, optional If `True`, it checks for whether or not the directory exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- tar_output_dir : `str` Path to the TAR output directory, which includes the resuls for each of the catalogues. """ # Figure directory - Output tar_output_dir = os.path.join(self.proj_dict['proc_dir'], 'TAR_files', self.cosmo_choice, self.halotype, self.survey) # Creating directory if create_dir: cfutils.Path_Folder(tar_output_dir) # Check for its existence if check_exist: if not (os.path.exists(tar_output_dir)): msg = '`tar_output_dir` ({0}) was not found!'.format( tar_output_dir) raise FileNotFoundError(msg) return tar_output_dir
def mass_func_output_dir(self, check_exist=False, create_dir=False): """ Output directory for the files with the info on the mass function used in this project. Parameters ----------- check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- mf_output_dir : `str` Path to the output directory for the files related to the mass function used in this project. """ # Mass Function - Output directory mf_output_dir = os.path.join(self.proj_dict['int_dir'], 'MF', self.cosmo_choice, self.survey) # Creating directory if create_dir: cfutils.Path_Folder(mf_output_dir) # Check for its existence if check_exist: if not (os.path.exists(mf_output_dir)): msg = '`mf_output_dir` ({0}) was not found!'.format( mf_output_dir) raise FileNotFoundError(msg) return mf_output_dir
def halobias_outdir(self, check_exist=False, create_dir=False): """ Output directory for the halobias files with the input galaxy information. Parameters ----------- check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- hb_files_out : `str` Path to the output directory for the halobias files with the input galaxy information. """ # Halobias file output directory hb_files_out = os.path.join(self.proj_dict['raw_dir'], 'hb_files', self.survey) # Creating directory if create_dir: cfutils.Path_Folder(hb_files_out) # Check for its existence if check_exist: if not (os.path.exists(hb_files_out)): msg = '`hb_files_out` ({0}) was not found!'.format( hb_files_out) raise FileNotFoundError(msg) return hb_files_out
def cookiecutter_paths(path='./'): """ Paths to main folders in the `Data Science` cookiecutter template. This structure was taken from : - https://drivendata.github.io/cookiecutter-data-science/ Parameters ---------- path : str, optional Path to the file within the `.git` repository Return ---------- param_dict : python dictionary Dictionary with info of the project that uses the Data Science cookiecutter template. Raises ---------- LSSUtils_Error : exception If `path` is not within a .git directory, it raises an error. """ # Base Path base_dir = git_root_dir(path) + '/' # Checking that directory exists if os.path.exists(base_dir): # Plot Directory plot_dir = os.path.join(base_dir, 'reports', 'figures/') # Source directory src_dir = os.path.join(base_dir, 'src', 'data/') # Data path data_dir = os.path.join(base_dir, 'data/') # External path ext_dir = os.path.join(data_dir, 'external/') # Processed path proc_dir = os.path.join(data_dir, 'processed/') # External path int_dir = os.path.join(data_dir, 'interim/') # External path raw_dir = os.path.join(data_dir, 'raw/') # Creating files for dir_ii in [plot_dir, src_dir, data_dir]: fd.Path_Folder(dir_ii) # Saving to dictionary param_dict = {} param_dict['base_dir'] = base_dir param_dict['plot_dir'] = plot_dir param_dict['src_dir'] = src_dir param_dict['data_dir'] = data_dir param_dict['ext_dir'] = ext_dir param_dict['proc_dir'] = proc_dir param_dict['int_dir'] = int_dir param_dict['raw_dir'] = raw_dir else: msg = '{0} `base_dir` ({1}) is not a Git directory! Exiting'.format( fd.Program_Msg(__file__), base_dir) raise LSSUtils_Error(msg) return param_dict
def directory_skeleton(param_dict, proj_dict): """ Creates the directory skeleton for the current project Parameters ---------- param_dict: python dictionary dictionary with `project` variables proj_dict: python dictionary dictionary with info of the project that uses the `Data Science` Cookiecutter template. Returns --------- proj_dict: python dictionary Dictionary with current and new paths to project directories """ file_msg = param_dict['Prog_msg'] ## Path to the simulation cubes to analyze sim_dir = os.path.join('/fs1/caldervf/CODES/vandy_group_statistics2', 'RESOLVE_Mocks_Cube_Color_Age', 'With_Weights', 'New_Mocks') # Checking that folder exists if not (os.path.exists(sim_dir)): msg = '{0} `sim_dir` ({1}) does not exist!'.format(file_msg, sim_dir) raise FileNotFoundError(msg) # Finding combination of the directory simfile = os.path.join( sim_dir, 'Mbaryon_9.0_Vpeak_0.2_u-r_Acc_Rate_100Myr_0.6_130.0.hdf5') # Check that file exists if not (os.path.exists(simfile)): msg = '{0} `simfile` ({1}) does not exist!'.format(file_msg, simfile) raise FileNotFoundError(msg) # # Output directory outdir = os.path.join(proj_dict['int_dir'], 'florez_age_matching_results') cfutils.Path_Folder(outdir) # # Saving to dictionary proj_dict['sim_dir'] = sim_dir proj_dict['simfile'] = simfile proj_dict['outdir'] = outdir return proj_dict
def analysis_out_dir(self, check_exist=True, create_dir=False): """ Path to the output directory for final analysis. Parameters ----------- check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- outdir_path : `str` Path to the output directory for the final analysis. """ # Check input parameters # `check_exist` if not (isinstance(check_exist, bool)): msg = '`check_exist` ({0}) must be of `boolean` type!'.format( type(check_exist)) raise TypeError(msg) # # `create_dir` if not (isinstance(create_dir, bool)): msg = '`create_dir` ({0}) must be of `boolean` type!'.format( type(create_dir)) raise TypeError(msg) # Path to directory outdir_path = os.path.join(self.proj_dict['proc_dir'], 'analysis') # Creating directory if create_dir: cfutils.Path_Folder(outdir_path) # Check for its existence if check_exist: if not (os.path.exists(outdir_path)): msg = '`outdir_path` ({0}) was not found!'.format(outdir_path) raise FileNotFoundError(msg) return outdir_path
def fig_outdir(self, hb_name, check_exist=False, create_dir=False): """ Output directory for the figures produced in this project. Parameters ----------- hb_name : `str` Name prefix of the file being analyzed. This is the name of the file that has the information of the galaxies. check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- fig_output_dir : `str` Path to the output directory for the files related to the mass function used in this project. """ # Figure directory - Output fig_output_dir = os.path.join(self.proj_dict['plot_dir'], self.cosmo_choice, self.halotype, self.survey, hb_name) # Creating directory if create_dir: cfutils.Path_Folder(fig_output_dir) # Check for its existence if check_exist: if not (os.path.exists(fig_output_dir)): msg = '`fig_output_dir` ({0}) was not found!'.format( fig_output_dir) raise FileNotFoundError(msg) return fig_output_dir
def url_files_download(url, ext, outdir, check_exist=False, create_dir=False, remove_files=False, bar_opt='tqdm'): """ Downloads the files from a URL to a local directory. The files that match a specific file extension, `ext`. Parameters ----------- url : `str` String of the URL ext : `str` File extension of the files in the URL. outdir : `str` Path to the output directory. This is the directory, to which the files with extensions `ext` will be saved. check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. remove_files : `bool`, optional If `True`, local files that are present that match the files at the URL will be replaced by the new versions. This variable is set to ``False`` by default. bar_opt : {'tqdm', 'native'} Option for which type of progress bar to use when downloading files. This variable is set to `tqdm` by default. Options: - 'tqdm' : Uses a tqdm-based progress bar - 'native': Used the wget-based native progress bar. """ file_msg = fd.Program_Msg(__file__) ## Checking for file type # 'URL' if not isinstance(url, str): msg = '{0} `url` ({1}) is not a valid type. It must be a STRING!' msg = msg.format(file_msg, type(url)) raise TypeError(msg) # File extension if not isinstance(ext, str): msg = '{0} `ext` ({1}) is not a valid type. It must be a STRING!' msg = msg.format(file_msg, type(ext)) raise TypeError(msg) # Output directory if not isinstance(outdir, str): msg = '{0} `outdir` ({1}) is not a valid type. It must be a STRING!' msg = msg.format(file_msg, type(outdir)) raise TypeError(msg) # `check_exist` if not (isinstance(check_exist, bool)): msg = '`check_exist` ({0}) must be of `boolean` type!'.format( type(check_exist)) raise TypeError(msg) # `create_dir` if not (isinstance(create_dir, bool)): msg = '`create_dir` ({0}) must be of `boolean` type!'.format( type(create_dir)) raise TypeError(msg) # `bar` - Type if not (isinstance(bar_opt, str)): msg = '`bar_opt` ({0}) must be of `boolean` type!'.format( type(bar_opt)) raise TypeError(msg) # Progress bar - Value if not (bar_opt in ['tqdm', 'native']): msg = '{0} `bar_opt` ({1}) is not a valid option! Exiting' msg = msg.format(file_msg, bar_opt) raise LSSUtils_Error(msg) ## ## List of files in the URL files_arr = url_file_list(url, ext) # Creating directory if create_dir: cfutils.Path_Folder(outdir) # Check for its existence if check_exist: if not (os.path.exists(outdir)): msg = '`outdir` ({0}) was not found!'.format(outdir) raise FileNotFoundError(msg) ## ## Downloading files to output directory if len(files_arr) > 0: if (bar_opt == 'tqdm'): tqdm_desc = 'Downloading files: ' for file_ii in tqdm(files_arr, desc=tqdm_desc): # Local file file_ii_local = os.path.join(outdir, os.path.basename(file_ii)) # Checking if local file exists if os.path.exists(file_ii_local): if remove_files: os.remove(file_ii_local) wget_opt = True else: wget_opt = False else: wget_opt = True ## ## Only downloading if necessary if wget_opt: wget.download(file_ii, out=outdir, bar=None) elif (bar_opt == 'native'): for file_ii in files_arr: # Local file file_ii_local = os.path.join(outdir, os.path.basename(file_ii)) # Checking if local file exists if os.path.exists(file_ii_local): if remove_files: os.remove(file_ii_local) wget_opt = True else: wget_opt = False else: wget_opt = True ## ## Only downloading if necessary if wget_opt: wget.download(file_ii, out=outdir) else: msg = '{0} Number of files is ZERO!'.format(file_msg) print(msg)
def catl_output_dir(self, hb_name, catl_kind='memb', perf=False, check_exist=False, create_dir=False): """ Output directory for the catalogues Parameters ------------ hb_name : `str` Name prefix of the file being analyzed. This is the name of the file that has the information of the galaxies. catl_kind : {'gal', 'memb', 'group'}, `str` Option for which kind of catalogue is being analyzed. This variable is set to ``memb`` by default. Options: - `gal` : Galaxy catalogue - `memb` : Group Member galaxy catalogue - `group` : Group galaxy catalogue perf : `bool`, optional If True, it returns (creates) the directory for the ``perfect`` catalogue. This variable is set to `False` by default. check_exist : `bool`, optional If `True`, it checks for whether or not the file exists. This variable is set to `False` by default. create_dir : `bool`, optional If `True`, it creates the directory if it does not exist. This variable is set to `False` by default. Returns --------- catl_output_dir : `str` Output directory for all the catalogues """ # Halobias file name - Type if not (isinstance(hb_name, str)): msg = '>>> `hb_name` ({1}) is not a valid type!' msg = msg.format(type(hb_name)) raise TypeError(msg) # Catalogue kind - Type if not (isinstance(catl_kind, str)): msg = '>>> `catl_kind` ({1}) is not a valid type!' msg = msg.format(type(catl_kind)) raise TypeError(msg) # Catalogue kind - Value calt_kind_arr = ['gal', 'memb', 'group'] if not (catl_kind in calt_kind_arr): msg = '>>> `catl_kind` ({1}) is not a valid input! Exiting!' msg = msg.format(catl_kind) raise TypeError(msg) # Perfect option - Type if not (isinstance(perf, bool)): msg = '>>> `perf` ({1}) is not a valid type!' msg = msg.format(type(perf)) raise TypeError(msg) # Check exists - Type if not (isinstance(check_exist, bool)): msg = '>>> `check_exist` ({1}) is not a valid type!' msg = msg.format(type(check_exist)) raise TypeError(msg) ## ## Output directory catl_outdir = os.path.join(self.proj_dict['proc_dir'], self.cosmo_choice, self.survey, hb_name) ## ## Type of Output directory catl_out_dict = { 'gal': 'galaxy_catalogues', 'memb': 'member_galaxy_catalogues', 'group': 'group_galaxy_catalogues' } if perf: catl_output_dir = os.path.join( catl_outdir, 'perf_{0}'.format(catl_out_dict[catl_kind])) else: catl_output_dir = os.path.join(catl_outdir, catl_out_dict[catl_kind]) # Creating directory if create_dir: cfutils.Path_Folder(catl_output_dir) # Check for its existence if check_exist: if not (os.path.exists(catl_output_dir)): msg = '`catl_output_dir` ({0}) was not found!'.format( catl_output_dir) raise FileNotFoundError(msg) return catl_output_dir