def input_catl_dir_path(self,
catl_kind='master',
check_exist=True,
create_dir=False):
"""
Path to the file being analyzed.
Parameters:
------------
catl_kind : {'master', 'random', 'redmapper'}, `str`
Option for which kind of catalogue is being analyzed.
Options:
- `master` : Catalogue of objects.
- `random` : Catalogue of random catalogues
- `redmapper` : Catalogue of RedMapper Clusters
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
------------
input_catl_dir : `str`
Path to the file being analyzed.
"""
# Check input parameters
# `check_exist`
if not (isinstance(check_exist, bool)):
msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
type(check_exist))
raise TypeError(msg)
#
# `create_dir`
if not (isinstance(create_dir, bool)):
msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
type(create_dir))
raise TypeError(msg)
#
# `catl_kind`
if not (catl_kind in ['master', 'redmapper', 'random']):
msg = '`catl_kind` ({1}) is not a valid input variable!'
msg = msg.format(catl_kind)
raise ValueError(msg)
# Path to directory
input_catl_dir = os.path.join(self.proj_dict['ext_dir'], catl_kind)
# Creating directory
if create_dir:
cfutils.Path_Folder(input_catl_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(input_catl_dir)):
msg = '`input_catl_dir` ({0}) was not found!'.format(
input_catl_dir)
raise FileNotFoundError(msg)
return input_catl_dir
def catl_filtered_dir(self,
catl_kind='master',
check_exist=True,
create_dir=False):
"""
Path to the `filtered` file for `master`, after having applied all
of the magnitude and redshift cuts.
Parameters:
------------
catl_kind : {'master'}, `str`
Option for which kind of catalogue is being analyzed.
Options:
- `master` : Catalogue of objects.
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
------------
filtered_dir path : `str`
Path to the `filtered` version of `master` file.
"""
# Check input parameters
# `check_exist`
if not (isinstance(check_exist, bool)):
msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
type(check_exist))
raise TypeError(msg)
#
# `create_dir`
if not (isinstance(create_dir, bool)):
msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
type(create_dir))
raise TypeError(msg)
#
# `catl_kind`
if not (catl_kind in ['master', 'redmapper', 'random']):
msg = '`catl_kind` ({1}) is not a valid input variable!'
msg = msg.format(catl_kind)
raise ValueError(msg)
# Path to directory
filtered_dir = os.path.join(self.proj_dict['int_dir'], 'filtered',
catl_kind)
# Creating directory
if create_dir:
cfutils.Path_Folder(filtered_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(filtered_dir)):
msg = '`filtered_dir` ({0}) was not found!'.format(
filtered_dir)
raise FileNotFoundError(msg)
return filtered_dir
def cosmo_outdir(self, check_exist=False, create_dir=False):
"""
Output directory for cosmology-related files.
Parameters
-----------
check_exist : `bool`, optional
If `True`, it checks for whether or not the directory exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
cosmo_dir : `str`
Path to the directory that contains cosmologica-related
files.
"""
# Figure directory - Output
cosmo_dir = os.path.join(self.proj_dict['raw_dir'], self.cosmo_choice,
'cosmo_dir')
# Creating directory
if create_dir:
cfutils.Path_Folder(cosmo_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(cosmo_dir)):
msg = '`cosmo_dir` ({0}) was not found!'.format(cosmo_dir)
raise FileNotFoundError(msg)
return cosmo_dir
def tar_outdir(self, check_exist=False, create_dir=False):
"""
Output directory for the TAR file produced in this project.
Parameters
-----------
check_exist : `bool`, optional
If `True`, it checks for whether or not the directory exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
tar_output_dir : `str`
Path to the TAR output directory, which includes the resuls for
each of the catalogues.
"""
# Figure directory - Output
tar_output_dir = os.path.join(self.proj_dict['proc_dir'], 'TAR_files',
self.cosmo_choice, self.halotype,
self.survey)
# Creating directory
if create_dir:
cfutils.Path_Folder(tar_output_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(tar_output_dir)):
msg = '`tar_output_dir` ({0}) was not found!'.format(
tar_output_dir)
raise FileNotFoundError(msg)
return tar_output_dir
def mass_func_output_dir(self, check_exist=False, create_dir=False):
"""
Output directory for the files with the info on the mass function
used in this project.
Parameters
-----------
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
mf_output_dir : `str`
Path to the output directory for the files related to the
mass function used in this project.
"""
# Mass Function - Output directory
mf_output_dir = os.path.join(self.proj_dict['int_dir'], 'MF',
self.cosmo_choice, self.survey)
# Creating directory
if create_dir:
cfutils.Path_Folder(mf_output_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(mf_output_dir)):
msg = '`mf_output_dir` ({0}) was not found!'.format(
mf_output_dir)
raise FileNotFoundError(msg)
return mf_output_dir
def halobias_outdir(self, check_exist=False, create_dir=False):
"""
Output directory for the halobias files with the
input galaxy information.
Parameters
-----------
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
hb_files_out : `str`
Path to the output directory for the halobias files with the
input galaxy information.
"""
# Halobias file output directory
hb_files_out = os.path.join(self.proj_dict['raw_dir'], 'hb_files',
self.survey)
# Creating directory
if create_dir:
cfutils.Path_Folder(hb_files_out)
# Check for its existence
if check_exist:
if not (os.path.exists(hb_files_out)):
msg = '`hb_files_out` ({0}) was not found!'.format(
hb_files_out)
raise FileNotFoundError(msg)
return hb_files_out
def cookiecutter_paths(path='./'):
"""
Paths to main folders in the `Data Science` cookiecutter template.
This structure was taken from :
- https://drivendata.github.io/cookiecutter-data-science/
Parameters
----------
path : str, optional
Path to the file within the `.git` repository
Return
----------
param_dict : python dictionary
Dictionary with info of the project that uses the Data Science
cookiecutter template.
Raises
----------
LSSUtils_Error : exception
If `path` is not within a .git directory, it raises an error.
"""
# Base Path
base_dir = git_root_dir(path) + '/'
# Checking that directory exists
if os.path.exists(base_dir):
# Plot Directory
plot_dir = os.path.join(base_dir, 'reports', 'figures/')
# Source directory
src_dir = os.path.join(base_dir, 'src', 'data/')
# Data path
data_dir = os.path.join(base_dir, 'data/')
# External path
ext_dir = os.path.join(data_dir, 'external/')
# Processed path
proc_dir = os.path.join(data_dir, 'processed/')
# External path
int_dir = os.path.join(data_dir, 'interim/')
# External path
raw_dir = os.path.join(data_dir, 'raw/')
# Creating files
for dir_ii in [plot_dir, src_dir, data_dir]:
fd.Path_Folder(dir_ii)
# Saving to dictionary
param_dict = {}
param_dict['base_dir'] = base_dir
param_dict['plot_dir'] = plot_dir
param_dict['src_dir'] = src_dir
param_dict['data_dir'] = data_dir
param_dict['ext_dir'] = ext_dir
param_dict['proc_dir'] = proc_dir
param_dict['int_dir'] = int_dir
param_dict['raw_dir'] = raw_dir
else:
msg = '{0} `base_dir` ({1}) is not a Git directory! Exiting'.format(
fd.Program_Msg(__file__), base_dir)
raise LSSUtils_Error(msg)
return param_dict
def directory_skeleton(param_dict, proj_dict):
"""
Creates the directory skeleton for the current project
Parameters
----------
param_dict: python dictionary
dictionary with `project` variables
proj_dict: python dictionary
dictionary with info of the project that uses the
`Data Science` Cookiecutter template.
Returns
---------
proj_dict: python dictionary
Dictionary with current and new paths to project directories
"""
file_msg = param_dict['Prog_msg']
## Path to the simulation cubes to analyze
sim_dir = os.path.join('/fs1/caldervf/CODES/vandy_group_statistics2',
'RESOLVE_Mocks_Cube_Color_Age', 'With_Weights',
'New_Mocks')
# Checking that folder exists
if not (os.path.exists(sim_dir)):
msg = '{0} `sim_dir` ({1}) does not exist!'.format(file_msg, sim_dir)
raise FileNotFoundError(msg)
# Finding combination of the directory
simfile = os.path.join(
sim_dir, 'Mbaryon_9.0_Vpeak_0.2_u-r_Acc_Rate_100Myr_0.6_130.0.hdf5')
# Check that file exists
if not (os.path.exists(simfile)):
msg = '{0} `simfile` ({1}) does not exist!'.format(file_msg, simfile)
raise FileNotFoundError(msg)
#
# Output directory
outdir = os.path.join(proj_dict['int_dir'], 'florez_age_matching_results')
cfutils.Path_Folder(outdir)
#
# Saving to dictionary
proj_dict['sim_dir'] = sim_dir
proj_dict['simfile'] = simfile
proj_dict['outdir'] = outdir
return proj_dict
def analysis_out_dir(self, check_exist=True, create_dir=False):
"""
Path to the output directory for final analysis.
Parameters
-----------
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
outdir_path : `str`
Path to the output directory for the final analysis.
"""
# Check input parameters
# `check_exist`
if not (isinstance(check_exist, bool)):
msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
type(check_exist))
raise TypeError(msg)
#
# `create_dir`
if not (isinstance(create_dir, bool)):
msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
type(create_dir))
raise TypeError(msg)
# Path to directory
outdir_path = os.path.join(self.proj_dict['proc_dir'], 'analysis')
# Creating directory
if create_dir:
cfutils.Path_Folder(outdir_path)
# Check for its existence
if check_exist:
if not (os.path.exists(outdir_path)):
msg = '`outdir_path` ({0}) was not found!'.format(outdir_path)
raise FileNotFoundError(msg)
return outdir_path
def fig_outdir(self, hb_name, check_exist=False, create_dir=False):
"""
Output directory for the figures produced in this project.
Parameters
-----------
hb_name : `str`
Name prefix of the file being analyzed. This is the name of the
file that has the information of the galaxies.
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
fig_output_dir : `str`
Path to the output directory for the files related to the
mass function used in this project.
"""
# Figure directory - Output
fig_output_dir = os.path.join(self.proj_dict['plot_dir'],
self.cosmo_choice, self.halotype,
self.survey, hb_name)
# Creating directory
if create_dir:
cfutils.Path_Folder(fig_output_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(fig_output_dir)):
msg = '`fig_output_dir` ({0}) was not found!'.format(
fig_output_dir)
raise FileNotFoundError(msg)
return fig_output_dir
def url_files_download(url,
ext,
outdir,
check_exist=False,
create_dir=False,
remove_files=False,
bar_opt='tqdm'):
"""
Downloads the files from a URL to a local directory. The files that
match a specific file extension, `ext`.
Parameters
-----------
url : `str`
String of the URL
ext : `str`
File extension of the files in the URL.
outdir : `str`
Path to the output directory. This is the directory, to which
the files with extensions `ext` will be saved.
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
remove_files : `bool`, optional
If `True`, local files that are present that match the files at
the URL will be replaced by the new versions. This variable is
set to ``False`` by default.
bar_opt : {'tqdm', 'native'}
Option for which type of progress bar to use when downloading files.
This variable is set to `tqdm` by default.
Options:
- 'tqdm' : Uses a tqdm-based progress bar
- 'native': Used the wget-based native progress bar.
"""
file_msg = fd.Program_Msg(__file__)
## Checking for file type
# 'URL'
if not isinstance(url, str):
msg = '{0} `url` ({1}) is not a valid type. It must be a STRING!'
msg = msg.format(file_msg, type(url))
raise TypeError(msg)
# File extension
if not isinstance(ext, str):
msg = '{0} `ext` ({1}) is not a valid type. It must be a STRING!'
msg = msg.format(file_msg, type(ext))
raise TypeError(msg)
# Output directory
if not isinstance(outdir, str):
msg = '{0} `outdir` ({1}) is not a valid type. It must be a STRING!'
msg = msg.format(file_msg, type(outdir))
raise TypeError(msg)
# `check_exist`
if not (isinstance(check_exist, bool)):
msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
type(check_exist))
raise TypeError(msg)
# `create_dir`
if not (isinstance(create_dir, bool)):
msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
type(create_dir))
raise TypeError(msg)
# `bar` - Type
if not (isinstance(bar_opt, str)):
msg = '`bar_opt` ({0}) must be of `boolean` type!'.format(
type(bar_opt))
raise TypeError(msg)
# Progress bar - Value
if not (bar_opt in ['tqdm', 'native']):
msg = '{0} `bar_opt` ({1}) is not a valid option! Exiting'
msg = msg.format(file_msg, bar_opt)
raise LSSUtils_Error(msg)
##
## List of files in the URL
files_arr = url_file_list(url, ext)
# Creating directory
if create_dir:
cfutils.Path_Folder(outdir)
# Check for its existence
if check_exist:
if not (os.path.exists(outdir)):
msg = '`outdir` ({0}) was not found!'.format(outdir)
raise FileNotFoundError(msg)
##
## Downloading files to output directory
if len(files_arr) > 0:
if (bar_opt == 'tqdm'):
tqdm_desc = 'Downloading files: '
for file_ii in tqdm(files_arr, desc=tqdm_desc):
# Local file
file_ii_local = os.path.join(outdir, os.path.basename(file_ii))
# Checking if local file exists
if os.path.exists(file_ii_local):
if remove_files:
os.remove(file_ii_local)
wget_opt = True
else:
wget_opt = False
else:
wget_opt = True
##
## Only downloading if necessary
if wget_opt:
wget.download(file_ii, out=outdir, bar=None)
elif (bar_opt == 'native'):
for file_ii in files_arr:
# Local file
file_ii_local = os.path.join(outdir, os.path.basename(file_ii))
# Checking if local file exists
if os.path.exists(file_ii_local):
if remove_files:
os.remove(file_ii_local)
wget_opt = True
else:
wget_opt = False
else:
wget_opt = True
##
## Only downloading if necessary
if wget_opt:
wget.download(file_ii, out=outdir)
else:
msg = '{0} Number of files is ZERO!'.format(file_msg)
print(msg)
def catl_output_dir(self,
hb_name,
catl_kind='memb',
perf=False,
check_exist=False,
create_dir=False):
"""
Output directory for the catalogues
Parameters
------------
hb_name : `str`
Name prefix of the file being analyzed. This is the name of the
file that has the information of the galaxies.
catl_kind : {'gal', 'memb', 'group'}, `str`
Option for which kind of catalogue is being analyzed. This
variable is set to ``memb`` by default.
Options:
- `gal` : Galaxy catalogue
- `memb` : Group Member galaxy catalogue
- `group` : Group galaxy catalogue
perf : `bool`, optional
If True, it returns (creates) the directory for the ``perfect``
catalogue. This variable is set to `False` by default.
check_exist : `bool`, optional
If `True`, it checks for whether or not the file exists.
This variable is set to `False` by default.
create_dir : `bool`, optional
If `True`, it creates the directory if it does not exist.
This variable is set to `False` by default.
Returns
---------
catl_output_dir : `str`
Output directory for all the catalogues
"""
# Halobias file name - Type
if not (isinstance(hb_name, str)):
msg = '>>> `hb_name` ({1}) is not a valid type!'
msg = msg.format(type(hb_name))
raise TypeError(msg)
# Catalogue kind - Type
if not (isinstance(catl_kind, str)):
msg = '>>> `catl_kind` ({1}) is not a valid type!'
msg = msg.format(type(catl_kind))
raise TypeError(msg)
# Catalogue kind - Value
calt_kind_arr = ['gal', 'memb', 'group']
if not (catl_kind in calt_kind_arr):
msg = '>>> `catl_kind` ({1}) is not a valid input! Exiting!'
msg = msg.format(catl_kind)
raise TypeError(msg)
# Perfect option - Type
if not (isinstance(perf, bool)):
msg = '>>> `perf` ({1}) is not a valid type!'
msg = msg.format(type(perf))
raise TypeError(msg)
# Check exists - Type
if not (isinstance(check_exist, bool)):
msg = '>>> `check_exist` ({1}) is not a valid type!'
msg = msg.format(type(check_exist))
raise TypeError(msg)
##
## Output directory
catl_outdir = os.path.join(self.proj_dict['proc_dir'],
self.cosmo_choice, self.survey, hb_name)
##
## Type of Output directory
catl_out_dict = {
'gal': 'galaxy_catalogues',
'memb': 'member_galaxy_catalogues',
'group': 'group_galaxy_catalogues'
}
if perf:
catl_output_dir = os.path.join(
catl_outdir, 'perf_{0}'.format(catl_out_dict[catl_kind]))
else:
catl_output_dir = os.path.join(catl_outdir,
catl_out_dict[catl_kind])
# Creating directory
if create_dir:
cfutils.Path_Folder(catl_output_dir)
# Check for its existence
if check_exist:
if not (os.path.exists(catl_output_dir)):
msg = '`catl_output_dir` ({0}) was not found!'.format(
catl_output_dir)
raise FileNotFoundError(msg)
return catl_output_dir