Exemple #1
0
    def input_catl_dir_path(self,
                            catl_kind='master',
                            check_exist=True,
                            create_dir=False):
        """
        Path to the file being analyzed.

        Parameters:
        ------------
        catl_kind : {'master', 'random', 'redmapper'}, `str`
            Option for which kind of catalogue is being analyzed.
            Options:
                - `master` : Catalogue of objects.
                - `random` : Catalogue of random catalogues
                - `redmapper` : Catalogue of RedMapper Clusters

        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ------------
        input_catl_dir : `str`
            Path to the file being analyzed.
        """
        # Check input parameters
        # `check_exist`
        if not (isinstance(check_exist, bool)):
            msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
                type(check_exist))
            raise TypeError(msg)
        #
        # `create_dir`
        if not (isinstance(create_dir, bool)):
            msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
                type(create_dir))
            raise TypeError(msg)
        #
        # `catl_kind`
        if not (catl_kind in ['master', 'redmapper', 'random']):
            msg = '`catl_kind` ({1}) is not a valid input variable!'
            msg = msg.format(catl_kind)
            raise ValueError(msg)
        # Path to directory
        input_catl_dir = os.path.join(self.proj_dict['ext_dir'], catl_kind)
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(input_catl_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(input_catl_dir)):
                msg = '`input_catl_dir` ({0}) was not found!'.format(
                    input_catl_dir)
                raise FileNotFoundError(msg)

        return input_catl_dir
Exemple #2
0
    def catl_filtered_dir(self,
                          catl_kind='master',
                          check_exist=True,
                          create_dir=False):
        """
        Path to the `filtered` file for `master`, after having applied all
        of the magnitude and redshift cuts.

        Parameters:
        ------------
        catl_kind : {'master'}, `str`
            Option for which kind of catalogue is being analyzed.
            Options:
                - `master` : Catalogue of objects.

        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ------------
        filtered_dir path : `str`
            Path to the `filtered` version of `master` file.
        """
        # Check input parameters
        # `check_exist`
        if not (isinstance(check_exist, bool)):
            msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
                type(check_exist))
            raise TypeError(msg)
        #
        # `create_dir`
        if not (isinstance(create_dir, bool)):
            msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
                type(create_dir))
            raise TypeError(msg)
        #
        # `catl_kind`
        if not (catl_kind in ['master', 'redmapper', 'random']):
            msg = '`catl_kind` ({1}) is not a valid input variable!'
            msg = msg.format(catl_kind)
            raise ValueError(msg)
        # Path to directory
        filtered_dir = os.path.join(self.proj_dict['int_dir'], 'filtered',
                                    catl_kind)
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(filtered_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(filtered_dir)):
                msg = '`filtered_dir` ({0}) was not found!'.format(
                    filtered_dir)
                raise FileNotFoundError(msg)

        return filtered_dir
    def cosmo_outdir(self, check_exist=False, create_dir=False):
        """
        Output directory for cosmology-related files.

        Parameters
        -----------
        check_exist : `bool`, optional
            If `True`, it checks for whether or not the directory exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        cosmo_dir : `str`
            Path to the directory that contains cosmologica-related
            files.
        """
        # Figure directory - Output
        cosmo_dir = os.path.join(self.proj_dict['raw_dir'], self.cosmo_choice,
                                 'cosmo_dir')
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(cosmo_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(cosmo_dir)):
                msg = '`cosmo_dir` ({0}) was not found!'.format(cosmo_dir)
                raise FileNotFoundError(msg)

        return cosmo_dir
    def tar_outdir(self, check_exist=False, create_dir=False):
        """
        Output directory for the TAR file produced in this project.

        Parameters
        -----------
        check_exist : `bool`, optional
            If `True`, it checks for whether or not the directory exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        tar_output_dir : `str`
            Path to the TAR output directory, which includes the resuls for
            each of the catalogues.
        """
        # Figure directory - Output
        tar_output_dir = os.path.join(self.proj_dict['proc_dir'], 'TAR_files',
                                      self.cosmo_choice, self.halotype,
                                      self.survey)
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(tar_output_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(tar_output_dir)):
                msg = '`tar_output_dir` ({0}) was not found!'.format(
                    tar_output_dir)
                raise FileNotFoundError(msg)

        return tar_output_dir
    def mass_func_output_dir(self, check_exist=False, create_dir=False):
        """
        Output directory for the files with the info on the mass function
        used in this project.

        Parameters
        -----------
        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        mf_output_dir : `str`
            Path to the output directory for the files related to the
            mass function used in this project.
        """
        # Mass Function - Output directory
        mf_output_dir = os.path.join(self.proj_dict['int_dir'], 'MF',
                                     self.cosmo_choice, self.survey)
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(mf_output_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(mf_output_dir)):
                msg = '`mf_output_dir` ({0}) was not found!'.format(
                    mf_output_dir)
                raise FileNotFoundError(msg)

        return mf_output_dir
    def halobias_outdir(self, check_exist=False, create_dir=False):
        """
        Output directory for the halobias files with the
        input galaxy information.

        Parameters
        -----------
        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        hb_files_out : `str`
            Path to the output directory for the halobias files with the
            input galaxy information.
        """
        # Halobias file output directory
        hb_files_out = os.path.join(self.proj_dict['raw_dir'], 'hb_files',
                                    self.survey)
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(hb_files_out)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(hb_files_out)):
                msg = '`hb_files_out` ({0}) was not found!'.format(
                    hb_files_out)
                raise FileNotFoundError(msg)

        return hb_files_out
def cookiecutter_paths(path='./'):
    """
    Paths to main folders in the `Data Science` cookiecutter template.
    This structure was taken from :
    - https://drivendata.github.io/cookiecutter-data-science/

    Parameters
    ----------
    path : str, optional
        Path to the file within the `.git` repository

    Return
    ----------
    param_dict : python dictionary
        Dictionary with info of the project that uses the Data Science
        cookiecutter template.

    Raises
    ----------
    LSSUtils_Error : exception
        If `path` is not within a .git directory, it raises an error.
    """
    # Base Path
    base_dir = git_root_dir(path) + '/'
    # Checking that directory exists
    if os.path.exists(base_dir):
        # Plot Directory
        plot_dir = os.path.join(base_dir, 'reports', 'figures/')
        # Source directory
        src_dir = os.path.join(base_dir, 'src', 'data/')
        # Data path
        data_dir = os.path.join(base_dir, 'data/')
        # External path
        ext_dir = os.path.join(data_dir, 'external/')
        # Processed path
        proc_dir = os.path.join(data_dir, 'processed/')
        # External path
        int_dir = os.path.join(data_dir, 'interim/')
        # External path
        raw_dir = os.path.join(data_dir, 'raw/')
        # Creating files
        for dir_ii in [plot_dir, src_dir, data_dir]:
            fd.Path_Folder(dir_ii)
        # Saving to dictionary
        param_dict = {}
        param_dict['base_dir'] = base_dir
        param_dict['plot_dir'] = plot_dir
        param_dict['src_dir'] = src_dir
        param_dict['data_dir'] = data_dir
        param_dict['ext_dir'] = ext_dir
        param_dict['proc_dir'] = proc_dir
        param_dict['int_dir'] = int_dir
        param_dict['raw_dir'] = raw_dir
    else:
        msg = '{0} `base_dir` ({1}) is not a Git directory! Exiting'.format(
            fd.Program_Msg(__file__), base_dir)
        raise LSSUtils_Error(msg)

    return param_dict
def directory_skeleton(param_dict, proj_dict):
    """
    Creates the directory skeleton for the current project

    Parameters
    ----------
    param_dict: python dictionary
        dictionary with `project` variables

    proj_dict: python dictionary
        dictionary with info of the project that uses the
        `Data Science` Cookiecutter template.

    Returns
    ---------
    proj_dict: python dictionary
        Dictionary with current and new paths to project directories
    """
    file_msg = param_dict['Prog_msg']
    ## Path to the simulation cubes to analyze
    sim_dir = os.path.join('/fs1/caldervf/CODES/vandy_group_statistics2',
                           'RESOLVE_Mocks_Cube_Color_Age', 'With_Weights',
                           'New_Mocks')
    # Checking that folder exists
    if not (os.path.exists(sim_dir)):
        msg = '{0} `sim_dir` ({1}) does not exist!'.format(file_msg, sim_dir)
        raise FileNotFoundError(msg)
    # Finding combination of the directory
    simfile = os.path.join(
        sim_dir, 'Mbaryon_9.0_Vpeak_0.2_u-r_Acc_Rate_100Myr_0.6_130.0.hdf5')
    # Check that file exists
    if not (os.path.exists(simfile)):
        msg = '{0} `simfile` ({1}) does not exist!'.format(file_msg, simfile)
        raise FileNotFoundError(msg)
    #
    # Output directory
    outdir = os.path.join(proj_dict['int_dir'], 'florez_age_matching_results')
    cfutils.Path_Folder(outdir)
    #
    # Saving to dictionary
    proj_dict['sim_dir'] = sim_dir
    proj_dict['simfile'] = simfile
    proj_dict['outdir'] = outdir

    return proj_dict
Exemple #9
0
    def analysis_out_dir(self, check_exist=True, create_dir=False):
        """
        Path to the output directory for final analysis.

        Parameters
        -----------
        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        outdir_path : `str`
            Path to the output directory for the final analysis.
        """
        # Check input parameters
        # `check_exist`
        if not (isinstance(check_exist, bool)):
            msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
                type(check_exist))
            raise TypeError(msg)
        #
        # `create_dir`
        if not (isinstance(create_dir, bool)):
            msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
                type(create_dir))
            raise TypeError(msg)
        # Path to directory
        outdir_path = os.path.join(self.proj_dict['proc_dir'], 'analysis')
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(outdir_path)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(outdir_path)):
                msg = '`outdir_path` ({0}) was not found!'.format(outdir_path)
                raise FileNotFoundError(msg)

        return outdir_path
    def fig_outdir(self, hb_name, check_exist=False, create_dir=False):
        """
        Output directory for the figures produced in this project.

        Parameters
        -----------
        hb_name : `str`
            Name prefix of the file being analyzed. This is the name of the
            file that has the information of the galaxies.

        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        fig_output_dir : `str`
            Path to the output directory for the files related to the
            mass function used in this project.
        """
        # Figure directory - Output
        fig_output_dir = os.path.join(self.proj_dict['plot_dir'],
                                      self.cosmo_choice, self.halotype,
                                      self.survey, hb_name)
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(fig_output_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(fig_output_dir)):
                msg = '`fig_output_dir` ({0}) was not found!'.format(
                    fig_output_dir)
                raise FileNotFoundError(msg)

        return fig_output_dir
def url_files_download(url,
                       ext,
                       outdir,
                       check_exist=False,
                       create_dir=False,
                       remove_files=False,
                       bar_opt='tqdm'):
    """
    Downloads the files from a URL to a local directory. The files that
    match a specific file extension, `ext`.

    Parameters
    -----------
    url : `str`
        String of the URL

    ext : `str`
        File extension of the files in the URL.

    outdir : `str`
        Path to the output directory. This is the directory, to which
        the files with extensions `ext` will be saved.

    check_exist : `bool`, optional
        If `True`, it checks for whether or not the file exists.
        This variable is set to `False` by default.

    create_dir : `bool`, optional
        If `True`, it creates the directory if it does not exist.
        This variable is set to `False` by default.

    remove_files : `bool`, optional
        If `True`, local files that are present that match the files at
        the URL will be replaced by the new versions. This variable is
        set to ``False`` by default.

    bar_opt : {'tqdm', 'native'}
        Option for which type of progress bar to use when downloading files.
        This variable is set to `tqdm` by default.
        Options:
            - 'tqdm' : Uses a tqdm-based progress bar
            - 'native': Used the wget-based native progress bar.
    """
    file_msg = fd.Program_Msg(__file__)
    ## Checking for file type
    # 'URL'
    if not isinstance(url, str):
        msg = '{0} `url` ({1}) is not a valid type. It must be a STRING!'
        msg = msg.format(file_msg, type(url))
        raise TypeError(msg)
    # File extension
    if not isinstance(ext, str):
        msg = '{0} `ext` ({1}) is not a valid type. It must be a STRING!'
        msg = msg.format(file_msg, type(ext))
        raise TypeError(msg)
    # Output directory
    if not isinstance(outdir, str):
        msg = '{0} `outdir` ({1}) is not a valid type. It must be a STRING!'
        msg = msg.format(file_msg, type(outdir))
        raise TypeError(msg)
    # `check_exist`
    if not (isinstance(check_exist, bool)):
        msg = '`check_exist` ({0}) must be of `boolean` type!'.format(
            type(check_exist))
        raise TypeError(msg)
    # `create_dir`
    if not (isinstance(create_dir, bool)):
        msg = '`create_dir` ({0}) must be of `boolean` type!'.format(
            type(create_dir))
        raise TypeError(msg)
    # `bar` - Type
    if not (isinstance(bar_opt, str)):
        msg = '`bar_opt` ({0}) must be of `boolean` type!'.format(
            type(bar_opt))
        raise TypeError(msg)
    # Progress bar - Value
    if not (bar_opt in ['tqdm', 'native']):
        msg = '{0} `bar_opt` ({1}) is not a valid option! Exiting'
        msg = msg.format(file_msg, bar_opt)
        raise LSSUtils_Error(msg)
    ##
    ## List of files in the URL
    files_arr = url_file_list(url, ext)
    # Creating directory
    if create_dir:
        cfutils.Path_Folder(outdir)
    # Check for its existence
    if check_exist:
        if not (os.path.exists(outdir)):
            msg = '`outdir` ({0}) was not found!'.format(outdir)
            raise FileNotFoundError(msg)
    ##
    ## Downloading files to output directory
    if len(files_arr) > 0:
        if (bar_opt == 'tqdm'):
            tqdm_desc = 'Downloading files: '
            for file_ii in tqdm(files_arr, desc=tqdm_desc):
                # Local file
                file_ii_local = os.path.join(outdir, os.path.basename(file_ii))
                # Checking if local file exists
                if os.path.exists(file_ii_local):
                    if remove_files:
                        os.remove(file_ii_local)
                        wget_opt = True
                    else:
                        wget_opt = False
                else:
                    wget_opt = True
                ##
                ## Only downloading if necessary
                if wget_opt:
                    wget.download(file_ii, out=outdir, bar=None)
        elif (bar_opt == 'native'):
            for file_ii in files_arr:
                # Local file
                file_ii_local = os.path.join(outdir, os.path.basename(file_ii))
                # Checking if local file exists
                if os.path.exists(file_ii_local):
                    if remove_files:
                        os.remove(file_ii_local)
                        wget_opt = True
                    else:
                        wget_opt = False
                else:
                    wget_opt = True
                ##
                ## Only downloading if necessary
                if wget_opt:
                    wget.download(file_ii, out=outdir)
    else:
        msg = '{0} Number of files is ZERO!'.format(file_msg)
        print(msg)
    def catl_output_dir(self,
                        hb_name,
                        catl_kind='memb',
                        perf=False,
                        check_exist=False,
                        create_dir=False):
        """
        Output directory for the catalogues

        Parameters
        ------------
        hb_name : `str`
            Name prefix of the file being analyzed. This is the name of the
            file that has the information of the galaxies.

        catl_kind : {'gal', 'memb', 'group'}, `str`
            Option for which kind of catalogue is being analyzed. This
            variable is set to ``memb`` by default.
            Options:
                - `gal` : Galaxy catalogue
                - `memb` : Group Member galaxy catalogue
                - `group` : Group galaxy catalogue

        perf : `bool`, optional
            If True, it returns (creates) the directory for the ``perfect``
            catalogue. This variable is set to `False` by default.

        check_exist : `bool`, optional
            If `True`, it checks for whether or not the file exists.
            This variable is set to `False` by default.

        create_dir : `bool`, optional
            If `True`, it creates the directory if it does not exist.
            This variable is set to `False` by default.

        Returns
        ---------
        catl_output_dir : `str`
            Output directory for all the catalogues
        """
        # Halobias file name - Type
        if not (isinstance(hb_name, str)):
            msg = '>>> `hb_name` ({1}) is not a valid type!'
            msg = msg.format(type(hb_name))
            raise TypeError(msg)
        # Catalogue kind - Type
        if not (isinstance(catl_kind, str)):
            msg = '>>> `catl_kind` ({1}) is not a valid type!'
            msg = msg.format(type(catl_kind))
            raise TypeError(msg)
        # Catalogue kind - Value
        calt_kind_arr = ['gal', 'memb', 'group']
        if not (catl_kind in calt_kind_arr):
            msg = '>>> `catl_kind` ({1}) is not a valid input! Exiting!'
            msg = msg.format(catl_kind)
            raise TypeError(msg)
        # Perfect option - Type
        if not (isinstance(perf, bool)):
            msg = '>>> `perf` ({1}) is not a valid type!'
            msg = msg.format(type(perf))
            raise TypeError(msg)
        # Check exists - Type
        if not (isinstance(check_exist, bool)):
            msg = '>>> `check_exist` ({1}) is not a valid type!'
            msg = msg.format(type(check_exist))
            raise TypeError(msg)
        ##
        ## Output directory
        catl_outdir = os.path.join(self.proj_dict['proc_dir'],
                                   self.cosmo_choice, self.survey, hb_name)
        ##
        ## Type of Output directory
        catl_out_dict = {
            'gal': 'galaxy_catalogues',
            'memb': 'member_galaxy_catalogues',
            'group': 'group_galaxy_catalogues'
        }
        if perf:
            catl_output_dir = os.path.join(
                catl_outdir, 'perf_{0}'.format(catl_out_dict[catl_kind]))
        else:
            catl_output_dir = os.path.join(catl_outdir,
                                           catl_out_dict[catl_kind])
        # Creating directory
        if create_dir:
            cfutils.Path_Folder(catl_output_dir)
        # Check for its existence
        if check_exist:
            if not (os.path.exists(catl_output_dir)):
                msg = '`catl_output_dir` ({0}) was not found!'.format(
                    catl_output_dir)
                raise FileNotFoundError(msg)

        return catl_output_dir