Beispiel #1
0
def load_user_config(args, log):
    """Load settings from the user's confiuration file, and add them to `args`.

    Settings are loaded from the configuration file in the user's home
    directory.  Those parameters are added (as attributes) to the `args`
    object.

    Arguments
    ---------
    args : `argparse.Namespace`
        Namespace object to which configuration attributes will be added.

    Returns
    -------
    args : `argparse.Namespace`
        Namespace object with added attributes.

    """
    if not os.path.exists(_CONFIG_PATH):
        err_str = (
            "Configuration file does not exists ({}).\n".format(_CONFIG_PATH) +
            "Run `python -m astrocats setup` to configure.")
        log_raise(log, err_str)

    config = json.load(open(_CONFIG_PATH, 'r'))
    setattr(args, _BASE_PATH_KEY, config[_BASE_PATH_KEY])
    log.debug("Loaded configuration: {}: {}".format(_BASE_PATH_KEY, config[
        _BASE_PATH_KEY]))
    return args
Beispiel #2
0
def load_user_config(args, log):
    """Load settings from the user's confiuration file, and add them to `args`.

    Settings are loaded from the configuration file in the user's home
    directory.  Those parameters are added (as attributes) to the `args`
    object.

    Arguments
    ---------
    args : `argparse.Namespace`
        Namespace object to which configuration attributes will be added.

    Returns
    -------
    args : `argparse.Namespace`
        Namespace object with added attributes.

    """
    if not os.path.exists(_CONFIG_PATH):
        err_str = (
            "Configuration file does not exists ({}).\n".format(_CONFIG_PATH) +
            "Run `python -m astrocats setup` to configure.")
        log_raise(log, err_str)

    config = json.load(open(_CONFIG_PATH, 'r'))
    setattr(args, _BASE_PATH_KEY, config[_BASE_PATH_KEY])
    log.debug("Loaded configuration: {}: {}".format(
        _BASE_PATH_KEY, config[_BASE_PATH_KEY]))
    return args
def do_wevers_2017(catalog=None):
    """
    """
    if catalog is None:
        log = utils.get_logger()
        task_str = "Wevers+2017"
        task_name = task_str
        task_dir = os.path.realpath(__file__)
        print("task_dir = ", task_dir)
        sys.exit(232)
    else:
        log = catalog.log
        log.debug("shen_2008.do_shen_2008()")
        task_str = catalog.get_current_task_str()
        task_name = catalog.current_task.name
        task_dir = catalog.get_current_task_repo()

    # Go through each element of the tables
    num = 0
    line_num = 0
    count = 0

    data_fname = os.path.join(task_dir, DATA_FILENAME)
    log.info("Input filename '{}'".format(data_fname))
    if not os.path.exists(data_fname):
        utils.log_raise(log, "File not found '{}'".format(data_fname), IOError)

    with tqdm.tqdm(desc=task_str, total=EXPECTED_TOTAL, dynamic_ncols=True) as pbar:

        with open(data_fname, 'r') as data:
            spamreader = csv.reader(data, delimiter='|')
            for row in spamreader:
                line_num += 1
                if len(row) == 0 or row[0].startswith('#'):
                    continue
                else:
                    count += 1
                    if count < 4:
                        continue

                bh_name = _add_entry_for_data_line(catalog, row)
                if bh_name is not None:
                    log.debug("{}: added '{}'".format(task_name, bh_name))
                    num += 1

                    if (JOURNAL_INTERNAL is not None) and (num % JOURNAL_INTERNAL == 0):
                        catalog.journal_entries()

                    if catalog.args.travis and (num > catalog.TRAVIS_QUERY_LIMIT):
                        log.warning("Exiting on travis limit")
                        break

                pbar.update(1)

    log.info("Added {} entries".format(num))
    if (num != EXPECTED_TOTAL) and (not catalog.args.travis):
        log.warning("Number of entries added {} does not match expectation {}!".format(
            num, EXPECTED_TOTAL))

    return
Beispiel #4
0
def main():
    """Primary entry point for all AstroCats catalogs.

    From this entry point, all internal catalogs can be accessed and their
    public methods executed (for example: import scripts).

    """
    from datetime import datetime

    # Initialize Command-Line and User-Config Settings, Log
    # -----------------------------------------------------

    beg_time = datetime.now()
    # Process command-line arguments to determine action
    # If no subcommand (e.g. 'import') is given, returns 'None' --> exit
    args, sub_clargs = load_command_line_args()
    if args is None:
        return

    # Create a logging object
    log = load_log(args)

    # Run configuration/setup interactive script
    if args.command == 'setup':
        setup_user_config(log)
        return

    git_vers = get_git()
    title_str = "Astrocats, version: {}, SHA: {}".format(__version__, git_vers)
    log.warning("\n\n{}\n{}\n{}\n".format(
        title_str, '=' * len(title_str), beg_time.ctime()))

    # Load the user settings from the home directory
    args = load_user_config(args, log)

    #
    # Choose Catalog and Operation(s) to perform
    # ------------------------------------------
    mod_name = args.command
    log.debug("Specified module: '{}'".format(mod_name))
    # Try to import the specified module
    try:
        mod = importlib.import_module('.' + mod_name, package='astrocats')
    except Exception as err:
        log.error("Import of specified module '{}' failed.".format(mod_name))
        log_raise(log, str(err), type(err))

    # Run the `main.main` method of the specified module
    mod.main.main(args, sub_clargs, log)

    end_time = datetime.now()
    log.warning("\nAll complete at {}, After {}".format(
        end_time, end_time - beg_time))
    return
Beispiel #5
0
def setup_user_config(log):
    """Setup a configuration file in the user's home directory.

    Currently this method stores default values to a fixed configuration
    filename.  It should be modified to run an interactive prompt session
    asking for parameters (or at least confirming the default ones).

    Arguments
    ---------
    log : `logging.Logger` object

    """
    log.warning("AstroCats Setup")
    log.warning("Configure filepath: '{}'".format(_CONFIG_PATH))

    # Create path to configuration file as needed
    config_path_dir = os.path.split(_CONFIG_PATH)[0]
    if not os.path.exists(config_path_dir):
        log.debug("Creating config directory '{}'".format(
            config_path_dir))
        os.makedirs(config_path_dir)

    if not os.path.isdir(config_path_dir):
        log_raise(log, "Configure path error '{}'".format(config_path_dir))

    # Determine default settings

    # Get this containing directory and use that as default data path
    def_base_path = os.path.abspath(os.path.dirname(__file__))
    log.warning("Setting '{}' to default path: '{}'".format(
        _BASE_PATH_KEY, def_base_path))
    config = {_BASE_PATH_KEY: def_base_path}

    # Write settings to configuration file
    json.dump(config, open(_CONFIG_PATH, 'w'))
    if not os.path.exists(def_base_path):
        log_raise(log, "Problem creating configuration file.")

    return
Beispiel #6
0
def setup_user_config(log):
    """Setup a configuration file in the user's home directory.

    Currently this method stores default values to a fixed configuration
    filename.  It should be modified to run an interactive prompt session
    asking for parameters (or at least confirming the default ones).

    Arguments
    ---------
    log : `logging.Logger` object

    """
    log.warning("AstroCats Setup")
    log.warning("Configure filepath: '{}'".format(_CONFIG_PATH))

    # Create path to configuration file as needed
    config_path_dir = os.path.split(_CONFIG_PATH)[0]
    if not os.path.exists(config_path_dir):
        log.debug("Creating config directory '{}'".format(config_path_dir))
        os.makedirs(config_path_dir)

    if not os.path.isdir(config_path_dir):
        log_raise(log, "Configure path error '{}'".format(config_path_dir))

    # Determine default settings

    # Get this containing directory and use that as default data path
    def_base_path = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
    log.warning("Setting '{}' to default path: '{}'".format(_BASE_PATH_KEY,
                                                            def_base_path))
    config = {_BASE_PATH_KEY: def_base_path}

    # Write settings to configuration file
    json.dump(config, open(_CONFIG_PATH, 'w'))
    if not os.path.exists(def_base_path):
        log_raise(log, "Problem creating configuration file.")

    return
def do_tremaine_2002(catalog):
    """
    """
    log = catalog.log
    log.debug("tremaine_2002.do_tremaine_2002()")
    task_str = catalog.get_current_task_str()
    task_name = catalog.current_task.name
    task_dir = catalog.get_current_task_repo()

    # Go through each element of the tables
    num = 0

    data_fname = os.path.join(task_dir, DATA_FILENAME)
    log.info("Input filename '{}'".format(data_fname))
    if not os.path.exists(data_fname):
        utils.log_raise(log, "File not found '{}'".format(data_fname), IOError)

    with tqdm.tqdm(desc=task_str, total=31, dynamic_ncols=True) as pbar:

        with open(data_fname, 'r') as data:
            spamreader = csv.reader(data, delimiter=' ')
            for row in spamreader:
                if len(row) <= 1 or row[0].startswith('#'):
                    continue

                bh_name = _add_entry_for_data_line(catalog, row)
                if bh_name is not None:
                    log.debug("{}: added '{}'".format(task_name, bh_name))
                    num += 1

                    if catalog.args.travis and (num > catalog.TRAVIS_QUERY_LIMIT):
                        log.warning("Exiting on travis limit")
                        break

                pbar.update(1)

    log.info("Added {} entries".format(num))
    return
Beispiel #8
0
def main():
    """Primary entry point for all AstroCats catalogs.

    From this entry point, all internal catalogs can be accessed and their
    public methods executed (for example: import scripts).

    """
    from datetime import datetime

    # Initialize Command-Line and User-Config Settings, Log
    # -----------------------------------------------------

    beg_time = datetime.now()
    # Process command-line arguments to determine action
    # If no subcommand (e.g. 'import') is given, returns 'None' --> exit
    args, sub_clargs = load_command_line_args()
    if args is None:
        return

    # Create a logging object
    log = load_log(args)

    # Run configuration/setup interactive script
    if args.command == 'setup':
        setup_user_config(log)
        return

    # Make sure configuration file exists, or that's what we're doing
    # (with the 'setup' subcommand)
    if not os.path.isfile(_CONFIG_PATH):
        raise RuntimeError("'{}' does not exist.  "
                           "Run `astrocats setup` to configure."
                           "".format(_CONFIG_PATH))

    git_vers = get_git()
    title_str = "Astrocats, version: {}, SHA: {}".format(__version__, git_vers)
    log.warning("\n\n{}\n{}\n{}\n".format(title_str, '=' * len(title_str),
                                          beg_time.ctime()))

    # Load the user settings from the home directory
    args = load_user_config(args, log)

    # Choose Catalog and Operation(s) to perform
    # ------------------------------------------
    mod_name = args.command
    log.debug("Importing specified module: '{}'".format(mod_name))
    # Try to import the specified module
    try:
        mod = importlib.import_module('.' + mod_name, package='astrocats')
    except Exception as err:
        log.error("Import of specified module '{}' failed.".format(mod_name))
        log_raise(log, str(err), type(err))

    # Run the `main.main` method of the specified module
    log.debug("Running `main.main()`")
    mod.main.main(args, sub_clargs, log)

    end_time = datetime.now()
    log.warning("\nAll complete at {}, After {}".format(end_time, end_time -
                                                        beg_time))
    return
def _add_entry_for_data_line(catalog, line):
    """

    Sample Entries:
    --------------
    000132.83+145608.0|000.386795|+14.935573| 0.3989| 18.898| -23.131| 45.356|  751|  303| 52251
        |0|1|1|0|   4026|    2.450|  8.119|   4699|    2.671|  8.114|       |         |
        |8.119|       |  10.293|Sloan|DR5|Simbad|NED

    000135.51-004206.7|000.397978|-00.701886| 3.5779| 19.183| -27.951| 47.015| 1489|  104| 52991
        |0|0|0|1|       |         |       |       |         |       |   6207|  271.312|  9.536
        |9.536|       |   3.937|Sloan|DR5|Simbad|NED

    Columns:
    -------
    [x]  0  SDSS     (A18)   SDSS-DR5 Object Designation (HHMMSS.ss+DDMMSS.s, J2000)
    [x]  1  RAJ2000  (F10.6) Right Ascension in decimal degrees (J2000)
    [x]  2  DEJ2000  (F10.6) Declination in decimal degrees (J2000)
    [x]  3  z        (F7.4)  Redshift
    [ ]  4  imag     (F7.3)  PSF i-band apparent magnitude (1)
    [x]  5  iMAG     (F8.3)  PSF i-band absolute magnitude, K-corrected to z=2
    [x]  6  logLbol  (F7.3)  ? log of bolometric luminosity (erg/s)
    [ ]  7  Plate    (I5)    Plate number
    [ ]  8  Fiber    (I5)    Fiber identification number
    [x]  9  ObsDate  (I6)    MJD date of spectroscopic observation
    [ ] 10  F        (I1)    [0/1] FIRST selection flag
    [ ] 11  R        (I1)    [0/1] ROSAT selection flag
    [ ] 12  U        (I1)    [0/1] Uniform selection flag
    [ ] 13  B        (I1)    [0/1] Broad Absoprtion Lines flag
    [x] 14  WHb      (I7)    ? Full-width at Half-Maximum of H{beta}
    [x] 15  L5100    (F9.3)  ? Monochromatic luminosity at 5100{AA}
    [x] 16  BHHb     (F7.3)  ? Virial mass of black hole estimated using H{beta} (log scale)
    [x] 17  WMgII    (I7)    ? Full-width at Half-Maximum of MgII
    [x] 18  L3000    (F9.3)  ? Monochromatic luminosity at 3000{AA}	[ucd=phys.luminosity]
    [x] 19  BHMgII   (F7.3)  ? Virial mass of black hole estimated using MgII (log scale)
    [x] 20  WCIV     (I7)    ? Full-width at Half-Maximum of CIV
    [x] 21  L1350    (F9.3)  ? Monochromatic luminosity at 1350{AA}
    [x] 22  BHCIV    (F7.3)  ? Virial mass of black hole estimated using CIV (log scale)
    [x] 23  BHvir    (F7.3)  ? Virial mass of black hole (3)
    [ ] 24  CIVMgII  (I7)    ? CIV-MgII blueshift
    [ ] 25  S  N	 (F8.3)  Mean spectrum signal-to-noise ratio
    [ ] 26  Sloan    (a5)    SDSS details from most recent SDSS data release

    """
    log = catalog.log
    # log.debug("shen_2008._add_entry_for_data_line()")
    if len(line) != NUM_COLUMNS:
        log.warning("length of line: '{}', expected {}!  '{}'".format(
            len(line), NUM_COLUMNS, line))
        return None

    line = [ll.strip() for ll in line]

    # [0] SDSS Galaxy/BH Name
    # -----------------------
    data_name = "SDSS" + line[0]
    name = catalog.add_entry(data_name)
    # Add this source
    source = catalog.entries[name].add_source(
        url=SOURCE_URL, bibcode=SOURCE_BIBCODE, name=SOURCE_NAME, secondary=False)

    task_name = catalog.current_task.name
    catalog.entries[name].add_data(BLACKHOLE.TASKS, task_name)

    # [1/2] RA/DEC
    catalog.entries[name].add_quantity(BLACKHOLE.RA, line[1], source)
    catalog.entries[name].add_quantity(BLACKHOLE.DEC, line[2], source)

    # [3] Redshift
    catalog.entries[name].add_quantity(BLACKHOLE.REDSHIFT, line[3], source)

    # [4] i-band apparent magnitude
    # photo_kwargs = {
    #     PHOTOMETRY.MAGNITUDE: line[4],
    #     PHOTOMETRY.SOURCE: source,
    #     PHOTOMETRY.HOST: False,
    #     PHOTOMETRY.U_LUMINOSITY: 'apparent Magnitude',
    #     PHOTOMETRY.DESCRIPTION: 'PSF i-band apparent magnitude',
    #     PHOTOMETRY.BAND: 'i',
    #     PHOTOMETRY.INCLUDES_HOST: True,
    # }
    # catalog.entries[name].add_photometry(**photo_kwargs)

    # [9] Observation date
    # convert from MJD to string YYYY/MM/DD
    # obs_date = ap.time.Time(int(line[9]), format='mjd').datetime.strftime('%Y/%m/%d')
    # obs_date = float(line[9])
    obs_date = line[9]

    # [5] i-band Absolute magnitude (z=2)
    if len(line[5]):
        photo_kwargs = {
            PHOTOMETRY.MAGNITUDE: line[5],
            PHOTOMETRY.SOURCE: source,
            PHOTOMETRY.HOST: False,
            PHOTOMETRY.U_LUMINOSITY: 'Absolute Magnitude',
            PHOTOMETRY.DESCRIPTION: 'PSF i-band absolute magnitude, K-corrected to z=2',
            PHOTOMETRY.BAND: 'i',
            PHOTOMETRY.INCLUDES_HOST: True,
            PHOTOMETRY.KCORRECTED: True,
            PHOTOMETRY.TIME: obs_date,
            PHOTOMETRY.U_TIME: 'MJD',
        }
        if not catalog.entries[name].add_photometry(**photo_kwargs):
            utils.log_raise(log, "Adding photometry failed for '{}'\n:{}".format(
                name, photo_kwargs))

    # [6] Luminosity Bolometric
    if len(line[6]):
        photo_kwargs = {
            PHOTOMETRY.LUMINOSITY: line[6],
            PHOTOMETRY.SOURCE: source,
            PHOTOMETRY.HOST: False,
            PHOTOMETRY.U_LUMINOSITY: 'log (L/[erg/s])',
            PHOTOMETRY.DESCRIPTION: 'Using bolometric corrections in Richards+2006b',
            PHOTOMETRY.BAND: 'bolometric',
            PHOTOMETRY.INCLUDES_HOST: True,
            PHOTOMETRY.TIME: obs_date,
            PHOTOMETRY.U_TIME: 'MJD',
        }
        if not catalog.entries[name].add_photometry(**photo_kwargs):
            utils.log_raise(log, "Adding photometry failed for '{}'\n:{}".format(
                name, photo_kwargs))

    # [14-22] FWHM, Mass, and Monochromatic Luminosities by line
    line_names = ["H-Beta", "Mg-II", "C-IV"]
    line_waves = ["5100", "3000", "1350"]
    line_vars = ["HBETA", "MGII", "CIV"]
    for ii, (nn, ww, vv) in enumerate(zip(line_names, line_waves, line_vars)):
        line_name = "{} ({} A)".format(nn, ww)

        # FWHM for this line
        col = 14 + ii*3
        if len(line[col]):
            var = getattr(BLACKHOLE, "FWHM_" + vv)
            desc = "Full-width at Half-Maximum of {}".format(line_name)
            quant_kwargs = {QUANTITY.U_VALUE: 'km/s', QUANTITY.DESCRIPTION: desc}
            catalog.entries[name].add_quantity(var, line[col], source, **quant_kwargs)

        # Luminosity from this line
        col = 15 + ii*3
        if len(line[col]):
            desc = 'Monochromatic luminosity for {} (lambda*L_lambda)'.format(line_name)
            photo_kwargs = {
                PHOTOMETRY.LUMINOSITY: line[col],
                PHOTOMETRY.SOURCE: source,
                PHOTOMETRY.HOST: False,
                PHOTOMETRY.U_LUMINOSITY: 'log (L/[erg/s])',
                PHOTOMETRY.DESCRIPTION: desc,
                PHOTOMETRY.WAVELENGTH: ww,
                PHOTOMETRY.U_WAVELENGTH: 'Angstrom',
                PHOTOMETRY.TIME: obs_date,
                PHOTOMETRY.U_TIME: 'MJD',
            }
            if not catalog.entries[name].add_photometry(**photo_kwargs):
                utils.log_raise(log, "Adding photometry failed for '{}' ({}) \n:{}".format(
                    name, nn, photo_kwargs))

        # Mass from this Line
        col = 16 + ii*3
        if len(line[col]):
            var = BLACKHOLE.MASS
            mass_method = getattr(BH_MASS_METHODS, "VIR_" + vv)
            desc = "Virial mass estimated using {}".format(line_name)
            quant_kwargs = {QUANTITY.U_VALUE: 'log(M/Msol)',
                            QUANTITY.DESCRIPTION: desc,
                            QUANTITY.KIND: mass_method}
            catalog.entries[name].add_quantity(var, line[col], source, **quant_kwargs)

    # [23] Mass from optimal line (based on redshift)
    val = line[23]
    if len(val):
        desc = ("Virial BH-Mass Using H-Beta for z < 0.7; "
                "Mg-ii for 0.7 < z < 1.9; and C-iv for z > 1.9")
        quant_kwargs = {QUANTITY.U_VALUE: 'log(M/Msol)',
                        QUANTITY.DESCRIPTION: desc,
                        QUANTITY.KIND: BH_MASS_METHODS.VIR}
        catalog.entries[name].add_quantity(BLACKHOLE.MASS, val, source, **quant_kwargs)

    return name
def do_wevers_2017(catalog=None):
    """
    """
    if catalog is None:
        import logging
        log = utils.get_logger(stream_level=logging.DEBUG)
        task_str = "Wevers+2017"
        task_name = task_str
        task_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir))
        task_dir = os.path.join(task_dir, "input", "internal", "")
        log.warning("Running task '{}' directly".format(task_str))
    else:
        log = catalog.log
        log.debug("shen_2008.do_shen_2008()")
        task_str = catalog.get_current_task_str()
        task_name = catalog.current_task.name
        task_dir = catalog.get_current_task_repo()

    # Go through each element of the tables
    num = 0
    line_num = 0
    count = 0

    data_fname = os.path.join(task_dir, DATA_FILENAME)
    log.info("Input filename '{}'".format(data_fname))
    if not os.path.exists(data_fname):
        utils.log_raise(log, "File not found '{}'".format(data_fname), IOError)

    import astropy as ap
    # with open(data_fname, 'r') as indata:
    #     data = indata.read()
    #
    # print(data)
    # data = ap.table.Table.read(data.encode('unicode-escape'), format='ascii.latex')
    # print(data)

    data = ap.table.Table.read(data_fname)  # , format='ascii.latex')
    print(data)

    return data

    with tqdm.tqdm(desc=task_str, total=EXPECTED_TOTAL, dynamic_ncols=True) as pbar:

        with open(data_fname, 'r') as data:
            spamreader = csv.reader(data, delimiter='|')
            for row in spamreader:
                line_num += 1
                if len(row) == 0 or row[0].startswith('#'):
                    continue
                else:
                    count += 1
                    if count < 4:
                        continue

                bh_name = _add_entry_for_data_line(catalog, row)
                if bh_name is not None:
                    log.debug("{}: added '{}'".format(task_name, bh_name))
                    num += 1

                    if (JOURNAL_INTERNAL is not None) and (num % JOURNAL_INTERNAL == 0):
                        catalog.journal_entries()

                    if catalog.args.travis and (num > catalog.TRAVIS_QUERY_LIMIT):
                        log.warning("Exiting on travis limit")
                        break

                pbar.update(1)

    log.info("Added {} entries".format(num))
    if (num != EXPECTED_TOTAL) and (not catalog.args.travis):
        log.warning("Number of entries added {} does not match expectation {}!".format(
            num, EXPECTED_TOTAL))

    return