def load_user_config(args, log):
"""Load settings from the user's confiuration file, and add them to `args`.
Settings are loaded from the configuration file in the user's home
directory. Those parameters are added (as attributes) to the `args`
object.
Arguments
---------
args : `argparse.Namespace`
Namespace object to which configuration attributes will be added.
Returns
-------
args : `argparse.Namespace`
Namespace object with added attributes.
"""
if not os.path.exists(_CONFIG_PATH):
err_str = (
"Configuration file does not exists ({}).\n".format(_CONFIG_PATH) +
"Run `python -m astrocats setup` to configure.")
log_raise(log, err_str)
config = json.load(open(_CONFIG_PATH, 'r'))
setattr(args, _BASE_PATH_KEY, config[_BASE_PATH_KEY])
log.debug("Loaded configuration: {}: {}".format(_BASE_PATH_KEY, config[
_BASE_PATH_KEY]))
return args
def load_user_config(args, log):
"""Load settings from the user's confiuration file, and add them to `args`.
Settings are loaded from the configuration file in the user's home
directory. Those parameters are added (as attributes) to the `args`
object.
Arguments
---------
args : `argparse.Namespace`
Namespace object to which configuration attributes will be added.
Returns
-------
args : `argparse.Namespace`
Namespace object with added attributes.
"""
if not os.path.exists(_CONFIG_PATH):
err_str = (
"Configuration file does not exists ({}).\n".format(_CONFIG_PATH) +
"Run `python -m astrocats setup` to configure.")
log_raise(log, err_str)
config = json.load(open(_CONFIG_PATH, 'r'))
setattr(args, _BASE_PATH_KEY, config[_BASE_PATH_KEY])
log.debug("Loaded configuration: {}: {}".format(
_BASE_PATH_KEY, config[_BASE_PATH_KEY]))
return args
def do_wevers_2017(catalog=None):
"""
"""
if catalog is None:
log = utils.get_logger()
task_str = "Wevers+2017"
task_name = task_str
task_dir = os.path.realpath(__file__)
print("task_dir = ", task_dir)
sys.exit(232)
else:
log = catalog.log
log.debug("shen_2008.do_shen_2008()")
task_str = catalog.get_current_task_str()
task_name = catalog.current_task.name
task_dir = catalog.get_current_task_repo()
# Go through each element of the tables
num = 0
line_num = 0
count = 0
data_fname = os.path.join(task_dir, DATA_FILENAME)
log.info("Input filename '{}'".format(data_fname))
if not os.path.exists(data_fname):
utils.log_raise(log, "File not found '{}'".format(data_fname), IOError)
with tqdm.tqdm(desc=task_str, total=EXPECTED_TOTAL, dynamic_ncols=True) as pbar:
with open(data_fname, 'r') as data:
spamreader = csv.reader(data, delimiter='|')
for row in spamreader:
line_num += 1
if len(row) == 0 or row[0].startswith('#'):
continue
else:
count += 1
if count < 4:
continue
bh_name = _add_entry_for_data_line(catalog, row)
if bh_name is not None:
log.debug("{}: added '{}'".format(task_name, bh_name))
num += 1
if (JOURNAL_INTERNAL is not None) and (num % JOURNAL_INTERNAL == 0):
catalog.journal_entries()
if catalog.args.travis and (num > catalog.TRAVIS_QUERY_LIMIT):
log.warning("Exiting on travis limit")
break
pbar.update(1)
log.info("Added {} entries".format(num))
if (num != EXPECTED_TOTAL) and (not catalog.args.travis):
log.warning("Number of entries added {} does not match expectation {}!".format(
num, EXPECTED_TOTAL))
return
def main():
"""Primary entry point for all AstroCats catalogs.
From this entry point, all internal catalogs can be accessed and their
public methods executed (for example: import scripts).
"""
from datetime import datetime
# Initialize Command-Line and User-Config Settings, Log
# -----------------------------------------------------
beg_time = datetime.now()
# Process command-line arguments to determine action
# If no subcommand (e.g. 'import') is given, returns 'None' --> exit
args, sub_clargs = load_command_line_args()
if args is None:
return
# Create a logging object
log = load_log(args)
# Run configuration/setup interactive script
if args.command == 'setup':
setup_user_config(log)
return
git_vers = get_git()
title_str = "Astrocats, version: {}, SHA: {}".format(__version__, git_vers)
log.warning("\n\n{}\n{}\n{}\n".format(
title_str, '=' * len(title_str), beg_time.ctime()))
# Load the user settings from the home directory
args = load_user_config(args, log)
#
# Choose Catalog and Operation(s) to perform
# ------------------------------------------
mod_name = args.command
log.debug("Specified module: '{}'".format(mod_name))
# Try to import the specified module
try:
mod = importlib.import_module('.' + mod_name, package='astrocats')
except Exception as err:
log.error("Import of specified module '{}' failed.".format(mod_name))
log_raise(log, str(err), type(err))
# Run the `main.main` method of the specified module
mod.main.main(args, sub_clargs, log)
end_time = datetime.now()
log.warning("\nAll complete at {}, After {}".format(
end_time, end_time - beg_time))
return
def setup_user_config(log):
"""Setup a configuration file in the user's home directory.
Currently this method stores default values to a fixed configuration
filename. It should be modified to run an interactive prompt session
asking for parameters (or at least confirming the default ones).
Arguments
---------
log : `logging.Logger` object
"""
log.warning("AstroCats Setup")
log.warning("Configure filepath: '{}'".format(_CONFIG_PATH))
# Create path to configuration file as needed
config_path_dir = os.path.split(_CONFIG_PATH)[0]
if not os.path.exists(config_path_dir):
log.debug("Creating config directory '{}'".format(
config_path_dir))
os.makedirs(config_path_dir)
if not os.path.isdir(config_path_dir):
log_raise(log, "Configure path error '{}'".format(config_path_dir))
# Determine default settings
# Get this containing directory and use that as default data path
def_base_path = os.path.abspath(os.path.dirname(__file__))
log.warning("Setting '{}' to default path: '{}'".format(
_BASE_PATH_KEY, def_base_path))
config = {_BASE_PATH_KEY: def_base_path}
# Write settings to configuration file
json.dump(config, open(_CONFIG_PATH, 'w'))
if not os.path.exists(def_base_path):
log_raise(log, "Problem creating configuration file.")
return
def setup_user_config(log):
"""Setup a configuration file in the user's home directory.
Currently this method stores default values to a fixed configuration
filename. It should be modified to run an interactive prompt session
asking for parameters (or at least confirming the default ones).
Arguments
---------
log : `logging.Logger` object
"""
log.warning("AstroCats Setup")
log.warning("Configure filepath: '{}'".format(_CONFIG_PATH))
# Create path to configuration file as needed
config_path_dir = os.path.split(_CONFIG_PATH)[0]
if not os.path.exists(config_path_dir):
log.debug("Creating config directory '{}'".format(config_path_dir))
os.makedirs(config_path_dir)
if not os.path.isdir(config_path_dir):
log_raise(log, "Configure path error '{}'".format(config_path_dir))
# Determine default settings
# Get this containing directory and use that as default data path
def_base_path = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
log.warning("Setting '{}' to default path: '{}'".format(_BASE_PATH_KEY,
def_base_path))
config = {_BASE_PATH_KEY: def_base_path}
# Write settings to configuration file
json.dump(config, open(_CONFIG_PATH, 'w'))
if not os.path.exists(def_base_path):
log_raise(log, "Problem creating configuration file.")
return
def do_tremaine_2002(catalog):
"""
"""
log = catalog.log
log.debug("tremaine_2002.do_tremaine_2002()")
task_str = catalog.get_current_task_str()
task_name = catalog.current_task.name
task_dir = catalog.get_current_task_repo()
# Go through each element of the tables
num = 0
data_fname = os.path.join(task_dir, DATA_FILENAME)
log.info("Input filename '{}'".format(data_fname))
if not os.path.exists(data_fname):
utils.log_raise(log, "File not found '{}'".format(data_fname), IOError)
with tqdm.tqdm(desc=task_str, total=31, dynamic_ncols=True) as pbar:
with open(data_fname, 'r') as data:
spamreader = csv.reader(data, delimiter=' ')
for row in spamreader:
if len(row) <= 1 or row[0].startswith('#'):
continue
bh_name = _add_entry_for_data_line(catalog, row)
if bh_name is not None:
log.debug("{}: added '{}'".format(task_name, bh_name))
num += 1
if catalog.args.travis and (num > catalog.TRAVIS_QUERY_LIMIT):
log.warning("Exiting on travis limit")
break
pbar.update(1)
log.info("Added {} entries".format(num))
return
def main():
"""Primary entry point for all AstroCats catalogs.
From this entry point, all internal catalogs can be accessed and their
public methods executed (for example: import scripts).
"""
from datetime import datetime
# Initialize Command-Line and User-Config Settings, Log
# -----------------------------------------------------
beg_time = datetime.now()
# Process command-line arguments to determine action
# If no subcommand (e.g. 'import') is given, returns 'None' --> exit
args, sub_clargs = load_command_line_args()
if args is None:
return
# Create a logging object
log = load_log(args)
# Run configuration/setup interactive script
if args.command == 'setup':
setup_user_config(log)
return
# Make sure configuration file exists, or that's what we're doing
# (with the 'setup' subcommand)
if not os.path.isfile(_CONFIG_PATH):
raise RuntimeError("'{}' does not exist. "
"Run `astrocats setup` to configure."
"".format(_CONFIG_PATH))
git_vers = get_git()
title_str = "Astrocats, version: {}, SHA: {}".format(__version__, git_vers)
log.warning("\n\n{}\n{}\n{}\n".format(title_str, '=' * len(title_str),
beg_time.ctime()))
# Load the user settings from the home directory
args = load_user_config(args, log)
# Choose Catalog and Operation(s) to perform
# ------------------------------------------
mod_name = args.command
log.debug("Importing specified module: '{}'".format(mod_name))
# Try to import the specified module
try:
mod = importlib.import_module('.' + mod_name, package='astrocats')
except Exception as err:
log.error("Import of specified module '{}' failed.".format(mod_name))
log_raise(log, str(err), type(err))
# Run the `main.main` method of the specified module
log.debug("Running `main.main()`")
mod.main.main(args, sub_clargs, log)
end_time = datetime.now()
log.warning("\nAll complete at {}, After {}".format(end_time, end_time -
beg_time))
return
def _add_entry_for_data_line(catalog, line):
"""
Sample Entries:
--------------
000132.83+145608.0|000.386795|+14.935573| 0.3989| 18.898| -23.131| 45.356| 751| 303| 52251
|0|1|1|0| 4026| 2.450| 8.119| 4699| 2.671| 8.114| | |
|8.119| | 10.293|Sloan|DR5|Simbad|NED
000135.51-004206.7|000.397978|-00.701886| 3.5779| 19.183| -27.951| 47.015| 1489| 104| 52991
|0|0|0|1| | | | | | | 6207| 271.312| 9.536
|9.536| | 3.937|Sloan|DR5|Simbad|NED
Columns:
-------
[x] 0 SDSS (A18) SDSS-DR5 Object Designation (HHMMSS.ss+DDMMSS.s, J2000)
[x] 1 RAJ2000 (F10.6) Right Ascension in decimal degrees (J2000)
[x] 2 DEJ2000 (F10.6) Declination in decimal degrees (J2000)
[x] 3 z (F7.4) Redshift
[ ] 4 imag (F7.3) PSF i-band apparent magnitude (1)
[x] 5 iMAG (F8.3) PSF i-band absolute magnitude, K-corrected to z=2
[x] 6 logLbol (F7.3) ? log of bolometric luminosity (erg/s)
[ ] 7 Plate (I5) Plate number
[ ] 8 Fiber (I5) Fiber identification number
[x] 9 ObsDate (I6) MJD date of spectroscopic observation
[ ] 10 F (I1) [0/1] FIRST selection flag
[ ] 11 R (I1) [0/1] ROSAT selection flag
[ ] 12 U (I1) [0/1] Uniform selection flag
[ ] 13 B (I1) [0/1] Broad Absoprtion Lines flag
[x] 14 WHb (I7) ? Full-width at Half-Maximum of H{beta}
[x] 15 L5100 (F9.3) ? Monochromatic luminosity at 5100{AA}
[x] 16 BHHb (F7.3) ? Virial mass of black hole estimated using H{beta} (log scale)
[x] 17 WMgII (I7) ? Full-width at Half-Maximum of MgII
[x] 18 L3000 (F9.3) ? Monochromatic luminosity at 3000{AA} [ucd=phys.luminosity]
[x] 19 BHMgII (F7.3) ? Virial mass of black hole estimated using MgII (log scale)
[x] 20 WCIV (I7) ? Full-width at Half-Maximum of CIV
[x] 21 L1350 (F9.3) ? Monochromatic luminosity at 1350{AA}
[x] 22 BHCIV (F7.3) ? Virial mass of black hole estimated using CIV (log scale)
[x] 23 BHvir (F7.3) ? Virial mass of black hole (3)
[ ] 24 CIVMgII (I7) ? CIV-MgII blueshift
[ ] 25 S N (F8.3) Mean spectrum signal-to-noise ratio
[ ] 26 Sloan (a5) SDSS details from most recent SDSS data release
"""
log = catalog.log
# log.debug("shen_2008._add_entry_for_data_line()")
if len(line) != NUM_COLUMNS:
log.warning("length of line: '{}', expected {}! '{}'".format(
len(line), NUM_COLUMNS, line))
return None
line = [ll.strip() for ll in line]
# [0] SDSS Galaxy/BH Name
# -----------------------
data_name = "SDSS" + line[0]
name = catalog.add_entry(data_name)
# Add this source
source = catalog.entries[name].add_source(
url=SOURCE_URL, bibcode=SOURCE_BIBCODE, name=SOURCE_NAME, secondary=False)
task_name = catalog.current_task.name
catalog.entries[name].add_data(BLACKHOLE.TASKS, task_name)
# [1/2] RA/DEC
catalog.entries[name].add_quantity(BLACKHOLE.RA, line[1], source)
catalog.entries[name].add_quantity(BLACKHOLE.DEC, line[2], source)
# [3] Redshift
catalog.entries[name].add_quantity(BLACKHOLE.REDSHIFT, line[3], source)
# [4] i-band apparent magnitude
# photo_kwargs = {
# PHOTOMETRY.MAGNITUDE: line[4],
# PHOTOMETRY.SOURCE: source,
# PHOTOMETRY.HOST: False,
# PHOTOMETRY.U_LUMINOSITY: 'apparent Magnitude',
# PHOTOMETRY.DESCRIPTION: 'PSF i-band apparent magnitude',
# PHOTOMETRY.BAND: 'i',
# PHOTOMETRY.INCLUDES_HOST: True,
# }
# catalog.entries[name].add_photometry(**photo_kwargs)
# [9] Observation date
# convert from MJD to string YYYY/MM/DD
# obs_date = ap.time.Time(int(line[9]), format='mjd').datetime.strftime('%Y/%m/%d')
# obs_date = float(line[9])
obs_date = line[9]
# [5] i-band Absolute magnitude (z=2)
if len(line[5]):
photo_kwargs = {
PHOTOMETRY.MAGNITUDE: line[5],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.HOST: False,
PHOTOMETRY.U_LUMINOSITY: 'Absolute Magnitude',
PHOTOMETRY.DESCRIPTION: 'PSF i-band absolute magnitude, K-corrected to z=2',
PHOTOMETRY.BAND: 'i',
PHOTOMETRY.INCLUDES_HOST: True,
PHOTOMETRY.KCORRECTED: True,
PHOTOMETRY.TIME: obs_date,
PHOTOMETRY.U_TIME: 'MJD',
}
if not catalog.entries[name].add_photometry(**photo_kwargs):
utils.log_raise(log, "Adding photometry failed for '{}'\n:{}".format(
name, photo_kwargs))
# [6] Luminosity Bolometric
if len(line[6]):
photo_kwargs = {
PHOTOMETRY.LUMINOSITY: line[6],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.HOST: False,
PHOTOMETRY.U_LUMINOSITY: 'log (L/[erg/s])',
PHOTOMETRY.DESCRIPTION: 'Using bolometric corrections in Richards+2006b',
PHOTOMETRY.BAND: 'bolometric',
PHOTOMETRY.INCLUDES_HOST: True,
PHOTOMETRY.TIME: obs_date,
PHOTOMETRY.U_TIME: 'MJD',
}
if not catalog.entries[name].add_photometry(**photo_kwargs):
utils.log_raise(log, "Adding photometry failed for '{}'\n:{}".format(
name, photo_kwargs))
# [14-22] FWHM, Mass, and Monochromatic Luminosities by line
line_names = ["H-Beta", "Mg-II", "C-IV"]
line_waves = ["5100", "3000", "1350"]
line_vars = ["HBETA", "MGII", "CIV"]
for ii, (nn, ww, vv) in enumerate(zip(line_names, line_waves, line_vars)):
line_name = "{} ({} A)".format(nn, ww)
# FWHM for this line
col = 14 + ii*3
if len(line[col]):
var = getattr(BLACKHOLE, "FWHM_" + vv)
desc = "Full-width at Half-Maximum of {}".format(line_name)
quant_kwargs = {QUANTITY.U_VALUE: 'km/s', QUANTITY.DESCRIPTION: desc}
catalog.entries[name].add_quantity(var, line[col], source, **quant_kwargs)
# Luminosity from this line
col = 15 + ii*3
if len(line[col]):
desc = 'Monochromatic luminosity for {} (lambda*L_lambda)'.format(line_name)
photo_kwargs = {
PHOTOMETRY.LUMINOSITY: line[col],
PHOTOMETRY.SOURCE: source,
PHOTOMETRY.HOST: False,
PHOTOMETRY.U_LUMINOSITY: 'log (L/[erg/s])',
PHOTOMETRY.DESCRIPTION: desc,
PHOTOMETRY.WAVELENGTH: ww,
PHOTOMETRY.U_WAVELENGTH: 'Angstrom',
PHOTOMETRY.TIME: obs_date,
PHOTOMETRY.U_TIME: 'MJD',
}
if not catalog.entries[name].add_photometry(**photo_kwargs):
utils.log_raise(log, "Adding photometry failed for '{}' ({}) \n:{}".format(
name, nn, photo_kwargs))
# Mass from this Line
col = 16 + ii*3
if len(line[col]):
var = BLACKHOLE.MASS
mass_method = getattr(BH_MASS_METHODS, "VIR_" + vv)
desc = "Virial mass estimated using {}".format(line_name)
quant_kwargs = {QUANTITY.U_VALUE: 'log(M/Msol)',
QUANTITY.DESCRIPTION: desc,
QUANTITY.KIND: mass_method}
catalog.entries[name].add_quantity(var, line[col], source, **quant_kwargs)
# [23] Mass from optimal line (based on redshift)
val = line[23]
if len(val):
desc = ("Virial BH-Mass Using H-Beta for z < 0.7; "
"Mg-ii for 0.7 < z < 1.9; and C-iv for z > 1.9")
quant_kwargs = {QUANTITY.U_VALUE: 'log(M/Msol)',
QUANTITY.DESCRIPTION: desc,
QUANTITY.KIND: BH_MASS_METHODS.VIR}
catalog.entries[name].add_quantity(BLACKHOLE.MASS, val, source, **quant_kwargs)
return name
def do_wevers_2017(catalog=None):
"""
"""
if catalog is None:
import logging
log = utils.get_logger(stream_level=logging.DEBUG)
task_str = "Wevers+2017"
task_name = task_str
task_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir))
task_dir = os.path.join(task_dir, "input", "internal", "")
log.warning("Running task '{}' directly".format(task_str))
else:
log = catalog.log
log.debug("shen_2008.do_shen_2008()")
task_str = catalog.get_current_task_str()
task_name = catalog.current_task.name
task_dir = catalog.get_current_task_repo()
# Go through each element of the tables
num = 0
line_num = 0
count = 0
data_fname = os.path.join(task_dir, DATA_FILENAME)
log.info("Input filename '{}'".format(data_fname))
if not os.path.exists(data_fname):
utils.log_raise(log, "File not found '{}'".format(data_fname), IOError)
import astropy as ap
# with open(data_fname, 'r') as indata:
# data = indata.read()
#
# print(data)
# data = ap.table.Table.read(data.encode('unicode-escape'), format='ascii.latex')
# print(data)
data = ap.table.Table.read(data_fname) # , format='ascii.latex')
print(data)
return data
with tqdm.tqdm(desc=task_str, total=EXPECTED_TOTAL, dynamic_ncols=True) as pbar:
with open(data_fname, 'r') as data:
spamreader = csv.reader(data, delimiter='|')
for row in spamreader:
line_num += 1
if len(row) == 0 or row[0].startswith('#'):
continue
else:
count += 1
if count < 4:
continue
bh_name = _add_entry_for_data_line(catalog, row)
if bh_name is not None:
log.debug("{}: added '{}'".format(task_name, bh_name))
num += 1
if (JOURNAL_INTERNAL is not None) and (num % JOURNAL_INTERNAL == 0):
catalog.journal_entries()
if catalog.args.travis and (num > catalog.TRAVIS_QUERY_LIMIT):
log.warning("Exiting on travis limit")
break
pbar.update(1)
log.info("Added {} entries".format(num))
if (num != EXPECTED_TOTAL) and (not catalog.args.travis):
log.warning("Number of entries added {} does not match expectation {}!".format(
num, EXPECTED_TOTAL))
return