def conf_logs(request):
""" Configure logging for the testing session. """
level = request.config.getoption("--logging-level")
init_logger(name=_LOGNAME, level=level, devmode=True)
logging.getLogger(_LOGNAME).info(
"Configured looper logger at level %s; attaching oldtests' logger %s",
str(level), __name__)
global _LOGGER
_LOGGER = logging.getLogger("looper.{}".format(__name__))
def interactive(prj_lines=PROJECT_CONFIG_LINES,
iface_lines=PIPELINE_INTERFACE_CONFIG_LINES,
merge_table_lines=MERGE_TABLE_LINES,
annotation_lines=SAMPLE_ANNOTATION_LINES,
project_kwargs=None,
logger_kwargs=None):
"""
Create Project and PipelineInterface instances from default or given data.
This is intended to provide easy access to instances of fundamental looper
object for interactive test-authorship-motivated work in an iPython
interpreter or Notebook. Test authorship is simplified if we provide
easy access to viable instances of these objects.
:param Iterable[str] prj_lines: project config lines
:param Iterable[str] iface_lines: pipeline interface config lines
:param Iterable[str] merge_table_lines: lines for a merge table file
:param Iterable[str] annotation_lines: lines for a sample annotations file
:param str | int loglevel: level at which to attend to log messages
:param dict project_kwargs: keyword arguments for Project constructor
:param dict logger_kwargs: keyword arguments for logging configuration
:param bool devmode: whether logging should be done in development mode;
this implies a more verbose level setting and a more information-rich
template for logging message formatting
:param str logfile: path to file to which to write logging messages
:return Project, PipelineInterface: one Project and one PipelineInterface,
"""
# Establish logging for interactive session.
looper_logger_kwargs = {"level": "DEBUG", "name": "looper"}
looper_logger_kwargs.update(logger_kwargs or {})
init_logger(**looper_logger_kwargs)
# TODO: don't work with tempfiles once ctors tolerate Iterable.
dirpath = tempfile.mkdtemp()
path_conf_file = write_temp(prj_lines,
dirpath=dirpath,
fname=P_CONFIG_FILENAME)
path_iface_file = write_temp(iface_lines,
dirpath=dirpath,
fname="pipeline_interface.yaml")
path_merge_table_file = write_temp(merge_table_lines,
dirpath=dirpath,
fname=MERGE_TABLE_FILENAME)
path_sample_annotation_file = write_temp(annotation_lines,
dirpath=dirpath,
fname=ANNOTATIONS_FILENAME)
prj = Project(path_conf_file, **(project_kwargs or {}))
iface = PipelineInterface(path_iface_file)
for path in [
path_conf_file, path_iface_file, path_merge_table_file,
path_sample_annotation_file
]:
os.unlink(path)
return prj, iface
def conf_logs(request):
""" Configure logging for the testing session. """
level = request.config.getoption("--logging-level")
logname = "peppy"
init_logger(name=logname, level=level, devmode=True)
logging.getLogger(logname).info(
"Configured pep logger at level %s; attaching tests' logger %s",
str(level), __name__)
global _LOGGER
_LOGGER = logging.getLogger("peppy.{}".format(__name__))
def est_log(**kwargs):
"""
Establish logging, e.g. for an interactive session.
:param dict kwargs: keyword arguments for logger setup.
:return logging.Logger: looper logger
"""
kwds = copy.copy(kwargs)
if "name" in kwds:
print("Ignoring {} and setting fixed values for logging names".format(
kwds["name"]))
del kwds["name"]
init_logger(name="peppy", **kwds)
return init_logger(name="looper", **kwds)
def interactive(prj_lines=PROJECT_CONFIG_LINES,
iface_lines=PIPELINE_INTERFACE_CONFIG_LINES,
sample_subannotation_lines=SAMPLE_SUBANNOTATION_LINES,
annotation_lines=SAMPLE_ANNOTATION_LINES,
project_kwargs=None,
logger_kwargs=None):
"""
Create Project instance from default or given data.
This is intended to provide easy access to instances of fundamental pep
object for interactive test-authorship-motivated work in an iPython
interpreter or Notebook. Test authorship is simplified if we provide
easy access to viable instances of these objects.
:param Iterable[str] prj_lines: project config lines
:param Iterable[str] iface_lines: pipeline interface config lines
:param Iterable[str] sample_subannotation_lines: lines for a merge table file
:param Iterable[str] annotation_lines: lines for a sample annotations file
:param dict project_kwargs: keyword arguments for Project constructor
:param dict logger_kwargs: keyword arguments for logging configuration
:return Project: configured Project
"""
# Establish logging for interactive session.
pep_logger_kwargs = {"level": "DEBUG", "name": "peppy"}
pep_logger_kwargs.update(logger_kwargs or {})
init_logger(**pep_logger_kwargs)
# TODO: don't work with tempfiles once ctors tolerate Iterable.
dirpath = tempfile.mkdtemp()
path_conf_file = _write_temp(prj_lines,
dirpath=dirpath,
fname=P_CONFIG_FILENAME)
path_iface_file = _write_temp(iface_lines,
dirpath=dirpath,
fname="pipeline_interface.yaml")
path_sample_subannotation_file = _write_temp(sample_subannotation_lines,
dirpath=dirpath,
fname=SUBSAMPLES_FILENAME)
path_sample_annotation_file = _write_temp(annotation_lines,
dirpath=dirpath,
fname=ANNOTATIONS_FILENAME)
prj = Project(path_conf_file, **(project_kwargs or {}))
for path in [
path_conf_file, path_iface_file, path_sample_subannotation_file,
path_sample_annotation_file
]:
os.unlink(path)
return prj
def test_logfile(tmpdir, filename):
""" Validate file handler setting for created logger. """
fp = tmpdir.join(filename).strpath
log = init_logger(logfile=fp)
assert 1 == len(log.handlers)
h = _check_hdlr_kind(log, logging.FileHandler)
assert fp == h.stream.name
def test_logfile_and_stream(filename, stream, tmpdir):
""" Logging can be both stream and file. """
fp = tmpdir.join(filename).strpath
log = init_logger(logfile=fp, stream=stream)
assert 2 == len(log.handlers)
fh = _check_hdlr_kind(log, logging.FileHandler)
sh = _check_hdlr_kind(log, logging.StreamHandler, omit=logging.FileHandler)
assert fp == fh.stream.name
assert stream == sh.stream
def main():
""" Primary workflow """
parser = build_argparser()
args, remaining_args = parser.parse_known_args()
if args.command is None:
parser.print_help(sys.stderr)
sys.exit(1)
# Set the logging level.
if args.dbg:
# Debug mode takes precedence and will listen for all messages.
level = args.logging_level or logging.DEBUG
elif args.verbosity is not None:
# Verbosity-framed specification trumps logging_level.
level = _LEVEL_BY_VERBOSITY[args.verbosity]
else:
# Normally, we're not in debug mode, and there's not verbosity.
level = LOGGING_LEVEL
logger_kwargs = {"level": level, "devmode": args.dbg}
init_logger(name="peppy", **logger_kwargs)
global _LOGGER
_LOGGER = init_logger(name=PKG_NAME, **logger_kwargs)
if args.command == "convert":
_LOGGER.debug("Creating a Project object from: {}".format(args.pep))
p = Project(args.pep)
convert_project(p, args.format)
_LOGGER.info("Conversion successful")
sys.exit(0)
if args.command == "list":
list_formats()
sys.exit(0)
sys.exit(0)
def test_all_defaults(attr, check):
""" Check the values on the logger that result from all default arguments. """
logger = init_logger()
if hasattr(check, "__call__"):
fails = list(
itertools.chain(*[check(obj) for obj in getattr(logger, attr)]))
if fails:
pytest.fail("Failures:\n{}".format("\n".join(fails)))
else:
try:
exp, err_fmt = check
except ValueError:
exp, err_fmt, transform = check
else:
transform = None
obs = getattr(logger, attr)
obs = transform(obs) if transform else obs
assert exp == obs, err_fmt.format(exp, obs)
def main():
"""Primary workflow"""
parser = logmuse.add_logging_options(build_argparser())
# args, remaining_args = parser.parse_known_args()
args = parser.parse_args()
logger_kwargs = {"level": args.verbosity, "devmode": args.logdev}
logmuse.init_logger("yacman", **logger_kwargs)
global _LOGGER
_LOGGER = logmuse.logger_via_cli(args)
if not args.command:
parser.print_help()
_LOGGER.error("No command given")
sys.exit(1)
if args.command == "init":
divcfg = args.config
_LOGGER.debug("Initializing divvy configuration")
is_writable(os.path.dirname(divcfg), check_exist=False)
divvy_init(divcfg, DEFAULT_CONFIG_FILEPATH)
sys.exit(0)
_LOGGER.debug("Divvy config: {}".format(args.config))
divcfg = select_divvy_config(args.config)
_LOGGER.info("Using divvy config: {}".format(divcfg))
dcc = ComputingConfiguration(filepath=divcfg)
if args.command == "list":
# Output header via logger and content via print so the user can
# redirect the list from stdout if desired without the header as clutter
_LOGGER.info("Available compute packages:\n")
print("{}".format("\n".join(dcc.list_compute_packages())))
sys.exit(1)
# Any non-divvy arguments will be passed along as key-value pairs
# that can be used to populate the template.
# keys = [str.replace(x, "--", "") for x in remaining_args[::2]]
# cli_vars = dict(zip(keys, remaining_args[1::2]))
if args.compute:
cli_vars = {y[0]: y[1] for y in [x.split("=") for x in args.compute]}
else:
cli_vars = {}
if args.command == "write" or args.command == "submit":
try:
dcc.activate_package(args.package)
except AttributeError:
parser.print_help(sys.stderr)
sys.exit(1)
if args.settings:
_LOGGER.info("Loading settings file: %s", args.settings)
with open(args.settings, "r") as f:
vars_groups = [cli_vars, yaml.load(f, SafeLoader)]
else:
vars_groups = [cli_vars]
_LOGGER.debug(vars_groups)
if args.command == "write":
dcc.write_script(args.outfile, vars_groups)
elif args.command == "submit":
dcc.submit(args.outfile, vars_groups)
# Project configuration, particularly for logging.
import logmuse
from ._version import __version__
from .henge import *
__classes__ = ["Henge"]
__all__ = __classes__ + ["connect_mongo", "split_schema", "NotFoundException"]
logmuse.init_logger("henge", propagate=True)
# Project configuration, particularly for logging.
import logmuse
from ._version import __version__
__classes__ = []
__all__ = __classes__ + []
logmuse.init_logger("bulker")
# Project configuration, particularly for logging.
import logmuse
from ._version import __version__
__classes__ = []
__all__ = __classes__ + []
logmuse.init_logger("packagename")
import logmuse
logmuse.init_logger("seqcolapi")
def main():
""" Primary workflow """
parser = logmuse.add_logging_options(build_argparser())
args, remaining_args = parser.parse_known_args()
logger_kwargs = {"level": args.verbosity, "devmode": args.logdev}
logmuse.init_logger(name="yacman", **logger_kwargs)
global _LOGGER
_LOGGER = logmuse.logger_via_cli(args)
_LOGGER.debug("Command given: {}".format(args.command))
if not args.command:
parser.print_help()
_LOGGER.error("No command given")
sys.exit(1)
if args.command == "init":
bulkercfg = args.config
_LOGGER.debug("Initializing bulker configuration")
_is_writable(os.path.dirname(bulkercfg), check_exist=False)
bulker_init(bulkercfg, DEFAULT_CONFIG_FILEPATH, args.engine)
sys.exit(0)
bulkercfg = select_bulker_config(args.config)
bulker_config = yacman.YacAttMap(filepath=bulkercfg, writable=False)
if args.command == "list":
# Output header via logger and content via print so the user can
# redirect the list from stdout if desired without the header as clutter
if args.simple:
fmt = "{namespace}/{crate}:{tag}"
else:
_LOGGER.info("Available crates:")
fmt = "{namespace}/{crate}:{tag} -- {path}"
if bulker_config.bulker.crates:
for namespace, crates in bulker_config.bulker.crates.items():
for crate, tags in crates.items():
for tag, path in tags.items():
print(
fmt.format(namespace=namespace,
crate=crate,
tag=tag,
path=path))
else:
_LOGGER.info(
"No crates available. Use 'bulker load' to load a crate.")
sys.exit(1)
# For all remaining commands we need a crate identifier
_LOGGER.info("Bulker config: {}".format(bulkercfg))
if args.command == "activate":
try:
cratelist = parse_registry_paths(
args.crate_registry_paths,
bulker_config.bulker.default_namespace)
_LOGGER.debug(cratelist)
_LOGGER.info("Activating bulker crate: {}{}".format(
args.crate_registry_paths, " (Strict)" if args.strict else ""))
bulker_activate(bulker_config,
cratelist,
echo=args.echo,
strict=args.strict,
prompt=args.no_prompt)
except KeyError as e:
parser.print_help(sys.stderr)
_LOGGER.error("{} is not an available crate".format(e))
sys.exit(1)
except MissingCrateError as e:
_LOGGER.error("Missing crate: {}".format(e))
sys.exit(1)
except AttributeError as e:
_LOGGER.error(
"Your bulker config file is outdated, you need to re-initialize it: {}"
.format(e))
sys.exit(1)
if args.command == "run":
try:
cratelist = parse_registry_paths(args.crate_registry_paths)
_LOGGER.info("Activating crate: {}\n".format(
args.crate_registry_paths))
bulker_run(bulker_config, cratelist, args.cmd, strict=args.strict)
except KeyError as e:
parser.print_help(sys.stderr)
_LOGGER.error("{} is not an available crate".format(e))
sys.exit(1)
except MissingCrateError as e:
_LOGGER.error("Missing crate: {}".format(e))
sys.exit(1)
if args.command == "load":
bulker_config.make_writable()
manifest, cratevars = load_remote_registry_path(
bulker_config, args.crate_registry_paths, args.manifest)
exe_template_jinja = None
build_template_jinja = None
shell_template_jinja = None
exe_template = mkabs(bulker_config.bulker.executable_template,
os.path.dirname(bulker_config._file_path))
build_template = mkabs(bulker_config.bulker.build_template,
os.path.dirname(bulker_config._file_path))
try:
shell_template = mkabs(bulker_config.bulker.shell_template,
os.path.dirname(bulker_config._file_path))
except AttributeError:
_LOGGER.error(
"You need to re-initialize your bulker config or add a 'shell_template' attribute."
)
sys.exit(1)
try:
assert (os.path.exists(exe_template))
except AssertionError:
_LOGGER.error(
"Bulker config points to a missing executable template: {}".
format(exe_template))
sys.exit(1)
with open(exe_template, 'r') as f:
# with open(DOCKER_TEMPLATE, 'r') as f:
contents = f.read()
exe_template_jinja = jinja2.Template(contents)
try:
assert (os.path.exists(shell_template))
except AssertionError:
_LOGGER.error(
"Bulker config points to a missing shell template: {}".format(
shell_template))
sys.exit(1)
with open(shell_template, 'r') as f:
# with open(DOCKER_TEMPLATE, 'r') as f:
contents = f.read()
shell_template_jinja = jinja2.Template(contents)
if args.build:
try:
assert (os.path.exists(build_template))
except AssertionError:
_LOGGER.error(
"Bulker config points to a missing build template: {}".
format(build_template))
sys.exit(1)
_LOGGER.info(
"Building images with template: {}".format(build_template))
with open(build_template, 'r') as f:
contents = f.read()
build_template_jinja = jinja2.Template(contents)
bulker_load(manifest,
cratevars,
bulker_config,
exe_jinja2_template=exe_template_jinja,
shell_jinja2_template=shell_template_jinja,
crate_path=args.path,
build=build_template_jinja,
force=args.force)
if args.command == "inspect":
if args.crate_registry_paths == "":
_LOGGER.error(
"No active create. Inspect requires a provided crate, or a currently active create."
)
sys.exit(1)
manifest, cratevars = load_remote_registry_path(
bulker_config, args.crate_registry_paths, None)
manifest_name = cratevars['crate']
print("Bulker manifest: {}".format(args.crate_registry_paths))
crate_path = os.path.join(bulker_config.bulker.default_crate_folder,
cratevars['namespace'], manifest_name,
cratevars['tag'])
if not os.path.isabs(crate_path):
crate_path = os.path.join(os.path.dirname(bcfg._file_path),
crate_path)
print("Crate path: {}".format(crate_path))
import glob
filenames = glob.glob(os.path.join(crate_path, "*"))
available_commands = [
x for x in [os.path.basename(x) for x in filenames] if x[0] != "_"
]
print("Available commands: {}".format(available_commands))
python -c "import logmuse print(logmuse.LEVEL_BY_VERBOSITY)" parser = logmuse.add_logmuse_args(parser) args = parser.parse_args() lmargs = logmuse.retrieve_logmuse_args(args) global _LOGGER _LOGGER = logmuse.init_logger(lmargs) logmuse.init_logger(name="pararead", lmargs) non-CLI packages like pypiper and pararead should not need to know about logmuse, but when a CLI tool (like a pipeline, or a count_reads tool) imports one of these with a logger, and uses logmuse, it should "just work". file = "../microtest/data/bs_aln_k1k3.bam"
import logmuse
from ._version import __version__
logmuse.init_logger("refgenie")
# Project configuration, particularly for logging.
import logmuse
from ._version import __version__
from .bbconf import *
from .const import *
__classes__ = ["BedBaseConf"]
__all__ = __classes__ + ["get_bedbase_cfg"]
logmuse.init_logger("bbconf")
# Project configuration, particularly for logging.
import logmuse
from ._version import __version__
from .pipestat import *
from .helpers import *
__classes__ = ["PipestatManager"]
__all__ = __classes__
logmuse.init_logger("pipestat")
# Project configuration, particularly for logging. import logmuse from ._version import __version__ from .helpers import * from .pipestat import * __classes__ = ["PipestatManager"] __all__ = __classes__ logmuse.init_logger(PKG_NAME)
def test_propagate(kwargs, exp):
""" Determination of propagation flag considers root status and propagation. """
assert init_logger(**kwargs).propagate is exp
def test_stream(stream, exp):
""" Validate stream handler setting for created logger. """
log = init_logger(stream=stream)
assert 1 == len(log.handlers)
h = _check_hdlr_kind(log, logging.StreamHandler)
assert exp == h.stream
"""Project configuration, particularly for logging.
Project-scope constants may reside here, but more importantly, some setup here
will provide a logging infrastructure for all of the project's modules.
Individual modules and classes may provide separate configuration on a more
local level, but this will at least provide a foundation.
"""
import logmuse
from ._version import __version__
from .compute import ComputingConfiguration, select_divvy_config
from .const import *
from .utils import write_submit_script
__classes__ = ["ComputingConfiguration"]
__functions__ = ["select_divvy_config"]
__all__ = __classes__ + __functions__ + [write_submit_script.__name__]
logmuse.init_logger("divvy")
def test_single_attr(att, val):
""" Test successful setting of a simple, particular logger attribute. """
assert val == getattr(init_logger(**{att: val}), att)
# Project configuration, particularly for logging.
import logmuse
from .bedshift import Bedshift
from ._version import __version__
__classes__ = ["Bedshift"]
__all__ = __classes__ + []
logmuse.init_logger("bedshift")
def test_make_non_root_name_root():
""" Non-root name for root logger is prohibited. """
with pytest.raises(ValueError):
init_logger("root", make_root=False)
def test_make_root(make_root, exp):
""" Root status for logger has a couple of implications. """
log = init_logger(make_root=make_root)
assert exp == log.name
assert log.propagate is False
def main():
""" Primary workflow """
global _LOGGER
parsers = build_parser()
parser = parsers[0]
aux_parser = parsers[1]
aux_parser.suppress_defaults()
args, remaining_args = parser.parse_known_args()
if args.command is None:
parser.print_help(sys.stderr)
sys.exit(1)
if args.config_file is None:
m = "No project config defined"
try:
setattr(args, "config_file", read_cfg_from_dotfile())
except OSError:
print(m + " and dotfile does not exist: {}".format(dotfile_path()))
parser.print_help(sys.stderr)
sys.exit(1)
else:
print(m + ", using: {}. Read from dotfile ({}).".format(
read_cfg_from_dotfile(), dotfile_path()))
if args.command == "init":
sys.exit(
int(not init_dotfile(dotfile_path(), args.config_file, args.force))
)
args = enrich_args_via_cfg(args, aux_parser)
# Set the logging level.
if args.dbg:
# Debug mode takes precedence and will listen for all messages.
level = args.logging_level or logging.DEBUG
elif args.verbosity is not None:
# Verbosity-framed specification trumps logging_level.
level = _LEVEL_BY_VERBOSITY[args.verbosity]
else:
# Normally, we're not in debug mode, and there's not verbosity.
level = LOGGING_LEVEL
# Establish the project-root logger and attach one for this module.
logger_kwargs = {
"level": level,
"logfile": args.logfile,
"devmode": args.dbg
}
init_logger(name="peppy", **logger_kwargs)
init_logger(name="divvy", **logger_kwargs)
init_logger(name="eido", **logger_kwargs)
_LOGGER = init_logger(name=_PKGNAME, **logger_kwargs)
# lc = LooperConfig(select_looper_config(filename=args.looper_config))
# _LOGGER.debug("Determined genome config: {}".format(lc))
_LOGGER.info("Looper version: {}\nCommand: {}".format(
__version__, args.command))
if len(remaining_args) > 0:
_LOGGER.warning("Unrecognized arguments: {}".format(" ".join(
[str(x) for x in remaining_args])))
divcfg = select_divvy_config(filepath=args.divvy) \
if hasattr(args, "divvy") else None
# Initialize project
_LOGGER.debug("Building Project")
try:
p = Project(config_file=args.config_file,
amendments=args.amend,
divcfg_path=divcfg,
runp=args.command == "runp",
**{
attr: getattr(args, attr)
for attr in CLI_PROJ_ATTRS if attr in args
})
except yaml.parser.ParserError as e:
_LOGGER.error("Project config parse failed -- {}".format(e))
sys.exit(1)
selected_compute_pkg = p.selected_compute_package \
or DEFAULT_COMPUTE_RESOURCES_NAME
if p.dcc is not None and not p.dcc.activate_package(selected_compute_pkg):
_LOGGER.info("Failed to activate '{}' computing package. "
"Using the default one".format(selected_compute_pkg))
with ProjectContext(prj=p,
selector_attribute=args.sel_attr,
selector_include=args.sel_incl,
selector_exclude=args.sel_excl) as prj:
if args.command in ["run", "rerun"]:
run = Runner(prj)
try:
compute_kwargs = _proc_resources_spec(args)
run(args, rerun=(args.command == "rerun"), **compute_kwargs)
except IOError:
_LOGGER.error("{} pipeline_interfaces: '{}'".format(
prj.__class__.__name__, prj.pipeline_interface_sources))
raise
if args.command == "runp":
compute_kwargs = _proc_resources_spec(args)
collate = Collator(prj)
collate(args, **compute_kwargs)
if args.command == "destroy":
return Destroyer(prj)(args)
if args.command == "table":
Table(prj)()
if args.command == "report":
Report(prj)(args)
if args.command == "check":
Checker(prj)(flags=args.flags)
if args.command == "clean":
return Cleaner(prj)(args)
if args.command == "inspect":
inspect_project(p, args.snames, args.attr_limit)
