def main():
"""Parse command line options and set up an Where analysis
Do simple parsing of command line arguments. Set up config-files and show the configuration.
"""
util.check_help_and_version(doc_module=__name__)
# Start logging
log.init()
# Read command line options
pipeline = pipelines.get_from_options()
config.read_pipeline(pipeline)
if util.check_options("--doy"):
rundate = util.parse_args("doy", doc_module=__name__)
else:
rundate = util.parse_args("date", doc_module=__name__)
session = pipelines.get_session(rundate, pipeline)
# Set up the configuration for the analysis
setup_config(rundate, pipeline, session)
# Show current configuration
show_config(rundate, pipeline, session)
# Store configuration in library
store_config_to_library(rundate, pipeline, session)
def main():
"""Parse command line options and run the Where analysis
Do simple parsing of command line arguments. Set up config-files and start the analysis. See the help docstring at
the top of the file for more information about the workflow.
"""
# Start logging
log.init()
# Read command line options
if util.check_options("--doy"):
rundate = util.parse_args("doy", doc_module=__name__)
else:
rundate = util.parse_args("date", doc_module=__name__)
pipeline = pipelines.get_from_options()
session = pipelines.get_session(rundate, pipeline)
# Pretend to empty mailbox
pretend_to_empty_mailbox()
# Start an interactive session
if util.check_options("-I", "--interactive"):
from where.tools import interactive
interactive.interactive(rundate, pipeline, session)
return
# Set up the configuration for a new analysis or update an existing one
setup.setup_config(rundate, pipeline, session)
# Run the analysis
setup.add_timestamp(rundate, pipeline, session, "last run")
with timer(f"Finish pipeline {pipeline.upper()} in"):
pipelines.run(rundate, pipeline, session)
def main():
"""Invoke where_tools
To add a new tool, simply add a new .py-file with a registered plugin that
will be called when the tool is called.
"""
# Start logging
log.init(log_level="info")
# First read tool from command line, don't use util.parse_args() as that overrides -h for each tool
try:
tool = [a for a in sys.argv[1:] if not a.startswith("-")][0]
sys.argv.remove(tool)
except IndexError:
util._print_help_from_doc(__name__)
raise SystemExit
# Check that tool is available and figure out signature
try:
sig = plugins.signature(__name__, tool)
tool_module = plugins.get(__name__, tool).function.__module__
except exceptions.UnknownPluginError as err:
util._print_help_from_doc(__name__)
err.args = (
f"{err.args[0]}\n Available tools are {', '.join(plugins.names(__name__))}",
)
raise
# Parse parameters
util.check_help_and_version(doc_module=tool_module)
tool_args = dict()
for key, param in sig.parameters.items():
if param.annotation is None:
raise SystemExit(f"{param} in {tool} tool is not annotated")
if param.annotation == "datedoy":
if util.check_options("--doy"):
date = util.parse_args("doy", doc_module=__name__)
else:
date = util.parse_args("date", doc_module=__name__)
tool_args[key] = date
elif param.annotation == "pipeline":
tool_args[key] = pipelines.get_from_options()
config.read_pipeline(tool_args[key])
elif param.annotation == "option":
tool_args[key] = util.read_option_value(f"--{key}")
else:
tool_args[key] = util.parse_args(param.annotation,
doc_module=tool_module)
# Call tool
plugins.call(__name__, tool, **tool_args)
def main():
"""Parse command line options and run the Where analysis
Do simple parsing of command line arguments. Set up config-files and start the analysis. See the help docstring at
the top of the file for more information about the workflow.
"""
util.check_help_and_version(doc_module=__name__)
# Start logging
log.init(config.where.log.default_level.str)
log.debug(
f"Use {util.get_python_version()} on process {util.get_pid_and_server()}"
)
# Read command line options
pipeline = pipelines.get_from_options()
config.read_pipeline(pipeline)
if util.check_options("--doy"):
rundate = util.parse_args("doy", doc_module=__name__)
else:
rundate = util.parse_args("date", doc_module=__name__)
args, kwargs = util.options2args(sys.argv[1:])
# Start an interactive session
if util.check_options("-I", "--interactive"):
from where.tools import interactive # Local import because interactive imports many external packages
interactive.interactive(rundate, pipeline, **kwargs)
return
# Set up the configuration for a new analysis or update an existing one
unused_options = setup.setup_config(rundate, pipeline, *args, **kwargs)
pipeline_args, pipeline_kwargs = util.options2args(unused_options)
# Run the analysis
setup.add_timestamp(rundate, pipeline, "last run", **kwargs)
with Timer(f"Finish pipeline {pipeline.upper()} in"):
pipelines.run(rundate, pipeline, *pipeline_args, **pipeline_kwargs)
def main():
"""Parse command line options and loop over the Where analysis
Do simple parsing of command line arguments. Set up config-files and potentially start the analysis. See the help
docstring at the top of the file for more information about the workflow.
"""
util.check_help_and_version(doc_module=__name__)
log.init(log_level=config.where.log.default_level.str, prefix="Runner")
# Initialize
pipeline = pipelines.get_from_options()
config.read_pipeline(pipeline)
if util.check_options("--doy"):
from_date = util.parse_args("doy", doc_module=__name__)
to_date = util.parse_args("doy", doc_module=__name__)
sys.argv.remove("--doy")
else:
from_date = util.parse_args("date", doc_module=__name__)
to_date = util.parse_args("date", doc_module=__name__)
# Handle list of sessions
session_list = set(
util.read_option_value("--session", default="").replace(",",
" ").split())
sys.argv = [o for o in sys.argv if not o.startswith("--session=")]
# Start logging
file_vars = dict(**util.get_user_info())
log.file_init(
file_path=files.path("log_runner", file_vars=file_vars),
log_level=config.where.log.default_level.str,
prefix="Runner",
rotation=config.where.log.number_of_log_backups.int,
)
atexit.register(log_statistics)
# Should where_runner crash if Where crashes?
stop_on_error_opts = None
if util.check_options("--stop-on-error"):
stop_on_error_opts = True
elif util.check_options("--continue-on-error"):
stop_on_error_opts = False
stop_on_error = config.where.get("stop_on_error",
section="runner",
value=stop_on_error_opts).bool
error_logger = log.fatal if stop_on_error else log.error
# Loop over dates
rundate = from_date
while rundate <= to_date:
available_sessions = set(pipelines.list_sessions(rundate, pipeline))
sessions = available_sessions & session_list if session_list else available_sessions
where_args = remove_runner_args(sys.argv[1:])
for session in sorted(sessions):
cmd = f"{where.__executable__} {rundate:%Y %m %d} --session={session}".split(
) + where_args
log.info(f"Running '{' '.join(cmd)}'")
count("Number of analyses")
try:
subprocess.run(cmd,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except subprocess.CalledProcessError as err:
count("Failed analyses")
error_msg = err.stderr.decode().strip().split("\n")[-1]
error_logger(f"Command '{' '.join(cmd)}' failed: {error_msg}")
else:
count("Successful analyses")
copy_log_from_where(rundate, pipeline, session)
rundate += timedelta(days=1)
