def get_stimela_logger():
"""Returns Stimela's logger, or None if no Stimela installed"""
try:
import stimela
return stimela.logger()
except ImportError:
return None
def __init__(self,
indir=None, # input directory
outdir=None, # output directory
msdir=None, # MS directory
parameter_file=None,
task=None,
base=None,
binary=None,
description=None,
tag=None,
prefix=None,
parameters=[],
version=None):
self.indir = indir
self.outdir = outdir
if parameter_file:
cab = utils.readJson(parameter_file)
self.task = cab["task"]
self.base = cab["base"]
self.binary = cab["binary"]
self.tag = cab["tag"]
self.version = cab.get("version", "x.x.x")
if cab["msdir"]:
self.msdir = msdir
else:
self.msdir = None
self.description = cab["description"]
self.prefix = cab["prefix"]
parameters0 = cab["parameters"]
self.parameters = []
for param in parameters0:
default = param.get("default", param.get("value", None))
addme = Parameter(name=param["name"],
dtype=param["dtype"],
io=param.get("io", None),
info=param.get(
"info", None) or "No documentation. Bad! Very bad...",
default=default,
mapping=param.get("mapping", None),
required=param.get("required", False),
choices=param.get("choices", False),
check_io=param.get("check_io", True))
self.parameters.append(addme)
else:
self.task = task
self.base = base
self.binary = binary
self.prefix = prefix
self.parameters = parameters
self.description = description
self.msdir = msdir
self.tag = tag
self.version = version
self.log = stimela.logger()
def create_logger():
""" Creates logger and associated objects. Called upon import"""
global log, log_filehandler
log.setLevel(logging.DEBUG)
log.propagate = False
# init stimela logger as a sublogger
if stimela.is_logger_initialized():
raise RuntimeError("Stimela logger already initialized. This is a bug: you must have an incompatible version of Stimela.")
stimela.logger(STIMELA_LOGGER_NAME, propagate=True, console=False)
log_filehandler = DelayedFileHandler()
log_filehandler.setFormatter(stimela.log_boring_formatter)
log_filehandler.setLevel(logging.INFO)
log.addHandler(log_filehandler)
def pull(image, name, docker=True, directory=".", force=False):
"""
pull an image
"""
if docker:
fp = "docker://{0:s}".format(image)
else:
fp = image
if not os.path.exists(directory):
os.mkdir(directory)
image_path = os.path.abspath(os.path.join(directory, name))
if os.path.exists(image_path) and not force:
stimela.logger().info(f"Singularity image already exists at '{image_path}'. To replace it, please re-run with the 'force' option")
else:
utils.xrun(f"cd {directory} && singularity", ["pull",
"--force" if force else "", "--name",
name, fp])
return 0
def setup_job_log(self, log_name=None):
""" set up a log for the job on the host side
log_name: preferably unique name for this jobs log
log_dir: log base directory, None is current directory
"""
log_name = log_name or self.name
if log_name not in StimelaJob.logs_avail:
self.log = stimela.logger().getChild(log_name)
if self.logfile is not False:
log_dir = os.path.dirname(self.logfile) or "."
if not os.path.exists(log_dir):
os.mkdir(log_dir)
fh = logging.FileHandler(self.logfile, 'w', delay=True)
fh.setLevel(logging.INFO)
self.log.addHandler(fh)
self.log.propagate = True # propagate also to main stimela logger
StimelaJob.logs_avail[log_name] = self.log
else:
self.log = StimelaJob.logs_avail[log_name]
def __init__(self, name, data=None,
parameter_file_dir=None, ms_dir=None,
build_label=None,
singularity_image_dir=None, JOB_TYPE='docker',
cabpath=None,
logger=None,
msdir=None,
indir=None,
outdir=None,
log_dir=None, logfile=None, logfile_task=None,
cabspecs=None,
loglevel="INFO"):
"""
Deifine and manage a stimela recipe instance.
name : Name of stimela recipe
msdir : Path of MSs to be used during the execution of the recipe
parameter_file_dir : Will store task specific parameter files here
logger: if set to a logger object, uses the specified logger.
if None, sets up its own logger using the parameters below
loglevel: default logging level
log_dir: default directory for logfiles
logfile: name of logfile, False to disable recipe-level logfiles, or None to form a default name
logfile_task: name of task-level logfile, False to disable task-level logfiles, or None to form a default name.
logfile_task may contain a "{task}" entry which will be substituted for a task name.
"""
self.name = name
self.name_ = re.sub(r'\W', '_', name) # pausterized name
self.stimela_context = inspect.currentframe().f_back.f_globals
self.stimela_path = os.path.dirname(docker.__file__)
# Update I/O with values specified on command line
self.indir = indir
self.outdir = outdir
self.msdir = self.ms_dir = msdir or ms_dir
self.loglevel = self.stimela_context.get('_STIMELA_LOG_LEVEL', None) or loglevel
self.JOB_TYPE = self.stimela_context.get('_STIMELA_JOB_TYPE', None) or JOB_TYPE
self.cabpath = cabpath
self.cabspecs = cabspecs or {}
# set default name for task-level logfiles
self.logfile_task = "{0}/log-{1}-{{task}}".format(log_dir or ".", self.name_) \
if logfile_task is None else logfile_task
self._log_fh = None
if logger is not None:
self.log = logger
else:
logger = stimela.logger(loglevel=self.loglevel)
self.log = logger.getChild(name)
self.log.propagate = True # propagate to main stimela logger
# logfile is False: no logfile at recipe level
if logfile is not False:
# logfile is None: use default name
if logfile is None:
logfile = "{0}/log-{1}.txt".format(log_dir or ".", self.name_)
# reset default name for task-level logfiles based on logfile
self.logfile_task = os.path.splitext(logfile)[0] + "-{task}.txt" \
if logfile_task is None else logfile_task
# ensure directory exists
log_dir = os.path.dirname(logfile) or "."
if not os.path.exists(log_dir):
self.log.info('creating log directory {0:s}'.format(log_dir))
os.makedirs(log_dir)
self._log_fh = logging.FileHandler(logfile, 'w', delay=True)
self._log_fh.setLevel(getattr(logging, self.loglevel))
self._log_fh.setFormatter(stimela.log_formatter)
self.log.addHandler(self._log_fh)
self.resume_file = '.last_{}.json'.format(self.name_)
# set to default if not set
# create a folder to store config files
# if it doesn't exist. These config
# files can be resued to re-run the
# task
self.jobs = []
self.completed = []
self.failed = None
self.remaining = []
self.pid = os.getpid()
cmd_line_pf = self.stimela_context.get('_STIMELA_PULLFOLDER', None)
self.singularity_image_dir = cmd_line_pf or singularity_image_dir or PULLFOLDER
if self.singularity_image_dir and not self.JOB_TYPE:
self.JOB_TYPE = "singularity"
self.log.info('---------------------------------')
self.log.info('Stimela version {0}'.format(stimela.__version__))
self.log.info('Running: {:s}'.format(self.name))
self.log.info('---------------------------------')
self.workdir = None
self.__make_workdir()
self.parameter_file_dir = parameter_file_dir or f'{self.workdir}/stimela_parameter_files'
if not os.path.exists(self.parameter_file_dir):
self.log.info(
f'Config directory cannot be found. Will create {self.parameter_file_dir}')
os.mkdir(self.parameter_file_dir)
def __init__(self, name, data=None,
parameter_file_dir=None, ms_dir=None,
tag=None, build_label=None,
singularity_image_dir=None, JOB_TYPE='docker',
cabpath=None,
logger=None,
log_dir=None, logfile=None, logfile_task=None):
"""
Deifine and manage a stimela recipe instance.
name : Name of stimela recipe
msdir : Path of MSs to be used during the execution of the recipe
tag : Use cabs with a specific tag
parameter_file_dir : Will store task specific parameter files here
logger: if set to a logger object, uses the specified logger.
if None, sets up its own logger using the parameters below
loglevel: default logging level
log_dir: default directory for logfiles
logfile: name of logfile, False to disable recipe-level logfiles, or None to form a default name
logfile_task: name of task-level logfile, False to disable task-level logfiles, or None to form a default name.
logfile_task may contain a "{task}" entry which will be substituted for a task name.
"""
self.name = name
self.name_ = re.sub(r'\W', '_', name) # pausterized name
self.cabpath = cabpath
# set default name for task-level logfiles
self.logfile_task = "{0}/log-{1}-{{task}}".format(log_dir or ".", self.name) \
if logfile_task is None else logfile_task
if logger is not None:
self.log = logger
else:
self.log = stimela.logger().getChild(name)
self.log.propagate = True # propagate to main stimela logger
# logfile is False: no logfile at recipe level
if logfile is not False:
# logfile is None: use default name
if logfile is None:
logfile = "{0}/log-{1}.txt".format(log_dir or ".", self.name)
# reset default name for task-level logfiles based on logfile
self.logfile_task = os.path.splitext(logfile)[0] + "-{task}.txt" \
if logfile_task is None else logfile_task
# ensure directory exists
log_dir = os.path.dirname(logfile) or "."
if not os.path.exists(log_dir):
self.log.info('creating log directory {0:s}'.format(log_dir))
os.makedirs(log_dir)
fh = logging.FileHandler(logfile, 'w', delay=True)
fh.setLevel(logging.INFO)
fh.setFormatter(stimela.log_formatter)
self.log.addHandler(fh)
self.JOB_TYPE = JOB_TYPE
self.resume_file = '.last_{}.json'.format(self.name_)
self.stimela_context = inspect.currentframe().f_back.f_globals
self.stimela_path = os.path.dirname(docker.__file__)
self.build_label = build_label or stimela.CAB_USERNAME
self.ms_dir = ms_dir
if not os.path.exists(self.ms_dir):
self.log.info(
'MS directory \'{}\' does not exist. Will create it'.format(self.ms_dir))
os.mkdir(self.ms_dir)
self.tag = tag
# create a folder to store config files
# if it doesn't exist. These config
# files can be resued to re-run the
# task
self.parameter_file_dir = parameter_file_dir or "stimela_parameter_files"
if not os.path.exists(self.parameter_file_dir):
self.log.info(
'Config directory cannot be found. Will create ./{}'.format(self.parameter_file_dir))
os.mkdir(self.parameter_file_dir)
self.jobs = []
self.completed = []
self.failed = None
self.remaining = []
self.pid = os.getpid()
self.singularity_image_dir = singularity_image_dir
if self.singularity_image_dir and not self.JOB_TYPE:
self.JOB_TYPE = "singularity"
self.log.info('---------------------------------')
self.log.info('Stimela version {0}'.format(stimela.__version__))
self.log.info('Sphesihle Makhathini <*****@*****.**>')
self.log.info('Running: {:s}'.format(self.name))
self.log.info('---------------------------------')
def xrun(command,
options,
log=None,
logfile=None,
timeout=-1,
kill_callback=None):
command_name = command
# this part could be inside the container
command = " ".join([command] + list(map(str, options)))
log = log or get_stimela_logger()
if log is None:
return xrun_nolog(command, name=command_name)
# this part is never inside the container
import stimela
log = log or stimela.logger()
log.info("running " + command,
extra=dict(stimela_subprocess_output=(command_name, "start")))
start_time = time.time()
proc = subprocess.Popen([command],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1,
universal_newlines=True,
shell=True)
poller = Poller(log=log)
poller.register_process(proc)
proc_running = True
try:
while proc_running and poller.fdlabels:
fdlist = poller.poll(verbose=DEBUG > 0)
for fname, fobj in fdlist:
try:
line = fobj.readline()
except EOFError:
line = b''
empty_line = not line
line = (line.decode('utf-8')
if type(line) is bytes else line).rstrip()
# break out if process closes
if empty_line:
poller.unregister_file(fobj)
if proc.stdout not in poller and proc.stderr not in poller:
log.debug("The {} process has exited".format(command))
proc_running = None
break
continue
# dispatch output to log
line = _remove_ctrls(line)
# the extra attributes are filtered by e.g. the CARACal logger
if fobj is proc.stderr:
log.warning(
line,
extra=dict(stimela_subprocess_output=(command_name,
"stderr")))
else:
log.info(
line,
extra=dict(stimela_subprocess_output=(command_name,
"stdout")))
if timeout > 0 and time.time() > start_time + timeout:
log.error("timeout, killing {} process".format(command))
kill_callback() if callable(kill_callback) else proc.kill()
proc_running = False
proc.wait()
status = proc.returncode
except SystemExit as exc:
log.error("{} has exited with code {}".format(command, exc.code))
proc.wait()
status = exc.code
raise StimelaCabRuntimeError('{}: SystemExit with code {}'.format(
command_name, status))
except KeyboardInterrupt:
if callable(kill_callback):
log.warning(
"Ctrl+C caught: shutting down {} process, please give it a few moments"
.format(command_name))
kill_callback()
log.info("the {} process was shut down successfully".format(
command_name),
extra=dict(stimela_subprocess_output=(command_name,
"status")))
else:
log.warning(
"Ctrl+C caught, killing {} process".format(command_name))
proc.kill()
proc.wait()
raise StimelaCabRuntimeError(
'{} interrupted with Ctrl+C'.format(command_name))
except Exception as exc:
traceback.print_exc()
log.error("Exception caught: {}".format(str(exc)))
proc.wait()
raise StimelaCabRuntimeError("{} throws exception '{}'".format(
command_name, str(exc)))
if status:
raise StimelaCabRuntimeError("{} returns error code {}".format(
command_name, status))
return status
def main(argv):
# parse initial arguments to init basic switches and modes
parser = config_parser.basic_parser(argv)
options, _ = parser.parse_known_args(argv)
caracal.init_console_logging(boring=options.boring, debug=options.debug)
stimela.logger().setLevel(logging.DEBUG if options.debug else logging.INFO)
# user requests worker help
if options.worker_help:
if not print_worker_help(options.worker_help):
parser.error("unknown worker '{}'".format(options.worker_help))
return
# User requests default config => dump and exit
if options.get_default:
sample_config = SAMPLE_CONFIGS.get(options.get_default_template)
if sample_config is None:
parser.error("unknown default template '{}'".format(options.get_default_template))
sample_config_path = os.path.join(pckgdir, "sample_configurations", sample_config)
if not os.path.exists(sample_config_path):
raise RuntimeError("Missing sample config file {}. This is a bug, please report".format(sample_config))
# validate the file
try:
parser = config_parser.config_parser()
_, version = parser.validate_config(sample_config_path)
if version != SCHEMA_VERSION:
log.warning("Sample config file {} version is {}, current CARACal version is {}.".format(sample_config,
version,
SCHEMA_VERSION))
log.warning("Proceeding anyway, but please notify the CARACal team to ship a newer sample config!")
except config_parser.ConfigErrors as exc:
log.error("{}, list of errors follows:".format(exc))
for section, errors in exc.errors.items():
print(" {}:".format(section))
for err in errors:
print(" - {}".format(err))
sys.exit(1) # indicate failure
log.info("Initializing {1} from config template '{0}' (schema version {2})".format(options.get_default_template,
options.get_default, version))
shutil.copy2(sample_config_path, options.get_default)
return
if options.print_calibrator_standard:
cdb = mkct.calibrator_database()
log.info("Found the following reference calibrators (in CASA format):")
log.info(cdb)
return
# if config was not specified (i.e. stayed default), print help and exit
config_file = options.config
if config_file == caracal.DEFAULT_CONFIG:
parser.print_help()
sys.exit(1)
try:
parser = config_parser.config_parser()
config, version = parser.validate_config(config_file)
if version != SCHEMA_VERSION:
log.warning("Config file {} schema version is {}, current CARACal version is {}".format(config_file,
version, SCHEMA_VERSION))
log.warning("Will try to proceed anyway, but please be advised that configuration options may have changed.")
# populate parser with items from config
parser.populate_parser(config)
# reparse arguments
caracal.log.info("Loading pipeline configuration from {}".format(config_file), extra=dict(color="GREEN"))
options, config = parser.update_config_from_args(config, argv)
# raise warning on schema version
except config_parser.ConfigErrors as exc:
log.error("{}, list of errors follows:".format(exc))
for section, errors in exc.errors.items():
print(" {}:".format(section))
for err in errors:
print(" - {}".format(err))
sys.exit(1) # indicate failure
except Exception as exc:
traceback.print_exc()
log.error("Error parsing arguments or configuration: {}".format(exc))
if options.debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
sys.exit(1) # indicate failure
if options.report and options.no_reports:
log.error("-report contradicts --no-reports")
sys.exit(1)
log_logo()
# Very good idea to print user options into the log before running:
parser.log_options(config)
execute_pipeline(options, config, block=True)