def __run(debug=False):
""" Executes pipeline """
# with stream_director(log) as director: # stdout and stderr needs to go to the log as well -- nah
try:
# Obtain some divine knowledge
cdb = mkct.calibrator_database()
pipeline = worker_administrator(config,
workers_directory,
add_all_first=False, prefix=options.general_prefix,
configFileName=options.config, singularity_image_dir=options.singularity_image_dir,
container_tech=options.container_tech, start_worker=options.start_worker,
end_worker=options.end_worker, generate_reports=not options.no_reports)
if options.report:
pipeline.regenerate_reports()
else:
pipeline.run_workers()
except SystemExit as e:
# if e.code != 0:
log.error("A pipeline worker initiated sys.exit({0:}). This is likely a bug, please report.".format(e.code))
log.info(" More information can be found in the logfile at {0:s}".format(caracal.CARACAL_LOG))
log.info(" You are running version {0:s}".format(str(__version__)), extra=dict(logfile_only=True))
if debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
sys.exit(1) # indicate failure
except KeyboardInterrupt:
log.error("Ctrl+C received from user, shutting down. Goodbye!")
except Exception as exc:
log.error("{} [{}]".format(exc, type(exc).__name__), extra=dict(boldface=True))
log.info(" More information can be found in the logfile at {0:s}".format(caracal.CARACAL_LOG))
log.info(" You are running version {0:s}".format(str(__version__)), extra=dict(logfile_only=True))
for line in traceback.format_exc().splitlines():
log.error(line, extra=dict(traceback_report=True))
if debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
log.info("exiting with error code 1")
sys.exit(1) # indicate failure
def conflict(conflict_type, pipeline, wname, ms, config, flags_bw, flags_aw, read_version = 'version'):
av_flagversions = get_flags(pipeline, ms)
req_version = config['rewind_flags'][read_version]
if req_version == 'auto':
req_version = flags_bw
if conflict_type == 'would_overwrite_bw' or conflict_type == 'rewind_too_little':
log.error('Flag version conflicts for {0:s} . If you are running Caracal on multiple targets'.format(ms))
log.error('and/or .MS files please read the warning at the end of this message.')
log.error('---------------------------------------------------------------------------------------------------')
log.error('A worker named "{0:s}" was already run on the .MS file {1:s} with pipeline prefix "{2:s}".'.format(wname, ms, pipeline.prefix))
if conflict_type == 'rewind_too_little':
log.error('and you are rewinding to a later flag version: {0:s} .'.format(req_version))
log.error('Running "{0:s}" again will attempt to overwrite existing flag versions, it might get messy.'.format(wname))
log.error('Caracal will not overwrite the "{0:s}" flag versions unless you explicitely request that.'.format(wname))
log.error('The current flag versions of this MS are (from the oldest to the most recent):')
for vv in av_flagversions:
if vv == flags_bw:
log.error(' {0:s} <-- (this worker)'.format(vv))
elif vv == flags_aw:
log.error(' {0:s} <-- (this worker)'.format(vv))
elif config['rewind_flags']["enable"] and vv == req_version:
log.error(' {0:s} <-- (rewinding to this version)'.format(vv))
else:
log.error(' {0:s}'.format(vv))
log.error('You have the following options:')
log.error(' 1) If you are happy with the flags currently stored in the FLAG column of this MS and')
log.error(' want to append new flags to them, change the name of this worker in the configuration')
log.error(' file by appending "__n" to it (where n is an integer not already taken in the list')
log.error(' above). The new flags will be appended to the FLAG column, and new flag versions will')
log.error(' be added to the list above.')
log.error(' 2) If you want to discard the flags obtained during the previous run of "{0:s}" (and,'.format(wname))
log.error(' necessarily, all flags obtained thereafter; see list above) reset the "{0:s}" worker'.format(wname))
log.error(' to its starting flag version by setting in the configuration file:'.format(flags_bw))
log.error(' {0:s}:'.format(wname))
log.error(' rewind_flags:')
log.error(' enable: true')
log.error(' mode: reset_worker')
log.error(' This will rewind to the flag version {0:s}. You will loose all flags'.format(flags_bw))
log.error(' appended to the FLAG column after that version, and take it from there.')
log.error(' 3) If you want to discard the flags obtained during the previous run of "{0:s}" and'.format(wname))
log.error(' rewind to an even earlier flag version from the list above set:')
log.error(' {0:s}:'.format(wname))
log.error(' rewind_flags:')
log.error(' enable: true')
log.error(' mode: rewind_to_version')
log.error(' {0:s}: <version_name>'.format(read_version))
log.error(' This will rewind to the requested flag version. You will loose all flags appended')
log.error(' to the FLAG column after that version, and take it from there.')
log.error(' 4) If you really know what you are doing, allow Caracal to overwrite flag versions by setting:')
log.error(' {0:s}:'.format(wname))
log.error(' overwrite_flagvers: true')
log.error(' The worker "{0:s}" will be run again; the new flags will be appended to the current'.format(wname))
log.error(' FLAG column (or to whatever flag version you are rewinding to); the flag version')
log.error(' "{0:s}" will be overwritten and appended to the list above (or to'.format(flags_bw))
log.error(' that list truncated to the flag version you are rewinding to).')
log.error('---------------------------------------------------------------------------------------------------')
log.error('Warning - Your choice will be applied to all .MS files being processed by the worker "{0:s}".'.format(wname))
log.error('If using the rewind_flags mode "rewind_to_version", make sure to rewind to a flag version that')
log.error('exists for all .MS files. If using the rewind_flags mode "reset_worker" each .MS file is taken')
log.error('care of automatically and you do not need to worry about it.')
elif conflict_type == 'rewind_to_non_existing':
log.error('You have asked to rewind the flags of {0:s} to the version "{1:s}" but this version'.format(ms, req_version))
log.error('does not exist. The available flag versions for this .MS file are:')
for vv in av_flagversions:
log.error(' {0:s}'.format(vv))
log.error('Note that if you are running Caracal on multiple targets and/or .MS files you should rewind to a flag')
log.error('version that exists for all of them.')
raise RuntimeError('Flag version conflicts.')
def run_workers(self):
""" Runs the workers """
report_updated = False
for _name, _worker, i in self.workers:
try:
worker = __import__(_worker)
except ImportError:
traceback.print_exc()
raise ImportError('Worker "{0:s}" could not be found at {1:s}'.format(
_worker, self.workers_directory))
if self.config["general"]["cabs"]:
log.info("Configuring cab specification overrides")
cabspecs_general = self.parse_cabspec_dict(self.config["general"]["cabs"])
else:
cabspecs_general = {}
active_workers = []
# first, check that workers import, and check their configs
for _name, _worker, i in self.workers:
config = self.config[_name]
if 'enable' in config and not config['enable']:
self.skip.append(_worker)
continue
log.info("Configuring worker {}".format(_name))
try:
worker = __import__(_worker)
except ImportError:
log.error('Error importing worker "{0:s}" from {1:s}'.format(_worker, self.workers_directory))
raise
if hasattr(worker, 'check_config'):
worker.check_config(config)
# check for cab specs
cabspecs = cabspecs_general
if config["cabs"]:
cabspecs = cabspecs.copy()
cabspecs.update(self.parse_cabspec_dict(config["cabs"]))
active_workers.append((_name, worker, config, cabspecs))
# now run the actual pipeline
#for _name, _worker, i in self.workers:
for _name, worker, config, cabspecs in active_workers:
# Define stimela recipe instance for worker
# Also change logger name to avoid duplication of logging info
label = getattr(worker, 'LABEL', None)
if label is None:
# if label is not set, take filename, and split off _worker.py
label = os.path.basename(worker.__file__).rsplit("_", 1)[0]
recipe = stimela.Recipe(label,
ms_dir=self.msdir,
singularity_image_dir=self.singularity_image_dir,
log_dir=self.logs,
cabspecs=cabspecs,
logfile=False, # no logfiles for recipes
logfile_task=f'{self.logs}/log-{label}-{{task}}-{self.timeNow}.txt')
recipe.JOB_TYPE = self.container_tech
self.CURRENT_WORKER = _name
# Don't allow pipeline-wide resume
# functionality
os.system('rm -f {}'.format(recipe.resume_file))
# Get recipe steps
# 1st get correct section of config file
log.info("{0:s}: initializing".format(label), extra=dict(color="GREEN"))
worker.worker(self, recipe, config)
log.info("{0:s}: running".format(label))
recipe.run()
log.info("{0:s}: finished".format(label))
# this should be in the cab cleanup code, no?
casa_last = glob.glob(self.output + '/*.last')
for file_ in casa_last:
os.remove(file_)
# update report at end of worker if so configured
if self.generate_reports and config["report"]:
self.regenerate_reports()
report_updated = True
else:
report_updated = False
# generate final report
if self.config["general"]["final_report"] and self.generate_reports and not report_updated:
self.regenerate_reports()
log.info("pipeline run complete")
def setup_default_notebooks(notebooks, output_dir, prefix, config):
# setup logos
logodir = os.path.join(output_dir, ".logo")
if not os.path.exists(logodir):
os.mkdir(logodir)
for png in glob.glob(os.path.join(SOURCE_NOTEBOOK_DIR, "*.png")):
shutil.copy2(png, logodir)
for notebook in notebooks:
nbfile = notebook + ".ipynb"
nbdest = os.path.join(
output_dir, "{}-{}".format(prefix, nbfile) if prefix else nbfile)
# overwrite destination only if source is newer
dest_mtime = os.path.getmtime(nbdest) if os.path.exists(nbdest) else 0
# if source is a template, invoke jinja
nbsrc = os.path.join(SOURCE_NOTEBOOK_DIR, nbfile + ".j2")
if os.path.exists(nbsrc):
if os.path.getmtime(nbsrc) > dest_mtime:
global _j2env
if _j2env is None:
_j2env = jinja2.Environment(
loader=jinja2.PackageLoader('caracal', 'notebooks'),
autoescape=jinja2.select_autoescape(['html', 'xml']))
template = _j2env.get_template(nbfile + ".j2")
log.info("Creating standard notebook {} from template".format(
nbdest))
with open(nbdest, "wt") as file:
try:
print(template.render(**config), file=file)
except jinja2.TemplateError as exc:
log.error(
"Error rendering notebook template: {}".format(
exc),
extra=dict(boldface=True))
log.info(
" More information can be found in the logfile at {0:s}"
.format(caracal.CARACAL_LOG))
for line in traceback.format_exc().splitlines():
log.error(line, extra=dict(traceback_report=True))
log.info("This is not fatal, continuing")
else:
log.info(
"Standard notebook {} already exists, won't overwrite".
format(nbdest))
continue
# if source exists as is, copy
nbsrc = os.path.join(SOURCE_NOTEBOOK_DIR, nbfile)
if os.path.exists(nbsrc):
if os.path.getmtime(nbsrc) > dest_mtime:
log.info("Creating standard notebook {}".format(nbdest))
shutil.copyfile(nbsrc, nbdest)
else:
log.info(
"Standard notebook {} already exists, won't overwrite".
format(nbdest))
continue
log.error("Standard notebook {} does not exist".format(nbsrc))
def main(argv):
# parse initial arguments to init basic switches and modes
parser = config_parser.basic_parser(argv)
options, _ = parser.parse_known_args(argv)
caracal.init_console_logging(boring=options.boring, debug=options.debug)
stimela.logger().setLevel(logging.DEBUG if options.debug else logging.INFO)
# user requests worker help
if options.worker_help:
if not print_worker_help(options.worker_help):
parser.error("unknown worker '{}'".format(options.worker_help))
return
# User requests default config => dump and exit
if options.get_default:
sample_config = SAMPLE_CONFIGS.get(options.get_default_template)
if sample_config is None:
parser.error("unknown default template '{}'".format(options.get_default_template))
sample_config_path = os.path.join(pckgdir, "sample_configurations", sample_config)
if not os.path.exists(sample_config_path):
raise RuntimeError("Missing sample config file {}. This is a bug, please report".format(sample_config))
# validate the file
try:
parser = config_parser.config_parser()
_, version = parser.validate_config(sample_config_path)
if version != SCHEMA_VERSION:
log.warning("Sample config file {} version is {}, current CARACal version is {}.".format(sample_config,
version,
SCHEMA_VERSION))
log.warning("Proceeding anyway, but please notify the CARACal team to ship a newer sample config!")
except config_parser.ConfigErrors as exc:
log.error("{}, list of errors follows:".format(exc))
for section, errors in exc.errors.items():
print(" {}:".format(section))
for err in errors:
print(" - {}".format(err))
sys.exit(1) # indicate failure
log.info("Initializing {1} from config template '{0}' (schema version {2})".format(options.get_default_template,
options.get_default, version))
shutil.copy2(sample_config_path, options.get_default)
return
if options.print_calibrator_standard:
cdb = mkct.calibrator_database()
log.info("Found the following reference calibrators (in CASA format):")
log.info(cdb)
return
# if config was not specified (i.e. stayed default), print help and exit
config_file = options.config
if config_file == caracal.DEFAULT_CONFIG:
parser.print_help()
sys.exit(1)
try:
parser = config_parser.config_parser()
config, version = parser.validate_config(config_file)
if version != SCHEMA_VERSION:
log.warning("Config file {} schema version is {}, current CARACal version is {}".format(config_file,
version, SCHEMA_VERSION))
log.warning("Will try to proceed anyway, but please be advised that configuration options may have changed.")
# populate parser with items from config
parser.populate_parser(config)
# reparse arguments
caracal.log.info("Loading pipeline configuration from {}".format(config_file), extra=dict(color="GREEN"))
options, config = parser.update_config_from_args(config, argv)
# raise warning on schema version
except config_parser.ConfigErrors as exc:
log.error("{}, list of errors follows:".format(exc))
for section, errors in exc.errors.items():
print(" {}:".format(section))
for err in errors:
print(" - {}".format(err))
sys.exit(1) # indicate failure
except Exception as exc:
traceback.print_exc()
log.error("Error parsing arguments or configuration: {}".format(exc))
if options.debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
sys.exit(1) # indicate failure
if options.report and options.no_reports:
log.error("-report contradicts --no-reports")
sys.exit(1)
log_logo()
# Very good idea to print user options into the log before running:
parser.log_options(config)
execute_pipeline(options, config, block=True)