def parse_cabspec_dict(self, cabspec_seq):
"""Turns sequence of cabspecs into a Stimela cabspec dict"""
cabspecs = OrderedDict()
speclists = OrderedDict()
# collect all specs encountered, sort them by cab
for spec in cabspec_seq:
name, version, tag = spec["name"], spec.get("version") or None, spec.get("tag") or None
if not version and not tag:
log.warning(f"Neither version nor tag specified for cabspec {name}, ignoring")
continue
speclists.setdefault(name, []).append((version, tag))
# now process each cab's list of specs.
for name, speclist in speclists.items():
if len(speclist) == 1:
version, tag = speclist[0]
if version is None:
log.info(f" {name}: forcing tag {tag} for all invocations")
cabspecs[name] = dict(tag=tag, force=True)
continue
elif tag is None:
log.info(f" {name}: forcing version {version} for all invocations")
cabspecs[name] = dict(version=version)
continue
# else make dict of version: tag pairs
cabspecs[name] = dict(version={version: tag for version, tag in speclist}, force=True)
for version, tag in speclist:
log.info(f" {name}: using tag {tag} for version {version}")
return cabspecs
def make_symlink(link, target):
if os.path.lexists(link):
if os.path.islink(link):
os.unlink(link) # old symlink can go
else:
log.warning("{} already exists and is not a symlink, can't relink".format(link))
return False
if not os.path.lexists(link):
os.symlink(target, link)
log.info("{} links to {}".format(link, target))
def __run(debug=False):
""" Executes pipeline """
# with stream_director(log) as director: # stdout and stderr needs to go to the log as well -- nah
try:
# Obtain some divine knowledge
cdb = mkct.calibrator_database()
pipeline = worker_administrator(config,
workers_directory,
add_all_first=False, prefix=options.general_prefix,
configFileName=options.config, singularity_image_dir=options.singularity_image_dir,
container_tech=options.container_tech, start_worker=options.start_worker,
end_worker=options.end_worker, generate_reports=not options.no_reports)
if options.report:
pipeline.regenerate_reports()
else:
pipeline.run_workers()
except SystemExit as e:
# if e.code != 0:
log.error("A pipeline worker initiated sys.exit({0:}). This is likely a bug, please report.".format(e.code))
log.info(" More information can be found in the logfile at {0:s}".format(caracal.CARACAL_LOG))
log.info(" You are running version {0:s}".format(str(__version__)), extra=dict(logfile_only=True))
if debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
sys.exit(1) # indicate failure
except KeyboardInterrupt:
log.error("Ctrl+C received from user, shutting down. Goodbye!")
except Exception as exc:
log.error("{} [{}]".format(exc, type(exc).__name__), extra=dict(boldface=True))
log.info(" More information can be found in the logfile at {0:s}".format(caracal.CARACAL_LOG))
log.info(" You are running version {0:s}".format(str(__version__)), extra=dict(logfile_only=True))
for line in traceback.format_exc().splitlines():
log.error(line, extra=dict(traceback_report=True))
if debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
log.info("exiting with error code 1")
sys.exit(1) # indicate failure
def generate_report_notebooks(notebooks, output_dir, prefix, container_tech):
opts = ["--non-interactive"]
if container_tech == "docker":
opts.append("--docker")
elif container_tech == "singularity":
opts.append("--singularity")
else:
log.warning(
"Container technology '{}' not supported by radiopadre, skipping report rendering"
)
return
if caracal.DEBUG:
opts += ['-v', '2', '--container-debug']
## disabling as per https://github.com/caracal-pipeline/caracal/issues/1161
# # first time run with -u
# global _radiopadre_updated
# if not _radiopadre_updated:
# opts.append('--update')
# _radiopadre_updated = True
start_time = time.time()
log.info("Rendering report(s)")
for notebook in notebooks:
if prefix:
notebook = "{}-{}".format(prefix, notebook)
nbdest = os.path.join(output_dir, notebook + ".ipynb")
nbhtml = os.path.join(output_dir, notebook + ".html")
if os.path.exists(nbdest):
try:
xrun("run-radiopadre", opts + ["--nbconvert", nbdest], log=log)
except StimelaCabRuntimeError as exc:
log.warning(
"Report {} failed to render ({}). HTML report will not be available."
.format(nbhtml, exc))
# check that HTML file actually showed up (sometimes the container doesn't report an error)
if os.path.exists(
nbhtml) and os.path.getmtime(nbhtml) >= start_time:
log.info("Rendered report {}".format(nbhtml))
else:
log.warning("Report {} failed to render".format(nbhtml))
else:
log.warning(
"Report notebook {} not found, skipping report rendering".
format(nbdest))
def _wrapper(*args, **kwargs):
# Delay caracal log import
from caracal import log
log.warning("Skipping %s.\n%s", fn.__name__, msg)
return msg
def worker(pipeline, recipe, config):
uvrange = config['uvrange']
plotter = config['standard_plotter']
label_in = config['label_in']
nobs = pipeline.nobs
subdir = config['dirname']
output_dir = os.path.join(pipeline.diagnostic_plots, subdir) if subdir else pipeline.diagnostic_plots
if config['field'] == 'calibrators':
fields = ['bpcal', 'gcal', 'fcal']
elif config['field'] == 'target':
fields = ['target']
else:
fields = config['field'].split(',')
if set(fields).difference(['fcal', 'bpcal', 'gcal']):
raise ValueError("Eligible values for 'field': 'target', 'calibrators', 'fcal', 'bpcal' or 'gcal'. " \
"User selected {}".format(",".join(fields)))
log.info(f"plotting fields: {' '.join(fields)}")
for iobs in range(nobs):
label = config['label_plot']
mslist = pipeline.get_mslist(iobs, label_in, target=(config['field'] == 'target'))
for msname in mslist:
if not os.path.exists(os.path.join(pipeline.msdir, msname)):
raise IOError("MS {0:s} does not exist. Please check that is where it should be.".format(msname))
for msname in mslist:
log.info(f"plotting MS: {msname}")
msbase = os.path.splitext(msname)[0]
ms_info_dict = pipeline.get_msinfo(msname)
corr = config['correlation']
ms_corrs = ms_info_dict['CORR']['CORR_TYPE']
if corr == 'auto' or corr == 'all':
corr = ','.join(ms_corrs)
elif corr == 'diag' or corr == 'parallel':
corr = ','.join([c for c in ms_corrs if len(c) == 2 and c[0] == c[1]])
if not corr:
log.warning(f"No correlations found to plot for {msname}")
continue
log.info(f"plotting correlations: {corr}")
# new-school plots
if config['shadems']['enable']:
# make dict of substitutions
basesubst = OrderedDict(msbase=os.path.splitext(msname)[0])
# make a map: {(fields): field_type}, so we can loop over fields below, but only include unique fields
field_map = OrderedDict()
# make a reverse map: field_type -> "field_name,field_name"
field_map_names = OrderedDict()
# make set of all field names
all_fields = set()
for field_type in fields:
if (label_in != '') and (config['field'] == 'target'):
field_names = tuple(ms_info_dict['FIELD']['NAME'])
else:
field_names = tuple(getattr(pipeline, field_type)[iobs])
field_map.setdefault(field_names, field_type)
all_fields.update(field_names)
basesubst[field_type] = field_map_names[field_type] = ",".join(field_names)
basesubst["all_fields"] = ",".join(all_fields)
plot_args = []
def collect_plots(args, plotspecs, extra_args, subst=None):
"""Generic helper function to parse a list of shadems plot specs, and add them to plot_args"""
for iplot, plotspec in enumerate(plotspecs):
if plotspec:
plotspec = plotspec.format(**(subst or basesubst))
# add arguments from args, if not present in plotspec
plotspec = plotspec.split()
for arg, value in args.items():
arg = "--" + arg
if arg not in plotspec:
plotspec += [arg, value]
plotspec += extra_args
plot_args.append(" ".join(plotspec))
baseargs = OrderedDict(
png="{}-{}-{}".format(msbase, label,
"{field}{_Spw}{_Scan}{_Ant}-{label}{_alphalabel}{_colorlabel}{_suffix}.png"),
title="'{ms} {_field}{_Spw}{_Scan}{_Ant}{_title}{_Alphatitle}{_Colortitle}'",
col=config['shadems']['default_column'],
corr=corr.replace(' ', ''))
# collect generic plots
collect_plots(baseargs, config['shadems']['plots'], [])
# collect plots_by_corr
args = baseargs.copy()
args["field"] = ",".join(all_fields)
collect_plots(args, config['shadems']['plots_by_corr'], ["--iter-corr"])
# collect plots_by_field
for field_names, field_type in field_map.items():
args = baseargs.copy()
args["field"] = ",".join(field_names)
args["png"] = "{}-{}-{}-{}".format(msbase, label, field_type,
"{field}{_Spw}{_Scan}{_Ant}-{label}{_alphalabel}{_colorlabel}{_suffix}.png")
args["title"] = "'{ms} " + field_type + "{_field}{_Spw}{_Scan}{_Ant}{_title}{_Alphatitle}{_Colortitle}'"
subst = basesubst.copy()
subst["field"] = field_type
collect_plots(args, config['shadems']['plots_by_field'], ["--iter-field"])
# dispatch plots
if plot_args:
step = 'plot-shadems-ms{0:d}'.format(iobs)
recipe.add("cab/shadems_direct", step,
dict(ms=msname, args=plot_args,
ignore_errors=config["shadems"]["ignore_errors"]),
input=pipeline.input, output=output_dir,
label="{0:s}:: Plotting".format(step))
else:
log.warning("The shadems section is enabled, but doesn't specify any plot_by_field or plot_by_corr")
# old-school plots
# define plot attributes
diagnostic_plots = {}
diagnostic_plots["real_imag"] = dict(
plotms={"xaxis": "imag", "yaxis": "real",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "real", "yaxis": "imag"},
ragavi_vis={"xaxis": "real", "yaxis": "imaginary",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["amp_phase"] = dict(
plotms={"xaxis": "amp", "yaxis": "phase",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "amp", "yaxis": "phase"},
ragavi_vis={"xaxis": "phase", "yaxis": "amplitude",
"iter-axis": "corr", "canvas-width": 1080,
"canvas-height": 720})
diagnostic_plots["amp_ant"] = dict(
plotms={"xaxis": "antenna", "yaxis": "amp",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "ANTENNA1", "yaxis": "amp"},
ragavi_vis=None)
diagnostic_plots["amp_uvwave"] = dict(
plotms={"xaxis": "uvwave", "yaxis": "amp",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "UV", "yaxis": "amp"},
ragavi_vis={"xaxis": "uvwave", "yaxis": "amplitude",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["phase_uvwave"] = dict(
plotms={"xaxis": "uvwave", "yaxis": "phase",
"colouraxis": "baseline", "iteraxis": "corr"},
shadems={"xaxis": "UV", "yaxis": "phase"},
ragavi_vis={"xaxis": "uvwave", "yaxis": "phase",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["amp_scan"] = dict(
plotms={"xaxis": "scan", "yaxis": "amp"},
shadems={"xaxis": "SCAN_NUMBER", "yaxis": "amp"},
ragavi_vis={"xaxis": "scan", "yaxis": "amplitude",
"iter-axis": None,
"canvas-width": 1080, "canvas-height": 720})
diagnostic_plots["amp_chan"] = dict(
plotms={"xaxis": "chan", "yaxis": "amp"},
shadems={"xaxis": "CHAN", "yaxis": "amp"},
ragavi_vis={"xaxis": "channel", "yaxis": "amplitude",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
diagnostic_plots["phase_chan"] = dict(
plotms={"xaxis": "chan", "yaxis": "phase"},
shadems={"xaxis": "CHAN", "yaxis": "phase"},
ragavi_vis={"xaxis": "channel", "yaxis": "phase",
"iter-axis": "scan", "canvas-width": 300,
"canvas-height": 300})
if plotter.lower() != "none":
for plotname in diagnostic_plots:
if not pipeline.enable_task(config, plotname):
continue
opts = diagnostic_plots[plotname][plotter]
if opts is None:
log.warn("The plotter '{0:s}' cannot make the plot '{1:s}'".format(
plotter, plotname))
continue
elif plotter == "ragavi_vis":
opts["num-cores"] = config["num_cores"]
opts["mem-limit"] = config["mem_limit"]
# make map from field name to field_type, field_id
field_map = OrderedDict()
for field_type in fields:
for field in getattr(pipeline, field_type)[iobs]:
if label_in != '' and field_type == 'target':
fid = 0
else:
fid = utils.get_field_id(ms_info_dict, field)[0]
field_map.setdefault(field, (field_type, fid))
if plotter == "shadems":
corr = corr.replace(" ", "").split(",")
for it, co in enumerate(corr):
if co in ms_corrs:
corr[it] = str(ms_corrs.index(co))
corr = ",".join(corr)
# for each corr
for co in corr.split(","):
opts["corr"] = co
for field, (field_type, fid) in field_map.items():
globals()[plotter](pipeline, recipe, config,
plotname, msname, field,
iobs, label, msbase, opts,
ftype=field_type, fid=fid, output_dir=output_dir,
corr_label=ms_corrs[int(co)])
elif plotter == "ragavi_vis" and not opts["iter-axis"] == "corr":
# change the labels to indices
corr = corr.replace(" ", "").split(",")
for it, co in enumerate(corr):
if co in ms_corrs:
corr[it] = str(ms_corrs.index(co))
corr = ",".join(corr)
# for each corr
for co in corr.split(","):
opts["corr"] = co
for field, (field_type, fid) in field_map.items():
globals()[plotter](pipeline, recipe, config,
plotname, msname, field,
iobs, label, msbase, opts,
ftype=field_type, fid=fid, output_dir=output_dir,
corr_label=ms_corrs[int(co)])
else:
opts["corr"] = corr
for field, (field_type, fid) in field_map.items():
globals()[plotter](pipeline, recipe, config,
plotname, msname, field, iobs, label,
msbase, opts, ftype=field_type,
fid=fid, output_dir=output_dir)
def main(argv):
# parse initial arguments to init basic switches and modes
parser = config_parser.basic_parser(argv)
options, _ = parser.parse_known_args(argv)
caracal.init_console_logging(boring=options.boring, debug=options.debug)
stimela.logger().setLevel(logging.DEBUG if options.debug else logging.INFO)
# user requests worker help
if options.worker_help:
if not print_worker_help(options.worker_help):
parser.error("unknown worker '{}'".format(options.worker_help))
return
# User requests default config => dump and exit
if options.get_default:
sample_config = SAMPLE_CONFIGS.get(options.get_default_template)
if sample_config is None:
parser.error("unknown default template '{}'".format(options.get_default_template))
sample_config_path = os.path.join(pckgdir, "sample_configurations", sample_config)
if not os.path.exists(sample_config_path):
raise RuntimeError("Missing sample config file {}. This is a bug, please report".format(sample_config))
# validate the file
try:
parser = config_parser.config_parser()
_, version = parser.validate_config(sample_config_path)
if version != SCHEMA_VERSION:
log.warning("Sample config file {} version is {}, current CARACal version is {}.".format(sample_config,
version,
SCHEMA_VERSION))
log.warning("Proceeding anyway, but please notify the CARACal team to ship a newer sample config!")
except config_parser.ConfigErrors as exc:
log.error("{}, list of errors follows:".format(exc))
for section, errors in exc.errors.items():
print(" {}:".format(section))
for err in errors:
print(" - {}".format(err))
sys.exit(1) # indicate failure
log.info("Initializing {1} from config template '{0}' (schema version {2})".format(options.get_default_template,
options.get_default, version))
shutil.copy2(sample_config_path, options.get_default)
return
if options.print_calibrator_standard:
cdb = mkct.calibrator_database()
log.info("Found the following reference calibrators (in CASA format):")
log.info(cdb)
return
# if config was not specified (i.e. stayed default), print help and exit
config_file = options.config
if config_file == caracal.DEFAULT_CONFIG:
parser.print_help()
sys.exit(1)
try:
parser = config_parser.config_parser()
config, version = parser.validate_config(config_file)
if version != SCHEMA_VERSION:
log.warning("Config file {} schema version is {}, current CARACal version is {}".format(config_file,
version, SCHEMA_VERSION))
log.warning("Will try to proceed anyway, but please be advised that configuration options may have changed.")
# populate parser with items from config
parser.populate_parser(config)
# reparse arguments
caracal.log.info("Loading pipeline configuration from {}".format(config_file), extra=dict(color="GREEN"))
options, config = parser.update_config_from_args(config, argv)
# raise warning on schema version
except config_parser.ConfigErrors as exc:
log.error("{}, list of errors follows:".format(exc))
for section, errors in exc.errors.items():
print(" {}:".format(section))
for err in errors:
print(" - {}".format(err))
sys.exit(1) # indicate failure
except Exception as exc:
traceback.print_exc()
log.error("Error parsing arguments or configuration: {}".format(exc))
if options.debug:
log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.")
pdb.post_mortem(sys.exc_info()[2])
sys.exit(1) # indicate failure
if options.report and options.no_reports:
log.error("-report contradicts --no-reports")
sys.exit(1)
log_logo()
# Very good idea to print user options into the log before running:
parser.log_options(config)
execute_pipeline(options, config, block=True)