def parse_cabspec_dict(self, cabspec_seq): """Turns sequence of cabspecs into a Stimela cabspec dict""" cabspecs = OrderedDict() speclists = OrderedDict() # collect all specs encountered, sort them by cab for spec in cabspec_seq: name, version, tag = spec["name"], spec.get("version") or None, spec.get("tag") or None if not version and not tag: log.warning(f"Neither version nor tag specified for cabspec {name}, ignoring") continue speclists.setdefault(name, []).append((version, tag)) # now process each cab's list of specs. for name, speclist in speclists.items(): if len(speclist) == 1: version, tag = speclist[0] if version is None: log.info(f" {name}: forcing tag {tag} for all invocations") cabspecs[name] = dict(tag=tag, force=True) continue elif tag is None: log.info(f" {name}: forcing version {version} for all invocations") cabspecs[name] = dict(version=version) continue # else make dict of version: tag pairs cabspecs[name] = dict(version={version: tag for version, tag in speclist}, force=True) for version, tag in speclist: log.info(f" {name}: using tag {tag} for version {version}") return cabspecs
def make_symlink(link, target): if os.path.lexists(link): if os.path.islink(link): os.unlink(link) # old symlink can go else: log.warning("{} already exists and is not a symlink, can't relink".format(link)) return False if not os.path.lexists(link): os.symlink(target, link) log.info("{} links to {}".format(link, target))
def __run(debug=False): """ Executes pipeline """ # with stream_director(log) as director: # stdout and stderr needs to go to the log as well -- nah try: # Obtain some divine knowledge cdb = mkct.calibrator_database() pipeline = worker_administrator(config, workers_directory, add_all_first=False, prefix=options.general_prefix, configFileName=options.config, singularity_image_dir=options.singularity_image_dir, container_tech=options.container_tech, start_worker=options.start_worker, end_worker=options.end_worker, generate_reports=not options.no_reports) if options.report: pipeline.regenerate_reports() else: pipeline.run_workers() except SystemExit as e: # if e.code != 0: log.error("A pipeline worker initiated sys.exit({0:}). This is likely a bug, please report.".format(e.code)) log.info(" More information can be found in the logfile at {0:s}".format(caracal.CARACAL_LOG)) log.info(" You are running version {0:s}".format(str(__version__)), extra=dict(logfile_only=True)) if debug: log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.") pdb.post_mortem(sys.exc_info()[2]) sys.exit(1) # indicate failure except KeyboardInterrupt: log.error("Ctrl+C received from user, shutting down. Goodbye!") except Exception as exc: log.error("{} [{}]".format(exc, type(exc).__name__), extra=dict(boldface=True)) log.info(" More information can be found in the logfile at {0:s}".format(caracal.CARACAL_LOG)) log.info(" You are running version {0:s}".format(str(__version__)), extra=dict(logfile_only=True)) for line in traceback.format_exc().splitlines(): log.error(line, extra=dict(traceback_report=True)) if debug: log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.") pdb.post_mortem(sys.exc_info()[2]) log.info("exiting with error code 1") sys.exit(1) # indicate failure
def generate_report_notebooks(notebooks, output_dir, prefix, container_tech): opts = ["--non-interactive"] if container_tech == "docker": opts.append("--docker") elif container_tech == "singularity": opts.append("--singularity") else: log.warning( "Container technology '{}' not supported by radiopadre, skipping report rendering" ) return if caracal.DEBUG: opts += ['-v', '2', '--container-debug'] ## disabling as per https://github.com/caracal-pipeline/caracal/issues/1161 # # first time run with -u # global _radiopadre_updated # if not _radiopadre_updated: # opts.append('--update') # _radiopadre_updated = True start_time = time.time() log.info("Rendering report(s)") for notebook in notebooks: if prefix: notebook = "{}-{}".format(prefix, notebook) nbdest = os.path.join(output_dir, notebook + ".ipynb") nbhtml = os.path.join(output_dir, notebook + ".html") if os.path.exists(nbdest): try: xrun("run-radiopadre", opts + ["--nbconvert", nbdest], log=log) except StimelaCabRuntimeError as exc: log.warning( "Report {} failed to render ({}). HTML report will not be available." .format(nbhtml, exc)) # check that HTML file actually showed up (sometimes the container doesn't report an error) if os.path.exists( nbhtml) and os.path.getmtime(nbhtml) >= start_time: log.info("Rendered report {}".format(nbhtml)) else: log.warning("Report {} failed to render".format(nbhtml)) else: log.warning( "Report notebook {} not found, skipping report rendering". format(nbdest))
def _wrapper(*args, **kwargs): # Delay caracal log import from caracal import log log.warning("Skipping %s.\n%s", fn.__name__, msg) return msg
def worker(pipeline, recipe, config): uvrange = config['uvrange'] plotter = config['standard_plotter'] label_in = config['label_in'] nobs = pipeline.nobs subdir = config['dirname'] output_dir = os.path.join(pipeline.diagnostic_plots, subdir) if subdir else pipeline.diagnostic_plots if config['field'] == 'calibrators': fields = ['bpcal', 'gcal', 'fcal'] elif config['field'] == 'target': fields = ['target'] else: fields = config['field'].split(',') if set(fields).difference(['fcal', 'bpcal', 'gcal']): raise ValueError("Eligible values for 'field': 'target', 'calibrators', 'fcal', 'bpcal' or 'gcal'. " \ "User selected {}".format(",".join(fields))) log.info(f"plotting fields: {' '.join(fields)}") for iobs in range(nobs): label = config['label_plot'] mslist = pipeline.get_mslist(iobs, label_in, target=(config['field'] == 'target')) for msname in mslist: if not os.path.exists(os.path.join(pipeline.msdir, msname)): raise IOError("MS {0:s} does not exist. Please check that is where it should be.".format(msname)) for msname in mslist: log.info(f"plotting MS: {msname}") msbase = os.path.splitext(msname)[0] ms_info_dict = pipeline.get_msinfo(msname) corr = config['correlation'] ms_corrs = ms_info_dict['CORR']['CORR_TYPE'] if corr == 'auto' or corr == 'all': corr = ','.join(ms_corrs) elif corr == 'diag' or corr == 'parallel': corr = ','.join([c for c in ms_corrs if len(c) == 2 and c[0] == c[1]]) if not corr: log.warning(f"No correlations found to plot for {msname}") continue log.info(f"plotting correlations: {corr}") # new-school plots if config['shadems']['enable']: # make dict of substitutions basesubst = OrderedDict(msbase=os.path.splitext(msname)[0]) # make a map: {(fields): field_type}, so we can loop over fields below, but only include unique fields field_map = OrderedDict() # make a reverse map: field_type -> "field_name,field_name" field_map_names = OrderedDict() # make set of all field names all_fields = set() for field_type in fields: if (label_in != '') and (config['field'] == 'target'): field_names = tuple(ms_info_dict['FIELD']['NAME']) else: field_names = tuple(getattr(pipeline, field_type)[iobs]) field_map.setdefault(field_names, field_type) all_fields.update(field_names) basesubst[field_type] = field_map_names[field_type] = ",".join(field_names) basesubst["all_fields"] = ",".join(all_fields) plot_args = [] def collect_plots(args, plotspecs, extra_args, subst=None): """Generic helper function to parse a list of shadems plot specs, and add them to plot_args""" for iplot, plotspec in enumerate(plotspecs): if plotspec: plotspec = plotspec.format(**(subst or basesubst)) # add arguments from args, if not present in plotspec plotspec = plotspec.split() for arg, value in args.items(): arg = "--" + arg if arg not in plotspec: plotspec += [arg, value] plotspec += extra_args plot_args.append(" ".join(plotspec)) baseargs = OrderedDict( png="{}-{}-{}".format(msbase, label, "{field}{_Spw}{_Scan}{_Ant}-{label}{_alphalabel}{_colorlabel}{_suffix}.png"), title="'{ms} {_field}{_Spw}{_Scan}{_Ant}{_title}{_Alphatitle}{_Colortitle}'", col=config['shadems']['default_column'], corr=corr.replace(' ', '')) # collect generic plots collect_plots(baseargs, config['shadems']['plots'], []) # collect plots_by_corr args = baseargs.copy() args["field"] = ",".join(all_fields) collect_plots(args, config['shadems']['plots_by_corr'], ["--iter-corr"]) # collect plots_by_field for field_names, field_type in field_map.items(): args = baseargs.copy() args["field"] = ",".join(field_names) args["png"] = "{}-{}-{}-{}".format(msbase, label, field_type, "{field}{_Spw}{_Scan}{_Ant}-{label}{_alphalabel}{_colorlabel}{_suffix}.png") args["title"] = "'{ms} " + field_type + "{_field}{_Spw}{_Scan}{_Ant}{_title}{_Alphatitle}{_Colortitle}'" subst = basesubst.copy() subst["field"] = field_type collect_plots(args, config['shadems']['plots_by_field'], ["--iter-field"]) # dispatch plots if plot_args: step = 'plot-shadems-ms{0:d}'.format(iobs) recipe.add("cab/shadems_direct", step, dict(ms=msname, args=plot_args, ignore_errors=config["shadems"]["ignore_errors"]), input=pipeline.input, output=output_dir, label="{0:s}:: Plotting".format(step)) else: log.warning("The shadems section is enabled, but doesn't specify any plot_by_field or plot_by_corr") # old-school plots # define plot attributes diagnostic_plots = {} diagnostic_plots["real_imag"] = dict( plotms={"xaxis": "imag", "yaxis": "real", "colouraxis": "baseline", "iteraxis": "corr"}, shadems={"xaxis": "real", "yaxis": "imag"}, ragavi_vis={"xaxis": "real", "yaxis": "imaginary", "iter-axis": "scan", "canvas-width": 300, "canvas-height": 300}) diagnostic_plots["amp_phase"] = dict( plotms={"xaxis": "amp", "yaxis": "phase", "colouraxis": "baseline", "iteraxis": "corr"}, shadems={"xaxis": "amp", "yaxis": "phase"}, ragavi_vis={"xaxis": "phase", "yaxis": "amplitude", "iter-axis": "corr", "canvas-width": 1080, "canvas-height": 720}) diagnostic_plots["amp_ant"] = dict( plotms={"xaxis": "antenna", "yaxis": "amp", "colouraxis": "baseline", "iteraxis": "corr"}, shadems={"xaxis": "ANTENNA1", "yaxis": "amp"}, ragavi_vis=None) diagnostic_plots["amp_uvwave"] = dict( plotms={"xaxis": "uvwave", "yaxis": "amp", "colouraxis": "baseline", "iteraxis": "corr"}, shadems={"xaxis": "UV", "yaxis": "amp"}, ragavi_vis={"xaxis": "uvwave", "yaxis": "amplitude", "iter-axis": "scan", "canvas-width": 300, "canvas-height": 300}) diagnostic_plots["phase_uvwave"] = dict( plotms={"xaxis": "uvwave", "yaxis": "phase", "colouraxis": "baseline", "iteraxis": "corr"}, shadems={"xaxis": "UV", "yaxis": "phase"}, ragavi_vis={"xaxis": "uvwave", "yaxis": "phase", "iter-axis": "scan", "canvas-width": 300, "canvas-height": 300}) diagnostic_plots["amp_scan"] = dict( plotms={"xaxis": "scan", "yaxis": "amp"}, shadems={"xaxis": "SCAN_NUMBER", "yaxis": "amp"}, ragavi_vis={"xaxis": "scan", "yaxis": "amplitude", "iter-axis": None, "canvas-width": 1080, "canvas-height": 720}) diagnostic_plots["amp_chan"] = dict( plotms={"xaxis": "chan", "yaxis": "amp"}, shadems={"xaxis": "CHAN", "yaxis": "amp"}, ragavi_vis={"xaxis": "channel", "yaxis": "amplitude", "iter-axis": "scan", "canvas-width": 300, "canvas-height": 300}) diagnostic_plots["phase_chan"] = dict( plotms={"xaxis": "chan", "yaxis": "phase"}, shadems={"xaxis": "CHAN", "yaxis": "phase"}, ragavi_vis={"xaxis": "channel", "yaxis": "phase", "iter-axis": "scan", "canvas-width": 300, "canvas-height": 300}) if plotter.lower() != "none": for plotname in diagnostic_plots: if not pipeline.enable_task(config, plotname): continue opts = diagnostic_plots[plotname][plotter] if opts is None: log.warn("The plotter '{0:s}' cannot make the plot '{1:s}'".format( plotter, plotname)) continue elif plotter == "ragavi_vis": opts["num-cores"] = config["num_cores"] opts["mem-limit"] = config["mem_limit"] # make map from field name to field_type, field_id field_map = OrderedDict() for field_type in fields: for field in getattr(pipeline, field_type)[iobs]: if label_in != '' and field_type == 'target': fid = 0 else: fid = utils.get_field_id(ms_info_dict, field)[0] field_map.setdefault(field, (field_type, fid)) if plotter == "shadems": corr = corr.replace(" ", "").split(",") for it, co in enumerate(corr): if co in ms_corrs: corr[it] = str(ms_corrs.index(co)) corr = ",".join(corr) # for each corr for co in corr.split(","): opts["corr"] = co for field, (field_type, fid) in field_map.items(): globals()[plotter](pipeline, recipe, config, plotname, msname, field, iobs, label, msbase, opts, ftype=field_type, fid=fid, output_dir=output_dir, corr_label=ms_corrs[int(co)]) elif plotter == "ragavi_vis" and not opts["iter-axis"] == "corr": # change the labels to indices corr = corr.replace(" ", "").split(",") for it, co in enumerate(corr): if co in ms_corrs: corr[it] = str(ms_corrs.index(co)) corr = ",".join(corr) # for each corr for co in corr.split(","): opts["corr"] = co for field, (field_type, fid) in field_map.items(): globals()[plotter](pipeline, recipe, config, plotname, msname, field, iobs, label, msbase, opts, ftype=field_type, fid=fid, output_dir=output_dir, corr_label=ms_corrs[int(co)]) else: opts["corr"] = corr for field, (field_type, fid) in field_map.items(): globals()[plotter](pipeline, recipe, config, plotname, msname, field, iobs, label, msbase, opts, ftype=field_type, fid=fid, output_dir=output_dir)
def main(argv): # parse initial arguments to init basic switches and modes parser = config_parser.basic_parser(argv) options, _ = parser.parse_known_args(argv) caracal.init_console_logging(boring=options.boring, debug=options.debug) stimela.logger().setLevel(logging.DEBUG if options.debug else logging.INFO) # user requests worker help if options.worker_help: if not print_worker_help(options.worker_help): parser.error("unknown worker '{}'".format(options.worker_help)) return # User requests default config => dump and exit if options.get_default: sample_config = SAMPLE_CONFIGS.get(options.get_default_template) if sample_config is None: parser.error("unknown default template '{}'".format(options.get_default_template)) sample_config_path = os.path.join(pckgdir, "sample_configurations", sample_config) if not os.path.exists(sample_config_path): raise RuntimeError("Missing sample config file {}. This is a bug, please report".format(sample_config)) # validate the file try: parser = config_parser.config_parser() _, version = parser.validate_config(sample_config_path) if version != SCHEMA_VERSION: log.warning("Sample config file {} version is {}, current CARACal version is {}.".format(sample_config, version, SCHEMA_VERSION)) log.warning("Proceeding anyway, but please notify the CARACal team to ship a newer sample config!") except config_parser.ConfigErrors as exc: log.error("{}, list of errors follows:".format(exc)) for section, errors in exc.errors.items(): print(" {}:".format(section)) for err in errors: print(" - {}".format(err)) sys.exit(1) # indicate failure log.info("Initializing {1} from config template '{0}' (schema version {2})".format(options.get_default_template, options.get_default, version)) shutil.copy2(sample_config_path, options.get_default) return if options.print_calibrator_standard: cdb = mkct.calibrator_database() log.info("Found the following reference calibrators (in CASA format):") log.info(cdb) return # if config was not specified (i.e. stayed default), print help and exit config_file = options.config if config_file == caracal.DEFAULT_CONFIG: parser.print_help() sys.exit(1) try: parser = config_parser.config_parser() config, version = parser.validate_config(config_file) if version != SCHEMA_VERSION: log.warning("Config file {} schema version is {}, current CARACal version is {}".format(config_file, version, SCHEMA_VERSION)) log.warning("Will try to proceed anyway, but please be advised that configuration options may have changed.") # populate parser with items from config parser.populate_parser(config) # reparse arguments caracal.log.info("Loading pipeline configuration from {}".format(config_file), extra=dict(color="GREEN")) options, config = parser.update_config_from_args(config, argv) # raise warning on schema version except config_parser.ConfigErrors as exc: log.error("{}, list of errors follows:".format(exc)) for section, errors in exc.errors.items(): print(" {}:".format(section)) for err in errors: print(" - {}".format(err)) sys.exit(1) # indicate failure except Exception as exc: traceback.print_exc() log.error("Error parsing arguments or configuration: {}".format(exc)) if options.debug: log.warning("you are running with -debug enabled, dropping you into pdb. Use Ctrl+D to exit.") pdb.post_mortem(sys.exc_info()[2]) sys.exit(1) # indicate failure if options.report and options.no_reports: log.error("-report contradicts --no-reports") sys.exit(1) log_logo() # Very good idea to print user options into the log before running: parser.log_options(config) execute_pipeline(options, config, block=True)