def cancel_jobs(output, verbose): """Cancels jobs.""" filename = os.path.join(output, "cancel_jobs.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="a") logger.info(get_cli_string()) for _ in range(60): cluster, promoted = Cluster.deserialize( output, try_promote_to_submitter=True, deserialize_jobs=True, ) if not promoted: logger.info("Did not get promoted. Sleep.") time.sleep(1) continue if cluster.is_complete(): cluster.demote_from_submitter() logger.info("All jobs are already finished.") sys.exit(0) submitter = JobSubmitter.load(output) submitter.cancel_jobs(cluster) sys.exit(0) logger.error("Failed to get promoted to submitter.") sys.exit(1)
def submit(config_file, output, force, verbose=False): """Submit the pipeline for execution.""" if os.path.exists(output): if force: shutil.rmtree(output) else: print( f"{output} already exists. Delete it or use '--force' to overwrite.", file=sys.stderr, ) sys.exit(1) os.makedirs(output, exist_ok=True) filename = os.path.join(output, "pipeline_submit.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level) logger.info(get_cli_string()) mgr = PipelineManager.create(config_file, output) try: mgr.submit_next_stage(1) except Exception: logger.exception("Pipeline execution failed") raise logging.shutdown() sys.exit(0)
def auto_config(extension, inputs, job_post_process_config_file, config_file, verbose): """Automatically create a configuration.""" level = logging.DEBUG if verbose else logging.WARNING setup_logging("auto_config", None, console_level=level) if job_post_process_config_file is not None: module, class_name, data = JobPostProcess.load_config_from_file( job_post_process_config_file) JobPostProcess(module, class_name, data) # ensure everything ok job_post_process_config = { "module": module, "class": class_name, "data": data } else: job_post_process_config = None # User extension registry = Registry() if not registry.is_registered(extension): raise InvalidExtension(f"Extension '{extension}' is not registered.") cli = registry.get_extension_class(extension, ExtensionClassType.CLI) config = cli.auto_config(*inputs, job_post_process_config=job_post_process_config) print(f"Created configuration with {config.get_num_jobs()} jobs.") config.dump(config_file) print(f"Dumped configuration to {config_file}.\n")
def show_events(output, names, categories=False, json_fmt=False, names_only=False, categories_only=False, verbose=False): """Shows the events after jobs run. \b Examples: jade show-events jade show-events unhandled_error jade show-events error -c jade show-events --names-only jade show-events --categories-only """ level = logging.DEBUG if verbose else logging.WARNING setup_logging("show_results", None, console_level=level) results = EventsSummary(output) if names_only: results.show_event_names() elif categories_only: results.show_event_categories() elif json_fmt: print(results.to_json()) else: if not names: names = results.list_unique_names() for name in names: if categories: results.show_events_in_category(name) else: results.show_events(name)
def create( filename, append_job_name, append_output_dir, config_file, cancel_on_blocking_job_failure, minutes_per_job, shuffle, strip_whitespace, verbose, ): """Create a config file from a filename with a list of executable commands.""" level = logging.DEBUG if verbose else logging.WARNING setup_logging("auto_config", None, console_level=level) config = GenericCommandConfiguration.auto_config( filename, cancel_on_blocking_job_failure=cancel_on_blocking_job_failure, minutes_per_job=minutes_per_job, append_job_name=append_job_name, append_output_dir=append_output_dir, ) if shuffle: config.shuffle_jobs() print(f"Created configuration with {config.get_num_jobs()} jobs.") indent = None if strip_whitespace else 2 config.dump(config_file, indent=indent) print(f"Dumped configuration to {config_file}.\n")
def manager_node(output_dir, job_id, verbose): """Print the name of the manager node to the console. Requires a single job in the batch.""" level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, None, console_level=level) try: cluster, _ = Cluster.deserialize(output_dir, deserialize_jobs=True) except InvalidConfiguration: print( f"{output_dir} is not a JADE output directory used in cluster mode", file=sys.stderr) sys.exit(1) if cluster.is_complete(): print("All jobs are already complete.") sys.exit() groups = make_submission_group_lookup( [cluster.config.submission_groups[0]]) hpc_mgr = HpcManager(groups, output_dir) if job_id not in cluster.job_status.hpc_job_ids: print(f"job_id={job_id} is not active", file=sys.stderr) sys.exit(1) node = hpc_mgr.list_active_nodes(job_id)[0] print(node)
def _run_manager(job_name, output_dir, verbose, manager_script_and_args): filename = os.path.join(output_dir, f"run_multi_node_job_manager__{job_name}.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="w") logger.info("Run manager on %s: %s", socket.gethostname(), get_cli_string()) # Note that the manager receives its own hostname. output = {} check_run_command(f"jade cluster hostnames {output_dir}", output) hostnames = [x for x in output["stdout"].split() if x != ""] logger.info("Manager found %s hostnames: %s", len(hostnames), hostnames) cmd = " ".join(manager_script_and_args) logger.info("Run manager script [%s]", cmd) os.environ["JADE_OUTPUT_DIR"] = output_dir os.environ["JADE_COMPUTE_NODE_NAMES"] = " ".join(hostnames) start = time.time() ret = run_command(cmd) logger.info("Finished job. duration = %s seconds", time.time() - start) return ret
def hostnames(output_dir, job_id, verbose): """Show the hostnames of active nodes participating in the batch.""" level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, None, console_level=level) try: cluster, _ = Cluster.deserialize(output_dir, deserialize_jobs=True) except InvalidConfiguration: print( f"{output_dir} is not a JADE output directory used in cluster mode", file=sys.stderr) sys.exit(1) if cluster.is_complete(): print("All jobs are already complete.") sys.exit() groups = make_submission_group_lookup( [cluster.config.submission_groups[0]]) hpc_mgr = HpcManager(groups, output_dir) nodes = [] for _job_id in cluster.job_status.hpc_job_ids: if job_id is not None and _job_id != job_id: continue nodes += hpc_mgr.list_active_nodes(_job_id) if not nodes: print("No nodes were detected.", file=sys.stderr) sys.exit(1) print(" ".join(nodes))
def create(filename, config_file, verbose): """Create a config file from a filename with a list of executable commands.""" level = logging.DEBUG if verbose else logging.WARNING setup_logging("auto_config", None, console_level=level) config = GenericCommandConfiguration.auto_config(filename) print(f"Created configuration with {config.get_num_jobs()} jobs.") config.dump(config_file) print(f"Dumped configuration to {config_file}.\n")
def test_setup_logging(mock_get_logger, mock_dict_config): """Should called dictConfig and getLogger methods""" # Call name = "this_is_a_logger_name" filename = "this_is_a_file_name" setup_logging(name, filename) # Assertions mock_dict_config.assert_called_once() mock_get_logger.assert_called_with(name)
def show_times(output_dirs, verbose): """Show the run times of all allocated jobs.""" level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, None, console_level=level) job_ids = [] for output in output_dirs: path = Path(output) try: cluster, _ = Cluster.deserialize(path, deserialize_jobs=False) except InvalidConfiguration: print( f"{output} is not a JADE output directory used in cluster mode", file=sys.stderr) sys.exit(1) job_ids += [ x.name.split("_")[2].replace(".e", "") for x in path.glob("*.e") ] job_ids.sort(key=lambda x: int(x)) groups = make_submission_group_lookup( [cluster.config.submission_groups[0]]) hpc_mgr = HpcManager(groups, output) total_duration = timedelta(seconds=0) table = PrettyTable() table.field_names = HpcJobStats._fields total_aus = 0 if os.environ.get("NREL_CLUSTER") == "eagle": au_parser = get_nrel_eagle_aus else: au_parser = None for job_id in job_ids: stats = hpc_mgr.get_job_stats(job_id) if stats is None: continue duration = stats.end - stats.start if stats.state == HpcJobStatus.COMPLETE and isinstance( stats.end, datetime): total_duration += duration data = stats._asdict() data["state"] = data["state"].value if au_parser is not None: total_aus += au_parser(duration, stats.qos) table.add_row(data.values()) print(table) print(f"\nTotal duration = {total_duration}") print("Total hours = {:.2f}".format(total_duration.total_seconds() / 3600)) if au_parser is not None: print("Total AUs = {:.2f}".format(total_aus))
def show_results(failed, output, successful, post_process, job_name, verbose): """Shows the results of a batch of jobs.""" level = logging.DEBUG if verbose else logging.WARNING setup_logging("show_results", None, console_level=level) if post_process: JobPostProcess.show_results(output, job_name) sys.exit(0) results = ResultsSummary(output) results.show_results(only_failed=failed, only_successful=successful)
def submit_jobs(config_file, per_node_batch_size, hpc_config, local, max_nodes, output, poll_interval, num_processes, rotate_logs, verbose, restart_failed, restart_missing, reports, try_add_blocked_jobs): """Submits jobs for execution, locally or on HPC.""" os.makedirs(output, exist_ok=True) previous_results = [] if restart_failed: failed_job_config = create_config_from_previous_run( config_file, output, result_type='failed') previous_results = ResultsSummary(output).get_successful_results() config_file = "failed_job_inputs.json" failed_job_config.dump(config_file) if restart_missing: missing_job_config = create_config_from_previous_run( config_file, output, result_type='missing') config_file = "missing_job_inputs.json" missing_job_config.dump(config_file) previous_results = ResultsSummary(output).list_results() if rotate_logs: rotate_filenames(output, ".log") filename = os.path.join(output, "submit_jobs.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level) logger.info(get_cli_string()) event_file = os.path.join(output, "submit_jobs_events.log") # This effectively means no console logging. setup_logging("event", event_file, console_level=logging.ERROR, file_level=logging.INFO) mgr = JobSubmitter(config_file, hpc_config=hpc_config, output=output) ret = mgr.submit_jobs( per_node_batch_size=per_node_batch_size, max_nodes=max_nodes, force_local=local, verbose=verbose, num_processes=num_processes, poll_interval=poll_interval, previous_results=previous_results, reports=reports, try_add_blocked_jobs=try_add_blocked_jobs, ) sys.exit(ret.value)
def run_worker(job_name, output_dir, verbose, poll_interval=60): """Run a worker instance.""" hostname = socket.gethostname() filename = os.path.join(output_dir, f"run_multi_node_job_worker__{job_name}__{hostname}.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="w") logger.info("Run worker: %s", get_cli_string()) shutdown_file = _get_shutdown_file(job_name, output_dir) while not shutdown_file.exists(): logger.debug("sleep for %s seconds", poll_interval) time.sleep(poll_interval) logger.info("Detected shutdown.") return 0
def resubmit_jobs(output, failed, missing, verbose): """Resubmit failed and missing jobs.""" event_file = os.path.join(output, "submit_jobs_events.log") setup_event_logging(event_file, mode="a") filename = os.path.join(output, "submit_jobs.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="a") cluster, promoted = Cluster.deserialize( output, try_promote_to_submitter=True, deserialize_jobs=True, ) if not cluster.is_complete(): cluster.demote_from_submitter() print( "resubmit-jobs requires that the existing submission be complete", file=sys.stderr) sys.exit(1) assert promoted jobs_to_resubmit = _get_jobs_to_resubmit(cluster, output, failed, missing) updated_blocking_jobs_by_name = _update_with_blocking_jobs( jobs_to_resubmit, output) _reset_results(output, jobs_to_resubmit) cluster.prepare_for_resubmission(jobs_to_resubmit, updated_blocking_jobs_by_name) ret = 1 try: mgr = JobSubmitter.load(output) status = mgr.submit_jobs(cluster) if status == Status.IN_PROGRESS: print(f"Resubmitted {len(jobs_to_resubmit)} jobs in {output}") ret = 0 else: ret = status.value except Exception: logger.exception("Failed to resubmit jobs") raise finally: cluster.demote_from_submitter() sys.exit(ret)
def run_worker(job, manager_node, output_dir, verbose, poll_interval=60): """Run a worker instance.""" logger.error("in worker manager_node=%s job=%s", manager_node, job.name) hostname = socket.gethostname() filename = os.path.join( output_dir, f"run_spark_job_worker__{hostname}__{job.name}.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="w") logger.info("Run worker: %s", get_cli_string()) # Ignore errors. Spark may not be running. run_command(job.model.spark_config.get_stop_worker()) # Give the master a head start. time.sleep(10) job_output = Path(output_dir) / JOBS_OUTPUT_DIR / job.name logs_dir = job_output / "spark" / "logs" job_conf_dir = job_output / "spark" / "conf" workers_dir = job_output / "spark" / "workers" _set_env_variables(job, job_conf_dir, logs_dir) worker_memory = _get_worker_memory_str(job, is_master=False) cmd = _get_worker_command(job, manager_node, worker_memory) ret = 1 output = {} for _ in range(5): output.clear() logger.info("Run spark worker: [%s]", cmd) ret = run_command(cmd, output=output) if ret == 0: break if ret != 0: logger.error("Failed to start spark worker: %s: %s", ret, output) shutdown_file = _get_shutdown_file(job.name, output_dir) while not shutdown_file.exists(): logger.debug("sleep for %s seconds", poll_interval) time.sleep(poll_interval) logger.info("Detected shutdown.") check_run_command(job.model.spark_config.get_stop_worker()) if job.model.spark_config.collect_worker_logs: shutil.copytree(Path(os.environ["SPARK_WORKER_DIR"]), workers_dir / hostname) return 0
def prune_files(output): """Deletes intermediate scripts, config files, and log files that are not needed if the job results were successful.""" setup_logging(__name__, None, console_level=logging.INFO) base_path = Path(output) count = 0 for path in itertools.chain( # Keep submit_jobs.log* files because there aren't many of them and they are useful. base_path.glob("submit_jobs_events*.log*"), base_path.glob("run_jobs_batch*.log*"), base_path.glob("config_batch*.json"), base_path.glob("*.sh"), ): path.unlink() count += 1 print(f"Deleted {count} files from {output}.")
def run_jobs(config_file, output, num_processes, verbose): """Starts jobs on HPC.""" match = re.search(r"batch_(\d+)\.json", config_file) assert match batch_id = match.group(1) os.makedirs(output, exist_ok=True) filename = os.path.join(output, f"run_jobs_batch_{batch_id}.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=logging.ERROR) logger.info(get_cli_string()) mgr = JobRunner(config_file, output=output, batch_id=batch_id) ret = mgr.run_jobs(verbose=verbose, num_processes=num_processes) sys.exit(ret.value)
def test_resource_stats(): with tempfile.TemporaryDirectory() as tmpdir: event_file = os.path.join(tmpdir, "events.log") setup_logging("event", event_file, console_level=logging.ERROR, file_level=logging.INFO) resource_monitor = ResourceMonitor("test") count = 2 found_cpu = 0 found_disk = 0 found_mem = 0 found_net = 0 for i in range(count): resource_monitor.log_resource_stats() summary = EventsSummary(tmpdir) assert len(summary.list_events(EVENT_NAME_CPU_STATS)) == count assert len(summary.list_events(EVENT_NAME_DISK_STATS)) == count assert len(summary.list_events(EVENT_NAME_MEMORY_STATS)) == count assert len(summary.list_events(EVENT_NAME_NETWORK_STATS)) == count viewers = [ CpuStatsViewer(summary), DiskStatsViewer(summary), MemoryStatsViewer(summary), NetworkStatsViewer(summary), ] for viewer in viewers: df = viewer.get_dataframe("test") assert len(df) == 2 if isinstance(viewer, MemoryStatsViewer): mem_df = viewer.get_dataframe("test") averages = viewer._calc_batch_averages("test") for field, val in averages.items(): assert val == df[field].mean() output = {} cmd = f"jade stats show -o {tmpdir} cpu disk mem net" ret = run_command(cmd, output=output) assert ret == 0 for term in ("IOPS", "read_bytes", "bytes_recv", "idle"): assert term in output["stdout"]
def submit_next_stage(output, stage_num, return_code, verbose=False): """Internal command to submit the next stage of the pipeline for execution.""" filename = os.path.join(output, "pipeline_submit.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="a") logger.info(get_cli_string()) mgr = PipelineManager.load(output) try: mgr.submit_next_stage(stage_num, return_code=return_code) except Exception: logger.exception("Pipeline execution failed") raise logging.shutdown() sys.exit(0)
def am_i_manager(output_dir, verbose): """Print 'true' or 'false' depending on whether the current node is the manager node.""" level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, None, console_level=level) try: cluster, _ = Cluster.deserialize(output_dir, deserialize_jobs=True) except InvalidConfiguration: print( f"{output_dir} is not a JADE output directory used in cluster mode", file=sys.stderr) sys.exit(1) if cluster.is_complete(): print("All jobs are already complete.") sys.exit() groups = make_submission_group_lookup( [cluster.config.submission_groups[0]]) hpc_mgr = HpcManager(groups, output_dir) am_manager = hpc_mgr.am_i_manager() print(str(am_manager).lower(), end="")
def list_active_ids(output_dirs, verbose): """List the HPC job IDs that are pending or running.""" # TODO: add flag for only pending or only running level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, None, console_level=level) job_ids = [] for output in output_dirs: path = Path(output) try: cluster, _ = Cluster.deserialize(path, deserialize_jobs=True) except InvalidConfiguration: print( f"{output} is not a JADE output directory used in cluster mode", file=sys.stderr) sys.exit(1) if not cluster.is_complete(): job_ids += list(cluster.iter_hpc_job_ids()) job_ids.sort(key=lambda x: int(x)) print(" ".join(job_ids))
def run_jobs(config_file, distributed_submitter, output, num_processes, verbose): """Starts jobs on HPC.""" match = re.search(r"batch_(\d+)\.json", config_file) assert match batch_id = match.group(1) os.makedirs(output, exist_ok=True) mgr = JobRunner(config_file, output=output, batch_id=batch_id) # Logging has to get enabled after the JobRunner is created because we need the node ID # is what makes the file unique. filename = os.path.join(output, f"run_jobs_batch_{batch_id}_{mgr.node_id}.log") level = logging.DEBUG if verbose else logging.INFO setup_event_logging(mgr.event_filename) logger = setup_logging(__name__, filename, file_level=level, console_level=level, mode="w") logger.info(get_cli_string()) group = mgr.config.get_default_submission_group() if group.submitter_params.node_setup_script: cmd = f"{group.submitter_params.node_setup_script} {config_file} {output}" ret = run_command(cmd) if ret != 0: logger.error("Failed to run node setup script %s: %s", cmd, ret) sys.exit(ret) status = mgr.run_jobs(distributed_submitter=distributed_submitter, verbose=verbose, num_processes=num_processes) ret = status.value if group.submitter_params.node_shutdown_script: cmd = f"{group.submitter_params.node_shutdown_script} {config_file} {output}" ret2 = run_command(cmd) if ret2 != 0: logger.error("Failed to run node shutdown script %s: %s", cmd, ret2) if status == Status.GOOD and distributed_submitter: start = time.time() _try_submit_jobs(output, verbose=verbose) logger.info("try-submit-jobs took %s seconds", time.time() - start) sys.exit(ret)
def show_results(failed, output, successful, post_process, job_name, verbose): """Shows the results of a batch of jobs.""" if not Path(output).exists(): print(f"{output} does not exist", file=sys.stderr) sys.exit(1) level = logging.DEBUG if verbose else logging.WARNING setup_logging("show_results", None, console_level=level) if post_process: JobPostProcess.show_results(output, job_name) sys.exit(0) try: results = ResultsSummary(output) except InvalidConfiguration: print( f"No results are available in {output}. To check status of in-progress jobs run " f"'jade show-status -o {output}'", file=sys.stderr, ) sys.exit(1) results.show_results(only_failed=failed, only_successful=successful)
def submit(config_file, output, verbose=False): """Submit the pipeline for execution.""" global logger os.makedirs(output, exist_ok=True) filename = os.path.join(output, "pipeline_submit.log") level = logging.DEBUG if verbose else logging.INFO logger = setup_logging(__name__, filename, file_level=level, console_level=level) logger.info(get_cli_string()) mgr = PipelineManager(config_file, output) try: mgr.submit(verbose=verbose) except Exception: logger.exception("Pipeline execution failed") raise sys.exit(0)
def __init__( self, config_file, output=OUTPUT_DIR, batch_id=0, ): super(JobRunner, self).__init__(config_file, output) self._intf, self._intf_type = self._create_node_interface() self._batch_id = batch_id self._event_file = os.path.join( output, f"run_jobs_batch_{batch_id}_events.log", ) self._event_logger = setup_logging("event", self._event_file, console_level=logging.ERROR, file_level=logging.INFO) logger.debug("Constructed JobRunner output=%s batch=%s", output, batch_id)
def _run_cluster_master(job, manager_node, output_dir, verbose, manager_script_and_args): filename = os.path.join(output_dir, f"run_spark_cluster__{job.name}.log") level = logging.DEBUG if verbose else logging.INFO setup_logging(__name__, filename, file_level=level, console_level=level, mode="w") logger.info("Run cluster master on %s job=%s: %s", socket.gethostname(), job.name, get_cli_string()) job_output = Path(output_dir) / JOBS_OUTPUT_DIR / job.name if job_output.exists(): shutil.rmtree(job_output) job_output.mkdir(parents=True) events_dir = job_output / "spark" / "events" events_dir.mkdir(parents=True) logs_dir = job_output / "spark" / "logs" logs_dir.mkdir() workers_dir = job_output / "spark" / "workers" workers_dir.mkdir() # Make a job-specific conf directory because the log and event files need to be per-job. job_conf_dir = job_output / "spark" / "conf" shutil.copytree( Path(job.model.spark_config.conf_dir) / "conf", job_conf_dir) _fix_spark_conf_file(job_conf_dir, events_dir) _set_env_variables(job, job_conf_dir, logs_dir) # Ignore errors. Spark may not be running. run_command(job.model.spark_config.get_stop_worker()) run_command(job.model.spark_config.get_stop_history_server()) run_command(job.model.spark_config.get_stop_master()) # It would be better to start all workers from the master. Doing so would require that # Spark processes on the master node be able to ssh into the worker nodes. # I haven't spent the time to figure out to do that inside Singularity containers. master_cmd = job.model.spark_config.get_start_master() logger.info("Run spark master: [%s]", master_cmd) check_run_command(master_cmd) history_cmd = job.model.spark_config.get_start_history_server() logger.info("Run spark history server: [%s]", history_cmd) check_run_command(history_cmd) worker_memory = _get_worker_memory_str(job, is_master=True) worker_cmd = _get_worker_command(job, manager_node, memory=worker_memory) logger.info("Run spark worker: [%s]", worker_cmd) check_run_command(worker_cmd) # Wait for workers. # TODO: find a way to check programmatically with the rest api # or parse the logs time.sleep(15) args = list(manager_script_and_args) + [ _get_cluster(manager_node), str(job_output) ] if job.model.spark_config.run_user_script_inside_container: user_cmd = str(job.model.spark_config.get_run_user_script() ) + " " + " ".join(args) else: user_cmd = " ".join(args) logger.info("Run user script [%s]", user_cmd) start = time.time() ret = run_command(user_cmd) logger.info("Finished job. duration = %s seconds", time.time() - start) # Delay to ensure the history is saved. time.sleep(10) metrics = SparkMetrics("localhost", history=True) try: metrics.generate_metrics(job_output / "spark_metrics") except Exception: logger.exception("Failed to generate metrics") check_run_command(job.model.spark_config.get_stop_worker()) check_run_command(job.model.spark_config.get_stop_history_server()) check_run_command(job.model.spark_config.get_stop_master()) if job.model.spark_config.collect_worker_logs: shutil.copytree(Path(os.environ["SPARK_WORKER_DIR"]), workers_dir / socket.gethostname()) return ret
def pipeline(): """Manage JADE execution pipeline.""" setup_logging("pipeline", None)
def extensions(): """Manage JADE extensions.""" setup_logging("extensions", None)
def config(): """Manage a JADE configuration.""" setup_logging("config", None)