def run_demo_auto(args): run_demo_manual(args) start_background_scheduler() logging.info( "Starting Airflow Webserver in background\n- to open webserver follow the link {}" .format(get_webserver_url())) with Mute(): webserver(get_airflow_default_args("webserver"))
def run_init(args): asset_conf("init") logging.info("Init cwl-airflow") update_config(args) create_folders() export_dags() logging.info("Init Airflow DB") with Mute(): initdb(argparse.Namespace())
def exit_if_unsupported_feature(cwl_file, exit_code=33): with Mute(): tool = load_cwl(cwl_file).tool if tool["class"] == "Workflow" and [ step["id"] for step in tool["steps"] if "scatter" in step ]: logging.warning( "Failed to submit workflow\n- {} - scatter is not supported". format(cwl_file)) sys.exit(exit_code)
def submit_job(args): asset_conf() logging.info( "Load workflow\n- workflow: {workflow}\n- job: {job}\n- uid: {uid}" .format(**vars(args))) exit_if_unsupported_feature(args.workflow) export_job_file(args) if getattr(args, "run", None): logging.info("Run Airflow Scheduler") with Mute(): add_run_info(args) scheduler(args) print(get_workflow_output(args.dag_id))
def main(argsl=None): if argsl is None: argsl = sys.argv[1:] argsl.append("") # To avoid raising error when argsl is empty args, _ = arg_parser().parse_known_args(argsl) args = normalize_args(args, skip_list=[ "func", "uid", "limit", "dag_timeout", "threads", "web_interval", "web_workers", "run", "auto", "manual", "list", "quiet" ]) with Mute(): reset_root_logger(args.quiet) args.func(args)
def start_background_scheduler(): logging.info("Starting Airflow Scheduler in background") scheduler_thread = Process(target=scheduler, args=(get_airflow_default_args("scheduler"), )) with Mute(): scheduler_thread.start()
#!/usr/bin/env python from __future__ import print_function import sys import os import argparse import uuid import logging from cwl_airflow.utils.mute import Mute with Mute(): # Suppress output from airflow.settings import AIRFLOW_HOME from cwl_airflow.utils.logger import reset_root_logger from airflow.bin.cli import scheduler, webserver, initdb from cwl_airflow.utils.func import ( export_job_file, add_run_info, update_config, export_dags, create_folders, get_demo_workflow, get_updated_args, start_background_scheduler, get_airflow_default_args, clean_jobs, get_webserver_url, asset_conf, copy_demo_workflows) from cwl_airflow.utils.utils import (get_workflow_output, normalize_args, exit_if_unsupported_feature, conf_get_default) def arg_parser(): parent_parser = argparse.ArgumentParser(add_help=False) general_parser = argparse.ArgumentParser(description='cwl-airflow') general_parser.add_argument( "-q", "--quiet", dest='quiet', action="store_true",