def run_demo_auto(args):
run_demo_manual(args)
start_background_scheduler()
logging.info(
"Starting Airflow Webserver in background\n- to open webserver follow the link {}"
.format(get_webserver_url()))
with Mute():
webserver(get_airflow_default_args("webserver"))
def run_init(args):
asset_conf("init")
logging.info("Init cwl-airflow")
update_config(args)
create_folders()
export_dags()
logging.info("Init Airflow DB")
with Mute():
initdb(argparse.Namespace())
def exit_if_unsupported_feature(cwl_file, exit_code=33):
with Mute():
tool = load_cwl(cwl_file).tool
if tool["class"] == "Workflow" and [
step["id"] for step in tool["steps"] if "scatter" in step
]:
logging.warning(
"Failed to submit workflow\n- {} - scatter is not supported".
format(cwl_file))
sys.exit(exit_code)
def submit_job(args):
asset_conf()
logging.info(
"Load workflow\n- workflow: {workflow}\n- job: {job}\n- uid: {uid}"
.format(**vars(args)))
exit_if_unsupported_feature(args.workflow)
export_job_file(args)
if getattr(args, "run", None):
logging.info("Run Airflow Scheduler")
with Mute():
add_run_info(args)
scheduler(args)
print(get_workflow_output(args.dag_id))
def main(argsl=None):
if argsl is None:
argsl = sys.argv[1:]
argsl.append("") # To avoid raising error when argsl is empty
args, _ = arg_parser().parse_known_args(argsl)
args = normalize_args(args,
skip_list=[
"func", "uid", "limit", "dag_timeout", "threads",
"web_interval", "web_workers", "run", "auto",
"manual", "list", "quiet"
])
with Mute():
reset_root_logger(args.quiet)
args.func(args)
def start_background_scheduler():
logging.info("Starting Airflow Scheduler in background")
scheduler_thread = Process(target=scheduler,
args=(get_airflow_default_args("scheduler"), ))
with Mute():
scheduler_thread.start()
#!/usr/bin/env python
from __future__ import print_function
import sys
import os
import argparse
import uuid
import logging
from cwl_airflow.utils.mute import Mute
with Mute(): # Suppress output
from airflow.settings import AIRFLOW_HOME
from cwl_airflow.utils.logger import reset_root_logger
from airflow.bin.cli import scheduler, webserver, initdb
from cwl_airflow.utils.func import (
export_job_file, add_run_info, update_config, export_dags,
create_folders, get_demo_workflow, get_updated_args,
start_background_scheduler, get_airflow_default_args, clean_jobs,
get_webserver_url, asset_conf, copy_demo_workflows)
from cwl_airflow.utils.utils import (get_workflow_output, normalize_args,
exit_if_unsupported_feature,
conf_get_default)
def arg_parser():
parent_parser = argparse.ArgumentParser(add_help=False)
general_parser = argparse.ArgumentParser(description='cwl-airflow')
general_parser.add_argument(
"-q",
"--quiet",
dest='quiet',
action="store_true",