def dag_trigger(args): """ Creates a dag run for the specified dag """ api_client = get_current_api_client() log = LoggingMixin().log try: message = api_client.trigger_dag(dag_id=args.dag_id, run_id=args.run_id, conf=args.conf, execution_date=args.exec_date) except OSError as err: log.error(err) raise AirflowException(err) log.info(message)
def dag_delete(args): """ Deletes all DB records related to the specified dag """ api_client = get_current_api_client() log = LoggingMixin().log if args.yes or input( "This will drop all existing records related to the specified DAG. " "Proceed? (y/n)").upper() == "Y": try: message = api_client.delete_dag(dag_id=args.dag_id) except OSError as err: log.error(err) raise AirflowException(err) log.info(message) else: print("Bail.")
catchup=False, max_active_runs=1, schedule_interval="@daily", concurrency=4, ) log = LoggingMixin().log ddl_sql_file_name = "../create_tables.sql" sql_path = path.join(path.dirname(path.abspath(__file__)), ddl_sql_file_name) sql_content = None try: with open(sql_path) as reader: sql_content = reader.read() except Exception as err: log.error(f"Failure when reading file {sql_path}") def fetch_riot_items_data(): api_key = Variable.get("RIOT_API_KEY") # VALIDATORS validator_staging_game_match = DataQualityValidator( sql_statement=f"SELECT COUNT(*) FROM {table_name_staging_game_match}", result_to_assert=0, should_assert_for_equality=False, ) validator_fact_game_match = DataQualityValidator( sql_statement=f"SELECT COUNT(*) FROM {table_name_fact_game_match}", result_to_assert=0, should_assert_for_equality=False,
def worker(args): """Starts Airflow Celery worker""" env = os.environ.copy() env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME if not settings.validate_session(): log = LoggingMixin().log log.error("Worker exiting... database connection precheck failed! ") sys.exit(1) # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker # pylint: disable=redefined-outer-name autoscale = args.autoscale if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") worker = worker.worker(app=celery_app) # pylint: disable=redefined-outer-name options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('core', 'LOGGING_LEVEL'), } if conf.has_option("celery", "pool"): options["pool"] = conf.get("celery", "pool") if args.daemon: pid, stdout, stderr, log_file = setup_locations( "worker", args.pid, args.stdout, args.stderr, args.log_file) handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(pid, -1), files_preserve=[handle], stdout=stdout, stderr=stderr, ) with ctx: sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True) worker.run(**options) sub_proc.kill() stdout.close() stderr.close() else: signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True) worker.run(**options) sub_proc.kill()