def dag_trigger(args):
"""
Creates a dag run for the specified dag
"""
api_client = get_current_api_client()
log = LoggingMixin().log
try:
message = api_client.trigger_dag(dag_id=args.dag_id,
run_id=args.run_id,
conf=args.conf,
execution_date=args.exec_date)
except OSError as err:
log.error(err)
raise AirflowException(err)
log.info(message)
def dag_delete(args):
"""
Deletes all DB records related to the specified dag
"""
api_client = get_current_api_client()
log = LoggingMixin().log
if args.yes or input(
"This will drop all existing records related to the specified DAG. "
"Proceed? (y/n)").upper() == "Y":
try:
message = api_client.delete_dag(dag_id=args.dag_id)
except OSError as err:
log.error(err)
raise AirflowException(err)
log.info(message)
else:
print("Bail.")
catchup=False,
max_active_runs=1,
schedule_interval="@daily",
concurrency=4,
)
log = LoggingMixin().log
ddl_sql_file_name = "../create_tables.sql"
sql_path = path.join(path.dirname(path.abspath(__file__)), ddl_sql_file_name)
sql_content = None
try:
with open(sql_path) as reader:
sql_content = reader.read()
except Exception as err:
log.error(f"Failure when reading file {sql_path}")
def fetch_riot_items_data():
api_key = Variable.get("RIOT_API_KEY")
# VALIDATORS
validator_staging_game_match = DataQualityValidator(
sql_statement=f"SELECT COUNT(*) FROM {table_name_staging_game_match}",
result_to_assert=0,
should_assert_for_equality=False,
)
validator_fact_game_match = DataQualityValidator(
sql_statement=f"SELECT COUNT(*) FROM {table_name_fact_game_match}",
result_to_assert=0,
should_assert_for_equality=False,
def worker(args):
"""Starts Airflow Celery worker"""
env = os.environ.copy()
env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME
if not settings.validate_session():
log = LoggingMixin().log
log.error("Worker exiting... database connection precheck failed! ")
sys.exit(1)
# Celery worker
from airflow.executors.celery_executor import app as celery_app
from celery.bin import worker # pylint: disable=redefined-outer-name
autoscale = args.autoscale
if autoscale is None and conf.has_option("celery", "worker_autoscale"):
autoscale = conf.get("celery", "worker_autoscale")
worker = worker.worker(app=celery_app) # pylint: disable=redefined-outer-name
options = {
'optimization': 'fair',
'O': 'fair',
'queues': args.queues,
'concurrency': args.concurrency,
'autoscale': autoscale,
'hostname': args.celery_hostname,
'loglevel': conf.get('core', 'LOGGING_LEVEL'),
}
if conf.has_option("celery", "pool"):
options["pool"] = conf.get("celery", "pool")
if args.daemon:
pid, stdout, stderr, log_file = setup_locations(
"worker", args.pid, args.stdout, args.stderr, args.log_file)
handle = setup_logging(log_file)
stdout = open(stdout, 'w+')
stderr = open(stderr, 'w+')
ctx = daemon.DaemonContext(
pidfile=TimeoutPIDLockFile(pid, -1),
files_preserve=[handle],
stdout=stdout,
stderr=stderr,
)
with ctx:
sub_proc = subprocess.Popen(['airflow', 'serve_logs'],
env=env,
close_fds=True)
worker.run(**options)
sub_proc.kill()
stdout.close()
stderr.close()
else:
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
sub_proc = subprocess.Popen(['airflow', 'serve_logs'],
env=env,
close_fds=True)
worker.run(**options)
sub_proc.kill()