def main():
dbnd_bootstrap()
from airflow.models import TaskInstance
from airflow.utils.log.logging_mixin import redirect_stderr, redirect_stdout
from airflow.utils.timezone import utcnow
from dbnd import config
from dbnd._core.constants import TaskExecutorType
from dbnd._core.settings import RunConfig
from dbnd._core.task_run.task_run import TaskRun
from dbnd.tasks.basics import dbnd_sanity_check
from dbnd_examples.dbnd_airflow import bash_dag
airflow_task_log = logging.getLogger("airflow.task")
task = bash_dag.t3
execution_date = utcnow()
ti = TaskInstance(task, execution_date)
ti.init_run_context(raw=False)
logger.warning("Running with task_log %s",
airflow_task_log.handlers[0].handler.baseFilename)
with redirect_stdout(airflow_task_log, logging.INFO), redirect_stderr(
airflow_task_log, logging.WARN):
logger.warning("from redirect")
logger.warning("after patch")
with config({RunConfig.task_executor_type: TaskExecutorType.local}):
run = dbnd_sanity_check.dbnd_run()
tr = run.root_task_run # type: TaskRun
logger.warning("TR: %s %s %s", tr, tr.task_tracker_url,
tr.log.local_log_file)
def main():
try:
from airflow import conf
except ImportError:
from airflow.configuration import conf
from airflow.jobs.scheduler_job import SchedulerJob
from airflow.models import DagBag
from dbnd import dbnd_bootstrap
from dbnd._core.log.logging_utils import create_file_handler
from dbnd_airflow.executors.simple_executor import InProcessExecutor
from test_dbnd_airflow.scenarios.scheduler_perf_experiment import (
dag_folder,
dag_id,
log_scheduler,
)
dbnd_bootstrap()
conf.set("core", "unit_test_mode", "True")
logging.root.addHandler(create_file_handler(log_file=log_scheduler))
dag_bag = DagBag(dag_folder=dag_folder)
scheduler_job = SchedulerJob(
dag_ids=[dag_id],
subdir=dag_folder,
do_pickle=False,
num_runs=3,
executor=InProcessExecutor(dag_bag=dag_bag),
)
scheduler_job.run()
def dbnd_operator__execute(dbnd_operator, context):
from dbnd._core.current import try_get_databand_run
from dbnd._core.run.databand_run import DatabandRun
from targets import target
run = try_get_databand_run()
if not run:
# we are not inside dbnd run, probably we are running from native airflow
# let's try to load it:
try:
executor_config = dbnd_operator.executor_config
logger.info("context: %s", context)
logger.info("task.executor_config: %s",
dbnd_operator.executor_config)
logger.info("ti.executor_config: %s",
context["ti"].executor_config)
driver_dump = executor_config["DatabandExecutor"].get(
"dbnd_driver_dump")
print(
"Running dbnd task %s from %s" %
(dbnd_operator.dbnd_task_id, driver_dump),
file=sys.__stderr__,
)
if executor_config["DatabandExecutor"].get(
"remove_airflow_std_redirect", False):
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
dbnd_bootstrap()
dbnd_airflow_bootstrap()
run = DatabandRun.load_run(dump_file=target(driver_dump),
disable_tracking_api=False)
except Exception as e:
print(
"Failed to load dbnd task in native airflow execution! Exception: %s"
% (e, ),
file=sys.__stderr__,
)
dump_trace()
raise
with run.run_context() as dr:
task_run = run.get_task_run_by_id(dbnd_operator.dbnd_task_id)
ret_value = task_run.runner.execute(airflow_context=context)
else:
task_run = run.get_task_run_by_id(dbnd_operator.dbnd_task_id)
ret_value = task_run.runner.execute(airflow_context=context)
return ret_value
else:
logger.error(" MANUAL finished with %s", exit_code)
if __name__ == "__main__":
# this part will be executed by direct call from the heartbeat_manual task
# via Popen
print("Message via print stdout", file=sys.stdout)
print("Message via print __stderr__", file=sys.__stderr__)
print("Message via print stderr", file=sys.stderr)
import logging
logger = logging.getLogger(__name__)
logging.info("Message via logging root")
logger.info("Message via logger")
from dbnd import config
with config({"core": {"tracker": ""}}):
dbnd_bootstrap()
context = get_databand_context()
print("CONTEXT CONTEXT CONTEXT")
print("CONTEXT :Message via print stdout", file=sys.stdout)
print("CONTEXT :Message via print __stderr__", file=sys.__stderr__)
print("CONTEXT :Message via print stderr", file=sys.stderr)
logging.info("CONTEXT : Message via logging root")
logger.info("CONTEXT : Message via logger")
tracking_store = get_databand_context().tracking_store