def _run_task_by_executor(args, dag, ti):
"""
Sends the task to the executor for execution. This can result in the task being started by another host
if the executor implementation does
"""
pickle_id = None
if args.ship_dag:
try:
# Running remotely, so pickling the DAG
with create_session() as session:
pickle = DagPickle(dag)
session.add(pickle)
pickle_id = pickle.id
# TODO: This should be written to a log
print('Pickled dag {dag} as pickle_id: {pickle_id}'.format(
dag=dag, pickle_id=pickle_id))
except Exception as e:
print('Could not pickle the DAG')
print(e)
raise e
executor = ExecutorLoader.get_default_executor()
executor.start()
print("Sending to executor.")
executor.queue_task_instance(
ti,
mark_success=args.mark_success,
pickle_id=pickle_id,
ignore_all_deps=args.ignore_all_dependencies,
ignore_depends_on_past=args.ignore_depends_on_past,
ignore_task_deps=args.ignore_dependencies,
ignore_ti_state=args.force,
pool=args.pool)
executor.heartbeat()
executor.end()
def test_should_support_plugins(self):
with conf_vars({
("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor"
}):
executor = ExecutorLoader.get_default_executor()
self.assertIsNotNone(executor)
self.assertIn("FakeExecutor", executor.__class__.__name__)
def __init__(
self,
dag_folder=None,
executor=None,
include_examples=conf.getboolean('core', 'LOAD_EXAMPLES'),
safe_mode=conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE'),
store_serialized_dags=False,
):
# do not use default arg in signature, to fix import cycle on plugin load
if executor is None:
from airflow.executors.executor_loader import ExecutorLoader
executor = ExecutorLoader.get_default_executor()
dag_folder = dag_folder or settings.DAGS_FOLDER
self.dag_folder = dag_folder
self.dags = {}
# the file's last modified timestamp when we last read it
self.file_last_changed = {}
self.executor = executor
self.import_errors = {}
self.has_logged = False
self.store_serialized_dags = store_serialized_dags
self.collect_dags(
dag_folder=dag_folder,
include_examples=include_examples,
safe_mode=safe_mode)
def get_executor_under_test(dotted_path):
"""
Create and return a MockExecutor
"""
from airflow.executors.executor_loader import ExecutorLoader
if dotted_path == "MockExecutor":
try:
# Run against master and 1.10.x releases
from tests.test_utils.mock_executor import MockExecutor as Executor
except ImportError:
from tests.executors.test_executor import TestExecutor as Executor
else:
Executor = ExecutorLoader.load_executor(dotted_path)
# Change this to try other executors
class ShortCircuitExecutor(ShortCircuitExecutorMixin, Executor):
"""
Placeholder class that implements the inheritance hierarchy
"""
scheduler_job = None
return ShortCircuitExecutor
def test_should_support_custom_path(self):
with conf_vars({
("core", "executor"): f"tests.executors.test_executor_loader.FakeExecutor"
}):
executor = ExecutorLoader.get_default_executor()
self.assertIsNotNone(executor)
self.assertIn("FakeExecutor", executor.__class__.__name__)
def test_should_support_executor_from_core(self, executor_name):
with conf_vars({
("core", "executor"): executor_name
}):
executor = ExecutorLoader.get_default_executor()
self.assertIsNotNone(executor)
self.assertIn(executor_name, executor.__class__.__name__)
def __init__(self, executor=None, heartrate=None, *args, **kwargs):
self.hostname = get_hostname()
self.executor = executor or ExecutorLoader.get_default_executor()
self.executor_class = self.executor.__class__.__name__
self.start_date = timezone.utcnow()
self.latest_heartbeat = timezone.utcnow()
if heartrate is not None:
self.heartrate = heartrate
self.unixname = getpass.getuser()
self.max_tis_per_query = conf.getint('scheduler', 'max_tis_per_query')
super().__init__(*args, **kwargs)
def test_should_support_plugin(self):
executors_modules.append(
make_module('airflow.executors.' + TEST_PLUGIN_NAME,
[FakeExecutor]))
self.addCleanup(self.remove_executor_module)
with conf_vars({
("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor"
}):
executor = ExecutorLoader.get_default_executor()
self.assertIsNotNone(executor)
self.assertIn("FakeExecutor", executor.__class__.__name__)
def test_should_support_plugin(self):
plugins_manager.plugins = [
FakePlugin()
]
self.addCleanup(self.remove_executor_plugins)
with conf_vars({
("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor"
}):
executor = ExecutorLoader.get_default_executor()
self.assertIsNotNone(executor)
self.assertIn("FakeExecutor", executor.__class__.__name__)
def _run(args, dag, ti):
if args.local:
run_job = jobs.LocalTaskJob(
task_instance=ti,
mark_success=args.mark_success,
pickle_id=args.pickle,
ignore_all_deps=args.ignore_all_dependencies,
ignore_depends_on_past=args.ignore_depends_on_past,
ignore_task_deps=args.ignore_dependencies,
ignore_ti_state=args.force,
pool=args.pool)
run_job.run()
elif args.raw:
ti._run_raw_task( # pylint: disable=protected-access
mark_success=args.mark_success,
job_id=args.job_id,
pool=args.pool,
)
else:
pickle_id = None
if args.ship_dag:
try:
# Running remotely, so pickling the DAG
with db.create_session() as session:
pickle = DagPickle(dag)
session.add(pickle)
pickle_id = pickle.id
# TODO: This should be written to a log
print('Pickled dag {dag} as pickle_id: {pickle_id}'.format(
dag=dag, pickle_id=pickle_id))
except Exception as e:
print('Could not pickle the DAG')
print(e)
raise e
executor = ExecutorLoader.get_default_executor()
executor.start()
print("Sending to executor.")
executor.queue_task_instance(
ti,
mark_success=args.mark_success,
pickle_id=pickle_id,
ignore_all_deps=args.ignore_all_dependencies,
ignore_depends_on_past=args.ignore_depends_on_past,
ignore_task_deps=args.ignore_dependencies,
ignore_ti_state=args.force,
pool=args.pool)
executor.heartbeat()
executor.end()
def get_executor_under_test(dotted_path):
"""
Create and return a MockExecutor
"""
from airflow.executors.executor_loader import ExecutorLoader
if dotted_path == "MockExecutor":
from tests.test_utils.mock_executor import MockExecutor as executor
else:
executor = ExecutorLoader.load_executor(dotted_path)
executor_cls = type(executor)
# Change this to try other executors
class ShortCircuitExecutor(ShortCircuitExecutorMixin, executor_cls):
"""
Placeholder class that implements the inheritance hierarchy
"""
scheduler_job = None
return ShortCircuitExecutor
def executor(self):
return ExecutorLoader.get_default_executor()
def test_should_support_custom_path(self):
with conf_vars({("core", "executor"): "tests.executors.test_executor_loader.FakeExecutor"}):
executor = ExecutorLoader.get_default_executor()
assert executor is not None
assert "FakeExecutor" == executor.__class__.__name__
def test_should_support_plugins(self):
with conf_vars({("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor"}):
executor = ExecutorLoader.get_default_executor()
assert executor is not None
assert "FakeExecutor" == executor.__class__.__name__
def test_should_support_executor_from_core(self, executor_name):
with conf_vars({("core", "executor"): executor_name}):
executor = ExecutorLoader.get_default_executor()
assert executor is not None
assert executor_name == executor.__class__.__name__