def _run_task_by_executor(args, dag, ti): """ Sends the task to the executor for execution. This can result in the task being started by another host if the executor implementation does """ pickle_id = None if args.ship_dag: try: # Running remotely, so pickling the DAG with create_session() as session: pickle = DagPickle(dag) session.add(pickle) pickle_id = pickle.id # TODO: This should be written to a log print('Pickled dag {dag} as pickle_id: {pickle_id}'.format( dag=dag, pickle_id=pickle_id)) except Exception as e: print('Could not pickle the DAG') print(e) raise e executor = ExecutorLoader.get_default_executor() executor.start() print("Sending to executor.") executor.queue_task_instance( ti, mark_success=args.mark_success, pickle_id=pickle_id, ignore_all_deps=args.ignore_all_dependencies, ignore_depends_on_past=args.ignore_depends_on_past, ignore_task_deps=args.ignore_dependencies, ignore_ti_state=args.force, pool=args.pool) executor.heartbeat() executor.end()
def test_should_support_plugins(self): with conf_vars({ ("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor" }): executor = ExecutorLoader.get_default_executor() self.assertIsNotNone(executor) self.assertIn("FakeExecutor", executor.__class__.__name__)
def __init__( self, dag_folder=None, executor=None, include_examples=conf.getboolean('core', 'LOAD_EXAMPLES'), safe_mode=conf.getboolean('core', 'DAG_DISCOVERY_SAFE_MODE'), store_serialized_dags=False, ): # do not use default arg in signature, to fix import cycle on plugin load if executor is None: from airflow.executors.executor_loader import ExecutorLoader executor = ExecutorLoader.get_default_executor() dag_folder = dag_folder or settings.DAGS_FOLDER self.dag_folder = dag_folder self.dags = {} # the file's last modified timestamp when we last read it self.file_last_changed = {} self.executor = executor self.import_errors = {} self.has_logged = False self.store_serialized_dags = store_serialized_dags self.collect_dags( dag_folder=dag_folder, include_examples=include_examples, safe_mode=safe_mode)
def get_executor_under_test(dotted_path): """ Create and return a MockExecutor """ from airflow.executors.executor_loader import ExecutorLoader if dotted_path == "MockExecutor": try: # Run against master and 1.10.x releases from tests.test_utils.mock_executor import MockExecutor as Executor except ImportError: from tests.executors.test_executor import TestExecutor as Executor else: Executor = ExecutorLoader.load_executor(dotted_path) # Change this to try other executors class ShortCircuitExecutor(ShortCircuitExecutorMixin, Executor): """ Placeholder class that implements the inheritance hierarchy """ scheduler_job = None return ShortCircuitExecutor
def test_should_support_custom_path(self): with conf_vars({ ("core", "executor"): f"tests.executors.test_executor_loader.FakeExecutor" }): executor = ExecutorLoader.get_default_executor() self.assertIsNotNone(executor) self.assertIn("FakeExecutor", executor.__class__.__name__)
def test_should_support_executor_from_core(self, executor_name): with conf_vars({ ("core", "executor"): executor_name }): executor = ExecutorLoader.get_default_executor() self.assertIsNotNone(executor) self.assertIn(executor_name, executor.__class__.__name__)
def __init__(self, executor=None, heartrate=None, *args, **kwargs): self.hostname = get_hostname() self.executor = executor or ExecutorLoader.get_default_executor() self.executor_class = self.executor.__class__.__name__ self.start_date = timezone.utcnow() self.latest_heartbeat = timezone.utcnow() if heartrate is not None: self.heartrate = heartrate self.unixname = getpass.getuser() self.max_tis_per_query = conf.getint('scheduler', 'max_tis_per_query') super().__init__(*args, **kwargs)
def test_should_support_plugin(self): executors_modules.append( make_module('airflow.executors.' + TEST_PLUGIN_NAME, [FakeExecutor])) self.addCleanup(self.remove_executor_module) with conf_vars({ ("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor" }): executor = ExecutorLoader.get_default_executor() self.assertIsNotNone(executor) self.assertIn("FakeExecutor", executor.__class__.__name__)
def test_should_support_plugin(self): plugins_manager.plugins = [ FakePlugin() ] self.addCleanup(self.remove_executor_plugins) with conf_vars({ ("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor" }): executor = ExecutorLoader.get_default_executor() self.assertIsNotNone(executor) self.assertIn("FakeExecutor", executor.__class__.__name__)
def _run(args, dag, ti): if args.local: run_job = jobs.LocalTaskJob( task_instance=ti, mark_success=args.mark_success, pickle_id=args.pickle, ignore_all_deps=args.ignore_all_dependencies, ignore_depends_on_past=args.ignore_depends_on_past, ignore_task_deps=args.ignore_dependencies, ignore_ti_state=args.force, pool=args.pool) run_job.run() elif args.raw: ti._run_raw_task( # pylint: disable=protected-access mark_success=args.mark_success, job_id=args.job_id, pool=args.pool, ) else: pickle_id = None if args.ship_dag: try: # Running remotely, so pickling the DAG with db.create_session() as session: pickle = DagPickle(dag) session.add(pickle) pickle_id = pickle.id # TODO: This should be written to a log print('Pickled dag {dag} as pickle_id: {pickle_id}'.format( dag=dag, pickle_id=pickle_id)) except Exception as e: print('Could not pickle the DAG') print(e) raise e executor = ExecutorLoader.get_default_executor() executor.start() print("Sending to executor.") executor.queue_task_instance( ti, mark_success=args.mark_success, pickle_id=pickle_id, ignore_all_deps=args.ignore_all_dependencies, ignore_depends_on_past=args.ignore_depends_on_past, ignore_task_deps=args.ignore_dependencies, ignore_ti_state=args.force, pool=args.pool) executor.heartbeat() executor.end()
def get_executor_under_test(dotted_path): """ Create and return a MockExecutor """ from airflow.executors.executor_loader import ExecutorLoader if dotted_path == "MockExecutor": from tests.test_utils.mock_executor import MockExecutor as executor else: executor = ExecutorLoader.load_executor(dotted_path) executor_cls = type(executor) # Change this to try other executors class ShortCircuitExecutor(ShortCircuitExecutorMixin, executor_cls): """ Placeholder class that implements the inheritance hierarchy """ scheduler_job = None return ShortCircuitExecutor
def executor(self): return ExecutorLoader.get_default_executor()
def test_should_support_custom_path(self): with conf_vars({("core", "executor"): "tests.executors.test_executor_loader.FakeExecutor"}): executor = ExecutorLoader.get_default_executor() assert executor is not None assert "FakeExecutor" == executor.__class__.__name__
def test_should_support_plugins(self): with conf_vars({("core", "executor"): f"{TEST_PLUGIN_NAME}.FakeExecutor"}): executor = ExecutorLoader.get_default_executor() assert executor is not None assert "FakeExecutor" == executor.__class__.__name__
def test_should_support_executor_from_core(self, executor_name): with conf_vars({("core", "executor"): executor_name}): executor = ExecutorLoader.get_default_executor() assert executor is not None assert executor_name == executor.__class__.__name__