def run_flow(self) -> None: """ Run the flow from specified flow_file_path location using the default executor """ # Call on_start callback if specified if self.on_start: self.on_start() try: from prefect.engine import ( get_default_flow_runner_class, get_default_executor_class, ) # Load serialized flow from file and run it with the executor with open( prefect.context.get("flow_file_path", "/root/.prefect/flow_env.prefect"), "rb", ) as f: flow = cloudpickle.load(f) runner_cls = get_default_flow_runner_class() executor_cls = get_default_executor_class() runner_cls(flow=flow).run(executor=executor_cls) except Exception as exc: self.logger.exception( "Unexpected error raised during flow run: {}".format(exc)) raise exc finally: # Call on_exit callback if specified if self.on_exit: self.on_exit()
def execute( # type: ignore self, storage: "Storage", flow_location: str, **kwargs: Any) -> None: """ Run a flow from the `flow_location` here using the specified executor and executor kwargs. Args: - storage (Storage): the storage object that contains information relating to where and how the flow is stored - flow_location (str): the location of the Flow to execute - **kwargs (Any): additional keyword arguments to pass to the runner """ try: from prefect.engine import ( get_default_executor_class, get_default_flow_runner_class, ) # Load serialized flow from file and run it with a DaskExecutor with open(flow_location, "rb") as f: flow = cloudpickle.load(f) with set_temporary_config( {"engine.executor.default_class": self.executor}): executor = get_default_executor_class() executor = executor(**self.executor_kwargs) runner_cls = get_default_flow_runner_class() runner_cls(flow=flow).run(executor=executor) except Exception as exc: self.logger.exception( "Unexpected error raised during flow run: {}".format(exc)) raise exc
def test_default_executor_responds_to_config(): with utilities.configuration.set_temporary_config({ "engine.executor.default_class": "prefect.engine.executors.LocalDaskExecutor" }): assert engine.get_default_executor_class( ) is engine.executors.LocalDaskExecutor
def execute(self, flow: "Flow", **kwargs: Any) -> None: # type: ignore """ Run the provided flow here using the specified executor and executor kwargs. Args: - flow (Flow): the Flow object - **kwargs (Any): additional keyword arguments to pass to the runner """ # Call on_start callback if specified if self.on_start: self.on_start() try: from prefect.engine import ( get_default_executor_class, get_default_flow_runner_class, ) # Run flow with default executor class with set_temporary_config( {"engine.executor.default_class": self.executor}): executor = get_default_executor_class() executor = executor(**self.executor_kwargs) runner_cls = get_default_flow_runner_class() runner_cls(flow=flow).run(executor=executor) except Exception as exc: self.logger.exception( "Unexpected error raised during flow run: {}".format(exc)) raise exc finally: # Call on_exit callback if specified if self.on_exit: self.on_exit()
def run_flow(self) -> None: """ Run the flow from specified flow_file_path location using the default executor """ # Call on_start callback if specified if self.on_start: self.on_start() try: from prefect.engine import ( get_default_flow_runner_class, get_default_executor_class, ) # Load serialized flow from file and run it with the executor with open( prefect.context.get("flow_file_path", "/root/.prefect/flow_env.prefect"), "rb", ) as f: flow = cloudpickle.load(f) ## populate global secrets secrets = prefect.context.get("secrets", {}) for secret in flow.storage.secrets: secrets[secret] = prefect.tasks.secrets.PrefectSecret( name=secret).run() with prefect.context(secrets=secrets): runner_cls = get_default_flow_runner_class() executor_cls = get_default_executor_class()( **self.executor_kwargs) runner_cls(flow=flow).run(executor=executor_cls) except Exception as exc: self.logger.exception( "Unexpected error raised during flow run: {}".format(exc)) raise exc finally: # Call on_exit callback if specified if self.on_exit: self.on_exit()
def run_flow(self) -> None: """ Run the flow using the default executor Raises: - ValueError: if no `flow_run_id` is found in context """ # Call on_start callback if specified if self.on_start: self.on_start() try: from prefect.engine import ( get_default_flow_runner_class, get_default_executor_class, ) flow_run_id = prefect.context.get("flow_run_id") if not flow_run_id: raise ValueError("No flow run ID found in context.") query = { "query": { with_args("flow_run", { "where": { "id": { "_eq": flow_run_id } } }): { "flow": { "name": True, "storage": True, }, } } } client = Client() result = client.graphql(query) flow_run = result.data.flow_run[0] flow_data = flow_run.flow storage_schema = prefect.serialization.storage.StorageSchema() storage = storage_schema.load(flow_data.storage) ## populate global secrets secrets = prefect.context.get("secrets", {}) for secret in storage.secrets: secrets[secret] = prefect.tasks.secrets.PrefectSecret( name=secret).run() with prefect.context(secrets=secrets): flow = storage.get_flow(storage.flows[flow_data.name]) runner_cls = get_default_flow_runner_class() if hasattr(self, "executor_kwargs"): executor_cls = get_default_executor_class()( **self.executor_kwargs) # type: ignore else: executor_cls = get_default_executor_class() runner_cls(flow=flow).run(executor=executor_cls) except Exception as exc: self.logger.exception( "Unexpected error raised during flow run: {}".format(exc)) raise exc finally: # Call on_exit callback if specified if self.on_exit: self.on_exit()
def test_default_executor(): assert engine.get_default_executor_class() is engine.executors.LocalExecutor
def test_default_executor_with_bad_config(): with utilities.configuration.set_temporary_config( {"engine.executor.default_class": "prefect.engine.bad import path"} ): with pytest.warns(UserWarning): assert engine.get_default_executor_class() is engine.executors.LocalExecutor
def test_default_executor(): assert engine.get_default_executor_class( ) is engine.executors.SynchronousExecutor
def test_default_executor_responds_to_config_object(): with utilities.configuration.set_temporary_config( {"engine.executor.default_class": engine.executors.LocalExecutor}): assert engine.get_default_executor_class( ) is engine.executors.LocalExecutor