def deploy_flows(self, flow_runs: list) -> None: """ Deploy flow runs on your local machine as Docker containers Args: - flow_runs (list): A list of GraphQLResult flow run objects """ for flow_run in flow_runs: storage = StorageSchema().load(flow_run.flow.storage) if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format(flow_run.id) ) continue env_vars = self.populate_env_vars(flow_run=flow_run) if not self.no_pull: self.logger.debug("Pulling image {}...".format(storage.name)) try: self.docker_client.pull(storage.name) except docker.errors.APIError: self.logger.error("Issue pulling image {}".format(storage.name)) # Create a container container = self.docker_client.create_container( storage.name, command="prefect execute cloud-flow", environment=env_vars ) # Start the container self.docker_client.start(container=container.get("Id"))
def deploy_flows(self, flow_runs: list) -> None: """ Deploy flow runs on your local machine as Docker containers Args: - flow_runs (list): A list of GraphQLResult flow run objects """ for flow_run in flow_runs: self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) storage = StorageSchema().load(flow_run.flow.storage) if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): msg = "Storage for flow run {} is not of type Docker.".format( flow_run.id) state_msg = "Agent {} failed to run flow: ".format( self.name) + msg self.client.set_flow_run_state(flow_run.id, version=flow_run.version, state=Failed(state_msg)) self.logger.error(msg) continue env_vars = self.populate_env_vars(flow_run=flow_run) if not self.no_pull and storage.registry_url: self.logger.info("Pulling image {}...".format(storage.name)) try: pull_output = self.docker_client.pull(storage.name, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info("Successfully pulled image {}...".format( storage.name)) except docker.errors.APIError as exc: msg = "Issue pulling image {}".format(storage.name) state_msg = ( "Agent {} failed to pull image for flow: ".format( self.name) + msg) self.client.set_flow_run_state(flow_run.id, version=flow_run.version, state=Failed(msg)) self.logger.error(msg) # Create a container self.logger.debug("Creating Docker container {}".format( storage.name)) container = self.docker_client.create_container( storage.name, command="prefect execute cloud-flow", environment=env_vars) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) self.docker_client.start(container=container.get("Id"))
def replace_job_spec_yaml(self, flow_run: GraphQLResult) -> dict: """ Populate metadata and variables in the job_spec.yaml file for flow runs Args: - flow_run (GraphQLResult): A flow run object Returns: - dict: a dictionary representing the populated yaml object """ with open(path.join(path.dirname(__file__), "job_spec.yaml"), "r") as job_file: job = yaml.safe_load(job_file) identifier = str(uuid.uuid4())[:8] job_name = "prefect-job-{}".format(identifier) # Populate job metadata for identification job["metadata"]["name"] = job_name job["metadata"]["labels"]["app"] = job_name job["metadata"]["labels"]["identifier"] = identifier job["metadata"]["labels"]["flow_run_id"] = flow_run.id # type: ignore job["metadata"]["labels"]["flow_id"] = flow_run.flow.id # type: ignore job["spec"]["template"]["metadata"]["labels"]["app"] = job_name job["spec"]["template"]["metadata"]["labels"][ "flow_run_id" ] = flow_run.id # type: ignore job["spec"]["template"]["metadata"]["labels"]["identifier"] = identifier # Use flow storage image for job job["spec"]["template"]["spec"]["containers"][0]["image"] = ( StorageSchema().load(flow_run.flow.storage).name # type: ignore ) self.logger.debug( "Using image {} for job".format( StorageSchema().load(flow_run.flow.storage).name # type: ignore ) ) # Populate environment variables for flow run execution env = job["spec"]["template"]["spec"]["containers"][0]["env"] env[0]["value"] = config.cloud.api or "https://api.prefect.io" env[1]["value"] = config.cloud.agent.auth_token env[2]["value"] = flow_run.id # type: ignore env[3]["value"] = os.getenv("NAMESPACE", "default") # Use image pull secrets if provided job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"] = os.getenv( "IMAGE_PULL_SECRETS", "" ) return job
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info( "Deploying flow run {}".format(flow_run.id) # type: ignore ) storage = StorageSchema().load(flow_run.flow.storage) if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format(flow_run.id) ) raise ValueError("Unsupported Storage type") env_vars = self.populate_env_vars(flow_run=flow_run) if not self.no_pull and storage.registry_url: self.logger.info("Pulling image {}...".format(storage.name)) pull_output = self.docker_client.pull( storage.name, stream=True, decode=True ) for line in pull_output: self.logger.debug(line) self.logger.info("Successfully pulled image {}...".format(storage.name)) # Create a container self.logger.debug("Creating Docker container {}".format(storage.name)) container = self.docker_client.create_container( storage.name, command="prefect execute cloud-flow", environment=env_vars ) # Start the container self.logger.debug( "Starting Docker container with ID {}".format(container.get("Id")) ) self.docker_client.start(container=container.get("Id")) self.logger.debug("Docker container {} started".format(container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ storage = StorageSchema().load(flow_run.flow.storage) if isinstance(storage, Docker): self.logger.error( "Flow run %s has an unsupported storage type: `%s`", flow_run.id, type(storage).__name__, ) raise TypeError("Unsupported Storage type: %s" % type(storage).__name__) run_config = self._get_run_config(flow_run, LocalRun) env = self.populate_env_vars(flow_run, run_config=run_config) working_dir = None if run_config is None else run_config.working_dir if working_dir and not os.path.exists(working_dir): msg = f"Flow run {flow_run.id} has a nonexistent `working_dir` configured: {working_dir}" self.logger.error(msg) raise ValueError(msg) stdout = sys.stdout if self.show_flow_logs else DEVNULL # note: we will allow these processes to be orphaned if the agent were to exit # before the flow runs have completed. The lifecycle of the agent should not # dictate the lifecycle of the flow run. However, if the user has elected to # show flow logs, these log entries will continue to stream to the users terminal # until these child processes exit, even if the agent has already exited. p = Popen( [sys.executable, "-m", "prefect", "execute", "flow-run"], stdout=stdout, stderr=STDOUT, env=env, cwd=working_dir, ) self.processes.add(p) self.logger.debug( "Submitted flow run {} to process PID {}".format(flow_run.id, p.pid) ) if self.block_during_deploy: self.logger.debug(f"Waiting flow run {flow_run.id} to complete") # We use communicate rather than .wait() to avoid potential hanging # of processes. See Popen docs for more info: # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.wait p.communicate() self.logger.debug(f"Flow run {flow_run.id} completed") return "PID: {}".format(p.pid)
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) if not isinstance( StorageSchema().load(flow_run.flow.storage), (Local, Azure, GCS, S3, GitHub, Webhook), ): self.logger.error( "Storage for flow run {} is not a supported type.".format( flow_run.id)) raise ValueError("Unsupported Storage type") env_vars = self.populate_env_vars(flow_run=flow_run) current_env = os.environ.copy() current_env.update(env_vars) python_path = [] if current_env.get("PYTHONPATH"): python_path.append(current_env.get("PYTHONPATH")) python_path.append(os.getcwd()) if self.import_paths: python_path += self.import_paths current_env["PYTHONPATH"] = ":".join(python_path) stdout = sys.stdout if self.show_flow_logs else DEVNULL # note: we will allow these processes to be orphaned if the agent were to exit # before the flow runs have completed. The lifecycle of the agent should not # dictate the lifecycle of the flow run. However, if the user has elected to # show flow logs, these log entries will continue to stream to the users terminal # until these child processes exit, even if the agent has already exited. p = Popen( get_flow_run_command(flow_run).split(" "), stdout=stdout, stderr=STDOUT, env=current_env, ) self.processes.add(p) self.logger.debug("Submitted flow run {} to process PID {}".format( flow_run.id, p.pid)) return "PID: {}".format(p.pid)
def replace_job_spec_json(self, flow_run: GraphQLResult) -> dict: """ Populate metadata and variables in the job_spec.nomad file for flow runs Args: - flow_run (GraphQLResult): A flow run objects Returns: - dict: a dictionary representing the populated json object """ with open(path.join(path.dirname(__file__), "job_spec.nomad"), "r") as job_file: job = json.load(job_file) job["Job"]["ID"] = flow_run.id # type: ignore job["Job"]["Name"] = "prefect-job-{}".format(str(uuid.uuid4())[:8]) job["Job"]["TaskGroups"][0]["Name"] = "prefect-job-{}".format( flow_run.id # type: ignore ) job["Job"]["TaskGroups"][0]["Tasks"][0][ "Name"] = flow_run.id # type: ignore job["Job"]["TaskGroups"][0]["Tasks"][0]["Config"]["image"] = ( StorageSchema().load(flow_run.flow.storage).name # type: ignore ) env = job["Job"]["TaskGroups"][0]["Tasks"][0]["Env"] env["PREFECT__CLOUD__API"] = config.cloud.api or "https://api.prefect.io" env["PREFECT__CLOUD__AGENT__AUTH_TOKEN"] = config.cloud.agent.auth_token env["PREFECT__CONTEXT__FLOW_RUN_ID"] = flow_run.id # type: ignore env["PREFECT__CONTEXT__NAMESPACE"] = os.getenv("NAMESPACE", "default") return job
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on to a k8s cluster as jobs Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) # Require Docker storage if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format( flow_run.id)) raise ValueError("Unsupported Storage type") job_spec = self.replace_job_spec_yaml(flow_run) self.logger.debug("Creating namespaced job {}".format( job_spec["metadata"]["name"])) job = self.batch_client.create_namespaced_job( namespace=self.namespace or os.getenv("NAMESPACE", "default"), body=job_spec) self.logger.debug("Job {} created".format(job.metadata.name)) return "Job {}".format(job.metadata.name)
def deploy_flows(self, flow_runs: list) -> None: """ Deploy flow runs on to a k8s cluster as jobs Args: - flow_runs (list): A list of GraphQLResult flow run objects """ for flow_run in flow_runs: self.logger.debug("Deploying flow run {}".format( flow_run.id) # type: ignore ) # Require Docker storage if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format( flow_run.id)) continue job_spec = self.replace_job_spec_yaml(flow_run) self.logger.debug("Creating namespaced job {}".format( job_spec["metadata"]["name"])) self.batch_client.create_namespaced_job(namespace=os.getenv( "NAMESPACE", "default"), body=job_spec)
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on to a Nomad cluster as jobs Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format( flow_run.id)) raise ValueError("Unsupported Storage type") # 'import requests' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import requests job_spec = self.replace_job_spec_json(flow_run) nomad_host = os.getenv("NOMAD_HOST", "http://127.0.0.1:4646") requests.post(path.join(nomad_host, "v1/jobs"), json=job_spec) return "Job ID: {}".format(job_spec["Job"]["ID"])
def deploy_flows(self, flow_runs: list) -> None: """ Deploy flow runs to Fargate Args: - flow_runs (list): A list of GraphQLResult flow run objects """ for flow_run in flow_runs: self.logger.debug("Deploying flow run {}".format( flow_run.id) # type: ignore ) # Require Docker storage if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format( flow_run.id)) continue # check if task definition exists self.logger.debug("Checking for task definition") if not self._verify_task_definition_exists(flow_run): self.logger.debug("No task definition found") self._create_task_definition(flow_run) # run task self._run_task(flow_run)
def _from_flow_data(cls, flow_data: dict, **kwargs: Any) -> "FlowView": """ Instantiate a `FlowView` from serialized data This method deserializes objects into their Prefect types. Args: - flow_data: The dict of serialized data - **kwargs: Additional kwargs are passed to __init__ and overrides attributes from `flow_data` """ flow_data = flow_data.copy() flow_id = flow_data.pop("id") flow_group_data = flow_data.pop("flow_group") flow_group_labels = flow_group_data["labels"] project_name = flow_data.pop("project")["name"] storage = StorageSchema().load(flow_data.pop("storage")) run_config = RunConfigSchema().load(flow_data.pop("run_config")) # Combine the data from `flow_data` with `kwargs` flow_args = { **dict( flow_id=flow_id, project_name=project_name, storage=storage, flow_group_labels=flow_group_labels, run_config=run_config, **flow_data, ), **kwargs, } return cls(**flow_args)
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy a flow run as an ECS task. Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ run_config = self._get_run_config(flow_run, ECSRun) assert isinstance(run_config, ECSRun) # mypy if run_config.task_definition_arn is None: # Register a new task definition self.logger.debug("Registering new task definition for flow %s", flow_run.flow.id) taskdef = self.generate_task_definition(flow_run, run_config) resp = self.ecs_client.register_task_definition(**taskdef) taskdef_arn = resp["taskDefinition"]["taskDefinitionArn"] new_taskdef_arn = True self.logger.debug( "Registered task definition %s for flow %s", taskdef_arn, flow_run.flow.id, ) else: from prefect.serialization.storage import StorageSchema from prefect.storage import Docker if isinstance(StorageSchema().load(flow_run.flow.storage), Docker): raise ValueError( "Cannot provide `task_definition_arn` when using `Docker` storage" ) taskdef_arn = run_config.task_definition_arn new_taskdef_arn = False self.logger.debug("Using task definition %s for flow %s", taskdef_arn, flow_run.flow.id) # Get kwargs to pass to run_task kwargs = self.get_run_task_kwargs(flow_run, run_config) resp = self.ecs_client.run_task(taskDefinition=taskdef_arn, **kwargs) # Always deregister the task definition if a new one was registered if new_taskdef_arn: self.logger.debug("Deregistering task definition %s", taskdef_arn) self.ecs_client.deregister_task_definition( taskDefinition=taskdef_arn) if resp.get("tasks"): task_arn = resp["tasks"][0]["taskArn"] self.logger.debug("Started task %r for flow run %r", task_arn, flow_run.id) return f"Task {task_arn}" raise ValueError( "Failed to start task for flow run {0}. Failures: {1}".format( flow_run.id, resp.get("failures")))
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs to Fargate Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) # create copies of kwargs to apply overrides as needed flow_task_definition_kwargs = copy.deepcopy( self.task_definition_kwargs) flow_task_run_kwargs = copy.deepcopy(self.task_run_kwargs) if self.use_external_kwargs: # override from external kwargs self._override_kwargs(flow_run, flow_task_definition_kwargs, flow_task_run_kwargs) # set proper task_definition_name and tags based on enable_task_revisions flag if self.enable_task_revisions: # set task definition name self.task_definition_name = slugify(flow_run.flow.name) self._add_flow_tags(flow_run, flow_task_definition_kwargs) else: self.task_definition_name = "prefect-task-{}".format( # type: ignore flow_run.flow.id[:8] # type: ignore ) # type: ignore # Require Docker storage if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format( flow_run.id)) raise ValueError("Unsupported Storage type") # check if task definition exists self.logger.debug("Checking for task definition") if not self._verify_task_definition_exists(flow_run): self.logger.debug("No task definition found") self._create_task_definition(flow_run, flow_task_definition_kwargs) # run task task_arn = self._run_task(flow_run, flow_task_run_kwargs) self.logger.debug("Run created for task {}".format(task_arn)) return "Task ARN: {}".format(task_arn)
def get_flow_image(flow_run: GraphQLResult, default: str = None) -> str: """ Retrieve the image to use for this flow run deployment. Args: - flow_run (GraphQLResult): A GraphQLResult flow run object - default (str, optional): A default image to use. If not specified, The `prefecthq/prefect` image corresponding with the flow's prefect version will be used. Returns: - str: a full image name to use for this flow run Raises: - ValueError: if deployment attempted on unsupported Storage type and `image` not present in environment metadata """ from prefect.storage import Docker from prefect.serialization.storage import StorageSchema from prefect.serialization.run_config import RunConfigSchema from prefect.serialization.environment import EnvironmentSchema has_run_config = getattr(flow_run, "run_config", None) is not None has_environment = getattr(flow_run.flow, "environment", None) is not None storage = StorageSchema().load(flow_run.flow.storage) # Not having an environment implies run-config based flow, even if # run_config is None. if has_run_config or not has_environment: # Precedence: # - Image on docker storage # - Image on run_config # - Provided default # - `prefecthq/prefect` for flow's core version if isinstance(storage, Docker): return storage.name if has_run_config: run_config = RunConfigSchema().load(flow_run.run_config) if getattr(run_config, "image", None) is not None: return run_config.image if default is not None: return default # core_version should always be present, but just in case version = flow_run.flow.get("core_version") or "latest" cleaned_version = version.split("+")[0] return f"prefecthq/prefect:{cleaned_version}" else: environment = EnvironmentSchema().load(flow_run.flow.environment) if hasattr(environment, "metadata") and hasattr( environment.metadata, "image"): return environment.metadata.get("image") elif isinstance(storage, Docker): return storage.name raise ValueError( f"Storage for flow run {flow_run.id} is not of type Docker and " f"environment has no `image` attribute in the metadata field.")
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info( "Deploying flow run {}".format(flow_run.id) # type: ignore ) if not isinstance( StorageSchema().load(flow_run.flow.storage), (Local, Azure, GCS, S3) ): self.logger.error( "Storage for flow run {} is not a supported type.".format(flow_run.id) ) raise ValueError("Unsupported Storage type") env_vars = self.populate_env_vars(flow_run=flow_run) current_env = os.environ.copy() current_env.update(env_vars) python_path = [] if current_env.get("PYTHONPATH"): python_path.append(current_env.get("PYTHONPATH")) python_path.append(os.getcwd()) if self.import_paths: python_path += self.import_paths current_env["PYTHONPATH"] = ":".join(python_path) stdout = sys.stdout if self.show_flow_logs else PIPE p = Popen( ["prefect", "execute", "cloud-flow"], stdout=stdout, stderr=STDOUT, env=current_env, ) self.processes.append(p) self.logger.debug( "Submitted flow run {} to process PID {}".format(flow_run.id, p.pid) ) return "PID: {}".format(p.pid)
def test_deprecated_storage_classes(cls_name, args): import prefect from prefect.serialization.storage import StorageSchema cls = getattr(prefect.storage, cls_name) old_cls = getattr(prefect.environments.storage, cls_name) with pytest.warns(UserWarning, match="deprecated"): old_obj = old_cls(*args) # Old cls is subclass of new class assert isinstance(old_obj, cls) # Serialization roundtrips to new class new = StorageSchema().load(old_obj.serialize()) assert type(new) is cls
def get_flow_image(flow_run: GraphQLResult) -> str: """ Retrieve the image to use for this flow run deployment. Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: a full image name to use for this flow run Raises: - ValueError: if deployment attempted on unsupported Storage type and `image` not present in environment metadata """ from prefect.environments.storage import Docker from prefect.serialization.storage import StorageSchema from prefect.serialization.run_config import RunConfigSchema from prefect.serialization.environment import EnvironmentSchema has_run_config = getattr(flow_run.flow, "run_config", None) is not None has_environment = getattr(flow_run.flow, "environment", None) is not None storage = StorageSchema().load(flow_run.flow.storage) # Not having an environment implies run-config based flow, even if # run_config is None. if has_run_config or not has_environment: if isinstance(storage, Docker): return storage.name elif has_run_config: run_config = RunConfigSchema().load(flow_run.flow.run_config) if getattr(run_config, "image", None) is not None: return run_config.image # No image found on run-config, and no environment present. Use default. # core_version should always be present, but just in case version = flow_run.flow.get("core_version") or "latest" cleaned_version = version.split("+")[0] return f"prefecthq/prefect:all_extras-{cleaned_version}" else: environment = EnvironmentSchema().load(flow_run.flow.environment) if hasattr(environment, "metadata") and hasattr(environment.metadata, "image"): return environment.metadata.get("image") elif isinstance(storage, Docker): return storage.name raise ValueError( f"Storage for flow run {flow_run.id} is not of type Docker and " f"environment has no `image` attribute in the metadata field." )
def deploy_flows(self, flow_runs: list) -> None: """ Deploy flow runs on to a Nomad cluster as jobs Args: - flow_runs (list): A list of GraphQLResult flow run objects """ for flow_run in flow_runs: if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format(flow_run.id) ) continue job_spec = self.replace_job_spec_json(flow_run) nomad_host = os.getenv("NOMAD_HOST", "http://127.0.0.1:4646") requests.post(path.join(nomad_host, "v1/jobs"), json=job_spec)
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format(flow_run.id)) # type: ignore storage = StorageSchema().load(flow_run.flow.storage) if isinstance(storage, Docker): self.logger.error( "Flow run %s has an unsupported storage type: `%s`", flow_run.id, type(storage).__name__, ) raise TypeError("Unsupported Storage type: %s" % type(storage).__name__)
def _create_task_definition( self, flow_run: GraphQLResult, flow_task_definition_kwargs: dict, container_definitions_kwargs: dict, task_definition_name: str, ) -> None: """ Create a task definition for the flow that each flow run will use. This function is only called when a flow is run for the first time. Args: - flow_runs (list): A list of GraphQLResult flow run objects - flow_task_definition_kwargs (dict): kwargs to use for registration - container_definitions_kwargs (dict): container definitions kwargs to use for registration - task_definition_name (str): task definition name to use """ self.logger.debug("Using image {} for task definition".format( StorageSchema().load(flow_run.flow.storage).name # type: ignore )) container_definitions = [{ "name": "flow", "image": StorageSchema().load(flow_run.flow.storage) # type: ignore .name, "command": ["/bin/sh", "-c", "prefect execute cloud-flow"], "environment": [ { "name": "PREFECT__CLOUD__API", "value": config.cloud.api or "https://api.prefect.io", }, { "name": "PREFECT__CLOUD__AGENT__LABELS", "value": str(self.labels), }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": str(self.log_to_cloud).lower(), }, { "name": "PREFECT__LOGGING__LEVEL", "value": "DEBUG" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, ], "secrets": [], "mountPoints": [], "essential": True, }] for key, value in self.env_vars.items(): container_definitions[0]["environment"].append( dict(name=key, value=value)) # apply container definitions to "containerDefinitions" key of task definition # do not allow override of static envars from Prefect base task definition, which may include self.env_vars base_envar_keys = [ x["name"] for x in container_definitions[0]["environment"] ] self.logger.debug( "Removing static Prefect envars from container_definitions_kwargs if exists" ) container_definitions_environment = [ x for x in container_definitions_kwargs.get("environment", []) if x["name"] not in base_envar_keys ] container_definitions[0]["environment"].extend( container_definitions_environment) container_definitions[0]["secrets"] = container_definitions_kwargs.get( "secrets", []) container_definitions[0][ "mountPoints"] = container_definitions_kwargs.get( "mountPoints", []) # Register task definition self.logger.debug("Registering task definition {}".format( task_definition_name # type: ignore )) if self.launch_type: flow_task_definition_kwargs["requiresCompatibilities"] = [ self.launch_type ] self.boto3_client.register_task_definition( family=task_definition_name, # type: ignore containerDefinitions=container_definitions, networkMode="awsvpc", **flow_task_definition_kwargs)
def _create_task_definition( self, flow_run: GraphQLResult, flow_task_definition_kwargs: dict, task_definition_name: str, ) -> None: """ Create a task definition for the flow that each flow run will use. This function is only called when a flow is run for the first time. Args: - flow_runs (list): A list of GraphQLResult flow run objects - flow_task_definition_kwargs (dict): kwargs to use for registration - task_definition_name (str): task definition name to use """ self.logger.debug("Using image {} for task definition".format( StorageSchema().load(flow_run.flow.storage).name # type: ignore )) container_definitions = [{ "name": "flow", "image": StorageSchema().load(flow_run.flow.storage) # type: ignore .name, "command": ["/bin/sh", "-c", "prefect execute cloud-flow"], "environment": [ { "name": "PREFECT__CLOUD__API", "value": config.cloud.api or "https://api.prefect.io", }, { "name": "PREFECT__CLOUD__AGENT__LABELS", "value": str(self.labels), }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": str(self.log_to_cloud).lower(), }, { "name": "PREFECT__LOGGING__LEVEL", "value": "DEBUG" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, ], "essential": True, }] for key, value in self.env_vars.items(): container_definitions[0]["environment"].append( dict(name=key, value=value)) # Register task definition self.logger.debug("Registering task definition {}".format( task_definition_name # type: ignore )) self.boto3_client.register_task_definition( family=task_definition_name, # type: ignore containerDefinitions=container_definitions, requiresCompatibilities=["FARGATE"], networkMode="awsvpc", **flow_task_definition_kwargs)
def _create_task_definition(self, flow_run: GraphQLResult) -> None: """ Create a task definition for the flow that each flow run will use. This function is only called when a flow is run for the first time. Args: - flow_runs (list): A list of GraphQLResult flow run objects """ container_definitions = [{ "name": "flow", "image": StorageSchema().load(flow_run.flow.storage) # type: ignore .name, "command": ["/bin/sh", "-c", "prefect execute cloud-flow"], "environment": [ { "name": "PREFECT__CLOUD__API", "value": config.cloud.api or "https://api.prefect.io", }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true" }, { "name": "PREFECT__LOGGING__LEVEL", "value": "DEBUG" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, ], "essential": True, }] # Assign repository credentials if they are specified if self.repository_credentials: container_definitions[0]["repositoryCredentials"] = { "credentialsParameter": self.repository_credentials } # Register task definition self.boto3_client.register_task_definition( family="prefect-task-{}".format( flow_run.flow.id[:8]), # type: ignore containerDefinitions=container_definitions, requiresCompatibilities=["FARGATE"], networkMode="awsvpc", cpu=self.task_cpu, memory=self.task_memory, )
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker storage = StorageSchema().load(flow_run.flow.storage) if not isinstance(StorageSchema().load(flow_run.flow.storage), Docker): self.logger.error( "Storage for flow run {} is not of type Docker.".format( flow_run.id)) raise ValueError("Unsupported Storage type") env_vars = self.populate_env_vars(flow_run=flow_run) if not self.no_pull and storage.registry_url: self.logger.info("Pulling image {}...".format(storage.name)) pull_output = self.docker_client.pull(storage.name, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info("Successfully pulled image {}...".format( storage.name)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(storage.name)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update( extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) container = self.docker_client.create_container( storage.name, command="prefect execute cloud-flow", environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, ) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: proc = multiprocessing.Process( target=self.stream_container_logs, kwargs={"container_id": container.get("Id")}, ) proc.start() self.processes.append(proc) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def replace_job_spec_yaml(self, flow_run: GraphQLResult) -> dict: """ Populate metadata and variables in the job_spec.yaml file for flow runs Args: - flow_run (GraphQLResult): A flow run object Returns: - dict: a dictionary representing the populated yaml object """ with open(path.join(path.dirname(__file__), "job_spec.yaml"), "r") as job_file: job = yaml.safe_load(job_file) identifier = str(uuid.uuid4())[:8] job_name = "prefect-job-{}".format(identifier) # Populate job metadata for identification job["metadata"]["name"] = job_name job["metadata"]["labels"]["app"] = job_name job["metadata"]["labels"]["identifier"] = identifier job["metadata"]["labels"]["flow_run_id"] = flow_run.id # type: ignore job["metadata"]["labels"]["flow_id"] = flow_run.flow.id # type: ignore job["spec"]["template"]["metadata"]["labels"]["app"] = job_name job["spec"]["template"]["metadata"]["labels"][ "flow_run_id"] = flow_run.id # type: ignore job["spec"]["template"]["metadata"]["labels"][ "identifier"] = identifier # Use flow storage image for job job["spec"]["template"]["spec"]["containers"][0]["image"] = ( StorageSchema().load(flow_run.flow.storage).name # type: ignore ) self.logger.debug("Using image {} for job".format(StorageSchema().load( flow_run.flow.storage).name # type: ignore )) # Populate environment variables for flow run execution env = job["spec"]["template"]["spec"]["containers"][0]["env"] env[0]["value"] = config.cloud.api or "https://api.prefect.io" env[1]["value"] = config.cloud.agent.auth_token env[2]["value"] = flow_run.id # type: ignore env[3]["value"] = flow_run.flow.id # type: ignore env[4]["value"] = os.getenv("NAMESPACE", "default") env[5]["value"] = str(self.labels) env[6]["value"] = str(self.log_to_cloud).lower() # append all user provided values for key, value in self.env_vars.items(): env.append(dict(name=key, value=value)) # Use image pull secrets if provided job["spec"]["template"]["spec"]["imagePullSecrets"][0][ "name"] = os.getenv("IMAGE_PULL_SECRETS", "") # Set resource requirements if provided resources = job["spec"]["template"]["spec"]["containers"][0][ "resources"] if os.getenv("JOB_MEM_REQUEST"): resources["requests"]["memory"] = os.getenv("JOB_MEM_REQUEST") if os.getenv("JOB_MEM_LIMIT"): resources["limits"]["memory"] = os.getenv("JOB_MEM_LIMIT") if os.getenv("JOB_CPU_REQUEST"): resources["requests"]["cpu"] = os.getenv("JOB_CPU_REQUEST") if os.getenv("JOB_CPU_LIMIT"): resources["limits"]["cpu"] = os.getenv("JOB_CPU_LIMIT") return job
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id)) # type: ignore storage = StorageSchema().load(flow_run.flow.storage) if isinstance(storage, Docker): self.logger.error( "Flow run %s has an unsupported storage type: `%s`", flow_run.id, type(storage).__name__, ) raise TypeError("Unsupported Storage type: %s" % type(storage).__name__) # If the flow is using a run_config, load it if getattr(flow_run.flow, "run_config", None) is not None: run_config = RunConfigSchema().load(flow_run.flow.run_config) if not isinstance(run_config, LocalRun): self.logger.error( "Flow run %s has a `run_config` of type `%s`, only `LocalRun` is supported", flow_run.id, type(run_config).__name__, ) raise TypeError("Unsupported RunConfig type: %s" % type(run_config).__name__) else: run_config = None env = self.populate_env_vars(flow_run, run_config=run_config) working_dir = None if run_config is None else run_config.working_dir if working_dir and not os.path.exists(working_dir): msg = f"Flow run {flow_run.id} has a nonexistent `working_dir` configured: {working_dir}" self.logger.error(msg) raise ValueError(msg) stdout = sys.stdout if self.show_flow_logs else DEVNULL # note: we will allow these processes to be orphaned if the agent were to exit # before the flow runs have completed. The lifecycle of the agent should not # dictate the lifecycle of the flow run. However, if the user has elected to # show flow logs, these log entries will continue to stream to the users terminal # until these child processes exit, even if the agent has already exited. p = Popen( get_flow_run_command(flow_run).split(" "), stdout=stdout, stderr=STDOUT, env=env, cwd=working_dir, ) self.processes.add(p) self.logger.debug("Submitted flow run {} to process PID {}".format( flow_run.id, p.pid)) return "PID: {}".format(p.pid)