def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) if not isinstance( StorageSchema().load(flow_run.flow.storage), (Local, Azure, GCS, S3, GitHub, Webhook), ): self.logger.error( "Storage for flow run {} is not a supported type.".format( flow_run.id)) raise ValueError("Unsupported Storage type") env_vars = self.populate_env_vars(flow_run=flow_run) current_env = os.environ.copy() current_env.update(env_vars) python_path = [] if current_env.get("PYTHONPATH"): python_path.append(current_env.get("PYTHONPATH")) python_path.append(os.getcwd()) if self.import_paths: python_path += self.import_paths current_env["PYTHONPATH"] = ":".join(python_path) stdout = sys.stdout if self.show_flow_logs else DEVNULL # note: we will allow these processes to be orphaned if the agent were to exit # before the flow runs have completed. The lifecycle of the agent should not # dictate the lifecycle of the flow run. However, if the user has elected to # show flow logs, these log entries will continue to stream to the users terminal # until these child processes exit, even if the agent has already exited. p = Popen( get_flow_run_command(flow_run).split(" "), stdout=stdout, stderr=STDOUT, env=current_env, ) self.processes.add(p) self.logger.debug("Submitted flow run {} to process PID {}".format( flow_run.id, p.pid)) return "PID: {}".format(p.pid)
def generate_task_definition(self, flow_run: GraphQLResult) -> Dict[str, Any]: """Generate an Vertex task definition from a flow run Args: - flow_run (GraphQLResult): A flow run object Returns: - dict: a dictionary representation of an Vertex task definition """ run_config = self._get_run_config(flow_run, VertexRun) assert isinstance(run_config, VertexRun) # mypy image = get_flow_image(flow_run) job_name = slugify.slugify( flow_run.flow.name + "-" + flow_run.name, max_length=255, word_boundary=True, save_order=True, ) machine_type = run_config.machine_type command = get_flow_run_command(flow_run) env = self.populate_env_vars(flow_run) env_list = self._to_env_list(env) # Start with a default taskdef taskdef = { "display_name": job_name, "job_spec": { "worker_pool_specs": [ {"machine_spec": {"machine_type": machine_type}, "replica_count": 1} ] }, } # type: Dict[str, Any] if run_config.worker_pool_specs is not None: taskdef["job_spec"]["worker_pool_specs"] = run_config.worker_pool_specs if run_config.network is not None: taskdef["job_spec"]["network"] = run_config.network if run_config.service_account is not None: taskdef["job_spec"]["service_account"] = run_config.service_account else: taskdef["job_spec"]["service_account"] = self.service_account if run_config.scheduling is not None: taskdef["job_spec"]["scheduling"] = run_config.scheduling # We always set the container spec on the zeroth pool spec to ensure it will run the flow taskdef["job_spec"]["worker_pool_specs"][0]["container_spec"] = { "image_uri": image, "command": command.split(), "args": [], "env": env_list, } return taskdef
def test_get_flow_run_command_works_if_core_version_not_on_response(): legacy_command = "prefect execute cloud-flow" flow_run = GraphQLResult({ "flow": GraphQLResult({ "storage": Local().serialize(), "id": "id", }), "id": "id", }) assert get_flow_run_command(flow_run) == legacy_command
def test_get_flow_run_command(core_version, command): flow_run = GraphQLResult({ "flow": GraphQLResult({ "storage": Local().serialize(), "id": "id", "core_version": core_version, }), "id": "id", }) assert get_flow_run_command(flow_run) == command
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ self.logger.info("Deploying flow run {}".format( flow_run.id)) # type: ignore # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker run_config = self._get_run_config(flow_run, DockerRun) assert run_config is None or isinstance(run_config, DockerRun) # mypy image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run, image, run_config=run_config) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) registry = image.split("/")[0] if self.reg_allow_list and registry not in self.reg_allow_list: self.logger.error( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) raise ValueError( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) else: pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info( "Successfully pulled image {}...".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update( extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None # At the time of creation, you can only connect a container to a single network, # however you can create more connections after creation. # Connect first network in the creation step. If no network is connected here the container # is connected to the default `bridge` network. # The rest of the networks are connected after creation. if self.networks: networking_config = self.docker_client.create_networking_config({ self.networks[0]: self.docker_client.create_endpoint_config() }) # Try fallback on old, deprecated, behaviour. if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) labels = { "io.prefect.flow-name": flow_run.flow.name, "io.prefect.flow-id": flow_run.flow.id, "io.prefect.flow-run-id": flow_run.id, } container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, labels=labels, ) # Connect the rest of the networks if self.networks: for network in self.networks[1:]: self.docker_client.connect_container_to_network( container=container, net_id=network) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) if self.networks: self.logger.debug( "Adding container with ID {} to docker networks: {}.".format( container.get("Id"), self.networks)) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: self.stream_flow_logs(container.get("Id")) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def generate_task_definition(self, flow_run: GraphQLResult, run_config: ECSRun) -> Dict[str, Any]: """Generate an ECS task definition from a flow run Args: - flow_run (GraphQLResult): A flow run object - run_config (ECSRun): The flow's run config Returns: - dict: a dictionary representation of an ECS task definition """ if run_config.task_definition: taskdef = deepcopy(run_config.task_definition) elif run_config.task_definition_path: self.logger.debug( "Loading task definition template from %r", run_config.task_definition_path, ) template_bytes = read_bytes_from_path( run_config.task_definition_path) taskdef = yaml.safe_load(template_bytes) else: taskdef = deepcopy(self.task_definition) slug = slugify.slugify( flow_run.flow.name, max_length=255 - len("prefect-"), word_boundary=True, save_order=True, ) family = f"prefect-{slug}" tags = self.get_task_definition_tags(flow_run) taskdef["family"] = family taskdef_tags = [{"key": k, "value": v} for k, v in tags.items()] for entry in taskdef.get("tags", []): if entry["key"] not in tags: taskdef_tags.append(entry) taskdef["tags"] = taskdef_tags # Get the flow container (creating one if it doesn't already exist) containers = taskdef.setdefault("containerDefinitions", []) for container in containers: if container.get("name") == "flow": break else: container = {"name": "flow"} containers.append(container) # Set flow image container["image"] = image = get_flow_image(flow_run) # Set flow run command container["command"] = [ "/bin/sh", "-c", get_flow_run_command(flow_run) ] # Set taskRoleArn if configured if run_config.task_role_arn: taskdef["taskRoleArn"] = run_config.task_role_arn # Populate static environment variables from the following sources, # with precedence: # - Static environment variables, hardcoded below # - Values in the task definition template env = { "PREFECT__CLOUD__USE_LOCAL_SECRETS": "false", "PREFECT__CONTEXT__IMAGE": image, "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudFlowRunner", "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudTaskRunner", } container_env = [{"name": k, "value": v} for k, v in env.items()] for entry in container.get("environment", []): if entry["name"] not in env: container_env.append(entry) container["environment"] = container_env # Set resource requirements, if provided # Also ensure that cpu/memory are strings not integers if run_config.cpu: taskdef["cpu"] = str(run_config.cpu) elif "cpu" in taskdef: taskdef["cpu"] = str(taskdef["cpu"]) if run_config.memory: taskdef["memory"] = str(run_config.memory) elif "memory" in taskdef: taskdef["memory"] = str(taskdef["memory"]) return taskdef
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker run_config = self._get_run_config(flow_run, DockerRun) assert run_config is None or isinstance(run_config, DockerRun) # mypy image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run, image, run_config=run_config) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) registry = image.split("/")[0] if self.reg_allow_list and registry not in self.reg_allow_list: self.logger.error( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) raise ValueError( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) else: pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info("Successfully pulled image {}".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) # By default, auto-remove containers host_config: Dict[str, Any] = {"auto_remove": True} # By default, no ports ports = None # Set up a host gateway for local communication; check the docker version since # this is not supported by older versions docker_engine_version = parse_version( self.docker_client.version()["Version"]) host_gateway_version = parse_version("20.10.0") if docker_engine_version < host_gateway_version: warnings.warn( "`host.docker.internal` could not be automatically resolved to your " "local host. This feature is not supported on Docker Engine " f"v{docker_engine_version}, upgrade to v{host_gateway_version}+ if you " "encounter issues.") else: # Compatibility for linux -- https://github.com/docker/cli/issues/2290 # Only supported by Docker v20.10.0+ which is our minimum recommend version host_config["extra_hosts"] = { "host.docker.internal": "host-gateway" } container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if run_config is not None and run_config.host_config: # The host_config passed from the run_config will overwrite defaults host_config.update(run_config.host_config) if run_config is not None and run_config.ports: ports = run_config.ports networking_config = None # At the time of creation, you can only connect a container to a single network, # however you can create more connections after creation. # Connect first network in the creation step. If no network is connected here the container # is connected to the default `bridge` network. # The rest of the networks are connected after creation. if self.networks: networking_config = self.docker_client.create_networking_config({ self.networks[0]: self.docker_client.create_endpoint_config() }) labels = { "io.prefect.flow-name": flow_run.flow.name, "io.prefect.flow-id": flow_run.flow.id, "io.prefect.flow-run-id": flow_run.id, } # Generate a container name to match the flow run name, ensuring it is docker # compatible and unique. Must match `[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end container_name = slugified_name = ( slugify( flow_run.name, lowercase=False, # Docker does not limit length but URL limits apply eventually so # limit the length for safety max_length=250, # Docker allows these characters for container names regex_pattern=r"[^a-zA-Z0-9_.-]+", ).lstrip( # Docker does not allow leading underscore, dash, or period "_-.") # Docker does not allow 0 character names so use the flow run id if name # would be empty after cleaning or flow_run.id) # Create the container with retries on name conflicts index = 0 # will be bumped on name colissions while True: try: container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, name=container_name, volumes=container_mount_paths, host_config=self.docker_client.create_host_config( **host_config), networking_config=networking_config, labels=labels, ports=ports, ) except docker.errors.APIError as exc: if "Conflict" in str(exc) and "container name" in str(exc): index += 1 container_name = f"{slugified_name}-{index}" else: raise else: break # Connect the rest of the networks if self.networks: for network in self.networks[1:]: self.docker_client.connect_container_to_network( container=container, net_id=network) # Start the container self.logger.debug( f"Starting Docker container with ID {container.get('Id')} and " f"name {container_name!r}") if self.networks: self.logger.debug( "Adding container with ID {} to docker networks: {}.".format( container.get("Id"), self.networks)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: self.stream_flow_logs(container.get("Id")) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def generate_job_spec_from_environment(self, flow_run: GraphQLResult, image: str = None) -> dict: """ Populate a k8s job spec. This spec defines a k8s job that handles executing a flow. This method runs each time the agent receives a flow to run. That job spec can optionally be customized by setting the following environment variables on the agent. - `NAMESPACE`: the k8s namespace the job will run in, defaults to `"default"` - `JOB_MEM_REQUEST`: memory requested, for example, `256Mi` for 256 MB. If this environment variable is not set, the cluster's defaults will be used. - `JOB_MEM_LIMIT`: memory limit, for example, `512Mi` For 512 MB. If this environment variable is not set, the cluster's defaults will be used. - `JOB_CPU_REQUEST`: CPU requested, defaults to `"100m"` - `JOB_CPU_LIMIT`: CPU limit, defaults to `"100m"` - `IMAGE_PULL_POLICY`: policy for pulling images. Defaults to `"IfNotPresent"`. - `IMAGE_PULL_SECRETS`: name of an existing k8s secret that can be used to pull images. This is necessary if your flow uses an image that is in a non-public container registry, such as Amazon ECR, or in a public registry that requires authentication to avoid hitting rate limits. To specify multiple image pull secrets, provide a comma-delimited string with no spaces, like `"some-secret,other-secret"`. - `SERVICE_ACCOUNT_NAME`: name of a service account to run the job as. By default, none is specified. - `YAML_TEMPLATE`: a path to where the YAML template should be loaded from. defaults to the embedded `job_spec.yaml`. Args: - flow_run (GraphQLResult): A flow run object - image (str, optional): The full name of an image to use for the job Returns: - dict: a dictionary representation of a k8s job for flow execution """ identifier = str(uuid.uuid4())[:8] yaml_path = os.getenv( "YAML_TEMPLATE", os.path.join(os.path.dirname(__file__), "job_spec.yaml")) with open(yaml_path, "r") as job_file: job = yaml.safe_load(job_file) job_name = "prefect-job-{}".format(identifier) # Populate job metadata for identification k8s_labels = { "prefect.io/identifier": identifier, "prefect.io/flow_run_id": flow_run.id, # type: ignore "prefect.io/flow_id": flow_run.flow.id, # type: ignore } job["metadata"]["name"] = job_name job["metadata"]["labels"].update(**k8s_labels) job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels) # Use provided image for job if image is None: image = get_flow_image(flow_run=flow_run) job["spec"]["template"]["spec"]["containers"][0]["image"] = image self.logger.debug("Using image {} for job".format(image)) # Datermine flow run command job["spec"]["template"]["spec"]["containers"][0]["args"] = [ get_flow_run_command(flow_run) ] # Populate environment variables for flow run execution env = job["spec"]["template"]["spec"]["containers"][0]["env"] env[0]["value"] = config.cloud.api or "https://api.prefect.io" env[1]["value"] = config.cloud.agent.auth_token env[2]["value"] = flow_run.id # type: ignore env[3]["value"] = flow_run.flow.id # type: ignore env[4]["value"] = self.namespace env[5]["value"] = str(self.labels) env[6]["value"] = str(self.log_to_cloud).lower() env[7]["value"] = self.env_vars.get("PREFECT__LOGGING__LEVEL", config.logging.level) # append all user provided values for key, value in self.env_vars.items(): env.append(dict(name=key, value=value)) # Use image pull secrets if provided if self.image_pull_secrets: for idx, secret_name in enumerate(self.image_pull_secrets): # this check preserves behavior from previous releases, # where prefect would only overwrite the first entry in # imagePullSecrets if idx == 0: job["spec"]["template"]["spec"]["imagePullSecrets"][0] = { "name": secret_name } else: job["spec"]["template"]["spec"]["imagePullSecrets"].append( {"name": secret_name}) else: del job["spec"]["template"]["spec"]["imagePullSecrets"] # Set resource requirements if provided resources = job["spec"]["template"]["spec"]["containers"][0][ "resources"] if os.getenv("JOB_MEM_REQUEST"): resources["requests"]["memory"] = os.getenv("JOB_MEM_REQUEST") if os.getenv("JOB_MEM_LIMIT"): resources["limits"]["memory"] = os.getenv("JOB_MEM_LIMIT") if os.getenv("JOB_CPU_REQUEST"): resources["requests"]["cpu"] = os.getenv("JOB_CPU_REQUEST") if os.getenv("JOB_CPU_LIMIT"): resources["limits"]["cpu"] = os.getenv("JOB_CPU_LIMIT") if self.volume_mounts: job["spec"]["template"]["spec"]["containers"][0][ "volumeMounts"] = self.volume_mounts else: del job["spec"]["template"]["spec"]["containers"][0][ "volumeMounts"] if self.volumes: job["spec"]["template"]["spec"]["volumes"] = self.volumes else: del job["spec"]["template"]["spec"]["volumes"] if os.getenv("IMAGE_PULL_POLICY"): job["spec"]["template"]["spec"]["containers"][0][ "imagePullPolicy"] = os.getenv("IMAGE_PULL_POLICY") if self.service_account_name: job["spec"]["template"]["spec"][ "serviceAccountName"] = self.service_account_name return job
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker run_config = self._get_run_config(flow_run, DockerRun) assert run_config is None or isinstance(run_config, DockerRun) # mypy image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run, image, run_config=run_config) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) registry = image.split("/")[0] if self.reg_allow_list and registry not in self.reg_allow_list: self.logger.error( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) raise ValueError( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) else: pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info("Successfully pulled image {}".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) networking_config = None # At the time of creation, you can only connect a container to a single network, # however you can create more connections after creation. # Connect first network in the creation step. If no network is connected here the container # is connected to the default `bridge` network. # The rest of the networks are connected after creation. if self.networks: networking_config = self.docker_client.create_networking_config({ self.networks[0]: self.docker_client.create_endpoint_config() }) # Try fallback on old, deprecated, behaviour. if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) labels = { "io.prefect.flow-name": flow_run.flow.name, "io.prefect.flow-id": flow_run.flow.id, "io.prefect.flow-run-id": flow_run.id, } # Generate a container name to match the flow run name, ensuring it is docker # compatible and unique. Must match `[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end container_name = slugified_name = ( slugify( flow_run.name, lowercase=False, # Docker does not limit length but URL limits apply eventually so # limit the length for safety max_length=250, # Docker allows these characters for container names regex_pattern=r"[^a-zA-Z0-9_.-]+", ).lstrip( # Docker does not allow leading underscore, dash, or period "_-.") # Docker does not allow 0 character names so use the flow run id if name # would be empty after cleaning or flow_run.id) # Create the container with retries on name conflicts index = 0 # will be bumped on name colissions while True: try: container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, name=container_name, volumes=container_mount_paths, host_config=self.docker_client.create_host_config( **host_config), networking_config=networking_config, labels=labels, ) except docker.errors.APIError as exc: if "Conflict" in str(exc) and "container name" in str(exc): index += 1 container_name = f"{slugified_name}-{index}" else: raise else: break # Connect the rest of the networks if self.networks: for network in self.networks[1:]: self.docker_client.connect_container_to_network( container=container, net_id=network) # Start the container self.logger.debug( f"Starting Docker container with ID {container.get('Id')} and " f"name {container_name!r}") if self.networks: self.logger.debug( "Adding container with ID {} to docker networks: {}.".format( container.get("Id"), self.networks)) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: self.stream_flow_logs(container.get("Id")) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def get_run_task_kwargs(self, flow_run: GraphQLResult, run_config: ECSRun) -> Dict[str, Any]: """Generate kwargs to pass to `ECS.client.run_task` for a flow run Args: - flow_run (GraphQLResult): A flow run object - run_config (ECSRun): The flow's run config Returns: - dict: kwargs to pass to `ECS.client.run_task` """ # Set agent defaults out = deepcopy(self.run_task_kwargs) out["launchType"] = self.launch_type if self.cluster: out["cluster"] = self.cluster # Apply run-config kwargs, if any if run_config.run_task_kwargs: out = merge_run_task_kwargs(out, run_config.run_task_kwargs) # Find or create the flow container overrides overrides = out.setdefault("overrides", {}) container_overrides = overrides.setdefault("containerOverrides", []) for container in container_overrides: if container.get("name") == "flow": break else: container = {"name": "flow"} container_overrides.append(container) # Set taskRoleArn if configured if run_config.task_role_arn: overrides["taskRoleArn"] = run_config.task_role_arn elif self.task_role_arn: overrides["taskRoleArn"] = self.task_role_arn # Set executionRoleArn if configured if run_config.execution_role_arn: overrides["executionRoleArn"] = run_config.execution_role_arn elif self.execution_role_arn: overrides["executionRoleArn"] = self.execution_role_arn # Set resource requirements, if provided # Also ensure that cpu/memory are strings not integers if run_config.cpu: overrides["cpu"] = str(run_config.cpu) elif "cpu" in overrides: overrides["cpu"] = str(overrides["cpu"]) if run_config.memory: overrides["memory"] = str(run_config.memory) elif "memory" in overrides: overrides["memory"] = str(overrides["memory"]) # Set flow run command container["command"] = [ "/bin/sh", "-c", get_flow_run_command(flow_run) ] # Populate environment variables from the following sources, # with precedence: # - Values required for flow execution, hardcoded below # - Values set on the ECSRun object # - Values set using the `--env` CLI flag on the agent env = self.env_vars.copy() if run_config.env: env.update(run_config.env) env.update({ "PREFECT__CLOUD__USE_LOCAL_SECRETS": "false", "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudFlowRunner", "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudTaskRunner", "PREFECT__BACKEND": config.backend, "PREFECT__CLOUD__API": config.cloud.api, "PREFECT__CONTEXT__FLOW_RUN_ID": flow_run.id, "PREFECT__CONTEXT__FLOW_ID": flow_run.flow.id, "PREFECT__LOGGING__LOG_TO_CLOUD": str(self.log_to_cloud).lower(), "PREFECT__CLOUD__AUTH_TOKEN": config.cloud.agent.auth_token, "PREFECT__CLOUD__AGENT__LABELS": str(self.labels), }) container_env = [{"name": k, "value": v} for k, v in env.items()] for entry in container.get("environment", []): if entry["name"] not in env: container_env.append(entry) container["environment"] = container_env return out
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run=flow_run) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) registry = image.split("/")[0] if self.reg_allow_list and registry not in self.reg_allow_list: self.logger.error( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) raise ValueError( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) else: pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info( "Successfully pulled image {}...".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update( extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, ) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: proc = multiprocessing.Process( target=self.stream_container_logs, kwargs={"container_id": container.get("Id")}, ) proc.start() self.processes.append(proc) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def generate_job_spec_from_run_config(self, flow_run: GraphQLResult) -> dict: """Generate a k8s job spec for a flow run. Args: - flow_run (GraphQLResult): A flow run object Returns: - dict: a dictionary representation of a k8s job for flow execution """ run_config = RunConfigSchema().load(flow_run.flow.run_config) if run_config.job_template: job = run_config.job_template else: job_template_path = run_config.job_template_path or self.job_template_path self.logger.debug("Loading job template from %r", job_template_path) template_bytes = read_bytes_from_path(job_template_path) job = yaml.safe_load(template_bytes) identifier = uuid.uuid4().hex[:8] job_name = f"prefect-job-{identifier}" # Populate job metadata for identification k8s_labels = { "prefect.io/identifier": identifier, "prefect.io/flow_run_id": flow_run.id, # type: ignore "prefect.io/flow_id": flow_run.flow.id, # type: ignore } _get_or_create(job, "metadata.labels") _get_or_create(job, "spec.template.metadata.labels") job["metadata"]["name"] = job_name job["metadata"]["labels"].update(**k8s_labels) job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels) # Get the first container, which is used for the prefect job containers = _get_or_create(job, "spec.template.spec.containers", []) if not containers: containers.append({}) container = containers[0] # Set container image container["image"] = image = get_flow_image(flow_run) # Set flow run command container["args"] = [get_flow_run_command(flow_run)] # Populate environment variables from the following sources, # with precedence: # - Values required for flow execution, hardcoded below # - Values set on the KubernetesRun object # - Values set using the `--env` CLI flag on the agent # - Values in the job template env = self.env_vars.copy() if run_config.env: env.update(run_config.env) env.update({ "PREFECT__CLOUD__API": config.cloud.api, "PREFECT__CLOUD__AUTH_TOKEN": config.cloud.agent.auth_token, "PREFECT__CLOUD__USE_LOCAL_SECRETS": "false", "PREFECT__CONTEXT__FLOW_RUN_ID": flow_run.id, "PREFECT__CONTEXT__FLOW_ID": flow_run.flow.id, "PREFECT__CONTEXT__IMAGE": image, "PREFECT__LOGGING__LOG_TO_CLOUD": str(self.log_to_cloud).lower(), "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudFlowRunner", "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudTaskRunner", }) container_env = [{"name": k, "value": v} for k, v in env.items()] for entry in container.get("env", []): if entry["name"] not in env: container_env.append(entry) container["env"] = container_env # Set resource requirements if provided _get_or_create(container, "resources.requests") _get_or_create(container, "resources.limits") resources = container["resources"] if run_config.memory_request: resources["requests"]["memory"] = run_config.memory_request if run_config.memory_limit: resources["limits"]["memory"] = run_config.memory_limit if run_config.cpu_request: resources["requests"]["cpu"] = run_config.cpu_request if run_config.cpu_limit: resources["limits"]["cpu"] = run_config.cpu_limit return job
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment Raises: - ValueError: if deployment attempted on unsupported Storage type """ self.logger.info("Deploying flow run {}".format( flow_run.id)) # type: ignore storage = StorageSchema().load(flow_run.flow.storage) if isinstance(storage, Docker): self.logger.error( "Flow run %s has an unsupported storage type: `%s`", flow_run.id, type(storage).__name__, ) raise TypeError("Unsupported Storage type: %s" % type(storage).__name__) # If the flow is using a run_config, load it if getattr(flow_run.flow, "run_config", None) is not None: run_config = RunConfigSchema().load(flow_run.flow.run_config) if not isinstance(run_config, LocalRun): self.logger.error( "Flow run %s has a `run_config` of type `%s`, only `LocalRun` is supported", flow_run.id, type(run_config).__name__, ) raise TypeError("Unsupported RunConfig type: %s" % type(run_config).__name__) else: run_config = None env = self.populate_env_vars(flow_run, run_config=run_config) working_dir = None if run_config is None else run_config.working_dir if working_dir and not os.path.exists(working_dir): msg = f"Flow run {flow_run.id} has a nonexistent `working_dir` configured: {working_dir}" self.logger.error(msg) raise ValueError(msg) stdout = sys.stdout if self.show_flow_logs else DEVNULL # note: we will allow these processes to be orphaned if the agent were to exit # before the flow runs have completed. The lifecycle of the agent should not # dictate the lifecycle of the flow run. However, if the user has elected to # show flow logs, these log entries will continue to stream to the users terminal # until these child processes exit, even if the agent has already exited. p = Popen( get_flow_run_command(flow_run).split(" "), stdout=stdout, stderr=STDOUT, env=env, cwd=working_dir, ) self.processes.add(p) self.logger.debug("Submitted flow run {} to process PID {}".format( flow_run.id, p.pid)) return "PID: {}".format(p.pid)
def replace_job_spec_yaml(self, flow_run: GraphQLResult, image: str) -> dict: """ Populate a k8s job spec. This spec defines a k8s job that handles executing a flow. This method runs each time the agent receives a flow to run. That job spec can optionally be customized by setting the following environment variables on the agent. - `NAMESPACE`: the k8s namespace the job will run in, defaults to `"default"` - `JOB_MEM_REQUEST`: memory requested, for example, `256Mi` for 256 MB. If this environment variable is not set, the cluster's defaults will be used. - `JOB_MEM_LIMIT`: memory limit, for example, `512Mi` For 512 MB. If this environment variable is not set, the cluster's defaults will be used. - `JOB_CPU_REQUEST`: CPU requested, defaults to `"100m"` - `JOB_CPU_LIMIT`: CPU limit, defaults to `"100m"` - `IMAGE_PULL_POLICY`: policy for pulling images. Defaults to `"IfNotPresent"`. - `IMAGE_PULL_SECRETS`: name of an existing k8s secret that can be used to pull images. This is necessary if your flow uses an image that is in a non-public container registry, such as Amazon ECR. - `SERVICE_ACCOUNT_NAME`: name of a service account to run the job as. By default, none is specified. Args: - flow_run (GraphQLResult): A flow run object - image (str): The full name of an image to use for the job Returns: - dict: a dictionary representation of a k8s job for flow execution """ with open(path.join(path.dirname(__file__), "job_spec.yaml"), "r") as job_file: job = yaml.safe_load(job_file) identifier = str(uuid.uuid4())[:8] job_name = "prefect-job-{}".format(identifier) # Populate job metadata for identification k8s_labels = { "prefect.io/identifier": identifier, "prefect.io/flow_run_id": flow_run.id, # type: ignore "prefect.io/flow_id": flow_run.flow.id, # type: ignore } job["metadata"]["name"] = job_name job["metadata"]["labels"].update(**k8s_labels) job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels) # Use provided image for job job["spec"]["template"]["spec"]["containers"][0]["image"] = image self.logger.debug("Using image {} for job".format(image)) # Datermine flow run command job["spec"]["template"]["spec"]["containers"][0]["args"] = [ get_flow_run_command(flow_run) ] # Populate environment variables for flow run execution env = job["spec"]["template"]["spec"]["containers"][0]["env"] env[0]["value"] = config.cloud.api or "https://api.prefect.io" env[1]["value"] = config.cloud.agent.auth_token env[2]["value"] = flow_run.id # type: ignore env[3]["value"] = flow_run.flow.id # type: ignore env[4]["value"] = os.getenv("NAMESPACE", "default") env[5]["value"] = str(self.labels) env[6]["value"] = str(self.log_to_cloud).lower() # append all user provided values for key, value in self.env_vars.items(): env.append(dict(name=key, value=value)) # Use image pull secrets if provided job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"] = os.getenv( "IMAGE_PULL_SECRETS", "" ) # Set resource requirements if provided resources = job["spec"]["template"]["spec"]["containers"][0]["resources"] if os.getenv("JOB_MEM_REQUEST"): resources["requests"]["memory"] = os.getenv("JOB_MEM_REQUEST") if os.getenv("JOB_MEM_LIMIT"): resources["limits"]["memory"] = os.getenv("JOB_MEM_LIMIT") if os.getenv("JOB_CPU_REQUEST"): resources["requests"]["cpu"] = os.getenv("JOB_CPU_REQUEST") if os.getenv("JOB_CPU_LIMIT"): resources["limits"]["cpu"] = os.getenv("JOB_CPU_LIMIT") if os.getenv("IMAGE_PULL_POLICY"): job["spec"]["template"]["spec"]["containers"][0][ "imagePullPolicy" ] = os.getenv("IMAGE_PULL_POLICY") if os.getenv("SERVICE_ACCOUNT_NAME"): job["spec"]["template"]["spec"]["serviceAccountName"] = os.getenv( "SERVICE_ACCOUNT_NAME" ) return job
networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()} ) <<<<<<< HEAD labels = { "io.prefect.flow-name": flow_run.flow.name, "io.prefect.flow-id": flow_run.flow.id, "io.prefect.flow-run-id": flow_run.id, } ======= >>>>>>> prefect clone container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, <<<<<<< HEAD labels=labels, ======= >>>>>>> prefect clone ) # Start the container self.logger.debug( "Starting Docker container with ID {}".format(container.get("Id")) ) if self.network:
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs to Fargate Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ self.logger.info("Deploying flow run {}".format(flow_run.id)) # type: ignore # create copies of kwargs to apply overrides as needed flow_task_definition_kwargs = copy.deepcopy(self.task_definition_kwargs) flow_task_run_kwargs = copy.deepcopy(self.task_run_kwargs) flow_container_definitions_kwargs = copy.deepcopy( self.container_definitions_kwargs ) # create task_definition_name dict for passing into verify method task_definition_dict = {} if self.use_external_kwargs: # override from external kwargs self._override_kwargs( flow_run, flow_task_definition_kwargs, flow_task_run_kwargs, flow_container_definitions_kwargs, ) # set proper task_definition_name and tags based on enable_task_revisions flag if self.enable_task_revisions: # set task definition name task_definition_dict["task_definition_name"] = slugify(flow_run.flow.name) self._add_flow_tags(flow_run, flow_task_definition_kwargs) else: task_definition_dict["task_definition_name"] = "prefect-task-{}".format( # type: ignore flow_run.flow.id[:8] # type: ignore ) # type: ignore image = get_flow_image(flow_run=flow_run) flow_run_command = get_flow_run_command(flow_run=flow_run) # check if task definition exists self.logger.debug("Checking for task definition") if not self._verify_task_definition_exists(flow_run, task_definition_dict): self.logger.debug("No task definition found") self._create_task_definition( image=image, flow_task_definition_kwargs=flow_task_definition_kwargs, container_definitions_kwargs=flow_container_definitions_kwargs, task_definition_name=task_definition_dict["task_definition_name"], flow_run_command=flow_run_command, ) # run task task_arn = self._run_task( flow_run, flow_task_run_kwargs, task_definition_dict["task_definition_name"] ) self.logger.debug("Run created for task {}".format(task_arn)) return "Task ARN: {}".format(task_arn)
def get_run_task_kwargs(self, flow_run: GraphQLResult, run_config: ECSRun, taskdef: Dict[str, Any]) -> Dict[str, Any]: """Generate kwargs to pass to `ECS.client.run_task` for a flow run Args: - flow_run (GraphQLResult): A flow run object - run_config (ECSRun): The flow's run config - taskdef (Dict): The ECS task definition used in ECS.client.run_task Returns: - dict: kwargs to pass to `ECS.client.run_task` """ # Set agent defaults out = deepcopy(self.run_task_kwargs) # Use launchType only if capacity provider is not specified if not out.get("capacityProviderStrategy"): out["launchType"] = self.launch_type if self.cluster: out["cluster"] = self.cluster # Apply run-config kwargs, if any if run_config.run_task_kwargs: out = merge_run_task_kwargs(out, run_config.run_task_kwargs) # Find or create the flow container overrides overrides = out.setdefault("overrides", {}) container_overrides = overrides.setdefault("containerOverrides", []) for container in container_overrides: if container.get("name") == "flow": break else: container = {"name": "flow"} container_overrides.append(container) # Task roles and execution roles should be retrieved from the following sources, # in order: # - An ARN passed explicitly on run_config. # - An ARN present on the task definition used for run_task. In this case, we # don't need to provide an override at all. # - An ARN passed in to the ECSAgent when instantiated, either programmatically # or via the CLI. if run_config.task_role_arn: overrides["taskRoleArn"] = run_config.task_role_arn elif taskdef.get("taskRoleArn"): overrides["taskRoleArn"] = taskdef["taskRoleArn"] elif self.task_role_arn: overrides["taskRoleArn"] = self.task_role_arn if run_config.execution_role_arn: overrides["executionRoleArn"] = run_config.execution_role_arn elif taskdef.get("executionRoleArn"): overrides["executionRoleArn"] = taskdef["executionRoleArn"] elif self.execution_role_arn and not taskdef.get("executionRoleArn"): overrides["executionRoleArn"] = self.execution_role_arn # Set resource requirements, if provided # Also ensure that cpu/memory are strings not integers if run_config.cpu: overrides["cpu"] = str(run_config.cpu) elif "cpu" in overrides: overrides["cpu"] = str(overrides["cpu"]) if run_config.memory: overrides["memory"] = str(run_config.memory) elif "memory" in overrides: overrides["memory"] = str(overrides["memory"]) # Set flow run command container["command"] = [ "/bin/sh", "-c", get_flow_run_command(flow_run) ] # Add `PREFECT__LOGGING__LEVEL` environment variable env = {"PREFECT__LOGGING__LEVEL": config.logging.level} # Populate environment variables from the following sources, # with precedence: # - Values required for flow execution, hardcoded below # - Values set on the ECSRun object # - Values set using the `--env` CLI flag on the agent env.update(self.env_vars) if run_config.env: env.update(run_config.env) env.update({ "PREFECT__CLOUD__USE_LOCAL_SECRETS": "false", "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudFlowRunner", "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudTaskRunner", "PREFECT__BACKEND": config.backend, "PREFECT__CLOUD__API": config.cloud.api, "PREFECT__CONTEXT__FLOW_RUN_ID": flow_run.id, "PREFECT__CONTEXT__FLOW_ID": flow_run.flow.id, "PREFECT__CLOUD__SEND_FLOW_RUN_LOGS": str(self.log_to_cloud).lower(), "PREFECT__CLOUD__API_KEY": self.flow_run_api_key or "", "PREFECT__CLOUD__TENANT_ID": ( # Providing a tenant id is only necessary when authenticating self.client.tenant_id if self.flow_run_api_key else ""), "PREFECT__CLOUD__AGENT__LABELS": str(self.labels), # Backwards compatibility variable for containers on Prefect <0.15.0 "PREFECT__LOGGING__LOG_TO_CLOUD": str(self.log_to_cloud).lower(), # Backwards compatibility variable for containers on Prefect <1.0.0 "PREFECT__CLOUD__AUTH_TOKEN": self.flow_run_api_key or "", }) container_env = [{"name": k, "value": v} for k, v in env.items()] for entry in container.get("environment", []): if entry["name"] not in env: container_env.append(entry) container["environment"] = container_env return out
def generate_job_spec_from_run_config(self, flow_run: GraphQLResult, run_config: KubernetesRun) -> dict: """Generate a k8s job spec for a flow run. Args: - flow_run (GraphQLResult): A flow run object - run_config (KubernetesRun): The flow run's run_config Returns: - dict: a dictionary representation of a k8s job for flow execution """ if run_config.job_template: job = run_config.job_template else: job_template_path = run_config.job_template_path or self.job_template_path self.logger.debug("Loading job template from %r", job_template_path) template_bytes = read_bytes_from_path(job_template_path) job = yaml.safe_load(template_bytes) identifier = uuid.uuid4().hex[:8] job_name = f"prefect-job-{identifier}" # Populate job metadata for identification k8s_labels = { "prefect.io/identifier": identifier, "prefect.io/flow_run_id": flow_run.id, # type: ignore "prefect.io/flow_id": flow_run.flow.id, # type: ignore } _get_or_create(job, "metadata.labels") _get_or_create(job, "spec.template.metadata.labels") job["metadata"]["name"] = job_name job["metadata"]["labels"].update(**k8s_labels) job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels) pod_spec = job["spec"]["template"]["spec"] # Configure `service_account_name` if specified if run_config.service_account_name is not None: # On run-config, always override service_account_name = (run_config.service_account_name ) # type: Optional[str] elif "serviceAccountName" in pod_spec and ( run_config.job_template or run_config.job_template_path): # On run-config job-template, no override service_account_name = None else: # Use agent value, if provided service_account_name = self.service_account_name if service_account_name is not None: pod_spec["serviceAccountName"] = service_account_name # Configure `image_pull_secrets` if specified if run_config.image_pull_secrets is not None: # On run-config, always override image_pull_secrets = (run_config.image_pull_secrets ) # type: Optional[Iterable[str]] elif "imagePullSecrets" in pod_spec and (run_config.job_template or run_config.job_template_path): # On run-config job template, no override image_pull_secrets = None else: # Use agent, if provided image_pull_secrets = self.image_pull_secrets if image_pull_secrets is not None: pod_spec["imagePullSecrets"] = [{ "name": s } for s in image_pull_secrets] # Default restartPolicy to Never _get_or_create(job, "spec.template.spec.restartPolicy", "Never") # Get the first container, which is used for the prefect job containers = _get_or_create(job, "spec.template.spec.containers", []) if not containers: containers.append({}) container = containers[0] # Set container image container["image"] = image = get_flow_image( flow_run, default=container.get("image")) # Set flow run command container["args"] = get_flow_run_command(flow_run).split() # Populate environment variables from the following sources, # with precedence: # - Values required for flow execution, hardcoded below # - Values set on the KubernetesRun object # - Values set using the `--env` CLI flag on the agent # - Values in the job template env = {"PREFECT__LOGGING__LEVEL": config.logging.level} env.update(self.env_vars) if run_config.env: env.update(run_config.env) env.update({ "PREFECT__BACKEND": config.backend, "PREFECT__CLOUD__AGENT__LABELS": str(self.labels), "PREFECT__CLOUD__API": config.cloud.api, "PREFECT__CLOUD__AUTH_TOKEN": config.cloud.agent.auth_token, "PREFECT__CLOUD__USE_LOCAL_SECRETS": "false", "PREFECT__CONTEXT__FLOW_RUN_ID": flow_run.id, "PREFECT__CONTEXT__FLOW_ID": flow_run.flow.id, "PREFECT__CONTEXT__IMAGE": image, "PREFECT__LOGGING__LOG_TO_CLOUD": str(self.log_to_cloud).lower(), "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudFlowRunner", "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS": "prefect.engine.cloud.CloudTaskRunner", }) container_env = [{"name": k, "value": v} for k, v in env.items()] for entry in container.get("env", []): if entry["name"] not in env: container_env.append(entry) container["env"] = container_env # Set resource requirements if provided _get_or_create(container, "resources.requests") _get_or_create(container, "resources.limits") resources = container["resources"] if run_config.memory_request: resources["requests"]["memory"] = run_config.memory_request if run_config.memory_limit: resources["limits"]["memory"] = run_config.memory_limit if run_config.cpu_request: resources["requests"]["cpu"] = run_config.cpu_request if run_config.cpu_limit: resources["limits"]["cpu"] = run_config.cpu_limit return job
msg = f"Flow run {flow_run.id} has a nonexistent `working_dir` configured: {working_dir}" self.logger.error(msg) raise ValueError(msg) stdout = sys.stdout if self.show_flow_logs else DEVNULL # note: we will allow these processes to be orphaned if the agent were to exit # before the flow runs have completed. The lifecycle of the agent should not # dictate the lifecycle of the flow run. However, if the user has elected to # show flow logs, these log entries will continue to stream to the users terminal # until these child processes exit, even if the agent has already exited. p = Popen( <<<<<<< HEAD [sys.executable, "-m", "prefect", "execute", "flow-run"], ======= get_flow_run_command(flow_run).split(" "), >>>>>>> prefect clone stdout=stdout, stderr=STDOUT, env=env, cwd=working_dir, ) self.processes.add(p) self.logger.debug( "Submitted flow run {} to process PID {}".format(flow_run.id, p.pid) ) return "PID: {}".format(p.pid) def populate_env_vars( self, flow_run: GraphQLResult, run_config: LocalRun = None