def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
Raises:
- ValueError: if deployment attempted on unsupported Storage type
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id) # type: ignore
)
if not isinstance(
StorageSchema().load(flow_run.flow.storage),
(Local, Azure, GCS, S3, GitHub, Webhook),
):
self.logger.error(
"Storage for flow run {} is not a supported type.".format(
flow_run.id))
raise ValueError("Unsupported Storage type")
env_vars = self.populate_env_vars(flow_run=flow_run)
current_env = os.environ.copy()
current_env.update(env_vars)
python_path = []
if current_env.get("PYTHONPATH"):
python_path.append(current_env.get("PYTHONPATH"))
python_path.append(os.getcwd())
if self.import_paths:
python_path += self.import_paths
current_env["PYTHONPATH"] = ":".join(python_path)
stdout = sys.stdout if self.show_flow_logs else DEVNULL
# note: we will allow these processes to be orphaned if the agent were to exit
# before the flow runs have completed. The lifecycle of the agent should not
# dictate the lifecycle of the flow run. However, if the user has elected to
# show flow logs, these log entries will continue to stream to the users terminal
# until these child processes exit, even if the agent has already exited.
p = Popen(
get_flow_run_command(flow_run).split(" "),
stdout=stdout,
stderr=STDOUT,
env=current_env,
)
self.processes.add(p)
self.logger.debug("Submitted flow run {} to process PID {}".format(
flow_run.id, p.pid))
return "PID: {}".format(p.pid)
def generate_task_definition(self, flow_run: GraphQLResult) -> Dict[str, Any]:
"""Generate an Vertex task definition from a flow run
Args:
- flow_run (GraphQLResult): A flow run object
Returns:
- dict: a dictionary representation of an Vertex task definition
"""
run_config = self._get_run_config(flow_run, VertexRun)
assert isinstance(run_config, VertexRun) # mypy
image = get_flow_image(flow_run)
job_name = slugify.slugify(
flow_run.flow.name + "-" + flow_run.name,
max_length=255,
word_boundary=True,
save_order=True,
)
machine_type = run_config.machine_type
command = get_flow_run_command(flow_run)
env = self.populate_env_vars(flow_run)
env_list = self._to_env_list(env)
# Start with a default taskdef
taskdef = {
"display_name": job_name,
"job_spec": {
"worker_pool_specs": [
{"machine_spec": {"machine_type": machine_type}, "replica_count": 1}
]
},
} # type: Dict[str, Any]
if run_config.worker_pool_specs is not None:
taskdef["job_spec"]["worker_pool_specs"] = run_config.worker_pool_specs
if run_config.network is not None:
taskdef["job_spec"]["network"] = run_config.network
if run_config.service_account is not None:
taskdef["job_spec"]["service_account"] = run_config.service_account
else:
taskdef["job_spec"]["service_account"] = self.service_account
if run_config.scheduling is not None:
taskdef["job_spec"]["scheduling"] = run_config.scheduling
# We always set the container spec on the zeroth pool spec to ensure it will run the flow
taskdef["job_spec"]["worker_pool_specs"][0]["container_spec"] = {
"image_uri": image,
"command": command.split(),
"args": [],
"env": env_list,
}
return taskdef
def test_get_flow_run_command_works_if_core_version_not_on_response():
legacy_command = "prefect execute cloud-flow"
flow_run = GraphQLResult({
"flow":
GraphQLResult({
"storage": Local().serialize(),
"id": "id",
}),
"id":
"id",
})
assert get_flow_run_command(flow_run) == legacy_command
def test_get_flow_run_command(core_version, command):
flow_run = GraphQLResult({
"flow":
GraphQLResult({
"storage": Local().serialize(),
"id": "id",
"core_version": core_version,
}),
"id":
"id",
})
assert get_flow_run_command(flow_run) == command
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id)) # type: ignore
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
run_config = self._get_run_config(flow_run, DockerRun)
assert run_config is None or isinstance(run_config, DockerRun) # mypy
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run,
image,
run_config=run_config)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
registry = image.split("/")[0]
if self.reg_allow_list and registry not in self.reg_allow_list:
self.logger.error(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
raise ValueError(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
else:
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info(
"Successfully pulled image {}...".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if sys.platform.startswith("linux") and self.docker_interface:
docker_internal_ip = get_docker_ip()
host_config.update(
extra_hosts={"host.docker.internal": docker_internal_ip})
networking_config = None
# At the time of creation, you can only connect a container to a single network,
# however you can create more connections after creation.
# Connect first network in the creation step. If no network is connected here the container
# is connected to the default `bridge` network.
# The rest of the networks are connected after creation.
if self.networks:
networking_config = self.docker_client.create_networking_config({
self.networks[0]:
self.docker_client.create_endpoint_config()
})
# Try fallback on old, deprecated, behaviour.
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()})
labels = {
"io.prefect.flow-name": flow_run.flow.name,
"io.prefect.flow-id": flow_run.flow.id,
"io.prefect.flow-run-id": flow_run.id,
}
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(**host_config),
networking_config=networking_config,
labels=labels,
)
# Connect the rest of the networks
if self.networks:
for network in self.networks[1:]:
self.docker_client.connect_container_to_network(
container=container, net_id=network)
# Start the container
self.logger.debug("Starting Docker container with ID {}".format(
container.get("Id")))
if self.networks:
self.logger.debug(
"Adding container with ID {} to docker networks: {}.".format(
container.get("Id"), self.networks))
if self.network:
self.logger.debug("Adding container to docker network: {}".format(
self.network))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
self.stream_flow_logs(container.get("Id"))
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
def generate_task_definition(self, flow_run: GraphQLResult,
run_config: ECSRun) -> Dict[str, Any]:
"""Generate an ECS task definition from a flow run
Args:
- flow_run (GraphQLResult): A flow run object
- run_config (ECSRun): The flow's run config
Returns:
- dict: a dictionary representation of an ECS task definition
"""
if run_config.task_definition:
taskdef = deepcopy(run_config.task_definition)
elif run_config.task_definition_path:
self.logger.debug(
"Loading task definition template from %r",
run_config.task_definition_path,
)
template_bytes = read_bytes_from_path(
run_config.task_definition_path)
taskdef = yaml.safe_load(template_bytes)
else:
taskdef = deepcopy(self.task_definition)
slug = slugify.slugify(
flow_run.flow.name,
max_length=255 - len("prefect-"),
word_boundary=True,
save_order=True,
)
family = f"prefect-{slug}"
tags = self.get_task_definition_tags(flow_run)
taskdef["family"] = family
taskdef_tags = [{"key": k, "value": v} for k, v in tags.items()]
for entry in taskdef.get("tags", []):
if entry["key"] not in tags:
taskdef_tags.append(entry)
taskdef["tags"] = taskdef_tags
# Get the flow container (creating one if it doesn't already exist)
containers = taskdef.setdefault("containerDefinitions", [])
for container in containers:
if container.get("name") == "flow":
break
else:
container = {"name": "flow"}
containers.append(container)
# Set flow image
container["image"] = image = get_flow_image(flow_run)
# Set flow run command
container["command"] = [
"/bin/sh", "-c", get_flow_run_command(flow_run)
]
# Set taskRoleArn if configured
if run_config.task_role_arn:
taskdef["taskRoleArn"] = run_config.task_role_arn
# Populate static environment variables from the following sources,
# with precedence:
# - Static environment variables, hardcoded below
# - Values in the task definition template
env = {
"PREFECT__CLOUD__USE_LOCAL_SECRETS":
"false",
"PREFECT__CONTEXT__IMAGE":
image,
"PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudFlowRunner",
"PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudTaskRunner",
}
container_env = [{"name": k, "value": v} for k, v in env.items()]
for entry in container.get("environment", []):
if entry["name"] not in env:
container_env.append(entry)
container["environment"] = container_env
# Set resource requirements, if provided
# Also ensure that cpu/memory are strings not integers
if run_config.cpu:
taskdef["cpu"] = str(run_config.cpu)
elif "cpu" in taskdef:
taskdef["cpu"] = str(taskdef["cpu"])
if run_config.memory:
taskdef["memory"] = str(run_config.memory)
elif "memory" in taskdef:
taskdef["memory"] = str(taskdef["memory"])
return taskdef
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
run_config = self._get_run_config(flow_run, DockerRun)
assert run_config is None or isinstance(run_config, DockerRun) # mypy
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run,
image,
run_config=run_config)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
registry = image.split("/")[0]
if self.reg_allow_list and registry not in self.reg_allow_list:
self.logger.error(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
raise ValueError(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
else:
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info("Successfully pulled image {}".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
# By default, auto-remove containers
host_config: Dict[str, Any] = {"auto_remove": True}
# By default, no ports
ports = None
# Set up a host gateway for local communication; check the docker version since
# this is not supported by older versions
docker_engine_version = parse_version(
self.docker_client.version()["Version"])
host_gateway_version = parse_version("20.10.0")
if docker_engine_version < host_gateway_version:
warnings.warn(
"`host.docker.internal` could not be automatically resolved to your "
"local host. This feature is not supported on Docker Engine "
f"v{docker_engine_version}, upgrade to v{host_gateway_version}+ if you "
"encounter issues.")
else:
# Compatibility for linux -- https://github.com/docker/cli/issues/2290
# Only supported by Docker v20.10.0+ which is our minimum recommend version
host_config["extra_hosts"] = {
"host.docker.internal": "host-gateway"
}
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if run_config is not None and run_config.host_config:
# The host_config passed from the run_config will overwrite defaults
host_config.update(run_config.host_config)
if run_config is not None and run_config.ports:
ports = run_config.ports
networking_config = None
# At the time of creation, you can only connect a container to a single network,
# however you can create more connections after creation.
# Connect first network in the creation step. If no network is connected here the container
# is connected to the default `bridge` network.
# The rest of the networks are connected after creation.
if self.networks:
networking_config = self.docker_client.create_networking_config({
self.networks[0]:
self.docker_client.create_endpoint_config()
})
labels = {
"io.prefect.flow-name": flow_run.flow.name,
"io.prefect.flow-id": flow_run.flow.id,
"io.prefect.flow-run-id": flow_run.id,
}
# Generate a container name to match the flow run name, ensuring it is docker
# compatible and unique. Must match `[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end
container_name = slugified_name = (
slugify(
flow_run.name,
lowercase=False,
# Docker does not limit length but URL limits apply eventually so
# limit the length for safety
max_length=250,
# Docker allows these characters for container names
regex_pattern=r"[^a-zA-Z0-9_.-]+",
).lstrip(
# Docker does not allow leading underscore, dash, or period
"_-.")
# Docker does not allow 0 character names so use the flow run id if name
# would be empty after cleaning
or flow_run.id)
# Create the container with retries on name conflicts
index = 0 # will be bumped on name colissions
while True:
try:
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
name=container_name,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(
**host_config),
networking_config=networking_config,
labels=labels,
ports=ports,
)
except docker.errors.APIError as exc:
if "Conflict" in str(exc) and "container name" in str(exc):
index += 1
container_name = f"{slugified_name}-{index}"
else:
raise
else:
break
# Connect the rest of the networks
if self.networks:
for network in self.networks[1:]:
self.docker_client.connect_container_to_network(
container=container, net_id=network)
# Start the container
self.logger.debug(
f"Starting Docker container with ID {container.get('Id')} and "
f"name {container_name!r}")
if self.networks:
self.logger.debug(
"Adding container with ID {} to docker networks: {}.".format(
container.get("Id"), self.networks))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
self.stream_flow_logs(container.get("Id"))
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
def generate_job_spec_from_environment(self,
flow_run: GraphQLResult,
image: str = None) -> dict:
"""
Populate a k8s job spec. This spec defines a k8s job that handles
executing a flow. This method runs each time the agent receives
a flow to run.
That job spec can optionally be customized by setting the
following environment variables on the agent.
- `NAMESPACE`: the k8s namespace the job will run in, defaults to `"default"`
- `JOB_MEM_REQUEST`: memory requested, for example, `256Mi` for 256 MB. If this
environment variable is not set, the cluster's defaults will be used.
- `JOB_MEM_LIMIT`: memory limit, for example, `512Mi` For 512 MB. If this
environment variable is not set, the cluster's defaults will be used.
- `JOB_CPU_REQUEST`: CPU requested, defaults to `"100m"`
- `JOB_CPU_LIMIT`: CPU limit, defaults to `"100m"`
- `IMAGE_PULL_POLICY`: policy for pulling images. Defaults to `"IfNotPresent"`.
- `IMAGE_PULL_SECRETS`: name of an existing k8s secret that can be used to pull
images. This is necessary if your flow uses an image that is in a non-public
container registry, such as Amazon ECR, or in a public registry that requires
authentication to avoid hitting rate limits. To specify multiple image pull
secrets, provide a comma-delimited string with no spaces, like
`"some-secret,other-secret"`.
- `SERVICE_ACCOUNT_NAME`: name of a service account to run the job as.
By default, none is specified.
- `YAML_TEMPLATE`: a path to where the YAML template should be loaded from. defaults
to the embedded `job_spec.yaml`.
Args:
- flow_run (GraphQLResult): A flow run object
- image (str, optional): The full name of an image to use for the job
Returns:
- dict: a dictionary representation of a k8s job for flow execution
"""
identifier = str(uuid.uuid4())[:8]
yaml_path = os.getenv(
"YAML_TEMPLATE",
os.path.join(os.path.dirname(__file__), "job_spec.yaml"))
with open(yaml_path, "r") as job_file:
job = yaml.safe_load(job_file)
job_name = "prefect-job-{}".format(identifier)
# Populate job metadata for identification
k8s_labels = {
"prefect.io/identifier": identifier,
"prefect.io/flow_run_id": flow_run.id, # type: ignore
"prefect.io/flow_id": flow_run.flow.id, # type: ignore
}
job["metadata"]["name"] = job_name
job["metadata"]["labels"].update(**k8s_labels)
job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels)
# Use provided image for job
if image is None:
image = get_flow_image(flow_run=flow_run)
job["spec"]["template"]["spec"]["containers"][0]["image"] = image
self.logger.debug("Using image {} for job".format(image))
# Datermine flow run command
job["spec"]["template"]["spec"]["containers"][0]["args"] = [
get_flow_run_command(flow_run)
]
# Populate environment variables for flow run execution
env = job["spec"]["template"]["spec"]["containers"][0]["env"]
env[0]["value"] = config.cloud.api or "https://api.prefect.io"
env[1]["value"] = config.cloud.agent.auth_token
env[2]["value"] = flow_run.id # type: ignore
env[3]["value"] = flow_run.flow.id # type: ignore
env[4]["value"] = self.namespace
env[5]["value"] = str(self.labels)
env[6]["value"] = str(self.log_to_cloud).lower()
env[7]["value"] = self.env_vars.get("PREFECT__LOGGING__LEVEL",
config.logging.level)
# append all user provided values
for key, value in self.env_vars.items():
env.append(dict(name=key, value=value))
# Use image pull secrets if provided
if self.image_pull_secrets:
for idx, secret_name in enumerate(self.image_pull_secrets):
# this check preserves behavior from previous releases,
# where prefect would only overwrite the first entry in
# imagePullSecrets
if idx == 0:
job["spec"]["template"]["spec"]["imagePullSecrets"][0] = {
"name": secret_name
}
else:
job["spec"]["template"]["spec"]["imagePullSecrets"].append(
{"name": secret_name})
else:
del job["spec"]["template"]["spec"]["imagePullSecrets"]
# Set resource requirements if provided
resources = job["spec"]["template"]["spec"]["containers"][0][
"resources"]
if os.getenv("JOB_MEM_REQUEST"):
resources["requests"]["memory"] = os.getenv("JOB_MEM_REQUEST")
if os.getenv("JOB_MEM_LIMIT"):
resources["limits"]["memory"] = os.getenv("JOB_MEM_LIMIT")
if os.getenv("JOB_CPU_REQUEST"):
resources["requests"]["cpu"] = os.getenv("JOB_CPU_REQUEST")
if os.getenv("JOB_CPU_LIMIT"):
resources["limits"]["cpu"] = os.getenv("JOB_CPU_LIMIT")
if self.volume_mounts:
job["spec"]["template"]["spec"]["containers"][0][
"volumeMounts"] = self.volume_mounts
else:
del job["spec"]["template"]["spec"]["containers"][0][
"volumeMounts"]
if self.volumes:
job["spec"]["template"]["spec"]["volumes"] = self.volumes
else:
del job["spec"]["template"]["spec"]["volumes"]
if os.getenv("IMAGE_PULL_POLICY"):
job["spec"]["template"]["spec"]["containers"][0][
"imagePullPolicy"] = os.getenv("IMAGE_PULL_POLICY")
if self.service_account_name:
job["spec"]["template"]["spec"][
"serviceAccountName"] = self.service_account_name
return job
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
run_config = self._get_run_config(flow_run, DockerRun)
assert run_config is None or isinstance(run_config, DockerRun) # mypy
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run,
image,
run_config=run_config)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
registry = image.split("/")[0]
if self.reg_allow_list and registry not in self.reg_allow_list:
self.logger.error(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
raise ValueError(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
else:
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info("Successfully pulled image {}".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
networking_config = None
# At the time of creation, you can only connect a container to a single network,
# however you can create more connections after creation.
# Connect first network in the creation step. If no network is connected here the container
# is connected to the default `bridge` network.
# The rest of the networks are connected after creation.
if self.networks:
networking_config = self.docker_client.create_networking_config({
self.networks[0]:
self.docker_client.create_endpoint_config()
})
# Try fallback on old, deprecated, behaviour.
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()})
labels = {
"io.prefect.flow-name": flow_run.flow.name,
"io.prefect.flow-id": flow_run.flow.id,
"io.prefect.flow-run-id": flow_run.id,
}
# Generate a container name to match the flow run name, ensuring it is docker
# compatible and unique. Must match `[a-zA-Z0-9][a-zA-Z0-9_.-]+` in the end
container_name = slugified_name = (
slugify(
flow_run.name,
lowercase=False,
# Docker does not limit length but URL limits apply eventually so
# limit the length for safety
max_length=250,
# Docker allows these characters for container names
regex_pattern=r"[^a-zA-Z0-9_.-]+",
).lstrip(
# Docker does not allow leading underscore, dash, or period
"_-.")
# Docker does not allow 0 character names so use the flow run id if name
# would be empty after cleaning
or flow_run.id)
# Create the container with retries on name conflicts
index = 0 # will be bumped on name colissions
while True:
try:
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
name=container_name,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(
**host_config),
networking_config=networking_config,
labels=labels,
)
except docker.errors.APIError as exc:
if "Conflict" in str(exc) and "container name" in str(exc):
index += 1
container_name = f"{slugified_name}-{index}"
else:
raise
else:
break
# Connect the rest of the networks
if self.networks:
for network in self.networks[1:]:
self.docker_client.connect_container_to_network(
container=container, net_id=network)
# Start the container
self.logger.debug(
f"Starting Docker container with ID {container.get('Id')} and "
f"name {container_name!r}")
if self.networks:
self.logger.debug(
"Adding container with ID {} to docker networks: {}.".format(
container.get("Id"), self.networks))
if self.network:
self.logger.debug("Adding container to docker network: {}".format(
self.network))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
self.stream_flow_logs(container.get("Id"))
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
def get_run_task_kwargs(self, flow_run: GraphQLResult,
run_config: ECSRun) -> Dict[str, Any]:
"""Generate kwargs to pass to `ECS.client.run_task` for a flow run
Args:
- flow_run (GraphQLResult): A flow run object
- run_config (ECSRun): The flow's run config
Returns:
- dict: kwargs to pass to `ECS.client.run_task`
"""
# Set agent defaults
out = deepcopy(self.run_task_kwargs)
out["launchType"] = self.launch_type
if self.cluster:
out["cluster"] = self.cluster
# Apply run-config kwargs, if any
if run_config.run_task_kwargs:
out = merge_run_task_kwargs(out, run_config.run_task_kwargs)
# Find or create the flow container overrides
overrides = out.setdefault("overrides", {})
container_overrides = overrides.setdefault("containerOverrides", [])
for container in container_overrides:
if container.get("name") == "flow":
break
else:
container = {"name": "flow"}
container_overrides.append(container)
# Set taskRoleArn if configured
if run_config.task_role_arn:
overrides["taskRoleArn"] = run_config.task_role_arn
elif self.task_role_arn:
overrides["taskRoleArn"] = self.task_role_arn
# Set executionRoleArn if configured
if run_config.execution_role_arn:
overrides["executionRoleArn"] = run_config.execution_role_arn
elif self.execution_role_arn:
overrides["executionRoleArn"] = self.execution_role_arn
# Set resource requirements, if provided
# Also ensure that cpu/memory are strings not integers
if run_config.cpu:
overrides["cpu"] = str(run_config.cpu)
elif "cpu" in overrides:
overrides["cpu"] = str(overrides["cpu"])
if run_config.memory:
overrides["memory"] = str(run_config.memory)
elif "memory" in overrides:
overrides["memory"] = str(overrides["memory"])
# Set flow run command
container["command"] = [
"/bin/sh", "-c", get_flow_run_command(flow_run)
]
# Populate environment variables from the following sources,
# with precedence:
# - Values required for flow execution, hardcoded below
# - Values set on the ECSRun object
# - Values set using the `--env` CLI flag on the agent
env = self.env_vars.copy()
if run_config.env:
env.update(run_config.env)
env.update({
"PREFECT__CLOUD__USE_LOCAL_SECRETS":
"false",
"PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudFlowRunner",
"PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudTaskRunner",
"PREFECT__BACKEND":
config.backend,
"PREFECT__CLOUD__API":
config.cloud.api,
"PREFECT__CONTEXT__FLOW_RUN_ID":
flow_run.id,
"PREFECT__CONTEXT__FLOW_ID":
flow_run.flow.id,
"PREFECT__LOGGING__LOG_TO_CLOUD":
str(self.log_to_cloud).lower(),
"PREFECT__CLOUD__AUTH_TOKEN":
config.cloud.agent.auth_token,
"PREFECT__CLOUD__AGENT__LABELS":
str(self.labels),
})
container_env = [{"name": k, "value": v} for k, v in env.items()]
for entry in container.get("environment", []):
if entry["name"] not in env:
container_env.append(entry)
container["environment"] = container_env
return out
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id) # type: ignore
)
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run=flow_run)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
registry = image.split("/")[0]
if self.reg_allow_list and registry not in self.reg_allow_list:
self.logger.error(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
raise ValueError(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
else:
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info(
"Successfully pulled image {}...".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if sys.platform.startswith("linux") and self.docker_interface:
docker_internal_ip = get_docker_ip()
host_config.update(
extra_hosts={"host.docker.internal": docker_internal_ip})
networking_config = None
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()})
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(**host_config),
networking_config=networking_config,
)
# Start the container
self.logger.debug("Starting Docker container with ID {}".format(
container.get("Id")))
if self.network:
self.logger.debug("Adding container to docker network: {}".format(
self.network))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
proc = multiprocessing.Process(
target=self.stream_container_logs,
kwargs={"container_id": container.get("Id")},
)
proc.start()
self.processes.append(proc)
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
def generate_job_spec_from_run_config(self,
flow_run: GraphQLResult) -> dict:
"""Generate a k8s job spec for a flow run.
Args:
- flow_run (GraphQLResult): A flow run object
Returns:
- dict: a dictionary representation of a k8s job for flow execution
"""
run_config = RunConfigSchema().load(flow_run.flow.run_config)
if run_config.job_template:
job = run_config.job_template
else:
job_template_path = run_config.job_template_path or self.job_template_path
self.logger.debug("Loading job template from %r",
job_template_path)
template_bytes = read_bytes_from_path(job_template_path)
job = yaml.safe_load(template_bytes)
identifier = uuid.uuid4().hex[:8]
job_name = f"prefect-job-{identifier}"
# Populate job metadata for identification
k8s_labels = {
"prefect.io/identifier": identifier,
"prefect.io/flow_run_id": flow_run.id, # type: ignore
"prefect.io/flow_id": flow_run.flow.id, # type: ignore
}
_get_or_create(job, "metadata.labels")
_get_or_create(job, "spec.template.metadata.labels")
job["metadata"]["name"] = job_name
job["metadata"]["labels"].update(**k8s_labels)
job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels)
# Get the first container, which is used for the prefect job
containers = _get_or_create(job, "spec.template.spec.containers", [])
if not containers:
containers.append({})
container = containers[0]
# Set container image
container["image"] = image = get_flow_image(flow_run)
# Set flow run command
container["args"] = [get_flow_run_command(flow_run)]
# Populate environment variables from the following sources,
# with precedence:
# - Values required for flow execution, hardcoded below
# - Values set on the KubernetesRun object
# - Values set using the `--env` CLI flag on the agent
# - Values in the job template
env = self.env_vars.copy()
if run_config.env:
env.update(run_config.env)
env.update({
"PREFECT__CLOUD__API":
config.cloud.api,
"PREFECT__CLOUD__AUTH_TOKEN":
config.cloud.agent.auth_token,
"PREFECT__CLOUD__USE_LOCAL_SECRETS":
"false",
"PREFECT__CONTEXT__FLOW_RUN_ID":
flow_run.id,
"PREFECT__CONTEXT__FLOW_ID":
flow_run.flow.id,
"PREFECT__CONTEXT__IMAGE":
image,
"PREFECT__LOGGING__LOG_TO_CLOUD":
str(self.log_to_cloud).lower(),
"PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudFlowRunner",
"PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudTaskRunner",
})
container_env = [{"name": k, "value": v} for k, v in env.items()]
for entry in container.get("env", []):
if entry["name"] not in env:
container_env.append(entry)
container["env"] = container_env
# Set resource requirements if provided
_get_or_create(container, "resources.requests")
_get_or_create(container, "resources.limits")
resources = container["resources"]
if run_config.memory_request:
resources["requests"]["memory"] = run_config.memory_request
if run_config.memory_limit:
resources["limits"]["memory"] = run_config.memory_limit
if run_config.cpu_request:
resources["requests"]["cpu"] = run_config.cpu_request
if run_config.cpu_limit:
resources["limits"]["cpu"] = run_config.cpu_limit
return job
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
Raises:
- ValueError: if deployment attempted on unsupported Storage type
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id)) # type: ignore
storage = StorageSchema().load(flow_run.flow.storage)
if isinstance(storage, Docker):
self.logger.error(
"Flow run %s has an unsupported storage type: `%s`",
flow_run.id,
type(storage).__name__,
)
raise TypeError("Unsupported Storage type: %s" %
type(storage).__name__)
# If the flow is using a run_config, load it
if getattr(flow_run.flow, "run_config", None) is not None:
run_config = RunConfigSchema().load(flow_run.flow.run_config)
if not isinstance(run_config, LocalRun):
self.logger.error(
"Flow run %s has a `run_config` of type `%s`, only `LocalRun` is supported",
flow_run.id,
type(run_config).__name__,
)
raise TypeError("Unsupported RunConfig type: %s" %
type(run_config).__name__)
else:
run_config = None
env = self.populate_env_vars(flow_run, run_config=run_config)
working_dir = None if run_config is None else run_config.working_dir
if working_dir and not os.path.exists(working_dir):
msg = f"Flow run {flow_run.id} has a nonexistent `working_dir` configured: {working_dir}"
self.logger.error(msg)
raise ValueError(msg)
stdout = sys.stdout if self.show_flow_logs else DEVNULL
# note: we will allow these processes to be orphaned if the agent were to exit
# before the flow runs have completed. The lifecycle of the agent should not
# dictate the lifecycle of the flow run. However, if the user has elected to
# show flow logs, these log entries will continue to stream to the users terminal
# until these child processes exit, even if the agent has already exited.
p = Popen(
get_flow_run_command(flow_run).split(" "),
stdout=stdout,
stderr=STDOUT,
env=env,
cwd=working_dir,
)
self.processes.add(p)
self.logger.debug("Submitted flow run {} to process PID {}".format(
flow_run.id, p.pid))
return "PID: {}".format(p.pid)
def replace_job_spec_yaml(self, flow_run: GraphQLResult, image: str) -> dict:
"""
Populate a k8s job spec. This spec defines a k8s job that handles
executing a flow. This method runs each time the agent receives
a flow to run.
That job spec can optionally be customized by setting the
following environment variables on the agent.
- `NAMESPACE`: the k8s namespace the job will run in, defaults to `"default"`
- `JOB_MEM_REQUEST`: memory requested, for example, `256Mi` for 256 MB. If this
environment variable is not set, the cluster's defaults will be used.
- `JOB_MEM_LIMIT`: memory limit, for example, `512Mi` For 512 MB. If this
environment variable is not set, the cluster's defaults will be used.
- `JOB_CPU_REQUEST`: CPU requested, defaults to `"100m"`
- `JOB_CPU_LIMIT`: CPU limit, defaults to `"100m"`
- `IMAGE_PULL_POLICY`: policy for pulling images. Defaults to `"IfNotPresent"`.
- `IMAGE_PULL_SECRETS`: name of an existing k8s secret that can be used to pull
images. This is necessary if your flow uses an image that is in a non-public
container registry, such as Amazon ECR.
- `SERVICE_ACCOUNT_NAME`: name of a service account to run the job as.
By default, none is specified.
Args:
- flow_run (GraphQLResult): A flow run object
- image (str): The full name of an image to use for the job
Returns:
- dict: a dictionary representation of a k8s job for flow execution
"""
with open(path.join(path.dirname(__file__), "job_spec.yaml"), "r") as job_file:
job = yaml.safe_load(job_file)
identifier = str(uuid.uuid4())[:8]
job_name = "prefect-job-{}".format(identifier)
# Populate job metadata for identification
k8s_labels = {
"prefect.io/identifier": identifier,
"prefect.io/flow_run_id": flow_run.id, # type: ignore
"prefect.io/flow_id": flow_run.flow.id, # type: ignore
}
job["metadata"]["name"] = job_name
job["metadata"]["labels"].update(**k8s_labels)
job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels)
# Use provided image for job
job["spec"]["template"]["spec"]["containers"][0]["image"] = image
self.logger.debug("Using image {} for job".format(image))
# Datermine flow run command
job["spec"]["template"]["spec"]["containers"][0]["args"] = [
get_flow_run_command(flow_run)
]
# Populate environment variables for flow run execution
env = job["spec"]["template"]["spec"]["containers"][0]["env"]
env[0]["value"] = config.cloud.api or "https://api.prefect.io"
env[1]["value"] = config.cloud.agent.auth_token
env[2]["value"] = flow_run.id # type: ignore
env[3]["value"] = flow_run.flow.id # type: ignore
env[4]["value"] = os.getenv("NAMESPACE", "default")
env[5]["value"] = str(self.labels)
env[6]["value"] = str(self.log_to_cloud).lower()
# append all user provided values
for key, value in self.env_vars.items():
env.append(dict(name=key, value=value))
# Use image pull secrets if provided
job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"] = os.getenv(
"IMAGE_PULL_SECRETS", ""
)
# Set resource requirements if provided
resources = job["spec"]["template"]["spec"]["containers"][0]["resources"]
if os.getenv("JOB_MEM_REQUEST"):
resources["requests"]["memory"] = os.getenv("JOB_MEM_REQUEST")
if os.getenv("JOB_MEM_LIMIT"):
resources["limits"]["memory"] = os.getenv("JOB_MEM_LIMIT")
if os.getenv("JOB_CPU_REQUEST"):
resources["requests"]["cpu"] = os.getenv("JOB_CPU_REQUEST")
if os.getenv("JOB_CPU_LIMIT"):
resources["limits"]["cpu"] = os.getenv("JOB_CPU_LIMIT")
if os.getenv("IMAGE_PULL_POLICY"):
job["spec"]["template"]["spec"]["containers"][0][
"imagePullPolicy"
] = os.getenv("IMAGE_PULL_POLICY")
if os.getenv("SERVICE_ACCOUNT_NAME"):
job["spec"]["template"]["spec"]["serviceAccountName"] = os.getenv(
"SERVICE_ACCOUNT_NAME"
)
return job
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()}
)
<<<<<<< HEAD
labels = {
"io.prefect.flow-name": flow_run.flow.name,
"io.prefect.flow-id": flow_run.flow.id,
"io.prefect.flow-run-id": flow_run.id,
}
=======
>>>>>>> prefect clone
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(**host_config),
networking_config=networking_config,
<<<<<<< HEAD
labels=labels,
=======
>>>>>>> prefect clone
)
# Start the container
self.logger.debug(
"Starting Docker container with ID {}".format(container.get("Id"))
)
if self.network:
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs to Fargate
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
self.logger.info("Deploying flow run {}".format(flow_run.id)) # type: ignore
# create copies of kwargs to apply overrides as needed
flow_task_definition_kwargs = copy.deepcopy(self.task_definition_kwargs)
flow_task_run_kwargs = copy.deepcopy(self.task_run_kwargs)
flow_container_definitions_kwargs = copy.deepcopy(
self.container_definitions_kwargs
)
# create task_definition_name dict for passing into verify method
task_definition_dict = {}
if self.use_external_kwargs:
# override from external kwargs
self._override_kwargs(
flow_run,
flow_task_definition_kwargs,
flow_task_run_kwargs,
flow_container_definitions_kwargs,
)
# set proper task_definition_name and tags based on enable_task_revisions flag
if self.enable_task_revisions:
# set task definition name
task_definition_dict["task_definition_name"] = slugify(flow_run.flow.name)
self._add_flow_tags(flow_run, flow_task_definition_kwargs)
else:
task_definition_dict["task_definition_name"] = "prefect-task-{}".format( # type: ignore
flow_run.flow.id[:8] # type: ignore
) # type: ignore
image = get_flow_image(flow_run=flow_run)
flow_run_command = get_flow_run_command(flow_run=flow_run)
# check if task definition exists
self.logger.debug("Checking for task definition")
if not self._verify_task_definition_exists(flow_run, task_definition_dict):
self.logger.debug("No task definition found")
self._create_task_definition(
image=image,
flow_task_definition_kwargs=flow_task_definition_kwargs,
container_definitions_kwargs=flow_container_definitions_kwargs,
task_definition_name=task_definition_dict["task_definition_name"],
flow_run_command=flow_run_command,
)
# run task
task_arn = self._run_task(
flow_run, flow_task_run_kwargs, task_definition_dict["task_definition_name"]
)
self.logger.debug("Run created for task {}".format(task_arn))
return "Task ARN: {}".format(task_arn)
def get_run_task_kwargs(self, flow_run: GraphQLResult, run_config: ECSRun,
taskdef: Dict[str, Any]) -> Dict[str, Any]:
"""Generate kwargs to pass to `ECS.client.run_task` for a flow run
Args:
- flow_run (GraphQLResult): A flow run object
- run_config (ECSRun): The flow's run config
- taskdef (Dict): The ECS task definition used in ECS.client.run_task
Returns:
- dict: kwargs to pass to `ECS.client.run_task`
"""
# Set agent defaults
out = deepcopy(self.run_task_kwargs)
# Use launchType only if capacity provider is not specified
if not out.get("capacityProviderStrategy"):
out["launchType"] = self.launch_type
if self.cluster:
out["cluster"] = self.cluster
# Apply run-config kwargs, if any
if run_config.run_task_kwargs:
out = merge_run_task_kwargs(out, run_config.run_task_kwargs)
# Find or create the flow container overrides
overrides = out.setdefault("overrides", {})
container_overrides = overrides.setdefault("containerOverrides", [])
for container in container_overrides:
if container.get("name") == "flow":
break
else:
container = {"name": "flow"}
container_overrides.append(container)
# Task roles and execution roles should be retrieved from the following sources,
# in order:
# - An ARN passed explicitly on run_config.
# - An ARN present on the task definition used for run_task. In this case, we
# don't need to provide an override at all.
# - An ARN passed in to the ECSAgent when instantiated, either programmatically
# or via the CLI.
if run_config.task_role_arn:
overrides["taskRoleArn"] = run_config.task_role_arn
elif taskdef.get("taskRoleArn"):
overrides["taskRoleArn"] = taskdef["taskRoleArn"]
elif self.task_role_arn:
overrides["taskRoleArn"] = self.task_role_arn
if run_config.execution_role_arn:
overrides["executionRoleArn"] = run_config.execution_role_arn
elif taskdef.get("executionRoleArn"):
overrides["executionRoleArn"] = taskdef["executionRoleArn"]
elif self.execution_role_arn and not taskdef.get("executionRoleArn"):
overrides["executionRoleArn"] = self.execution_role_arn
# Set resource requirements, if provided
# Also ensure that cpu/memory are strings not integers
if run_config.cpu:
overrides["cpu"] = str(run_config.cpu)
elif "cpu" in overrides:
overrides["cpu"] = str(overrides["cpu"])
if run_config.memory:
overrides["memory"] = str(run_config.memory)
elif "memory" in overrides:
overrides["memory"] = str(overrides["memory"])
# Set flow run command
container["command"] = [
"/bin/sh", "-c", get_flow_run_command(flow_run)
]
# Add `PREFECT__LOGGING__LEVEL` environment variable
env = {"PREFECT__LOGGING__LEVEL": config.logging.level}
# Populate environment variables from the following sources,
# with precedence:
# - Values required for flow execution, hardcoded below
# - Values set on the ECSRun object
# - Values set using the `--env` CLI flag on the agent
env.update(self.env_vars)
if run_config.env:
env.update(run_config.env)
env.update({
"PREFECT__CLOUD__USE_LOCAL_SECRETS":
"false",
"PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudFlowRunner",
"PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudTaskRunner",
"PREFECT__BACKEND":
config.backend,
"PREFECT__CLOUD__API":
config.cloud.api,
"PREFECT__CONTEXT__FLOW_RUN_ID":
flow_run.id,
"PREFECT__CONTEXT__FLOW_ID":
flow_run.flow.id,
"PREFECT__CLOUD__SEND_FLOW_RUN_LOGS":
str(self.log_to_cloud).lower(),
"PREFECT__CLOUD__API_KEY":
self.flow_run_api_key or "",
"PREFECT__CLOUD__TENANT_ID": (
# Providing a tenant id is only necessary when authenticating
self.client.tenant_id if self.flow_run_api_key else ""),
"PREFECT__CLOUD__AGENT__LABELS":
str(self.labels),
# Backwards compatibility variable for containers on Prefect <0.15.0
"PREFECT__LOGGING__LOG_TO_CLOUD":
str(self.log_to_cloud).lower(),
# Backwards compatibility variable for containers on Prefect <1.0.0
"PREFECT__CLOUD__AUTH_TOKEN":
self.flow_run_api_key or "",
})
container_env = [{"name": k, "value": v} for k, v in env.items()]
for entry in container.get("environment", []):
if entry["name"] not in env:
container_env.append(entry)
container["environment"] = container_env
return out
def generate_job_spec_from_run_config(self, flow_run: GraphQLResult,
run_config: KubernetesRun) -> dict:
"""Generate a k8s job spec for a flow run.
Args:
- flow_run (GraphQLResult): A flow run object
- run_config (KubernetesRun): The flow run's run_config
Returns:
- dict: a dictionary representation of a k8s job for flow execution
"""
if run_config.job_template:
job = run_config.job_template
else:
job_template_path = run_config.job_template_path or self.job_template_path
self.logger.debug("Loading job template from %r",
job_template_path)
template_bytes = read_bytes_from_path(job_template_path)
job = yaml.safe_load(template_bytes)
identifier = uuid.uuid4().hex[:8]
job_name = f"prefect-job-{identifier}"
# Populate job metadata for identification
k8s_labels = {
"prefect.io/identifier": identifier,
"prefect.io/flow_run_id": flow_run.id, # type: ignore
"prefect.io/flow_id": flow_run.flow.id, # type: ignore
}
_get_or_create(job, "metadata.labels")
_get_or_create(job, "spec.template.metadata.labels")
job["metadata"]["name"] = job_name
job["metadata"]["labels"].update(**k8s_labels)
job["spec"]["template"]["metadata"]["labels"].update(**k8s_labels)
pod_spec = job["spec"]["template"]["spec"]
# Configure `service_account_name` if specified
if run_config.service_account_name is not None:
# On run-config, always override
service_account_name = (run_config.service_account_name
) # type: Optional[str]
elif "serviceAccountName" in pod_spec and (
run_config.job_template or run_config.job_template_path):
# On run-config job-template, no override
service_account_name = None
else:
# Use agent value, if provided
service_account_name = self.service_account_name
if service_account_name is not None:
pod_spec["serviceAccountName"] = service_account_name
# Configure `image_pull_secrets` if specified
if run_config.image_pull_secrets is not None:
# On run-config, always override
image_pull_secrets = (run_config.image_pull_secrets
) # type: Optional[Iterable[str]]
elif "imagePullSecrets" in pod_spec and (run_config.job_template or
run_config.job_template_path):
# On run-config job template, no override
image_pull_secrets = None
else:
# Use agent, if provided
image_pull_secrets = self.image_pull_secrets
if image_pull_secrets is not None:
pod_spec["imagePullSecrets"] = [{
"name": s
} for s in image_pull_secrets]
# Default restartPolicy to Never
_get_or_create(job, "spec.template.spec.restartPolicy", "Never")
# Get the first container, which is used for the prefect job
containers = _get_or_create(job, "spec.template.spec.containers", [])
if not containers:
containers.append({})
container = containers[0]
# Set container image
container["image"] = image = get_flow_image(
flow_run, default=container.get("image"))
# Set flow run command
container["args"] = get_flow_run_command(flow_run).split()
# Populate environment variables from the following sources,
# with precedence:
# - Values required for flow execution, hardcoded below
# - Values set on the KubernetesRun object
# - Values set using the `--env` CLI flag on the agent
# - Values in the job template
env = {"PREFECT__LOGGING__LEVEL": config.logging.level}
env.update(self.env_vars)
if run_config.env:
env.update(run_config.env)
env.update({
"PREFECT__BACKEND":
config.backend,
"PREFECT__CLOUD__AGENT__LABELS":
str(self.labels),
"PREFECT__CLOUD__API":
config.cloud.api,
"PREFECT__CLOUD__AUTH_TOKEN":
config.cloud.agent.auth_token,
"PREFECT__CLOUD__USE_LOCAL_SECRETS":
"false",
"PREFECT__CONTEXT__FLOW_RUN_ID":
flow_run.id,
"PREFECT__CONTEXT__FLOW_ID":
flow_run.flow.id,
"PREFECT__CONTEXT__IMAGE":
image,
"PREFECT__LOGGING__LOG_TO_CLOUD":
str(self.log_to_cloud).lower(),
"PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudFlowRunner",
"PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS":
"prefect.engine.cloud.CloudTaskRunner",
})
container_env = [{"name": k, "value": v} for k, v in env.items()]
for entry in container.get("env", []):
if entry["name"] not in env:
container_env.append(entry)
container["env"] = container_env
# Set resource requirements if provided
_get_or_create(container, "resources.requests")
_get_or_create(container, "resources.limits")
resources = container["resources"]
if run_config.memory_request:
resources["requests"]["memory"] = run_config.memory_request
if run_config.memory_limit:
resources["limits"]["memory"] = run_config.memory_limit
if run_config.cpu_request:
resources["requests"]["cpu"] = run_config.cpu_request
if run_config.cpu_limit:
resources["limits"]["cpu"] = run_config.cpu_limit
return job
msg = f"Flow run {flow_run.id} has a nonexistent `working_dir` configured: {working_dir}"
self.logger.error(msg)
raise ValueError(msg)
stdout = sys.stdout if self.show_flow_logs else DEVNULL
# note: we will allow these processes to be orphaned if the agent were to exit
# before the flow runs have completed. The lifecycle of the agent should not
# dictate the lifecycle of the flow run. However, if the user has elected to
# show flow logs, these log entries will continue to stream to the users terminal
# until these child processes exit, even if the agent has already exited.
p = Popen(
<<<<<<< HEAD
[sys.executable, "-m", "prefect", "execute", "flow-run"],
=======
get_flow_run_command(flow_run).split(" "),
>>>>>>> prefect clone
stdout=stdout,
stderr=STDOUT,
env=env,
cwd=working_dir,
)
self.processes.add(p)
self.logger.debug(
"Submitted flow run {} to process PID {}".format(flow_run.id, p.pid)
)
return "PID: {}".format(p.pid)
def populate_env_vars(
self, flow_run: GraphQLResult, run_config: LocalRun = None
