def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id) # type: ignore
)
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run=flow_run)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info("Successfully pulled image {}...".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if sys.platform.startswith("linux") and self.docker_interface:
docker_internal_ip = get_docker_ip()
host_config.update(
extra_hosts={"host.docker.internal": docker_internal_ip})
networking_config = None
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()})
container = self.docker_client.create_container(
image,
command="prefect execute cloud-flow",
environment=env_vars,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(**host_config),
networking_config=networking_config,
)
# Start the container
self.logger.debug("Starting Docker container with ID {}".format(
container.get("Id")))
if self.network:
self.logger.debug("Adding container to docker network: {}".format(
self.network))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
proc = multiprocessing.Process(
target=self.stream_container_logs,
kwargs={"container_id": container.get("Id")},
)
proc.start()
self.processes.append(proc)
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if sys.platform.startswith("linux") and self.docker_interface:
docker_internal_ip = get_docker_ip()
host_config.update(extra_hosts={"host.docker.internal": docker_internal_ip})
networking_config = None
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()}
)
<<<<<<< HEAD
labels = {
"io.prefect.flow-name": flow_run.flow.name,
"io.prefect.flow-id": flow_run.flow.id,
"io.prefect.flow-run-id": flow_run.id,
}
def start(
version,
skip_pull,
no_upgrade,
no_ui,
postgres_port,
hasura_port,
graphql_port,
ui_port,
server_port,
no_postgres_port,
no_hasura_port,
no_graphql_port,
no_ui_port,
no_server_port,
):
"""
This command spins up all infrastructure and services for the Prefect Core server
\b
Options:
--version, -v TEXT The server image versions to use (for example, '0.10.0' or 'master')
Defaults to the current installed Prefect version.
--skip-pull Flag to skip pulling new images (if available)
--no-upgrade, -n Flag to avoid running a database upgrade when the database spins up
--no-ui, -u Flag to avoid starting the UI
\b
--postgres-port TEXT Port used to serve Postgres, defaults to '5432'
--hasura-port TEXT Port used to serve Hasura, defaults to '3001'
--graphql-port TEXT Port used to serve the GraphQL API, defaults to '4001'
--ui-port TEXT Port used to serve the UI, defaults to '8080'
--server-port TEXT Port used to serve the Core server, defaults to '4200'
\b
--no-postgres-port Disable port map of Postgres to host
--no-hasura-port Disable port map of Hasura to host
--no-graphql-port Disable port map of the GraphQL API to host
--no-ui-port Disable port map of the UI to host
--no-server-port Disable port map of the Core server to host
"""
docker_dir = Path(__file__).parents[0]
compose_dir_path = docker_dir
# Remove port mappings if specified
if (no_postgres_port or no_hasura_port or no_graphql_port or no_ui_port
or no_server_port or platform_is_linux()):
temp_dir = tempfile.gettempdir()
temp_path = os.path.join(temp_dir, "docker-compose.yml")
shutil.copy2(os.path.join(docker_dir, "docker-compose.yml"), temp_path)
with open(temp_path, "r") as file:
y = yaml.safe_load(file)
if no_postgres_port:
del y["services"]["postgres"]["ports"]
if no_hasura_port:
del y["services"]["hasura"]["ports"]
if no_graphql_port:
del y["services"]["graphql"]["ports"]
if no_ui_port:
del y["services"]["ui"]["ports"]
if no_server_port:
del y["services"]["apollo"]["ports"]
if platform_is_linux():
docker_internal_ip = get_docker_ip()
for service in list(y["services"]):
y["services"][service]["extra_hosts"] = [
"host.docker.internal:{}".format(docker_internal_ip)
]
with open(temp_path, "w") as f:
y = yaml.safe_dump(y, f)
compose_dir_path = temp_dir
# Temporary config set for port allocation
with set_temporary_config({
"server.database.host_port": postgres_port,
"server.hasura.host_port": hasura_port,
"server.graphql.host_port": graphql_port,
"server.ui.host_port": ui_port,
"server.host_port": server_port,
}):
env = make_env()
if "PREFECT_SERVER_TAG" not in env:
env.update(PREFECT_SERVER_TAG=version or ("master" if len(
prefect.__version__.split("+")) > 1 else prefect.__version__))
if "PREFECT_SERVER_DB_CMD" not in env:
cmd = ("prefect-server database upgrade -y"
if not no_upgrade else "echo 'DATABASE MIGRATIONS SKIPPED'")
env.update(PREFECT_SERVER_DB_CMD=cmd)
proc = None
try:
if not skip_pull:
subprocess.check_call(["docker-compose", "pull"],
cwd=compose_dir_path,
env=env)
cmd = ["docker-compose", "up"]
if no_ui:
cmd += ["--scale", "ui=0"]
proc = subprocess.Popen(cmd, cwd=compose_dir_path, env=env)
while True:
time.sleep(0.5)
except:
click.secho(
"Exception caught; killing services (press ctrl-C to force)",
fg="white",
bg="red",
)
subprocess.check_output(["docker-compose", "down"],
cwd=compose_dir_path,
env=env)
if proc:
proc.kill()
raise
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id)) # type: ignore
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
run_config = self._get_run_config(flow_run, DockerRun)
assert run_config is None or isinstance(run_config, DockerRun) # mypy
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run,
image,
run_config=run_config)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
registry = image.split("/")[0]
if self.reg_allow_list and registry not in self.reg_allow_list:
self.logger.error(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
raise ValueError(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
else:
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info(
"Successfully pulled image {}...".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if sys.platform.startswith("linux") and self.docker_interface:
docker_internal_ip = get_docker_ip()
host_config.update(
extra_hosts={"host.docker.internal": docker_internal_ip})
networking_config = None
# At the time of creation, you can only connect a container to a single network,
# however you can create more connections after creation.
# Connect first network in the creation step. If no network is connected here the container
# is connected to the default `bridge` network.
# The rest of the networks are connected after creation.
if self.networks:
networking_config = self.docker_client.create_networking_config({
self.networks[0]:
self.docker_client.create_endpoint_config()
})
# Try fallback on old, deprecated, behaviour.
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()})
labels = {
"io.prefect.flow-name": flow_run.flow.name,
"io.prefect.flow-id": flow_run.flow.id,
"io.prefect.flow-run-id": flow_run.id,
}
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(**host_config),
networking_config=networking_config,
labels=labels,
)
# Connect the rest of the networks
if self.networks:
for network in self.networks[1:]:
self.docker_client.connect_container_to_network(
container=container, net_id=network)
# Start the container
self.logger.debug("Starting Docker container with ID {}".format(
container.get("Id")))
if self.networks:
self.logger.debug(
"Adding container with ID {} to docker networks: {}.".format(
container.get("Id"), self.networks))
if self.network:
self.logger.debug("Adding container to docker network: {}".format(
self.network))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
self.stream_flow_logs(container.get("Id"))
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
def deploy_flow(self, flow_run: GraphQLResult) -> str:
"""
Deploy flow runs on your local machine as Docker containers
Args:
- flow_run (GraphQLResult): A GraphQLResult flow run object
Returns:
- str: Information about the deployment
"""
self.logger.info("Deploying flow run {}".format(
flow_run.id)) # type: ignore
# 'import docker' is expensive time-wise, we should do this just-in-time to keep
# the 'import prefect' time low
import docker
if getattr(flow_run.flow, "run_config", None) is not None:
run_config = RunConfigSchema().load(flow_run.flow.run_config)
if not isinstance(run_config, DockerRun):
self.logger.error(
"Flow run %s has a `run_config` of type `%s`, only `DockerRun` is supported",
flow_run.id,
type(run_config).__name__,
)
raise TypeError("Unsupported RunConfig type: %s" %
type(run_config).__name__)
else:
run_config = None
image = get_flow_image(flow_run=flow_run)
env_vars = self.populate_env_vars(flow_run, run_config=run_config)
if not self.no_pull and len(image.split("/")) > 1:
self.logger.info("Pulling image {}...".format(image))
registry = image.split("/")[0]
if self.reg_allow_list and registry not in self.reg_allow_list:
self.logger.error(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
raise ValueError(
"Trying to pull image from a Docker registry '{}' which"
" is not in the reg_allow_list".format(registry))
else:
pull_output = self.docker_client.pull(image,
stream=True,
decode=True)
for line in pull_output:
self.logger.debug(line)
self.logger.info(
"Successfully pulled image {}...".format(image))
# Create any named volumes (if they do not already exist)
for named_volume_name in self.named_volumes:
try:
self.docker_client.inspect_volume(name=named_volume_name)
except docker.errors.APIError:
self.logger.debug(
"Creating named volume {}".format(named_volume_name))
self.docker_client.create_volume(
name=named_volume_name,
driver="local",
labels={"prefect_created": "true"},
)
# Create a container
self.logger.debug("Creating Docker container {}".format(image))
host_config = {"auto_remove": True} # type: dict
container_mount_paths = self.container_mount_paths
if container_mount_paths:
host_config.update(binds=self.host_spec)
if sys.platform.startswith("linux") and self.docker_interface:
docker_internal_ip = get_docker_ip()
host_config.update(
extra_hosts={"host.docker.internal": docker_internal_ip})
networking_config = None
if self.network:
networking_config = self.docker_client.create_networking_config(
{self.network: self.docker_client.create_endpoint_config()})
container = self.docker_client.create_container(
image,
command=get_flow_run_command(flow_run),
environment=env_vars,
volumes=container_mount_paths,
host_config=self.docker_client.create_host_config(**host_config),
networking_config=networking_config,
)
# Start the container
self.logger.debug("Starting Docker container with ID {}".format(
container.get("Id")))
if self.network:
self.logger.debug("Adding container to docker network: {}".format(
self.network))
self.docker_client.start(container=container.get("Id"))
if self.show_flow_logs:
self.stream_flow_logs(container.get("Id"))
self.logger.debug("Docker container {} started".format(
container.get("Id")))
return "Container ID: {}".format(container.get("Id"))
