def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ self.logger.info("Deploying flow run {}".format( flow_run.id) # type: ignore ) # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run=flow_run) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info("Successfully pulled image {}...".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update( extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) container = self.docker_client.create_container( image, command="prefect execute cloud-flow", environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, ) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: proc = multiprocessing.Process( target=self.stream_container_logs, kwargs={"container_id": container.get("Id")}, ) proc.start() self.processes.append(proc) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update(extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()} ) <<<<<<< HEAD labels = { "io.prefect.flow-name": flow_run.flow.name, "io.prefect.flow-id": flow_run.flow.id, "io.prefect.flow-run-id": flow_run.id, }
def start( version, skip_pull, no_upgrade, no_ui, postgres_port, hasura_port, graphql_port, ui_port, server_port, no_postgres_port, no_hasura_port, no_graphql_port, no_ui_port, no_server_port, ): """ This command spins up all infrastructure and services for the Prefect Core server \b Options: --version, -v TEXT The server image versions to use (for example, '0.10.0' or 'master') Defaults to the current installed Prefect version. --skip-pull Flag to skip pulling new images (if available) --no-upgrade, -n Flag to avoid running a database upgrade when the database spins up --no-ui, -u Flag to avoid starting the UI \b --postgres-port TEXT Port used to serve Postgres, defaults to '5432' --hasura-port TEXT Port used to serve Hasura, defaults to '3001' --graphql-port TEXT Port used to serve the GraphQL API, defaults to '4001' --ui-port TEXT Port used to serve the UI, defaults to '8080' --server-port TEXT Port used to serve the Core server, defaults to '4200' \b --no-postgres-port Disable port map of Postgres to host --no-hasura-port Disable port map of Hasura to host --no-graphql-port Disable port map of the GraphQL API to host --no-ui-port Disable port map of the UI to host --no-server-port Disable port map of the Core server to host """ docker_dir = Path(__file__).parents[0] compose_dir_path = docker_dir # Remove port mappings if specified if (no_postgres_port or no_hasura_port or no_graphql_port or no_ui_port or no_server_port or platform_is_linux()): temp_dir = tempfile.gettempdir() temp_path = os.path.join(temp_dir, "docker-compose.yml") shutil.copy2(os.path.join(docker_dir, "docker-compose.yml"), temp_path) with open(temp_path, "r") as file: y = yaml.safe_load(file) if no_postgres_port: del y["services"]["postgres"]["ports"] if no_hasura_port: del y["services"]["hasura"]["ports"] if no_graphql_port: del y["services"]["graphql"]["ports"] if no_ui_port: del y["services"]["ui"]["ports"] if no_server_port: del y["services"]["apollo"]["ports"] if platform_is_linux(): docker_internal_ip = get_docker_ip() for service in list(y["services"]): y["services"][service]["extra_hosts"] = [ "host.docker.internal:{}".format(docker_internal_ip) ] with open(temp_path, "w") as f: y = yaml.safe_dump(y, f) compose_dir_path = temp_dir # Temporary config set for port allocation with set_temporary_config({ "server.database.host_port": postgres_port, "server.hasura.host_port": hasura_port, "server.graphql.host_port": graphql_port, "server.ui.host_port": ui_port, "server.host_port": server_port, }): env = make_env() if "PREFECT_SERVER_TAG" not in env: env.update(PREFECT_SERVER_TAG=version or ("master" if len( prefect.__version__.split("+")) > 1 else prefect.__version__)) if "PREFECT_SERVER_DB_CMD" not in env: cmd = ("prefect-server database upgrade -y" if not no_upgrade else "echo 'DATABASE MIGRATIONS SKIPPED'") env.update(PREFECT_SERVER_DB_CMD=cmd) proc = None try: if not skip_pull: subprocess.check_call(["docker-compose", "pull"], cwd=compose_dir_path, env=env) cmd = ["docker-compose", "up"] if no_ui: cmd += ["--scale", "ui=0"] proc = subprocess.Popen(cmd, cwd=compose_dir_path, env=env) while True: time.sleep(0.5) except: click.secho( "Exception caught; killing services (press ctrl-C to force)", fg="white", bg="red", ) subprocess.check_output(["docker-compose", "down"], cwd=compose_dir_path, env=env) if proc: proc.kill() raise
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ self.logger.info("Deploying flow run {}".format( flow_run.id)) # type: ignore # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker run_config = self._get_run_config(flow_run, DockerRun) assert run_config is None or isinstance(run_config, DockerRun) # mypy image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run, image, run_config=run_config) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) registry = image.split("/")[0] if self.reg_allow_list and registry not in self.reg_allow_list: self.logger.error( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) raise ValueError( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) else: pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info( "Successfully pulled image {}...".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update( extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None # At the time of creation, you can only connect a container to a single network, # however you can create more connections after creation. # Connect first network in the creation step. If no network is connected here the container # is connected to the default `bridge` network. # The rest of the networks are connected after creation. if self.networks: networking_config = self.docker_client.create_networking_config({ self.networks[0]: self.docker_client.create_endpoint_config() }) # Try fallback on old, deprecated, behaviour. if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) labels = { "io.prefect.flow-name": flow_run.flow.name, "io.prefect.flow-id": flow_run.flow.id, "io.prefect.flow-run-id": flow_run.id, } container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, labels=labels, ) # Connect the rest of the networks if self.networks: for network in self.networks[1:]: self.docker_client.connect_container_to_network( container=container, net_id=network) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) if self.networks: self.logger.debug( "Adding container with ID {} to docker networks: {}.".format( container.get("Id"), self.networks)) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: self.stream_flow_logs(container.get("Id")) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))
def deploy_flow(self, flow_run: GraphQLResult) -> str: """ Deploy flow runs on your local machine as Docker containers Args: - flow_run (GraphQLResult): A GraphQLResult flow run object Returns: - str: Information about the deployment """ self.logger.info("Deploying flow run {}".format( flow_run.id)) # type: ignore # 'import docker' is expensive time-wise, we should do this just-in-time to keep # the 'import prefect' time low import docker if getattr(flow_run.flow, "run_config", None) is not None: run_config = RunConfigSchema().load(flow_run.flow.run_config) if not isinstance(run_config, DockerRun): self.logger.error( "Flow run %s has a `run_config` of type `%s`, only `DockerRun` is supported", flow_run.id, type(run_config).__name__, ) raise TypeError("Unsupported RunConfig type: %s" % type(run_config).__name__) else: run_config = None image = get_flow_image(flow_run=flow_run) env_vars = self.populate_env_vars(flow_run, run_config=run_config) if not self.no_pull and len(image.split("/")) > 1: self.logger.info("Pulling image {}...".format(image)) registry = image.split("/")[0] if self.reg_allow_list and registry not in self.reg_allow_list: self.logger.error( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) raise ValueError( "Trying to pull image from a Docker registry '{}' which" " is not in the reg_allow_list".format(registry)) else: pull_output = self.docker_client.pull(image, stream=True, decode=True) for line in pull_output: self.logger.debug(line) self.logger.info( "Successfully pulled image {}...".format(image)) # Create any named volumes (if they do not already exist) for named_volume_name in self.named_volumes: try: self.docker_client.inspect_volume(name=named_volume_name) except docker.errors.APIError: self.logger.debug( "Creating named volume {}".format(named_volume_name)) self.docker_client.create_volume( name=named_volume_name, driver="local", labels={"prefect_created": "true"}, ) # Create a container self.logger.debug("Creating Docker container {}".format(image)) host_config = {"auto_remove": True} # type: dict container_mount_paths = self.container_mount_paths if container_mount_paths: host_config.update(binds=self.host_spec) if sys.platform.startswith("linux") and self.docker_interface: docker_internal_ip = get_docker_ip() host_config.update( extra_hosts={"host.docker.internal": docker_internal_ip}) networking_config = None if self.network: networking_config = self.docker_client.create_networking_config( {self.network: self.docker_client.create_endpoint_config()}) container = self.docker_client.create_container( image, command=get_flow_run_command(flow_run), environment=env_vars, volumes=container_mount_paths, host_config=self.docker_client.create_host_config(**host_config), networking_config=networking_config, ) # Start the container self.logger.debug("Starting Docker container with ID {}".format( container.get("Id"))) if self.network: self.logger.debug("Adding container to docker network: {}".format( self.network)) self.docker_client.start(container=container.get("Id")) if self.show_flow_logs: self.stream_flow_logs(container.get("Id")) self.logger.debug("Docker container {} started".format( container.get("Id"))) return "Container ID: {}".format(container.get("Id"))