def restore( container: Optional[Tuple[str, str]], backup_file: str, force: bool ) -> None: if not container: print_and_exit( "The restore procedure requires {} running, please start your stack", SERVICE_NAME, ) docker = Docker() log.info("Starting restore on {}...", SERVICE_NAME) backup_path = f"/backup/{SERVICE_NAME}/{backup_file}" dump_file = backup_file.replace(".gz", "") dump_path = f"/tmp/{dump_file}" docker.exec_command(container, user="******", command=f"cp {backup_path} /tmp/") docker.exec_command( container, user="******", command=f"gunzip -kf /tmp/{backup_file}" ) # Executed as root docker.exec_command(container, user="******", command=f"chown postgres {dump_path}") # By using pg_dumpall the resulting dump can be restored with psql: docker.exec_command( container, user="******", command=f"psql -U sqluser -f {dump_path} postgres", ) log.info("Restore from data{} completed", backup_path)
def tuning(ram: int, cpu: int) -> None: verify_available_images( [SERVICE_NAME], Application.data.compose_config, Application.data.base_services, ) docker = Docker() container = docker.get_container(SERVICE_NAME) command = f"neo4j-admin memrec --memory {ram}" if container: docker.exec_command(container, user="******", command=command) else: docker.compose.create_volatile_container(SERVICE_NAME, command=command) # output = temporary_stream.getvalue().split("\\") # print(output) # Don't allocate more than 31g of heap, # since this will disable pointer compression, also known as "compressed oops", # in the JVM and make less effective use of the heap. # heap = min(ram * 0.4, 31 * GB) # print(f"NEO4J_HEAP_SIZE: {bytes_to_str(heap)}") # print(f"NEO4J_PAGECACHE_SIZE: {bytes_to_str(ram * 0.3)}") log.info("Use 'dbms.memory.heap.max_size' as NEO4J_HEAP_SIZE") log.info("Use 'dbms.memory.pagecache.size' as NEO4J_PAGECACHE_SIZE") log.info("Keep enough free memory for lucene indexes " "(check size reported in the output, if any)")
def scale( scaling: str = typer.Argument(..., help="scale SERVICE to NUM_REPLICA"), wait: bool = typer.Option( False, "--wait", help="Wait service convergence", show_default=False, ), ) -> None: Application.print_command( Application.serialize_parameter("--wait", wait, IF=wait), Application.serialize_parameter("", scaling), ) Application.get_controller().controller_init() options = scaling.split("=") if len(options) == 2: service, nreplicas = options else: scale_var = f"DEFAULT_SCALE_{scaling.upper()}" nreplicas = glom(Configuration.specs, f"variables.env.{scale_var}", default="1") service = scaling docker = Docker() service_name = docker.get_service(service) scales: Dict[Union[str, Service], int] = {} try: scales[service_name] = int(nreplicas) except ValueError: print_and_exit("Invalid number of replicas: {}", nreplicas) # Stop core services non compatible with scale with 2+ instances if scales[service_name] >= 2: core_services = list(Application.data.base_services.keys()) if service in core_services and service not in supported_services: print_and_exit( "Service {} is not guaranteed to support the scale, " "can't accept the request", service, ) docker.registry.ping() verify_available_images( [service], Application.data.compose_config, Application.data.base_services, ) try: docker.client.service.scale(scales, detach=not wait) # Can happens in case of scale before start except NoSuchService: print_and_exit( "No such service: {}, have you started your stack?", service_name )
def password(container: Tuple[str, str], old_password: str, new_password: str) -> None: docker = Docker() user = Application.env.get("RABBITMQ_USER") docker.exec_command( container, user=services.get_default_user(SERVICE_NAME), command=f'rabbitmqctl change_password "{user}" "{new_password}"', )
def test_reload_prod(capfd: Capture, faker: Faker) -> None: create_project( capfd=capfd, name=random_project_name(faker), auth="no", frontend="angular", ) init_project(capfd, " --prod ", "--force") start_registry(capfd) pull_images(capfd) start_project(capfd) time.sleep(5) exec_command(capfd, "reload backend", "Reloading gunicorn (PID #") exec_command( capfd, "reload", "Can't reload the frontend if not explicitly requested", "Services reloaded", ) docker = Docker() container = docker.get_container("frontend") assert container is not None docker.client.container.stop(container[0]) exec_command(capfd, "reload frontend", "Reloading frontend...") container = docker.get_container("frontend") if Configuration.swarm_mode: # frontend reload is always execute in compose mode # => the container retrieved from docker.get_container in swarm mode is None assert container is None # Let's retrieve the container name in compose mode: Configuration.swarm_mode = False docker = Docker() container = docker.get_container("frontend") # Let's restore the docker client Configuration.swarm_mode = True docker = Docker() assert container is not None docker.client.container.remove(container[0], force=True) exec_command(capfd, "reload frontend", "Reloading frontend...") exec_command( capfd, "reload frontend backend", "Can't reload frontend and other services at once", ) exec_command(capfd, "remove", "Stack removed")
def list_cmd( element_type: ElementTypes = typer.Argument( ..., help="Type of element to be listed"), ) -> None: Application.print_command(Application.serialize_parameter( "", element_type)) Application.get_controller().controller_init() table: List[List[str]] = [] if element_type == ElementTypes.env: log.info("List env variables:\n") headers = ["Key", "Value"] env = read_env() for var in sorted(env): val = env.get(var) or "" table.append([var, val]) if element_type == ElementTypes.services: log.info("List of active services:\n") headers = ["Name", "Image", "Status", "Path"] docker = Docker() services_status = docker.get_services_status(Configuration.project) for name, service in Application.data.compose_config.items(): if name in Application.data.active_services: image = service.image build = service.build status = services_status.get(name, "N/A") if build: build_path = str(build.context.relative_to(os.getcwd())) else: build_path = "" table.append([name, image, status, build_path]) if element_type == ElementTypes.submodules: log.info("List of submodules:\n") headers = ["Repo", "Branch", "Path"] for name in Application.gits: repo = Application.gits.get(name) if repo and repo.working_dir: branch = git.get_active_branch(repo) or "N/A" path = str(repo.working_dir).replace(os.getcwd(), "") # to be replacecd with removeprefix if path.startswith("/"): path = path[1:] table.append([name, branch, path]) print("") print(tabulate(table, tablefmt=TABLE_FORMAT, headers=headers))
def password(container: Tuple[str, str], old_password: str, new_password: str) -> None: docker = Docker() docker.exec_command( container, user=services.get_default_user(SERVICE_NAME), command=f"""bin/cypher-shell \" ALTER CURRENT USER SET PASSWORD FROM '{old_password}' TO '{new_password}'; \"""", )
def status( services: List[str] = typer.Argument( None, help="Services to be inspected", shell_complete=Application.autocomplete_service, ), ) -> None: Application.print_command(Application.serialize_parameter("", services)) Application.get_controller().controller_init(services) docker = Docker() docker.status(Application.data.services)
def logs( services: List[str] = typer.Argument( None, help="Services to be inspected", shell_complete=Application.autocomplete_service, ), follow: bool = typer.Option( False, "--follow", "-f", help="Follow logs", show_default=False, ), tail: int = typer.Option( "500", "--tail", "-t", help="Number of lines to show", ), ) -> None: Application.print_command( Application.serialize_parameter("--follow", follow, IF=follow), Application.serialize_parameter("--tail", tail, IF=tail), Application.serialize_parameter("", services), ) Application.get_controller().controller_init(services) services = Application.data.services docker = Docker() try: docker.compose.logs(services, follow=follow, tail=tail) except KeyboardInterrupt: # pragma: no cover log.info("Stopped by keyboard")
def scale(scaling: str = typer.Argument( ..., help="scale SERVICE to NUM_REPLICA")) -> None: Application.print_command(Application.serialize_parameter("", scaling)) Application.get_controller().controller_init() options = scaling.split("=") if len(options) != 2: scale_var = f"DEFAULT_SCALE_{scaling.upper()}" nreplicas = glom(Configuration.specs, f"variables.env.{scale_var}", default="1") service = scaling else: service, nreplicas = options if isinstance(nreplicas, str) and not nreplicas.isnumeric(): print_and_exit("Invalid number of replicas: {}", nreplicas) verify_available_images( [service], Application.data.compose_config, Application.data.base_services, ) docker = Docker() docker.compose.start_containers([service], scales={service: int(nreplicas)})
def backup( container: Optional[Tuple[str, str]], now: datetime, force: bool, dry_run: bool ) -> None: if not container: print_and_exit( "The backup procedure requires {} running, please start your stack", SERVICE_NAME, ) docker = Docker() log.info("Starting backup on {}...", SERVICE_NAME) # This double step is required because postgres user is uid 70 # It is not fixed with host uid as the other SERVICE_NAMEs tmp_backup_path = f"/tmp/{now}.sql" # Creating backup on a tmp folder as postgres user if not dry_run: log.info("Executing pg_dumpall...") docker.exec_command( container, user="******", command=f"pg_dumpall --clean -U sqluser -f {tmp_backup_path}", ) # Compress the sql with best compression ratio if not dry_run: log.info("Compressing the backup file...") docker.exec_command( container, user="******", command=f"gzip -9 {tmp_backup_path}" ) # Verify the gz integrity if not dry_run: log.info("Verifying the integrity of the backup file...") docker.exec_command( container, user="******", command=f"gzip -t {tmp_backup_path}.gz" ) # Move the backup from /tmp to /backup (as root user) backup_path = f"/backup/{SERVICE_NAME}/{now}.sql.gz" if not dry_run: docker.exec_command( container, user="******", command=f"mv {tmp_backup_path}.gz {backup_path}" ) log.info("Backup completed: data{}", backup_path)
def pull( services: List[str] = typer.Argument( None, help="Services to be pulled", shell_complete=Application.autocomplete_service, ), include_all: bool = typer.Option( False, "--all", help="Include both core and custom images", show_default=False, ), quiet: bool = typer.Option( False, "--quiet", help="Pull without printing progress information", show_default=False, ), ) -> None: Application.print_command( Application.serialize_parameter("--all", include_all, IF=include_all), Application.serialize_parameter("--quiet", quiet, IF=quiet), Application.serialize_parameter("", services), ) Application.get_controller().controller_init(services) docker = Docker() if Configuration.swarm_mode: docker.registry.ping() docker.registry.login() image: str = "" images: Set[str] = set() for service in Application.data.active_services: if Application.data.services and service not in Application.data.services: continue if base_image := glom(Application.data.base_services, f"{service}.image", default=""): images.add(base_image) image = glom(Application.data.compose_config, f"{service}.image", default="") # include custom services without a bulid to base images build = glom(Application.data.compose_config, f"{service}.build", default="") if image and (include_all or not build): images.add(image)
def password(container: Tuple[str, str], old_password: str, new_password: str) -> None: # https://dev.mysql.com/doc/refman/8.0/en/set-password.html docker = Docker() user = Application.env.get("ALCHEMY_USER") pwd = Application.env.get("MYSQL_ROOT_PASSWORD") db = Application.env.get("ALCHEMY_DB") docker.exec_command( container, user=services.get_default_user(SERVICE_NAME), command=f""" mysql -uroot -p\"{pwd}\" -D\"{db}\" -e "ALTER USER '{user}'@'%' IDENTIFIED BY '{new_password}';" """, )
def verify_available_images( services: List[str], compose_config: ComposeServices, base_services: ComposeServices, is_run_command: bool = False, ) -> None: docker = Docker() # All template builds (core only) templates = find_templates_build(base_services, include_image=True) clean_core_services = get_non_redundant_services(templates, services) for service in sorted(clean_core_services): for image, data in templates.items(): data_services = data["services"] if data["service"] != service and service not in data_services: continue if Configuration.swarm_mode and not is_run_command: image_exists = docker.registry.verify_image(image) else: image_exists = docker.client.image.exists(image) if not image_exists: if is_run_command: print_and_exit("Missing {} image, add {opt} option", image, opt=RED("--pull")) else: print_and_exit( "Missing {} image, execute {command}", image, command=RED(f"rapydo pull {service}"), ) # All builds used for the current configuration (core + custom) builds = find_templates_build(compose_config, include_image=True) clean_services = get_non_redundant_services(builds, services) for service in clean_services: for image, data in builds.items(): data_services = data["services"] if data["service"] != service and service not in data_services: continue if Configuration.swarm_mode and not is_run_command: image_exists = docker.registry.verify_image(image) else: image_exists = docker.client.image.exists(image) if not image_exists: action = "build" if data["path"] else "pull" print_and_exit( "Missing {} image, execute {command}", image, command=RED(f"rapydo {action} {service}"), )
def password(container: Tuple[str, str], old_password: str, new_password: str) -> None: docker = Docker() # Interactively: # \password username # Non interactively: # https://ubiq.co/database-blog/how-to-change-user-password-in-postgresql user = Application.env.get("ALCHEMY_USER") db = Application.env.get("ALCHEMY_DB") docker.exec_command( container, user=services.get_default_user(SERVICE_NAME), command=f""" psql -U {user} -d {db} -c \" ALTER USER {user} WITH PASSWORD \'{new_password}\'; \" """, )
def dump() -> None: Application.print_command() Application.get_controller().controller_init() docker = Docker() docker.compose.dump_config( Application.data.services, v1_compatibility=not Configuration.swarm_mode ) log.info("Config dump: {}", COMPOSE_FILE)
def stop(services: List[str] = typer.Argument( None, help="Services to be stopped", shell_complete=Application.autocomplete_service, )) -> None: Application.print_command(Application.serialize_parameter("", services)) Application.get_controller().controller_init(services) docker = Docker() docker.client.compose.stop(Application.data.services) log.info("Stack stopped")
def wait_stack_deploy(docker: Docker) -> None: MAX = 60 for i in range(0, MAX): try: if docker.get_running_services(): break log.info("Stack is still starting, waiting... [{}/{}]", i + 1, MAX) time.sleep(1) # Can happens when the stack is near to be deployed except DockerException: # pragma: no cover pass
def backup(container: Optional[Tuple[str, str]], now: datetime, force: bool, dry_run: bool) -> None: if container and not force: print_and_exit( "RabbitMQ is running and the backup will temporary stop it. " "If you want to continue add --force flag") docker = Docker() if container and not dry_run: docker.remove(SERVICE_NAME) backup_path = f"/backup/{SERVICE_NAME}/{now}.tar.gz" log.info("Starting backup on {}...", SERVICE_NAME) if not dry_run: log.info("Executing rabbitmq mnesia...") docker.compose.create_volatile_container( SERVICE_NAME, command=f"tar -zcf {backup_path} -C /var/lib/rabbitmq mnesia") # Verify the gz integrity if not dry_run: log.info("Verifying the integrity of the backup file...") docker.compose.create_volatile_container( SERVICE_NAME, command=f"gzip -t {backup_path}") log.info("Backup completed: data{}", backup_path) if container and not dry_run: docker.start(SERVICE_NAME)
def restore(container: Optional[Tuple[str, str]], backup_file: str, force: bool) -> None: if container and not force: print_and_exit( "RabbitMQ is running and the restore will temporary stop it. " "If you want to continue add --force flag") docker = Docker() if container: docker.remove(SERVICE_NAME) backup_path = f"/backup/{SERVICE_NAME}/{backup_file}" command = f"tar -xf {backup_path} -C /var/lib/rabbitmq/" log.info("Starting restore on {}...", SERVICE_NAME) docker.compose.create_volatile_container(SERVICE_NAME, command=command) log.info("Restore from data{} completed", backup_path) if container: docker.start(SERVICE_NAME)
def test_split_command() -> None: cmd = Docker.split_command(None) assert isinstance(cmd, list) assert len(cmd) == 0 cmd = Docker.split_command("") assert isinstance(cmd, list) assert len(cmd) == 0 cmd = Docker.split_command("a") assert isinstance(cmd, list) assert len(cmd) == 1 assert cmd[0] == "a" cmd = Docker.split_command("a b") assert isinstance(cmd, list) assert len(cmd) == 2 assert cmd[0] == "a" assert cmd[1] == "b" cmd = Docker.split_command("a b c") assert isinstance(cmd, list) assert len(cmd) == 3 assert cmd[0] == "a" assert cmd[1] == "b" assert cmd[2] == "c" cmd = Docker.split_command("a 'b c'") assert isinstance(cmd, list) assert len(cmd) == 2 assert cmd[0] == "a" assert cmd[1] == "b c"
def start( services: List[str] = typer.Argument( None, help="Services to be started", shell_complete=Application.autocomplete_service, ), force: bool = typer.Option( False, "--force", "-f", help="Force containers restart", show_default=False, ), ) -> None: Application.print_command(Application.serialize_parameter("", services)) Application.get_controller().controller_init(services) docker = Docker() if Configuration.swarm_mode: docker.registry.ping() verify_available_images( Application.data.services, Application.data.compose_config, Application.data.base_services, ) if Configuration.swarm_mode: docker.compose.dump_config(Application.data.services) docker.swarm.deploy() if force: for service in Application.data.services: docker.client.service.update( f"{Configuration.project}_{service}", detach=True, force=True) wait_stack_deploy(docker) else: docker.compose.start_containers(Application.data.services, force=force) log.info("Stack started")
def logs( services: List[str] = typer.Argument( None, help="Services to be inspected", shell_complete=Application.autocomplete_service, ), follow: bool = typer.Option( False, "--follow", "-f", help="Follow logs", show_default=False, ), tail: int = typer.Option( "500", "--tail", "-t", help="Number of lines to show", ), ) -> None: Application.print_command( Application.serialize_parameter("--follow", follow, IF=follow), Application.serialize_parameter("--tail", tail), Application.serialize_parameter("", services), ) Application.get_controller().controller_init(services) if follow and len(Application.data.services) > 1: print_and_exit("Follow flag is not supported on multiple services") for service in Application.data.services: if service == "frontend": timestamps = True else: timestamps = False docker = Docker() try: docker.swarm.logs(service, follow, tail, timestamps) except KeyboardInterrupt: # pragma: no cover log.info("Stopped by keyboard") print("")
def get_container_start_date(capfd: Capture, service: str, wait: bool = False) -> datetime: if Configuration.swarm_mode and wait: time.sleep(5) # This is needed to debug and wait the service rollup to complete # Status is both for debug and to delay the get_container exec_command(capfd, "status") # Optional is needed because docker.get_container returns Optional[str] container: Optional[Tuple[str, str]] = None docker = Docker() if service == REGISTRY: # a tuple to have the same type of get_container container = (REGISTRY, "") else: container = docker.get_container(service, slot=1) assert container is not None return docker.client.container.inspect(container[0]).state.started_at
def password(container: Tuple[str, str], old_password: str, new_password: str) -> None: docker = Docker() # restapi init need the env variable to be updated but can't be done after # the restart because it often fails because unable to re-connect to # services in a short time and some long sleep would be needed # => applied a workaround to be able to execute it before the restart docker = Docker() docker.exec_command( container, user=services.get_default_user(SERVICE_NAME), command=f"""/bin/bash -c ' AUTH_DEFAULT_PASSWORD=\"{new_password}\" restapi init --force-user ' """, )
def backup(container: Optional[Tuple[str, str]], now: datetime, force: bool, dry_run: bool) -> None: docker = Docker() log.info("Starting backup on {}...", SERVICE_NAME) backup_path = f"/backup/{SERVICE_NAME}/{now}.tar.gz" # If running, ask redis to synchronize the database if container: docker.exec_command( container, user="******", command="sh -c 'redis-cli --pass \"$REDIS_PASSWORD\" save'", ) command = f"tar -zcf {backup_path} -C /data dump.rdb appendonly.aof" if not dry_run: log.info("Compressing the data files...") if container: docker.exec_command(container, user="******", command=command) else: docker.compose.create_volatile_container(SERVICE_NAME, command=command) # Verify the gz integrity command = f"gzip -t {backup_path}" if not dry_run: log.info("Verifying the integrity of the backup file...") if container: docker.exec_command(container, user="******", command=command) else: docker.compose.create_volatile_container(SERVICE_NAME, command=command) log.info("Backup completed: data{}", backup_path)
def join( manager: bool = typer.Option( False, "--manager", show_default=False, help="join new node with manager role" ) ) -> None: Application.print_command( Application.serialize_parameter("--manager", manager, IF=manager), ) Application.get_controller().controller_init() docker = Docker() manager_address = "N/A" # Search for the manager address for node in docker.client.node.list(): role = node.spec.role state = node.status.state availability = node.spec.availability if ( role == "manager" and state == "ready" and availability == "active" and node.manager_status ): manager_address = node.manager_status.addr if manager: log.info("To add a manager to this swarm, run the following command:") token = docker.swarm.get_token("manager") else: log.info("To add a worker to this swarm, run the following command:") token = docker.swarm.get_token("worker") print("") print(f"docker swarm join --token {token} {manager_address}") print("")
def backup(container: Optional[Tuple[str, str]], now: datetime, force: bool, dry_run: bool) -> None: if container and not force: print_and_exit( "Neo4j is running and the backup will temporary stop it. " "If you want to continue add --force flag") docker = Docker() if container and not dry_run: docker.remove(SERVICE_NAME) backup_path = f"/backup/{SERVICE_NAME}/{now}.dump" command = f"neo4j-admin dump --to={backup_path} --database=neo4j" log.info("Starting backup on {}...", SERVICE_NAME) if not dry_run: docker.compose.create_volatile_container(SERVICE_NAME, command=command) log.info("Backup completed: data{}", backup_path) if container and not dry_run: docker.start(SERVICE_NAME)
def remove( services: List[str] = typer.Argument( None, help="Services to be removed", shell_complete=Application.autocomplete_service, ), ) -> None: Application.print_command(Application.serialize_parameter("", services)) remove_extras: List[str] = [] for extra in ( REGISTRY, "adminer", "swaggerui", ): if services and extra in services: # services is a tuple, even if defined as List[str] ... services = list(services) services.pop(services.index(extra)) remove_extras.append(extra) Application.get_controller().controller_init(services) docker = Docker() if remove_extras: for extra_service in remove_extras: if not docker.client.container.exists(extra_service): log.error("Service {} is not running", extra_service) continue docker.client.container.remove(extra_service, force=True) log.info("Service {} removed", extra_service) # Nothing more to do if not services: return all_services = Application.data.services == Application.data.active_services if all_services: docker.swarm.remove() # This is needed because docker stack remove does not support a --wait flag # To make the remove command sync and chainable with a start command engine = Application.env.get("DEPLOY_ENGINE", "swarm") network_name = f"{Configuration.project}_{engine}_default" wait_network_removal(docker, network_name) log.info("Stack removed") else: if not docker.swarm.stack_is_running(): print_and_exit( "Stack {} is not running, deploy it with {command}", Configuration.project, command=RED("rapydo start"), ) scales: Dict[Union[str, Service], int] = {} for service in Application.data.services: service_name = Docker.get_service(service) scales[service_name] = 0 docker.client.service.scale(scales, detach=False) log.info("Services removed")
def test_base(capfd: Capture, faker: Faker) -> None: execute_outside(capfd, "reload") project_name = random_project_name(faker) create_project( capfd=capfd, name=project_name, auth="no", frontend="no", services=["fail2ban"], ) init_project(capfd) exec_command(capfd, "reload", "No service reloaded") exec_command(capfd, "reload backend", "No service reloaded") exec_command(capfd, "reload invalid", "No such service: invalid") exec_command(capfd, "reload backend invalid", "No such service: invalid") start_registry(capfd) pull_images(capfd) start_project(capfd) exec_command(capfd, "reload backend", "Reloading Flask...") if Configuration.swarm_mode: service = "backend" exec_command( capfd, "start backend", "Stack started", ) exec_command( capfd, "scale backend=2 --wait", f"{project_name}_backend scaled to 2", "Service converged", ) else: service = "fail2ban" exec_command( capfd, "scale fail2ban=2", "Scaling services: fail2ban=2...", "Services scaled: fail2ban=2", ) time.sleep(4) docker = Docker() container1 = docker.get_container(service, slot=1) container2 = docker.get_container(service, slot=2) assert container1 is not None assert container2 is not None assert container1 != container2 exec_command( capfd, f"reload {service}", f"Executing command on {container1[0]}", f"Executing command on {container2[0]}", ) exec_command(capfd, "shell backend -u root 'rm /usr/local/bin/reload'") exec_command( capfd, "reload backend", "Service backend does not support the reload command" ) exec_command(capfd, "remove", "Stack removed")