def clone(online_url,
path,
branch='master',
do=False,
check=True,
expand_path=True):
if expand_path:
local_path = os.path.join(os.curdir, SUBMODULES_DIR, path)
else:
local_path = path
local_path_exists = os.path.exists(local_path)
if local_path_exists:
log.debug("Path {} already exists", local_path)
gitobj = Repo(local_path)
elif do:
gitobj = Repo.clone_from(url=online_url, to_path=local_path)
log.info("Cloned repo {}@{} as {}", online_url, branch, path)
else:
log.exit("Repo {} missing as {}. You should init your project".format(
online_url, local_path))
if do:
switch_branch(gitobj, branch)
if check:
compare_repository(gitobj, branch, online_url=online_url, path=path)
return gitobj
def exec_command(self,
service,
user=None,
command=None,
disable_tty=False,
nofailure=False):
"""
Execute a command on a running container
"""
shell_command, shell_args = self.split_command(command)
options = {
'SERVICE': service,
'COMMAND': shell_command,
'ARGS': shell_args,
'--index': '1',
'--user': user,
'-T': disable_tty,
'--env': None,
'--workdir': None,
# '-d': False,
'--detach': False,
'--privileged': False,
}
if shell_command is not None:
log.debug("Command: {}({}+{})", service.lower(), shell_command,
shell_args)
try:
out = self.command('exec_command', options, nofailure=nofailure)
except NoSuchService:
if nofailure:
raise AttributeError("Cannot find service: {}".format(service))
else:
log.exit("Cannot find a running container called {}", service)
else:
return out
def get_current_gid() -> int:
try:
return os.getgid()
# Can fail on Windows
except AttributeError as e: # pragma: no cover
log.debug(e)
return 0
def clone(
url: str, path: Path, branch: str, do: bool = False, check: bool = True
) -> Repo:
local_path = SUBMODULES_DIR.joinpath(path)
if local_path.exists():
log.debug("Path {} already exists", local_path)
gitobj = Repo(local_path)
elif do:
gitobj = Repo.clone_from(url=url, to_path=local_path)
log.info("Cloned {}@{} as {}", url, branch, path)
else:
print_and_exit(
"Repo {} missing as {}. You should init your project",
url,
local_path,
)
if do:
ret = switch_branch(gitobj, branch)
if not ret: # pragma: no cover
print_and_exit("Cannot switch repo {} to version {}", local_path, branch)
if check:
compare_repository(gitobj, branch, online_url=url)
return gitobj
def check_installed_software() -> None:
log.debug(
"python version: {}.{}.{}",
sys.version_info.major,
sys.version_info.minor,
sys.version_info.micro,
)
# 17.05 added support for multi-stage builds
# https://docs.docker.com/compose/compose-file/compose-file-v3/#compose-and-docker-compatibility-matrix
# 18.09.2 fixed the CVE-2019-5736 vulnerability
# 20.10.0 introduced copy --chmod and improved logging
Packages.check_program("docker",
min_version="20.10.0",
min_recommended_version="20.10.0")
if docker.compose.is_installed():
# too slow to verify the version on every commands... near half a seconds
# Sometimes a couple of seconds!
# v = docker.compose.version()
# log.debug("docker compose is installed: {}", v)
log.debug("docker compose is installed")
else: # pragma: no cover
print_and_exit(
"A mandatory dependency is missing: docker compose not found"
"\nInstallation guide: "
"https://docs.docker.com/compose/cli-command/#installing-compose-v2"
"\nor try the automated installation with {command}",
command=RED("rapydo install compose"),
)
def get_username(uid: int) -> str:
try:
import pwd
return pwd.getpwuid(uid).pw_name
# Can fail on Windows
except ImportError as e: # pragma: no cover
log.debug(e)
return str(uid)
def git_checks(ignore_submodule: List[str]) -> None:
for name, gitobj in Application.gits.items():
if name in ignore_submodule:
log.debug("Skipping checks on {}", name)
continue
if gitobj:
git.check_updates(name, gitobj)
git.check_unstaged(name, gitobj)
def check_program(
program: str,
min_version: Optional[str] = None,
max_version: Optional[str] = None,
min_recommended_version: Optional[str] = None,
) -> str:
"""
Verify if a binary exists and (optionally) its version
"""
found_version = Packages.get_bin_version(program)
if found_version is None:
hints = ""
if program == "docker": # pragma: no cover
install_cmd = RED("rapydo install docker")
hints = "\n\nTo install docker visit: https://get.docker.com"
hints += f"or execute {install_cmd}"
print_and_exit("A mandatory dependency is missing: {} not found{}",
program, hints)
v = Version(found_version)
if min_version is not None:
if Version(min_version) > v:
print_and_exit(
"Minimum supported version for {} is {}, found {}",
program,
min_version,
found_version,
)
if min_recommended_version is not None:
if Version(min_recommended_version) > v:
log.warning(
"Minimum recommended version for {} is {}, found {}",
program,
min_recommended_version,
found_version,
)
if max_version is not None:
if Version(max_version) < v:
print_and_exit(
"Maximum supported version for {} is {}, found {}",
program,
max_version,
found_version,
)
log.debug("{} version: {}", program, found_version)
return found_version
def command(self, command, options=None, nofailure=False):
# NOTE: debug defaults
# tmp = self.get_defaults(command)
# print("TEST", tmp, type(tmp))
# # exit(1)
compose_handler = self.get_handle()
method = getattr(compose_handler, command)
if options is None:
options = {}
if options.get('SERVICE', None) is None:
options['SERVICE'] = []
log.debug("{}'{}'", compose_log, command)
out = None
# sometimes this import stucks... importing here to avoid unnecessary waits
from docker.errors import APIError
try:
out = method(options=options)
except SystemExit as e:
# NOTE: we check the status here.
# System exit is received also when a normal command finished.
if e.code < 0:
log.warning("Invalid code returned: {}", e.code)
elif e.code > 0:
log.exit("Compose received: system.exit({})",
e.code,
error_code=e.code)
else:
log.verbose("Executed compose {} w/{}", command, options)
except (clierrors.UserError, cerrors.OperationFailedError,
BuildError) as e:
msg = "Failed command execution:\n{}".format(e)
if nofailure:
raise AttributeError(msg)
else:
log.exit(msg)
except APIError as e:
log.exit("Failed docker container:\n{}", e)
except (ProjectError, NoSuchService) as e:
log.exit(str(e))
else:
log.verbose("Executed compose {} w/{}", command, options)
return out
def git_update(ignore_submodule: List[str]) -> None:
for name, gitobj in Application.gits.items():
if name in ignore_submodule:
log.debug("Skipping update on {}", name)
continue
if gitobj and not git.can_be_updated(name, gitobj):
print_and_exit("Can't continue with updates")
controller_is_updated = False
for name, gitobj in Application.gits.items():
if name in ignore_submodule:
continue
if name == "do":
controller_is_updated = True
if gitobj:
git.update(name, gitobj)
if controller_is_updated:
installation_path = Packages.get_installation_path("rapydo")
# Can't be tested on GA since rapydo is alway installed from a folder
if not installation_path: # pragma: no cover
log.warning("Controller is not installed in editable mode, "
"rapydo is unable to update it")
elif Application.gits["do"].working_dir:
do_dir = Path(Application.gits["do"].working_dir)
if do_dir.is_symlink():
do_dir = do_dir.resolve()
# This can be used starting from py39
# do_dir = do_dir.readlink()
if do_dir == installation_path:
log.info("Controller installed from {} and updated",
installation_path)
else:
log.warning(
"Controller not updated because it is installed outside this "
"project. Installation path is {}, the current folder is {}",
installation_path,
do_dir,
)
else: # pragma: no cover
log.warning("Controller submodule folder can't be found")
def __init__(self) -> None:
self.template_dir = Path(__file__).resolve().parent.joinpath(TEMPLATE_DIR)
if not self.template_dir.is_dir():
print_and_exit("Template folder not found: {}", self.template_dir)
log.debug("Template folder: {}", self.template_dir)
loader = FileSystemLoader([TEMPLATE_DIR, self.template_dir])
self.env = Environment(
loader=loader,
undefined=DebugUndefined,
autoescape=True,
keep_trailing_newline=True,
)
self.env.filters["password"] = password
self.env.filters["username"] = username
def print_command(*parameters: Optional[str]) -> None:
pre_params = " ".join(
[p for p in Configuration.parameters if p is not None]).strip()
post_params = " ".join([p for p in parameters
if p is not None]).strip()
if pre_params:
pre_params = f"{pre_params} "
if post_params:
post_params = f" {post_params}"
log.debug(
"Command: rapydo {}{}{}",
pre_params,
Configuration.action,
post_params,
log_to_file=True,
)
def find_templates_override(services: ComposeServices,
templates: BuildInfo) -> Dict[str, str]:
builds: Dict[str, str] = {}
for service in services.values():
if service.build is not None and service.image not in templates:
baseimage = get_dockerfile_base_image(service.build.context,
templates)
if not baseimage.startswith("rapydo/"):
continue
vanilla_img = service.image
template_img = baseimage
log.debug("{} extends {}", vanilla_img, template_img)
builds[vanilla_img] = template_img
return builds
def find_active(services: ComposeServices) -> List[str]:
"""
Check only services involved in current mode,
which is equal to services 'activated' + 'depends_on'.
"""
dependencies: Dict[str, List[str]] = {}
base_actives: List[str] = []
for name, service in services.items():
dependencies[name] = list(service.depends_on.keys())
if service.environment and service.environment.get("ACTIVATE",
"0") == "1":
base_actives.append(name)
log.debug("Base active services: {}", ", ".join(base_actives))
# log.debug("Services dependencies: {}", ", ".join(dependencies))
active_services = walk_services(base_actives, dependencies)
return active_services
def create_projectrc(self) -> None:
templating = Templating()
t = templating.get_template(
"projectrc",
{
"project": Configuration.project,
"hostname": Configuration.hostname,
"swarm": Configuration.swarm_mode,
"production": Configuration.production,
"testing": Configuration.testing,
"services": self.active_services,
"env_variables": Configuration.environment,
},
)
templating.save_template(PROJECTRC, t, force=True)
Application.load_projectrc()
if not self.files:
log.debug("Created temporary default {} file", PROJECTRC)
PROJECTRC.unlink()
else:
log.info("Created default {} file", PROJECTRC)
def check_file_younger_than(
gitobj: Repo, filename: Path, timestamp: Union[str, float]
) -> Tuple[bool, float, datetime]:
try:
commits = gitobj.blame(rev="HEAD", file=str(filename))
except GitCommandError:
log.debug("Can't retrieve a commit history for {}", filename)
return False, 0, datetime.fromtimestamp(0)
# added a default date to prevent errors in case of new files with no blame commits
dates = [datetime.fromtimestamp(0).replace(tzinfo=pytz.utc)]
if commits:
for commit in commits:
current_blame = gitobj.commit(rev=str(commit[0]))
dates.append(current_blame.committed_datetime)
max_date = max(dates)
return (
bool(timestamp_from_string(timestamp) < max_date),
float(timestamp),
max_date,
)
def create_project(
project_name: str,
auth: str,
frontend: str,
services: List[str],
extend: Optional[str],
envs: Optional[List[str]] = None,
auto: bool = False,
force: bool = False,
force_current: bool = False,
add_optionals: bool = False,
path: Path = None,
) -> None:
project_scaffold = Project()
enable_postgres = auth == "postgres" or "postgres" in services
enable_mysql = auth == "mysql" or "mysql" in services
enable_neo4j = auth == "neo4j" or "neo4j" in services
enable_rabbit = "rabbit" in services
enable_redis = "redis" in services
enable_celery = "celery" in services
enable_flower = "flower" in services
enable_fail2ban = "fail2ban" in services
enable_ftp = "ftp" in services
enable_bot = "bot" in services
if auth == "postgres" or auth == "mysql":
auth = "sqlalchemy"
if auth == "no":
auth = NO_AUTHENTICATION
if frontend == "no":
frontend = NO_FRONTEND
if not force_current:
dirs = os.listdir(".")
if dirs and dirs != [".git"]:
print_and_exit(
"Current folder is not empty, cannot create a new project here.\n"
"Found: {}\n"
"Use --current to force the creation here",
", ".join(dirs[0:3]), # add first 3 files/folders found
)
celery_broker = None # Keep default value == REDIS
celery_backend = None # Keep default value == REDIS
if enable_celery:
if enable_rabbit:
celery_broker = "RABBIT"
else:
celery_broker = "REDIS"
enable_redis = True
if enable_redis:
celery_backend = "REDIS"
else:
celery_backend = "RABBIT"
env_variables = parse_env_variables(envs)
project_scaffold.load_project_scaffold(project_name, auth, services)
if frontend != NO_FRONTEND:
project_scaffold.load_frontend_scaffold(frontend)
# In case of errors this function will exit
project_scaffold.check_invalid_characters(project_name)
if project_name in project_scaffold.reserved_project_names:
print_and_exit(
"You selected a reserved name, invalid project name: {}",
project_name)
templating = Templating()
folders = project_scaffold.expected_folders + project_scaffold.data_folders
if add_optionals:
folders += project_scaffold.optionals_folders
for f in folders:
if f.exists():
log.debug("Project folder already exists: {}", f)
continue
if not auto:
print_and_exit("\nmkdir -p {}", f)
f.mkdir(parents=True, exist_ok=True)
for f in project_scaffold.suggested_gitkeep:
f.open("a").close()
files = project_scaffold.expected_files
if add_optionals:
files += project_scaffold.optionals_files
if path:
if path not in files:
print_and_exit("Invalid path, cannot upgrade {}", path)
else:
files = [path]
for p in files:
template = templating.get_template(
p.name,
{
"version": __version__,
"project": project_name,
"auth_service": auth,
"enable_postgres": enable_postgres,
"enable_mysql": enable_mysql,
"enable_neo4j": enable_neo4j,
"enable_rabbit": enable_rabbit,
"enable_redis": enable_redis,
"enable_celery": enable_celery,
"enable_flower": enable_flower,
"enable_fail2ban": enable_fail2ban,
"enable_ftp": enable_ftp,
"enable_bot": enable_bot,
"celery_broker": celery_broker,
"celery_backend": celery_backend,
"frontend": frontend,
"testing": Configuration.testing,
"extend": extend,
"services": services,
"env_variables": env_variables,
},
)
# automatic creation
if auto:
if p.exists() and not force:
log.info("Project file already exists: {}", p)
else:
templating.save_template(p, template, force=force)
continue
# manual creation
if p.exists():
log.info("Project file already exists: {}", p)
else:
print(f"\n{template}")
print_and_exit(str(p))
if not path:
for p in project_scaffold.raw_files:
# automatic creation
if auto:
if p.exists() and not force:
log.info("Project file already exists: {}", p)
else:
shutil.copyfile(templating.template_dir.joinpath(p.name),
p)
continue
# manual creation
if p.exists():
log.info("Project file already exists: {}", p)
else:
# print(f"Missing file: {p}")
print_and_exit("File is missing: {}", p)
def check(
no_git: bool = typer.Option(
False,
"--no-git",
"-s",
help="Skip checks on git commits",
show_default=False,
),
no_builds: bool = typer.Option(
False,
"--no-builds",
help="Skip check on docker builds",
show_default=False,
),
ignore_submodules: List[str] = typer.Option(
[],
"--ignore-submodule",
"-i",
help="Ignore submodule",
show_default=False,
shell_complete=Application.autocomplete_submodule,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--no-git", no_git, IF=no_git),
Application.serialize_parameter("--no-builds", no_builds,
IF=no_builds),
Application.serialize_parameter("--ignore-submodule",
ignore_submodules),
)
Application.get_controller().controller_init()
docker = Docker()
if Configuration.swarm_mode:
log.debug("Swarm is correctly initialized")
docker.swarm.check_resources()
if no_git:
log.info("Skipping git checks")
else:
log.info("Checking git (skip with --no-git)")
Application.git_checks(ignore_submodules)
if no_builds:
log.info("Skipping builds checks")
else:
log.info("Checking builds (skip with --no-builds)")
dimages: List[str] = []
for img in docker.client.images():
if img.repo_tags:
for i in img.repo_tags:
dimages.append(i)
all_builds = find_templates_build(Application.data.compose_config)
core_builds = find_templates_build(Application.data.base_services)
overriding_builds = find_templates_override(
Application.data.compose_config, core_builds)
for image_tag, build in all_builds.items():
services = build["services"]
if not any(x in Application.data.active_services
for x in services):
continue
if image_tag not in dimages:
if image_tag in core_builds:
log.warning(
"Missing {} image, execute {command}",
image_tag,
command=RED("rapydo pull"),
)
else:
log.warning(
"Missing {} image, execute {command}",
image_tag,
command=RED("rapydo build"),
)
continue
image_creation = get_image_creation(image_tag)
# Check if some recent commit modified the Dockerfile
d1, d2 = build_is_obsolete(image_creation, build.get("path"))
if d1 and d2:
tmp_from_image = overriding_builds.get(image_tag)
# This is the case of a build not overriding a core image,
# e.g nifi or geoserver. In that case from_image is faked to image_tag
# just to make print_obsolete to print 'build' instead of 'pull'
if not tmp_from_image and image_tag not in core_builds:
tmp_from_image = image_tag
print_obsolete(image_tag, d1, d2, build.get("service"),
tmp_from_image)
# if FROM image is newer, this build should be re-built
elif image_tag in overriding_builds:
from_img = overriding_builds.get(image_tag, "")
from_build: Optional[TemplateInfo] = core_builds.get(from_img)
if not from_build: # pragma: no cover
log.critical("Malformed {} image, from build is missing",
image_tag)
continue
# Verify if template build exists
if from_img not in dimages: # pragma: no cover
log.warning(
"Missing template build for {} ({})\n{}",
from_build.get("services"),
from_img,
)
from_timestamp = get_image_creation(from_img)
# Verify if template build is obsolete or not
d1, d2 = build_is_obsolete(from_timestamp,
from_build.get("path"))
if d1 and d2: # pragma: no cover
print_obsolete(from_img, d1, d2, from_build.get("service"))
if from_timestamp > image_creation:
b = image_creation.strftime(DATE_FORMAT)
c = from_timestamp.strftime(DATE_FORMAT)
print_obsolete(image_tag, b, c, build.get("service"),
from_img)
templating = Templating()
for filename in Application.project_scaffold.fixed_files:
if templating.file_changed(str(filename)):
log.warning(
"{} changed, please execute {command}",
filename,
command=RED(f"rapydo upgrade --path {filename}"),
)
compose_version = "Unknown"
buildx_version = "Unknown"
m = re.search(
r"^Docker Compose version (v[0-9]+\.[0-9]+\.[0-9]+)$",
docker.client.compose.version(),
)
if m:
compose_version = m.group(1)
m = re.search(
r"^github.com/docker/buildx (v[0-9]+\.[0-9]+\.[0-9]+) .*$",
docker.client.buildx.version(),
)
if m:
buildx_version = m.group(1)
if compose_version == COMPOSE_VERSION:
log.info("Compose is installed with version {}", COMPOSE_VERSION)
else: # pragma: no cover
cmd = RED("rapydo install compose")
fix_hint = f"You can update it with {cmd}"
log.warning(
"Compose is installed with version {}, expected version is {}.\n{}",
compose_version,
COMPOSE_VERSION,
fix_hint,
)
if buildx_version == BUILDX_VERSION:
log.info("Buildx is installed with version {}", BUILDX_VERSION)
else: # pragma: no cover
cmd = RED("rapydo install buildx")
fix_hint = f"You can update it with {cmd}"
log.warning(
"Buildx is installed with version {}, expected version is {}.\n{}",
buildx_version,
BUILDX_VERSION,
fix_hint,
)
for expired_passwords in get_expired_passwords():
log.warning(
"{} is expired on {}",
expired_passwords[0],
expired_passwords[1].strftime("%Y-%m-%d"),
)
log.info("Checks completed")
def controller_init(self,
services: Optional[Iterable[str]] = None) -> None:
if Configuration.create:
Application.check_installed_software()
return None
main_folder_error = Application.project_scaffold.check_main_folder()
if main_folder_error:
print_and_exit(main_folder_error)
if not Configuration.print_version:
Application.check_installed_software()
# if project is None, it is retrieve by project folder
Configuration.project = Application.project_scaffold.get_project(
Configuration.project)
Configuration.ABS_PROJECT_PATH = PROJECT_DIR.joinpath(
Configuration.project)
if Configuration.print_version:
self.read_specs(read_extended=True)
return None
log.debug("You are using RAPyDo version {}", __version__)
if Configuration.check:
log.info("Selected project: {}", Configuration.project)
else:
log.debug("Selected project: {}", Configuration.project)
if (Configuration.initialize or Configuration.update
or Configuration.check or Configuration.install):
Application.check_internet_connection()
if Configuration.install:
self.read_specs(read_extended=False)
return None
# Auth is not available yet, will be read by read_specs
Application.project_scaffold.load_project_scaffold(
Configuration.project, auth=None)
Application.preliminary_version_check()
# read project configuration
self.read_specs(read_extended=True)
# from read_specs
Application.project_scaffold.load_frontend_scaffold(
Configuration.frontend)
Application.verify_rapydo_version()
Application.project_scaffold.inspect_project_folder()
self.current_uid = system.get_current_uid()
self.current_gid = system.get_current_gid()
# Cannot be tested
if self.current_uid == ROOT_UID: # pragma: no cover
self.current_uid = BASE_UID
log.warning("Current user is 'root'")
else:
os_user = system.get_username(self.current_uid)
log.debug("Current UID: {} ({})", self.current_uid, os_user)
log.debug("Current GID: {}", self.current_gid)
if Configuration.initialize:
return None
Application.git_submodules()
if Configuration.update:
return None
self.make_env()
# Compose services and variables
base_services, compose_config = self.get_compose_configuration(
services)
if Configuration.action != "password":
self.check_placeholders_and_passwords(compose_config,
self.enabled_services)
Application.data = CommandsData(
files=self.files,
base_files=self.base_files,
services=self.enabled_services,
active_services=self.active_services,
base_services=base_services,
compose_config=compose_config,
)
return None
def backup(
service: SupportedServices = typer.Argument(..., help="Service name"),
force: bool = typer.Option(
False,
"--force",
help="Force the backup procedure",
show_default=False,
),
max_backups: int = typer.Option(
0,
"--max",
help=
"Maximum number of backups, older exceeding this number will be removed",
show_default=False,
),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Do not perform any backup or delete backup files",
show_default=False,
),
restart: List[str] = typer.Option(
[],
"--restart",
help=
"Service to be restarted once completed the backup (multiple allowed)",
shell_complete=Application.autocomplete_service,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--force", force, IF=force),
Application.serialize_parameter("--max", max_backups, IF=max_backups),
Application.serialize_parameter("--dry-run", dry_run, IF=dry_run),
Application.serialize_parameter("--restart", restart, IF=restart),
Application.serialize_parameter("", service.value),
)
if dry_run:
log.warning("Dry run mode is enabled")
Application.get_controller().controller_init()
service_name = service.value
verify_available_images(
[service_name],
Application.data.compose_config,
Application.data.base_services,
)
docker = Docker()
container = docker.get_container(service_name)
backup_dir = BACKUP_DIR.joinpath(service_name)
backup_dir.mkdir(parents=True, exist_ok=True)
if max_backups > 0:
backups = list(backup_dir.glob(get_date_pattern()))
if max_backups >= len(backups):
log.debug("Found {} backup files, maximum not reached",
len(backups))
else:
for f in sorted(backups)[:-max_backups]:
if not dry_run:
f.unlink()
log.warning(
"{} deleted because exceeding the max number of backup files ({})",
f.name,
max_backups,
)
module = BACKUP_MODULES.get(service.value)
if not module: # pragma: no cover
print_and_exit(f"{service.value} misconfiguration, module not found")
now = datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
module.backup(container=container, now=now, force=force, dry_run=dry_run)
if restart and not dry_run:
log.info("Restarting services in 20 seconds...")
time.sleep(10)
log.info("Restarting services in 10 seconds...")
time.sleep(10)
reload(docker, restart)
def run(
service: str = typer.Argument(
...,
help="Service name",
shell_complete=Application.autocomplete_allservice,
),
pull: bool = typer.Option(
False,
"--pull",
help="Pull the image before starting the container",
show_default=False,
),
debug: bool = typer.Option(
False,
"--debug",
help="Start the container in debug mode",
show_default=False,
),
command: str = typer.Option(
None,
"--command",
help="UNIX command to be executed in the container",
show_default=False,
),
user: str = typer.Option(
None,
"--user",
"-u",
help="User existing in selected service",
show_default=False,
),
first_port: Optional[int] = typer.Option(
None,
"--port",
"-p",
help="port to be associated to the current service interface",
),
detach: Optional[bool] = typer.Option(
None,
"--detach",
help="Start the container in detach mode (default for non-interfaces)",
show_default=False,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--pull", pull, IF=pull),
Application.serialize_parameter("--debug", debug, IF=debug),
Application.serialize_parameter("--command", command, IF=command),
Application.serialize_parameter("--user", user, IF=user),
Application.serialize_parameter("--port", first_port, IF=first_port),
Application.serialize_parameter("", service),
)
Configuration.FORCE_COMPOSE_ENGINE = True
Application.get_controller().controller_init()
Application.get_controller().check_placeholders_and_passwords(
Application.data.compose_config, [service])
if service == REGISTRY and not Configuration.swarm_mode:
print_and_exit("Can't start the registry in compose mode")
docker = Docker()
if Configuration.swarm_mode:
if service != REGISTRY:
docker.registry.ping()
else:
if docker.registry.ping(do_exit=False):
registry = docker.registry.get_host()
print_and_exit("The registry is already running at {}",
registry)
if docker.client.container.exists("registry"):
log.debug(
"The registry container is already existing, removing")
docker.client.container.remove("registry", force=True)
if not debug:
if user:
print_and_exit("Can't specify a user if debug mode is OFF")
if command:
print_and_exit("Can't specify a command if debug mode is OFF")
if user:
log.warning(
"Please remember that users in volatile containers are not mapped on"
" current uid and gid. You should not write or modify files on volumes"
" to prevent permissions errors")
if pull:
log.info("Pulling image for {}...", service)
docker.client.compose.pull([service])
else:
verify_available_images(
[service],
Application.data.compose_config,
Application.data.base_services,
is_run_command=True,
)
# This is equivalent to the old volatile command
if debug:
if not command:
command = "bash"
log.info("Starting {}...", service)
docker.compose.create_volatile_container(
service,
command=command,
user=user,
# if None the wrapper will automatically switch the default ones
# How to prevent ports on volatile containers?
# publish=None,
)
log.info("Service {} removed", service)
return None
# This is equivalent to the old registry command
if service == REGISTRY:
# @ symbol in secrets is not working
# https://github.com/bitnami/charts/issues/1954
# Other symbols like # and " also lead to configuration errors
os.environ["REGISTRY_HTTP_SECRET"] = password(
param_not_used="", length=96
# , symbols="%*,-.=?[]^_~"
)
publish_ports = get_publish_ports(service, first_port)
if detach is None:
if service == "swaggerui" or service == "adminer":
detach = False
else:
detach = True
log.info("Running {}...", service)
if service == "swaggerui":
if Configuration.production:
prot = "https"
else:
prot = "http"
port = publish_ports[0][0] if publish_ports else first_port
log.info(
"You can access SwaggerUI web page here: {}\n",
f"{prot}://{Configuration.hostname}:{port}",
)
if service == "adminer":
if Configuration.production:
prot = "https"
else:
prot = "http"
port = publish_ports[0][0] if publish_ports else first_port
log.info(
"You can access Adminer interface on: {}\n",
f"{prot}://{Configuration.hostname}:{port}",
)
docker.compose.create_volatile_container(service,
detach=detach,
publish=publish_ports)
def get_compose_configuration(
self,
enabled_services: Optional[Iterable[str]] = None
) -> Tuple[ComposeServices, ComposeServices]:
compose_files: List[Path] = []
MODE = f"{Configuration.stack}.yml"
customconf = Configuration.ABS_PROJECT_PATH.joinpath(
CONTAINERS_YAML_DIRNAME)
angular_loaded = False
def add(p: Path, f: str) -> None:
compose_files.append(p.joinpath(f))
if Configuration.load_backend:
add(CONFS_DIR, "backend.yml")
if Configuration.load_frontend:
if Configuration.frontend == ANGULAR:
add(CONFS_DIR, "angular.yml")
angular_loaded = True
if (Configuration.swarm_mode and Configuration.production
and not Configuration.FORCE_COMPOSE_ENGINE):
add(CONFS_DIR, "swarm_angular_prod_options.yml")
if Configuration.swarm_mode and not Configuration.FORCE_COMPOSE_ENGINE:
add(CONFS_DIR, "swarm_options.yml")
if Application.env.get("NFS_HOST"):
log.info("NFS Server is enabled")
add(CONFS_DIR, "volumes_nfs.yml")
else:
add(CONFS_DIR, "volumes_local.yml")
if Configuration.production:
add(CONFS_DIR, "production.yml")
else:
add(CONFS_DIR, "development.yml")
if angular_loaded:
add(CONFS_DIR, "angular-development.yml")
if self.extended_project and self.extended_project_path:
extendedconf = self.extended_project_path.joinpath(
CONTAINERS_YAML_DIRNAME)
# Only added if exists, this is the only non mandatory conf file
extended_mode_conf = extendedconf.joinpath(MODE)
if extended_mode_conf.exists():
compose_files.append(extended_mode_conf)
if Configuration.load_commons:
add(extendedconf, "commons.yml")
if Configuration.load_commons:
add(customconf, "commons.yml")
add(customconf, MODE)
# Read necessary files
self.files, self.base_files = configuration.read_composer_yamls(
compose_files)
# to build the config with files and variables
from controller.deploy.docker import Docker
docker = Docker(compose_files=self.base_files,
verify_swarm=not Configuration.initialize)
base_services = docker.compose.get_config().services
docker = Docker(compose_files=self.files,
verify_swarm=not Configuration.initialize)
compose_config = docker.compose.get_config().services
self.active_services = services.find_active(compose_config)
self.enabled_services = services.get_services(
Configuration.services_list or enabled_services,
default=self.active_services,
)
for service in self.enabled_services:
if service not in self.active_services:
print_and_exit("No such service: {}", service)
log.debug("Enabled services: {}", ", ".join(self.enabled_services))
self.create_datafile(list(compose_config.keys()), self.active_services)
return base_services, compose_config
def build(
services: List[str] = typer.Argument(
None,
help="Services to be built",
shell_complete=Application.autocomplete_service,
),
core: bool = typer.Option(
False,
"--core",
help="Include core images to the build list",
show_default=False,
),
force: bool = typer.Option(
False,
"--force",
"-f",
help="remove the cache to force the build",
show_default=False,
),
) -> bool:
Application.print_command(
Application.serialize_parameter("--core", core, IF=core),
Application.serialize_parameter("--force", force, IF=force),
Application.serialize_parameter("", services),
)
Application.get_controller().controller_init(services)
docker = Docker()
if docker.client.buildx.is_installed():
v = docker.client.buildx.version()
log.debug("docker buildx is installed: {}", v)
else: # pragma: no cover
print_and_exit(
"A mandatory dependency is missing: docker buildx not found"
"\nInstallation guide: https://github.com/docker/buildx#binary-release"
"\nor try the automated installation with {command}",
command=RED("rapydo install buildx"),
)
if Configuration.swarm_mode:
docker.registry.ping()
docker.registry.login()
images: Set[str] = set()
if core:
log.debug("Forcing rebuild of core builds")
# Create merged compose file with core files only
docker = Docker(compose_files=Application.data.base_files)
docker.compose.dump_config(Application.data.services,
set_registry=False)
log.debug("Compose configuration dumped on {}", COMPOSE_FILE)
docker.client.buildx.bake(
targets=Application.data.services,
files=[COMPOSE_FILE],
pull=True,
load=True,
cache=not force,
)
log.info("Core images built")
if Configuration.swarm_mode:
log.warning(
"Local registry push is not implemented yet for core images")
docker = Docker()
docker.compose.dump_config(Application.data.services, set_registry=False)
log.debug("Compose configuration dumped on {}", COMPOSE_FILE)
core_builds = find_templates_build(Application.data.base_services)
all_builds = find_templates_build(Application.data.compose_config)
services_with_custom_builds: List[str] = []
for image, build in all_builds.items():
if image not in core_builds:
# this is used to validate the target Dockerfile:
if p := build.get("path"):
get_dockerfile_base_image(p, core_builds)
services_with_custom_builds.extend(build["services"])
images.add(image)
def load_frontend_scaffold(self, frontend: Optional[str]) -> bool:
self.frontend = frontend
if self.frontend is None or self.frontend == NO_FRONTEND:
log.debug("No frontend framework enabled")
return False
self.expected_folders.append(self.p_path("frontend"))
if self.frontend == ANGULAR:
self.expected_folders.extend(
[
self.p_path("frontend", "app"),
self.p_path("frontend", "styles"),
self.p_path("frontend", "integration"),
self.p_path("frontend", "assets"),
self.p_path("frontend", "assets", "favicon"),
]
)
self.suggested_gitkeep.append(
DATA_DIR.joinpath(self.project, "frontend", GITKEEP)
)
self.suggested_gitkeep.append(
self.p_path("frontend", "integration", GITKEEP)
)
self.expected_files.extend(
[
self.p_path("frontend", "package.json"),
self.p_path("frontend", "styles", "style.scss"),
self.p_path("frontend", "styles", "variables.scss"),
self.p_path("frontend", "app", "customization.ts"),
self.p_path("frontend", "app", "custom.module.ts"),
self.p_path("frontend", "app", "custom.navbar.ts"),
self.p_path("frontend", "app", "custom.footer.ts"),
self.p_path("frontend", "app", "custom.profile.ts"),
self.p_path("frontend", "app", "custom.navbar.links.html"),
self.p_path("frontend", "app", "custom.navbar.brand.html"),
self.p_path("frontend", "app", "custom.footer.html"),
self.p_path("frontend", "app", "custom.profile.html"),
self.p_path("frontend", "app", "types.ts"),
]
)
self.raw_files.extend(
[
# Generated with https://realfavicongenerator.net
self.p_path(
"frontend", "assets", "favicon", "android-chrome-192x192.png"
),
self.p_path("frontend", "assets", "favicon", "browserconfig.xml"),
self.p_path("frontend", "assets", "favicon", "favicon-32x32.png"),
self.p_path("frontend", "assets", "favicon", "mstile-150x150.png"),
self.p_path(
"frontend", "assets", "favicon", "safari-pinned-tab.svg"
),
self.p_path(
"frontend", "assets", "favicon", "apple-touch-icon.png"
),
self.p_path("frontend", "assets", "favicon", "favicon-16x16.png"),
self.p_path("frontend", "assets", "favicon", "favicon.ico"),
self.p_path("frontend", "assets", "favicon", "site.webmanifest"),
]
)
frontend_data_dir = DATA_DIR.joinpath(self.project, "frontend")
self.data_folders.extend(
[
frontend_data_dir,
frontend_data_dir.joinpath("app"),
frontend_data_dir.joinpath("node_modules"),
DATA_DIR.joinpath(self.project, "karma"),
DATA_DIR.joinpath(self.project, "cypress"),
]
)
self.data_files.extend(
[
frontend_data_dir.joinpath("angular.json"),
frontend_data_dir.joinpath("karma.conf.js"),
frontend_data_dir.joinpath("package.json"),
frontend_data_dir.joinpath("polyfills.ts"),
frontend_data_dir.joinpath("tsconfig.json"),
frontend_data_dir.joinpath("tsconfig.app.json"),
frontend_data_dir.joinpath("tsconfig.spec.json"),
frontend_data_dir.joinpath("tsconfig.server.json"),
frontend_data_dir.joinpath("cypress.json"),
]
)
self.obsolete_files.extend(
[
self.p_path("frontend", "app", "app.routes.ts"),
self.p_path("frontend", "app", "app.declarations.ts"),
self.p_path("frontend", "app", "app.providers.ts"),
self.p_path("frontend", "app", "app.imports.ts"),
self.p_path("frontend", "app", "app.custom.navbar.ts"),
self.p_path("frontend", "app", "app.custom.navbar.html"),
self.p_path("frontend", "app", "app.entryComponents.ts"),
self.p_path("frontend", "app", "app.home.ts"),
self.p_path("frontend", "app", "app.home.html"),
self.p_path("frontend", "app", "custom.declarations.ts"),
self.p_path("frontend", "app", "custom.routes.ts"),
self.p_path("frontend", "app", "custom.project.options.ts"),
# Removed since 1.0
frontend_data_dir.joinpath("browserslist"),
# Removed since 1.2 (replaced with scss in styles)
self.p_path("frontend", "css"),
]
)
return True
def dump_config(
self,
services: List[str],
set_registry: bool = True,
v1_compatibility: bool = False,
) -> None:
compose_config = self.get_config_json()
clean_config: Dict[str, Any] = {
"version": compose_config.get("version", COMPOSE_FILE_VERSION),
"networks": {},
"volumes": {},
"services": {},
}
networks = set()
volumes = set()
binds: Set[Path] = set()
registry = self.docker_wrapper.registry.get_host()
# Remove unused services, networks and volumes from compose configuration
for key, value in compose_config.get("services", {}).items():
if key not in services:
continue
if Configuration.swarm_mode and set_registry and key != REGISTRY:
value["image"] = f"{registry}/{value['image']}"
if "healthcheck" in value and "test" in value["healthcheck"]:
# healtcheck commands can contain env variables double-escaped ($$)
# When dumped to docker-compose.yml the double escape is removed
# and when started the single escaped variable is not resolved
# and breaks the command. Let's double all the $ to restore the
# expected behavior and counteract the consumed $
value["healthcheck"]["test"] = [
t.replace("$", "$$") for t in value["healthcheck"]["test"]
]
for k, v in value.get("environment", {}).items():
# Empty variables are converted to None...
# and None variables are not passed to the container
# This check can be removed when will be no longer covered
if v is None:
value["environment"][k] = ""
# Ports are forced to be int to prevent failures with compose
for idx, port in enumerate(value.get("ports", [])):
target_port = system.to_int(port["target"])
published_port = system.to_int(port["published"])
if target_port is None or published_port is None:
print_and_exit( # pragma: no cover
"Can't convert service ports to integers: {}-{}",
port["target"],
port["published"],
)
port["target"] = target_port
port["published"] = published_port
value["ports"][idx] = port
clean_config["services"][key] = value
for k in value.get("networks", {}).keys():
networks.add(k)
for k in value.get("volumes", []):
source = k.get("source", "")
volume_type = k.get("type", "")
if source and volume_type == "volume":
volumes.add(source.split(":")[0])
elif source and volume_type == "bind":
# Remove unsupported option: 'create_host_path'
if v1_compatibility:
k.get("bind", {}).pop("create_host_path", None)
binds.add(Path(source.split(":")[0]))
# Remove replicas if both replicas and global mode are set
if "deploy" in value: # pragma: no cover
if "replicas" in value["deploy"] and "mode" in value["deploy"]:
if value["deploy"]["mode"] == "global":
value["deploy"].pop("replicas")
# Missing folders are then automatically created by the docker engine
# the runs with root privileges and so create folders as root
# and this can often lead to issues with permissions.
for b in binds:
if not b.exists():
self.create_local_path(b, "bind folder")
for net in networks:
clean_config["networks"][net] = compose_config["networks"].get(net)
for vol in volumes:
volume_config = compose_config["volumes"].get(vol)
if "driver_opts" in volume_config:
device_type = volume_config["driver_opts"].get("type", "local")
device = volume_config["driver_opts"].get("device", "")
if device_type == "nfs" and device:
# starting from py39
# device = device.removeprefix(":")
if device.startswith(":"):
device = device[1:]
d = Path(device)
if not d.exists():
self.create_local_path(d, "volume path")
clean_config["volumes"][vol] = volume_config
with open(COMPOSE_FILE, "w") as fh:
fh.write(yaml.dump(clean_config, default_flow_style=False))
log.debug("Compose configuration dumped on {}", COMPOSE_FILE)
def init(
create_projectrc: bool = typer.Option(
False,
"--force",
"-f",
help="Overwrite initialization files if already exist",
show_default=False,
),
submodules_path: Path = typer.Option(
None,
"--submodules-path",
help=
"Link all submodules in an existing folder instead of download them",
),
) -> None:
Application.print_command(
Application.serialize_parameter("--force",
create_projectrc,
IF=create_projectrc),
Application.serialize_parameter("--submodules-path",
submodules_path,
IF=submodules_path),
)
Application.get_controller().controller_init()
for p in Application.project_scaffold.data_folders:
if not p.exists():
p.mkdir(parents=True, exist_ok=True)
for p in Application.project_scaffold.data_files:
if not p.exists():
p.touch()
if not Configuration.projectrc and not Configuration.host_configuration:
create_projectrc = True
# We have to create the .projectrc twice
# One generic here with main options and another after the complete
# conf reading to set services variables
if create_projectrc:
Application.get_controller().create_projectrc()
Application.get_controller().read_specs(read_extended=False)
if submodules_path is not None:
if not submodules_path.exists():
print_and_exit("Local path not found: {}", submodules_path)
Application.git_submodules(from_path=submodules_path)
Application.get_controller().read_specs(read_extended=True)
Application.get_controller().make_env()
# Compose services and variables
Application.get_controller().get_compose_configuration()
# We have to create the .projectrc twice
# One generic with main options and another here
# when services are available to set specific configurations
if create_projectrc:
Application.get_controller().create_projectrc()
Application.get_controller().read_specs(read_extended=True)
Application.get_controller().make_env()
if Configuration.swarm_mode:
docker = Docker(verify_swarm=False)
if not docker.swarm.get_token():
docker.swarm.init()
log.info("Swarm is now initialized")
else:
log.debug("Swarm is already initialized")
if Configuration.frontend == ANGULAR:
yarn_lock = DATA_DIR.joinpath(Configuration.project, "frontend",
"yarn.lock")
if yarn_lock.exists():
yarn_lock.unlink()
log.info("Yarn lock file deleted")
log.info("Project initialized")
def check_updates(path, gitobj, fetch_remote='origin', remote_branch=None):
fetch(path, gitobj, fetch_remote)
branch = get_active_branch(gitobj)
if branch is None:
log.warning("{} repo is detached? Unable to verify updates!", path)
return False
if remote_branch is None:
remote_branch = branch
max_remote = 20
log.verbose("Inspecting {}/{}", path, branch)
# CHECKING COMMITS BEHIND (TO BE PULLED) #
behind_check = "{}..{}/{}".format(branch, fetch_remote, remote_branch)
commits_behind = gitobj.iter_commits(behind_check, max_count=max_remote)
try:
commits_behind_list = list(commits_behind)
except GitCommandError:
log.info(
"Remote branch {} not found for {} repo. Is it a local branch?".
format(branch, path))
else:
if len(commits_behind_list) > 0:
log.warning("{} repo should be updated!", path)
else:
log.debug("{} repo is updated", path)
for c in commits_behind_list:
message = c.message.strip().replace('\n', "")
sha = c.hexsha[0:7]
if len(message) > 60:
message = message[0:57] + "..."
log.warning("Missing commit from {}: {} ({})", path, sha, message)
# CHECKING COMMITS AHEAD (TO BE PUSHED) #
# if path != 'upstream' and remote_branch == branch:
if remote_branch == branch:
ahead_check = "{}/{}..{}".format(fetch_remote, remote_branch, branch)
commits_ahead = gitobj.iter_commits(ahead_check, max_count=max_remote)
try:
commits_ahead_list = list(commits_ahead)
except GitCommandError:
log.info(
"Remote branch {} not found for {}. Is it a local branch?".
format(branch, path))
else:
if len(commits_ahead_list) > 0:
log.warning("You have commits not pushed on {} repo", path)
else:
log.debug("You pushed all commits on {} repo", path)
for c in commits_ahead_list:
message = c.message.strip().replace('\n', "")
sha = c.hexsha[0:7]
if len(message) > 60:
message = message[0:57] + "..."
log.warning("Unpushed commit in {}: {} ({})", path, sha,
message)
return True
def reload(
services: List[str] = typer.Argument(
None,
help="Services to be reloaded",
shell_complete=Application.autocomplete_service,
),
) -> None:
Application.print_command(Application.serialize_parameter("", services))
Application.get_controller().controller_init(services)
docker = Docker()
running_services = docker.get_running_services()
if "frontend" in services and len(services) > 1:
print_and_exit("Can't reload frontend and other services at once")
reloaded = 0
for service in Application.data.services:
# Special case: frontend in production mode
if Configuration.production and service == "frontend":
# Only consider it if explicitly requested in input
if "frontend" not in services:
log.debug(
"Can't reload the frontend if not explicitly requested")
else:
log.info("Reloading frontend...")
# The frontend build stucks in swarm mode... let's start the container
# always in compose mode when using the reload comand
Configuration.FORCE_COMPOSE_ENGINE = True
Application.get_controller().controller_init([service])
docker = Docker()
docker.compose.start_containers([service], force=True)
reloaded += 1
continue
if service not in running_services:
continue
containers = docker.get_containers(service)
if not containers:
log.warning("Can't find any container for {}", service)
continue
try:
# get the first container from the containers dict
container = containers.get(list(containers.keys())[0])
# Just added for typing purpose
if not container: # pragma: no conver
log.warning("Can't find any container for {}", service)
continue
output = docker.exec_command(
container,
user="******",
command="ls /usr/local/bin/reload",
force_output_return=True,
)
# this is to consume the iterator and raise the exception with exit code
if output:
[_ for _ in output]
except DockerException as e:
# fail2ban fails with code 1
if "It returned with code 1" in str(e):
log.warning("Service {} does not support the reload command",
service)
continue
# backend fails with code 2
if "It returned with code 2" in str(e):
log.warning("Service {} does not support the reload command",
service)
continue
raise
docker.exec_command(containers,
user="******",
command="/usr/local/bin/reload")
reloaded += 1
if reloaded == 0:
log.info("No service reloaded")
else:
log.info("Services reloaded")
def shell(
service: str = typer.Argument(
...,
help="Service name",
shell_complete=Application.autocomplete_service),
command: str = typer.Argument(
"bash",
help="UNIX command to be executed on selected running service"),
user: Optional[str] = typer.Option(
None,
"--user",
"-u",
help="User existing in selected service",
show_default=False,
),
default_command: bool = typer.Option(
False,
"--default-command",
"--default",
help="Execute the default command configured for the container",
show_default=False,
),
no_tty: bool = typer.Option(
False,
"--no-tty",
help=
"Disable pseudo-tty allocation (useful for non-interactive script)",
show_default=False,
),
replica: int = typer.Option(
1,
"--replica",
"--slot",
help="Execute the command on the specified replica",
show_default=False,
),
broadcast: bool = typer.Option(
False,
"--broadcast",
help="Execute the command on all the replicas",
show_default=False,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--user", user, IF=user),
Application.serialize_parameter("--default",
default_command,
IF=default_command),
Application.serialize_parameter("", service),
Application.serialize_parameter("", command),
)
if no_tty:
log.warning("--no-tty option is deprecated, you can stop using it")
if replica > 1 and broadcast:
print_and_exit("--replica and --broadcast options are not compatible")
Application.get_controller().controller_init()
docker = Docker()
if not user:
user = services.get_default_user(service)
if default_command:
command = services.get_default_command(service)
log.debug("Requested command: {} with user: {}", command, user
or "default")
if broadcast:
containers = docker.get_containers(service)
if not containers:
print_and_exit("No running container found for {} service",
service)
docker.exec_command(containers, user=user, command=command)
else:
container = docker.get_container(service, slot=replica)
if not container:
if replica != 1:
print_and_exit("Replica number {} not found for {} service",
str(replica), service)
print_and_exit("No running container found for {} service",
service)
docker.exec_command(container, user=user, command=command)
def check_updates(path: str, gitobj: Repo) -> None:
fetch(path, gitobj)
branch = get_active_branch(gitobj)
if branch is None: # pragma: no cover
log.warning("Is {} repo detached? Unable to verify updates", path)
return None
# CHECKING COMMITS BEHIND (TO BE PULLED) #
commits_behind = gitobj.iter_commits(
f"{branch}..origin/{branch}", max_count=MAX_FETCHED_COMMITS
)
try:
commits_behind_list = list(commits_behind)
except GitCommandError: # pragma: no cover
log.info(
"Remote branch {} not found for {} repo. Is it a local branch?",
branch,
path,
)
else:
if not commits_behind_list:
log.debug("{} repo is updated", path)
else: # pragma: no cover
log.warning("{} repo should be updated!", path)
for c in commits_behind_list:
message = str(c.message).strip().replace("\n", "")
if message.startswith("#"):
continue
sha = c.hexsha[0:7]
if len(message) > 60:
message = message[0:57] + "..."
log.warning("... missing commit from {}: {} ({})", path, sha, message)
commits_ahead = gitobj.iter_commits(
f"origin/{branch}..{branch}", max_count=MAX_FETCHED_COMMITS
)
try:
commits_ahead_list = list(commits_ahead)
except GitCommandError: # pragma: no cover
log.info(
"Remote branch {} not found for {}. Is it a local branch?", branch, path
)
else:
if len(commits_ahead_list) > 0:
log.warning("You have commits not pushed on {} repo", path)
else:
log.debug("You pushed all commits on {} repo", path)
for c in commits_ahead_list:
message = str(c.message).strip().replace("\n", "")
sha = c.hexsha[0:7]
if len(message) > 60: # pragma: no cover
message = message[0:57] + "..."
log.warning("Unpushed commit in {}: {} ({})", path, sha, message)
