def create_template(
templating: Templating,
template_name: str,
target_path: Path,
name: str,
services: List[str],
auth: str,
force: bool,
project: str,
) -> None:
if not force and target_path.exists():
print_and_exit("{} already exists", target_path)
template = templating.get_template(
template_name,
{
"name": name,
"services": services,
"auth_service": auth,
"project": project
},
)
templating.save_template(target_path, template, force=force)
def create_task(
project_scaffold: Project,
name: str,
services: List[str],
auth: str,
force: bool,
add_tests: bool,
) -> None:
path = project_scaffold.p_path("backend", "tasks")
path = path.joinpath(f"{name}.py")
templating = Templating()
create_template(
templating,
"task_template.py",
path,
name,
services,
auth,
force,
project_scaffold.project,
)
log.info("Task created: {}", path)
if add_tests:
log.warning("Tests for tasks not implemented yet")
def update_projectrc(variables: Dict[str, str]) -> None:
today = date.today().strftime("%Y-%m-%d")
annotation = f"# {UPDATE_LABEL} {today}"
with open(PROJECTRC) as f:
lines = f.readlines()
append_additional_lines: List[str] = []
blanks = get_projectrc_variables_indentation(lines)
if blanks == 0: # pragma: no cover
print_and_exit(
"Malformed .projectrc file, can't find an env block")
pref = " " * blanks
for variable, value in variables.items():
for index, line in enumerate(lines):
# If the variable is found in .projectrc, let's update it
if line.strip().startswith(variable):
lines[
index] = f'{pref}{variable}: "{value}" {annotation}\n'
break
# if the variable is not found in .projectrc, let's append as additional
else:
append_additional_lines.append(
f'{pref}{variable}: "{value}" {annotation}\n')
templating = Templating()
templating.make_backup(PROJECTRC)
with open(PROJECTRC, "w") as f:
last_line = ""
for line in lines + append_additional_lines:
last_line = line
f.write(line)
if not line.endswith("\n"):
f.write("\n")
# If last line is not an empty line, let's add a newline at the end of file
if last_line.strip():
f.write("\n")
# Write again the .env file
Application.get_controller().load_projectrc()
Application.get_controller().read_specs(read_extended=True)
Application.get_controller().make_env()
def create_projectrc(self) -> None:
templating = Templating()
t = templating.get_template(
"projectrc",
{
"project": Configuration.project,
"hostname": Configuration.hostname,
"swarm": Configuration.swarm_mode,
"production": Configuration.production,
"testing": Configuration.testing,
"services": self.active_services,
"env_variables": Configuration.environment,
},
)
templating.save_template(PROJECTRC, t, force=True)
Application.load_projectrc()
if not self.files:
log.debug("Created temporary default {} file", PROJECTRC)
PROJECTRC.unlink()
else:
log.info("Created default {} file", PROJECTRC)
def create_endpoint(
project_scaffold: Project,
name: str,
services: List[str],
auth: str,
force: bool,
add_tests: bool,
) -> None:
path = project_scaffold.p_path("backend", "endpoints")
path = path.joinpath(f"{name}.py")
templating = Templating()
create_template(
templating,
"endpoint_template.py",
path,
name,
services,
auth,
force,
project_scaffold.project,
)
log.info("Endpoint created: {}", path)
if add_tests:
path = project_scaffold.p_path("backend", "tests")
path = path.joinpath(f"test_endpoints_{name}.py")
create_template(
templating,
"endpoint_test_template.py",
path,
name,
services,
auth,
force,
project_scaffold.project,
)
log.info("Tests scaffold created: {}", path)
def create_integration_test(
project_scaffold: Project,
name: str,
services: List[str],
auth: str,
force: bool,
add_tests: bool,
) -> None:
if add_tests:
print_and_exit(
"Add integration_test does not support --add-tests flag")
path = project_scaffold.p_path("frontend", "integration")
# Expected name is a route-like string, e.g. app/mypath/:my_id
# Let's replace the name with a path safe version
# -> app_mypath_my_id
# To be replaced with removeprefix
# Initial / always removed... than will be added
if name.startswith("/"):
name = name[1:]
filename = name.replace("/", "_").replace(":", "")
path = path.joinpath(f"{filename}.spec.ts")
templating = Templating()
create_template(
templating,
"cypress_template.spec.ts",
path,
f"/{name}",
services,
auth,
force,
project_scaffold.project,
)
log.info("Integration test created: {}", path)
def create_workflow(
project_scaffold: Project,
name: str,
services: List[str],
auth: str,
force: bool,
add_tests: bool,
) -> None:
if add_tests:
print_and_exit("Add workflow does not support --add-tests flag")
workflows = ["backend", "frontend", "cypress", "mypy"]
if name not in workflows:
print_and_exit("Invalid workflow name, expected: {}",
", ".join(workflows))
path = Path(".github", "workflows")
if not path.exists():
path.mkdir(parents=True)
filename = f"github_actions-{name}.yml"
path = path.joinpath(filename)
templating = Templating()
create_template(
templating,
filename,
path,
filename,
services,
auth,
force,
project_scaffold.project,
)
log.info("GitHub Actions workflow created: {}", path)
def create_project(
project_name: str,
auth: str,
frontend: str,
services: List[str],
extend: Optional[str],
envs: Optional[List[str]] = None,
auto: bool = False,
force: bool = False,
force_current: bool = False,
add_optionals: bool = False,
path: Path = None,
) -> None:
project_scaffold = Project()
enable_postgres = auth == "postgres" or "postgres" in services
enable_mysql = auth == "mysql" or "mysql" in services
enable_neo4j = auth == "neo4j" or "neo4j" in services
enable_rabbit = "rabbit" in services
enable_redis = "redis" in services
enable_celery = "celery" in services
enable_flower = "flower" in services
enable_fail2ban = "fail2ban" in services
enable_ftp = "ftp" in services
enable_bot = "bot" in services
if auth == "postgres" or auth == "mysql":
auth = "sqlalchemy"
if auth == "no":
auth = NO_AUTHENTICATION
if frontend == "no":
frontend = NO_FRONTEND
if not force_current:
dirs = os.listdir(".")
if dirs and dirs != [".git"]:
print_and_exit(
"Current folder is not empty, cannot create a new project here.\n"
"Found: {}\n"
"Use --current to force the creation here",
", ".join(dirs[0:3]), # add first 3 files/folders found
)
celery_broker = None # Keep default value == REDIS
celery_backend = None # Keep default value == REDIS
if enable_celery:
if enable_rabbit:
celery_broker = "RABBIT"
else:
celery_broker = "REDIS"
enable_redis = True
if enable_redis:
celery_backend = "REDIS"
else:
celery_backend = "RABBIT"
env_variables = parse_env_variables(envs)
project_scaffold.load_project_scaffold(project_name, auth, services)
if frontend != NO_FRONTEND:
project_scaffold.load_frontend_scaffold(frontend)
# In case of errors this function will exit
project_scaffold.check_invalid_characters(project_name)
if project_name in project_scaffold.reserved_project_names:
print_and_exit(
"You selected a reserved name, invalid project name: {}",
project_name)
templating = Templating()
folders = project_scaffold.expected_folders + project_scaffold.data_folders
if add_optionals:
folders += project_scaffold.optionals_folders
for f in folders:
if f.exists():
log.debug("Project folder already exists: {}", f)
continue
if not auto:
print_and_exit("\nmkdir -p {}", f)
f.mkdir(parents=True, exist_ok=True)
for f in project_scaffold.suggested_gitkeep:
f.open("a").close()
files = project_scaffold.expected_files
if add_optionals:
files += project_scaffold.optionals_files
if path:
if path not in files:
print_and_exit("Invalid path, cannot upgrade {}", path)
else:
files = [path]
for p in files:
template = templating.get_template(
p.name,
{
"version": __version__,
"project": project_name,
"auth_service": auth,
"enable_postgres": enable_postgres,
"enable_mysql": enable_mysql,
"enable_neo4j": enable_neo4j,
"enable_rabbit": enable_rabbit,
"enable_redis": enable_redis,
"enable_celery": enable_celery,
"enable_flower": enable_flower,
"enable_fail2ban": enable_fail2ban,
"enable_ftp": enable_ftp,
"enable_bot": enable_bot,
"celery_broker": celery_broker,
"celery_backend": celery_backend,
"frontend": frontend,
"testing": Configuration.testing,
"extend": extend,
"services": services,
"env_variables": env_variables,
},
)
# automatic creation
if auto:
if p.exists() and not force:
log.info("Project file already exists: {}", p)
else:
templating.save_template(p, template, force=force)
continue
# manual creation
if p.exists():
log.info("Project file already exists: {}", p)
else:
print(f"\n{template}")
print_and_exit(str(p))
if not path:
for p in project_scaffold.raw_files:
# automatic creation
if auto:
if p.exists() and not force:
log.info("Project file already exists: {}", p)
else:
shutil.copyfile(templating.template_dir.joinpath(p.name),
p)
continue
# manual creation
if p.exists():
log.info("Project file already exists: {}", p)
else:
# print(f"Missing file: {p}")
print_and_exit("File is missing: {}", p)
def check(
no_git: bool = typer.Option(
False,
"--no-git",
"-s",
help="Skip checks on git commits",
show_default=False,
),
no_builds: bool = typer.Option(
False,
"--no-builds",
help="Skip check on docker builds",
show_default=False,
),
ignore_submodules: List[str] = typer.Option(
[],
"--ignore-submodule",
"-i",
help="Ignore submodule",
show_default=False,
shell_complete=Application.autocomplete_submodule,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--no-git", no_git, IF=no_git),
Application.serialize_parameter("--no-builds", no_builds,
IF=no_builds),
Application.serialize_parameter("--ignore-submodule",
ignore_submodules),
)
Application.get_controller().controller_init()
docker = Docker()
if Configuration.swarm_mode:
log.debug("Swarm is correctly initialized")
docker.swarm.check_resources()
if no_git:
log.info("Skipping git checks")
else:
log.info("Checking git (skip with --no-git)")
Application.git_checks(ignore_submodules)
if no_builds:
log.info("Skipping builds checks")
else:
log.info("Checking builds (skip with --no-builds)")
dimages: List[str] = []
for img in docker.client.images():
if img.repo_tags:
for i in img.repo_tags:
dimages.append(i)
all_builds = find_templates_build(Application.data.compose_config)
core_builds = find_templates_build(Application.data.base_services)
overriding_builds = find_templates_override(
Application.data.compose_config, core_builds)
for image_tag, build in all_builds.items():
services = build["services"]
if not any(x in Application.data.active_services
for x in services):
continue
if image_tag not in dimages:
if image_tag in core_builds:
log.warning(
"Missing {} image, execute {command}",
image_tag,
command=RED("rapydo pull"),
)
else:
log.warning(
"Missing {} image, execute {command}",
image_tag,
command=RED("rapydo build"),
)
continue
image_creation = get_image_creation(image_tag)
# Check if some recent commit modified the Dockerfile
d1, d2 = build_is_obsolete(image_creation, build.get("path"))
if d1 and d2:
tmp_from_image = overriding_builds.get(image_tag)
# This is the case of a build not overriding a core image,
# e.g nifi or geoserver. In that case from_image is faked to image_tag
# just to make print_obsolete to print 'build' instead of 'pull'
if not tmp_from_image and image_tag not in core_builds:
tmp_from_image = image_tag
print_obsolete(image_tag, d1, d2, build.get("service"),
tmp_from_image)
# if FROM image is newer, this build should be re-built
elif image_tag in overriding_builds:
from_img = overriding_builds.get(image_tag, "")
from_build: Optional[TemplateInfo] = core_builds.get(from_img)
if not from_build: # pragma: no cover
log.critical("Malformed {} image, from build is missing",
image_tag)
continue
# Verify if template build exists
if from_img not in dimages: # pragma: no cover
log.warning(
"Missing template build for {} ({})\n{}",
from_build.get("services"),
from_img,
)
from_timestamp = get_image_creation(from_img)
# Verify if template build is obsolete or not
d1, d2 = build_is_obsolete(from_timestamp,
from_build.get("path"))
if d1 and d2: # pragma: no cover
print_obsolete(from_img, d1, d2, from_build.get("service"))
if from_timestamp > image_creation:
b = image_creation.strftime(DATE_FORMAT)
c = from_timestamp.strftime(DATE_FORMAT)
print_obsolete(image_tag, b, c, build.get("service"),
from_img)
templating = Templating()
for filename in Application.project_scaffold.fixed_files:
if templating.file_changed(str(filename)):
log.warning(
"{} changed, please execute {command}",
filename,
command=RED(f"rapydo upgrade --path {filename}"),
)
compose_version = "Unknown"
buildx_version = "Unknown"
m = re.search(
r"^Docker Compose version (v[0-9]+\.[0-9]+\.[0-9]+)$",
docker.client.compose.version(),
)
if m:
compose_version = m.group(1)
m = re.search(
r"^github.com/docker/buildx (v[0-9]+\.[0-9]+\.[0-9]+) .*$",
docker.client.buildx.version(),
)
if m:
buildx_version = m.group(1)
if compose_version == COMPOSE_VERSION:
log.info("Compose is installed with version {}", COMPOSE_VERSION)
else: # pragma: no cover
cmd = RED("rapydo install compose")
fix_hint = f"You can update it with {cmd}"
log.warning(
"Compose is installed with version {}, expected version is {}.\n{}",
compose_version,
COMPOSE_VERSION,
fix_hint,
)
if buildx_version == BUILDX_VERSION:
log.info("Buildx is installed with version {}", BUILDX_VERSION)
else: # pragma: no cover
cmd = RED("rapydo install buildx")
fix_hint = f"You can update it with {cmd}"
log.warning(
"Buildx is installed with version {}, expected version is {}.\n{}",
buildx_version,
BUILDX_VERSION,
fix_hint,
)
for expired_passwords in get_expired_passwords():
log.warning(
"{} is expired on {}",
expired_passwords[0],
expired_passwords[1].strftime("%Y-%m-%d"),
)
log.info("Checks completed")
def test_get_templating() -> None:
templating = Templating()
with pytest.raises(SystemExit):
templating.get_template("invalid", {})
def create_service(
project_scaffold: Project,
name: str,
services: List[str],
auth: str,
force: bool,
add_tests: bool,
) -> None:
path = project_scaffold.p_path("frontend", "app", "services")
path.mkdir(parents=True, exist_ok=True)
path = path.joinpath(f"{name}.ts")
templating = Templating()
create_template(
templating,
"service_template.ts",
path,
name,
services,
auth,
force,
project_scaffold.project,
)
log.info("Service created: {}", path)
module_path = project_scaffold.p_path("frontend", "app",
"custom.module.ts")
module = None
with open(module_path) as f:
module = f.read().splitlines()
normalized_name = name.title().replace(" ", "")
SNAME = f"{normalized_name}Service"
# Add service import
import_line = f"import {{ {SNAME} }} from '@app/services/{name}';"
for idx, row in enumerate(module):
if import_line in row:
log.info("Import already included in module file")
break
if row.strip().startswith("const") or row.strip().startswith("@"):
module = module[:idx] + [import_line, ""] + module[idx:]
log.info("Added {} to module file", import_line)
break
# Add service declaration
for idx, row in enumerate(module):
if row.strip().startswith("declarations"):
module = module[:idx + 1] + [f" {SNAME},"] + module[idx + 1:]
log.info("Added {} to module declarations", SNAME)
break
templating.make_backup(module_path)
# Save new module file
with open(module_path, "w") as f:
for row in module:
f.write(f"{row}\n")
f.write("\n")
if add_tests:
log.warning("Tests for services not implemented yet")
def create_component(
project_scaffold: Project,
name: str,
services: List[str],
auth: str,
force: bool,
add_tests: bool,
) -> None:
path = project_scaffold.p_path("frontend", "app", "components", name)
path.mkdir(parents=True, exist_ok=True)
# Used by Frontend during tests
is_sink_special_case = name == "sink"
if is_sink_special_case:
template_ts = "sink.ts"
template_html = "sink.html"
else:
template_ts = "component_template.ts"
template_html = "component_template.html"
cpath = path.joinpath(f"{name}.ts")
templating = Templating()
create_template(
templating,
template_ts,
cpath,
name,
services,
auth,
force,
project_scaffold.project,
)
hpath = path.joinpath(f"{name}.html")
create_template(
templating,
template_html,
hpath,
name,
services,
auth,
force,
project_scaffold.project,
)
log.info("Component created: {}", path)
module_path = project_scaffold.p_path("frontend", "app",
"custom.module.ts")
module = None
with open(module_path) as f:
module = f.read().splitlines()
normalized_name = name.title().replace(" ", "")
CNAME = f"{normalized_name}Component"
# Add component import
import_line = f"import {{ {CNAME} }} from '@app/components/{name}/{name}';"
for idx, row in enumerate(module):
if import_line in row:
log.info("Import already included in module file")
break
if row.strip().startswith("const") or row.strip().startswith("@"):
module = module[:idx] + [import_line, ""] + module[idx:]
log.info("Added {} to module file", import_line)
break
# Add sink route
if is_sink_special_case:
for idx, row in enumerate(module):
if row.strip().startswith("const routes: Routes = ["):
ROUTE = """
{
path: "app/sink",
component: SinkComponent,
},
"""
module = module[:idx + 1] + [ROUTE] + module[idx + 1:]
log.info("Added route to module declarations")
break
# Add component declaration
for idx, row in enumerate(module):
if row.strip().startswith("declarations"):
module = module[:idx + 1] + [f" {CNAME},"] + module[idx + 1:]
log.info("Added {} to module declarations", CNAME)
break
templating.make_backup(module_path)
# Save new module file
with open(module_path, "w") as f:
for row in module:
f.write(f"{row}\n")
f.write("\n")
if add_tests:
path = project_scaffold.p_path("frontend", "app", "components", name)
path = path.joinpath(f"{name}.spec.ts")
create_template(
templating,
"component_test_template.spec.ts",
path,
name,
services,
auth,
force,
project_scaffold.project,
)
log.info("Tests scaffold created: {}", path)