def create_template( templating: Templating, template_name: str, target_path: Path, name: str, services: List[str], auth: str, force: bool, project: str, ) -> None: if not force and target_path.exists(): print_and_exit("{} already exists", target_path) template = templating.get_template( template_name, { "name": name, "services": services, "auth_service": auth, "project": project }, ) templating.save_template(target_path, template, force=force)
def create_task( project_scaffold: Project, name: str, services: List[str], auth: str, force: bool, add_tests: bool, ) -> None: path = project_scaffold.p_path("backend", "tasks") path = path.joinpath(f"{name}.py") templating = Templating() create_template( templating, "task_template.py", path, name, services, auth, force, project_scaffold.project, ) log.info("Task created: {}", path) if add_tests: log.warning("Tests for tasks not implemented yet")
def update_projectrc(variables: Dict[str, str]) -> None: today = date.today().strftime("%Y-%m-%d") annotation = f"# {UPDATE_LABEL} {today}" with open(PROJECTRC) as f: lines = f.readlines() append_additional_lines: List[str] = [] blanks = get_projectrc_variables_indentation(lines) if blanks == 0: # pragma: no cover print_and_exit( "Malformed .projectrc file, can't find an env block") pref = " " * blanks for variable, value in variables.items(): for index, line in enumerate(lines): # If the variable is found in .projectrc, let's update it if line.strip().startswith(variable): lines[ index] = f'{pref}{variable}: "{value}" {annotation}\n' break # if the variable is not found in .projectrc, let's append as additional else: append_additional_lines.append( f'{pref}{variable}: "{value}" {annotation}\n') templating = Templating() templating.make_backup(PROJECTRC) with open(PROJECTRC, "w") as f: last_line = "" for line in lines + append_additional_lines: last_line = line f.write(line) if not line.endswith("\n"): f.write("\n") # If last line is not an empty line, let's add a newline at the end of file if last_line.strip(): f.write("\n") # Write again the .env file Application.get_controller().load_projectrc() Application.get_controller().read_specs(read_extended=True) Application.get_controller().make_env()
def create_projectrc(self) -> None: templating = Templating() t = templating.get_template( "projectrc", { "project": Configuration.project, "hostname": Configuration.hostname, "swarm": Configuration.swarm_mode, "production": Configuration.production, "testing": Configuration.testing, "services": self.active_services, "env_variables": Configuration.environment, }, ) templating.save_template(PROJECTRC, t, force=True) Application.load_projectrc() if not self.files: log.debug("Created temporary default {} file", PROJECTRC) PROJECTRC.unlink() else: log.info("Created default {} file", PROJECTRC)
def create_endpoint( project_scaffold: Project, name: str, services: List[str], auth: str, force: bool, add_tests: bool, ) -> None: path = project_scaffold.p_path("backend", "endpoints") path = path.joinpath(f"{name}.py") templating = Templating() create_template( templating, "endpoint_template.py", path, name, services, auth, force, project_scaffold.project, ) log.info("Endpoint created: {}", path) if add_tests: path = project_scaffold.p_path("backend", "tests") path = path.joinpath(f"test_endpoints_{name}.py") create_template( templating, "endpoint_test_template.py", path, name, services, auth, force, project_scaffold.project, ) log.info("Tests scaffold created: {}", path)
def create_integration_test( project_scaffold: Project, name: str, services: List[str], auth: str, force: bool, add_tests: bool, ) -> None: if add_tests: print_and_exit( "Add integration_test does not support --add-tests flag") path = project_scaffold.p_path("frontend", "integration") # Expected name is a route-like string, e.g. app/mypath/:my_id # Let's replace the name with a path safe version # -> app_mypath_my_id # To be replaced with removeprefix # Initial / always removed... than will be added if name.startswith("/"): name = name[1:] filename = name.replace("/", "_").replace(":", "") path = path.joinpath(f"{filename}.spec.ts") templating = Templating() create_template( templating, "cypress_template.spec.ts", path, f"/{name}", services, auth, force, project_scaffold.project, ) log.info("Integration test created: {}", path)
def create_workflow( project_scaffold: Project, name: str, services: List[str], auth: str, force: bool, add_tests: bool, ) -> None: if add_tests: print_and_exit("Add workflow does not support --add-tests flag") workflows = ["backend", "frontend", "cypress", "mypy"] if name not in workflows: print_and_exit("Invalid workflow name, expected: {}", ", ".join(workflows)) path = Path(".github", "workflows") if not path.exists(): path.mkdir(parents=True) filename = f"github_actions-{name}.yml" path = path.joinpath(filename) templating = Templating() create_template( templating, filename, path, filename, services, auth, force, project_scaffold.project, ) log.info("GitHub Actions workflow created: {}", path)
def create_project( project_name: str, auth: str, frontend: str, services: List[str], extend: Optional[str], envs: Optional[List[str]] = None, auto: bool = False, force: bool = False, force_current: bool = False, add_optionals: bool = False, path: Path = None, ) -> None: project_scaffold = Project() enable_postgres = auth == "postgres" or "postgres" in services enable_mysql = auth == "mysql" or "mysql" in services enable_neo4j = auth == "neo4j" or "neo4j" in services enable_rabbit = "rabbit" in services enable_redis = "redis" in services enable_celery = "celery" in services enable_flower = "flower" in services enable_fail2ban = "fail2ban" in services enable_ftp = "ftp" in services enable_bot = "bot" in services if auth == "postgres" or auth == "mysql": auth = "sqlalchemy" if auth == "no": auth = NO_AUTHENTICATION if frontend == "no": frontend = NO_FRONTEND if not force_current: dirs = os.listdir(".") if dirs and dirs != [".git"]: print_and_exit( "Current folder is not empty, cannot create a new project here.\n" "Found: {}\n" "Use --current to force the creation here", ", ".join(dirs[0:3]), # add first 3 files/folders found ) celery_broker = None # Keep default value == REDIS celery_backend = None # Keep default value == REDIS if enable_celery: if enable_rabbit: celery_broker = "RABBIT" else: celery_broker = "REDIS" enable_redis = True if enable_redis: celery_backend = "REDIS" else: celery_backend = "RABBIT" env_variables = parse_env_variables(envs) project_scaffold.load_project_scaffold(project_name, auth, services) if frontend != NO_FRONTEND: project_scaffold.load_frontend_scaffold(frontend) # In case of errors this function will exit project_scaffold.check_invalid_characters(project_name) if project_name in project_scaffold.reserved_project_names: print_and_exit( "You selected a reserved name, invalid project name: {}", project_name) templating = Templating() folders = project_scaffold.expected_folders + project_scaffold.data_folders if add_optionals: folders += project_scaffold.optionals_folders for f in folders: if f.exists(): log.debug("Project folder already exists: {}", f) continue if not auto: print_and_exit("\nmkdir -p {}", f) f.mkdir(parents=True, exist_ok=True) for f in project_scaffold.suggested_gitkeep: f.open("a").close() files = project_scaffold.expected_files if add_optionals: files += project_scaffold.optionals_files if path: if path not in files: print_and_exit("Invalid path, cannot upgrade {}", path) else: files = [path] for p in files: template = templating.get_template( p.name, { "version": __version__, "project": project_name, "auth_service": auth, "enable_postgres": enable_postgres, "enable_mysql": enable_mysql, "enable_neo4j": enable_neo4j, "enable_rabbit": enable_rabbit, "enable_redis": enable_redis, "enable_celery": enable_celery, "enable_flower": enable_flower, "enable_fail2ban": enable_fail2ban, "enable_ftp": enable_ftp, "enable_bot": enable_bot, "celery_broker": celery_broker, "celery_backend": celery_backend, "frontend": frontend, "testing": Configuration.testing, "extend": extend, "services": services, "env_variables": env_variables, }, ) # automatic creation if auto: if p.exists() and not force: log.info("Project file already exists: {}", p) else: templating.save_template(p, template, force=force) continue # manual creation if p.exists(): log.info("Project file already exists: {}", p) else: print(f"\n{template}") print_and_exit(str(p)) if not path: for p in project_scaffold.raw_files: # automatic creation if auto: if p.exists() and not force: log.info("Project file already exists: {}", p) else: shutil.copyfile(templating.template_dir.joinpath(p.name), p) continue # manual creation if p.exists(): log.info("Project file already exists: {}", p) else: # print(f"Missing file: {p}") print_and_exit("File is missing: {}", p)
def check( no_git: bool = typer.Option( False, "--no-git", "-s", help="Skip checks on git commits", show_default=False, ), no_builds: bool = typer.Option( False, "--no-builds", help="Skip check on docker builds", show_default=False, ), ignore_submodules: List[str] = typer.Option( [], "--ignore-submodule", "-i", help="Ignore submodule", show_default=False, shell_complete=Application.autocomplete_submodule, ), ) -> None: Application.print_command( Application.serialize_parameter("--no-git", no_git, IF=no_git), Application.serialize_parameter("--no-builds", no_builds, IF=no_builds), Application.serialize_parameter("--ignore-submodule", ignore_submodules), ) Application.get_controller().controller_init() docker = Docker() if Configuration.swarm_mode: log.debug("Swarm is correctly initialized") docker.swarm.check_resources() if no_git: log.info("Skipping git checks") else: log.info("Checking git (skip with --no-git)") Application.git_checks(ignore_submodules) if no_builds: log.info("Skipping builds checks") else: log.info("Checking builds (skip with --no-builds)") dimages: List[str] = [] for img in docker.client.images(): if img.repo_tags: for i in img.repo_tags: dimages.append(i) all_builds = find_templates_build(Application.data.compose_config) core_builds = find_templates_build(Application.data.base_services) overriding_builds = find_templates_override( Application.data.compose_config, core_builds) for image_tag, build in all_builds.items(): services = build["services"] if not any(x in Application.data.active_services for x in services): continue if image_tag not in dimages: if image_tag in core_builds: log.warning( "Missing {} image, execute {command}", image_tag, command=RED("rapydo pull"), ) else: log.warning( "Missing {} image, execute {command}", image_tag, command=RED("rapydo build"), ) continue image_creation = get_image_creation(image_tag) # Check if some recent commit modified the Dockerfile d1, d2 = build_is_obsolete(image_creation, build.get("path")) if d1 and d2: tmp_from_image = overriding_builds.get(image_tag) # This is the case of a build not overriding a core image, # e.g nifi or geoserver. In that case from_image is faked to image_tag # just to make print_obsolete to print 'build' instead of 'pull' if not tmp_from_image and image_tag not in core_builds: tmp_from_image = image_tag print_obsolete(image_tag, d1, d2, build.get("service"), tmp_from_image) # if FROM image is newer, this build should be re-built elif image_tag in overriding_builds: from_img = overriding_builds.get(image_tag, "") from_build: Optional[TemplateInfo] = core_builds.get(from_img) if not from_build: # pragma: no cover log.critical("Malformed {} image, from build is missing", image_tag) continue # Verify if template build exists if from_img not in dimages: # pragma: no cover log.warning( "Missing template build for {} ({})\n{}", from_build.get("services"), from_img, ) from_timestamp = get_image_creation(from_img) # Verify if template build is obsolete or not d1, d2 = build_is_obsolete(from_timestamp, from_build.get("path")) if d1 and d2: # pragma: no cover print_obsolete(from_img, d1, d2, from_build.get("service")) if from_timestamp > image_creation: b = image_creation.strftime(DATE_FORMAT) c = from_timestamp.strftime(DATE_FORMAT) print_obsolete(image_tag, b, c, build.get("service"), from_img) templating = Templating() for filename in Application.project_scaffold.fixed_files: if templating.file_changed(str(filename)): log.warning( "{} changed, please execute {command}", filename, command=RED(f"rapydo upgrade --path {filename}"), ) compose_version = "Unknown" buildx_version = "Unknown" m = re.search( r"^Docker Compose version (v[0-9]+\.[0-9]+\.[0-9]+)$", docker.client.compose.version(), ) if m: compose_version = m.group(1) m = re.search( r"^github.com/docker/buildx (v[0-9]+\.[0-9]+\.[0-9]+) .*$", docker.client.buildx.version(), ) if m: buildx_version = m.group(1) if compose_version == COMPOSE_VERSION: log.info("Compose is installed with version {}", COMPOSE_VERSION) else: # pragma: no cover cmd = RED("rapydo install compose") fix_hint = f"You can update it with {cmd}" log.warning( "Compose is installed with version {}, expected version is {}.\n{}", compose_version, COMPOSE_VERSION, fix_hint, ) if buildx_version == BUILDX_VERSION: log.info("Buildx is installed with version {}", BUILDX_VERSION) else: # pragma: no cover cmd = RED("rapydo install buildx") fix_hint = f"You can update it with {cmd}" log.warning( "Buildx is installed with version {}, expected version is {}.\n{}", buildx_version, BUILDX_VERSION, fix_hint, ) for expired_passwords in get_expired_passwords(): log.warning( "{} is expired on {}", expired_passwords[0], expired_passwords[1].strftime("%Y-%m-%d"), ) log.info("Checks completed")
def test_get_templating() -> None: templating = Templating() with pytest.raises(SystemExit): templating.get_template("invalid", {})
def create_service( project_scaffold: Project, name: str, services: List[str], auth: str, force: bool, add_tests: bool, ) -> None: path = project_scaffold.p_path("frontend", "app", "services") path.mkdir(parents=True, exist_ok=True) path = path.joinpath(f"{name}.ts") templating = Templating() create_template( templating, "service_template.ts", path, name, services, auth, force, project_scaffold.project, ) log.info("Service created: {}", path) module_path = project_scaffold.p_path("frontend", "app", "custom.module.ts") module = None with open(module_path) as f: module = f.read().splitlines() normalized_name = name.title().replace(" ", "") SNAME = f"{normalized_name}Service" # Add service import import_line = f"import {{ {SNAME} }} from '@app/services/{name}';" for idx, row in enumerate(module): if import_line in row: log.info("Import already included in module file") break if row.strip().startswith("const") or row.strip().startswith("@"): module = module[:idx] + [import_line, ""] + module[idx:] log.info("Added {} to module file", import_line) break # Add service declaration for idx, row in enumerate(module): if row.strip().startswith("declarations"): module = module[:idx + 1] + [f" {SNAME},"] + module[idx + 1:] log.info("Added {} to module declarations", SNAME) break templating.make_backup(module_path) # Save new module file with open(module_path, "w") as f: for row in module: f.write(f"{row}\n") f.write("\n") if add_tests: log.warning("Tests for services not implemented yet")
def create_component( project_scaffold: Project, name: str, services: List[str], auth: str, force: bool, add_tests: bool, ) -> None: path = project_scaffold.p_path("frontend", "app", "components", name) path.mkdir(parents=True, exist_ok=True) # Used by Frontend during tests is_sink_special_case = name == "sink" if is_sink_special_case: template_ts = "sink.ts" template_html = "sink.html" else: template_ts = "component_template.ts" template_html = "component_template.html" cpath = path.joinpath(f"{name}.ts") templating = Templating() create_template( templating, template_ts, cpath, name, services, auth, force, project_scaffold.project, ) hpath = path.joinpath(f"{name}.html") create_template( templating, template_html, hpath, name, services, auth, force, project_scaffold.project, ) log.info("Component created: {}", path) module_path = project_scaffold.p_path("frontend", "app", "custom.module.ts") module = None with open(module_path) as f: module = f.read().splitlines() normalized_name = name.title().replace(" ", "") CNAME = f"{normalized_name}Component" # Add component import import_line = f"import {{ {CNAME} }} from '@app/components/{name}/{name}';" for idx, row in enumerate(module): if import_line in row: log.info("Import already included in module file") break if row.strip().startswith("const") or row.strip().startswith("@"): module = module[:idx] + [import_line, ""] + module[idx:] log.info("Added {} to module file", import_line) break # Add sink route if is_sink_special_case: for idx, row in enumerate(module): if row.strip().startswith("const routes: Routes = ["): ROUTE = """ { path: "app/sink", component: SinkComponent, }, """ module = module[:idx + 1] + [ROUTE] + module[idx + 1:] log.info("Added route to module declarations") break # Add component declaration for idx, row in enumerate(module): if row.strip().startswith("declarations"): module = module[:idx + 1] + [f" {CNAME},"] + module[idx + 1:] log.info("Added {} to module declarations", CNAME) break templating.make_backup(module_path) # Save new module file with open(module_path, "w") as f: for row in module: f.write(f"{row}\n") f.write("\n") if add_tests: path = project_scaffold.p_path("frontend", "app", "components", name) path = path.joinpath(f"{name}.spec.ts") create_template( templating, "component_test_template.spec.ts", path, name, services, auth, force, project_scaffold.project, ) log.info("Tests scaffold created: {}", path)