def reset_mysql(project): """Reset the discovery mysql database""" from derex.runner.docker import check_services from derex.runner.docker import wait_for_mysql if project.runmode is not ProjectRunMode.debug: click.get_current_context().fail( "This command can only be run in `debug` runmode") if not check_services(["mysql"]): click.echo( "Mysql service not found.\nMaybe you forgot to run\nddc-services up -d" ) return wait_for_mysql() restore_dump_path = abspath_from_egg("derex.discovery", "derex/discovery/restore_dump.py") run_compose( [ "run", "--rm", "-v", f"{restore_dump_path}:/openedx/discovery/restore_dump.py", "discovery", "python", "/openedx/discovery/restore_dump.py", ], project=project, ) return 0
def reset_mysql(project: Project, dry_run: bool = False): """Run script from derex/openedx image to reset the mysql db. """ if project.runmode is not ProjectRunMode.debug: click.get_current_context().fail( "The command reset-mysql can only be run in `debug` runmode") logger.warning("Resetting mysql database") restore_dump_path = abspath_from_egg( "derex.runner", "derex/runner/restore_dump.py.source") assert (restore_dump_path ), "Could not find restore_dump.py in derex.runner distribution" run_compose( [ "run", "--rm", "-v", f"{restore_dump_path}:/restore_dump.py", "lms", "python", "/restore_dump.py", ], project=project, dry_run=dry_run, )
def openedx(version, target, push, only_print_image_name, docker_opts): """Build openedx image using docker. Defaults to dev image target.""" dockerdir = abspath_from_egg("derex.runner", "docker-definition/Dockerfile").parent build_arguments = [] for spec in version.value.items(): build_arguments.append("--build-arg") build_arguments.append(f"{spec[0].upper()}={spec[1]}") docker_image_prefix = version.value["docker_image_prefix"] image_name = f"{docker_image_prefix}-{target}:{__version__}" if only_print_image_name: click.echo(image_name) return push_arg = ",push=true" if push else "" command = [ "docker", "buildx", "build", str(dockerdir), "-t", image_name, *build_arguments, f"--target={target}", ] transifex_path = os.path.expanduser("~/.transifexrc") if os.path.exists(transifex_path): command.extend(["--secret", f"id=transifex,src={transifex_path}"]) if docker_opts: command.extend(docker_opts.format(**locals()).split()) print("Invoking\n" + " ".join(command), file=sys.stderr) os.execve(find_executable(command[0]), command, os.environ)
def _populate_settings(self): """If the project includes user defined settings, add ours to that directory to let the project's settings use the line from .derex import * Also add a base.py file with the above content if it does not exist. """ if self.settings_dir is None: return base_settings = self.settings_dir / "base.py" if not base_settings.is_file(): base_settings.write_text("from .derex import *\n") init = self.settings_dir / "__init__.py" if not init.is_file(): init.write_text('"""Settings for edX"""') our_settings_dir = abspath_from_egg( "derex.runner", "derex/runner/settings/README.rst").parent for source in our_settings_dir.glob("**/*.py"): destination = self.settings_dir / source.relative_to( our_settings_dir) if destination.is_file(): if destination.read_text() != source.read_text(): raise SettingsModified(filename=destination) else: if not destination.parent.is_dir(): destination.parent.mkdir(parents=True) destination.write_text(source.read_text()) destination.chmod(0o444)
def openedx(version, target, push, only_print_image_name, docker_opts): """Build openedx image using docker. Defaults to dev image target.""" dockerdir = abspath_from_egg("derex.runner", "docker-definition/Dockerfile").parent git_repo = version.value["git_repo"] git_branch = version.value["git_branch"] python_version = version.value.get("python_version", "3.6") docker_image_prefix = version.value["docker_image_prefix"] image_name = f"{docker_image_prefix}-{target}:{__version__}" if only_print_image_name: click.echo(image_name) return push_arg = ",push=true" if push else "" command = [ "docker", "buildx", "build", str(dockerdir), "-t", image_name, "--build-arg", f"PYTHON_VERSION={python_version}", "--build-arg", f"EDX_PLATFORM_VERSION={git_branch}", "--build-arg", f"EDX_PLATFORM_REPOSITORY={git_repo}", f"--target={target}", ] transifex_path = os.path.expanduser("~/.transifexrc") if os.path.exists(transifex_path): command.extend(["--secret", f"id=transifex,src={transifex_path}"]) if docker_opts: command.extend(docker_opts.format(**locals()).split()) print("Invoking\n" + " ".join(command), file=sys.stderr) os.execve(find_executable(command[0]), command, os.environ)
def settings_directory_path(self) -> Path: """Return an absolute path that will be mounted under lms/envs/derex_project and cms/envs/derex_project inside the container. If the project has local settings, we use that directory. Otherwise we use the directory bundled with `derex.runner` """ if self.settings_dir is not None: return self.settings_dir return abspath_from_egg("derex.runner", "derex/runner/settings/derex/base.py").parent
def compose_options() -> Dict[str, Union[str, List[str]]]: """See derex.runner.plugin_spec.compose_options docstring. """ options = [ "--project-name", "derex_services", "-f", str( abspath_from_egg("derex.runner", "derex/runner/compose_files/services.yml")), ] if asbool(os.environ.get("DEREX_ADMIN_SERVICES", True)): options += [ "-f", str( abspath_from_egg("derex.runner", "derex/runner/compose_files/admin.yml")), ] return { "options": options, "name": "base", "priority": "_begin", "variant": "services", }
def load_dump(relpath): """Loads a mysql dump into the derex mysql database. """ dump_path = abspath_from_egg("derex.runner", relpath) image = client.containers.get("mysql").image logger.info("Resetting email database") try: client.containers.run( image.tags[0], ["sh", "-c", f"mysql -h mysql -psecret < /dump/{dump_path.name}"], network="derex", volumes={dump_path.parent: { "bind": "/dump" }}, auto_remove=True, ) except docker.errors.ContainerError as exc: logger.exception(exc)
def generate_local_docker_compose(project: Project) -> Path: """This function is called every time ddc-project is run. It assembles a docker-compose file from the given configuration. It should execute as fast as possible. """ local_compose_path = project.private_filepath("docker-compose.yml") template_path = abspath_from_egg("derex.runner", "derex/runner/templates/local.yml.j2") final_image = None if image_exists(project.image_tag): final_image = project.image_tag if not image_exists(project.requirements_image_tag): click.echo("Building requirements image") build_requirements_image(project) tmpl = Template(template_path.read_text()) text = tmpl.render(project=project, final_image=final_image) local_compose_path.write_text(text) return local_compose_path
def reset_mysql_openedx(project: Project, dry_run: bool = False): """Run script from derex/openedx image to reset the mysql db. """ restore_dump_path = abspath_from_egg( "derex.runner", "derex/runner/restore_dump.py.source") assert (restore_dump_path ), "Could not find restore_dump.py in derex.runner distribution" run_compose( [ "run", "--rm", "-v", f"{restore_dump_path}:/restore_dump.py", "lms", "python", "/restore_dump.py", ], project=project, dry_run=dry_run, )
def forum_build(version, push, only_print_image_name, docker_opts): # TODO: generalize derex.runner build.openedx function so to # take the dockerfile_dir and secrets as arguments. # This way we can avoid repeating code here and just call # the build function with our arguments. # e.g. # return derex_build_cli.build( # dockerfile_dir, # version, # push, # only_print_image_name, # docker_opts, # secrets # ) """Build openedx image using docker. Defaults to dev image target.""" dockerfile_dir = abspath_from_egg("derex.forum", "docker_build/Dockerfile").parent build_arguments = [] for spec in version.value.items(): build_arguments.append("--build-arg") build_arguments.append(f"{spec[0].upper()}={spec[1]}") docker_image_prefix = version.value["docker_image_prefix"] image_name = f"{docker_image_prefix}:{__version__}" if only_print_image_name: click.echo(image_name) return push_arg = ",push=true" if push else "" command = [ "docker", "buildx", "build", str(dockerfile_dir), "-t", image_name, *build_arguments, ] if docker_opts: command.extend(docker_opts.format(**locals()).split()) print("Invoking\n" + " ".join(command), file=sys.stderr) os.execve(find_executable(command[0]), command, os.environ)
def load_fixtures(project): """Load fixtures from the plugin fixtures directory""" from derex.runner.compose_utils import run_compose fixtures_dir = project.get_plugin_directories(__package__).get("fixtures") if fixtures_dir is None: click.echo("No fixtures directory present for this plugin") return load_fixtures_path = abspath_from_egg("derex.discovery", "derex/discovery/load_fixtures.py") compose_args = [ "run", "--rm", "-v", f"{load_fixtures_path}:/openedx/discovery/load_fixtures.py", "discovery", "python", "/openedx/discovery/load_fixtures.py", ] run_compose(compose_args, project=project) return
def _populate_settings(self): """If the project includes user defined settings, add ours to that directory to let the project's settings use the line from .derex import * Also add a base.py file with the above content if it does not exist. """ if self.settings_dir is None: return base_settings = self.settings_dir / "base.py" if not base_settings.is_file(): base_settings.write_text("from .derex import *\n") init = self.settings_dir / "__init__.py" if not init.is_file(): init.write_text('"""Settings for edX"""') derex_runner_settings_dir = abspath_from_egg( "derex.runner", "derex/runner/settings/README.rst").parent self.update_default_settings(derex_runner_settings_dir, self.settings_dir)
class Project: """Represents a derex.runner project, i.e. a directory with a `derex.config.yaml` file and optionally a "themes", "settings" and "requirements" directory. The directory is inspected on object instantiation: changes will not be automatically picked up unless a new object is created. The project root directory can be passed in the `path` parameter, and defaults to the current directory. If files needed by derex outside of its private `.derex.` dir are missing they will be created, unless the `read_only` parameter is set to True. """ #: The root path to this project root: Path #: The name of the base image with dev goodies and precompiled assets base_image: str # Tne image name of the base image for the final production project build final_base_image: str # The named version of Open edX to use openedx_version: "OpenEdXVersions" #: The directory containing requirements, if defined requirements_dir: Optional[Path] = None #: The directory containing themes, if defined themes_dir: Optional[Path] = None # The directory containing project settings (that feed django.conf.settings) settings_dir: Optional[Path] = None # The directory containing project database fixtures (used on --reset-mysql) fixtures_dir: Optional[Path] = None # The directory where plugins can store their custom requirements, settings, # fixtures and themes. plugins_dir: Optional[Path] = None # The image name of the image that includes requirements requirements_image_name: str # The image name of the image that includes requirements and themes themes_image_name: str # The image name of the final image containing everything needed for this project image_name: str # Image prefix to construct the above image names if they're not specified. # Can include a private docker name, like registry.example.com/onlinecourses/edx-ironwood image_prefix: str # Path to a local docker-compose.yml file, if present local_compose: Optional[Path] = None # Volumes to mount the requirements. In case this is not None the requirements # directory will not be mounted directly, but this dictionary will be used. # Keys are paths on the host system and values are path inside the container requirements_volumes: Optional[Dict[str, str]] = None # Enum containing possible settings modules _available_settings = None _derex_django_path = abspath_from_egg("derex.runner", "derex_django/README.rst").parent @property def mysql_db_name(self) -> str: return self.config.get("mysql_db_name", f"{self.name}_openedx") @property def mongodb_db_name(self) -> str: return self.config.get("mongodb_db_name", f"{self.name}_openedx") @property def runmode(self) -> ProjectRunMode: """The run mode of this project, either debug or production. In debug mode django's runserver is used. Templates are reloaded on every request and assets do not need to be collected. In production mode gunicorn is run, and assets need to be compiled and collected. """ name = "runmode" mode_str = self._get_status(name) if mode_str is not None: if mode_str in ProjectRunMode.__members__: return ProjectRunMode[mode_str] # We found a string but we don't recognize it: warn the user logger.warning( f"Value `{mode_str}` found in `{self.private_filepath(name)}` " "is not valid as runmode " "(valid values are `debug` and `production`)") default = self.config.get(f"default_{name}") if default: if default not in ProjectRunMode.__members__: logger.warning( f"Value `{default}` found in config `{self.root / CONF_FILENAME}` " "is not a valid default for runmode " "(valid values are `debug` and `production`)") else: return ProjectRunMode[default] return next(iter(ProjectRunMode)) # Return the first by default @runmode.setter def runmode(self, value: ProjectRunMode): self._set_status("runmode", value.name) @property def settings(self): """Name of the module to use as DJANGO_SETTINGS_MODULE """ current_status = self._get_status("settings", "base") return self.get_available_settings()[current_status] @settings.setter def settings(self, value: IntEnum): self._set_status("settings", value.name) def settings_directory_path(self) -> Path: """Return an absolute path that will be mounted under lms/envs/derex_project and cms/envs/derex_project inside the container. If the project has local settings, we use that directory. Otherwise we use the directory bundled with `derex.runner` """ if self.settings_dir is not None: return self.settings_dir return abspath_from_egg("derex.runner", "derex/runner/settings/derex/base.py").parent def _get_status(self, name: str, default: Optional[str] = None) -> Optional[str]: """Read value for the desired status from the project directory. """ filepath = self.private_filepath(name) if filepath.exists(): return filepath.read_text() return default def _set_status(self, name: str, value: str): """Persist a status in the project directory. Each status will be written to a different file. """ if not self.private_filepath(name).parent.exists(): self.private_filepath(name).parent.mkdir() self.private_filepath(name).write_text(value) def private_filepath(self, name: str) -> Path: """Return the full file path to `name` rooted from the project private dir ".derex". >>> Project().private_filepath("filename.txt") "/path/to/project/.derex/filename.txt" """ return self.root / DEREX_RUNNER_PROJECT_DIR / name def __init__(self, path: Union[Path, str] = None, read_only: bool = False): logger.debug("Creating project object") # Load first, and only afterwards manipulate the folder # so that if an error occurs during loading we bail wout # before making any change self._load(path) if not read_only: self._populate_settings() if not (self.root / DEREX_RUNNER_PROJECT_DIR).exists(): (self.root / DEREX_RUNNER_PROJECT_DIR).mkdir() def _load(self, path: Union[Path, str] = None): """Load project configuraton from the given directory. """ if not path: path = os.getcwd() self.root = find_project_root(Path(path)) config_path = self.root / CONF_FILENAME self.config = yaml.load(config_path.open(), Loader=yaml.FullLoader) self.openedx_version = OpenEdXVersions[self.config.get( "openedx_version", "ironwood")] source_image_prefix = self.openedx_version.value["docker_image_prefix"] self.base_image = self.config.get( "base_image", f"{source_image_prefix}-dev:{__version__}") self.final_base_image = self.config.get( "final_base_image", f"{source_image_prefix}-nostatic:{__version__}") if "project_name" not in self.config: raise ValueError( f"A project_name was not specified in {config_path}") self.name = self.config["project_name"] if not re.search("^[0-9a-zA-Z-]+$", self.name): raise ValueError( f"`{self.name}` is not a valid name: A project_name can only contain letters, numbers and dashes" ) self.image_prefix = self.config.get("image_prefix", f"{self.name}/openedx") local_compose = self.root / "docker-compose.yml" if local_compose.is_file(): self.local_compose = local_compose requirements_dir = self.root / "requirements" if requirements_dir.is_dir(): self.requirements_dir = requirements_dir # We only hash text files inside the requirements image: # this way changes to code can be made effective by # mounting the requirements directory img_hash = get_requirements_hash(self.requirements_dir) self.requirements_image_name = ( f"{self.image_prefix}-requirements:{img_hash[:6]}") requirements_volumes: Dict[str, str] = {} # If the requirements directory contains any symlink we mount # their targets individually instead of the whole requirements directory for el in self.requirements_dir.iterdir(): if el.is_symlink(): self.requirements_volumes = requirements_volumes requirements_volumes[str( el.resolve())] = ("/openedx/derex.requirements/" + el.name) else: self.requirements_image_name = self.base_image themes_dir = self.root / "themes" if themes_dir.is_dir(): self.themes_dir = themes_dir img_hash = get_dir_hash( self.themes_dir ) # XXX some files are generated. We should ignore them when we hash the directory self.themes_image_name = f"{self.image_prefix}-themes:{img_hash[:6]}" else: self.themes_image_name = self.requirements_image_name settings_dir = self.root / "settings" if settings_dir.is_dir(): self.settings_dir = settings_dir # TODO: run some sanity checks on the settings dir and raise an # exception if they fail fixtures_dir = self.root / "fixtures" if fixtures_dir.is_dir(): self.fixtures_dir = fixtures_dir plugins_dir = self.root / "plugins" if plugins_dir.is_dir(): self.plugins_dir = plugins_dir self.image_name = self.themes_image_name def update_default_settings(self, default_settings_dir, destination_settings_dir): """Update default settings in a specified directory. Given a directory where to look for default settings modules recursively copy or update them into the destination directory. Additionally add a warning asking not to manually edit files. If files needs to be overwritten, print a diff. """ for source in default_settings_dir.glob("**/*.py"): destination = destination_settings_dir / source.relative_to( default_settings_dir) new_text = ("# DO NOT EDIT THIS FILE!\n" "# IT CAN BE OVERWRITTEN ON UPGRADE.\n" f"# Generated by derex.runner {__version__}\n\n" f"{source.read_text()}") if destination.is_file(): old_text = destination.read_text() if old_text != new_text: logger.warn( f"Replacing file {destination} with newer version") diff = tuple( difflib.unified_diff( old_text.splitlines(keepends=True), new_text.splitlines(keepends=True), )) logger.warn("".join(diff)) else: if not destination.parent.is_dir(): destination.parent.mkdir(parents=True) try: destination.write_text(new_text) except PermissionError: current_mode = stat.S_IMODE(os.lstat(destination).st_mode) # XXX Remove me: older versions of derex set a non-writable permission # for their files. This except branch is needed now (Easter 2020), but # when the pandemic is over we can probably remove it destination.chmod(current_mode | 0o700) destination.write_text(new_text) def _populate_settings(self): """If the project includes user defined settings, add ours to that directory to let the project's settings use the line from .derex import * Also add a base.py file with the above content if it does not exist. """ if self.settings_dir is None: return base_settings = self.settings_dir / "base.py" if not base_settings.is_file(): base_settings.write_text("from .derex import *\n") init = self.settings_dir / "__init__.py" if not init.is_file(): init.write_text('"""Settings for edX"""') derex_runner_settings_dir = abspath_from_egg( "derex.runner", "derex/runner/settings/README.rst").parent self.update_default_settings(derex_runner_settings_dir, self.settings_dir) def get_plugin_directories(self, plugin: str) -> Dict[str, Path]: """ Return a dictionary filled with paths to existing directories for custom requirements, settings, fixtures and themes for a plugin. """ plugin_directories = {} if self.plugins_dir: plugin_dir = self.plugins_dir / plugin if plugin_dir.exists(): for directory in [ "settings", "requirements", "fixtures", "themes" ]: if (plugin_dir / directory).exists(): plugin_directories[directory] = plugin_dir / directory return plugin_directories def get_available_settings(self): """Return an Enum object that includes possible settings for this project. This enum must be dynamic, since it depends on the contents of the project settings directory. For this reason we use the functional API for python Enum, which means we're limited to IntEnums. For this reason we'll be using `settings.name` instead of `settings.value` throughout the code. """ if self._available_settings is not None: return self._available_settings if self.settings_dir is None: available_settings = IntEnum("settings", "base") else: settings_names = [] for file in self.settings_dir.iterdir(): if file.suffix == ".py" and file.stem != "__init__": settings_names.append(file.stem) available_settings = IntEnum("settings", " ".join(settings_names)) self._available_settings = available_settings return available_settings def get_container_env(self): """Return a dictionary to be used as environment variables for all containers in this project. Variables are looked up inside the config according to the current settings for the project. """ settings = self.settings.name result = {} variables = self.config.get("variables", {}) for variable in variables: value = variables[variable][settings] if not isinstance(value, str): result[f"DEREX_JSON_{variable.upper()}"] = json.dumps(value) else: result[f"DEREX_{variable.upper()}"] = value return result def secret(self, name: str) -> str: return get_secret(DerexSecrets[name])
def generate_local_docker_compose(project: Project) -> Path: """TODO: Interim function waiting to be refactored into derex.runner """ local_compose_path = project.private_filepath( "docker-compose-discovery.yml") template_compose_path = abspath_from_egg( "derex.discovery", "derex/discovery/docker-compose-discovery.yml.j2") plugin_directories = project.get_plugin_directories(__package__) our_settings_dir = abspath_from_egg( "derex.discovery", "derex/discovery/settings/README.rst").parent settings_dir = our_settings_dir / "derex" active_settings = "base" if plugin_directories.get("settings"): settings_dir = plugin_directories.get("settings") if (plugin_directories.get("settings") / "{}.py".format(project.settings.name)).exists(): active_settings = project.settings.name else: logger.warning( f"{project.settings.name} settings module not found for {__package__} plugin. " "Running with default settings.") # Write out default read-only settings file # if they are not present base_settings = settings_dir / "base.py" if not base_settings.is_file(): base_settings.write_text("from .derex import *\n") init = settings_dir / "__init__.py" if not init.is_file(): init.write_text('"""Settings for edX Discovery Service"""') for source_code in our_settings_dir.glob("**/*.py"): destination = settings_dir / source_code.relative_to( our_settings_dir) if (destination.is_file() and destination.read_text() != source_code.read_text()): # TODO: Replace this warning with a call to a derex.runner # function which should take care of updating default settings logger.warning( f"WARNING: Default settings modified at {destination}. Replacing " ) if not destination.parent.is_dir(): destination.parent.mkdir(parents=True) try: destination.write_text(source_code.read_text()) except PermissionError: current_mode = stat.S_IMODE(os.lstat(destination).st_mode) # XXX Remove me: older versions of derex set a non-writable permission # for their files. This except branch is needed now (Easter 2020), but # when the pandemic is over we can probably remove it destination.chmod(current_mode | 0o700) destination.write_text(source_code.read_text()) tmpl = Template(template_compose_path.read_text()) text = tmpl.render( project=project, plugins_dirs=plugin_directories, settings_dir=settings_dir, active_settings=active_settings, ) local_compose_path.write_text(text) return local_compose_path