def reset_mysql(project):
"""Reset the discovery mysql database"""
from derex.runner.docker import check_services
from derex.runner.docker import wait_for_mysql
if project.runmode is not ProjectRunMode.debug:
click.get_current_context().fail(
"This command can only be run in `debug` runmode")
if not check_services(["mysql"]):
click.echo(
"Mysql service not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
wait_for_mysql()
restore_dump_path = abspath_from_egg("derex.discovery",
"derex/discovery/restore_dump.py")
run_compose(
[
"run",
"--rm",
"-v",
f"{restore_dump_path}:/openedx/discovery/restore_dump.py",
"discovery",
"python",
"/openedx/discovery/restore_dump.py",
],
project=project,
)
return 0
def reset_mysql(project: Project, dry_run: bool = False):
"""Run script from derex/openedx image to reset the mysql db.
"""
if project.runmode is not ProjectRunMode.debug:
click.get_current_context().fail(
"The command reset-mysql can only be run in `debug` runmode")
logger.warning("Resetting mysql database")
restore_dump_path = abspath_from_egg(
"derex.runner", "derex/runner/restore_dump.py.source")
assert (restore_dump_path
), "Could not find restore_dump.py in derex.runner distribution"
run_compose(
[
"run",
"--rm",
"-v",
f"{restore_dump_path}:/restore_dump.py",
"lms",
"python",
"/restore_dump.py",
],
project=project,
dry_run=dry_run,
)
def openedx(version, target, push, only_print_image_name, docker_opts):
"""Build openedx image using docker. Defaults to dev image target."""
dockerdir = abspath_from_egg("derex.runner",
"docker-definition/Dockerfile").parent
build_arguments = []
for spec in version.value.items():
build_arguments.append("--build-arg")
build_arguments.append(f"{spec[0].upper()}={spec[1]}")
docker_image_prefix = version.value["docker_image_prefix"]
image_name = f"{docker_image_prefix}-{target}:{__version__}"
if only_print_image_name:
click.echo(image_name)
return
push_arg = ",push=true" if push else ""
command = [
"docker",
"buildx",
"build",
str(dockerdir),
"-t",
image_name,
*build_arguments,
f"--target={target}",
]
transifex_path = os.path.expanduser("~/.transifexrc")
if os.path.exists(transifex_path):
command.extend(["--secret", f"id=transifex,src={transifex_path}"])
if docker_opts:
command.extend(docker_opts.format(**locals()).split())
print("Invoking\n" + " ".join(command), file=sys.stderr)
os.execve(find_executable(command[0]), command, os.environ)
def _populate_settings(self):
"""If the project includes user defined settings, add ours to that directory
to let the project's settings use the line
from .derex import *
Also add a base.py file with the above content if it does not exist.
"""
if self.settings_dir is None:
return
base_settings = self.settings_dir / "base.py"
if not base_settings.is_file():
base_settings.write_text("from .derex import *\n")
init = self.settings_dir / "__init__.py"
if not init.is_file():
init.write_text('"""Settings for edX"""')
our_settings_dir = abspath_from_egg(
"derex.runner", "derex/runner/settings/README.rst").parent
for source in our_settings_dir.glob("**/*.py"):
destination = self.settings_dir / source.relative_to(
our_settings_dir)
if destination.is_file():
if destination.read_text() != source.read_text():
raise SettingsModified(filename=destination)
else:
if not destination.parent.is_dir():
destination.parent.mkdir(parents=True)
destination.write_text(source.read_text())
destination.chmod(0o444)
def openedx(version, target, push, only_print_image_name, docker_opts):
"""Build openedx image using docker. Defaults to dev image target."""
dockerdir = abspath_from_egg("derex.runner", "docker-definition/Dockerfile").parent
git_repo = version.value["git_repo"]
git_branch = version.value["git_branch"]
python_version = version.value.get("python_version", "3.6")
docker_image_prefix = version.value["docker_image_prefix"]
image_name = f"{docker_image_prefix}-{target}:{__version__}"
if only_print_image_name:
click.echo(image_name)
return
push_arg = ",push=true" if push else ""
command = [
"docker",
"buildx",
"build",
str(dockerdir),
"-t",
image_name,
"--build-arg",
f"PYTHON_VERSION={python_version}",
"--build-arg",
f"EDX_PLATFORM_VERSION={git_branch}",
"--build-arg",
f"EDX_PLATFORM_REPOSITORY={git_repo}",
f"--target={target}",
]
transifex_path = os.path.expanduser("~/.transifexrc")
if os.path.exists(transifex_path):
command.extend(["--secret", f"id=transifex,src={transifex_path}"])
if docker_opts:
command.extend(docker_opts.format(**locals()).split())
print("Invoking\n" + " ".join(command), file=sys.stderr)
os.execve(find_executable(command[0]), command, os.environ)
def settings_directory_path(self) -> Path:
"""Return an absolute path that will be mounted under
lms/envs/derex_project and cms/envs/derex_project inside the
container.
If the project has local settings, we use that directory.
Otherwise we use the directory bundled with `derex.runner`
"""
if self.settings_dir is not None:
return self.settings_dir
return abspath_from_egg("derex.runner",
"derex/runner/settings/derex/base.py").parent
def compose_options() -> Dict[str, Union[str, List[str]]]:
"""See derex.runner.plugin_spec.compose_options docstring.
"""
options = [
"--project-name",
"derex_services",
"-f",
str(
abspath_from_egg("derex.runner",
"derex/runner/compose_files/services.yml")),
]
if asbool(os.environ.get("DEREX_ADMIN_SERVICES", True)):
options += [
"-f",
str(
abspath_from_egg("derex.runner",
"derex/runner/compose_files/admin.yml")),
]
return {
"options": options,
"name": "base",
"priority": "_begin",
"variant": "services",
}
def load_dump(relpath):
"""Loads a mysql dump into the derex mysql database.
"""
dump_path = abspath_from_egg("derex.runner", relpath)
image = client.containers.get("mysql").image
logger.info("Resetting email database")
try:
client.containers.run(
image.tags[0],
["sh", "-c", f"mysql -h mysql -psecret < /dump/{dump_path.name}"],
network="derex",
volumes={dump_path.parent: {
"bind": "/dump"
}},
auto_remove=True,
)
except docker.errors.ContainerError as exc:
logger.exception(exc)
def generate_local_docker_compose(project: Project) -> Path:
"""This function is called every time ddc-project is run.
It assembles a docker-compose file from the given configuration.
It should execute as fast as possible.
"""
local_compose_path = project.private_filepath("docker-compose.yml")
template_path = abspath_from_egg("derex.runner",
"derex/runner/templates/local.yml.j2")
final_image = None
if image_exists(project.image_tag):
final_image = project.image_tag
if not image_exists(project.requirements_image_tag):
click.echo("Building requirements image")
build_requirements_image(project)
tmpl = Template(template_path.read_text())
text = tmpl.render(project=project, final_image=final_image)
local_compose_path.write_text(text)
return local_compose_path
def reset_mysql_openedx(project: Project, dry_run: bool = False):
"""Run script from derex/openedx image to reset the mysql db.
"""
restore_dump_path = abspath_from_egg(
"derex.runner", "derex/runner/restore_dump.py.source")
assert (restore_dump_path
), "Could not find restore_dump.py in derex.runner distribution"
run_compose(
[
"run",
"--rm",
"-v",
f"{restore_dump_path}:/restore_dump.py",
"lms",
"python",
"/restore_dump.py",
],
project=project,
dry_run=dry_run,
)
def forum_build(version, push, only_print_image_name, docker_opts):
# TODO: generalize derex.runner build.openedx function so to
# take the dockerfile_dir and secrets as arguments.
# This way we can avoid repeating code here and just call
# the build function with our arguments.
# e.g.
# return derex_build_cli.build(
# dockerfile_dir,
# version,
# push,
# only_print_image_name,
# docker_opts,
# secrets
# )
"""Build openedx image using docker. Defaults to dev image target."""
dockerfile_dir = abspath_from_egg("derex.forum",
"docker_build/Dockerfile").parent
build_arguments = []
for spec in version.value.items():
build_arguments.append("--build-arg")
build_arguments.append(f"{spec[0].upper()}={spec[1]}")
docker_image_prefix = version.value["docker_image_prefix"]
image_name = f"{docker_image_prefix}:{__version__}"
if only_print_image_name:
click.echo(image_name)
return
push_arg = ",push=true" if push else ""
command = [
"docker",
"buildx",
"build",
str(dockerfile_dir),
"-t",
image_name,
*build_arguments,
]
if docker_opts:
command.extend(docker_opts.format(**locals()).split())
print("Invoking\n" + " ".join(command), file=sys.stderr)
os.execve(find_executable(command[0]), command, os.environ)
def load_fixtures(project):
"""Load fixtures from the plugin fixtures directory"""
from derex.runner.compose_utils import run_compose
fixtures_dir = project.get_plugin_directories(__package__).get("fixtures")
if fixtures_dir is None:
click.echo("No fixtures directory present for this plugin")
return
load_fixtures_path = abspath_from_egg("derex.discovery",
"derex/discovery/load_fixtures.py")
compose_args = [
"run",
"--rm",
"-v",
f"{load_fixtures_path}:/openedx/discovery/load_fixtures.py",
"discovery",
"python",
"/openedx/discovery/load_fixtures.py",
]
run_compose(compose_args, project=project)
return
def _populate_settings(self):
"""If the project includes user defined settings, add ours to that directory
to let the project's settings use the line
from .derex import *
Also add a base.py file with the above content if it does not exist.
"""
if self.settings_dir is None:
return
base_settings = self.settings_dir / "base.py"
if not base_settings.is_file():
base_settings.write_text("from .derex import *\n")
init = self.settings_dir / "__init__.py"
if not init.is_file():
init.write_text('"""Settings for edX"""')
derex_runner_settings_dir = abspath_from_egg(
"derex.runner", "derex/runner/settings/README.rst").parent
self.update_default_settings(derex_runner_settings_dir,
self.settings_dir)
class Project:
"""Represents a derex.runner project, i.e. a directory with a
`derex.config.yaml` file and optionally a "themes", "settings" and
"requirements" directory.
The directory is inspected on object instantiation: changes will not
be automatically picked up unless a new object is created.
The project root directory can be passed in the `path` parameter, and
defaults to the current directory.
If files needed by derex outside of its private `.derex.` dir are missing
they will be created, unless the `read_only` parameter is set to True.
"""
#: The root path to this project
root: Path
#: The name of the base image with dev goodies and precompiled assets
base_image: str
# Tne image name of the base image for the final production project build
final_base_image: str
# The named version of Open edX to use
openedx_version: "OpenEdXVersions"
#: The directory containing requirements, if defined
requirements_dir: Optional[Path] = None
#: The directory containing themes, if defined
themes_dir: Optional[Path] = None
# The directory containing project settings (that feed django.conf.settings)
settings_dir: Optional[Path] = None
# The directory containing project database fixtures (used on --reset-mysql)
fixtures_dir: Optional[Path] = None
# The directory where plugins can store their custom requirements, settings,
# fixtures and themes.
plugins_dir: Optional[Path] = None
# The image name of the image that includes requirements
requirements_image_name: str
# The image name of the image that includes requirements and themes
themes_image_name: str
# The image name of the final image containing everything needed for this project
image_name: str
# Image prefix to construct the above image names if they're not specified.
# Can include a private docker name, like registry.example.com/onlinecourses/edx-ironwood
image_prefix: str
# Path to a local docker-compose.yml file, if present
local_compose: Optional[Path] = None
# Volumes to mount the requirements. In case this is not None the requirements
# directory will not be mounted directly, but this dictionary will be used.
# Keys are paths on the host system and values are path inside the container
requirements_volumes: Optional[Dict[str, str]] = None
# Enum containing possible settings modules
_available_settings = None
_derex_django_path = abspath_from_egg("derex.runner",
"derex_django/README.rst").parent
@property
def mysql_db_name(self) -> str:
return self.config.get("mysql_db_name", f"{self.name}_openedx")
@property
def mongodb_db_name(self) -> str:
return self.config.get("mongodb_db_name", f"{self.name}_openedx")
@property
def runmode(self) -> ProjectRunMode:
"""The run mode of this project, either debug or production.
In debug mode django's runserver is used. Templates are reloaded
on every request and assets do not need to be collected.
In production mode gunicorn is run, and assets need to be compiled and collected.
"""
name = "runmode"
mode_str = self._get_status(name)
if mode_str is not None:
if mode_str in ProjectRunMode.__members__:
return ProjectRunMode[mode_str]
# We found a string but we don't recognize it: warn the user
logger.warning(
f"Value `{mode_str}` found in `{self.private_filepath(name)}` "
"is not valid as runmode "
"(valid values are `debug` and `production`)")
default = self.config.get(f"default_{name}")
if default:
if default not in ProjectRunMode.__members__:
logger.warning(
f"Value `{default}` found in config `{self.root / CONF_FILENAME}` "
"is not a valid default for runmode "
"(valid values are `debug` and `production`)")
else:
return ProjectRunMode[default]
return next(iter(ProjectRunMode)) # Return the first by default
@runmode.setter
def runmode(self, value: ProjectRunMode):
self._set_status("runmode", value.name)
@property
def settings(self):
"""Name of the module to use as DJANGO_SETTINGS_MODULE
"""
current_status = self._get_status("settings", "base")
return self.get_available_settings()[current_status]
@settings.setter
def settings(self, value: IntEnum):
self._set_status("settings", value.name)
def settings_directory_path(self) -> Path:
"""Return an absolute path that will be mounted under
lms/envs/derex_project and cms/envs/derex_project inside the
container.
If the project has local settings, we use that directory.
Otherwise we use the directory bundled with `derex.runner`
"""
if self.settings_dir is not None:
return self.settings_dir
return abspath_from_egg("derex.runner",
"derex/runner/settings/derex/base.py").parent
def _get_status(self,
name: str,
default: Optional[str] = None) -> Optional[str]:
"""Read value for the desired status from the project directory.
"""
filepath = self.private_filepath(name)
if filepath.exists():
return filepath.read_text()
return default
def _set_status(self, name: str, value: str):
"""Persist a status in the project directory.
Each status will be written to a different file.
"""
if not self.private_filepath(name).parent.exists():
self.private_filepath(name).parent.mkdir()
self.private_filepath(name).write_text(value)
def private_filepath(self, name: str) -> Path:
"""Return the full file path to `name` rooted from the
project private dir ".derex".
>>> Project().private_filepath("filename.txt")
"/path/to/project/.derex/filename.txt"
"""
return self.root / DEREX_RUNNER_PROJECT_DIR / name
def __init__(self, path: Union[Path, str] = None, read_only: bool = False):
logger.debug("Creating project object")
# Load first, and only afterwards manipulate the folder
# so that if an error occurs during loading we bail wout
# before making any change
self._load(path)
if not read_only:
self._populate_settings()
if not (self.root / DEREX_RUNNER_PROJECT_DIR).exists():
(self.root / DEREX_RUNNER_PROJECT_DIR).mkdir()
def _load(self, path: Union[Path, str] = None):
"""Load project configuraton from the given directory.
"""
if not path:
path = os.getcwd()
self.root = find_project_root(Path(path))
config_path = self.root / CONF_FILENAME
self.config = yaml.load(config_path.open(), Loader=yaml.FullLoader)
self.openedx_version = OpenEdXVersions[self.config.get(
"openedx_version", "ironwood")]
source_image_prefix = self.openedx_version.value["docker_image_prefix"]
self.base_image = self.config.get(
"base_image", f"{source_image_prefix}-dev:{__version__}")
self.final_base_image = self.config.get(
"final_base_image",
f"{source_image_prefix}-nostatic:{__version__}")
if "project_name" not in self.config:
raise ValueError(
f"A project_name was not specified in {config_path}")
self.name = self.config["project_name"]
if not re.search("^[0-9a-zA-Z-]+$", self.name):
raise ValueError(
f"`{self.name}` is not a valid name: A project_name can only contain letters, numbers and dashes"
)
self.image_prefix = self.config.get("image_prefix",
f"{self.name}/openedx")
local_compose = self.root / "docker-compose.yml"
if local_compose.is_file():
self.local_compose = local_compose
requirements_dir = self.root / "requirements"
if requirements_dir.is_dir():
self.requirements_dir = requirements_dir
# We only hash text files inside the requirements image:
# this way changes to code can be made effective by
# mounting the requirements directory
img_hash = get_requirements_hash(self.requirements_dir)
self.requirements_image_name = (
f"{self.image_prefix}-requirements:{img_hash[:6]}")
requirements_volumes: Dict[str, str] = {}
# If the requirements directory contains any symlink we mount
# their targets individually instead of the whole requirements directory
for el in self.requirements_dir.iterdir():
if el.is_symlink():
self.requirements_volumes = requirements_volumes
requirements_volumes[str(
el.resolve())] = ("/openedx/derex.requirements/" + el.name)
else:
self.requirements_image_name = self.base_image
themes_dir = self.root / "themes"
if themes_dir.is_dir():
self.themes_dir = themes_dir
img_hash = get_dir_hash(
self.themes_dir
) # XXX some files are generated. We should ignore them when we hash the directory
self.themes_image_name = f"{self.image_prefix}-themes:{img_hash[:6]}"
else:
self.themes_image_name = self.requirements_image_name
settings_dir = self.root / "settings"
if settings_dir.is_dir():
self.settings_dir = settings_dir
# TODO: run some sanity checks on the settings dir and raise an
# exception if they fail
fixtures_dir = self.root / "fixtures"
if fixtures_dir.is_dir():
self.fixtures_dir = fixtures_dir
plugins_dir = self.root / "plugins"
if plugins_dir.is_dir():
self.plugins_dir = plugins_dir
self.image_name = self.themes_image_name
def update_default_settings(self, default_settings_dir,
destination_settings_dir):
"""Update default settings in a specified directory.
Given a directory where to look for default settings modules recursively
copy or update them into the destination directory.
Additionally add a warning asking not to manually edit files.
If files needs to be overwritten, print a diff.
"""
for source in default_settings_dir.glob("**/*.py"):
destination = destination_settings_dir / source.relative_to(
default_settings_dir)
new_text = ("# DO NOT EDIT THIS FILE!\n"
"# IT CAN BE OVERWRITTEN ON UPGRADE.\n"
f"# Generated by derex.runner {__version__}\n\n"
f"{source.read_text()}")
if destination.is_file():
old_text = destination.read_text()
if old_text != new_text:
logger.warn(
f"Replacing file {destination} with newer version")
diff = tuple(
difflib.unified_diff(
old_text.splitlines(keepends=True),
new_text.splitlines(keepends=True),
))
logger.warn("".join(diff))
else:
if not destination.parent.is_dir():
destination.parent.mkdir(parents=True)
try:
destination.write_text(new_text)
except PermissionError:
current_mode = stat.S_IMODE(os.lstat(destination).st_mode)
# XXX Remove me: older versions of derex set a non-writable permission
# for their files. This except branch is needed now (Easter 2020), but
# when the pandemic is over we can probably remove it
destination.chmod(current_mode | 0o700)
destination.write_text(new_text)
def _populate_settings(self):
"""If the project includes user defined settings, add ours to that directory
to let the project's settings use the line
from .derex import *
Also add a base.py file with the above content if it does not exist.
"""
if self.settings_dir is None:
return
base_settings = self.settings_dir / "base.py"
if not base_settings.is_file():
base_settings.write_text("from .derex import *\n")
init = self.settings_dir / "__init__.py"
if not init.is_file():
init.write_text('"""Settings for edX"""')
derex_runner_settings_dir = abspath_from_egg(
"derex.runner", "derex/runner/settings/README.rst").parent
self.update_default_settings(derex_runner_settings_dir,
self.settings_dir)
def get_plugin_directories(self, plugin: str) -> Dict[str, Path]:
"""
Return a dictionary filled with paths to existing directories
for custom requirements, settings, fixtures and themes for
a plugin.
"""
plugin_directories = {}
if self.plugins_dir:
plugin_dir = self.plugins_dir / plugin
if plugin_dir.exists():
for directory in [
"settings", "requirements", "fixtures", "themes"
]:
if (plugin_dir / directory).exists():
plugin_directories[directory] = plugin_dir / directory
return plugin_directories
def get_available_settings(self):
"""Return an Enum object that includes possible settings for this project.
This enum must be dynamic, since it depends on the contents of the project
settings directory.
For this reason we use the functional API for python Enum, which means we're
limited to IntEnums. For this reason we'll be using `settings.name` instead
of `settings.value` throughout the code.
"""
if self._available_settings is not None:
return self._available_settings
if self.settings_dir is None:
available_settings = IntEnum("settings", "base")
else:
settings_names = []
for file in self.settings_dir.iterdir():
if file.suffix == ".py" and file.stem != "__init__":
settings_names.append(file.stem)
available_settings = IntEnum("settings", " ".join(settings_names))
self._available_settings = available_settings
return available_settings
def get_container_env(self):
"""Return a dictionary to be used as environment variables for all containers
in this project. Variables are looked up inside the config according to
the current settings for the project.
"""
settings = self.settings.name
result = {}
variables = self.config.get("variables", {})
for variable in variables:
value = variables[variable][settings]
if not isinstance(value, str):
result[f"DEREX_JSON_{variable.upper()}"] = json.dumps(value)
else:
result[f"DEREX_{variable.upper()}"] = value
return result
def secret(self, name: str) -> str:
return get_secret(DerexSecrets[name])
def generate_local_docker_compose(project: Project) -> Path:
"""TODO: Interim function waiting to be refactored into derex.runner
"""
local_compose_path = project.private_filepath(
"docker-compose-discovery.yml")
template_compose_path = abspath_from_egg(
"derex.discovery", "derex/discovery/docker-compose-discovery.yml.j2")
plugin_directories = project.get_plugin_directories(__package__)
our_settings_dir = abspath_from_egg(
"derex.discovery", "derex/discovery/settings/README.rst").parent
settings_dir = our_settings_dir / "derex"
active_settings = "base"
if plugin_directories.get("settings"):
settings_dir = plugin_directories.get("settings")
if (plugin_directories.get("settings") /
"{}.py".format(project.settings.name)).exists():
active_settings = project.settings.name
else:
logger.warning(
f"{project.settings.name} settings module not found for {__package__} plugin. "
"Running with default settings.")
# Write out default read-only settings file
# if they are not present
base_settings = settings_dir / "base.py"
if not base_settings.is_file():
base_settings.write_text("from .derex import *\n")
init = settings_dir / "__init__.py"
if not init.is_file():
init.write_text('"""Settings for edX Discovery Service"""')
for source_code in our_settings_dir.glob("**/*.py"):
destination = settings_dir / source_code.relative_to(
our_settings_dir)
if (destination.is_file()
and destination.read_text() != source_code.read_text()):
# TODO: Replace this warning with a call to a derex.runner
# function which should take care of updating default settings
logger.warning(
f"WARNING: Default settings modified at {destination}. Replacing "
)
if not destination.parent.is_dir():
destination.parent.mkdir(parents=True)
try:
destination.write_text(source_code.read_text())
except PermissionError:
current_mode = stat.S_IMODE(os.lstat(destination).st_mode)
# XXX Remove me: older versions of derex set a non-writable permission
# for their files. This except branch is needed now (Easter 2020), but
# when the pandemic is over we can probably remove it
destination.chmod(current_mode | 0o700)
destination.write_text(source_code.read_text())
tmpl = Template(template_compose_path.read_text())
text = tmpl.render(
project=project,
plugins_dirs=plugin_directories,
settings_dir=settings_dir,
active_settings=active_settings,
)
local_compose_path.write_text(text)
return local_compose_path
