def get_available_settings():
"""Return settings available on the current project"""
try:
project = Project()
except (ValueError, SettingsModified):
return None
return project.get_available_settings().__members__
def test_ddc_project_addition(minimal_project, mocker, capsys):
from derex.runner import hookimpl
class CustomAdditional:
@staticmethod
@hookimpl
def ddc_project_options(
project: Project, ) -> Dict[str, Union[str, List[str]]]:
"""See derex.runner.plugin_spec.ddc_project_options docstring
"""
return {
"options": ["custom-additional"],
"name": "custom",
"priority": ">local-derex",
}
with minimal_project:
docker_compose_path = Project().root / "docker-compose.yml"
with docker_compose_path.open("w") as fh:
fh.write("lms:\n image: foobar\n")
project = Project()
run_ddc_project([], project, dry_run=True)
output = capsys.readouterr().out
# The last option should be the path of the user docker compose file for this project
assert output.endswith(f"-f {docker_compose_path}\n")
def test_populate_settings(testproj):
with testproj as projdir:
default_settings_dir = Project().settings_directory_path()
assert default_settings_dir.is_dir()
create_settings_file(Path(projdir), "production")
project = Project(read_only=True)
assert default_settings_dir != project.settings_directory_path()
project._populate_settings()
assert (project.settings_dir / "base.py").is_file(), str(
sorted((project.settings_dir).iterdir())
)
assert (project.settings_dir / "derex").is_dir(), str(
sorted((project.settings_dir).iterdir())
)
base_py = project.settings_dir / "derex" / "base.py"
assert base_py.is_file()
assert (project.settings_dir / "derex" / "__init__.py").is_file()
assert os.access(str(base_py), os.W_OK)
# In case a settings file was already present, it should be overwritten,
# even if it lacks owner write permission.
base_py.write_text("# Changed")
base_py.chmod(0o444)
assert not os.access(str(base_py), os.W_OK)
project._populate_settings()
assert os.access(str(base_py), os.W_OK)
def test_container_variables_json_serialized(minimal_project):
with minimal_project:
conf_file = Project().root / CONF_FILENAME
config = {
"project_name": "minimal",
"variables": {
"ALL_JWT_AUTH": {
"default": {
"JWT_AUDIENCE": "jwt-audience",
"JWT_SECRET_KEY": "jwt-secret",
},
"production": {
"JWT_AUDIENCE": "prod-audience",
"JWT_SECRET_KEY": "prod-secret",
},
},
},
}
conf_file.write_text(yaml.dump(config))
create_settings_file(Project().root, "production")
project = Project()
env = project.get_container_env()
assert "DEREX_JSON_ALL_JWT_AUTH" in env
expected = config["variables"]["ALL_JWT_AUTH"]["default"]
assert expected == json.loads(env["DEREX_JSON_ALL_JWT_AUTH"])
project.settings = project._available_settings.production
env = project.get_container_env()
assert "DEREX_JSON_ALL_JWT_AUTH" in env
expected = config["variables"]["ALL_JWT_AUTH"]["production"]
assert expected == json.loads(env["DEREX_JSON_ALL_JWT_AUTH"])
def test_container_variables_json_serialized(testproj):
with testproj as projdir:
conf_file = Path(projdir) / "derex.config.yaml"
config = {
"project_name": "minimal",
"variables": {
"lms_ALL_JWT_AUTH": {
"base": {
"JWT_AUDIENCE": "jwt-audience",
"JWT_SECRET_KEY": "jwt-secret",
},
"production": {
"JWT_AUDIENCE": "prod-audience",
"JWT_SECRET_KEY": "prod-secret",
},
}
},
}
conf_file.write_text(yaml.dump(config))
create_settings_file(Path(projdir), "production")
project = Project()
env = project.get_container_env()
assert "DEREX_JSON_LMS_ALL_JWT_AUTH" in env
expected = json.loads(env["DEREX_JSON_LMS_ALL_JWT_AUTH"])
assert expected == config["variables"]["lms_ALL_JWT_AUTH"]["base"]
def generate_ddc_project_compose(project: Project) -> Path:
"""This function is called every time ddc-project is run.
It assembles a docker-compose file from the given configuration.
It should execute as fast as possible.
"""
project_compose_path = project.private_filepath("docker-compose.yml")
template_path = DDC_PROJECT_TEMPLATE_PATH
final_image = None
if image_exists(project.image_name):
final_image = project.image_name
if not image_exists(project.requirements_image_name):
logger.warning(f"Image {project.requirements_image_name} not found\n"
"Run\nderex build requirements\n to build it")
openedx_customizations = project.get_openedx_customizations()
tmpl = Template(template_path.read_text())
text = tmpl.render(
project=project,
final_image=final_image,
wsgi_py_path=WSGI_PY_PATH,
derex_django_path=DEREX_DJANGO_PATH,
openedx_customizations=openedx_customizations,
)
project_compose_path.write_text(text)
return project_compose_path
def test_project_name_constraints(minimal_project):
with minimal_project:
project_root = Project().root
conf_file = project_root / CONF_FILENAME
config = {"project_name": ";invalid;"}
conf_file.write_text(yaml.dump(config))
create_settings_file(project_root, "production")
with pytest.raises(ValueError):
Project()
def test_derex_runmode_wrong(minimal_project):
with minimal_project:
project = Project()
# Use low level API to inject invalid value
project._set_status("runmode", "garbage-not-a-valid-runmode")
result = runner.invoke(derex_cli_group, "runmode")
# Ensure presence of error message
assert "garbage-not-a-valid-runmode" in result.stderr
assert "is not valid as" in result.stderr
def test_derex_runmode_wrong(testproj):
from derex.runner.cli import derex
with testproj:
project = Project()
# Use low level API to inject invalid value
project._set_status("runmode", "garbage")
result = runner.invoke(derex, ["runmode"])
# Ensure presence of error message
assert "not valid" in result.stderr
def test_derex_build(workdir, sys_argv):
from derex.runner.cli import derex
from derex.runner.ddc import ddc_project
with workdir(COMPLETE_PROJ):
result = runner.invoke(derex, ["compile-theme"])
assert_result_ok(result)
with sys_argv(["ddc-project", "config"]):
ddc_project()
assert Project().name in result.output
assert os.path.isdir(Project().root / ".derex")
def test_complete_project(workdir):
with workdir(COMPLETE_PROJ / "themes"):
project = Project()
project_loaded_with_path = Project(COMPLETE_PROJ)
assert project.root == project_loaded_with_path.root
assert type(project.config) == dict
assert project.requirements_dir == COMPLETE_PROJ / "requirements"
assert project.themes_dir == COMPLETE_PROJ / "themes"
assert project.name == "complete"
assert project.requirements_image_name == "complete/openedx-requirements:6c92de"
def test_run_django_script(minimal_project):
with minimal_project:
from derex.runner.project import Project
from derex.runner.ddc import run_django_script
result = run_django_script(
Project(),
"import json; print(json.dumps(dict(foo='bar', one=1)))")
assert result == {"foo": "bar", "one": 1}
result = run_django_script(
Project(),
"import json; print('This is not { the JSON you re looking for')")
assert result is None
def test_run_script(testproj):
with testproj:
from derex.runner.project import Project
from derex.runner.compose_utils import run_script
result = run_script(
Project(),
"import json; print(json.dumps(dict(foo='bar', one=1)))")
assert result == {"foo": "bar", "one": 1}
result = run_script(
Project(),
"import json; print('This is not { the JSON you re looking for')")
assert result is None
def test_runmode(minimal_project):
with minimal_project:
project = Project()
# If no default is specified, the value should be debug
assert project.runmode == ProjectRunMode.debug
# If a default value is specified, it should be picked up
with (project.root / CONF_FILENAME).open("a") as fh:
fh.write("default_runmode: production\n")
assert Project().runmode == ProjectRunMode.production
Project().runmode = ProjectRunMode.debug
# Runmode changes should be persisted in the project directory
# and picked up by a second Project instance
assert Project().runmode == ProjectRunMode.debug
def test_derex_compile_theme(workdir_copy, sys_argv):
from derex.runner.cli import derex
with workdir_copy(COMPLETE_PROJ):
result = runner.invoke(derex, ["compile-theme"])
assert_result_ok(result)
assert os.path.isdir(Project().root / ".derex")
def ddc_project():
"""Proxy for docker-compose: writes a docker-compose.yml file with the
configuration of this project, and then run `docker-compose` on it.
You probably want do run `ddc-project up -d` and `ddc-project logs -f`.
"""
check_docker()
setup_logging()
try:
project = Project()
except ValueError:
click.echo("You need to run this command in a derex project")
sys.exit(1)
compose_args, dry_run = ddc_parse_args(sys.argv)
# If trying to start up containers, first check that needed services are running
is_start_cmd = any(param in compose_args for param in ["up", "start"])
if is_start_cmd and not check_services(["mysql", "mongodb", "rabbitmq"]):
click.echo(
"Mysql/mongo/rabbitmq services not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
run_compose(list(compose_args),
project=project,
dry_run=dry_run,
exit_afterwards=True)
def test_docker_compose_addition_per_runmode(testproj, mocker):
from derex.runner.compose_utils import get_compose_options
with testproj:
docker_compose_path = Path(testproj._tmpdir.name) / "docker-compose-debug.yml"
with docker_compose_path.open("w") as fh:
fh.write("lms:\n image: foobar\n")
project = Project()
opts = get_compose_options(args=[], variant="", project=project)
# The last option should be the path of the debug docker compose
assert opts[-1] == str(docker_compose_path)
project.runmode = ProjectRunMode.production
opts = get_compose_options(args=[], variant="", project=project)
# The last option should be the path of the production docker compose file
assert opts[-1] != str(docker_compose_path)
def test_ddc_project_minimal(sys_argv, mocker, minimal_project, capsys):
from derex.runner.ddc import ddc_project
from derex.runner.project import Project
"""Test the open edx ironwood docker compose shortcut."""
# It should check for services to be up before trying to do anything
wait_for_service = mocker.patch("derex.runner.ddc.wait_for_service")
with minimal_project:
for param in ["up", "start"]:
wait_for_service.return_value = 0
wait_for_service.side_effect = None
with sys_argv(["ddc-project", param, "--dry-run"]):
ddc_project()
assert "Would have run" in capsys.readouterr().out
wait_for_service.side_effect = RuntimeError(
"mysql service not found.\n"
"Maybe you forgot to run\n"
"ddc-services up -d")
with sys_argv(["ddc-project", param, "--dry-run"]):
with pytest.raises(SystemExit):
ddc_project()
assert "ddc-services up -d" in capsys.readouterr().out
with sys_argv(["ddc-project", "config"]):
ddc_project()
assert "worker" in capsys.readouterr().out
if Project().openedx_version.name == "juniper":
with sys_argv(["ddc-project", "config"]):
ddc_project()
assert (
"/derex/runner/compose_files/openedx_customizations/juniper/"
in capsys.readouterr().out)
def test_ddc_services(sys_argv, capsys, monkeypatch, complete_project):
"""Test the derex docker compose shortcut."""
from derex.runner.ddc import ddc_services
from derex.runner.project import Project
os.environ["DEREX_ADMIN_SERVICES"] = "False"
with sys_argv(["ddc-services", "config"]):
ddc_services()
output = capsys.readouterr().out
assert "mongodb" in output
assert "adminer" not in output
os.environ["DEREX_ADMIN_SERVICES"] = "True"
with sys_argv(["ddc-services", "config"]):
ddc_services()
output = capsys.readouterr().out
assert "adminer" in output
with complete_project:
monkeypatch.setenv("DEREX_ETC_PATH",
str(Project().root / "derex_etc_dir"))
with sys_argv(["ddc-services", "config"]):
ddc_services()
output = capsys.readouterr().out
assert "my-overridden-secret-password" in output
def derex(ctx):
"""Derex directs edX: commands to manage an Open edX installation
"""
# Optimize --help and bash completion by importing
from derex.runner.project import Project
try:
ctx.obj = Project()
except ProjectNotFound:
pass
except Exception as ex:
logger.error("\n".join(map(str, ex.args)))
sys.exit(1)
if ctx.invoked_subcommand:
return
click.echo(derex.get_help(ctx) + "\n")
from derex.runner.docker_utils import get_exposed_container_names
container_names = get_exposed_container_names()
if not container_names:
return
console = Console()
table = Table(
title="[bold green]These containers are running and exposing an HTTP server on port 80",
box=box.SIMPLE,
)
table.add_column("Name")
for container in container_names:
container = (f"[bold]{container[0]}",) + container[1:]
table.add_row(*container)
console.print(table)
def runmode(project: Project, runmode: Optional[ProjectRunMode], force):
"""Get/set project runmode (debug/production)"""
if runmode is None:
click.echo(project.runmode.name)
else:
if project.runmode == runmode:
click.echo(
f"The current project runmode is already {runmode.name}", err=True
)
return
if not force:
if runmode is ProjectRunMode.production:
if not HAS_MASTER_SECRET:
click.echo(
red("Set a master secret before switching to production"),
err=True,
)
sys.exit(1)
return 1
# We need https://github.com/Santandersecurityresearch/DrHeader/pull/102
# for the return 1 to work, but it's not released yet
previous_runmode = project.runmode
project.runmode = runmode
click.echo(
f"Switched runmode: {previous_runmode.name} → {runmode.name}", err=True
)
def test_docker_compose_addition(testproj, mocker):
from derex.runner import hookimpl
from derex.runner.compose_utils import get_compose_options
from derex.runner.plugins import setup_plugin_manager
class CustomAdditional:
@staticmethod
@hookimpl
def local_compose_options(project: Project) -> Dict[str, Union[str, List[str]]]:
"""See derex.runner.plugin_spec.compose_options docstring
"""
return {
"options": ["custom-additional"],
"name": "custom",
"priority": ">local-derex",
}
with testproj:
docker_compose_path = Path(testproj._tmpdir.name) / "docker-compose.yml"
with docker_compose_path.open("w") as fh:
fh.write("lms:\n image: foobar\n")
project = Project()
mgr = setup_plugin_manager()
mgr.register(CustomAdditional)
mocker.patch(
"derex.runner.compose_utils.setup_plugin_manager", return_value=mgr
)
opts = get_compose_options(args=[], variant="", project=project)
# The last option should be the path of the user docker compose file for this project
assert opts[-1] == str(docker_compose_path)
def test_runmode(testproj):
from derex.runner.utils import CONF_FILENAME
with testproj:
# If no default is specified, the value should be debug
assert Project().runmode == ProjectRunMode.debug
# If a default value is specified, it should be picked up
with (Path(testproj._tmpdir.name) / CONF_FILENAME).open("a") as fh:
fh.write("default_runmode: production\n")
assert Project().runmode == ProjectRunMode.production
Project().runmode = ProjectRunMode.production
# Runmode changes should be persisted in the project directory
# and picked up by a second Project instance
assert Project().runmode == ProjectRunMode.production
def create_settings_file(project_root: Path, filename: str):
"""Create an empty settings file inside the given project"""
settings_dir = project_root / "settings"
if not settings_dir.is_dir():
settings_dir.mkdir()
(settings_dir / "__init__.py").write_text("")
project = Project(read_only=True)
(project.settings_dir / f"{filename}.py").write_text("# Empty file")
def test_project_name_constraints(testproj):
with testproj as projdir:
conf_file = Path(projdir) / "derex.config.yaml"
config = {"project_name": ";invalid;"}
conf_file.write_text(yaml.dump(config))
create_settings_file(Path(projdir), "production")
with pytest.raises(ValueError):
Project()
def test_settings_enum(testproj):
with testproj:
assert Project().settings == Project().get_available_settings().base
create_settings_file(Project().root, "production")
Project().settings = Project().get_available_settings().production
assert Project().settings == Project().get_available_settings().production
def test_get_final_image(mocker):
from derex.runner.config import image_exists
mocker.patch(
"derex.runner.config.docker.APIClient",
return_value=mocker.Mock(images=mocker.Mock(
return_value=DOCKER_DAEMON_IMAGES_RESPONSE)),
)
project = Project(MINIMAL_PROJ)
image_exists(project)
def test_complete_project(workdir, complete_project):
with complete_project:
project_path = Project().root
project_loaded_with_path = Project(project_path)
with workdir(project_path / "themes"):
project = Project()
assert project.root == project_loaded_with_path.root
assert type(project.config) == dict
assert project.requirements_dir == project_path / "requirements"
assert project.themes_dir == project_path / "themes"
assert project.name == f"{project.openedx_version.name}-complete"
if project.openedx_version.name == "ironwood":
assert (project.requirements_image_name ==
f"{project.name}/openedx-requirements:6c92de")
elif project.openedx_version.name == "juniper":
assert (project.requirements_image_name ==
f"{project.name}/openedx-requirements:27703b")
def test_generate_ddc_test_compose(complete_project):
from derex.runner.compose_generation import generate_ddc_test_compose
with complete_project:
project = Project()
compose_file = generate_ddc_test_compose(project)
assert "cypress" in compose_file.read_text()
assert project.name in compose_file.read_text()
assert "/complete/e2e:/e2e" in compose_file.read_text()
def test_settings_enum(minimal_project):
with minimal_project:
assert (Project().settings.value ==
Project().get_available_settings().default.value)
create_settings_file(Project().root, "production")
Project().settings = Project().get_available_settings().production
assert (Project().settings.value ==
Project().get_available_settings().production.value)