def ddc_project():
"""Proxy for docker-compose: writes a docker-compose.yml file with the
configuration of this project, and then run `docker-compose` on it.
You probably want do run `ddc-project up -d` and `ddc-project logs -f`.
"""
check_docker()
setup_logging()
try:
project = Project()
except ValueError:
click.echo("You need to run this command in a derex project")
sys.exit(1)
compose_args, dry_run = ddc_parse_args(sys.argv)
# If trying to start up containers, first check that needed services are running
is_start_cmd = any(param in compose_args for param in ["up", "start"])
if is_start_cmd and not check_services(["mysql", "mongodb", "rabbitmq"]):
click.echo(
"Mysql/mongo/rabbitmq services not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
run_compose(list(compose_args),
project=project,
dry_run=dry_run,
exit_afterwards=True)
def reset_mysql(project):
"""Reset the discovery mysql database"""
from derex.runner.docker import check_services
from derex.runner.docker import wait_for_mysql
if project.runmode is not ProjectRunMode.debug:
click.get_current_context().fail(
"This command can only be run in `debug` runmode")
if not check_services(["mysql"]):
click.echo(
"Mysql service not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
wait_for_mysql()
restore_dump_path = abspath_from_egg("derex.discovery",
"derex/discovery/restore_dump.py")
run_compose(
[
"run",
"--rm",
"-v",
f"{restore_dump_path}:/openedx/discovery/restore_dump.py",
"discovery",
"python",
"/openedx/discovery/restore_dump.py",
],
project=project,
)
return 0
def update_minio(old_key: str):
"""Run minio to re-key data with the new secret. The output is very confusing, but it works.
If you read a red warning and "Rotation complete" at the end, it means rekeying has worked.
If your read your current SecretKey, it means the current credentials are correct and you don't need
to update your keys.
"""
from derex.runner.compose_utils import run_compose
# We need to stop minio after it's done re-keying. To this end, we use the expect package
script = "apk add expect --no-cache "
# We need to make sure the current credentials are not working...
script += ' && expect -c "spawn /usr/bin/minio server /data; expect "Endpoint" { close; exit 1 }"'
# ..but the old ones are
script += f' && if MINIO_SECRET_KEY="{old_key}" expect -c \'spawn /usr/bin/minio server /data; expect "Endpoint" {{ close; exit 1 }}\'; then exit 1; fi'
script += f' && export MINIO_ACCESS_KEY_OLD="$MINIO_ACCESS_KEY" MINIO_SECRET_KEY_OLD="{old_key}"'
expected_string = "Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs"
script += f" && expect -c 'spawn /usr/bin/minio server /data; expect \"{expected_string}\" {{ close; exit 0 }}'"
args = [
"run",
"--rm",
"--entrypoint",
"/bin/sh",
"-T",
"minio",
"-c",
script,
]
run_compose(args, exit_afterwards=True)
click.echo(f"Minio server rekeying finished")
def ddc_services():
"""Derex docker-compose: run docker-compose with additional parameters.
Adds docker compose file paths for services and administrative tools.
If the environment variable DEREX_ADMIN_SERVICES is set to a falsey value,
only the core ones will be started (mysql, mongodb etc) and the nice-to-have
will not (portainer and adminer).
Besides the regular docker-compose options it also accepts the --dry-run
option; in case it's specified docker-compose will not be invoked, but
a line will be printed showing what would have been invoked.
"""
check_docker()
setup_logging()
args, dry_run = ddc_parse_args(sys.argv)
run_compose(args, dry_run=dry_run, exit_afterwards=True)
def reset_rabbitmq(project):
"""Create rabbitmq vhost"""
from derex.runner.compose_utils import run_compose
vhost = f"{project.name}_edxqueue"
args = [
"exec",
"-T",
"rabbitmq",
"sh",
"-c",
f"""rabbitmqctl add_vhost {vhost}
rabbitmqctl set_permissions -p {vhost} guest ".*" ".*" ".*"
""",
]
run_compose(args)
click.echo(f"Rabbitmq vhost {vhost} created")
return 0
def refresh_course_metadata(project):
"""Run discovery `refresh_course_metadata` Django command"""
from derex.runner.docker import check_services
if not check_services(["elasticsearch"]):
click.echo(
"Elasticsearch service not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
run_compose(
[
"run", "--rm", "discovery", "python", "manage.py",
"refresh_course_metadata"
],
project=project,
)
return 0
def create_index(project):
"""Run discovery `install_es_indexes` Django command"""
from derex.runner.docker import check_services
if not check_services(["elasticsearch"]):
click.echo(
"Elasticsearch service not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
run_compose(
[
"run", "--rm", "discovery", "python", "manage.py",
"install_es_indexes"
],
project=project,
)
return 0
def reset_mysql_openedx(project: Project, dry_run: bool = False):
"""Run script from derex/openedx image to reset the mysql db.
"""
restore_dump_path = abspath_from_egg(
"derex.runner", "derex/runner/restore_dump.py.source")
assert (restore_dump_path
), "Could not find restore_dump.py in derex.runner distribution"
run_compose(
[
"run",
"--rm",
"-v",
f"{restore_dump_path}:/restore_dump.py",
"lms",
"python",
"/restore_dump.py",
],
project=project,
dry_run=dry_run,
)
def copy_database(source_db_name: str, destination_db_name: str):
"""
Copy an existing MySQL database. This actually involves exporting and importing back
the database with a different name.
"""
create_database(destination_db_name)
logger.info(f"Copying database {source_db_name} to {destination_db_name}")
run_compose([
"run",
"--rm",
"mysql",
"sh",
"-c",
f"""set -ex
mysqldump -h mysql -u root -psecret {source_db_name} --no-create-db |
mysql -h mysql --user=root -psecret {destination_db_name}
""",
])
logger.info(
f"Successfully copied database {source_db_name} to {destination_db_name}"
)
def compile_theme(project):
"""Compile theme sass files"""
from derex.runner.compose_utils import run_compose
if project.themes_dir is None:
click.echo("No theme directory present in this project")
return
themes = ",".join(el.name for el in project.themes_dir.iterdir())
uid = os.getuid()
args = [
"run",
"--rm",
"lms",
"sh",
"-c",
f"""set -ex
export PATH=/openedx/edx-platform/node_modules/.bin:$PATH # FIXME: this should not be necessary
paver compile_sass --theme-dirs /openedx/themes --themes {themes}
chown {uid}:{uid} /openedx/themes/* -R""",
]
run_compose(args, project=DebugBaseImageProject(), exit_afterwards=True)
def load_fixtures(project):
"""Load fixtures from the plugin fixtures directory"""
from derex.runner.compose_utils import run_compose
fixtures_dir = project.get_plugin_directories(__package__).get("fixtures")
if fixtures_dir is None:
click.echo("No fixtures directory present for this plugin")
return
load_fixtures_path = abspath_from_egg("derex.discovery",
"derex/discovery/load_fixtures.py")
compose_args = [
"run",
"--rm",
"-v",
f"{load_fixtures_path}:/openedx/discovery/load_fixtures.py",
"discovery",
"python",
"/openedx/discovery/load_fixtures.py",
]
run_compose(compose_args, project=project)
return
def update_index(project):
"""Run discovery `update_index` Django command"""
from derex.runner.docker import check_services
if not check_services(["elasticsearch"]):
click.echo(
"Elasticsearch service not found.\nMaybe you forgot to run\nddc-services up -d"
)
return
run_compose(
[
"run",
"--rm",
"discovery",
"python",
"manage.py",
"update_index",
"--disable-change-limit",
],
project=project,
)
return 0
