def ensure_in_docker(
# True to make this interactive (the ``-i`` flag in ``docker exec``.)
is_interactive: bool = False,
# Return value: True if already in Docker; the function calls ``sys.exit(0)``, ending the program, otherwise.
) -> bool:
if in_docker():
return True
# Get the name of the container running the Runestone servers.
res = subprocess.run(
'docker ps --filter "ancestor=runestone/server" --format "{{.Names}}"',
shell=True,
capture_output=True,
text=True,
)
runestone_container_name = res.stdout.strip()
if not runestone_container_name:
runestone_container_name = "production-runestone-1"
# Some subtleties:
#
# #. Single-quote each argument before passing it.
# #. Run it in the venv used when building Docker, since this avoids installing click globally.
# #. Use env vars defined in the `../Dockerfile`, rather than hard-coding paths. We want these env vars evaluated after the shell in Docker starts, not now, hence the use of ``\$`` and the surrounding double quotes.
# #. Use just the name, not the full path, of ``sys.argv[0]``, since the filesystem is different in Docker. We assume that this command will be either in the path (with the venv activated).
exec_name = Path(sys.argv[0]).name
quoted_args = "' '".join([exec_name] + sys.argv[1:])
xqt(f"docker exec -{'i' if is_interactive else ''}t {runestone_container_name} bash -c "
'"source \$RUNESTONE_PATH/.venv/bin/activate; '
f"'{quoted_args}'\"")
sys.exit(0)
def _stop_servers() -> None:
ensure_in_docker()
xqt(
"pkill celery",
"pkill -f gunicorn",
"service nginx stop",
check=False,
)
def shell(venv: bool) -> None:
"""
Open a Bash shell in the Docker container.
"""
# Ask for an interactive console.
ensure_in_docker(True)
# Skip a check, since the user will see any failures and because this raises an exception of the last command in the shell produced a non-zero exit code.
if venv:
xqt("poetry run bash", cwd=env.RUNESTONE_PATH, check=False)
else:
xqt("bash", check=False)
def run_bookserver(dev: bool) -> None:
assert in_docker()
w2p_parent = Path(env.WEB2PY_PATH).parent
bookserver_path = Path(f"{w2p_parent}/BookServer")
# See the `Volume detection strategy`_.
dev_bookserver = (bookserver_path / 'bookserver').is_dir()
run_bookserver_kwargs = dict(cwd=bookserver_path) if dev_bookserver else {}
run_bookserver_venv = ("poetry run " if dev_bookserver else
f"{sys.executable} -m ") + "bookserver "
xqt(
run_bookserver_venv + "--root /ns "
"--error_path /tmp "
"--gconfig /etc/gunicorn/gunicorn.conf.py "
"--bind unix:/run/gunicorn.sock " + ("--reload " if dev else "") + "&",
**run_bookserver_kwargs,
)
def _start_servers(dev: bool) -> None:
ensure_in_docker()
xqt(
"poetry run bookserver --root /ns "
"--error_path /tmp "
"--gconfig $RUNESTONE_PATH/docker/gunicorn_config/fastapi_config.py "
# This much match the address in `../nginx/sites-available/runestone.template`.
"--bind unix:/run/fastapi.sock " + ("--reload " if dev else "") + "&",
"service nginx start",
"poetry run gunicorn --config $RUNESTONE_PATH/docker/gunicorn_config/web2py_config.py &",
cwd=f"{env.RUNESTONE_PATH}/docker/gunicorn_config",
)
# Start the script to collect tickets and store them in the database. Most useful
# for a production environment with several worker containers
xqt(
f"cp {env.RUNESTONE_PATH}/scripts/tickets2db.py {env.WEB2PY_PATH}",
"python web2py.py -M -S runestone --run tickets2db.py &",
cwd=f"{env.WEB2PY_PATH}",
)
def test(bks: bool, rc: bool, rs: bool, passthrough: Tuple) -> None:
"""
Run unit tests.
PASSTHROUGH: These arguments are passed directly to the underlying "pytest" command. To pass options to this command, prefix this argument with "--". For example, use "docker_tools.py test -- -k test_just_this" instead of "docker_tools.py test -k test_just_this" (which produces an error).
"""
ensure_in_docker()
_stop_servers()
pytest = "$RUNESTONE_PATH/.venv/bin/pytest"
passthrough_args = " ".join(passthrough)
if bks:
xqt(f"{pytest} -v {passthrough_args}", cwd="/srv/BookServer")
if rc:
xqt(f"{pytest} -v {passthrough_args}", cwd="/srv/RunestoneComponents")
if rs:
xqt(
f"{pytest} -v applications/runestone/tests {passthrough_args}",
cwd=env.WEB2PY_PATH,
)
parser = argparse.ArgumentParser(
description='Run tests on the Web2Py Runestone server.')
parser.add_argument(
'--rebuildgrades',
action='store_true',
help='Reset the unit test based on current grading code.')
parser.add_argument('--skipdbinit',
action='store_true',
help='Skip initialization of the test database.')
parsed_args = parser.parse_args()
if parsed_args.rebuildgrades:
with pushd('../../..'):
print("recalculating grades tables")
xqt('{} web2py.py -S runestone -M -R applications/runestone/tests/make_clean_db_with_grades.py'
.format(sys.executable))
print("dumping the data")
xqt('pg_dump --no-owner runestone_test > applications/runestone/tests/runestone_test.sql'
)
sys.exit(0)
if parsed_args.skipdbinit:
print('Skipping DB initialization.')
else:
# make sure runestone_test is nice and clean
xqt('dropdb --echo --if-exists "{}"'.format(dbname),
'createdb --echo "{}"'.format(dbname),
'psql "{}" < runestone_test.sql'.format(dbname))
# Build the test book to add in db fields needed.
with pushd('test_book'):
# The runestone build process only look at ``DBURL``.
def _build_phase2(arm: bool, dev: bool, pic24: bool, tex: bool, rust: bool):
# Check the environment.
assert env.POSTGRES_PASSWORD, "Please export POSTGRES_PASSWORD."
assert env.RUNESTONE_HOST, "Please export RUNESTONE_HOST."
# This should always be `run in a venv`_.
assert sys.prefix != sys.base_prefix, "This should be running in a Python virtual environment."
w2p_parent = Path(env.WEB2PY_PATH).parent
bookserver_path = Path(f"{w2p_parent}/BookServer")
# _`Volume detection strategy`: don't check just ``BookServer`` -- the volume may be mounted, but may not point to an actual filesystem path if the developer didn't clone the BookServer repo. Instead, look for evidence that there are actually some files in this path.
dev_bookserver = (bookserver_path / 'bookserver').is_dir()
run_bookserver_kwargs = dict(cwd=bookserver_path) if dev_bookserver else {}
# Misc setup
# ^^^^^^^^^^
if env.CERTBOT_EMAIL:
xqt('certbot -n --agree-tos --email "$CERTBOT_EMAIL" --nginx --redirect -d "$RUNESTONE_HOST"'
)
print("You should be good for https")
else:
print(
"CERTBOT_EMAIL not set will not attempt certbot setup -- NO https!!"
)
# Install rsmanage.
xqt(
f"eatmydata {sys.executable} -m pip install -e $RUNESTONE_PATH/rsmanage",
)
if dev:
# Start up everything needed for vnc access.
xqt(
# Sometimes, previous runs leave this file behind, which causes Xvfb to output ``Fatal server error: Server is already active for display 0. If this server is no longer running, remove /tmp/.X0-lock and start again.``
f"rm -f /tmp/.X{env.DISPLAY.split(':', 1)[1]}-lock",
"Xvfb $DISPLAY &",
# Wait a bit for Xvfb to start up before running the following X applications.
"sleep 1",
"x11vnc -forever &",
"icewm-session &",
)
# Set up nginx
# ^^^^^^^^^^^^
# _`Set up nginx based on env vars.` See `nginx/sites-available/runestone`.
nginx_conf = Path(
f"{env.RUNESTONE_PATH}/docker/nginx/sites-available/runestone")
txt = replace_vars(
nginx_conf.read_text(),
dict(
RUNESTONE_HOST=env.RUNESTONE_HOST,
WEB2PY_PATH=env.WEB2PY_PATH,
LISTEN_PORT=443 if env.CERTBOT_EMAIL else 80,
PRODUCTION_ONLY=dedent("""\
# `server (http) <http://nginx.org/en/docs/http/ngx_http_core_module.html#server>`_: set configuration for a virtual server. This server closes the connection if there's no host match to prevent host spoofing.
server {
# `listen (http) <http://nginx.org/en/docs/http/ngx_http_core_module.html#listen>`_: Set the ``address`` and ``port`` for IP, or the ``path`` for a UNIX-domain socket on which the server will accept requests.
#
# I think that omitting the server_name_ directive causes this to match any host name not otherwise matched. TODO: does the use of ``default_server`` play into this? What is the purpose of ``default_server``?
listen 80 default_server;
# Also look for HTTPS connections.
listen 443 default_server;
# `return <https://nginx.org/en/docs/http/ngx_http_rewrite_module.html#return>`_: define a rewritten URL for the client. The non-standard code 444 closes a connection without sending a response header.
return 444;
}
""") if env.WEB2PY_CONFIG == "production" else "",
FORWARD_HTTP=dedent("""\
# Redirect from http to https. Copied from an `nginx blog <https://www.nginx.com/blog/creating-nginx-rewrite-rules/#https>`_.
server {
listen 80;
server_name ${RUNESTONE_HOST};
return 301 https://${RUNESTONE_HOST}$request_uri;
}
""") if env.CERTBOT_EMAIL else "",
))
Path("/etc/nginx/sites-available/runestone").write_text(txt)
xqt(
"ln -sf /etc/nginx/sites-available/runestone /etc/nginx/sites-enabled/runestone",
)
# Do dev installs
# ^^^^^^^^^^^^^^^
if dev_bookserver:
assert dev, "You must run ``docker-tools.py build --dev`` in order to install the dev version of the BookServer."
print("Installing development version of the BookServer.")
xqt(
# By default, Poetry creates a venv in the home directory of the current user (root). However, this isn't accessible when running as ``www-data``. So, tell it to create the venv in a `subdirectory of the project <https://python-poetry.org/docs/configuration/#virtualenvsin-project>`_ instead, which is accessible.
"poetry config virtualenvs.in-project true",
"poetry install",
cwd=bookserver_path)
rsc = Path(f"{w2p_parent}/RunestoneComponents")
# Use the same `volume detection strategy`_ as the BookServer.
if (rsc / "runestone").is_dir():
chdir(rsc)
# If the bookserver is in dev mode, then the Runestone Components is already installed there in dev mode. Install it again in the venv so that both are up to date.
# Otherwise, install them now.
if not dev:
print(
"Warning: you're installing a dev version of the components without running ``docker-tools.py build --dev``. The usual dev tools aren't installed."
)
print("Installing Development Version of Runestone Components")
xqt(
f"{sys.executable} -m pip install --upgrade -e .",
f"{sys.executable} -m runestone --version",
)
# Build the webpack after the Runestone Components are installed.
xqt(
"npm install",
"npm run build",
)
xqt(
# web2py needs write access to update logs, database schemas, etc. Give it group ownership with write permission to allow this.
f"chgrp -R www-data {Path(env.RUNESTONE_PATH).parent}",
f"chmod -R g+w {Path(env.RUNESTONE_PATH).parent}",
)
# Set up Postgres database
# ^^^^^^^^^^^^^^^^^^^^^^^^
# Wait until Postgres is ready using `pg_isready <https://www.postgresql.org/docs/current/app-pg-isready.html>`_. Note that its ``--timeout`` parameter applies only when it's waiting for a response from the Postgres server, not to the time spent retrying a refused connection.
print("Waiting for Postgres to start...")
if env.WEB2PY_CONFIG == "production":
effective_dburl = env.DBURL
elif env.WEB2PY_CONFIG == "test":
effective_dburl = env.TEST_DBURL
else:
effective_dburl = env.DEV_DBURL
for junk in range(5):
try:
xqt(f'pg_isready --dbname="{effective_dburl}"')
break
except Exception:
sleep(1)
else:
assert False, "Postgres not available."
print("Creating database if necessary...")
try:
xqt(f"psql {effective_dburl} -c ''")
except Exception:
# The expected format of a DBURL is ``postgresql://user:password@netloc/dbname``, a simplified form of the `connection URI <https://www.postgresql.org/docs/9.6/static/libpq-connect.html#LIBPQ-CONNSTRING>`_.
junk, dbname = effective_dburl.rsplit("/", 1)
xqt(f"PGPASSWORD=$POSTGRES_PASSWORD PGUSER=$POSTGRES_USER PGHOST=db createdb {dbname}"
)
print("Checking the State of Database and Migration Info")
p = xqt(f"psql {effective_dburl} -c '\d'", capture_output=True, text=True)
if p.stderr == "Did not find any relations.\n":
print("Populating database...")
# Populate the db with courses, users.
populate_script = dedent('''\
from bookserver.main import app
from fastapi.testclient import TestClient
with TestClient(app) as client:
pass
''')
xqt(
f'BOOK_SERVER_CONFIG=development DROP_TABLES=Yes {"poetry run python" if dev_bookserver else sys.executable} -c "{populate_script}"',
**run_bookserver_kwargs)
# Remove any existing web2py migration data, since this is out of date and confuses web2py (an empty db, but migration files claiming it's populated).
xqt("rm -f $RUNESTONE_PATH/databases/*")
else:
print("Database already populated.")
# TODO: any checking to see if the db is healthy? Perhaps run Alembic autogenerate to see if it wants to do anything?
# Start the servers
# ^^^^^^^^^^^^^^^^^
print("Starting Celery...")
# sudo doesn't pass root's env vars; provide only the env vars Celery needs when invoking it.
xqt('sudo -u www-data env "PATH=$PATH" "REDIS_URI=$REDIS_URI" /srv/venv/bin/celery --app=scheduled_builder worker --pool=threads --concurrency=3 --loglevel=info &',
cwd=f"{env.RUNESTONE_PATH}/modules")
print("starting nginx")
xqt("service nginx start")
print("starting uwsgi")
# Use uwsgi's "--virtualenv" option, since running it from a venv doesn't cause it to run apps in the same venv.
xqt("/srv/venv/bin/uwsgi --virtualenv /srv/venv --ini /etc/uwsgi/sites/runestone.ini &",
cwd=env.WEB2PY_PATH)
# To manually test out web2py, first ``service nginx stop`` then run ``python3 web2py.py --ip=0.0.0.0 --port=80 --password="******" -K runestone --no_gui -X``.
print("Starting FastAPI server")
run_bookserver(dev)
def build(arm: bool, dev: bool, passthrough: Tuple, pic24: bool, tex: bool,
rust: bool) -> None:
"""
When executed outside a Docker build, build a Docker container for the Runestone webservers.
PASSTHROUGH: These arguments are passed directly to the underlying "docker build" command. To pass options to this command, prefix this argument with "--". For example, use "docker_tools.py build -- -no-cache" instead of "docker_tools.py build -no-cache" (which produces an error).
Inside a Docker build, install all dependencies as root.
"""
# Are we inside the Docker build?
phase = env.IN_DOCKER
if not phase:
# No -- this is the first step in the install.
assert not in_docker()
# Step 1: prepare to run the Docker build
# ---------------------------------------
# Did we add the current user to a group?
did_group_add = False
# Do we need to use ``sudo`` to execute Docker?
docker_sudo = False
# Check to make sure Docker is installed.
try:
xqt("docker --version")
except subprocess.CalledProcessError as e:
check_install_curl()
print(f"Unable to run docker: {e} Installing Docker...")
# Use the `convenience script <https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script>`_.
xqt(
"curl -fsSL https://get.docker.com -o get-docker.sh",
"sudo sh ./get-docker.sh",
"rm get-docker.sh",
# This follows the `Docker docs <https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user>`__.`
"sudo usermod -aG docker ${USER}",
)
# The group add doesn't take effect until the user logs out then back in. Work around it for now.
did_group_add = True
docker_sudo = True
# ...and docker-compose.
try:
xqt("docker-compose --version")
except subprocess.CalledProcessError as e:
print("Unable to run docker-compose: {e} Installing...")
# This is from the `docker-compose install instructions <https://docs.docker.com/compose/install/#install-compose-on-linux-systems>`_.
xqt(
'sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose',
"sudo chmod +x /usr/local/bin/docker-compose",
)
# Are we inside the Runestone repo?
if not (wd / "uwsgi").is_dir():
change_dir = True
# No, we must be running from a downloaded script. Clone the runestone repo.
try:
xqt("git --version")
except Exception as e:
print(f"Unable to run git: {e} Installing...")
xqt("sudo apt-get install -y git")
print("Didn't find the runestone repo. Cloning...")
# Make this in a path that can eventually include web2py.
mkdir("web2py/applications", parents=True)
chdir("web2py/applications")
xqt("git clone https://github.com/RunestoneInteractive/RunestoneServer.git runestone"
)
chdir("runestone")
else:
# Make sure we're in the root directory of the web2py repo.
chdir(wd.parent)
change_dir = False
# Make sure the ``docker/.env`` file exists.
if not Path(".env").is_file():
xqt("cp docker/.env.prototype .env")
# Do the same for ``1.py``.
one_py = Path("models/1.py")
if not one_py.is_file():
# add a new setting so that institutions can run using a base book like thinkcspy as their course. On Runestone.academy we don't let anyone be an instructor for the base courses because they are open to anyone. This makes for a much less complicated deployment strategy for an institution that just wants to run their own server and use one or two books.
one_py.write_text(
dedent("""\
settings.docker_institution_mode = True
settings.jobe_key = ''
settings.jobe_server = 'http://jobe'
settings.bks = "ns"
settings.python_interpreter = "/srv/venv/bin/python3"
# This must match the secret in the BookServer's ``config.py`` ``settings.secret``.
settings.secret = "supersecret"
"""))
# For development, include extra volumes.
dc = Path("docker-compose.override.yml")
if dev and not dc.is_file():
dc.write_text(
dedent("""\
version: "3"
services:
runestone:
# Set up for VNC.
environment:
DISPLAY: ${DISPLAY}
ports:
- "5900:5900"
volumes:
- ../../../RunestoneComponents/:/srv/RunestoneComponents
- ../../../BookServer/:/srv/BookServer
# To make Chrome happy.
- /dev/shm:/dev/shm
"""))
# Ensure the user is in the ``www-data`` group.
print(
"Checking to see if the current user is in the www-data group...")
if "www-data" not in xqt("groups", capture_output=True,
text=True).stdout:
xqt('sudo usermod -a -G www-data "$USER"')
did_group_add = True
if dev:
if is_linux:
# To allow VNC access to the container. Not available on OS X.
check_install("gvncviewer -h", "gvncviewer")
# Allow VS Code / remote access to the container. dpkg isn't available on OS X .
check_install("dpkg -l openssh-server", "openssh-server")
# Run the Docker build.
xqt(f'ENABLE_BUILDKIT=1 {"sudo" if docker_sudo else ""} docker build -t runestone/server . --build-arg DOCKER_BUILD_ARGS="{" ".join(sys.argv[1:])}" --progress plain {" ".join(passthrough)}'
)
# Print thesse messages last; otherwise, it will be lost in all the build noise.
if change_dir:
print(
'\nDownloaded the RunestoneServer repo. You must "cd web2py/applications/runestone" before running this script again.'
)
if did_group_add:
print(
'\nAdded the current user to the www-data and/or docker group(s). You must log out and log back in for this to take effect, or run "su -s ${USER}".'
)
return
# Step 3 - startup script for container.
if phase == "2":
try:
_build_phase2(arm, dev, pic24, tex, rust)
print("Success! The Runestone servers are running.")
except Exception:
print(f"Failed to start the Runestone servers:")
print_exc()
# Notify listener user we're done.
print("=-=-= Runestone setup finished =-=-=")
# Flush now, so that text won't stay hidden in Python's buffers. The next step is to do nothing (where no flush occurs and the text would otherwise stay hidden).
sys.stdout.flush()
sys.stderr.flush()
# If this script exits, then Docker re-runs it. So, loop forever.
while True:
sleep(1000)
# Step 2: install Runestone dependencies
# ---------------------------------------
# This is run inside the Docker build, from the `Dockerfile`.
assert phase == "1", f"Unknown value of IN_DOCKER={phase}"
assert in_docker()
# It should always be `run in a venv <https://stackoverflow.com/a/1883251/16038919>`_.
assert in_venv, "This should be running in a Python virtual environment."
# Install required packages
# ^^^^^^^^^^^^^^^^^^^^^^^^^
# Add in Chrome repo. Copied from https://tecadmin.net/setup-selenium-with-chromedriver-on-debian/.
xqt(
"curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -",
)
Path("/etc/apt/sources.list.d/google-chrome.list").write_text(
"deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main")
# Add node.js per the `instructions <https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions>`_.
xqt("curl -fsSL https://deb.nodesource.com/setup_current.x | bash -")
xqt(
"apt-get update",
"apt-get install -y eatmydata",
# All one big command! Therefore, there are no commas after each line, but instead a trailing space.
"eatmydata apt-get install -y --no-install-recommends "
"gcc unzip "
# For jobe's runguard.
"sudo "
# Some books use the `Sphinx graphviz extension <https://www.sphinx-doc.org/en/master/usage/extensions/graphviz.html>`_, which needs the ``graphviz``` binary.
"graphviz "
# TODO: What is this for?
"libfreetype6-dev "
"postgresql-client "
# Just installing ``nodejs`` fails with messages about unmet dependencies. Adding ``yarn`` (which is never used) makes it happy. Solution from `SO <https://stackoverflow.com/a/67329755/16038919>`__.
"nodejs yarn "
# TODO: should this only be installed in the dev image?
"libpq-dev libxml2-dev libxslt1-dev "
"certbot python-certbot-nginx "
"rsync wget nginx "
# Useful tools for debug.
"nano less ")
# Build runguard and set up jobe users.
xqt("mkdir /var/www/jobe")
chdir("/var/www/jobe")
xqt(
"cp -r $RUNESTONE_PATH/docker/runguard .",
f"{sys.executable} $RUNESTONE_PATH/docker/runguard-install.py",
)
if arm:
xqt(
# Get the ``add-apt-repository`` tool.
"eatmydata apt-get install -y software-properties-common",
# Use it to add repo for the ARM tools.
"eatmydata add-apt-repository -y ppa:canonical-server/server-backports",
# Then install the ARM tools (and the QEMU emulator).
"eatmydata apt-get install -y qemu-system-arm gcc-arm-none-eabi libnewlib-arm-none-eabi build-essential",
)
if dev:
xqt(
# Tests use `html5validator <https://github.com/svenkreiss/html5validator>`_, which requires the JDK.
"eatmydata apt-get install -y --no-install-recommends openjdk-11-jre-headless git xvfb x11-utils google-chrome-stable lsof emacs-nox",
# Install Chromedriver. Based on https://tecadmin.net/setup-selenium-with-chromedriver-on-debian/.
"wget --no-verbose https://chromedriver.storage.googleapis.com/94.0.4606.61/chromedriver_linux64.zip",
"unzip chromedriver_linux64.zip",
"rm chromedriver_linux64.zip",
"mv chromedriver /usr/bin/chromedriver",
"chown root:root /usr/bin/chromedriver",
"chmod +x /usr/bin/chromedriver",
# Provide VNC access. TODO: just pass the correct DISPLAY value and ports and use X11 locally, but how? Notes on my failures:
#
# - Including ``network_mode: host`` in `../docker-compose.yml` works. However, this breaks everything else (port mapping, links, etc.). It suggests that the correct networking setup would make this work.
# - Passing ``volume: - /tmp/.X11-unix:/tmp/.X11-unix`` has no effect (on a Ubuntu 20.03.4 LTS host). Per the previous point, it seems that X11 is using TCP as its transport.
# - Mapping X11 ports via ``ports: - "6000-6063:6000-6063"`` doesn't work.
# - Setting ``DISPLAY`` to various values (from the host's ``hostname -I``, or various names to route to the host) doesn't work.
#
# Install a VNC server plus a simple window manager.
"eatmydata apt-get install -y x11vnc icewm",
)
if pic24:
# When changing the xc16 version, update the string below **and** the path added at the end of this block.
xc16_ver = "xc16-v1.70-full-install-linux64-installer.run"
mplabx_ver = "MPLABX-v5.50(1)-linux-installer.tar"
mplabx_sh = "MPLABX-v5.50-linux-installer.sh"
xqt(
# Install the xc16 compiler.
f"eatmydata wget --no-verbose https://ww1.microchip.com/downloads/en/DeviceDoc/{xc16_ver}",
f"chmod a+x {xc16_ver}",
# The installer complains if the netserver name isn't specified. This option isn't documented in the ``--help``. So, supply junk, and it seems to work.
f"eatmydata ./{xc16_ver} --mode unattended --netservername foo",
f"rm {xc16_ver}",
# MPLAB X install
#
# No longer required: per https://unix.stackexchange.com/questions/486806/steam-missing-32-bit-libraries-libx11-6, enable 32-bit libs.
#"eatmydata dpkg --add-architecture i386",
#"eatmydata apt-get update",
# No longer required: per https://microchipdeveloper.com/install:mplabx-lin64, install prereqs. The xc16 compiler is 32-bit.
#"eatmydata apt-get install -y lib32stdc++6 libc6:i386 libx11-6:i386 libxext6:i386 libstdc++6:i386 libexpat1:i386",
# Then download and install MPLAB X.
f'eatmydata wget --no-verbose "https://ww1.microchip.com/downloads/en/DeviceDoc/{mplabx_ver}"',
f'eatmydata tar -xf "{mplabx_ver}"',
f'rm "{mplabx_ver}"',
# Install just the IDE and the 16-bit tools. This program check to see if this is being run by root by looking at the ``USER`` env var, which Docker doesn't set. Fake it out.
f'USER=root eatmydata "./{mplabx_sh}" -- --mode unattended --ipe 0 --8bitmcu 0 --32bitmcu 0 --othermcu 0',
f'rm "{mplabx_sh}"',
)
# Add the path to the xc16 tools.
with open("/root/.bashrc", "a", encoding="utf-8") as f:
f.write(
dedent("""
export PATH=$PATH:/opt/microchip/xc16/v1.70/bin
"""))
# Just symlink mdb, since that's the only tool we use.
xqt(
"ln -sf /opt/microchip/mplabx/v5.50/mplab_platform/bin/mdb.sh /usr/local/bin/mdb",
)
# Microchip tools (mdb) needs write access to these directories.
mchp_packs = "/var/www/.mchp_packs"
java = "/var/www/.java"
for path in (mchp_packs, java):
xqt(
f"mkdir {path}",
f"chown www-data:www-data {path}",
)
if tex:
xqt("eatmydata apt-get install -y texlive-full xsltproc pdf2svg")
if rust:
xqt("eatmydata apt-get install -y cargo")
# Python/pip-related installs
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^
# Install web2py.
xqt(
"mkdir -p $WEB2PY_PATH",
# Make the www-data the owner and place its files in the www-data group. This is because web2py needs to write to this directory tree (log, errors, etc.).
"chown www-data:www-data $WEB2PY_PATH",
# Make any newly created directories have the www-group. Give the group write permission.
"chmod g+ws $WEB2PY_PATH",
)
w2p_parent = Path(env.WEB2PY_PATH).parent
xqt(
# Install additional components.
"eatmydata wget --no-verbose https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip",
"eatmydata unzip -q web2py_src.zip",
"rm -f web2py_src.zip",
cwd=w2p_parent,
)
# Wheel helps several other packages (uwsgi, etc.) build more cleanly. Otherwise, we get ``Using legacy 'setup.py install' for uwsgi, since package 'wheel' is not installed.``
xqt(f"eatmydata {sys.executable} -m pip install --upgrade wheel")
chdir(env.RUNESTONE_PATH)
if dev:
# The dev requirements include the main requirements file as well.
xqt(f"eatmydata {sys.executable} -m pip install -r requirements-dev.txt"
)
# For development purposes, `install Poetry <https://python-poetry.org/docs/master/#osx--linux--bashonwindows-install-instructions>`_.
xqt("curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | POETRY_HOME=/usr/local python -"
)
else:
xqt(f"eatmydata {sys.executable} -m pip install -r requirements.txt")
xqt(
f"eatmydata {sys.executable} -m pip install uwsgi uwsgitop bookserver myst-parser sphinx-reredirects",
"rm -rf $WEB2PY_PATH/.cache/*",
"cp scripts/routes.py $WEB2PY_PATH/routes.py",
)
# Set up config files
# ^^^^^^^^^^^^^^^^^^^
xqt(
# Set up for uwsgi
"mkdir -p $WEB2PY_PATH/logs",
"mkdir -p /run/uwsgi",
"mkdir -p /etc/uwsgi/sites",
"cp $RUNESTONE_PATH/docker/uwsgi/sites/runestone.ini /etc/uwsgi/sites/runestone.ini",
# TODO: is this ever used?
"cp $RUNESTONE_PATH/docker/systemd/system/uwsgi.service /etc/systemd/system/uwsgi.service",
"ln -sf /etc/systemd/system/uwsgi.service /etc/systemd/system/multi-user.target.wants/uwsgi.service",
"cp $RUNESTONE_PATH/docker/wsgihandler.py $WEB2PY_PATH/wsgihandler.py",
# Set up nginx (partially -- more in step 3 below).
"rm /etc/nginx/sites-enabled/default",
# Send nginx logs to stdout/stderr, so they'll show up in Docker logs.
"ln -sf /dev/stdout /var/log/nginx/access.log",
"ln -sf /dev/stderr /var/log/nginx/error.log",
# Set up gunicorn
"mkdir -p /etc/gunicorn",
"cp $RUNESTONE_PATH/docker/gunicorn/gunicorn.conf.py /etc/gunicorn",
# Set up web2py routing.
"cp $RUNESTONE_PATH/docker/routes.py $WEB2PY_PATH",
# ``sphinxcontrib.paverutils.run_sphinx`` lacks venv support -- it doesn't use ``sys.executable``, so it doesn't find ``sphinx-build`` in the system path when executing ``/srv/venv/bin/runestone`` directly, instead of activating the venv first (where it does work). As a huge, ugly hack, symlink it to make it available in the system path.
"ln -sf /srv/venv/bin/sphinx-build /usr/local/bin",
)
# Create a default auth key for web2py.
print("Creating auth key")
xqt("mkdir -p $RUNESTONE_PATH/private")
(Path(env.RUNESTONE_PATH) / "private/auth.key"
).write_text("sha512:16492eda-ba33-48d4-8748-98d9bbdf8d33")
# Clean up after install.
xqt(
"eatmydata sudo apt-get -y autoclean",
"eatmydata sudo apt-get -y autoremove",
"rm -rf /tmp/* /var/tmp/*",
)
# Remove all the files from the local repo, since they will be replaced by the volume. This must be the last step, since it deletes the script as well.
xqt("rm -rf $RUNESTONE_PATH")
],
check=True)
from ci_utils import chdir, env, is_linux, mkdir, xqt
# Third-party
# -----------
# This comes after importing ``ci_utils``, since we use that to install click if necessary.
in_venv = sys.prefix != sys.base_prefix
try:
import click
except ImportError:
print("Installing click...")
# Outside a venv, install locally.
user = '' if in_venv else '--user'
xqt(
f"{sys.executable} -m pip install {user} --upgrade pip",
f"{sys.executable} -m pip install {user} --upgrade click",
)
# If pip is upgraded, it won't find click. `Re-load sys.path <https://stackoverflow.com/a/25384923/16038919>`_ to fix this.
import site
from importlib import reload
reload(site)
import click
# ``build`` command
# =================
# Create a series of subcommands for this CLI.
@click.group()
def cli() -> None:
pass
assert (not empty1) and (not empty2)
os.environ['PGPASSWORD'] = pgpassword
os.environ['PGUSER'] = pguser
parser = argparse.ArgumentParser(description='Run tests on the Web2Py Runestone server.')
parser.add_argument('--rebuildgrades', action='store_true',
help='Reset the unit test based on current grading code.')
parser.add_argument('--skipdbinit', action='store_true',
help='Skip initialization of the test database.')
# Per https://docs.python.org/2/library/argparse.html#partial-parsing, gather any known args. These will be passed to pytest.
parsed_args, extra_args = parser.parse_known_args()
if parsed_args.rebuildgrades:
with pushd('../../..'):
print("recalculating grades tables")
xqt('{} web2py.py -S runestone -M -R applications/runestone/tests/make_clean_db_with_grades.py'.format(sys.executable))
print("dumping the data")
xqt('pg_dump --no-owner runestone_test > applications/runestone/tests/runestone_test.sql')
sys.exit(0)
if parsed_args.skipdbinit:
print('Skipping DB initialization.')
else:
# make sure runestone_test is nice and clean.
xqt('dropdb --echo --if-exists "{}"'.format(dbname),
'createdb --echo "{}"'.format(dbname),
'psql "{}" < runestone_test.sql'.format(dbname))
# Build the test book to add in db fields needed.
with pushd('test_book'):
# The runestone build process only looks at ``DBURL``.
os.environ['DBURL'] = os.environ['TEST_DBURL']
parser.add_argument('--skipdbinit',
action='store_true',
help='Skip initialization of the test database.')
parser.add_argument('--runold',
action='store_true',
help='Force old tests to run ')
# Per https://docs.python.org/2/library/argparse.html#partial-parsing, gather any known args. These will be passed to pytest.
parsed_args, extra_args = parser.parse_known_args()
if parsed_args.skipdbinit:
print('Skipping DB initialization.')
else:
# Make sure runestone_test is nice and clean -- this will remove many
# tables that web2py will then re-create.
xqt('rsmanage --verbose initdb --reset --force')
# Copy the test book to the books directory.
rmtree('../books/test_course_1', ignore_errors=True)
# Sometimes this fails for no good reason on Windows. Retry.
for retry in range(100):
try:
copytree('test_course_1', '../books/test_course_1')
break
except WindowsError:
if retry == 99:
raise
# Build the test book to add in db fields needed.
#xqt('rsmanage addcourse --course-name=test_course_1 --basecourse=test_course_1')
with pushd('../books/test_course_1'):
# The runestone build process only looks at ``DBURL``.
parser = argparse.ArgumentParser(
description='Run tests on the Web2Py Runestone server.')
parser.add_argument(
'--rebuildgrades',
action='store_true',
help='Reset the unit test based on current grading code.')
parser.add_argument('--skipdbinit',
action='store_true',
help='Skip initialization of the test database.')
# Per https://docs.python.org/2/library/argparse.html#partial-parsing, gather any known args. These will be passed to pytest.
parsed_args, extra_args = parser.parse_known_args()
if parsed_args.rebuildgrades:
with pushd('../../..'):
print("recalculating grades tables")
xqt('{} web2py.py -S runestone -M -R applications/runestone/tests/make_clean_db_with_grades.py'
.format(sys.executable))
print("dumping the data")
xqt('pg_dump --no-owner runestone_test > applications/runestone/tests/runestone_test.sql'
)
sys.exit(0)
if parsed_args.skipdbinit:
print('Skipping DB initialization.')
else:
# make sure runestone_test is nice and clean.
xqt('dropdb --echo --if-exists "{}"'.format(dbname),
'createdb --echo "{}"'.format(dbname),
'psql "{}" < runestone_test.sql'.format(dbname))
# Build the test book to add in db fields needed.
with pushd('test_book'):
# The runestone build process only looks at ``DBURL``.
parser.add_argument('--skipdbinit',
action='store_true',
help='Skip initialization of the test database.')
parser.add_argument('--runold',
action='store_true',
help='Force old tests to run ')
# Per https://docs.python.org/2/library/argparse.html#partial-parsing, gather any known args. These will be passed to pytest.
parsed_args, extra_args = parser.parse_known_args()
if parsed_args.skipdbinit:
print('Skipping DB initialization.')
else:
# Make sure runestone_test is nice and clean -- this will remove many
# tables that web2py will then re-create.
xqt('rsmanage --verbose initdb --reset --force')
# Copy the test book to the books directory.
rmtree('../books/test_course_1', ignore_errors=True)
# Sometimes this fails for no good reason on Windows. Retry.
for retry in range(100):
try:
copytree('test_course_1', '../books/test_course_1')
break
except WindowsError:
if retry == 99:
raise
# Build the test book to add in db fields needed.
#xqt('rsmanage addcourse --course-name=test_course_1 --basecourse=test_course_1')
with pushd('../books/test_course_1'):
# The runestone build process only looks at ``DBURL``.
