def setup_contest_files(options):
""" Setup all the contest specific files and directories """
contest_root = options.root_dir
local_repo = options.local_repo
compiled_dir = os.path.join(contest_root, "compiled")
if not os.path.exists(compiled_dir):
os.mkdir(compiled_dir)
run_cmd("chown {0}: {1}".format(options.username, compiled_dir))
map_dir = os.path.join(contest_root, "maps")
if not os.path.exists(map_dir):
os.mkdir(map_dir)
run_cmd("chown {0}: {1}".format(options.username, map_dir))
log_dir = os.path.join(contest_root, "logs")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
run_cmd("chown {0}: {1}".format(options.username, log_dir))
worker_dir = os.path.join(contest_root, local_repo, "worker")
si_filename = os.path.join(TEMPLATE_DIR, "worker_server_info.py.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(contest_root=contest_root,
repo_dir=local_repo, log_dir=log_dir,
map_dir=map_dir, compiled_dir=compiled_dir,
api_url=options.api_url, api_key=options.api_key)
with CD(worker_dir):
if not os.path.exists("server_info.py"):
with open("server_info.py", "w") as si_file:
si_file.write(si_contents)
run_cmd("chown {0}:{0} server_info.py".format(options.username))
if os.stat(local_repo).st_uid != pwd.getpwnam(options.username).pw_uid:
run_cmd("chown -R {0}: {1}".format(options.username, local_repo))
def setup_database(opts):
""" Setup database for contest use """
import MySQLdb
try:
password_opt = ""
if opts.database_password:
password_opt = "-p'%s'" % (opts.database_password, )
run_cmd("echo 'quit' | mysql -u %s %s %s" %
(opts.database_user, password_opt, opts.database_name))
except CmdError:
with MySQLdb.connect(host="127.0.0.1",
user="******",
passwd=opts.database_root_password) as cursor:
cursor.execute(SETUP_SQL["creation"] % (opts.database_name, ))
if opts.database_user != "root":
if opts.database_password:
cursor.execute(
SETUP_SQL["user_grant_passwd"] %
(opts.database_user, opts.database_password))
else:
cursor.execute(SETUP_SQL["user_grant_nopasswd"] %
(opts.database_user, ))
cursor.execute(SETUP_SQL["database_perms"] %
(opts.database_name, opts.database_user))
password_opt = ""
if opts.database_password:
password_opt = "-p'%s'" % (opts.database_password, )
schema_dir = os.path.join(opts.root_dir, opts.local_repo, "sql")
schema_files = os.listdir(schema_dir)
schema_files = [f for f in schema_files if f.endswith(".sql")]
schema_files.sort()
for sf in schema_files:
sp = os.path.join(schema_dir, sf)
run_cmd("mysql -u %s %s %s < %s" %
(opts.database_user, password_opt, opts.database_name, sp))
def install_groovy(download_base):
""" Install the Groovy language """
if os.path.exists("/usr/bin/groovy"):
return
with CD("/root"):
run_cmd("curl '%s/groovy.deb' > groovy.deb" % (download_base,))
run_cmd("dpkg -i groovy.deb")
def create_jail_user(username):
""" Setup a jail user with the given username """
run_cmd("useradd -g jailusers -d /home/jailuser %s" % (username,))
# Add rule to drop any network communication from this user
run_cmd("iptables -A OUTPUT -m owner --uid-owner %s -j DROP" % (username,))
# Create user specific chroot
chroot_dir = "/srv/chroot"
jail_dir = os.path.join(chroot_dir, username)
os.makedirs(os.path.join(jail_dir, "scratch"))
os.makedirs(os.path.join(jail_dir, "root"))
home_dir = os.path.join(jail_dir, "home/home/jailuser")
os.makedirs(home_dir)
run_cmd("chown %s:jailusers %s" % (username, home_dir))
run_cmd("chown :jailkeeper %s" % (jail_dir,))
run_cmd("chmod g=rwx %s" % (jail_dir,))
fs_line = "unionfs-fuse#%s=rw:%s=ro:%s=ro %s fuse cow,allow_other,noauto 0 0" % (
os.path.join(jail_dir, "scratch"),
os.path.join(jail_dir, "home"),
os.path.join(chroot_dir, "aic-base"),
os.path.join(jail_dir, "root")
)
append_line("/etc/fstab", fs_line)
cfg_filename = os.path.join(TEMPLATE_DIR,
"chroot_configs/chroot.d/jailuser.template")
with open(cfg_filename, 'r') as cfg_file:
cfg = cfg_file.read()
schroot_filename = os.path.join("/etc/schroot/chroot.d", username)
with open(schroot_filename, 'w') as schroot_file:
schroot_file.write(cfg.format(jailname=username))
def install_groovy():
""" Install the Groovy language """
if os.path.exists("/usr/bin/groovy"):
return
with CD("/root"):
run_cmd("curl 'http://dist.groovy.codehaus.org/distributions/installers/deb/groovy_1.7.8-1_all.deb' > groovy_1.7.8-1_all.deb")
run_cmd("dpkg -i groovy_1.7.8-1_all.deb")
def install_website_packages():
""" Install system packages required for the website """
pkg_list = ["memcached", "php-memcache", "zip", "nodejs",
"cvs", "openjdk-11-jdk", "ant",
"python-setuptools", "dvipng", "texlive-latex-base"]
run_cmd("curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -")
install_apt_packages(pkg_list)
def setup_database(opts):
""" Setup database for contest use """
import MySQLdb
try:
password_opt = ""
if opts.database_password:
password_opt = "-p'%s'" % (opts.database_password,)
run_cmd("echo 'quit' | mysql -u %s %s %s" % (opts.database_user,
password_opt, opts.database_name))
except CmdError:
with MySQLdb.connect(host="127.0.0.1", user="******",
passwd=opts.database_root_password) as cursor:
cursor.execute(SETUP_SQL["creation"] % (opts.database_name,))
if opts.database_user != "root":
if opts.database_password:
cursor.execute(SETUP_SQL["user_grant_passwd"]
% (opts.database_user, opts.database_password))
else:
cursor.execute(SETUP_SQL["user_grant_nopasswd"]
% (opts.database_user,))
cursor.execute(SETUP_SQL["database_perms"]
% (opts.database_name, opts.database_user))
password_opt = ""
if opts.database_password:
password_opt = "-p'%s'" % (opts.database_password,)
schema_dir = os.path.join(opts.local_repo, "sql")
schema_files = os.listdir(schema_dir)
schema_files = [f for f in schema_files if f.endswith(".sql")]
schema_files.sort()
for sf in schema_files:
sp = os.path.join(schema_dir, sf)
run_cmd("mysql -u %s %s %s < %s" % (opts.database_user,
password_opt, opts.database_name, sp))
def install_groovy(download_base):
""" Install the Groovy language """
if os.path.exists("/usr/bin/groovy"):
return
with CD("/root"):
run_cmd("curl '%s/groovy.deb' > groovy.deb" % (download_base, ))
run_cmd("dpkg -i groovy.deb")
def setup_contest_files(options):
""" Setup all the contest specific files and directories """
contest_root = options.root_dir
local_repo = options.local_repo
compiled_dir = os.path.join(contest_root, "compiled")
if not os.path.exists(compiled_dir):
os.mkdir(compiled_dir)
run_cmd("chown {0}: {1}".format(options.username, compiled_dir))
map_dir = os.path.join(contest_root, "maps")
if not os.path.exists(map_dir):
os.mkdir(map_dir)
run_cmd("chown {0}: {1}".format(options.username, map_dir))
if not os.path.exists(options.log_dir):
os.mkdir(options.log_dir)
run_cmd("chown {0}: {1}".format(options.username, options.log_dir))
worker_dir = os.path.join(contest_root, local_repo, "worker")
si_filename = os.path.join(TEMPLATE_DIR, "worker_server_info.py.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(contest_root=contest_root,
repo_dir=local_repo,
log_dir=options.log_dir,
map_dir=map_dir,
compiled_dir=compiled_dir,
api_url=options.api_url,
api_key=options.api_key)
with CD(worker_dir):
if not os.path.exists("server_info.py"):
with open("server_info.py", "w") as si_file:
si_file.write(si_contents)
run_cmd("chown {0}:{0} server_info.py".format(options.username))
if os.stat(local_repo).st_uid != pwd.getpwnam(options.username).pw_uid:
run_cmd("chown -R {0}: {1}".format(options.username, local_repo))
def create_jail_user(username):
""" Setup a jail user with the given username """
run_cmd("useradd -g jailusers -d /home/jailuser %s" % (username, ))
# Add rule to drop any network communication from this user
run_cmd("iptables -A OUTPUT -m owner --uid-owner %s -j DROP" %
(username, ))
# Create user specific chroot
chroot_dir = "/srv/chroot"
jail_dir = os.path.join(chroot_dir, username)
os.makedirs(os.path.join(jail_dir, "scratch"))
os.makedirs(os.path.join(jail_dir, "root"))
home_dir = os.path.join(jail_dir, "home/home/jailuser")
os.makedirs(home_dir)
run_cmd("chown %s:jailusers %s" % (username, home_dir))
run_cmd("chown :jailkeeper %s" % (jail_dir, ))
run_cmd("chmod g=rwx %s" % (jail_dir, ))
fs_line = "unionfs-fuse#%s=rw:%s=ro:%s=ro %s fuse cow,allow_other,noauto 0 0" % (
os.path.join(jail_dir, "scratch"), os.path.join(jail_dir, "home"),
os.path.join(chroot_dir, "aic-base"), os.path.join(jail_dir, "root"))
append_line("/etc/fstab", fs_line)
cfg_filename = os.path.join(TEMPLATE_DIR,
"chroot_configs/chroot.d/jailuser.template")
with open(cfg_filename, 'r') as cfg_file:
cfg = cfg_file.read()
schroot_filename = os.path.join("/etc/schroot/chroot.d", username)
with open(schroot_filename, 'w') as schroot_file:
schroot_file.write(cfg.format(jailname=username))
def install_groovy():
""" Install the Groovy language """
if os.path.exists("/usr/bin/groovy"):
return
with CD("/root"):
run_cmd("curl 'http://dist.groovy.codehaus.org/distributions/installers/deb/groovy_1.7.8-1_all.deb' > groovy_1.7.8-1_all.deb")
run_cmd("dpkg -i groovy_1.7.8-1_all.deb")
def install_dmd():
""" Install the D language """
if os.path.exists("/usr/bin/dmd"):
return
install_apt_packages("gcc-multilib")
with CD("/root"):
run_cmd("curl 'http://ftp.digitalmars.com/dmd_2.055-0_amd64.deb' > dmd_2.055-0_amd64.deb")
run_cmd("dpkg -i dmd_2.055-0_amd64.deb")
def install_racket(download_base):
""" Install the Racket language"""
if os.path.exists("/usr/bin/racket"):
return
with CD("/root"):
run_cmd("curl '%s/racket.sh' > racket.sh" % (download_base,))
run_cmd('echo -e "\n4\n" | sh racket.sh')
os.symlink("/usr/racket/bin/racket", "/usr/bin/racket")
def install_dart(download_base):
""" Install the Dart language """
if os.path.exists("/usr/bin/frogsh"):
return
with CD("/root"):
run_cmd("curl '%s/dart.tgz' | tar xz" % (download_base, ))
os.rename("dart-frogsh-r1499", "/usr/share/dart")
os.symlink("/usr/share/dart/frog/frogsh", "/usr/bin/frogsh")
def install_dart(download_base):
""" Install the Dart language """
if os.path.exists("/usr/bin/frogsh"):
return
with CD("/root"):
run_cmd("curl '%s/dart.tgz' | tar xz" % (download_base,))
os.rename("dart-frogsh-r1499", "/usr/share/dart")
os.symlink("/usr/share/dart/frog/frogsh", "/usr/bin/frogsh")
def install_dmd(download_base):
""" Install the D language """
if os.path.exists("/usr/bin/dmd"):
return
install_apt_packages("gcc-multilib")
with CD("/root"):
run_cmd("curl '%s/dmd.deb' > dmd.deb" % (download_base,))
run_cmd("dpkg -i dmd.deb")
def install_dmd():
""" Install the D language """
if os.path.exists("/usr/bin/dmd"):
return
install_apt_packages("gcc-multilib")
with CD("/root"):
run_cmd("curl 'http://ftp.digitalmars.com/dmd_2.053-0_amd64.deb' > dmd_2.053-0_amd64.deb")
run_cmd("dpkg -i dmd_2.053-0_amd64.deb")
def install_dmd(download_base):
""" Install the D language """
if os.path.exists("/usr/bin/dmd"):
return
install_apt_packages("gcc-multilib")
with CD("/root"):
run_cmd("curl '%s/dmd.deb' > dmd.deb" % (download_base, ))
run_cmd("dpkg -i dmd.deb")
def install_pypy(download_base):
""" Install pypy """
if os.path.exists("/usr/bin/pypy"):
return
with CD("/root"):
run_cmd("curl '%s/pypy.tar.bz2' | tar xj" % (download_base, ))
os.rename("pypy-1.7", "/usr/share/pypy-1.7")
os.symlink("/usr/share/pypy-1.7/bin/pypy", "/usr/bin/pypy")
def install_pypy(download_base):
""" Install pypy """
if os.path.exists("/usr/bin/pypy"):
return
with CD("/root"):
run_cmd("curl '%s/pypy.tar.bz2' | tar xj" % (download_base,))
os.rename("pypy-1.7", "/usr/share/pypy-1.7")
os.symlink("/usr/share/pypy-1.7/bin/pypy", "/usr/bin/pypy")
def install_racket(download_base):
""" Install the Racket language"""
if os.path.exists("/usr/bin/racket"):
return
with CD("/root"):
run_cmd("curl '%s/racket.sh' > racket.sh" % (download_base, ))
run_cmd('echo -e "\n4\n" | sh racket.sh')
os.symlink("/usr/racket/bin/racket", "/usr/bin/racket")
def setup_local_repo(opts):
""" Create or update the local source repository as needed """
with CD(opts.root_dir):
if not os.path.exists(os.path.join(opts.root_dir, opts.local_repo)):
run_cmd("git clone -b %s %s %s" %
(opts.src_branch, opts.src_repo, opts.local_repo))
else:
with CD(os.path.join(opts.root_dir, opts.local_repo)):
run_cmd("git pull %s %s" % (opts.src_repo, opts.src_branch))
def install_scala():
""" Install the Scala language """
if os.path.exists("/usr/bin/scala"):
return
with CD("/root"):
run_cmd("curl 'http://www.scala-lang.org/downloads/distrib/files/scala-2.8.1.final.tgz' | tar xz")
os.rename("scala-2.8.1.final", "/usr/share/scala")
os.symlink("/usr/share/scala/bin/scala", "/usr/bin/scala")
os.symlink("/usr/share/scala/bin/scalac", "/usr/bin/scalac")
def install_scala(download_base):
""" Install the Scala language """
if os.path.exists("/usr/bin/scala"):
return
with CD("/root"):
run_cmd("curl '%s/scala.tgz' | tar xz" % (download_base,))
os.rename("scala-2.9.0.1", "/usr/share/scala")
os.symlink("/usr/share/scala/bin/scala", "/usr/bin/scala")
os.symlink("/usr/share/scala/bin/scalac", "/usr/bin/scalac")
def install_dmd():
""" Install the D language """
# FIXME: This is broken because the package below is for i386 only and we
# are using amd64 workers
if os.path.exists("/usr/bin/dmd"):
return
with CD("/root"):
run_cmd("curl 'http://ftp.digitalmars.com/dmd_2.052-0_i386.deb' > dmd_2.052-0_i386.deb")
run_cmd("dpkg -i dmd_2.052-0_i386.deb")
def setup_local_repo(opts):
""" Create or update the local source repository as needed """
with CD(opts.root_dir):
if not os.path.exists(os.path.join(opts.root_dir, opts.local_repo)):
run_cmd("git clone -b %s %s %s" % (opts.src_branch,
opts.src_repo, opts.local_repo))
else:
with CD(os.path.join(opts.root_dir, opts.local_repo)):
run_cmd("git pull %s %s" % (opts.src_repo, opts.src_branch))
def install_scala():
""" Install the Scala language """
if os.path.exists("/usr/bin/scala"):
return
with CD("/root"):
run_cmd("curl 'http://www.scala-lang.org/downloads/distrib/files/scala-2.9.0.1.tgz' | tar xz")
os.rename("scala-2.9.0.1", "/usr/share/scala")
os.symlink("/usr/share/scala/bin/scala", "/usr/bin/scala")
os.symlink("/usr/share/scala/bin/scalac", "/usr/bin/scalac")
def install_scala(download_base):
""" Install the Scala language """
if os.path.exists("/usr/bin/scala"):
return
with CD("/root"):
run_cmd("curl '%s/scala.tgz' | tar xz" % (download_base, ))
os.rename("scala-2.9.0.1", "/usr/share/scala")
os.symlink("/usr/share/scala/bin/scala", "/usr/bin/scala")
os.symlink("/usr/share/scala/bin/scalac", "/usr/bin/scalac")
def setup_language_repo(opts):
""" Download languages not part of OS distribution locally for workers """
download_dir = os.path.join(opts.local_repo, "website/langs")
try:
os.mkdir(download_dir)
except OSError:
if not os.path.isdir(download_dir):
raise
retrieve_cmd = os.path.join(opts.local_repo, "setup/retrieve_languages.py")
run_cmd("%s %s" % (retrieve_cmd, download_dir))
def setup_language_repo(opts):
""" Download languages not part of OS distribution locally for workers """
download_dir = os.path.join(opts.local_repo, "website/langs")
try:
os.mkdir(download_dir)
except OSError:
if not os.path.isdir(download_dir):
raise
retrieve_cmd = os.path.join(opts.local_repo, "setup/retrieve_languages.py")
run_cmd("%s %s" % (retrieve_cmd, download_dir))
def setup_jailusers(options):
""" Create and configure the jail users """
create_jail_group(options)
iptablesload_path = "/etc/network/if-pre-up.d/iptablesload"
if not os.path.exists(iptablesload_path):
with open(iptablesload_path, "w") as loadfile:
loadfile.write(IPTABLES_LOAD)
os.chmod(iptablesload_path, 0744)
setup_base_chroot(options)
for user_num in range(1, 33):
create_jail_user("jailuser%s" % (user_num,))
run_cmd("iptables-save > /etc/iptables.rules")
def setup_base_files(opts):
""" Setup all the contest specific files and directories """
sub_dir = os.path.join(opts.root_dir, "submissions")
if not os.path.exists(sub_dir):
os.mkdir(sub_dir)
run_cmd("chown {0}:{0} {1}".format(opts.username, sub_dir))
map_dir = os.path.join(opts.root_dir, "maps")
if not os.path.exists(map_dir):
os.mkdir(map_dir)
run_cmd("chown {0}:{0} {1}".format(opts.username, map_dir))
si_filename = os.path.join(TEMPLATE_DIR, "server_info.py.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(contest_root=opts.root_dir,
database_user=opts.database_user,
database_password=opts.database_password,
database_name=opts.database_name,
map_dir=map_dir, sub_dir=sub_dir)
manager_dir = os.path.join(opts.root_dir, opts.local_repo, "manager")
with CD(manager_dir):
if not os.path.exists("server_info.py"):
with open("server_info.py", "w") as si_file:
si_file.write(si_contents)
run_cmd("chmod 600 server_info.py")
run_cmd("chown -R {0}:{0} {1}".format(opts.username, opts.root_dir))
def setup_base_files(opts):
""" Setup all the contest specific files and directories """
sub_dir = os.path.join(opts.root_dir, "submissions")
if not os.path.exists(sub_dir):
os.mkdir(sub_dir)
run_cmd("chown {0}:{0} {1}".format(opts.username, sub_dir))
map_dir = os.path.join(opts.root_dir, "maps")
if not os.path.exists(map_dir):
os.mkdir(map_dir)
run_cmd("chown {0}:{0} {1}".format(opts.username, map_dir))
si_filename = os.path.join(TEMPLATE_DIR, "server_info.py.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(contest_root=opts.root_dir,
database_user=opts.database_user,
database_password=opts.database_password,
database_name=opts.database_name,
map_dir=map_dir,
sub_dir=sub_dir)
manager_dir = os.path.join(opts.root_dir, opts.local_repo, "manager")
with CD(manager_dir):
if not os.path.exists("server_info.py"):
with open("server_info.py", "w") as si_file:
si_file.write(si_contents)
run_cmd("chmod 600 server_info.py")
run_cmd("chown -R {0}:{0} {1}".format(opts.username, opts.root_dir))
def setup_contest_files(opts):
""" Setup all the contest specific files and directories """
contest_root = opts.root_dir
compiled_dir = os.path.join(contest_root, "compiled")
if not os.path.exists(compiled_dir):
os.mkdir(compiled_dir)
run_cmd("chown {0}:{0} {1}".format(opts.username, compiled_dir))
map_dir = os.path.join(contest_root, "maps")
if not os.path.exists(map_dir):
os.mkdir(map_dir)
run_cmd("chown {0}:{0} {1}".format(opts.username, map_dir))
worker_dir = os.path.join(contest_root, opts.local_repo, "worker")
si_filename = os.path.join(TEMPLATE_DIR, "worker_server_info.py.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(contest_root=contest_root,
map_dir=map_dir,
compiled_dir=compiled_dir,
api_url=opts.api_url,
api_key=opts.api_key)
with CD(worker_dir):
if not os.path.exists("server_info.py"):
with open("server_info.py", "w") as si_file:
si_file.write(si_contents)
run_cmd("chmod 600 server_info.py")
run_cmd("chown -R {0}:{0} {1}".format(opts.username, contest_root))
def setup_jailusers(contest_root):
""" Create and configure the jail users """
worker_dir = os.path.join(contest_root, "aichallenge", "worker")
with CD(worker_dir):
run_cmd("python create_jail_users.py 32")
org_mode = os.stat("/etc/sudoers")[0]
os.chmod("/etc/sudoers", 0640)
append_line("/etc/sudoers", "contest ALL = (%jailusers) NOPASSWD: ALL")
os.chmod("/etc/sudoers", org_mode)
run_cmd("iptables-save > /etc/iptables.rules")
iptablesload_path = "/etc/network/if-pre-up.d/iptablesload"
if not os.path.exists(iptablesload_path):
with open(iptablesload_path, "w") as loadfile:
loadfile.write(IPTABLES_LOAD)
os.chmod(iptablesload_path, 0744)
def install_clojure(download_base):
""" Install the Clojure language """
if os.path.exists("/usr/share/java/clojure.jar"):
return
with CD("/root"):
run_cmd("curl '%s/clojure.zip' > clojure.zip" % (download_base,))
run_cmd("unzip clojure.zip")
run_cmd("cp clojure-1.3.0/clojure-1.3.0.jar /usr/share/java/clojure.jar")
run_cmd("chmod a+r /usr/share/java/clojure.jar")
def install_clojure(download_base):
""" Install the Clojure language """
if os.path.exists("/usr/share/java/clojure.jar"):
return
with CD("/root"):
run_cmd("curl '%s/clojure.zip' > clojure.zip" % (download_base, ))
run_cmd("unzip clojure.zip")
run_cmd(
"cp clojure-1.3.0/clojure-1.3.0.jar /usr/share/java/clojure.jar")
run_cmd("chmod a+r /usr/share/java/clojure.jar")
def install_golang():
""" Install golang from a mercurial release """
RELEASE_TAG = "release.r56"
if os.path.exists("/usr/local/bin/godoc"):
return
pkg_list = ["bison", "ed", "gawk", "libc6-dev", "make",
"python-setuptools", "build-essential", "mercurial"]
install_apt_packages(pkg_list)
try:
os.makedirs("/usr/local/src")
except OSError:
pass
with CD("/usr/local/src"):
run_cmd("hg clone -r %s https://go.googlecode.com/hg/ go"
% (RELEASE_TAG,))
append_line("/root/.bashrc", "export GOROOT=/usr/local/src/go")
append_line("/root/.bashrc", "export GOBIN=/usr/local/bin")
with CD("/usr/local/src/go/src"):
run_cmd("export GOBIN=/usr/local/bin; ./all.bash")
def create_jail_group(options):
""" Create user group for jail users and set limits on it """
if not file_contains("/etc/group", "^jailusers"):
run_cmd("groupadd jailusers")
limits_conf = "/etc/security/limits.conf"
if not file_contains(limits_conf, "@jailusers"):
# limit jailuser processes to:
# 10 processes or system threads
append_line(limits_conf, "@jailusers hard nproc 10 # ai-contest")
# 20 minutes of cpu time
append_line(limits_conf, "@jailusers hard cpu 20 # ai-contest")
# slightly more than 1GB of ram
append_line(limits_conf, "@jailusers hard rss 1048600 # ai-contest")
if not file_contains("/etc/sudoers",
"^%s.+jailusers" % (options.username,)):
org_mode = os.stat("/etc/sudoers")[0]
os.chmod("/etc/sudoers", 0640)
append_line("/etc/sudoers",
"%s ALL = (%%jailusers) NOPASSWD: ALL" % (options.username,))
os.chmod("/etc/sudoers", org_mode)
def main(argv=["worker_setup.py"]):
""" Completely set everything up from a fresh ec2 instance """
opts = get_options(argv)
with Environ("DEBIAN_FRONTEND", "noninteractive"):
if opts.update_system:
run_cmd("apt-get update")
run_cmd("apt-get upgrade -y")
if opts.install_required:
install_required_packages()
if opts.install_utilities:
install_utility_packages()
if opts.install_languages:
install_all_languages()
if opts.packages_only:
return
setup_contest_files(opts)
if opts.create_jails:
setup_jailusers(opts)
start_script = os.path.join(opts.root_dir, opts.local_repo,
"worker/start_worker.sh")
if opts.install_cronjob:
cron_file = "/etc/cron.d/ai-contest"
if not file_contains(cron_file, start_script):
append_line(cron_file, "@reboot root %s" % (start_script,))
if opts.run_worker:
run_cmd(start_script)
def setup_website(opts):
""" Configure apache to serve the website and set a server_info.php """
website_root = os.path.join(opts.root_dir, opts.local_repo, "website")
subs_dir = os.path.join(opts.root_dir, "submissions")
si_filename = os.path.join(TEMPLATE_DIR, "server_info.php.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(
sub_dir=subs_dir,
database_user=opts.database_user,
database_password=opts.database_password,
database_name=opts.database_name,
)
with CD(website_root):
if not os.path.exists("server_info.php"):
with open("server_info.php", "w") as si_file:
si_file.write(si_contents)
log_dir = os.path.join(opts.root_dir, "html_logs")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
run_cmd("chown %s:www-data %s" % (opts.username, log_dir))
os.chmod(log_dir, 0775)
site_config = "/etc/apache2/sites-available/ai-contest"
if not os.path.exists(site_config):
site_filename = os.path.join(TEMPLATE_DIR, "apache_site.template")
with open(site_filename, "r") as site_file:
site_template = site_file.read()
site_contents = site_template.format(web_hostname="ai-contest.com",
web_root=website_root,
log_dir=log_dir)
with open(site_config, "w") as site_file:
site_file.write(site_contents)
if opts.website_as_default:
enabled_link = "/etc/apache2/sites-enabled/000-default"
else:
enabled_link = "/etc/apache2/sites-enabled/ai-contest"
if os.path.exists(enabled_link):
os.remove(enabled_link)
os.symlink(site_config, enabled_link)
run_cmd("/etc/init.d/apache2 restart")
def create_jail_group(options):
""" Create user group for jail users and set limits on it """
if not file_contains("/etc/group", "^jailusers"):
run_cmd("groupadd jailusers")
run_cmd("groupadd jailkeeper")
run_cmd("usermod -a -G jailkeeper %s" % (options.username,))
limits_conf = "/etc/security/limits.conf"
if not file_contains(limits_conf, "@jailusers"):
# limit jailuser processes to:
# 25 processes or system threads
append_line(limits_conf, "@jailusers hard nproc 25 # ai-contest")
# 20 minutes of cpu time
append_line(limits_conf, "@jailusers hard cpu 20 # ai-contest")
# slightly more than 1.5GB of ram
append_line(limits_conf, "@jailusers hard rss 1580000 # ai-contest")
if not file_contains("/etc/sudoers",
"^%s.+jailusers" % (options.username,)):
org_mode = os.stat("/etc/sudoers")[0]
os.chmod("/etc/sudoers", 0640)
append_line("/etc/sudoers",
"%s ALL = (%%jailusers) NOPASSWD: ALL" % (options.username,))
append_line("/etc/sudoers",
"%s ALL = (ALL) NOPASSWD: /bin/mount, /bin/umount" % (
options.username,))
os.chmod("/etc/sudoers", org_mode)
def main(argv=["worker_setup.py"]):
""" Completely set everything up from a fresh ec2 instance """
opts = get_options(argv)
opts.arch = 'i386'
with Environ("DEBIAN_FRONTEND", "noninteractive"):
if opts.update_system:
run_cmd("apt-get update")
run_cmd("apt-get upgrade -y")
if opts.install_required:
install_required_packages()
if opts.install_utilities:
install_utility_packages()
if opts.install_pkg_languages:
install_packaged_languages()
if opts.install_languages:
install_all_languages(opts)
if opts.install_jailguard:
install_jailguard(opts)
if opts.create_jails:
setup_base_chroot(opts)
if opts.packages_only:
return
setup_contest_files(opts)
if opts.create_jails:
setup_base_jail(opts)
setup_jailusers(opts)
start_script = os.path.join(opts.root_dir, "worker/start_worker.sh")
if opts.install_cronjob:
cron_file = "/etc/cron.d/ai-contest"
if not file_contains(cron_file, start_script):
append_line(cron_file, "@reboot %s %s"
% (opts.username, start_script,))
if opts.run_worker:
run_cmd("sudo -u %s %s" % (opts.username, start_script))
def install_nodejs():
""" Install node.js """
if os.path.exists("/usr/local/bin/node"):
return
install_apt_packages("make")
try:
os.makedirs("/usr/local/src/nodejs")
except OSError:
pass
with CD("/usr/local/src/nodejs"):
run_cmd("curl 'http://nodejs.org/dist/node-v0.4.1.tar.gz' | tar -xz")
with CD("/usr/local/src/nodejs/node-v0.4.1"):
run_cmd("./configure")
run_cmd("make")
run_cmd("make install")
def install_clojure():
""" Install the Clojure language """
if os.path.exists("/usr/share/java/clojure.jar"):
return
with CD("/root"):
run_cmd("curl 'http://cloud.github.com/downloads/clojure/clojure/clojure-1.2.0.zip' > clojure-1.2.0.zip")
run_cmd("unzip clojure-1.2.0.zip")
run_cmd("cp clojure-1.2.0/clojure.jar /usr/share/java")
def install_clojure():
""" Install the Clojure language """
if os.path.exists("/usr/share/java/clojure.jar"):
return
with CD("/root"):
run_cmd("curl 'http://cloud.github.com/downloads/clojure/clojure/clojure-1.2.0.zip' > clojure-1.2.0.zip")
run_cmd("unzip clojure-1.2.0.zip")
run_cmd("cp clojure-1.2.0/clojure.jar /usr/share/java")
def setup_website(opts):
""" Configure apache to serve the website and set a server_info.php """
website_root = os.path.join(opts.root_dir, opts.local_repo, "website")
subs_dir = os.path.join(opts.root_dir, "submissions")
si_filename = os.path.join(TEMPLATE_DIR, "server_info.php.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(sub_dir=subs_dir,
database_user=opts.database_user,
database_password=opts.database_password,
database_name=opts.database_name,
)
with CD(website_root):
if not os.path.exists("server_info.php"):
with open("server_info.php", "w") as si_file:
si_file.write(si_contents)
log_dir = os.path.join(opts.root_dir, "html_logs")
if not os.path.exists(log_dir):
os.mkdir(log_dir)
run_cmd("chown %s:www-data %s" % (opts.username, log_dir))
os.chmod(log_dir, 0775)
site_config = "/etc/apache2/sites-available/ai-contest"
if not os.path.exists(site_config):
site_filename = os.path.join(TEMPLATE_DIR, "apache_site.template")
with open(site_filename, "r") as site_file:
site_template = site_file.read()
site_contents = site_template.format(web_hostname="ai-contest.com",
web_root=website_root, log_dir=log_dir)
with open(site_config, "w") as site_file:
site_file.write(site_contents)
if opts.website_as_default:
enabled_link = "/etc/apache2/sites-enabled/000-default"
else:
enabled_link = "/etc/apache2/sites-enabled/ai-contest"
if os.path.exists(enabled_link):
os.remove(enabled_link)
os.symlink(site_config, enabled_link)
run_cmd("/etc/init.d/apache2 restart")
def install_coffeescript(download_base):
""" Install coffeescript """
if os.path.exists("/usr/local/bin/coffee"):
return
with CD("/root"):
run_cmd("curl '%s/coffeescript.tgz' > coffeescript.tgz" % (download_base,))
run_cmd("tar xzf coffeescript.tgz")
with CD("jashkenas-coffee-script-1a652a9"):
run_cmd("bin/cake install")
def install_coffeescript(download_base):
""" Install coffeescript """
if os.path.exists("/usr/local/bin/coffee"):
return
with CD("/root"):
run_cmd("curl '%s/coffeescript.tgz' > coffeescript.tgz" %
(download_base, ))
run_cmd("tar xzf coffeescript.tgz")
with CD("jashkenas-coffee-script-1a652a9"):
run_cmd("bin/cake install")
def setup_base_jail(options):
""" Create and configure base jail """
run_cmd("schroot -c aic-base -- %s/setup/worker_setup.py --chroot-setup"
% (os.path.join(options.root_dir, options.local_repo),))
create_jail_group(options)
iptablesload_path = "/etc/network/if-pre-up.d/iptablesload"
if not os.path.exists(iptablesload_path):
with open(iptablesload_path, "w") as loadfile:
loadfile.write(IPTABLES_LOAD)
os.chmod(iptablesload_path, 0744)
worker_dir = os.path.join(options.root_dir, options.local_repo, "worker")
with CD(worker_dir):
user_info = pwd.getpwnam(options.username)
cuid = user_info.pw_uid
cgid = user_info.pw_gid
jgid = grp.getgrnam("jailusers").gr_gid
run_cmd("gcc -DCONTEST_UID=%d -DCONTEST_GID=%d -DJAIL_GID=%d jail_own.c -o jail_own" % (cuid, cgid, jgid))
run_cmd("chown root:%s jail_own" % (cgid,))
run_cmd("chmod u=rwxs,g=rwx,o= jail_own")
def setup_base_jail(options):
""" Create and configure base jail """
run_cmd("schroot -c aic-base -- %s/setup/worker_setup.py --chroot-setup"
% (os.path.join(options.root_dir, options.local_repo),))
create_jail_group(options)
iptablesload_path = "/etc/network/if-pre-up.d/iptablesload"
if not os.path.exists(iptablesload_path):
with open(iptablesload_path, "w") as loadfile:
loadfile.write(IPTABLES_LOAD)
os.chmod(iptablesload_path, 0744)
worker_dir = os.path.join(options.root_dir, options.local_repo, "worker")
with CD(worker_dir):
user_info = pwd.getpwnam(options.username)
cuid = user_info.pw_uid
cgid = user_info.pw_gid
jgid = grp.getgrnam("jailusers").gr_gid
run_cmd("gcc -DCONTEST_UID=%d -DCONTEST_GID=%d -DJAIL_GID=%d jail_own.c -o jail_own" % (cuid, cgid, jgid))
run_cmd("chown root:%s jail_own" % (cgid,))
run_cmd("chmod u=rwxs,g=rwx,o= jail_own")
def setup_base_chroot(options):
""" Create and setup the base chroot jail users will run in. """
install_apt_packages(["debootstrap", "schroot", "unionfs-fuse", "gcc"])
chroot_dir = "/srv/chroot"
base_chroot_dir = os.path.join(chroot_dir, "aic-base")
if not os.path.exists(base_chroot_dir):
os.makedirs(base_chroot_dir)
run_cmd("debootstrap --variant=buildd --arch %s saucy \
%s %s" % (options.arch, base_chroot_dir, options.os_url))
with CD(TEMPLATE_DIR):
run_cmd("cp chroot_configs/chroot.d/aic-base /etc/schroot/chroot.d/")
with open("chroot_configs/sources.list.template", "r") as sl_file:
sources_contents = sl_file.read()
chroot_filename = "%s/etc/apt/sources.list" % (base_chroot_dir,)
with open(chroot_filename, "w") as sources_file:
sources_file.write(sources_contents)
run_cmd("cp -r chroot_configs/ai-jail /etc/schroot/ai-jail")
deb_archives = "/var/cache/apt/archives/"
run_cmd("cp {0}*.deb {1}{0}".format(deb_archives, base_chroot_dir))
run_cmd("schroot -c aic-base -- /bin/sh -c \"\
DEBIANFRONTEND=noninteractive;\
apt-get update; apt-get upgrade -y\"")
run_cmd("schroot -c aic-base -- apt-get install -y python")
run_cmd("schroot -c aic-base -- %s/setup/worker_setup.py --chroot-base"
% (os.path.join(options.root_dir, options.local_repo, "worker"),))
def install_golang(download_base):
""" Install golang """
with CD("/root"):
run_cmd("curl '%s/golang.deb' > golang.deb" % (download_base,))
run_cmd("dpkg -i golang.deb")
def install_nodejs(download_base):
""" Install node.js """
install_apt_packages(["rlwrap"])
with CD("/root"):
run_cmd("curl '%s/nodejs.deb' > nodejs.deb" % (download_base, ))
run_cmd("dpkg -i nodejs.deb")
def setup_base_chroot(options):
""" Create and setup the base chroot jail users will run in. """
install_apt_packages(["debootstrap", "schroot", "unionfs-fuse", "gcc"])
chroot_dir = "/srv/chroot"
base_chroot_dir = os.path.join(chroot_dir, "aic-base")
if not os.path.exists(base_chroot_dir):
os.makedirs(base_chroot_dir)
run_cmd("debootstrap --variant=buildd --arch %s natty \
%s http://us-east-1.ec2.archive.ubuntu.com/ubuntu/" % (options.arch, base_chroot_dir,))
with CD(TEMPLATE_DIR):
run_cmd("cp chroot_configs/chroot.d/aic-base /etc/schroot/chroot.d/")
run_cmd("cp chroot_configs/sources.list %s/etc/apt/"
% (base_chroot_dir,))
run_cmd("cp -r chroot_configs/ai-jail /etc/schroot/ai-jail")
deb_archives = "/var/cache/apt/archives/"
run_cmd("cp {0}*.deb {1}{0}".format(deb_archives, base_chroot_dir))
run_cmd("schroot -c aic-base -- /bin/sh -c \"\
DEBIANFRONTEND=noninteractive;\
apt-get update; apt-get upgrade -y\"")
run_cmd("schroot -c aic-base -- apt-get install -y python")
run_cmd("schroot -c aic-base -- %s/setup/worker_setup.py --chroot-base"
% (os.path.join(options.root_dir, options.local_repo),))
def setup_website(opts):
""" Configure apache to serve the website and set a server_info.php """
website_root = os.path.join(opts.local_repo, "website")
si_filename = os.path.join(TEMPLATE_DIR, "server_info.php.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(upload_dir=opts.upload_dir,
map_dir=opts.map_dir, replay_dir=opts.replay_dir,
log_dir=opts.log_dir, repo_dir=opts.local_repo,
database_user=opts.database_user,
database_password=opts.database_password,
database_name=opts.database_name,
api_url=opts.website_hostname
)
with CD(website_root):
if not os.path.exists("server_info.php"):
with open("server_info.php", "w") as si_file:
si_file.write(si_contents)
# setup pygments flavored markdown
run_cmd("easy_install ElementTree")
run_cmd("easy_install Markdown")
run_cmd("easy_install Pygments")
if not os.path.exists("aichallenge.wiki"):
run_cmd("git clone git://github.com/aichallenge/aichallenge.wiki.git")
run_cmd("python setup.py")
with CD(os.path.join(opts.local_repo, "ants/dist/starter_bots")):
run_cmd("make")
run_cmd("make install")
if not os.path.exists(os.path.join(website_root, "worker-src.tgz")):
create_worker_archive.main(website_root)
visualizer_path = os.path.join(opts.local_repo, "ants/visualizer")
plugin_path = "/usr/share/icedtea-web/plugin.jar"
if not os.path.exists(os.path.join(website_root, "visualizer")):
with CD(visualizer_path):
run_cmd("ant deploy -Djava.plugin=%s -Ddeploy.path=%s"
% (plugin_path, website_root))
setup_language_repo(opts)
site_config = "/etc/apache2/sites-available/" + opts.website_hostname
if not os.path.exists(site_config):
site_filename = os.path.join(TEMPLATE_DIR, "apache_site.template")
with open(site_filename, "r") as site_file:
site_template = site_file.read()
site_contents = site_template.format(web_hostname=opts.website_hostname,
web_root=website_root,
log_dir=opts.log_dir,
map_dir=opts.map_dir,
replay_dir=opts.replay_dir)
with open(site_config, "w") as site_file:
site_file.write(site_contents)
if opts.website_as_default:
enabled_link = "/etc/apache2/sites-enabled/000-default"
else:
enabled_link = "/etc/apache2/sites-enabled/" + opts.website_hostname
if os.path.exists(enabled_link):
os.remove(enabled_link)
os.symlink(site_config, enabled_link)
run_cmd("a2enmod rewrite")
run_cmd("/etc/init.d/apache2 restart")
run_cmd("chown -R {0}:{0} {1}".format(opts.username, website_root))
def install_jailguard(options):
worker_dir = os.path.join(options.local_repo, "worker")
run_cmd("cp %s/jailguard.py /usr/local/bin/" % (worker_dir,))
def install_nodejs(download_base):
""" Install node.js """
install_apt_packages(["rlwrap"])
with CD("/root"):
run_cmd("curl '%s/nodejs.deb' > nodejs.deb" % (download_base,))
run_cmd("dpkg -i nodejs.deb")
def setup_jailusers(options):
""" Create and configure the jail users """
for user_num in range(1, 33):
create_jail_user("jailuser%s" % (user_num,))
run_cmd("iptables-save > /etc/iptables.rules")
def setup_base_files(opts):
""" Setup all the contest specific files and directories """
if not os.path.exists(opts.upload_dir):
os.mkdir(opts.upload_dir)
run_cmd("chown {0}:www-data {1}".format(opts.username, opts.upload_dir))
os.chmod(opts.upload_dir, 0775)
if not os.path.exists(opts.map_dir):
os.mkdir(opts.map_dir)
run_cmd("chown {0}:www-data {1}".format(opts.username, opts.map_dir))
if not os.path.exists(opts.replay_dir):
os.mkdir(opts.replay_dir)
run_cmd("chown {0}:www-data {1}".format(opts.username, opts.replay_dir))
os.chmod(opts.replay_dir, 0775)
if not os.path.exists(opts.log_dir):
os.mkdir(opts.log_dir)
run_cmd("chown {0}:www-data {1}".format(opts.username, opts.log_dir))
os.chmod(opts.log_dir, 0775)
si_filename = os.path.join(TEMPLATE_DIR, "server_info.py.template")
with open(si_filename, 'r') as si_file:
si_template = si_file.read()
si_contents = si_template.format(contest_root=opts.root_dir,
database_user=opts.database_user,
database_password=opts.database_password,
database_name=opts.database_name,
map_dir=opts.map_dir, upload_dir=opts.upload_dir,
log_dir=opts.log_dir)
manager_dir = os.path.join(opts.local_repo, "manager")
with CD(manager_dir):
if not os.path.exists("server_info.py"):
with open("server_info.py", "w") as si_file:
si_file.write(si_contents)
run_cmd("chown {0}:{0} server_info.py".format(opts.username))
if os.stat(opts.local_repo).st_uid != pwd.getpwnam(opts.username).pw_uid:
run_cmd("chown -R {0}:{0} {1}".format(opts.username, opts.local_repo))
