def setup_participant(player, temp_dir):
"""Download and set up the bot for a game"""
player.bot_id = backend.get_player_compiled_bot_id(player.player_id)
bot_dir = "%s_%s" % (player.player_index, player.player_id)
bot_dir = os.path.join(temp_dir, bot_dir)
os.mkdir(bot_dir)
bot_archive_path = backend.download_bot(player.bot_id,
bot_dir,
is_compiled=True)
archive.unpack(bot_archive_path)
# Make the start script executable
os.chmod(os.path.join(bot_dir, 'run.sh'), 0o755)
# Give the bot user ownership of their directory, worker still have access through each user's default group
bot_user = "******" % player.player_index
bot_group = "bot_%s" % player.player_index
bot_cgroup = "bot_%s" % player.player_index
# We want 770 so that the bot can create files still; leading 2
# is equivalent to g+s which forces new files to be owned by the group
util.give_ownership(bot_dir, 0o2770, bot_group)
bot_command = BOT_COMMAND.format(cgroup=bot_cgroup,
bot_dir=bot_dir,
bot_group=bot_group,
bot_user=bot_user,
runfile='run.sh')
return bot_command, bot_dir
def downloadUsers(users):
for user in users:
userDir = str(user["userID"])
if os.path.isdir(userDir):
shutil.rmtree(userDir)
os.mkdir(userDir)
archive.unpack(backend.storeBotLocally(user["userID"], userDir))
def downloadUsers(users):
for user in users:
userDir = str(user["userID"])
if os.path.isdir(userDir):
shutil.rmtree(userDir)
os.mkdir(userDir)
archive.unpack(backend.storeBotLocally(user["userID"], userDir))
def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json):
try:
# ignore the return value
archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json)
return True
except Exception, e:
print('unpack e.message:%s\t' % e.message)
print('unpack %s failed' % os.path.basename(fullpkg_path))
os.remove(fullpkg_path)
return False
def anonymize(dicom_in, dicom_out,
tags_to_keep=None, forced_values=None):
"""
Configures the Anonymizer and runs it on DICOM files.
"""
if not os.path.exists(dicom_in):
print "The DICOM input does not exist."
return
is_dicom_in_archive = is_archive(dicom_in)
is_dicom_out_archive = is_archive(dicom_out)
if is_dicom_in_archive:
wip_dicom_in = mkdtemp()
unpack(dicom_in, wip_dicom_in)
else:
wip_dicom_in = os.path.abspath(dicom_in)
if is_dicom_out_archive:
wip_dicom_out = mkdtemp()
else:
wip_dicom_out = os.path.abspath(dicom_out)
if os.path.isfile(wip_dicom_in):
if os.path.exists(wip_dicom_out) and os.path.isdir(wip_dicom_out):
wip_dicom_out = os.path.join(
wip_dicom_out, os.path.basename(wip_dicom_in))
anon = Anonymizer(wip_dicom_in, wip_dicom_out, tags_to_keep,
forced_values)
anon.run()
elif os.path.isdir(wip_dicom_in):
if os.path.isfile(wip_dicom_out):
print "Since the input is a directory, an output directory is expected."
return
if not os.path.exists(wip_dicom_out):
os.makedirs(wip_dicom_out)
for root, dirs, files in os.walk(wip_dicom_in):
for name in files:
current_file = os.path.join(root, name)
subdir = root.replace(
wip_dicom_in + '/', '').replace(wip_dicom_in, '')
file_out = os.path.join(
wip_dicom_out, subdir, os.path.basename(current_file))
if not os.path.exists(os.path.dirname(file_out)):
os.makedirs(os.path.dirname(file_out))
anon = Anonymizer(current_file, file_out, tags_to_keep,
forced_values)
anon.run()
else:
print "The input file type is not handled by this tool."
if is_dicom_in_archive:
shutil.rmtree(wip_dicom_in)
if is_dicom_out_archive:
pack(os.path.abspath(dicom_out),
glob(os.path.join(wip_dicom_out, '*')))
shutil.rmtree(wip_dicom_out)
def install_not_git_package(package, package_info, local_pkgs_path,
package_url, bsp_package_path, package_name):
result = True
# Download a package of compressed package type.
if not package.download(package_info['ver'], local_pkgs_path, package_url):
return False
pkg_dir = package.get_filename(package_info['ver'])
pkg_dir = os.path.splitext(pkg_dir)[0]
package_path = os.path.join(local_pkgs_path,
package.get_filename(package_info['ver']))
if not archive.package_integrity_test(package_path):
print("package : %s is invalid" % package_path.encode("utf-8"))
return False
# unpack package
if not os.path.exists(pkg_dir):
try:
if not archive.unpack(package_path, bsp_package_path, package_info,
package_name):
result = False
except Exception as e:
result = False
logging.error('Error message: {0}'.format(e))
else:
print("The file does not exist.")
return result
def executeCompileTask(user, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
print("Compiling a bot with userID %s\n" % str(user["userID"]))
try:
workingPath = "workingPath"
makePath(workingPath)
botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True)
archive.unpack(botPath)
while len([name for name in os.listdir(workingPath) if os.path.isfile(os.path.join(workingPath, name))]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1:
singleFolder = glob.glob(os.path.join(workingPath, "*"))[0]
bufferFolder = os.path.join(workingPath, SECRET_FOLDER)
os.mkdir(bufferFolder)
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename))
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename))
os.rmdir(bufferFolder)
# Rm symlinks
os.system("find "+workingPath+" -type l -delete")
language, errors = compile_anything(workingPath)
didCompile = True if errors == None else False
except Exception as e:
language = "Other"
errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()]
didCompile = False
if didCompile:
print("Bot did compile\n")
archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip"))
backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip"))
else:
print("Bot did not compile\n")
print("Bot errors %s\n" % str(errors))
backend.compileResult(int(user["userID"]), didCompile, language, errors=(None if didCompile else "\n".join(errors)))
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
def executeCompileTask(user, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
print("Compiling a bot with userID %s\n" % str(user["userID"]))
try:
workingPath = "workingPath"
makePath(workingPath)
botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True)
archive.unpack(botPath)
while len([name for name in os.listdir(workingPath) if os.path.isfile(os.path.join(workingPath, name))]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1:
singleFolder = glob.glob(os.path.join(workingPath, "*"))[0]
bufferFolder = os.path.join(workingPath, SECRET_FOLDER)
os.mkdir(bufferFolder)
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename))
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename))
os.rmdir(bufferFolder)
# Rm symlinks
os.system("find "+workingPath+" -type l -delete")
language, errors = compile_anything(workingPath)
didCompile = True if errors == None else False
except Exception as e:
language = "Other"
errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()]
didCompile = False
if didCompile:
print("Bot did compile\n")
archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip"))
backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip"))
else:
print("Bot did not compile\n")
print("Bot errors %s\n" % str(errors))
backend.compileResult(int(user["userID"]), didCompile, language, errors=(None if didCompile else "\n".join(errors)))
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
def filesToDevice():
init()
try:
archive.assertReady()
drive.assertReady()
warnUser()
if ok('\nReady to initialize the hard drive. All data will be erased. LAST CHANCE to abort.'
):
drive.initDrive()
drive.setupEfi()
drive.setupSys()
drive.mount(False)
if ok('\nReady to unpack the system onto the new partition.'):
archive.unpack()
if ok('\nReady to update UUIDs.'):
drive.updateUuids()
print 'All done!'
except UserAbort:
print('Process aborted by user.')
except Exception, err:
print('Error: {}'.format(err))
def executeCompileTask(user, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
print("Compiling a bot with userID %s" % (user["userID"]))
try:
workingPath = "workingPath"
makePath(workingPath)
botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True)
archive.unpack(botPath)
while len([name for name in os.listdir(workingPath) if os.path.isfile(name)]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1:
singleFolder = glob.glob(os.path.join(workingPath, "*"))[0]
bufferFolder = os.path.join(workingPath, SECRET_FOLDER)
os.mkdir(bufferFolder)
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename))
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename))
os.rmdir(bufferFolder)
language, errors = compile_anything(workingPath)
didCompile = True if errors == None else False
except Exception as e:
language = "Other"
errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()]
didCompile = False
if didCompile:
print("Bot did compile")
archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip"))
backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip"))
else:
print("Bot did not compile")
print(str(errors))
sendEmail("Halite Bot Compilation Error", "<h2>The bot that you recently submitted to the Halite competition would not compile on our servers.</h2> <p>Our autocompile script <b>thought that your bot was written in \""+language+".\"</b> If that is incorrect, please change your code's file extensions to <code>cpp</code> and <code>h</code> for C++11, <code>java</code> for Java 7, <code>py</code> for Python3, and <code>rs</code> for Rust 1.10. Make sure to include a <code>Cargo.toml</code> file if you are using Rust. Please make sure that your <b>main file is named MyBot</b> (not main, not BasicBot).</p> <b>Here is a description of the compilation error</b>:<br><pre><code>"+"<br>".join(errors)+"</code></pre>", user["email"])
backend.compileResult(int(user["userID"]), didCompile, language)
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
def runGame(width, height, users):
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
shutil.copy(ENVIRONMENT, os.path.join(temp_dir, ENVIRONMENT))
command = [
"./" + ENVIRONMENT,
"-d",
"{} {}".format(width, height),
"-q",
"-o",
]
# Make sure bots have access to the temp dir as a whole
# Otherwise, Python can't import modules from the bot dir
# Based on strace, Python lstat()s the full dir path to the dir it's
# in, and fails when it tries to lstat the temp dir, which this
# fixes
os.chmod(temp_dir, 0o755)
for user_index, user in enumerate(users):
bot_dir = "{}_{}".format(user["user_id"], user["bot_id"])
bot_dir = os.path.join(temp_dir, bot_dir)
os.mkdir(bot_dir)
archive.unpack(
backend.storeBotLocally(user["user_id"], user["bot_id"],
bot_dir))
# Make the start script executable
os.chmod(os.path.join(bot_dir, RUNFILE), 0o755)
# Give the bot user ownership of their directory
# We should set up each user's default group as a group that the
# worker is also a part of. Then we always have access to their
# files, but not vice versa.
# https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group
bot_user = "******".format(user_index)
bot_cgroup = "bot_{}".format(user_index)
# We want 775 so that the bot can create files still; leading 2
# is equivalent to g+s which forces new files to be owned by the
# group
give_ownership(bot_dir, "bots", 0o2775)
command.append(
BOT_COMMAND.format(
cgroup=bot_cgroup,
bot_dir=bot_dir,
bot_user=bot_user,
runfile=RUNFILE,
))
command.append("{} v{}".format(user["username"],
user["version_number"]))
logging.debug("Run game command %s\n" % command)
logging.debug("Waiting for game output...\n")
lines = subprocess.Popen(
command,
stdout=subprocess.PIPE).stdout.read().decode('utf-8').split('\n')
logging.debug("\n-----Here is game output: -----")
logging.debug("\n".join(lines))
logging.debug("--------------------------------\n")
# tempdir will automatically be cleaned up, but we need to do things
# manually because the bot might have made files it owns
for user_index, user in enumerate(users):
bot_user = "******".format(user_index)
rm_as_user(bot_user, temp_dir)
# The processes won't necessarily be automatically cleaned up, so
# let's do it ourselves
util.kill_processes_as(bot_user, "bash")
return lines
def unpack(self, fullpkg_path, path):
try:
archive.unpack(fullpkg_path, path)
except Exception, e:
print 'unpack %s failed' % os.path.basename(fullpkg_path)
def handle_compile_task(bot_id):
"""Downloads and compiles a bot, then posts the compiled bot archive back through the API"""
errors = []
util.rm_everything_owned_in_tmp_and_home_by("bot_compilation")
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
try:
bot_path = backend.download_bot(bot_id,
temp_dir,
is_compiled=False)
archive.unpack(bot_path)
# Make sure things are in the top-level directory so people can zip their bot from inside or outside the directory
while len([
name for name in os.listdir(temp_dir)
if os.path.isfile(os.path.join(temp_dir, name))
]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1:
with tempfile.TemporaryDirectory(
dir=TEMP_DIR) as buffer_folder:
single_folder = glob.glob(os.path.join(temp_dir, "*"))[0]
for filename in os.listdir(single_folder):
shutil.move(os.path.join(single_folder, filename),
buffer_folder)
os.rmdir(single_folder)
for filename in os.listdir(buffer_folder):
shutil.move(os.path.join(buffer_folder, filename),
temp_dir)
# Delete any symlinks
subprocess.call(["find", temp_dir, "-type", "l", "-delete"])
# Replace DOS line endings with unix line endings in sh scripts
try:
logging.debug(
"Replacing DOS line endings with UNIX line endings...")
ps_find = subprocess.Popen([
'find', temp_dir, '-name', '*.sh', '-type', 'f', '-print0'
],
stdout=subprocess.PIPE)
dos2unix_output = subprocess.check_output(
['xargs', '-0', 'dos2unix'], stdin=ps_find.stdout)
ps_find.wait()
logging.debug(dos2unix_output)
except Exception as ex:
logging.error(
"An error occured while converting dos line endings in sh files",
ex)
# Give the compilation user access
os.chmod(temp_dir, 0o755)
# Compilation user needs to be able to write to the directory and create files
util.give_ownership(temp_dir, 0o2770, "bot_compilation")
# Reset cwd before compilation, in case it was in a deleted temporary folder
os.chdir(os.path.dirname(os.path.realpath(sys.argv[0])))
language, more_errors = compiler.compile_anything(temp_dir)
did_compile = more_errors is None
if more_errors:
errors.extend(more_errors)
except Exception as ex:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc(),
str(ex)] + errors
did_compile = False
try:
if did_compile:
logging.debug('Bot did compile')
# Make things group-readable
subprocess.call([
"sudo", "-H", "-u", "bot_compilation", "-s", "chmod", "-R",
"g+r", temp_dir
],
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL)
archive_path = os.path.join(temp_dir, str(bot_id) + ".zip")
archive.pack(temp_dir, archive_path)
backend.upload_bot(bot_id, archive_path)
else:
logging.debug("Bot did not compile")
logging.debug("Bot errors: %s" % "\n".join(errors))
backend.send_compilation_result(bot_id, did_compile, language,
"\n".join(errors))
except Exception as ex:
logging.error("Bot did not upload", ex)
errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc())
backend.send_compilation_result(bot_id, False, language,
"\n".join(errors))
finally:
# Remove files as bot user (Python will clean up tempdir, but we don't necessarily have permissions to clean up files)
util.chmod_recursive("bot_compilation", temp_dir, "755")
util.rm_as_user("bot_compilation", temp_dir)
util.rm_everything_owned_in_tmp_and_home_by("bot_compilation")
def executeCompileTask(user_id, bot_id, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
logging.debug("Compiling a bot with userID %s\n" % str(user_id))
errors = []
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
try:
bot_path = backend.storeBotLocally(user_id, bot_id, temp_dir,
is_compile=True)
archive.unpack(bot_path)
# Make sure things are in the top-level directory
while len([
name for name in os.listdir(temp_dir)
if os.path.isfile(os.path.join(temp_dir, name))
]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1:
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as bufferFolder:
singleFolder = glob.glob(os.path.join(temp_dir, "*"))[0]
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), bufferFolder)
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), temp_dir)
# Context manager takes care of buffer folder
# Delete any symlinks
subprocess.call(["find", temp_dir, "-type", "l", "-delete"])
# Give the compilation user access
os.chmod(temp_dir, 0o755)
# User needs to be able to write to the directory and create files
give_ownership(temp_dir, "bots", 0o2770)
# Reset cwd before compilation, in case it was in a
# deleted temporary folder
os.chdir(os.path.dirname(os.path.realpath(sys.argv[0])))
language, more_errors = compiler.compile_anything(temp_dir)
didCompile = more_errors is None
if more_errors:
errors.extend(more_errors)
except Exception:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors
didCompile = False
try:
if didCompile:
logging.debug("Bot did compile\n")
archive_path = os.path.join(temp_dir, str(user_id)+".zip")
archive.zipFolder(temp_dir, archive_path)
backend.storeBotRemotely(user_id, bot_id, archive_path)
else:
logging.debug("Bot did not compile\n")
logging.debug("Bot errors %s\n" % str(errors))
backend.compileResult(user_id, bot_id, didCompile, language,
errors=(None if didCompile else "\n".join(errors)))
except Exception as e:
logging.error("Bot did not upload", e)
errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc())
backend.compileResult(user_id, bot_id, False, language,
errors="\n".join(errors))
finally:
# Remove files as bot user (Python will clean up tempdir, but we don't
# necessarily have permissions to clean up files)
rm_as_user("bot_compilation", temp_dir)
def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json):
try:
archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json)
except Exception, e:
print('e.message:%s\t' % e.message)
print('unpack %s failed' % os.path.basename(fullpkg_path))
def setupParticipant(user_index, user, temp_dir):
"""
Download and set up the bot for a game participant.
"""
# Include username to deal with duplicate bots
bot_dir = "{}_{}_{}".format(user["user_id"], user["bot_id"], user["username"])
bot_dir = os.path.join(temp_dir, bot_dir)
os.mkdir(bot_dir)
archive.unpack(backend.storeBotLocally(user["user_id"],
user["bot_id"], bot_dir))
if user.get("requires_compilation"):
compile_dir = bot_dir + '_compile'
try:
# Move to temp directory to avoid permission problems
# (can't chown files created by compile user back to us)
shutil.move(bot_dir, compile_dir)
# Give the compilation user access
os.chmod(compile_dir, 0o2755)
# User needs to be able to write to the directory
give_ownership(compile_dir, "bots", 0o2774)
language, errors = compiler.compile_anything(compile_dir)
didCompile = errors is None
except Exception:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors
didCompile = False
if not didCompile:
# Abort and upload an error log
rm_as_user("bot_compilation", compile_dir)
raise OndemandCompileError(language, '\n'.join(errors))
# Move back to original directory
try:
shutil.copytree(compile_dir, bot_dir)
except shutil.Error as e:
logging.error("Could not compile bot ondemand", e)
rm_as_user("bot_compilation", compile_dir)
# Make the start script executable
os.chmod(os.path.join(bot_dir, RUNFILE), 0o755)
# Give the bot user ownership of their directory
# We should set up each user's default group as a group that the
# worker is also a part of. Then we always have access to their
# files, but not vice versa.
# https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group
bot_user = "******".format(user_index)
bot_group = "bots_{}".format(user_index)
bot_cgroup = "bot_{}".format(user_index)
# We want 775 so that the bot can create files still; leading 2
# is equivalent to g+s which forces new files to be owned by the
# group
give_ownership(bot_dir, bot_group, 0o2775)
bot_command = BOT_COMMAND.format(
cgroup=bot_cgroup,
bot_dir=bot_dir,
bot_group=bot_group,
bot_user=bot_user,
runfile=RUNFILE,
)
bot_name = "{} v{}".format(user["username"], user["version_number"])
return bot_command, bot_name, bot_dir
def _unpack_file(self, input_file, output_dir):
g_log.info("Extracting %s to %s..." % (input_file, output_dir))
self._remove_file(archive.get_output_path(input_file, output_dir))
archive.unpack(input_file, output_dir)
g_log.info("Extracting %s... DONE" % (input_file))
