def setup_participant(player, temp_dir): """Download and set up the bot for a game""" player.bot_id = backend.get_player_compiled_bot_id(player.player_id) bot_dir = "%s_%s" % (player.player_index, player.player_id) bot_dir = os.path.join(temp_dir, bot_dir) os.mkdir(bot_dir) bot_archive_path = backend.download_bot(player.bot_id, bot_dir, is_compiled=True) archive.unpack(bot_archive_path) # Make the start script executable os.chmod(os.path.join(bot_dir, 'run.sh'), 0o755) # Give the bot user ownership of their directory, worker still have access through each user's default group bot_user = "******" % player.player_index bot_group = "bot_%s" % player.player_index bot_cgroup = "bot_%s" % player.player_index # We want 770 so that the bot can create files still; leading 2 # is equivalent to g+s which forces new files to be owned by the group util.give_ownership(bot_dir, 0o2770, bot_group) bot_command = BOT_COMMAND.format(cgroup=bot_cgroup, bot_dir=bot_dir, bot_group=bot_group, bot_user=bot_user, runfile='run.sh') return bot_command, bot_dir
def downloadUsers(users): for user in users: userDir = str(user["userID"]) if os.path.isdir(userDir): shutil.rmtree(userDir) os.mkdir(userDir) archive.unpack(backend.storeBotLocally(user["userID"], userDir))
def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): try: # ignore the return value archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) return True except Exception, e: print('unpack e.message:%s\t' % e.message) print('unpack %s failed' % os.path.basename(fullpkg_path)) os.remove(fullpkg_path) return False
def anonymize(dicom_in, dicom_out, tags_to_keep=None, forced_values=None): """ Configures the Anonymizer and runs it on DICOM files. """ if not os.path.exists(dicom_in): print "The DICOM input does not exist." return is_dicom_in_archive = is_archive(dicom_in) is_dicom_out_archive = is_archive(dicom_out) if is_dicom_in_archive: wip_dicom_in = mkdtemp() unpack(dicom_in, wip_dicom_in) else: wip_dicom_in = os.path.abspath(dicom_in) if is_dicom_out_archive: wip_dicom_out = mkdtemp() else: wip_dicom_out = os.path.abspath(dicom_out) if os.path.isfile(wip_dicom_in): if os.path.exists(wip_dicom_out) and os.path.isdir(wip_dicom_out): wip_dicom_out = os.path.join( wip_dicom_out, os.path.basename(wip_dicom_in)) anon = Anonymizer(wip_dicom_in, wip_dicom_out, tags_to_keep, forced_values) anon.run() elif os.path.isdir(wip_dicom_in): if os.path.isfile(wip_dicom_out): print "Since the input is a directory, an output directory is expected." return if not os.path.exists(wip_dicom_out): os.makedirs(wip_dicom_out) for root, dirs, files in os.walk(wip_dicom_in): for name in files: current_file = os.path.join(root, name) subdir = root.replace( wip_dicom_in + '/', '').replace(wip_dicom_in, '') file_out = os.path.join( wip_dicom_out, subdir, os.path.basename(current_file)) if not os.path.exists(os.path.dirname(file_out)): os.makedirs(os.path.dirname(file_out)) anon = Anonymizer(current_file, file_out, tags_to_keep, forced_values) anon.run() else: print "The input file type is not handled by this tool." if is_dicom_in_archive: shutil.rmtree(wip_dicom_in) if is_dicom_out_archive: pack(os.path.abspath(dicom_out), glob(os.path.join(wip_dicom_out, '*'))) shutil.rmtree(wip_dicom_out)
def install_not_git_package(package, package_info, local_pkgs_path, package_url, bsp_package_path, package_name): result = True # Download a package of compressed package type. if not package.download(package_info['ver'], local_pkgs_path, package_url): return False pkg_dir = package.get_filename(package_info['ver']) pkg_dir = os.path.splitext(pkg_dir)[0] package_path = os.path.join(local_pkgs_path, package.get_filename(package_info['ver'])) if not archive.package_integrity_test(package_path): print("package : %s is invalid" % package_path.encode("utf-8")) return False # unpack package if not os.path.exists(pkg_dir): try: if not archive.unpack(package_path, bsp_package_path, package_info, package_name): result = False except Exception as e: result = False logging.error('Error message: {0}'.format(e)) else: print("The file does not exist.") return result
def executeCompileTask(user, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" print("Compiling a bot with userID %s\n" % str(user["userID"])) try: workingPath = "workingPath" makePath(workingPath) botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True) archive.unpack(botPath) while len([name for name in os.listdir(workingPath) if os.path.isfile(os.path.join(workingPath, name))]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1: singleFolder = glob.glob(os.path.join(workingPath, "*"))[0] bufferFolder = os.path.join(workingPath, SECRET_FOLDER) os.mkdir(bufferFolder) for filename in os.listdir(singleFolder): shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename)) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename)) os.rmdir(bufferFolder) # Rm symlinks os.system("find "+workingPath+" -type l -delete") language, errors = compile_anything(workingPath) didCompile = True if errors == None else False except Exception as e: language = "Other" errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()] didCompile = False if didCompile: print("Bot did compile\n") archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip")) backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip")) else: print("Bot did not compile\n") print("Bot errors %s\n" % str(errors)) backend.compileResult(int(user["userID"]), didCompile, language, errors=(None if didCompile else "\n".join(errors))) if os.path.isdir(workingPath): shutil.rmtree(workingPath)
def filesToDevice(): init() try: archive.assertReady() drive.assertReady() warnUser() if ok('\nReady to initialize the hard drive. All data will be erased. LAST CHANCE to abort.' ): drive.initDrive() drive.setupEfi() drive.setupSys() drive.mount(False) if ok('\nReady to unpack the system onto the new partition.'): archive.unpack() if ok('\nReady to update UUIDs.'): drive.updateUuids() print 'All done!' except UserAbort: print('Process aborted by user.') except Exception, err: print('Error: {}'.format(err))
def executeCompileTask(user, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" print("Compiling a bot with userID %s" % (user["userID"])) try: workingPath = "workingPath" makePath(workingPath) botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True) archive.unpack(botPath) while len([name for name in os.listdir(workingPath) if os.path.isfile(name)]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1: singleFolder = glob.glob(os.path.join(workingPath, "*"))[0] bufferFolder = os.path.join(workingPath, SECRET_FOLDER) os.mkdir(bufferFolder) for filename in os.listdir(singleFolder): shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename)) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename)) os.rmdir(bufferFolder) language, errors = compile_anything(workingPath) didCompile = True if errors == None else False except Exception as e: language = "Other" errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()] didCompile = False if didCompile: print("Bot did compile") archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip")) backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip")) else: print("Bot did not compile") print(str(errors)) sendEmail("Halite Bot Compilation Error", "<h2>The bot that you recently submitted to the Halite competition would not compile on our servers.</h2> <p>Our autocompile script <b>thought that your bot was written in \""+language+".\"</b> If that is incorrect, please change your code's file extensions to <code>cpp</code> and <code>h</code> for C++11, <code>java</code> for Java 7, <code>py</code> for Python3, and <code>rs</code> for Rust 1.10. Make sure to include a <code>Cargo.toml</code> file if you are using Rust. Please make sure that your <b>main file is named MyBot</b> (not main, not BasicBot).</p> <b>Here is a description of the compilation error</b>:<br><pre><code>"+"<br>".join(errors)+"</code></pre>", user["email"]) backend.compileResult(int(user["userID"]), didCompile, language) if os.path.isdir(workingPath): shutil.rmtree(workingPath)
def runGame(width, height, users): with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: shutil.copy(ENVIRONMENT, os.path.join(temp_dir, ENVIRONMENT)) command = [ "./" + ENVIRONMENT, "-d", "{} {}".format(width, height), "-q", "-o", ] # Make sure bots have access to the temp dir as a whole # Otherwise, Python can't import modules from the bot dir # Based on strace, Python lstat()s the full dir path to the dir it's # in, and fails when it tries to lstat the temp dir, which this # fixes os.chmod(temp_dir, 0o755) for user_index, user in enumerate(users): bot_dir = "{}_{}".format(user["user_id"], user["bot_id"]) bot_dir = os.path.join(temp_dir, bot_dir) os.mkdir(bot_dir) archive.unpack( backend.storeBotLocally(user["user_id"], user["bot_id"], bot_dir)) # Make the start script executable os.chmod(os.path.join(bot_dir, RUNFILE), 0o755) # Give the bot user ownership of their directory # We should set up each user's default group as a group that the # worker is also a part of. Then we always have access to their # files, but not vice versa. # https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group bot_user = "******".format(user_index) bot_cgroup = "bot_{}".format(user_index) # We want 775 so that the bot can create files still; leading 2 # is equivalent to g+s which forces new files to be owned by the # group give_ownership(bot_dir, "bots", 0o2775) command.append( BOT_COMMAND.format( cgroup=bot_cgroup, bot_dir=bot_dir, bot_user=bot_user, runfile=RUNFILE, )) command.append("{} v{}".format(user["username"], user["version_number"])) logging.debug("Run game command %s\n" % command) logging.debug("Waiting for game output...\n") lines = subprocess.Popen( command, stdout=subprocess.PIPE).stdout.read().decode('utf-8').split('\n') logging.debug("\n-----Here is game output: -----") logging.debug("\n".join(lines)) logging.debug("--------------------------------\n") # tempdir will automatically be cleaned up, but we need to do things # manually because the bot might have made files it owns for user_index, user in enumerate(users): bot_user = "******".format(user_index) rm_as_user(bot_user, temp_dir) # The processes won't necessarily be automatically cleaned up, so # let's do it ourselves util.kill_processes_as(bot_user, "bash") return lines
def unpack(self, fullpkg_path, path): try: archive.unpack(fullpkg_path, path) except Exception, e: print 'unpack %s failed' % os.path.basename(fullpkg_path)
def handle_compile_task(bot_id): """Downloads and compiles a bot, then posts the compiled bot archive back through the API""" errors = [] util.rm_everything_owned_in_tmp_and_home_by("bot_compilation") with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: try: bot_path = backend.download_bot(bot_id, temp_dir, is_compiled=False) archive.unpack(bot_path) # Make sure things are in the top-level directory so people can zip their bot from inside or outside the directory while len([ name for name in os.listdir(temp_dir) if os.path.isfile(os.path.join(temp_dir, name)) ]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1: with tempfile.TemporaryDirectory( dir=TEMP_DIR) as buffer_folder: single_folder = glob.glob(os.path.join(temp_dir, "*"))[0] for filename in os.listdir(single_folder): shutil.move(os.path.join(single_folder, filename), buffer_folder) os.rmdir(single_folder) for filename in os.listdir(buffer_folder): shutil.move(os.path.join(buffer_folder, filename), temp_dir) # Delete any symlinks subprocess.call(["find", temp_dir, "-type", "l", "-delete"]) # Replace DOS line endings with unix line endings in sh scripts try: logging.debug( "Replacing DOS line endings with UNIX line endings...") ps_find = subprocess.Popen([ 'find', temp_dir, '-name', '*.sh', '-type', 'f', '-print0' ], stdout=subprocess.PIPE) dos2unix_output = subprocess.check_output( ['xargs', '-0', 'dos2unix'], stdin=ps_find.stdout) ps_find.wait() logging.debug(dos2unix_output) except Exception as ex: logging.error( "An error occured while converting dos line endings in sh files", ex) # Give the compilation user access os.chmod(temp_dir, 0o755) # Compilation user needs to be able to write to the directory and create files util.give_ownership(temp_dir, 0o2770, "bot_compilation") # Reset cwd before compilation, in case it was in a deleted temporary folder os.chdir(os.path.dirname(os.path.realpath(sys.argv[0]))) language, more_errors = compiler.compile_anything(temp_dir) did_compile = more_errors is None if more_errors: errors.extend(more_errors) except Exception as ex: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc(), str(ex)] + errors did_compile = False try: if did_compile: logging.debug('Bot did compile') # Make things group-readable subprocess.call([ "sudo", "-H", "-u", "bot_compilation", "-s", "chmod", "-R", "g+r", temp_dir ], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) archive_path = os.path.join(temp_dir, str(bot_id) + ".zip") archive.pack(temp_dir, archive_path) backend.upload_bot(bot_id, archive_path) else: logging.debug("Bot did not compile") logging.debug("Bot errors: %s" % "\n".join(errors)) backend.send_compilation_result(bot_id, did_compile, language, "\n".join(errors)) except Exception as ex: logging.error("Bot did not upload", ex) errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc()) backend.send_compilation_result(bot_id, False, language, "\n".join(errors)) finally: # Remove files as bot user (Python will clean up tempdir, but we don't necessarily have permissions to clean up files) util.chmod_recursive("bot_compilation", temp_dir, "755") util.rm_as_user("bot_compilation", temp_dir) util.rm_everything_owned_in_tmp_and_home_by("bot_compilation")
def executeCompileTask(user_id, bot_id, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" logging.debug("Compiling a bot with userID %s\n" % str(user_id)) errors = [] with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: try: bot_path = backend.storeBotLocally(user_id, bot_id, temp_dir, is_compile=True) archive.unpack(bot_path) # Make sure things are in the top-level directory while len([ name for name in os.listdir(temp_dir) if os.path.isfile(os.path.join(temp_dir, name)) ]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as bufferFolder: singleFolder = glob.glob(os.path.join(temp_dir, "*"))[0] for filename in os.listdir(singleFolder): shutil.move(os.path.join(singleFolder, filename), bufferFolder) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move(os.path.join(bufferFolder, filename), temp_dir) # Context manager takes care of buffer folder # Delete any symlinks subprocess.call(["find", temp_dir, "-type", "l", "-delete"]) # Give the compilation user access os.chmod(temp_dir, 0o755) # User needs to be able to write to the directory and create files give_ownership(temp_dir, "bots", 0o2770) # Reset cwd before compilation, in case it was in a # deleted temporary folder os.chdir(os.path.dirname(os.path.realpath(sys.argv[0]))) language, more_errors = compiler.compile_anything(temp_dir) didCompile = more_errors is None if more_errors: errors.extend(more_errors) except Exception: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors didCompile = False try: if didCompile: logging.debug("Bot did compile\n") archive_path = os.path.join(temp_dir, str(user_id)+".zip") archive.zipFolder(temp_dir, archive_path) backend.storeBotRemotely(user_id, bot_id, archive_path) else: logging.debug("Bot did not compile\n") logging.debug("Bot errors %s\n" % str(errors)) backend.compileResult(user_id, bot_id, didCompile, language, errors=(None if didCompile else "\n".join(errors))) except Exception as e: logging.error("Bot did not upload", e) errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc()) backend.compileResult(user_id, bot_id, False, language, errors="\n".join(errors)) finally: # Remove files as bot user (Python will clean up tempdir, but we don't # necessarily have permissions to clean up files) rm_as_user("bot_compilation", temp_dir)
def unpack(self, fullpkg_path, path, pkg, pkgs_name_in_json): try: archive.unpack(fullpkg_path, path, pkg, pkgs_name_in_json) except Exception, e: print('e.message:%s\t' % e.message) print('unpack %s failed' % os.path.basename(fullpkg_path))
def setupParticipant(user_index, user, temp_dir): """ Download and set up the bot for a game participant. """ # Include username to deal with duplicate bots bot_dir = "{}_{}_{}".format(user["user_id"], user["bot_id"], user["username"]) bot_dir = os.path.join(temp_dir, bot_dir) os.mkdir(bot_dir) archive.unpack(backend.storeBotLocally(user["user_id"], user["bot_id"], bot_dir)) if user.get("requires_compilation"): compile_dir = bot_dir + '_compile' try: # Move to temp directory to avoid permission problems # (can't chown files created by compile user back to us) shutil.move(bot_dir, compile_dir) # Give the compilation user access os.chmod(compile_dir, 0o2755) # User needs to be able to write to the directory give_ownership(compile_dir, "bots", 0o2774) language, errors = compiler.compile_anything(compile_dir) didCompile = errors is None except Exception: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors didCompile = False if not didCompile: # Abort and upload an error log rm_as_user("bot_compilation", compile_dir) raise OndemandCompileError(language, '\n'.join(errors)) # Move back to original directory try: shutil.copytree(compile_dir, bot_dir) except shutil.Error as e: logging.error("Could not compile bot ondemand", e) rm_as_user("bot_compilation", compile_dir) # Make the start script executable os.chmod(os.path.join(bot_dir, RUNFILE), 0o755) # Give the bot user ownership of their directory # We should set up each user's default group as a group that the # worker is also a part of. Then we always have access to their # files, but not vice versa. # https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group bot_user = "******".format(user_index) bot_group = "bots_{}".format(user_index) bot_cgroup = "bot_{}".format(user_index) # We want 775 so that the bot can create files still; leading 2 # is equivalent to g+s which forces new files to be owned by the # group give_ownership(bot_dir, bot_group, 0o2775) bot_command = BOT_COMMAND.format( cgroup=bot_cgroup, bot_dir=bot_dir, bot_group=bot_group, bot_user=bot_user, runfile=RUNFILE, ) bot_name = "{} v{}".format(user["username"], user["version_number"]) return bot_command, bot_name, bot_dir
def _unpack_file(self, input_file, output_dir): g_log.info("Extracting %s to %s..." % (input_file, output_dir)) self._remove_file(archive.get_output_path(input_file, output_dir)) archive.unpack(input_file, output_dir) g_log.info("Extracting %s... DONE" % (input_file))