def test_extract_tar(self): '''test_extract_tar will test extraction of a tar.gz file ''' print("Testing utils.extract_tar...") # First create a temporary tar file from utils import extract_tar from glob import glob import tarfile # Create and close a temporary tar.gz print("Case 1: Testing tar.gz...") creation_dir = tempfile.mkdtemp() archive, files = create_test_tar(creation_dir) # Extract to different directory extract_dir = tempfile.mkdtemp() extract_tar(archive=archive, output_folder=extract_dir) extracted_files = [ x.replace(extract_dir, '') for x in glob("%s/tmp/*" % (extract_dir)) ] [self.assertTrue(x in files) for x in extracted_files] # Clean up for dirname in [extract_dir, creation_dir]: shutil.rmtree(dirname) print("Case 2: Testing tar...") creation_dir = tempfile.mkdtemp() archive, files = create_test_tar(creation_dir, compressed=False) # Extract to different directory extract_dir = tempfile.mkdtemp() extract_tar(archive=archive, output_folder=extract_dir) extracted_files = [ x.replace(extract_dir, '') for x in glob("%s/tmp/*" % (extract_dir)) ] [self.assertTrue(x in files) for x in extracted_files] print("Case 3: Testing that extract_tar returns None on error...") creation_dir = tempfile.mkdtemp() archive, files = create_test_tar(creation_dir, compressed=False) extract_dir = tempfile.mkdtemp() shutil.rmtree(extract_dir) output = extract_tar(archive=archive, output_folder=extract_dir) self.assertEqual(output, None)
def test_extract_tar(self): '''test_extract_tar will test extraction of a tar.gz file ''' print("Testing utils.extract_tar...") # First create a temporary tar file from utils import extract_tar from glob import glob import tarfile # Create and close a temporary tar.gz print("Case 1: Testing tar.gz...") creation_dir = tempfile.mkdtemp() archive,files = create_test_tar(creation_dir) # Extract to different directory extract_dir = tempfile.mkdtemp() extract_tar(archive=archive, output_folder=extract_dir) extracted_files = [x.replace(extract_dir,'') for x in glob("%s/tmp/*" %(extract_dir))] [self.assertTrue(x in files) for x in extracted_files] # Clean up for dirname in [extract_dir,creation_dir]: shutil.rmtree(dirname) print("Case 2: Testing tar...") creation_dir = tempfile.mkdtemp() archive,files = create_test_tar(creation_dir,compressed=False) # Extract to different directory extract_dir = tempfile.mkdtemp() extract_tar(archive=archive, output_folder=extract_dir) extracted_files = [x.replace(extract_dir,'') for x in glob("%s/tmp/*" %(extract_dir))] [self.assertTrue(x in files) for x in extracted_files] print("Case 3: Testing that extract_tar returns None on error...") creation_dir = tempfile.mkdtemp() archive,files = create_test_tar(creation_dir,compressed=False) extract_dir = tempfile.mkdtemp() shutil.rmtree(extract_dir) output = extract_tar(archive=archive, output_folder=extract_dir) self.assertEqual(output,None)
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error( "root file system not specified OR defined as environmental variable, exiting!" ) sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image = args.shub manifest = get_shub_manifest(image) if args.pull_folder == None: cache_base = get_cache(subfolder="shub", disable_cache=args.disable_cache) else: cache_base = args.pull_folder # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" % (cache_base, image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." % image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug( "Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s", singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" % singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" % singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace image = parse_image_uri(image=image, uri="docker://") namespace = image['namespace'] repo_name = image['repo_name'] repo_tag = image['repo_tag'] # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace, repo_name, repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache=args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" % (cache_base, image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz, singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec = "Cmd" else: spec = "Entrypoint" cmd = get_config(manifest, spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." % (spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def run(args): # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None and args.shub == None: logger.error("root file system not specified OR defined as environmental variable, exiting!") sys.exit(1) if singularity_rootfs != None: logger.info("Root file system defined as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to download a Singularity image? if args.shub != None: image_id = int(args.shub) manifest = get_shub_manifest(image_id) cache_base = get_cache(subfolder="shub", disable_cache = args.disable_cache) # The image name is the md5 hash, download if it's not there image_name = get_image_name(manifest) image_file = "%s/%s" %(cache_base,image_name) if not os.path.exists(image_file): image_file = download_image(manifest=manifest, download_folder=cache_base) else: print("Image already exists at %s, skipping download." %image_file) logger.info("Singularity Hub Image Download: %s", image_file) # If singularity_rootfs is provided, write metadata to it if singularity_rootfs != None: logger.debug("Writing SINGULARITY_RUNDIR and SINGULARITY_IMAGE to %s",singularity_rootfs) write_file("%s/SINGULARITY_RUNDIR" %singularity_rootfs, os.path.dirname(image_file)) write_file("%s/SINGULARITY_IMAGE" %singularity_rootfs, image_file) # Do we have a docker image specified? elif args.docker != None: # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) image = args.docker logger.info("Docker image: %s", image) # Input Parsing ---------------------------- # Parse image name, repo name, and namespace # First split the docker image name by / image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) == 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = "library" image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace,repo_name,repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(repo_name=repo_name, namespace=namespace, registry=args.registry, auth=auth) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache = args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" %(cache_base,image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # Extract image and remove tar output = extract_tar(targz,singularity_rootfs) if output is None: logger.error("Error extracting image: %s", targz) sys.exit(1) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec="Cmd" else: spec="Entrypoint" cmd = get_config(manifest,spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." %(spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def main(): parser = argparse.ArgumentParser( description="bootstrap Docker images for Singularity containers") # Name of the docker image, required parser.add_argument( "--docker", dest='docker', help= "name of Docker image to bootstrap, in format library/ubuntu:latest", type=str, default=None) # root file system of singularity image parser.add_argument("--rootfs", dest='rootfs', help="the path for the root filesystem to extract to", type=str, default=None) # Docker registry (default is registry-1.docker.io parser.add_argument( "--registry", dest='registry', help="the registry path to use, to replace registry-1.docker.io", type=str, default=None) # Flag to add the Docker CMD as a runscript parser.add_argument( "--cmd", dest='includecmd', action="store_true", help= "boolean to specify that the CMD should be included as a runscript (default is not included)", default=False) # Flag to indicate a token is not required parser.add_argument( "--no-token", dest='notoken', action="store_true", help= "boolean to specify that the CMD should be included as a runscript (default is not included)", default=False) try: args = parser.parse_args() except: parser.print_help() sys.exit(0) # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None: print( "ERROR: root file system not specified or defined as environmental variable, exiting!" ) sys.exit(1) # Does the registry require a token? doauth = True if args.notoken == True: doauth = False # Does the user want to include the CMD as runscript? includecmd = args.includecmd # Do we have a docker image specified? if args.docker != None: image = args.docker # INPUT PARSING ------------------------------------------- # Parse image name, repo name, and namespace # First split the docker image name by / image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) == 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = "library" image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" # Tell the user the namespace, repo name and tag print("%s/%s:%s" % (namespace, repo_name, repo_tag)) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=doauth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest, registry=args.registry, auth=doauth) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Create a temporary directory for targzs tmpdir = tempfile.mkdtemp() layers = [] for image_id in images: # Download the layer targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=tmpdir, registry=args.registry, auth=doauth) layers.append(targz) # in case we want a list at the end # @chrisfilo suggestion to try compiling into one tar.gz # Extract image and remove tar extract_tar(targz, singularity_rootfs) os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: cmd = get_config(manifest) # default is spec="Cmd" # Only add runscript if command is defined if cmd != None: print("Adding Docker CMD as Singularity runscript...") runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # change permission of runscript to 0755 (default) change_permissions("%s/singularity" % (singularity_rootfs))
def main(): logger.info("\n*** STARTING DOCKER BOOTSTRAP PYTHON PORTION ****") parser = argparse.ArgumentParser( description="bootstrap Docker images for Singularity containers") # Name of the docker image, required parser.add_argument( "--docker", dest='docker', help= "name of Docker image to bootstrap, in format library/ubuntu:latest", type=str, default=None) # root file system of singularity image parser.add_argument("--rootfs", dest='rootfs', help="the path for the root filesystem to extract to", type=str, default=None) # Docker registry (default is registry-1.docker.io parser.add_argument( "--registry", dest='registry', help="the registry path to use, to replace registry-1.docker.io", type=str, default=None) # Flag to add the Docker CMD as a runscript parser.add_argument( "--cmd", dest='includecmd', action="store_true", help= "boolean to specify that CMD should be used instead of ENTRYPOINT as the runscript.", default=False) parser.add_argument("--username", dest='username', help="username for registry authentication", default=None) parser.add_argument("--password", dest='password', help="password for registry authentication", default=None) # Flag to disable cache parser.add_argument("--no-cache", dest='disable_cache', action="store_true", help="boolean to specify disabling the cache.", default=False) try: args = parser.parse_args() except: logger.error("Input args to %s improperly set, exiting.", os.path.abspath(__file__)) parser.print_help() sys.exit(0) # Find root filesystem location if args.rootfs != None: singularity_rootfs = args.rootfs logger.info("Root file system defined by command line variable as %s", singularity_rootfs) else: singularity_rootfs = os.environ.get("SINGULARITY_ROOTFS", None) if singularity_rootfs == None: logger.error( "root file system not specified OR defined as environmental variable, exiting!" ) sys.exit(1) logger.info("Root file system defined by env variable as %s", singularity_rootfs) # Does the registry require authentication? auth = None if args.username is not None and args.password is not None: auth = basic_auth_header(args.username, args.password) logger.info("Username for registry authentication: %s", args.username) # Does the user want to override default Entrypoint and use CMD as runscript? includecmd = args.includecmd logger.info("Including Docker command as Runscript? %s", includecmd) # Do we have a docker image specified? if args.docker != None: image = args.docker logger.info("Docker image: %s", image) # INPUT PARSING ------------------------------------------- # Parse image name, repo name, and namespace # First split the docker image name by / image = image.split('/') # If there are two parts, we have namespace with repo (and maybe tab) if len(image) == 2: namespace = image[0] image = image[1] # Otherwise, we must be using library namespace else: namespace = "library" image = image[0] # Now split the docker image name by : image = image.split(':') if len(image) == 2: repo_name = image[0] repo_tag = image[1] # Otherwise, assume latest of an image else: repo_name = image[0] repo_tag = "latest" # Tell the user the namespace, repo name and tag logger.info("Docker image path: %s/%s:%s", namespace, repo_name, repo_tag) # IMAGE METADATA ------------------------------------------- # Use Docker Registry API (version 2.0) to get images ids, manifest # Get an image manifest - has image ids to parse, and will be # used later to get Cmd manifest = get_manifest(repo_name=repo_name, namespace=namespace, repo_tag=repo_tag, registry=args.registry, auth=auth) # Get images from manifest using version 2.0 of Docker Registry API images = get_images(manifest=manifest, registry=args.registry, auth=auth) # DOWNLOAD LAYERS ------------------------------------------- # Each is a .tar.gz file, obtained from registry with curl # Get the cache (or temporary one) for docker cache_base = get_cache(subfolder="docker", disable_cache=args.disable_cache) layers = [] for image_id in images: # Download the layer, if we don't have it targz = "%s/%s.tar.gz" % (cache_base, image_id) if not os.path.exists(targz): targz = get_layer(image_id=image_id, namespace=namespace, repo_name=repo_name, download_folder=cache_base, registry=args.registry, auth=auth) layers.append(targz) # in case we want a list at the end # @chrisfilo suggestion to try compiling into one tar.gz # Extract image and remove tar extract_tar(targz, singularity_rootfs) if args.disable_cache == True: os.remove(targz) # If the user wants to include the CMD as runscript, generate it here if includecmd == True: spec = "Cmd" else: spec = "Entrypoint" cmd = get_config(manifest, spec=spec) # Only add runscript if command is defined if cmd != None: print("Adding Docker %s as Singularity runscript..." % (spec.upper())) print(cmd) runscript = create_runscript(cmd=cmd, base_dir=singularity_rootfs) # When we finish, clean up images if args.disable_cache == True: shutil.rmtree(cache_base) logger.info("*** FINISHING DOCKER BOOTSTRAP PYTHON PORTION ****\n")
def judge(src_code, language, test_data_dir, time_limit, memory_limit, file_size_limit=10 * 1024 * 1024, check_method='line'): """ :param src_code: 源代码 :param language: 语言:c++, java, c, python, python3, go, ruby :param test_data_dir: 测试数据文件夹 :param time_limit: 时间限制 单位:s :param memory_limit: 内存限制 单位:MB :param file_size_limit: 文件大小限制 (update: 似乎被docker.py转为Byte)单位:block 查看utils.py中generate_ulimits函数说明 :param check_method: 查看checker.py文件 :return: """ language = str(language).capitalize() if language not in LANG: raise CrazyBoxError('No support for the language: %s', language) language = LANG[language] suffix = language['suffix'] exe_suffix = language['exe_suffix'] if 'exe_suffix' in language else '' # info用来给维护者debug msg用来显示给前台用户 result = { 'status': None, 'info': '', 'msg': '', 'time': 0, 'memory': 0, # ms KB 'compile_time': None, 'compile_exit_code': None, 'detail': [] } with working_volume() as volume_name, compress_code(src_code, suffix) as file_name: src_path = os.path.join(WORKING_DIR, file_name + suffix) exe_path = os.path.join(WORKING_DIR, file_name + exe_suffix) compile_cmd = replace_arg(language['compile_command'], src_path, exe_path) compile_time_limit = language['compile_max_cpu_time'] / 1000 compile_memory_limit = language['compile_max_memory'] / 1024 / 1024 ret = _compile(file_name, volume_name, compile_cmd, compile_time_limit, compile_memory_limit) result['compile_time'] = ret['duration'] result['compile_exit_code'] = ret['exit_code'] if ret['exit_code'] != 0: err = ret['stderr'].decode() logger.warning('complied failed(exit code: %s, time: %s): %s', ret['exit_code'], ret['duration'], err) result['status'] = CE result['info'] = err result['msg'] = err return result run_cmd = replace_arg(language['run_command'], src_path, exe_path) name_list = [] for data_name in os.listdir(test_data_dir): if not os.path.isfile(os.path.join(test_data_dir, data_name)): continue name, file_suffix = os.path.splitext(data_name) if file_suffix == '.in' and os.path.exists( os.path.join(test_data_dir, name + '.out')): name_list.append(name) name_list.sort() for data_name in name_list: logger.info('----running on case: %s----', data_name) ret, tar_path = _run(file_name, volume_name, run_cmd, test_data_dir, data_name, time_limit, memory_limit * 2, file_size_limit) with extract_tar(tar_path) as out_file_path: in_file_path = os.path.join(test_data_dir, data_name + '.in') answer_file_path = os.path.join(test_data_dir, data_name + '.out') used_time = str(int(ret['duration'] * 1000)) + ' ms' if ret['duration'] else None sub_result = { 'test': data_name, 'time': used_time, 'memory': None, 'exit code': ret['exit_code'], 'checker exit code': None, 'verdict': None, 'input': None, 'output': None, 'answer': None, 'log': None } if ret['exit_code'] != 0: if ret['exit_code'] == 153: info = 'File size limit exceeded : %s MB' % ( file_size_limit / 1024 / 1024) msg = 'Output limit exceed on test %s' % data_name sub_result['verdict'] = 'Output Limit Exceed' result['status'] = OLE elif ret['oom_killed']: info = 'memory limit exceeded : %s MB' % memory_limit msg = 'Memory limit exceed on test %s' % data_name sub_result['verdict'] = 'Memory Limit Exceed' result['status'] = MLE elif ret['timeout']: info = 'time limit exceeded : %s s' % time_limit msg = 'Time limit exceed on test %s' % data_name sub_result['verdict'] = 'Time Limit Exceed' result['status'] = TLE else: info = ret['stdout'].decode( ) + '\n' + ret['stderr'].decode() msg = 'Runtime error on test %s' % data_name sub_result['verdict'] = 'Runtime Error' result['status'] = RE logger.warning(info) result['info'] = info result['msg'] = msg result['detail'].append(sub_result) return result used_maximum_memory = _run(file_name, volume_name, run_cmd, test_data_dir, data_name, time_limit, memory_limit * 2, file_size_limit, use_time=True) if int(used_maximum_memory) / 1024 >= memory_limit: result[ 'info'] = 'memory limit exceeded : %s MB' % memory_limit result[ 'msg'] = 'Memory limit exceed on test %s' % data_name sub_result['verdict'] = 'Memory Limit Exceed' result['detail'].append(sub_result) return result result['time'] = max(result['time'], int(ret['duration'] * 1000)) result['memory'] = max(result['memory'], int(used_maximum_memory)) sub_result['memory'] = str(used_maximum_memory) + ' KB' sub_result['checker exit code'], sub_result['log'], \ sub_result['input'], sub_result['output'], sub_result['answer'] \ = check(in_file_path, out_file_path, answer_file_path, check_method) code = sub_result['checker exit code'] # info msg status if code == 0: sub_result['verdict'] = 'OK' result['detail'].append(sub_result) else: if code == 1: sub_result['verdict'] = 'Wrong Answer' info = msg = 'Wrong answer on test %s' % data_name elif code == 2: sub_result['verdict'] = 'Presentation Error' info = msg = 'Presentation error on test %s' % data_name else: sub_result['verdict'] = 'Judgement Failed' info = 'check method({}) error: {}'.format( check_method, os.path.join(test_data_dir, data_name)) msg = 'judge failed, please contact manager.' result['info'] = info result['msg'] = msg result['detail'].append(sub_result) return result result['status'] = AC return result