def command_grade_local(args): """ The 'local' sub-sub-command of the 'grade' sub-command simulates running a grader on a sample submission from the local file system. """ # TODO: Support arguments to pass to the graders. d = utils.docker_client(args) try: volume_str = common.mk_submission_volume_str(args.dir) logging.debug("Volume string: %s", volume_str) container = d.create_container( image=args.containerId, user='******' % 1000, host_config=docker.utils.create_host_config( binds=[volume_str, ], network_mode='none', mem_limit='1g', memswap_limit='1g', ), ) except: logging.error( "Could not set up the container to run the grade command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise run_container(d, container, args)
def command_ls(args): "Implements the ls subcommand" d = utils.docker_client(args) command = [] if args.l: command.append('-l') if args.human: command.append('-h') if args.a: command.append('-a') command.append(args.dir) logging.debug("Commands: %s", command) try: container = d.create_container( image=args.imageId, entrypoint="/bin/ls", command=command) except: logging.error( "Could not set up the container to run the ls command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise d.start(container) exit_code = d.wait(container, timeout=2) if exit_code != 0: logging.warn("The ls command did not exit cleanly within the " "container. Exit code: %s", exit_code) command_output = d.logs(container) # Use sys.stdout to avoid extra trailing newline. (Py3.x compatible) sys.stdout.write(command_output) if not args.no_rm: d.remove_container(container)
def command_cat(args): "Implements the cat subcommand" d = utils.docker_client(args) try: container = d.create_container(image=args.imageId, entrypoint="/bin/cat", command=args.file) except: logging.error( "Could not set up the container to run the cat command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise d.start(container) exit_code = d.wait(container, timeout=2) if exit_code != 0: logging.warn( "The cat command did not exit cleanly within the " "container. Exit code: %s", exit_code) command_output = d.logs(container) if type(command_output) is bytes: command_output = command_output.decode("utf-8") # Use sys.stdout to avoid extra trailing newline. (Py3.x compatible) sys.stdout.write(command_output)
def command_ls(args): "Implements the ls subcommand" d = utils.docker_client(args) command = [] if args.l: command.append('-l') if args.human: command.append('-h') if args.a: command.append('-a') command.append(args.dir) logging.debug("Commands: %s", command) try: container = d.create_container(image=args.imageId, entrypoint="/bin/ls", command=command) except: logging.error( "Could not set up the container to run the ls command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise d.start(container) exit_code = d.wait(container, timeout=2) if exit_code != 0: logging.warn( "The ls command did not exit cleanly within the " "container. Exit code: %s", exit_code) command_output = d.logs(container) # Use sys.stdout to avoid extra trailing newline. (Py3.x compatible) sys.stdout.write(command_output) if not args.no_rm: d.remove_container(container)
def command_grade_local(args): """ The 'local' sub-sub-command of the 'grade' sub-command simulates running a grader on a sample submission from the local file system. """ d = utils.docker_client(args) memory_limit = compute_memory_limit(args) try: volume_str = common.mk_submission_volume_str(args.dir) logging.debug("Volume string: %s", volume_str) extra_hosts = { 'somehost': "192.168.1.2", } host_config = d.create_host_config( binds=[ volume_str, ], network_mode='none', mem_limit=memory_limit, memswap_limit=memory_limit, extra_hosts=extra_hosts, ) user = '******' % 1000 if 'args' in args and len(args.args) > 0: # Handle additional command-line arguments which will # be passed to the grader. inspect = d.inspect_image(image=args.imageId) cmd = inspect['Config']['Entrypoint'] if not type(cmd) is list: logging.error('ENTRYPOINT in Dockerfile must be a list in ' + 'order to pass in command-line arguments') raise cmd.extend(args.args) container = d.create_container( image=args.imageId, entrypoint=cmd, user=user, host_config=host_config, hostname='somehost', ) else: container = d.create_container( image=args.imageId, user=user, host_config=host_config, ) except: logging.error( "Could not set up the container to run the grade command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise run_container(d, container, args)
def command_grade_local(args): """ The 'local' sub-sub-command of the 'grade' sub-command simulates running a grader on a sample submission from the local file system. """ d = utils.docker_client(args) memory_limit = compute_memory_limit(args) try: volume_str = common.mk_submission_volume_str(args.dir) logging.debug("Volume string: %s", volume_str) host_config = d.create_host_config( binds=[volume_str, ], network_mode='none', mem_limit=memory_limit, memswap_limit=memory_limit, ) user = '******' % 1000 if 'args' in args and len(args.args) > 0: # Handle additional command-line arguments which will # be passed to the grader. inspect = d.inspect_image(image=args.imageId) cmd = inspect['Config']['Entrypoint'] if not type(cmd) is list: logging.error('ENTRYPOINT in Dockerfile must be a list in ' + 'order to pass in command-line arguments') raise cmd.extend(args.args) container = d.create_container( image=args.imageId, entrypoint=cmd, user=user, host_config=host_config, ) else: container = d.create_container( image=args.imageId, user=user, host_config=host_config, ) except: logging.error( "Could not set up the container to run the grade command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise run_container(d, container, args)
def command_cat(args): "Implements the cat subcommand" d = utils.docker_client(args) try: container = d.create_container( image=args.imageId, entrypoint="/bin/cat", command=args.file) except: logging.error( "Could not set up the container to run the cat command in. Most " "likely, this means that you specified an inappropriate container " "id.") raise d.start(container) exit_code = d.wait(container, timeout=2) if exit_code != 0: logging.warn("The cat command did not exit cleanly within the " "container. Exit code: %s", exit_code) command_output = d.logs(container) # Use sys.stdout to avoid extra trailing newline. (Py3.x compatible) sys.stdout.write(command_output)
def command_upload(args): "Implements the upload subcommand" d = utils.docker_client(args) image = get_container_image(args, d) oauth2_instance = oauth2.build_oauth2(args) auth = oauth2_instance.build_authorizer() # TODO: use transloadit's signatures for upload signing. # authorization = authorize_upload(args, auth) # Generate a random uuid for upload. upload_id = uuid.uuid4().hex transloadit_host = idle_transloadit_server(args) upload_url = 'https://%(host)s/assemblies/%(id)s' % { 'host': transloadit_host, 'id': upload_id, } if args.upload_to_requestbin is not None: upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin if not args.quiet > 0: sys.stdout.write( 'About to upload to server:\n\t%(transloadit_host)s\n' 'with upload id:\n\t%(upload_id)s\nStatus API:\n' '\t%(upload_url)s\nUploading...' % { 'transloadit_host': transloadit_host, 'upload_id': upload_id, 'upload_url': upload_url, }) sys.stdout.flush() p = multiprocessing.Process(target=upload, args=(args, upload_url, image)) p.daemon = True # Auto-kill when the main process exits. p.start() time.sleep(20) # Yield control to the child process to kick off upload. upload_information = None while p.is_alive(): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: logging.warn( 'Upload information retrieved before upload completed??! %s', upload_information) break time.sleep(10) # 10 seconds p.join(1) # Join to clean up zombie. # TODO: make time waiting for transloadit to finish processing configurable for i in xrange(300): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: break time.sleep(5) if upload_information is None: logging.error( 'Upload did not complete within expected time limits. Upload ' 'URL: %s', upload_url) return 1 # Register the grader with Coursera to initiate the image cleaning process logging.debug('Grader upload info is: %s', upload_information) # Rebuild an authorizer to ensure it's fresh and not expired auth = oauth2_instance.build_authorizer() grader_id = register_grader(auth, args, bucket=upload_information[0], key=upload_information[1]) return update_assignments(auth, grader_id, args)
def command_sanity(args): "Implements the sanity subcommand" # Sanity check should: # - Check docker versions # - Check for existance of entrypoint (that isn't /bin/bash) # - Check over copy commands // make sure they look reasonable. # - Check the entrypoint if it's a shell script for keywords like "sudo" # or chroot. if not args.skip_environment: logging.info("Checking local docker demon...") conn = utils.docker_client(args) docker_version = conn.version() logging.info("Docker version: %s", docker_version["Version"]) if semver.match(docker_version["Version"], '>1.5.0'): if semver.match(docker_version["Version"], '<=1.9.1'): logging.info("Docker version ok.") else: logging.error("Docker version must be <=1.9.1") else: logging.error("Docker version must be >1.5.0.") if "docker_file" in args and args.docker_file is not None: try: docker_file = DockerfileParser(args.docker_file) except: logging.error("Could not parse Dockerfile at: %s", args.docker_file) else: structure = docker_file.structure seen_entrypoint = False for cmd in structure: if cmd['instruction'].lower() == 'copy': if not cmd['value'].startswith('/'): logging.warn( 'Line %(lineno)s: Copy destination should always ' 'start with a /.', { "lineno": cmd['startline'], }) if cmd['instruction'].lower() == 'from': if "ubuntu" in cmd['value']: logging.info( 'Line %(lineno)s: We recommend using ' 'debian, or other smaller base images.', { "lineno": cmd['startline'], }) if cmd['instruction'].lower() == 'entrypoint': if seen_entrypoint: logging.warn( 'Line %(lineno)s: Re-defining entrypoint ' 'of container.', { "lineno": cmd['startline'], }) seen_entrypoint = True if 'bash' in cmd['value']: logging.warn( 'Line %(lineno)s: Please mark your grading script ' 'or binary as the ENTRYPOINT, and not bash', { 'lineno': cmd['startline'], }) if cmd['instruction'].lower() == 'expose': logging.warn( 'Line %(lineno)s: EXPOSE commands do not work for ' 'graders', { 'lineno': cmd['startline'], }) if cmd['instruction'].lower() == 'env': logging.warn( 'Line %(lineno)s: ENV-based environment variables are ' 'stripped in the production environment for security ' 'reasons. Please set any environment variables you ' 'need in your grading script.', { 'lineno': cmd['startline'], }) if cmd['instruction'].lower() == 'volume': logging.warn( 'Line %(lineno)s: VOLUME commands are stripped in ' 'the production environment, and will likely not work ' 'as expected.', { 'lineno': cmd['startline'], }) if not seen_entrypoint: logging.warn('Your Dockerfile must define an ENTRYPOINT.') else: logging.info("No Dockerfile provided... skipping file checks.")
def command_upload(args): "Implements the upload subcommand" d = utils.docker_client(args) image = get_container_image(args, d) oauth2_instance = oauth2.build_oauth2(args) auth = oauth2_instance.build_authorizer() # TODO: use transloadit's signatures for upload signing. # authorization = authorize_upload(args, auth) # Generate a random uuid for upload. upload_id = uuid.uuid4().hex transloadit_host = idle_transloadit_server(args) upload_url = 'https://%(host)s/assemblies/%(id)s' % { 'host': transloadit_host, 'id': upload_id, } if args.upload_to_requestbin is not None: upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin if not args.quiet > 0: sys.stdout.write( 'About to upload to server:\n\t%(transloadit_host)s\n' 'with upload id:\n\t%(upload_id)s\nStatus API:\n' '\t%(upload_url)s\nUploading...' % { 'transloadit_host': transloadit_host, 'upload_id': upload_id, 'upload_url': upload_url, }) sys.stdout.flush() p = multiprocessing.Process(target=upload, args=(args, upload_url, image)) p.daemon = True # Auto-kill when the main process exits. p.start() time.sleep(20) # Yield control to the child process to kick off upload. upload_information = None while p.is_alive(): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: logging.warn( 'Upload information retrieved before upload completed??! %s', upload_information) break time.sleep(10) # 10 seconds p.join(1) # Join to clean up zombie. # TODO: make time waiting for transloadit to finish processing configurable for i in xrange(300): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: break time.sleep(5) if upload_information is None: logging.error( 'Upload did not complete within expected time limits. Upload ' 'URL: %s', upload_url) return 1 # Register the grader with Coursera to initiate the image cleaning process logging.debug('Grader upload info is: %s', upload_information) # Rebuild an authorizer to ensure it's fresh and not expired auth = oauth2_instance.build_authorizer() grader_cpu = None if hasattr(args, 'grader_cpu') and args.grader_cpu is not None: grader_cpu = args.grader_cpu * 1024 register_request = { 'courseId': args.course, 'bucket': upload_information[0], 'key': upload_information[1], 'reservedCpu': grader_cpu, 'reservedMemory': getattr(args, 'grader_memory_limit', None), 'wallClockTimeout': getattr(args, 'grading_timeout', None), } logging.debug('About to POST data to register endpoint: %s', json.dumps(register_request)) register_result = requests.post(args.register_endpoint, data=json.dumps(register_request), auth=auth) if register_result.status_code != 201: # Created logging.error('Failed to register grader (%s) with Coursera: %s', upload_information[1], register_result.text) return 1 try: grader_id = register_result.json()['elements'][0]['executorId'] location = register_result.headers['location'] except: logging.exception( 'Could not parse the response from the Coursera register grader ' 'endpoint: %s', register_result.text) return 1 logging.info('The grader status API is at: %s', location) return update_assignments(auth, grader_id, args)
def command_upload(args): "Implements the upload subcommand" d = utils.docker_client(args) image = get_container_image(args, d) oauth2_instance = oauth2.build_oauth2(args) auth = oauth2_instance.build_authorizer() # TODO: use transloadit's signatures for upload signing. # authorization = authorize_upload(args, auth) # Generate a random uuid for upload. upload_id = uuid.uuid4().hex transloadit_host = idle_transloadit_server(args) upload_url = 'https://%(host)s/assemblies/%(id)s' % { 'host': transloadit_host, 'id': upload_id, } if args.upload_to_requestbin is not None: upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin if not args.quiet > 0: sys.stdout.write( 'About to upload to server:\n\t%(transloadit_host)s\n' 'with upload id:\n\t%(upload_id)s\nStatus API:\n' '\t%(upload_url)s\nUploading...' % { 'transloadit_host': transloadit_host, 'upload_id': upload_id, 'upload_url': upload_url, }) sys.stdout.flush() p = multiprocessing.Process(target=upload, args=(args, upload_url, image)) p.daemon = True # Auto-kill when the main process exits. p.start() time.sleep(10) # Yield control to the child process to kick off upload. upload_information = None while p.is_alive(): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: logging.warn( 'Upload information retrieved before upload completed??! %s', upload_information) break time.sleep(10) # 10 seconds p.join(1) # Join to clean up zombie. # TODO: make time waiting for transloadit to finish processing configurable for i in xrange(300): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: break time.sleep(5) if upload_information is None: logging.error( 'Upload did not complete within expected time limits. Upload ' 'URL: %s', upload_url) return 1 # Register the grader with Coursera to initiate the image cleaning process logging.debug('Grader upload info is: %s', upload_information) # Rebuild an authorizer to ensure it's fresh and not expired auth = oauth2_instance.build_authorizer() register_request = { 'courseId': args.course, 'bucket': upload_information[0], 'key': upload_information[1], } logging.debug('About to POST data to register endpoint: %s', json.dumps(register_request)) register_result = requests.post( args.register_endpoint, data=json.dumps(register_request), auth=auth) if register_result.status_code != 201: # Created logging.error( 'Failed to register grader (%s) with Coursera: %s', upload_information[1], register_result.text) return 1 try: grader_id = register_result.json()['elements'][0]['executorId'] location = register_result.headers['location'] except: logging.exception( 'Could not parse the response from the Coursera register grader ' 'endpoint: %s', register_result.text) return 1 logging.info('The grader status API is at: %s', location) update_assignment_params = { 'action': args.update_part_action, 'id': '%s~%s' % (args.course, args.item), 'partId': args.part, 'executorId': grader_id, } update_result = requests.post( args.update_part_endpoint, params=update_assignment_params, auth=auth) if update_result.status_code != 200: logging.error( 'Unable to update the assignment to use the new grader. Param: %s ' 'URL: %s Response: %s', update_assignment_params, update_result.url, update_result.text) return 1 logging.info('Successfully updated assignment part %s to new executor %s', args.part, grader_id) return 0
def command_upload(args): "Implements the upload subcommand" d = utils.docker_client(args) image = get_container_image(args, d) oauth2_instance = oauth2.build_oauth2(args) auth = oauth2_instance.build_authorizer() # TODO: use transloadit's signatures for upload signing. # authorization = authorize_upload(args, auth) # Generate a random uuid for upload. upload_id = uuid.uuid4().hex transloadit_host = idle_transloadit_server(args) upload_url = "https://%(host)s/assemblies/%(id)s" % {"host": transloadit_host, "id": upload_id} if args.upload_to_requestbin is not None: upload_url = "http://requestb.in/%s" % args.upload_to_requestbin if not args.quiet > 0: sys.stdout.write( "About to upload to server:\n\t%(transloadit_host)s\n" "with upload id:\n\t%(upload_id)s\nStatus API:\n" "\t%(upload_url)s\nUploading..." % {"transloadit_host": transloadit_host, "upload_id": upload_id, "upload_url": upload_url} ) sys.stdout.flush() p = multiprocessing.Process(target=upload, args=(args, upload_url, image)) p.daemon = True # Auto-kill when the main process exits. p.start() time.sleep(20) # Yield control to the child process to kick off upload. upload_information = None while p.is_alive(): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: logging.warn("Upload information retrieved before upload completed??! %s", upload_information) break time.sleep(10) # 10 seconds p.join(1) # Join to clean up zombie. # TODO: make time waiting for transloadit to finish processing configurable for i in xrange(300): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: break time.sleep(5) if upload_information is None: logging.error("Upload did not complete within expected time limits. Upload " "URL: %s", upload_url) return 1 # Register the grader with Coursera to initiate the image cleaning process logging.debug("Grader upload info is: %s", upload_information) # Rebuild an authorizer to ensure it's fresh and not expired auth = oauth2_instance.build_authorizer() grader_cpu = None if hasattr(args, "grader_cpu") and args.grader_cpu is not None: grader_cpu = args.grader_cpu * 1024 register_request = { "courseId": args.course, "bucket": upload_information[0], "key": upload_information[1], "reservedCpu": grader_cpu, "reservedMemory": getattr(args, "grader_memory_limit", None), "wallClockTimeout": getattr(args, "grading_timeout", None), } logging.debug("About to POST data to register endpoint: %s", json.dumps(register_request)) register_result = requests.post(args.register_endpoint, data=json.dumps(register_request), auth=auth) if register_result.status_code != 201: # Created logging.error("Failed to register grader (%s) with Coursera: %s", upload_information[1], register_result.text) return 1 try: grader_id = register_result.json()["elements"][0]["executorId"] location = register_result.headers["location"] except: logging.exception( "Could not parse the response from the Coursera register grader " "endpoint: %s", register_result.text ) return 1 logging.info("The grader status API is at: %s", location) return update_assignments(auth, grader_id, args)