def command_grade_local(args):
"""
The 'local' sub-sub-command of the 'grade' sub-command simulates running a
grader on a sample submission from the local file system.
"""
# TODO: Support arguments to pass to the graders.
d = utils.docker_client(args)
try:
volume_str = common.mk_submission_volume_str(args.dir)
logging.debug("Volume string: %s", volume_str)
container = d.create_container(
image=args.containerId,
user='******' % 1000,
host_config=docker.utils.create_host_config(
binds=[volume_str, ],
network_mode='none',
mem_limit='1g',
memswap_limit='1g',
),
)
except:
logging.error(
"Could not set up the container to run the grade command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
run_container(d, container, args)
def command_ls(args):
"Implements the ls subcommand"
d = utils.docker_client(args)
command = []
if args.l:
command.append('-l')
if args.human:
command.append('-h')
if args.a:
command.append('-a')
command.append(args.dir)
logging.debug("Commands: %s", command)
try:
container = d.create_container(
image=args.imageId,
entrypoint="/bin/ls",
command=command)
except:
logging.error(
"Could not set up the container to run the ls command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
d.start(container)
exit_code = d.wait(container, timeout=2)
if exit_code != 0:
logging.warn("The ls command did not exit cleanly within the "
"container. Exit code: %s", exit_code)
command_output = d.logs(container)
# Use sys.stdout to avoid extra trailing newline. (Py3.x compatible)
sys.stdout.write(command_output)
if not args.no_rm:
d.remove_container(container)
def command_cat(args):
"Implements the cat subcommand"
d = utils.docker_client(args)
try:
container = d.create_container(image=args.imageId,
entrypoint="/bin/cat",
command=args.file)
except:
logging.error(
"Could not set up the container to run the cat command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
d.start(container)
exit_code = d.wait(container, timeout=2)
if exit_code != 0:
logging.warn(
"The cat command did not exit cleanly within the "
"container. Exit code: %s", exit_code)
command_output = d.logs(container)
if type(command_output) is bytes:
command_output = command_output.decode("utf-8")
# Use sys.stdout to avoid extra trailing newline. (Py3.x compatible)
sys.stdout.write(command_output)
def command_ls(args):
"Implements the ls subcommand"
d = utils.docker_client(args)
command = []
if args.l:
command.append('-l')
if args.human:
command.append('-h')
if args.a:
command.append('-a')
command.append(args.dir)
logging.debug("Commands: %s", command)
try:
container = d.create_container(image=args.imageId,
entrypoint="/bin/ls",
command=command)
except:
logging.error(
"Could not set up the container to run the ls command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
d.start(container)
exit_code = d.wait(container, timeout=2)
if exit_code != 0:
logging.warn(
"The ls command did not exit cleanly within the "
"container. Exit code: %s", exit_code)
command_output = d.logs(container)
# Use sys.stdout to avoid extra trailing newline. (Py3.x compatible)
sys.stdout.write(command_output)
if not args.no_rm:
d.remove_container(container)
def command_grade_local(args):
"""
The 'local' sub-sub-command of the 'grade' sub-command simulates running a
grader on a sample submission from the local file system.
"""
d = utils.docker_client(args)
memory_limit = compute_memory_limit(args)
try:
volume_str = common.mk_submission_volume_str(args.dir)
logging.debug("Volume string: %s", volume_str)
extra_hosts = {
'somehost': "192.168.1.2",
}
host_config = d.create_host_config(
binds=[
volume_str,
],
network_mode='none',
mem_limit=memory_limit,
memswap_limit=memory_limit,
extra_hosts=extra_hosts,
)
user = '******' % 1000
if 'args' in args and len(args.args) > 0:
# Handle additional command-line arguments which will
# be passed to the grader.
inspect = d.inspect_image(image=args.imageId)
cmd = inspect['Config']['Entrypoint']
if not type(cmd) is list:
logging.error('ENTRYPOINT in Dockerfile must be a list in ' +
'order to pass in command-line arguments')
raise
cmd.extend(args.args)
container = d.create_container(
image=args.imageId,
entrypoint=cmd,
user=user,
host_config=host_config,
hostname='somehost',
)
else:
container = d.create_container(
image=args.imageId,
user=user,
host_config=host_config,
)
except:
logging.error(
"Could not set up the container to run the grade command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
run_container(d, container, args)
def command_grade_local(args):
"""
The 'local' sub-sub-command of the 'grade' sub-command simulates running a
grader on a sample submission from the local file system.
"""
d = utils.docker_client(args)
memory_limit = compute_memory_limit(args)
try:
volume_str = common.mk_submission_volume_str(args.dir)
logging.debug("Volume string: %s", volume_str)
host_config = d.create_host_config(
binds=[volume_str, ],
network_mode='none',
mem_limit=memory_limit,
memswap_limit=memory_limit,
)
user = '******' % 1000
if 'args' in args and len(args.args) > 0:
# Handle additional command-line arguments which will
# be passed to the grader.
inspect = d.inspect_image(image=args.imageId)
cmd = inspect['Config']['Entrypoint']
if not type(cmd) is list:
logging.error('ENTRYPOINT in Dockerfile must be a list in ' +
'order to pass in command-line arguments')
raise
cmd.extend(args.args)
container = d.create_container(
image=args.imageId,
entrypoint=cmd,
user=user,
host_config=host_config,
)
else:
container = d.create_container(
image=args.imageId,
user=user,
host_config=host_config,
)
except:
logging.error(
"Could not set up the container to run the grade command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
run_container(d, container, args)
def command_cat(args):
"Implements the cat subcommand"
d = utils.docker_client(args)
try:
container = d.create_container(
image=args.imageId,
entrypoint="/bin/cat",
command=args.file)
except:
logging.error(
"Could not set up the container to run the cat command in. Most "
"likely, this means that you specified an inappropriate container "
"id.")
raise
d.start(container)
exit_code = d.wait(container, timeout=2)
if exit_code != 0:
logging.warn("The cat command did not exit cleanly within the "
"container. Exit code: %s", exit_code)
command_output = d.logs(container)
# Use sys.stdout to avoid extra trailing newline. (Py3.x compatible)
sys.stdout.write(command_output)
def command_upload(args):
"Implements the upload subcommand"
d = utils.docker_client(args)
image = get_container_image(args, d)
oauth2_instance = oauth2.build_oauth2(args)
auth = oauth2_instance.build_authorizer()
# TODO: use transloadit's signatures for upload signing.
# authorization = authorize_upload(args, auth)
# Generate a random uuid for upload.
upload_id = uuid.uuid4().hex
transloadit_host = idle_transloadit_server(args)
upload_url = 'https://%(host)s/assemblies/%(id)s' % {
'host': transloadit_host,
'id': upload_id,
}
if args.upload_to_requestbin is not None:
upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin
if not args.quiet > 0:
sys.stdout.write(
'About to upload to server:\n\t%(transloadit_host)s\n'
'with upload id:\n\t%(upload_id)s\nStatus API:\n'
'\t%(upload_url)s\nUploading...' % {
'transloadit_host': transloadit_host,
'upload_id': upload_id,
'upload_url': upload_url,
})
sys.stdout.flush()
p = multiprocessing.Process(target=upload, args=(args, upload_url, image))
p.daemon = True # Auto-kill when the main process exits.
p.start()
time.sleep(20) # Yield control to the child process to kick off upload.
upload_information = None
while p.is_alive():
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
logging.warn(
'Upload information retrieved before upload completed??! %s',
upload_information)
break
time.sleep(10) # 10 seconds
p.join(1) # Join to clean up zombie.
# TODO: make time waiting for transloadit to finish processing configurable
for i in xrange(300):
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
break
time.sleep(5)
if upload_information is None:
logging.error(
'Upload did not complete within expected time limits. Upload '
'URL: %s', upload_url)
return 1
# Register the grader with Coursera to initiate the image cleaning process
logging.debug('Grader upload info is: %s', upload_information)
# Rebuild an authorizer to ensure it's fresh and not expired
auth = oauth2_instance.build_authorizer()
grader_id = register_grader(auth,
args,
bucket=upload_information[0],
key=upload_information[1])
return update_assignments(auth, grader_id, args)
def command_sanity(args):
"Implements the sanity subcommand"
# Sanity check should:
# - Check docker versions
# - Check for existance of entrypoint (that isn't /bin/bash)
# - Check over copy commands // make sure they look reasonable.
# - Check the entrypoint if it's a shell script for keywords like "sudo"
# or chroot.
if not args.skip_environment:
logging.info("Checking local docker demon...")
conn = utils.docker_client(args)
docker_version = conn.version()
logging.info("Docker version: %s", docker_version["Version"])
if semver.match(docker_version["Version"], '>1.5.0'):
if semver.match(docker_version["Version"], '<=1.9.1'):
logging.info("Docker version ok.")
else:
logging.error("Docker version must be <=1.9.1")
else:
logging.error("Docker version must be >1.5.0.")
if "docker_file" in args and args.docker_file is not None:
try:
docker_file = DockerfileParser(args.docker_file)
except:
logging.error("Could not parse Dockerfile at: %s",
args.docker_file)
else:
structure = docker_file.structure
seen_entrypoint = False
for cmd in structure:
if cmd['instruction'].lower() == 'copy':
if not cmd['value'].startswith('/'):
logging.warn(
'Line %(lineno)s: Copy destination should always '
'start with a /.', {
"lineno": cmd['startline'],
})
if cmd['instruction'].lower() == 'from':
if "ubuntu" in cmd['value']:
logging.info(
'Line %(lineno)s: We recommend using '
'debian, or other smaller base images.', {
"lineno": cmd['startline'],
})
if cmd['instruction'].lower() == 'entrypoint':
if seen_entrypoint:
logging.warn(
'Line %(lineno)s: Re-defining entrypoint '
'of container.', {
"lineno": cmd['startline'],
})
seen_entrypoint = True
if 'bash' in cmd['value']:
logging.warn(
'Line %(lineno)s: Please mark your grading script '
'or binary as the ENTRYPOINT, and not bash', {
'lineno': cmd['startline'],
})
if cmd['instruction'].lower() == 'expose':
logging.warn(
'Line %(lineno)s: EXPOSE commands do not work for '
'graders', {
'lineno': cmd['startline'],
})
if cmd['instruction'].lower() == 'env':
logging.warn(
'Line %(lineno)s: ENV-based environment variables are '
'stripped in the production environment for security '
'reasons. Please set any environment variables you '
'need in your grading script.', {
'lineno': cmd['startline'],
})
if cmd['instruction'].lower() == 'volume':
logging.warn(
'Line %(lineno)s: VOLUME commands are stripped in '
'the production environment, and will likely not work '
'as expected.', {
'lineno': cmd['startline'],
})
if not seen_entrypoint:
logging.warn('Your Dockerfile must define an ENTRYPOINT.')
else:
logging.info("No Dockerfile provided... skipping file checks.")
def command_upload(args):
"Implements the upload subcommand"
d = utils.docker_client(args)
image = get_container_image(args, d)
oauth2_instance = oauth2.build_oauth2(args)
auth = oauth2_instance.build_authorizer()
# TODO: use transloadit's signatures for upload signing.
# authorization = authorize_upload(args, auth)
# Generate a random uuid for upload.
upload_id = uuid.uuid4().hex
transloadit_host = idle_transloadit_server(args)
upload_url = 'https://%(host)s/assemblies/%(id)s' % {
'host': transloadit_host,
'id': upload_id,
}
if args.upload_to_requestbin is not None:
upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin
if not args.quiet > 0:
sys.stdout.write(
'About to upload to server:\n\t%(transloadit_host)s\n'
'with upload id:\n\t%(upload_id)s\nStatus API:\n'
'\t%(upload_url)s\nUploading...' % {
'transloadit_host': transloadit_host,
'upload_id': upload_id,
'upload_url': upload_url,
})
sys.stdout.flush()
p = multiprocessing.Process(target=upload, args=(args, upload_url, image))
p.daemon = True # Auto-kill when the main process exits.
p.start()
time.sleep(20) # Yield control to the child process to kick off upload.
upload_information = None
while p.is_alive():
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
logging.warn(
'Upload information retrieved before upload completed??! %s',
upload_information)
break
time.sleep(10) # 10 seconds
p.join(1) # Join to clean up zombie.
# TODO: make time waiting for transloadit to finish processing configurable
for i in xrange(300):
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
break
time.sleep(5)
if upload_information is None:
logging.error(
'Upload did not complete within expected time limits. Upload '
'URL: %s', upload_url)
return 1
# Register the grader with Coursera to initiate the image cleaning process
logging.debug('Grader upload info is: %s', upload_information)
# Rebuild an authorizer to ensure it's fresh and not expired
auth = oauth2_instance.build_authorizer()
grader_cpu = None
if hasattr(args, 'grader_cpu') and args.grader_cpu is not None:
grader_cpu = args.grader_cpu * 1024
register_request = {
'courseId': args.course,
'bucket': upload_information[0],
'key': upload_information[1],
'reservedCpu': grader_cpu,
'reservedMemory': getattr(args, 'grader_memory_limit', None),
'wallClockTimeout': getattr(args, 'grading_timeout', None),
}
logging.debug('About to POST data to register endpoint: %s',
json.dumps(register_request))
register_result = requests.post(args.register_endpoint,
data=json.dumps(register_request),
auth=auth)
if register_result.status_code != 201: # Created
logging.error('Failed to register grader (%s) with Coursera: %s',
upload_information[1], register_result.text)
return 1
try:
grader_id = register_result.json()['elements'][0]['executorId']
location = register_result.headers['location']
except:
logging.exception(
'Could not parse the response from the Coursera register grader '
'endpoint: %s', register_result.text)
return 1
logging.info('The grader status API is at: %s', location)
return update_assignments(auth, grader_id, args)
def command_upload(args):
"Implements the upload subcommand"
d = utils.docker_client(args)
image = get_container_image(args, d)
oauth2_instance = oauth2.build_oauth2(args)
auth = oauth2_instance.build_authorizer()
# TODO: use transloadit's signatures for upload signing.
# authorization = authorize_upload(args, auth)
# Generate a random uuid for upload.
upload_id = uuid.uuid4().hex
transloadit_host = idle_transloadit_server(args)
upload_url = 'https://%(host)s/assemblies/%(id)s' % {
'host': transloadit_host,
'id': upload_id,
}
if args.upload_to_requestbin is not None:
upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin
if not args.quiet > 0:
sys.stdout.write(
'About to upload to server:\n\t%(transloadit_host)s\n'
'with upload id:\n\t%(upload_id)s\nStatus API:\n'
'\t%(upload_url)s\nUploading...' % {
'transloadit_host': transloadit_host,
'upload_id': upload_id,
'upload_url': upload_url,
})
sys.stdout.flush()
p = multiprocessing.Process(target=upload, args=(args, upload_url, image))
p.daemon = True # Auto-kill when the main process exits.
p.start()
time.sleep(10) # Yield control to the child process to kick off upload.
upload_information = None
while p.is_alive():
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
logging.warn(
'Upload information retrieved before upload completed??! %s',
upload_information)
break
time.sleep(10) # 10 seconds
p.join(1) # Join to clean up zombie.
# TODO: make time waiting for transloadit to finish processing configurable
for i in xrange(300):
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
break
time.sleep(5)
if upload_information is None:
logging.error(
'Upload did not complete within expected time limits. Upload '
'URL: %s',
upload_url)
return 1
# Register the grader with Coursera to initiate the image cleaning process
logging.debug('Grader upload info is: %s', upload_information)
# Rebuild an authorizer to ensure it's fresh and not expired
auth = oauth2_instance.build_authorizer()
register_request = {
'courseId': args.course,
'bucket': upload_information[0],
'key': upload_information[1],
}
logging.debug('About to POST data to register endpoint: %s',
json.dumps(register_request))
register_result = requests.post(
args.register_endpoint,
data=json.dumps(register_request),
auth=auth)
if register_result.status_code != 201: # Created
logging.error(
'Failed to register grader (%s) with Coursera: %s',
upload_information[1],
register_result.text)
return 1
try:
grader_id = register_result.json()['elements'][0]['executorId']
location = register_result.headers['location']
except:
logging.exception(
'Could not parse the response from the Coursera register grader '
'endpoint: %s',
register_result.text)
return 1
logging.info('The grader status API is at: %s', location)
update_assignment_params = {
'action': args.update_part_action,
'id': '%s~%s' % (args.course, args.item),
'partId': args.part,
'executorId': grader_id,
}
update_result = requests.post(
args.update_part_endpoint,
params=update_assignment_params,
auth=auth)
if update_result.status_code != 200:
logging.error(
'Unable to update the assignment to use the new grader. Param: %s '
'URL: %s Response: %s',
update_assignment_params,
update_result.url,
update_result.text)
return 1
logging.info('Successfully updated assignment part %s to new executor %s',
args.part, grader_id)
return 0
def command_upload(args):
"Implements the upload subcommand"
d = utils.docker_client(args)
image = get_container_image(args, d)
oauth2_instance = oauth2.build_oauth2(args)
auth = oauth2_instance.build_authorizer()
# TODO: use transloadit's signatures for upload signing.
# authorization = authorize_upload(args, auth)
# Generate a random uuid for upload.
upload_id = uuid.uuid4().hex
transloadit_host = idle_transloadit_server(args)
upload_url = "https://%(host)s/assemblies/%(id)s" % {"host": transloadit_host, "id": upload_id}
if args.upload_to_requestbin is not None:
upload_url = "http://requestb.in/%s" % args.upload_to_requestbin
if not args.quiet > 0:
sys.stdout.write(
"About to upload to server:\n\t%(transloadit_host)s\n"
"with upload id:\n\t%(upload_id)s\nStatus API:\n"
"\t%(upload_url)s\nUploading..."
% {"transloadit_host": transloadit_host, "upload_id": upload_id, "upload_url": upload_url}
)
sys.stdout.flush()
p = multiprocessing.Process(target=upload, args=(args, upload_url, image))
p.daemon = True # Auto-kill when the main process exits.
p.start()
time.sleep(20) # Yield control to the child process to kick off upload.
upload_information = None
while p.is_alive():
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
logging.warn("Upload information retrieved before upload completed??! %s", upload_information)
break
time.sleep(10) # 10 seconds
p.join(1) # Join to clean up zombie.
# TODO: make time waiting for transloadit to finish processing configurable
for i in xrange(300):
upload_information = poll_transloadit(args, upload_url)
if upload_information is not None:
break
time.sleep(5)
if upload_information is None:
logging.error("Upload did not complete within expected time limits. Upload " "URL: %s", upload_url)
return 1
# Register the grader with Coursera to initiate the image cleaning process
logging.debug("Grader upload info is: %s", upload_information)
# Rebuild an authorizer to ensure it's fresh and not expired
auth = oauth2_instance.build_authorizer()
grader_cpu = None
if hasattr(args, "grader_cpu") and args.grader_cpu is not None:
grader_cpu = args.grader_cpu * 1024
register_request = {
"courseId": args.course,
"bucket": upload_information[0],
"key": upload_information[1],
"reservedCpu": grader_cpu,
"reservedMemory": getattr(args, "grader_memory_limit", None),
"wallClockTimeout": getattr(args, "grading_timeout", None),
}
logging.debug("About to POST data to register endpoint: %s", json.dumps(register_request))
register_result = requests.post(args.register_endpoint, data=json.dumps(register_request), auth=auth)
if register_result.status_code != 201: # Created
logging.error("Failed to register grader (%s) with Coursera: %s", upload_information[1], register_result.text)
return 1
try:
grader_id = register_result.json()["elements"][0]["executorId"]
location = register_result.headers["location"]
except:
logging.exception(
"Could not parse the response from the Coursera register grader " "endpoint: %s", register_result.text
)
return 1
logging.info("The grader status API is at: %s", location)
return update_assignments(auth, grader_id, args)
