def create_run_manager(w):
if args.batch_queue is None:
# We defer importing the run managers so their dependencies are lazily loaded
from docker_run import DockerRunManager
from docker_client import DockerClient
from docker_image_manager import DockerImageManager
logging.info("Using local docker client for run submission.")
docker = DockerClient()
image_manager = DockerImageManager(docker, args.work_dir, max_images_bytes)
cpuset = parse_cpuset_args(args.cpuset)
gpuset = parse_gpuset_args(docker, args.gpuset)
return DockerRunManager(docker, bundle_service, image_manager, w, args.network_prefix, cpuset, gpuset)
else:
try:
import boto3
except ImportError:
logging.exception("Missing dependencies, please install boto3 to enable AWS support.")
import sys
sys.exit(1)
from aws_batch import AwsBatchRunManager
logging.info("Using AWS Batch queue %s for run submission.", args.batch_queue)
batch_client = boto3.client('batch')
return AwsBatchRunManager(batch_client, args.batch_queue, bundle_service, w)
def callback(self, ch, method, properties, body):
data = json.loads(body)
submit_data = SubmitData(data["id"], data["problem_tag"],
data["source_code"], data["author"],
data["language"])
if self.check_language_exists(submit_data.language):
return JudgeResult("WA", "Cannot run with specified language")
# TODO: docker走らす系の処理
docker_client = DockerClient(submit_data)
res = docker_client.fetch_problem_testcases(self.db_session)
print(res)
def scan(images=True):
""" scanning method that will scan all images or containers """
client = DockerClient()
emu = Emulator()
objs = client.images() if images is True else client.containers()
# If there are no images/containers on the machine, objs will be ['']
if objs == ['']:
return
# does actual work here!
for im in objs:
try:
emu.create_dirs()
mount_obj(emu.tmp_image_dir, im, client.info()['Storage Driver'])
if emu.is_applicable():
print "scanning " + im[:12]
emu.intial_setup()
emu.chroot_and_run()
emu.unmount()
else:
print im[:12] + " is not RHEL based"
unmount_obj(emu.tmp_image_dir, client.info()['Storage Driver'])
emu.remove_dirs()
except MountError as dme:
raise ValueError(str(dme))
emu.gather_data(images)
print >> sys.stderr, """
Permissions on password file are too lax.
Only the user should be allowed to access the file.
On Linux, run:
chmod 600 %s""" % args.password_file
exit(1)
with open(args.password_file) as f:
username = f.readline().strip()
password = f.readline().strip()
else:
username = raw_input('Username: '******'%(asctime)s %(message)s',
level=logging.DEBUG)
max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
worker = Worker(args.id, args.tag, args.work_dir, max_work_dir_size_bytes,
args.shared_file_system, args.slots,
BundleServiceClient(args.server, username, password),
DockerClient())
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
print 'Worker started.'
worker.run()
def main():
parser = argparse.ArgumentParser(description='CodaLab worker.')
parser.add_argument('--tag',
help='Tag that allows for scheduling runs on specific '
'workers.')
parser.add_argument(
'--server',
default='https://worksheets.codalab.org',
help='URL of the CodaLab server, in the format '
'<http|https>://<hostname>[:<port>] (e.g., https://worksheets.codalab.org)'
)
parser.add_argument('--work-dir',
default='codalab-worker-scratch',
help='Directory where to store temporary bundle data, '
'including dependencies and the data from run '
'bundles.')
parser.add_argument('--max-work-dir-size',
type=str,
metavar='SIZE',
default='10g',
help='Maximum size of the temporary bundle data '
'(e.g., 3, 3k, 3m, 3g, 3t).')
parser.add_argument(
'--max-image-cache-size',
type=str,
metavar='SIZE',
help='Limit the disk space used to cache Docker images '
'for worker jobs to the specified amount (e.g. '
'3, 3k, 3m, 3g, 3t). If the limit is exceeded, '
'the least recently used images are removed first. '
'Worker will not remove any images if this option '
'is not specified.')
parser.add_argument('--slots',
type=int,
default=1,
help='Number of slots to use for running bundles. '
'A single bundle takes up a single slot.')
parser.add_argument('--password-file',
help='Path to the file containing the username and '
'password for logging into the bundle service, '
'each on a separate line. If not specified, the '
'password is read from standard input.')
parser.add_argument('--verbose',
action='store_true',
help='Whether to output verbose log messages.')
parser.add_argument('--id',
default='%s(%d)' % (socket.gethostname(), os.getpid()),
help='Internal use: ID to use for the worker.')
parser.add_argument(
'--shared-file-system',
action='store_true',
help='Internal use: Whether the file system containing '
'bundle data is shared between the bundle service '
'and the worker.')
args = parser.parse_args()
# Get the username and password.
logger.info('Connecting to %s' % args.server)
if args.password_file:
if os.stat(args.password_file).st_mode & (stat.S_IRWXG | stat.S_IRWXO):
print >> sys.stderr, """
Permissions on password file are too lax.
Only the user should be allowed to access the file.
On Linux, run:
chmod 600 %s""" % args.password_file
exit(1)
with open(args.password_file) as f:
username = f.readline().strip()
password = f.readline().strip()
else:
username = os.environ.get('CODALAB_USERNAME')
if username is None:
username = raw_input('Username: '******'CODALAB_PASSWORD')
if password is None:
password = getpass.getpass()
# Set up logging.
if args.verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.INFO)
max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
if args.max_image_cache_size is None:
max_images_bytes = None
else:
max_images_bytes = parse_size(args.max_image_cache_size)
worker = Worker(args.id, args.tag, args.work_dir, max_work_dir_size_bytes,
max_images_bytes, args.shared_file_system, args.slots,
BundleServiceClient(args.server, username, password),
DockerClient())
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
logger.info('Worker started.')
worker.run()
def __init__(self, mountpoint, live=False, mnt_mkdir=False):
Mount.__init__(self, mountpoint, live)
self.client = docker.Client()
self.docker_client = DockerClient()
self.mnt_mkdir = mnt_mkdir
from flask import Flask, request, abort, jsonify
from docker_client import DockerClient
import json
app = Flask(__name__)
docker = DockerClient()
@app.route("/docker/image/pull", methods=["POST"])
def image_pull():
try:
docker_img = json.loads(request.data).get("docker_img")
image_tag = json.loads(request.data).get("image_tag")
image = docker.pull_docker_image(docker_img, image_tag)
return jsonify({
"message": "the image: {}:{} has been updated".format(docker_img, image_tag),
"image_id": image.id
})
except Exception as ex:
print(ex)
abort(status=500)
@app.route("/docker/container/update", methods=["POST"])
def update_container():
try:
def main():
parser = argparse.ArgumentParser(description='CodaLab worker.')
parser.add_argument('--tag',
help='Tag that allows for scheduling runs on specific '
'workers.')
parser.add_argument(
'--server',
default='https://worksheets.codalab.org',
help='URL of the CodaLab server, in the format '
'<http|https>://<hostname>[:<port>] (e.g., https://worksheets.codalab.org)'
)
parser.add_argument('--work-dir',
default='codalab-worker-scratch',
help='Directory where to store temporary bundle data, '
'including dependencies and the data from run '
'bundles.')
parser.add_argument('--network-prefix',
default='codalab_worker_network',
help='Docker network name prefix')
parser.add_argument(
'--cpuset',
type=str,
metavar='CPUSET_STR',
default='ALL',
help='Comma-separated list of CPUs in which to allow bundle execution, '
'(e.g., \"0,2,3\", \"1\").')
parser.add_argument(
'--gpuset',
type=str,
metavar='GPUSET_STR',
default='ALL',
help='Comma-separated list of GPUs in which to allow bundle execution '
'(e.g., \"0,1\", \"1\").')
parser.add_argument('--max-work-dir-size',
type=str,
metavar='SIZE',
default='10g',
help='Maximum size of the temporary bundle data '
'(e.g., 3, 3k, 3m, 3g, 3t).')
parser.add_argument(
'--max-dependencies-serialized-length',
type=int,
default=60000,
help='Maximum length of serialized json of dependency list of worker '
'(e.g., 50, 30000, 60000).')
parser.add_argument(
'--max-image-cache-size',
type=str,
metavar='SIZE',
help='Limit the disk space used to cache Docker images '
'for worker jobs to the specified amount (e.g. '
'3, 3k, 3m, 3g, 3t). If the limit is exceeded, '
'the least recently used images are removed first. '
'Worker will not remove any images if this option '
'is not specified.')
parser.add_argument('--password-file',
help='Path to the file containing the username and '
'password for logging into the bundle service, '
'each on a separate line. If not specified, the '
'password is read from standard input.')
parser.add_argument('--verbose',
action='store_true',
help='Whether to output verbose log messages.')
parser.add_argument('--id',
default='%s(%d)' % (socket.gethostname(), os.getpid()),
help='Internal use: ID to use for the worker.')
parser.add_argument(
'--shared-file-system',
action='store_true',
help='Internal use: Whether the file system containing '
'bundle data is shared between the bundle service '
'and the worker.')
args = parser.parse_args()
# Get the username and password.
logger.info('Connecting to %s' % args.server)
if args.password_file:
if os.stat(args.password_file).st_mode & (stat.S_IRWXG | stat.S_IRWXO):
print >> sys.stderr, """
Permissions on password file are too lax.
Only the user should be allowed to access the file.
On Linux, run:
chmod 600 %s""" % args.password_file
exit(1)
with open(args.password_file) as f:
username = f.readline().strip()
password = f.readline().strip()
else:
username = os.environ.get('CODALAB_USERNAME')
if username is None:
username = raw_input('Username: '******'CODALAB_PASSWORD')
if password is None:
password = getpass.getpass()
# Set up logging.
if args.verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.INFO)
max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
if args.max_image_cache_size is None:
max_images_bytes = None
else:
max_images_bytes = parse_size(args.max_image_cache_size)
docker_client = DockerClient()
# transform/verify cpuset and gpuset
cpuset = parse_cpuset_args(args.cpuset)
gpuset = parse_gpuset_args(docker_client, args.gpuset)
worker = Worker(args.id, args.tag, args.work_dir, cpuset, gpuset,
max_work_dir_size_bytes,
args.max_dependencies_serialized_length, max_images_bytes,
args.shared_file_system,
BundleServiceClient(args.server, username, password),
docker_client, args.network_prefix)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
# BEGIN: DO NOT CHANGE THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING
# THIS IS HERE TO KEEP TEST-CLI FROM HANGING
print('Worker started.')
# END
worker.run()
