def task_put(self, task):
limits = KolejkaLimits()
limits.cpus = self.config.cpus
limits.memory = self.config.memory
limits.pids = self.config.pids
limits.storage = self.config.storage
limits.time = self.config.time
limits.network = self.config.network
task.limits.update(limits)
if not self.instance_session:
self.login()
for k, f in task.files.items():
if not f.reference or not self.blob_check(
blob_reference=f.reference):
f.reference = None
if f.path:
f.reference = self.blob_put(os.path.join(
task.path, f.path))['key']
else:
raise
info = self.post('/task/task/', data=json.dumps(task.dump()))
if info.status_code == 200:
task = KolejkaTask(None)
task.load(info.json()['task'])
return task
else:
print(info)
print(info.text)
def settings(request):
settings = django.conf.settings
if request.method == 'GET':
response = dict()
response['blob_hash_algorithm'] = settings.BLOB_HASH_ALGORITHM
limits = KolejkaLimits(
cpus=settings.LIMIT_CPUS,
memory=settings.LIMIT_MEMORY,
pids=settings.LIMIT_PIDS,
storage=settings.LIMIT_STORAGE,
network=settings.LIMIT_NETWORK,
time=settings.LIMIT_TIME,
)
response['limits'] = limits.dump()
return JsonResponse(response)
def limits(self, limits=KolejkaLimits()):
if limits.memory is not None:
assert 'memory' in self.groups
limit_file = self.group_path('memory',
filename='memory.limit_in_bytes')
with open(limit_file, 'w') as f:
f.write(str(limits.memory))
logging.debug('Limited session %s memory to %s bytes' %
(self.id, limits.memory))
if limits.cpus is not None:
assert 'cpuset' in self.groups
cpuset_cpus = self.available_cpus()
logging.debug('Available cpus: %s',
','.join([str(c) for c in cpuset_cpus]))
cpus_offset = limits.cpus_offset or 0
if len(cpuset_cpus) < cpus_offset + limits.cpus:
cpus_offset = 0
if len(cpuset_cpus) > cpus_offset + limits.cpus:
cpuset_cpus = cpuset_cpus[0:limits.cpus]
limit_file = self.group_path('cpuset', filename='cpuset.cpus')
with open(limit_file, 'w') as f:
f.write(','.join([str(c) for c in cpuset_cpus]))
logging.debug('Limited session %s cpus to %s' %
(self.id, ','.join([str(c) for c in cpuset_cpus])))
if limits.pids is not None:
assert 'pids' in self.groups
limit_file = self.group_path('pids', filename='pids.max')
with open(limit_file, 'w') as f:
f.write(str(limits.pids))
logging.debug('Limited session %s pids to %s' %
(self.id, limits.pids))
def run_command(self, command, stdin: Optional[Path],
stdout: Optional[Path], stderr: Optional[Path], env, user,
group):
from kolejka import observer
from kolejka.common import KolejkaLimits
with ExitStack() as stack:
stdin_file = stdin and stack.enter_context(
self.get_file_handle(stdin, 'r'))
stdout_file = stdout and stack.enter_context(
self.get_file_handle(stdout, 'w'))
stderr_file = stderr and stack.enter_context(
self.get_file_handle(stderr, 'w'))
execution_status = observer.run(
command,
stdin=stdin_file,
stdout=stdout_file,
stderr=stderr_file,
env=env,
limits=KolejkaLimits(**self.limits),
preexec_fn=self.get_change_user_function(user=user,
group=group))
execution_status.stdout = stdout
execution_status.stderr = stderr
return execution_status
def settings(request):
if not request.user.is_authenticated:
return HttpResponseForbidden()
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
settings = django.conf.settings
response = dict()
response['blob_hash_algorithm'] = settings.BLOB_HASH_ALGORITHM
limits = KolejkaLimits(
cpus=settings.LIMIT_CPUS,
memory=settings.LIMIT_MEMORY,
swap=settings.LIMIT_SWAP,
pids=settings.LIMIT_PIDS,
storage=settings.LIMIT_STORAGE,
image=settings.LIMIT_IMAGE,
workspace=settings.LIMIT_WORKSPACE,
network=settings.LIMIT_NETWORK,
time=settings.LIMIT_TIME,
)
response['limits'] = limits.dump()
return OKResponse(response)
def run(args, limits=None, **kwargs):
if limits is None:
limits = KolejkaLimits()
def target(conn, args, limits, kwargs):
client = KolejkaObserverClient()
client.attach()
client.limit(limits)
res = subprocess.run(args, **kwargs)
stats = client.stats()
client.close()
#TODO: add stats to res, handle exceptions
conn.send(res)
conn.close()
p_conn, c_conn = multiprocessing.Pipe()
proc = Process(target=target, args=(c_conn, args, limits, kwargs))
proc.start()
res = p_conn.recv()
proc.join()
return res
def start_command(self, command, stdin_path, stdout_path, stdout_append,
stdout_max_bytes, stderr_path, stderr_append,
stderr_max_bytes, environment, work_path, user, group,
limits):
import kolejka.observer.runner
from kolejka.common import KolejkaLimits
change_user, change_group, change_groups = self.get_user_group_groups(
user, group)
resources = self.get_resources(limits)
limits = KolejkaLimits(
cpus=limits.cores,
memory=limits.memory,
swap=0,
pids=limits.pids,
time=limits.real_time,
)
#TODO: cpu_time !!!!!
stdin_file = self.read_file(stdin_path)
stdout_file, stdout_writer = self.file_writer(
stdout_path, stdout_append, max_bytes=stdout_max_bytes)
stderr_file, stderr_writer = self.file_writer(
stderr_path, stderr_append, max_bytes=stderr_max_bytes)
writers = (stdout_writer, stderr_writer)
process = kolejka.observer.runner.start(
command,
limits=limits,
user=change_user,
group=change_group,
groups=change_groups,
resources=resources,
stdin=stdin_file,
stdout=stdout_file,
stderr=stderr_file,
env=environment,
cwd=work_path,
)
stdin_file.close()
stdout_file.close()
stderr_file.close()
return (process, writers)
def __init__(self, params):
self.session_id = params.get('session_id', None)
self.secret = params.get('secret', None)
self.limits = KolejkaLimits()
self.limits.load(params.get('limits', {}))
def limits(self, session_id, limits=KolejkaLimits()):
assert session_id in self.sessions
return self.sessions[session_id].limits(limits=limits)
def dequeue(request):
if request.method == 'POST':
if not request.user.is_authenticated():
return HttpResponseForbidden()
#TODO: Check that user can get tasks
tasks = list()
params = json.loads(str(request.read(), request.encoding or 'utf-8'))
concurency = params.get('concurency', 1)
limits = KolejkaLimits()
limits.load(params.get('limits', dict()))
tags = set(params.get('tags', list()))
resources = KolejkaLimits()
resources.update(limits)
available_tasks = Task.objects.filter(
assignee=None).order_by('time_create')[0:100]
for t in available_tasks:
if len(tasks) > concurency:
break
tt = t.task()
if len(tasks) > 0 and tt.exclusive:
continue
if not set(tt.requires).issubset(tags):
continue
if resources.cpus is not None and (
tt.limits.cpus is None or tt.limits.cpus > resources.cpus):
continue
if resources.memory is not None and (
tt.limits.memory is None
or tt.limits.memory > resources.memory):
continue
if resources.pids is not None and (
tt.limits.pids is None or tt.limits.pids > resources.pids):
continue
if resources.storage is not None and (
tt.limits.storage is None
or tt.limits.storage > resources.storage):
continue
if resources.network is not None and (tt.limits.network is None
or tt.limits.network
and not resources.network):
continue
if resources.time is not None and (
tt.limits.time is None or tt.limits.time > resources.time):
continue
tasks.append(tt.dump())
t.assignee = request.user
t.save()
if resources.cpus is not None:
resources.cpus -= tt.limits.cpus
if resources.memory is not None:
resources.memory -= tt.limits.memory
if resources.pids is not None:
resources.pids -= tt.limits.pids
if resources.storage is not None:
resources.storage -= tt.limits.storage
if tt.exclusive:
break
response = dict()
response['tasks'] = tasks
return JsonResponse(response)
return HttpResponseNotAllowed(['POST'])
def dequeue(request):
if not request.user.has_perm('task.process_task'):
return HttpResponseForbidden()
if request.method != 'POST':
return HttpResponseNotAllowed(['POST'])
content_type = ContentType.objects.get_for_model(Task)
tasks = list()
params = json.loads(str(request.read(), request.encoding or 'utf-8'))
concurency = params.get('concurency', 1)
limits = KolejkaLimits()
limits.load(params.get('limits', dict()))
tags = set(params.get('tags', list()))
resources = KolejkaLimits()
resources.update(limits)
image_usage = dict()
available_tasks = Task.objects.filter(assignee__isnull=True).order_by('time_create')[0:100]
for t in available_tasks:
if len(tasks) > concurency:
break
tt = t.task()
if len(tasks) > 0 and tt.exclusive:
print('At exclusive')
continue
if not set(tt.requires).issubset(tags):
continue
if resources.cpus is not None and (tt.limits.cpus is None or tt.limits.cpus > resources.cpus):
continue
if tt.limits.gpus is not None and tt.limits.gpus > 0:
if resources.gpus is not None and (tt.limits.gpus is None or tt.limits.gpus > resources.gpus):
continue
if resources.memory is not None and (tt.limits.memory is None or tt.limits.memory > resources.memory):
continue
if resources.swap is not None and (tt.limits.swap is None or tt.limits.swap > resources.swap):
continue
if resources.pids is not None and (tt.limits.pids is None or tt.limits.pids > resources.pids):
continue
if resources.storage is not None and (tt.limits.storage is None or tt.limits.storage > resources.storage):
continue
if resources.image is not None:
if tt.limits.image is None:
continue
image_usage_add = max(image_usage.get(tt.image, 0), tt.limits.image) - image_usage.get(tt.image, 0)
if image_usage_add > resources.image:
continue
if resources.workspace is not None and (tt.limits.workspace is None or tt.limits.workspace > resources.workspace):
continue
if resources.network is not None and (tt.limits.network is None or tt.limits.network and not resources.network):
continue
if resources.time is not None and (tt.limits.time is None or tt.limits.time > resources.time):
continue
tasks.append(tt.dump())
t.assignee = request.user
t.time_assign = django.utils.timezone.now()
t.save()
if resources.cpus is not None:
resources.cpus -= tt.limits.cpus
if resources.gpus is not None:
resources.gpus -= tt.limits.gpus
if resources.memory is not None:
resources.memory -= tt.limits.memory
if resources.swap is not None:
resources.swap -= tt.limits.swap
if resources.pids is not None:
resources.pids -= tt.limits.pids
if resources.storage is not None:
resources.storage -= tt.limits.storage
if resources.image is not None:
resources.image -= image_usage_add
image_usage[tt.image] = max(image_usage.get(tt.image, 0), tt.limits.image)
if resources.workspace is not None:
resources.workspace -= tt.limits.workspace
if tt.exclusive:
break
response = dict()
response['tasks'] = tasks
return OKResponse(response)
def foreman():
config = foreman_config()
limits = KolejkaLimits()
limits.cpus = config.cpus
limits.memory = config.memory
limits.pids = config.pids
limits.storage = config.storage
limits.time = config.time
limits.network = config.network
client = KolejkaClient()
while True:
try:
tasks = client.dequeue(config.concurency, limits, config.tags)
if len(tasks) == 0:
time.sleep(config.interval)
else:
while len(tasks) > 0:
resources = KolejkaLimits()
resources.update(limits)
processes = list()
cpus_offset = 0
for task in tasks:
if len(processes) >= config.concurency:
break
if task.exclusive and len(processes) > 0:
break
task.limits.update(limits)
task.limits.cpus_offset = cpus_offset
ok = True
if resources.cpus is not None and task.limits.cpus > resources.cpus:
ok = False
if resources.memory is not None and task.limits.memory > resources.memory:
ok = False
if resources.pids is not None and task.limits.pids > resources.pids:
ok = False
if resources.storage is not None and task.limits.storage > resources.storage:
ok = False
if ok:
proc = Thread(target=foreman_single,
args=(config.temp_path, client,
task))
proc.start()
processes.append(proc)
cpus_offset += task.limits.cpus
if resources.cpus is not None:
resources.cpus -= task.limits.cpus
if resources.memory is not None:
resources.memory -= task.limits.memory
if resources.pids is not None:
resources.pids -= task.limits.pids
if resources.storage is not None:
resources.storage -= task.limits.storage
tasks = tasks[1:]
if task.exclusive:
break
else:
break
for proc in processes:
proc.join()
except:
time.sleep(config.interval)
def stage0(task_path, result_path, temp_path=None, consume_task_folder=False):
config = worker_config()
cgs = ControlGroupSystem()
task = KolejkaTask(task_path)
if not task.id:
task.id = uuid.uuid4().hex
logging.warning('Assigned id {} to the task'.format(task.id))
if not task.image:
logging.error('Task does not define system image')
sys.exit(1)
if not task.args:
logging.error('Task does not define args')
sys.exit(1)
if not task.files.is_local:
logging.error('Task contains non-local files')
sys.exit(1)
limits = KolejkaLimits()
limits.cpus = config.cpus
limits.memory = config.memory
limits.swap = config.swap
limits.pids = config.pids
limits.storage = config.storage
limits.image = config.image
limits.workspace = config.workspace
limits.time = config.time
limits.network = config.network
limits.gpus = config.gpus
task.limits.update(limits)
docker_task = 'kolejka_worker_{}'.format(task.id)
docker_cleanup = [
['docker', 'kill', docker_task],
['docker', 'rm', docker_task],
]
with tempfile.TemporaryDirectory(dir=temp_path) as jailed_path:
#TODO jailed_path size remains unlimited?
logging.debug('Using {} as temporary directory'.format(jailed_path))
jailed_task_path = os.path.join(jailed_path, 'task')
os.makedirs(jailed_task_path, exist_ok=True)
jailed_result_path = os.path.join(jailed_path, 'result')
os.makedirs(jailed_result_path, exist_ok=True)
jailed = KolejkaTask(os.path.join(jailed_path, 'task'))
jailed.load(task.dump())
jailed.files.clear()
volumes = list()
check_python_volume()
if os.path.exists(OBSERVER_SOCKET):
volumes.append((OBSERVER_SOCKET, OBSERVER_SOCKET, 'rw'))
else:
logging.warning('Observer is not running.')
volumes.append(
(jailed_result_path, os.path.join(WORKER_DIRECTORY,
'result'), 'rw'))
for key, val in task.files.items():
if key != TASK_SPEC:
src_path = os.path.join(task.path, val.path)
dst_path = os.path.join(jailed_path, 'task', key)
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
if consume_task_folder:
shutil.move(src_path, dst_path)
else:
shutil.copy(src_path, dst_path)
jailed.files.add(key)
jailed.files.add(TASK_SPEC)
#jailed.limits = KolejkaLimits() #TODO: Task is limited by docker, no need to limit it again?
jailed.commit()
volumes.append((jailed.path, os.path.join(WORKER_DIRECTORY,
'task'), 'rw'))
if consume_task_folder:
try:
shutil.rmtree(task_path)
except:
logging.warning('Failed to remove {}'.format(task_path))
pass
for spath in [os.path.dirname(__file__)]:
stage1 = os.path.join(spath, 'stage1.sh')
if os.path.isfile(stage1):
volumes.append(
(stage1, os.path.join(WORKER_DIRECTORY,
'stage1.sh'), 'ro'))
break
for spath in [os.path.dirname(__file__)]:
stage2 = os.path.join(spath, 'stage2.py')
if os.path.isfile(stage2):
volumes.append(
(stage2, os.path.join(WORKER_DIRECTORY,
'stage2.py'), 'ro'))
break
docker_call = ['docker', 'run']
docker_call += ['--detach']
docker_call += ['--name', docker_task]
docker_call += [
'--entrypoint',
os.path.join(WORKER_DIRECTORY, 'stage1.sh')
]
for key, val in task.environment.items():
docker_call += ['--env', '{}={}'.format(key, val)]
docker_call += ['--hostname', WORKER_HOSTNAME]
docker_call += ['--init']
if task.limits.cpus is not None:
docker_call += [
'--cpuset-cpus', ','.join([
str(c) for c in cgs.limited_cpuset(cgs.full_cpuset(
), task.limits.cpus, task.limits.cpus_offset)
])
]
if task.limits.gpus is not None and task.limits.gpus > 0:
check_gpu_runtime_availability()
gpus = ','.join(
map(
str,
limited_gpuset(full_gpuset(), task.limits.gpus,
task.limits.gpus_offset)))
docker_call += [
'--runtime=nvidia', '--shm-size=1g', '--gpus',
f'"device={gpus}"'
]
if task.limits.memory is not None:
docker_call += ['--memory', str(task.limits.memory)]
if task.limits.swap is not None:
docker_call += [
'--memory-swap',
str(task.limits.memory + task.limits.swap)
]
if task.limits.storage is not None:
docker_info_run = subprocess.run(
['docker', 'system', 'info', '--format', '{{json .Driver}}'],
stdout=subprocess.PIPE,
check=True)
storage_driver = str(
json.loads(str(docker_info_run.stdout, 'utf-8')))
if storage_driver == 'overlay2':
docker_info_run = subprocess.run([
'docker', 'system', 'info', '--format',
'{{json .DriverStatus}}'
],
stdout=subprocess.PIPE,
check=True)
storage_fs = dict(
json.loads(str(docker_info_run.stdout,
'utf-8')))['Backing Filesystem']
if storage_fs in ['xfs']:
storage_limit = task.limits.storage
docker_call += [
'--storage-opt', 'size=' + str(storage_limit)
]
else:
logging.warning(
"Storage limit on {} ({}) is not supported".format(
storage_driver, storage_fs))
else:
logging.warning("Storage limit on {} is not supported".format(
storage_driver))
if task.limits.network is not None:
if not task.limits.network:
docker_call += ['--network=none']
docker_call += ['--cap-add', 'SYS_NICE']
if task.limits.pids is not None:
docker_call += ['--pids-limit', str(task.limits.pids)]
if task.limits.time is not None:
docker_call += [
'--stop-timeout',
str(int(math.ceil(task.limits.time.total_seconds())))
]
docker_call += [
'--volume',
'{}:{}:{}'.format(WORKER_PYTHON_VOLUME,
os.path.join(WORKER_DIRECTORY, 'python3'), 'ro')
]
for v in volumes:
docker_call += [
'--volume', '{}:{}:{}'.format(os.path.realpath(v[0]), v[1],
v[2])
]
docker_call += ['--workdir', WORKER_DIRECTORY]
docker_image = task.image
docker_call += [docker_image]
docker_call += ['--consume']
if config.debug:
docker_call += ['--debug']
if config.verbose:
docker_call += ['--verbose']
docker_call += [os.path.join(WORKER_DIRECTORY, 'task')]
docker_call += [os.path.join(WORKER_DIRECTORY, 'result')]
logging.debug('Docker call : {}'.format(docker_call))
pull_image = config.pull
if not pull_image:
docker_inspect_run = subprocess.run(
['docker', 'image', 'inspect', docker_image],
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
if docker_inspect_run.returncode != 0:
pull_image = True
if pull_image:
subprocess.run(['docker', 'pull', docker_image], check=True)
for docker_clean in docker_cleanup:
silent_call(docker_clean)
if os.path.exists(result_path):
shutil.rmtree(result_path)
os.makedirs(result_path, exist_ok=True)
result = KolejkaResult(result_path)
result.id = task.id
result.limits = task.limits
result.stdout = task.stdout
result.stderr = task.stderr
start_time = datetime.datetime.now()
docker_run = subprocess.run(docker_call, stdout=subprocess.PIPE)
cid = str(docker_run.stdout, 'utf-8').strip()
logging.info('Started container {}'.format(cid))
try:
if task.limits.gpus is not None and task.limits.gpus > 0:
result.stats.update(
gpu_stats(gpus=limited_gpuset(full_gpuset(
), task.limits.gpus, task.limits.gpus_offset)))
except:
pass
time.sleep(0.1)
while True:
try:
docker_state_run = subprocess.run(
['docker', 'inspect', '--format', '{{json .State}}', cid],
stdout=subprocess.PIPE)
state = json.loads(str(docker_state_run.stdout, 'utf-8'))
except:
break
try:
result.stats.update(cgs.name_stats(cid))
if task.limits.gpus is not None and task.limits.gpus > 0:
result.stats.update(
gpu_stats(gpus=limited_gpuset(full_gpuset(
), task.limits.gpus, task.limits.gpus_offset)))
except:
pass
time.sleep(0.1)
if not state['Running']:
result.result = state['ExitCode']
try:
result.stats.time = dateutil.parser.parse(
state['FinishedAt']) - dateutil.parser.parse(
state['StartedAt'])
except:
result.stats.time = None
break
if task.limits.time is not None and datetime.datetime.now(
) - start_time > task.limits.time + datetime.timedelta(seconds=2):
docker_kill_run = subprocess.run(
['docker', 'kill', docker_task])
subprocess.run(['docker', 'logs', cid], stdout=subprocess.PIPE)
try:
summary = KolejkaResult(jailed_result_path)
result.stats.update(summary.stats)
except:
pass
stop_time = datetime.datetime.now()
if result.stats.time is None:
result.stats.time = stop_time - start_time
result.stats.pids.usage = None
result.stats.memory.usage = None
result.stats.memory.swap = None
for dirpath, dirnames, filenames in os.walk(jailed_result_path):
for filename in filenames:
abspath = os.path.join(dirpath, filename)
realpath = os.path.realpath(abspath)
if realpath.startswith(
os.path.realpath(jailed_result_path) + '/'):
relpath = abspath[len(jailed_result_path) + 1:]
if relpath != RESULT_SPEC:
destpath = os.path.join(result.path, relpath)
os.makedirs(os.path.dirname(destpath), exist_ok=True)
shutil.move(realpath, destpath)
os.chmod(destpath, 0o640)
result.files.add(relpath)
result.commit()
os.chmod(result.spec_path, 0o640)
for docker_clean in docker_cleanup:
silent_call(docker_clean)
def foreman():
config = foreman_config()
limits = KolejkaLimits()
limits.cpus = config.cpus
limits.memory = config.memory
limits.swap = config.swap
limits.pids = config.pids
limits.storage = config.storage
limits.image = config.image
limits.workspace = config.workspace
limits.time = config.time
limits.network = config.network
limits.gpus = config.gpus
client = KolejkaClient()
while True:
try:
tasks = client.dequeue(config.concurency, limits, config.tags)
if len(tasks) == 0:
time.sleep(config.interval)
else:
check_python_volume()
while len(tasks) > 0:
resources = KolejkaLimits()
resources.update(limits)
image_usage = dict()
processes = list()
cpus_offset = 0
gpus_offset = 0
for task in tasks:
if len(processes) >= config.concurency:
break
if task.exclusive and len(processes) > 0:
break
task.limits.update(limits)
task.limits.cpus_offset = cpus_offset
task.limits.gpus_offset = gpus_offset
ok = True
if resources.cpus is not None and task.limits.cpus > resources.cpus:
ok = False
if task.limits.gpus is not None and task.limits.gpus > 0:
if resources.gpus is None or task.limits.gpus > resources.gpus:
ok = False
if resources.memory is not None and task.limits.memory > resources.memory:
ok = False
if resources.gpus is not None:
if task.limits.gpus > resources.gpus:
ok = False
if resources.swap is not None and task.limits.swap > resources.swap:
ok = False
if resources.pids is not None and task.limits.pids > resources.pids:
ok = False
if resources.storage is not None and task.limits.storage > resources.storage:
ok = False
if resources.image is not None:
image_usage_add = max(
image_usage.get(task.image, 0),
task.limits.image) - image_usage.get(
task.image, 0)
if image_usage_add > resources.image:
ok = False
if resources.workspace is not None and task.limits.workspace > resources.workspace:
ok = False
if ok:
proc = Process(target=foreman_single,
args=(config.temp_path, task))
processes.append(proc)
cpus_offset += task.limits.cpus
if resources.cpus is not None:
resources.cpus -= task.limits.cpus
gpus_offset += task.limits.gpus
if resources.gpus is not None:
resources.gpus -= task.limits.gpus
if resources.memory is not None:
resources.memory -= task.limits.memory
if resources.swap is not None:
resources.swap -= task.limits.swap
if resources.pids is not None:
resources.pids -= task.limits.pids
if resources.storage is not None:
resources.storage -= task.limits.storage
if resources.image is not None:
resources.image -= image_usage_add
image_usage[task.image] = max(
image_usage.get(task.image, 0),
task.limits.image)
if resources.workspace is not None:
resources.workspace -= task.limits.workspace
tasks = tasks[1:]
if task.exclusive:
break
else:
break
if config.image is not None:
manage_images(config.pull, config.image, image_usage,
[task.image for task in tasks])
for proc in processes:
proc.start()
for proc in processes:
proc.join()
except KeyboardInterrupt:
raise
except:
traceback.print_exc()
time.sleep(config.interval)
def task(request, key=''):
if request.method == 'POST':
if key != '':
return HttpResponseForbidden()
if not request.user.has_perm('task.add_task'):
return HttpResponseForbidden()
t = KolejkaTask(None)
t.load(request.read())
for image_re, image_sub in settings.IMAGE_NAME_MAPS:
t.image = re.sub(r'^' + image_re + r'$', image_sub, t.image)
accept_image = False
for image_re in settings.LIMIT_IMAGE_NAME:
if re.match(image_re, t.image):
accept_image = True
break
if not accept_image:
return FAILResponse(
message='Image {} is not accepted by the server'.format(
t.image))
local_image = False
for image_re in settings.LOCAL_IMAGE_NAMES:
if re.match(image_re, t.image):
local_image = True
break
t.id = uuid.uuid4().hex
for k, f in t.files.items():
if not f.reference:
return FAILResponse(
message='File {} does not have a reference'.format(k))
f.path = None
refs = list()
for k, f in t.files.items():
try:
ref = Reference.objects.get(key=f.reference)
except Reference.DoesNotExist:
return FAILResponse(
message='Reference for file {} is unknown'.format(k))
if not ref.public:
if not request.user.has_perm(
'blob.view_reference') and request.user != ref.user:
return FAILResponse(
message='Reference for file {} is unknown'.format(k))
refs.append(ref)
limits = KolejkaLimits(
cpus=settings.LIMIT_CPUS,
memory=settings.LIMIT_MEMORY,
swap=settings.LIMIT_SWAP,
pids=settings.LIMIT_PIDS,
storage=settings.LIMIT_STORAGE,
network=settings.LIMIT_NETWORK,
time=settings.LIMIT_TIME,
image=settings.LIMIT_IMAGE,
workspace=settings.LIMIT_WORKSPACE,
)
t.limits.update(limits)
if settings.IMAGE_REGISTRY is not None and settings.IMAGE_REGISTRY_NAME is not None and not local_image:
try:
subprocess.run(['docker', 'pull', t.image], check=True)
docker_inspect_run = subprocess.run([
'docker', 'image', 'inspect', '--format', '{{json .Id}}',
t.image
],
stdout=subprocess.PIPE,
check=True)
image_id = str(
json.loads(str(docker_inspect_run.stdout,
'utf-8'))).split(':')[-1]
logging.info(image_id)
docker_inspect_run = subprocess.run([
'docker', 'image', 'inspect', '--format', '{{json .Size}}',
t.image
],
stdout=subprocess.PIPE,
check=True)
image_size = int(
json.loads(str(docker_inspect_run.stdout, 'utf-8')))
except:
return FAILResponse(
message='Image {} could not be pulled'.format(t.image))
if t.limits.image is not None and image_size > t.limits.image:
return FAILResponse(
message='Image {} exceeds image size limit {}'.format(
t.image, t.limits.image))
image_name = settings.IMAGE_REGISTRY + '/' + settings.IMAGE_REGISTRY_NAME + ':' + image_id
try:
subprocess.run(['docker', 'tag', t.image, image_name],
check=True)
subprocess.run(['docker', 'push', image_name], check=True)
subprocess.run(['docker', 'rmi', image_name], check=True)
except:
return FAILResponse(
message='Image {} could not be pushed to local repository'.
format(t.image))
t.image = image_name
t.limits.image = image_size
task = models.Task(user=request.user,
key=t.id,
description=json.dumps(t.dump()))
task.save()
for ref in refs:
task.files.add(ref)
response = dict()
response['task'] = task.task().dump()
return OKResponse(response)
if not request.user.is_authenticated:
return HttpResponseForbidden()
try:
task = models.Task.objects.get(key=key)
except models.Task.DoesNotExist:
return HttpResponseNotFound()
if not request.user.has_perm(
'task.view_task'
) and request.user != task.user and request.user != task.assignee:
return HttpResponseForbidden()
if request.method == 'PUT':
response = dict()
response['task'] = task.task().dump()
return OKResponse(response)
if request.method == 'DELETE':
if not request.user.has_perm(
'task.delete_task') and request.user != task.user:
return HttpResponseForbidden()
task.delete()
return OKResponse({'deleted': True})
if request.method == 'GET' or request.method == 'HEAD':
response = dict()
response['task'] = task.task().dump()
return OKResponse(response)
return HttpResponseNotAllowed(['HEAD', 'GET', 'POST', 'PUT', 'DELETE'])
def task_put(self, task):
limits = KolejkaLimits()
limits.cpus = self.config.cpus
limits.memory = self.config.memory
limits.swap = self.config.swap
limits.pids = self.config.pids
limits.storage = self.config.storage
limits.image = self.config.image
limits.workspace = self.config.workspace
limits.time = self.config.time
limits.network = self.config.network
limits.gpus = self.config.gpus
task.limits.update(limits)
if not self.instance_session:
self.login()
for f in task.files.values():
if not f.reference or not self.blob_check(
blob_reference=f.reference):
assert f.path
f.reference = self.blob_put(os.path.join(task.path,
f.path))['key']
response = self.post('/task/task/', data=json.dumps(task.dump()))
task = KolejkaTask(None)
task.load(response.json()['task'])
return task
