def dequeue(request):
if request.method == 'POST':
if not request.user.is_authenticated():
return HttpResponseForbidden()
#TODO: Check that user can get tasks
tasks = list()
params = json.loads(str(request.read(), request.encoding or 'utf-8'))
concurency = params.get('concurency', 1)
limits = KolejkaLimits()
limits.load(params.get('limits', dict()))
tags = set(params.get('tags', list()))
resources = KolejkaLimits()
resources.update(limits)
available_tasks = Task.objects.filter(
assignee=None).order_by('time_create')[0:100]
for t in available_tasks:
if len(tasks) > concurency:
break
tt = t.task()
if len(tasks) > 0 and tt.exclusive:
continue
if not set(tt.requires).issubset(tags):
continue
if resources.cpus is not None and (
tt.limits.cpus is None or tt.limits.cpus > resources.cpus):
continue
if resources.memory is not None and (
tt.limits.memory is None
or tt.limits.memory > resources.memory):
continue
if resources.pids is not None and (
tt.limits.pids is None or tt.limits.pids > resources.pids):
continue
if resources.storage is not None and (
tt.limits.storage is None
or tt.limits.storage > resources.storage):
continue
if resources.network is not None and (tt.limits.network is None
or tt.limits.network
and not resources.network):
continue
if resources.time is not None and (
tt.limits.time is None or tt.limits.time > resources.time):
continue
tasks.append(tt.dump())
t.assignee = request.user
t.save()
if resources.cpus is not None:
resources.cpus -= tt.limits.cpus
if resources.memory is not None:
resources.memory -= tt.limits.memory
if resources.pids is not None:
resources.pids -= tt.limits.pids
if resources.storage is not None:
resources.storage -= tt.limits.storage
if tt.exclusive:
break
response = dict()
response['tasks'] = tasks
return JsonResponse(response)
return HttpResponseNotAllowed(['POST'])
def foreman():
config = foreman_config()
limits = KolejkaLimits()
limits.cpus = config.cpus
limits.memory = config.memory
limits.pids = config.pids
limits.storage = config.storage
limits.time = config.time
limits.network = config.network
client = KolejkaClient()
while True:
try:
tasks = client.dequeue(config.concurency, limits, config.tags)
if len(tasks) == 0:
time.sleep(config.interval)
else:
while len(tasks) > 0:
resources = KolejkaLimits()
resources.update(limits)
processes = list()
cpus_offset = 0
for task in tasks:
if len(processes) >= config.concurency:
break
if task.exclusive and len(processes) > 0:
break
task.limits.update(limits)
task.limits.cpus_offset = cpus_offset
ok = True
if resources.cpus is not None and task.limits.cpus > resources.cpus:
ok = False
if resources.memory is not None and task.limits.memory > resources.memory:
ok = False
if resources.pids is not None and task.limits.pids > resources.pids:
ok = False
if resources.storage is not None and task.limits.storage > resources.storage:
ok = False
if ok:
proc = Thread(target=foreman_single,
args=(config.temp_path, client,
task))
proc.start()
processes.append(proc)
cpus_offset += task.limits.cpus
if resources.cpus is not None:
resources.cpus -= task.limits.cpus
if resources.memory is not None:
resources.memory -= task.limits.memory
if resources.pids is not None:
resources.pids -= task.limits.pids
if resources.storage is not None:
resources.storage -= task.limits.storage
tasks = tasks[1:]
if task.exclusive:
break
else:
break
for proc in processes:
proc.join()
except:
time.sleep(config.interval)
def dequeue(request):
if not request.user.has_perm('task.process_task'):
return HttpResponseForbidden()
if request.method != 'POST':
return HttpResponseNotAllowed(['POST'])
content_type = ContentType.objects.get_for_model(Task)
tasks = list()
params = json.loads(str(request.read(), request.encoding or 'utf-8'))
concurency = params.get('concurency', 1)
limits = KolejkaLimits()
limits.load(params.get('limits', dict()))
tags = set(params.get('tags', list()))
resources = KolejkaLimits()
resources.update(limits)
image_usage = dict()
available_tasks = Task.objects.filter(assignee__isnull=True).order_by('time_create')[0:100]
for t in available_tasks:
if len(tasks) > concurency:
break
tt = t.task()
if len(tasks) > 0 and tt.exclusive:
print('At exclusive')
continue
if not set(tt.requires).issubset(tags):
continue
if resources.cpus is not None and (tt.limits.cpus is None or tt.limits.cpus > resources.cpus):
continue
if tt.limits.gpus is not None and tt.limits.gpus > 0:
if resources.gpus is not None and (tt.limits.gpus is None or tt.limits.gpus > resources.gpus):
continue
if resources.memory is not None and (tt.limits.memory is None or tt.limits.memory > resources.memory):
continue
if resources.swap is not None and (tt.limits.swap is None or tt.limits.swap > resources.swap):
continue
if resources.pids is not None and (tt.limits.pids is None or tt.limits.pids > resources.pids):
continue
if resources.storage is not None and (tt.limits.storage is None or tt.limits.storage > resources.storage):
continue
if resources.image is not None:
if tt.limits.image is None:
continue
image_usage_add = max(image_usage.get(tt.image, 0), tt.limits.image) - image_usage.get(tt.image, 0)
if image_usage_add > resources.image:
continue
if resources.workspace is not None and (tt.limits.workspace is None or tt.limits.workspace > resources.workspace):
continue
if resources.network is not None and (tt.limits.network is None or tt.limits.network and not resources.network):
continue
if resources.time is not None and (tt.limits.time is None or tt.limits.time > resources.time):
continue
tasks.append(tt.dump())
t.assignee = request.user
t.time_assign = django.utils.timezone.now()
t.save()
if resources.cpus is not None:
resources.cpus -= tt.limits.cpus
if resources.gpus is not None:
resources.gpus -= tt.limits.gpus
if resources.memory is not None:
resources.memory -= tt.limits.memory
if resources.swap is not None:
resources.swap -= tt.limits.swap
if resources.pids is not None:
resources.pids -= tt.limits.pids
if resources.storage is not None:
resources.storage -= tt.limits.storage
if resources.image is not None:
resources.image -= image_usage_add
image_usage[tt.image] = max(image_usage.get(tt.image, 0), tt.limits.image)
if resources.workspace is not None:
resources.workspace -= tt.limits.workspace
if tt.exclusive:
break
response = dict()
response['tasks'] = tasks
return OKResponse(response)
def foreman():
config = foreman_config()
limits = KolejkaLimits()
limits.cpus = config.cpus
limits.memory = config.memory
limits.swap = config.swap
limits.pids = config.pids
limits.storage = config.storage
limits.image = config.image
limits.workspace = config.workspace
limits.time = config.time
limits.network = config.network
limits.gpus = config.gpus
client = KolejkaClient()
while True:
try:
tasks = client.dequeue(config.concurency, limits, config.tags)
if len(tasks) == 0:
time.sleep(config.interval)
else:
check_python_volume()
while len(tasks) > 0:
resources = KolejkaLimits()
resources.update(limits)
image_usage = dict()
processes = list()
cpus_offset = 0
gpus_offset = 0
for task in tasks:
if len(processes) >= config.concurency:
break
if task.exclusive and len(processes) > 0:
break
task.limits.update(limits)
task.limits.cpus_offset = cpus_offset
task.limits.gpus_offset = gpus_offset
ok = True
if resources.cpus is not None and task.limits.cpus > resources.cpus:
ok = False
if task.limits.gpus is not None and task.limits.gpus > 0:
if resources.gpus is None or task.limits.gpus > resources.gpus:
ok = False
if resources.memory is not None and task.limits.memory > resources.memory:
ok = False
if resources.gpus is not None:
if task.limits.gpus > resources.gpus:
ok = False
if resources.swap is not None and task.limits.swap > resources.swap:
ok = False
if resources.pids is not None and task.limits.pids > resources.pids:
ok = False
if resources.storage is not None and task.limits.storage > resources.storage:
ok = False
if resources.image is not None:
image_usage_add = max(
image_usage.get(task.image, 0),
task.limits.image) - image_usage.get(
task.image, 0)
if image_usage_add > resources.image:
ok = False
if resources.workspace is not None and task.limits.workspace > resources.workspace:
ok = False
if ok:
proc = Process(target=foreman_single,
args=(config.temp_path, task))
processes.append(proc)
cpus_offset += task.limits.cpus
if resources.cpus is not None:
resources.cpus -= task.limits.cpus
gpus_offset += task.limits.gpus
if resources.gpus is not None:
resources.gpus -= task.limits.gpus
if resources.memory is not None:
resources.memory -= task.limits.memory
if resources.swap is not None:
resources.swap -= task.limits.swap
if resources.pids is not None:
resources.pids -= task.limits.pids
if resources.storage is not None:
resources.storage -= task.limits.storage
if resources.image is not None:
resources.image -= image_usage_add
image_usage[task.image] = max(
image_usage.get(task.image, 0),
task.limits.image)
if resources.workspace is not None:
resources.workspace -= task.limits.workspace
tasks = tasks[1:]
if task.exclusive:
break
else:
break
if config.image is not None:
manage_images(config.pull, config.image, image_usage,
[task.image for task in tasks])
for proc in processes:
proc.start()
for proc in processes:
proc.join()
except KeyboardInterrupt:
raise
except:
traceback.print_exc()
time.sleep(config.interval)