def login(request):
if request.method == 'POST':
username = request.POST['username']
password = request.POST['password']
user = authenticate(request, username=username, password=password)
if user is not None:
auth_login(request, user)
return OKResponse({})
return FAILResponse({})
if request.method == 'GET':
return OKResponse({})
return HttpResponseNotAllowed(['GET', 'POST'])
def blob(request, key):
try:
blob = models.Blob.objects.get(key=key)
except models.Blob.DoesNotExist:
return HttpResponseNotFound()
if not request.user.has_perm('blob.view_blob'):
return HttpResponseForbidden()
if request.method == 'POST': #CREATE NEW REFERENCE TO THE BLOB
reference = models.Reference(user=request.user, blob=blob)
reference.save()
blob.save()
response = dict()
response['reference'] = {
'key': reference.key,
'blob': reference.blob.key,
'size': reference.blob.size,
'time_create': reference.time_create,
'time_access': reference.time_access,
}
return OKResponse(response)
if request.method == 'PUT': #QUERY BLOB DATA
response = dict()
response['blob'] = {
'key': blob.key,
'size': blob.size,
'time_create': blob.time_create,
'time_access': blob.time_access,
'references': blob.reference_set.count(),
}
return OKResponse(response)
if request.method == 'DELETE': #DELETE BLOB AND ALL REFERENCES
if not request.user.has_perm('blob.delete_blob'):
return HttpResponseForbidden()
blob.reference_set.all().delete()
blob.delete()
return OKResponse({'deleted': True})
if request.method == 'GET' or request.method == 'HEAD':
blob.save()
return StreamingHttpResponse(blob.open(),
content_type='application/octet-stream')
return HttpResponseNotAllowed(['HEAD', 'GET', 'POST', 'PUT', 'DELETE'])
def stats(request):
if not request.user.is_authenticated:
return HttpResponseForbidden()
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
response = dict()
response['task_count'] = Task.objects.count()
response['task_resolved_count'] = Task.objects.filter(result__isnull=False).count()
response['task_unresolved_count'] = Task.objects.filter(result__isnull=True).count()
response['task_unassigned_count'] = Task.objects.filter(assignee__isnull=True).count()
assignees = dict([ (v['assignee_username'], v['task_count']) for v in Task.objects.filter(assignee__isnull=False, result__isnull=True).annotate(assignee_username=F('assignee__username')).values('assignee_username').annotate(task_count=Count('assignee_username')) ])
response['task_assigned_count'] = sum(assignees.values())
response['task_assignees'] = assignees
return OKResponse(response)
def settings(request):
if not request.user.is_authenticated:
return HttpResponseForbidden()
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
settings = django.conf.settings
response = dict()
response['blob_hash_algorithm'] = settings.BLOB_HASH_ALGORITHM
limits = KolejkaLimits(
cpus=settings.LIMIT_CPUS,
memory=settings.LIMIT_MEMORY,
swap=settings.LIMIT_SWAP,
pids=settings.LIMIT_PIDS,
storage=settings.LIMIT_STORAGE,
image=settings.LIMIT_IMAGE,
workspace=settings.LIMIT_WORKSPACE,
network=settings.LIMIT_NETWORK,
time=settings.LIMIT_TIME,
)
response['limits'] = limits.dump()
return OKResponse(response)
def dequeue(request):
if not request.user.has_perm('task.process_task'):
return HttpResponseForbidden()
if request.method != 'POST':
return HttpResponseNotAllowed(['POST'])
content_type = ContentType.objects.get_for_model(Task)
tasks = list()
params = json.loads(str(request.read(), request.encoding or 'utf-8'))
concurency = params.get('concurency', 1)
limits = KolejkaLimits()
limits.load(params.get('limits', dict()))
tags = set(params.get('tags', list()))
resources = KolejkaLimits()
resources.update(limits)
image_usage = dict()
available_tasks = Task.objects.filter(assignee__isnull=True).order_by('time_create')[0:100]
for t in available_tasks:
if len(tasks) > concurency:
break
tt = t.task()
if len(tasks) > 0 and tt.exclusive:
print('At exclusive')
continue
if not set(tt.requires).issubset(tags):
continue
if resources.cpus is not None and (tt.limits.cpus is None or tt.limits.cpus > resources.cpus):
continue
if tt.limits.gpus is not None and tt.limits.gpus > 0:
if resources.gpus is not None and (tt.limits.gpus is None or tt.limits.gpus > resources.gpus):
continue
if resources.memory is not None and (tt.limits.memory is None or tt.limits.memory > resources.memory):
continue
if resources.swap is not None and (tt.limits.swap is None or tt.limits.swap > resources.swap):
continue
if resources.pids is not None and (tt.limits.pids is None or tt.limits.pids > resources.pids):
continue
if resources.storage is not None and (tt.limits.storage is None or tt.limits.storage > resources.storage):
continue
if resources.image is not None:
if tt.limits.image is None:
continue
image_usage_add = max(image_usage.get(tt.image, 0), tt.limits.image) - image_usage.get(tt.image, 0)
if image_usage_add > resources.image:
continue
if resources.workspace is not None and (tt.limits.workspace is None or tt.limits.workspace > resources.workspace):
continue
if resources.network is not None and (tt.limits.network is None or tt.limits.network and not resources.network):
continue
if resources.time is not None and (tt.limits.time is None or tt.limits.time > resources.time):
continue
tasks.append(tt.dump())
t.assignee = request.user
t.time_assign = django.utils.timezone.now()
t.save()
if resources.cpus is not None:
resources.cpus -= tt.limits.cpus
if resources.gpus is not None:
resources.gpus -= tt.limits.gpus
if resources.memory is not None:
resources.memory -= tt.limits.memory
if resources.swap is not None:
resources.swap -= tt.limits.swap
if resources.pids is not None:
resources.pids -= tt.limits.pids
if resources.storage is not None:
resources.storage -= tt.limits.storage
if resources.image is not None:
resources.image -= image_usage_add
image_usage[tt.image] = max(image_usage.get(tt.image, 0), tt.limits.image)
if resources.workspace is not None:
resources.workspace -= tt.limits.workspace
if tt.exclusive:
break
response = dict()
response['tasks'] = tasks
return OKResponse(response)
def reference(request, key=''):
if request.method == 'POST': #CREATE A NEW REFERENCE BY SENDING A BLOB
if key != '':
return HttpResponseForbidden()
if not request.user.has_perm('blob.add_reference'):
return HttpResponseForbidden()
hasher = hashlib.new(settings.BLOB_HASH_ALGORITHM)
temp_file = models.Blob.blob_temp_path()
file_size = 0
blob = None
try:
with open(temp_file, 'wb') as tf:
while True:
buf = request.read(READ_BUF_SIZE)
if len(buf) == 0:
break
file_size += len(buf)
tf.write(buf)
hasher.update(buf)
key = hasher.hexdigest()
blob, created = models.Blob.objects.get_or_create(key=key,
size=file_size)
if not os.path.exists(blob.store_path):
os.rename(temp_file, blob.store_path)
blob.activate()
finally:
if os.path.exists(temp_file):
os.unlink(temp_file)
reference = models.Reference(blob=blob, user=request.user)
reference.save()
response = dict()
response['reference'] = {
'key': reference.key,
'blob': reference.blob.key,
'size': reference.blob.size,
'time_create': reference.time_create,
'time_access': reference.time_access,
}
return OKResponse(response)
try:
reference = models.Reference.objects.get(key=key)
except models.Reference.DoesNotExist:
return HttpResponseNotFound()
if not reference.public and request.user != reference.user:
if not request.user.has_perm('blob.view_reference'):
return HttpResponseForbidden()
if request.method == 'PUT': #QUERY REFERENCE DATA
response = dict()
response['reference'] = {
'key': reference.key,
'blob': reference.blob.key,
'size': reference.blob.size,
'time_create': reference.time_create,
'time_access': reference.time_access,
}
return OKResponse(response)
if request.method == 'DELETE': #DELETE REFERENCE
if not request.user.has_perm(
'blob.delete_reference') and request.user != reference.user:
return HttpResponseForbidden()
reference.delete()
return OKResponse({'deleted': True})
if request.method == 'GET' or request.method == 'HEAD': #GET REFERENCED BLOB
reference.blob.save()
reference.save()
return StreamingHttpResponse(reference.blob.open(),
content_type='application/octet-stream')
return HttpResponseNotAllowed(['HEAD', 'GET', 'POST', 'PUT', 'DELETE'])
def logout(request):
auth_logout(request)
return OKResponse({})
def task(request, key=''):
if request.method == 'POST':
if key != '':
return HttpResponseForbidden()
if not request.user.has_perm('task.add_task'):
return HttpResponseForbidden()
t = KolejkaTask(None)
t.load(request.read())
for image_re, image_sub in settings.IMAGE_NAME_MAPS:
t.image = re.sub(r'^' + image_re + r'$', image_sub, t.image)
accept_image = False
for image_re in settings.LIMIT_IMAGE_NAME:
if re.match(image_re, t.image):
accept_image = True
break
if not accept_image:
return FAILResponse(
message='Image {} is not accepted by the server'.format(
t.image))
local_image = False
for image_re in settings.LOCAL_IMAGE_NAMES:
if re.match(image_re, t.image):
local_image = True
break
t.id = uuid.uuid4().hex
for k, f in t.files.items():
if not f.reference:
return FAILResponse(
message='File {} does not have a reference'.format(k))
f.path = None
refs = list()
for k, f in t.files.items():
try:
ref = Reference.objects.get(key=f.reference)
except Reference.DoesNotExist:
return FAILResponse(
message='Reference for file {} is unknown'.format(k))
if not ref.public:
if not request.user.has_perm(
'blob.view_reference') and request.user != ref.user:
return FAILResponse(
message='Reference for file {} is unknown'.format(k))
refs.append(ref)
limits = KolejkaLimits(
cpus=settings.LIMIT_CPUS,
memory=settings.LIMIT_MEMORY,
swap=settings.LIMIT_SWAP,
pids=settings.LIMIT_PIDS,
storage=settings.LIMIT_STORAGE,
network=settings.LIMIT_NETWORK,
time=settings.LIMIT_TIME,
image=settings.LIMIT_IMAGE,
workspace=settings.LIMIT_WORKSPACE,
)
t.limits.update(limits)
if settings.IMAGE_REGISTRY is not None and settings.IMAGE_REGISTRY_NAME is not None and not local_image:
try:
subprocess.run(['docker', 'pull', t.image], check=True)
docker_inspect_run = subprocess.run([
'docker', 'image', 'inspect', '--format', '{{json .Id}}',
t.image
],
stdout=subprocess.PIPE,
check=True)
image_id = str(
json.loads(str(docker_inspect_run.stdout,
'utf-8'))).split(':')[-1]
logging.info(image_id)
docker_inspect_run = subprocess.run([
'docker', 'image', 'inspect', '--format', '{{json .Size}}',
t.image
],
stdout=subprocess.PIPE,
check=True)
image_size = int(
json.loads(str(docker_inspect_run.stdout, 'utf-8')))
except:
return FAILResponse(
message='Image {} could not be pulled'.format(t.image))
if t.limits.image is not None and image_size > t.limits.image:
return FAILResponse(
message='Image {} exceeds image size limit {}'.format(
t.image, t.limits.image))
image_name = settings.IMAGE_REGISTRY + '/' + settings.IMAGE_REGISTRY_NAME + ':' + image_id
try:
subprocess.run(['docker', 'tag', t.image, image_name],
check=True)
subprocess.run(['docker', 'push', image_name], check=True)
subprocess.run(['docker', 'rmi', image_name], check=True)
except:
return FAILResponse(
message='Image {} could not be pushed to local repository'.
format(t.image))
t.image = image_name
t.limits.image = image_size
task = models.Task(user=request.user,
key=t.id,
description=json.dumps(t.dump()))
task.save()
for ref in refs:
task.files.add(ref)
response = dict()
response['task'] = task.task().dump()
return OKResponse(response)
if not request.user.is_authenticated:
return HttpResponseForbidden()
try:
task = models.Task.objects.get(key=key)
except models.Task.DoesNotExist:
return HttpResponseNotFound()
if not request.user.has_perm(
'task.view_task'
) and request.user != task.user and request.user != task.assignee:
return HttpResponseForbidden()
if request.method == 'PUT':
response = dict()
response['task'] = task.task().dump()
return OKResponse(response)
if request.method == 'DELETE':
if not request.user.has_perm(
'task.delete_task') and request.user != task.user:
return HttpResponseForbidden()
task.delete()
return OKResponse({'deleted': True})
if request.method == 'GET' or request.method == 'HEAD':
response = dict()
response['task'] = task.task().dump()
return OKResponse(response)
return HttpResponseNotAllowed(['HEAD', 'GET', 'POST', 'PUT', 'DELETE'])
def result(request, key=''):
if request.method == 'POST':
if key != '':
return HttpResponseForbidden()
if not request.user.has_perm('task.process_task'):
return HttpResponseForbidden()
r = KolejkaResult(None)
r.load(request.read())
try:
task = models.Task.objects.get(key=r.id)
except models.Task.DoesNotExist:
return HttpResponseForbidden()
if not request.user.has_perm(
'task.add_result') and request.user != task.assignee:
return HttpResponseForbidden()
for k, f in r.files.items():
if not f.reference:
return FAILResponse(
message='File {} does not have a reference'.format(k))
f.path = None
refs = list()
for k, f in r.files.items():
try:
ref = Reference.objects.get(key=f.reference)
except Reference.DoesNotExist:
return FAILResponse(
message='Reference for file {} is unknown'.format(k))
if not ref.public:
if request.user != ref.user:
return FAILResponse(
message=
'Reference for file {} belongs to a different user'.
format(k))
refs.append(ref)
result, created = models.Result.objects.get_or_create(
task=task, user=request.user, description=json.dumps(r.dump()))
result.save()
for ref in refs:
ref.user = task.user
ref.save()
result.files.add(ref)
response = dict()
response['result'] = result.result().dump()
return OKResponse(response)
if not request.user.is_authenticated:
return HttpResponseForbidden()
try:
task = models.Task.objects.get(key=key)
result = models.Result.objects.get(task=task)
except models.Task.DoesNotExist:
return FAILResponse(message='Task {} is unknown'.format(key))
except models.Result.DoesNotExist:
return HttpResponseNotFound()
if not request.user.has_perm(
'task.view_result') and request.user != task.user:
return HttpResponseForbidden()
if request.method == 'PUT':
response = dict()
response['result'] = result.result().dump()
return OKResponse(response)
if request.method == 'DELETE':
if not request.user.has_perm(
'task.delete_result') and request.user != task.user:
return HttpResponseForbidden()
result.delete()
return OKResponse({'deleted': True})
if request.method == 'GET' or request.method == 'HEAD':
response = dict()
response['result'] = result.result().dump()
return OKResponse(response)
return HttpResponseNotAllowed(['HEAD', 'GET', 'POST', 'PUT', 'DELETE'])