def create_or_update(dl_model_id, name, model_file, weights_file,
labelmap_file, interpretation_file, owner, storage,
is_shared):
def get_abs_path(share_path):
if not share_path:
return share_path
share_root = settings.SHARE_ROOT
relpath = os.path.normpath(share_path).lstrip('/')
if '..' in relpath.split(os.path.sep):
raise Exception('Permission denied')
abspath = os.path.abspath(os.path.join(share_root, relpath))
if os.path.commonprefix([share_root, abspath]) != share_root:
raise Exception('Bad file path on share: ' + abspath)
return abspath
def save_file_as_tmp(data):
if not data:
return None
fd, filename = tempfile.mkstemp()
with open(filename, 'wb') as tmp_file:
for chunk in data.chunks():
tmp_file.write(chunk)
os.close(fd)
return filename
is_create_request = dl_model_id is None
if is_create_request:
dl_model_id = create_empty(owner=owner)
run_tests = bool(model_file or weights_file or labelmap_file
or interpretation_file)
if storage != "local":
model_file = get_abs_path(model_file)
weights_file = get_abs_path(weights_file)
labelmap_file = get_abs_path(labelmap_file)
interpretation_file = get_abs_path(interpretation_file)
else:
model_file = save_file_as_tmp(model_file)
weights_file = save_file_as_tmp(weights_file)
labelmap_file = save_file_as_tmp(labelmap_file)
interpretation_file = save_file_as_tmp(interpretation_file)
if owner:
restricted = not has_admin_role(owner)
else:
restricted = not has_admin_role(
AnnotationModel.objects.get(pk=dl_model_id).owner)
rq_id = "auto_annotation.create.{}".format(dl_model_id)
queue = django_rq.get_queue("default")
queue.enqueue_call(func=_update_dl_model_thread,
args=(dl_model_id, name, is_shared, model_file,
weights_file, labelmap_file, interpretation_file,
run_tests, storage == "local", is_create_request,
restricted),
job_id=rq_id)
return rq_id
def update_model(request, mid):
if request.method != 'POST':
return HttpResponseBadRequest("Only POST requests are accepted")
try:
params = request.POST
storage = params["storage"]
name = params.get("name")
is_shared = params.get("shared")
is_shared = is_shared.lower() == "true" if is_shared else None
if is_shared and not has_admin_role(request.user):
raise Exception("Only admin can create shared models")
files = request.FILES
model = files.get("xml")
weights = files.get("bin")
labelmap = files.get("json")
interpretation_script = files.get("py")
rq_id = model_manager.create_or_update(
dl_model_id=mid,
name=name,
model_file=model,
weights_file=weights,
labelmap_file=labelmap,
interpretation_file=interpretation_script,
owner=None,
storage=storage,
is_shared=is_shared,
)
return JsonResponse({"id": rq_id})
except Exception as e:
return HttpResponseBadRequest(str(e))
def create_model(request):
if request.method != 'POST':
return HttpResponseBadRequest("Only POST requests are accepted")
try:
params = request.POST
storage = params["storage"]
name = params["name"]
is_shared = params["shared"].lower() == "true"
if is_shared and not has_admin_role(request.user):
raise Exception("Only admin can create shared models")
files = request.FILES if storage == "local" else params
model = files["xml"]
weights = files["bin"]
labelmap = files["json"]
interpretation_script = files["py"]
owner = request.user
rq_id = model_manager.create_or_update(
dl_model_id=None,
name=name,
model_file=model,
weights_file=weights,
labelmap_file=labelmap,
interpretation_file=interpretation_script,
owner=owner,
storage=storage,
is_shared=is_shared,
)
return JsonResponse({"id": rq_id})
except Exception as e:
return HttpResponseBadRequest(str(e))
def get_queryset(self):
queryset = super().get_queryset()
user = self.request.user
# Don't filter queryset for admin
if auth.has_admin_role(user) or self.detail:
return queryset
else:
return queryset.filter(Q(owner=user)).distinct()
def task_data(request, task_id, data_path):
"""serving user's task data with permission checking.
CVAT doesn't provide APIs to access all task data, e.g. uploaded videos.
Using django-xsendfile in order to serve files with web server
https://github.com/johnsensible/django-sendfile
"""
db_tasks = Task.objects.filter(pk=task_id)
if not auth.has_admin_role(request.user):
db_tasks.filter(Q(owner=request.user))
if len(db_tasks) < 1:
raise Http404
db_task = db_tasks[0]
file_path = os.path.abspath(os.path.realpath(os.path.join(db_task.get_task_dirname(), data_path)))
return sendfile.sendfile(request, file_path)
def get_meta_info(request):
try:
tids = request.data
response = {
"admin": has_admin_role(request.user),
"models": [],
"run": {},
}
dl_model_list = list(
AnnotationModel.objects.filter(
Q(owner=request.user) | Q(primary=True)
| Q(shared=True)).order_by('-created_date'))
for dl_model in dl_model_list:
labels = []
if dl_model.labelmap_file and os.path.exists(
dl_model.labelmap_file.name):
with dl_model.labelmap_file.open('r') as f:
labels = list(json.load(f)["label_map"].values())
response["models"].append({
"id": dl_model.id,
"name": dl_model.name,
"primary": dl_model.primary,
"uploadDate": dl_model.created_date,
"updateDate": dl_model.updated_date,
"labels": labels,
"owner": dl_model.owner.id,
})
queue = django_rq.get_queue("low")
for tid in tids:
rq_id = "auto_annotation.run.{}".format(tid)
job = queue.fetch_job(rq_id)
if job is not None:
response["run"][tid] = {
"status": job.get_status(),
"rq_id": rq_id,
}
return JsonResponse(response)
except Exception as e:
return HttpResponseBadRequest(str(e))
def start_annotation(request, mid, tid):
slogger.glob.info(
"auto annotation create request for task {} via DL model {}".format(
tid, mid))
try:
db_task = TaskModel.objects.get(pk=tid)
queue = django_rq.get_queue("low")
job = queue.fetch_job("auto_annotation.run.{}".format(tid))
if job is not None and (job.is_started or job.is_queued):
raise Exception("The process is already running")
data = json.loads(request.body.decode('utf-8'))
should_reset = data["reset"]
user_defined_labels_mapping = data["labels"]
dl_model = AnnotationModel.objects.get(pk=mid)
model_file_path = dl_model.model_file.name
weights_file_path = dl_model.weights_file.name
labelmap_file = dl_model.labelmap_file.name
convertation_file_path = dl_model.interpretation_file.name
restricted = not has_admin_role(dl_model.owner)
db_labels = db_task.label_set.prefetch_related(
"attributespec_set").all()
db_attributes = {
db_label.id: {
db_attr.name: db_attr.id
for db_attr in db_label.attributespec_set.all()
}
for db_label in db_labels
}
db_labels = {db_label.name: db_label.id for db_label in db_labels}
model_labels = {
value: key
for key, value in load_labelmap(labelmap_file).items()
}
labels_mapping = {}
for user_model_label, user_db_label in user_defined_labels_mapping.items(
):
if user_model_label in model_labels and user_db_label in db_labels:
labels_mapping[int(
model_labels[user_model_label])] = db_labels[user_db_label]
if not labels_mapping:
raise Exception("No labels found for annotation")
rq_id = "auto_annotation.run.{}".format(tid)
queue.enqueue_call(func=model_manager.run_inference_thread,
args=(
tid,
model_file_path,
weights_file_path,
labels_mapping,
db_attributes,
convertation_file_path,
should_reset,
request.user,
restricted,
),
job_id=rq_id,
timeout=604800) # 7 days
slogger.task[tid].info("auto annotation job enqueued")
except Exception as ex:
try:
slogger.task[tid].exception(
"exception was occurred during annotation request",
exc_info=True)
except Exception as logger_ex:
slogger.glob.exception(
"exception was occurred during create auto annotation request for task {}: {}"
.format(tid, str(logger_ex)),
exc_info=True)
return HttpResponseBadRequest(str(ex))
return JsonResponse({"id": rq_id})
