def __init__(self, task_data, include_images=False):
super().__init__()
self._categories = self._load_categories(task_data)
dm_items = []
if include_images:
frame_provider = FrameProvider(task_data.db_task.data)
for frame_data in task_data.group_by_frame(include_empty=True):
loader = None
if include_images:
loader = lambda p, i=frame_data.idx: frame_provider.get_frame(
i,
quality=frame_provider.Quality.ORIGINAL,
out_type=frame_provider.Type.NUMPY_ARRAY)[0]
dm_image = Image(path=frame_data.name,
loader=loader,
size=(frame_data.height, frame_data.width))
dm_anno = self._read_cvat_anno(frame_data, task_data)
dm_item = datumaro.DatasetItem(
id=osp.splitext(frame_data.name)[0],
annotations=dm_anno,
image=dm_image,
attributes={'frame': frame_data.frame})
dm_items.append(dm_item)
self._items = dm_items
def _get_image(self, db_task, frame, quality):
if quality is None or quality == "original":
quality = FrameProvider.Quality.ORIGINAL
elif quality == "compressed":
quality = FrameProvider.Quality.COMPRESSED
else:
raise ValidationError(
'`{}` lambda function was run '.format(self.id) +
'with wrong arguments (quality={})'.format(quality),
code=status.HTTP_400_BAD_REQUEST)
frame_provider = FrameProvider(db_task.data)
image = frame_provider.get_frame(frame, quality=quality)
return base64.b64encode(image[0].getvalue()).decode('utf-8')
async def upload_images(cvat_project_id, training_id, api):
project = Project.objects.get(pk=cvat_project_id)
tasks: List[Task] = project.tasks.all()
for task in tasks:
frame_provider = FrameProvider(task)
frames = frame_provider.get_frames()
for i, (buffer, _) in enumerate(frames):
training_image_id = api.upload_image(training_id=training_id,
buffer=buffer)
if training_image_id:
TrainingProjectImage.objects.create(
project=project,
task=task,
idx=i,
training_image_id=training_image_id)
def create_thread(tid, labels_mapping, user):
try:
TRESHOLD = 0.5
# Init rq job
job = rq.get_current_job()
job.meta['progress'] = 0
job.save_meta()
# Get job indexes and segment length
db_task = TaskModel.objects.get(pk=tid)
# Get image list
image_list = FrameProvider(db_task.data)
# Run auto annotation by tf
result = None
slogger.glob.info("tf annotation with tensorflow framework for task {}".format(tid))
result = run_tensorflow_annotation(image_list, labels_mapping, TRESHOLD)
if result is None:
slogger.glob.info('tf annotation for task {} canceled by user'.format(tid))
return
# Modify data format and save
result = convert_to_cvat_format(result)
serializer = LabeledDataSerializer(data = result)
if serializer.is_valid(raise_exception=True):
put_task_data(tid, result)
slogger.glob.info('tf annotation for task {} done'.format(tid))
except Exception as ex:
try:
slogger.task[tid].exception('exception was occured during tf annotation of the task', exc_info=True)
except:
slogger.glob.exception('exception was occured during tf annotation of the task {}'.format(tid), exc_info=True)
raise ex
def dump_media_files(task_data: TaskData,
img_dir: str,
project_data: ProjectData = None):
ext = ''
if task_data.meta['task']['mode'] == 'interpolation':
ext = FrameProvider.VIDEO_FRAME_EXT
frame_provider = FrameProvider(task_data.db_task.data)
frames = frame_provider.get_frames(frame_provider.Quality.ORIGINAL,
frame_provider.Type.BUFFER)
for frame_id, (frame_data, _) in enumerate(frames):
frame_name = task_data.frame_info[frame_id]['path'] if project_data is None \
else project_data.frame_info[(task_data.db_task.id, frame_id)]['path']
img_path = osp.join(img_dir, frame_name + ext)
os.makedirs(osp.dirname(img_path), exist_ok=True)
with open(img_path, 'wb') as f:
f.write(frame_data.getvalue())
def upload_images_job(task_id: int):
if TrainingProjectImage.objects.filter(task_id=task_id).count() is 0:
task = Task.objects.get(pk=task_id)
frame_provider = FrameProvider(task.data)
frames = frame_provider.get_frames()
api = TrainingServerAPI(
host=task.project.training_project.host,
username=task.project.training_project.username,
password=task.project.training_project.password,
)
for i, (buffer, _) in enumerate(frames):
training_image_id = api.upload_image(
training_id=task.project.training_project.training_id,
buffer=buffer)
if training_image_id:
TrainingProjectImage.objects.create(
task=task, idx=i, training_image_id=training_image_id)
def _export(dst_file, task_data, anno_callback, save_images=False):
with TemporaryDirectory() as temp_dir:
with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f:
anno_callback(f, task_data)
if save_images:
img_dir = osp.join(temp_dir, 'images')
frame_provider = FrameProvider(task_data.db_task.data)
frames = frame_provider.get_frames(
frame_provider.Quality.ORIGINAL,
frame_provider.Type.NUMPY_ARRAY)
for frame_id, (frame_data, _) in enumerate(frames):
frame_name = task_data.frame_info[frame_id]['path']
if '.' in frame_name:
save_image(osp.join(img_dir, frame_name),
frame_data, jpeg_quality=100, create_dir=True)
else:
save_image(osp.join(img_dir, frame_name + '.png'),
frame_data, create_dir=True)
make_zip_archive(temp_dir, dst_file)
def _export(dst_file, task_data, anno_callback, save_images=False):
with TemporaryDirectory() as temp_dir:
with open(osp.join(temp_dir, 'annotations.xml'), 'wb') as f:
anno_callback(f, task_data)
if save_images:
ext = ''
if task_data.meta['task']['mode'] == 'interpolation':
ext = FrameProvider.VIDEO_FRAME_EXT
img_dir = osp.join(temp_dir, 'images')
frame_provider = FrameProvider(task_data.db_task.data)
frames = frame_provider.get_frames(frame_provider.Quality.ORIGINAL,
frame_provider.Type.BUFFER)
for frame_id, (frame_data, _) in enumerate(frames):
frame_name = task_data.frame_info[frame_id]['path']
img_path = osp.join(img_dir, frame_name + ext)
os.makedirs(osp.dirname(img_path), exist_ok=True)
with open(img_path, 'wb') as f:
f.write(frame_data.getvalue())
make_zip_archive(temp_dir, dst_file)
def _create(self):
self._project = Project.generate(self._project_dir)
self._project.add_source('task_%s' % self._db_task.id, {
'format': _TASK_IMAGES_EXTRACTOR,
})
self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR,
lambda url: CvatImagesExtractor(url,
FrameProvider(self._db_task.data)))
self._init_dataset()
self._dataset.define_categories(self._generate_categories())
self.save()
def run_inference_thread(tid, model_file, weights_file, labels_mapping, attributes, convertation_file, reset, user, restricted=True):
def update_progress(job, progress):
job.refresh()
if "cancel" in job.meta:
del job.meta["cancel"]
job.save()
return False
job.meta["progress"] = progress
job.save_meta()
return True
try:
job = rq.get_current_job()
job.meta["progress"] = 0
job.save_meta()
db_task = TaskModel.objects.get(pk=tid)
result = None
slogger.glob.info("auto annotation with openvino toolkit for task {}".format(tid))
result = run_inference_engine_annotation(
data=ImageLoader(FrameProvider(db_task.data)),
model_file=model_file,
weights_file=weights_file,
labels_mapping=labels_mapping,
attribute_spec=attributes,
convertation_file= convertation_file,
job=job,
update_progress=update_progress,
restricted=restricted
)
if result is None:
slogger.glob.info("auto annotation for task {} canceled by user".format(tid))
return
serializer = LabeledDataSerializer(data = result)
if serializer.is_valid(raise_exception=True):
if reset:
put_task_data(tid, user, result)
else:
patch_task_data(tid, user, result, "create")
slogger.glob.info("auto annotation for task {} done".format(tid))
except Exception as e:
try:
slogger.task[tid].exception("exception was occurred during auto annotation of the task", exc_info=True)
except Exception as ex:
slogger.glob.exception("exception was occurred during auto annotation of the task {}: {}".format(tid, str(ex)), exc_info=True)
raise ex
raise e
def __init__(self, task_data, include_images=False):
super().__init__()
self._categories = self._load_categories(task_data)
dm_items = []
is_video = task_data.meta['task']['mode'] == 'interpolation'
ext = ''
if is_video:
ext = FrameProvider.VIDEO_FRAME_EXT
if include_images:
frame_provider = FrameProvider(task_data.db_task.data)
if is_video:
# optimization for videos: use numpy arrays instead of bytes
# some formats or transforms can require image data
def _make_image(i, **kwargs):
loader = lambda _: frame_provider.get_frame(
i,
quality=frame_provider.Quality.ORIGINAL,
out_type=frame_provider.Type.NUMPY_ARRAY)[0]
return Image(loader=loader, **kwargs)
else:
# for images use encoded data to avoid recoding
def _make_image(i, **kwargs):
loader = lambda _: frame_provider.get_frame(
i,
quality=frame_provider.Quality.ORIGINAL,
out_type=frame_provider.Type.BUFFER)[0].getvalue()
return ByteImage(data=loader, **kwargs)
for frame_data in task_data.group_by_frame(include_empty=True):
image_args = {
'path': frame_data.name + ext,
'size': (frame_data.height, frame_data.width),
}
if include_images:
dm_image = _make_image(frame_data.idx, **image_args)
else:
dm_image = Image(**image_args)
dm_anno = self._read_cvat_anno(frame_data, task_data)
dm_item = datumaro.DatasetItem(
id=osp.splitext(frame_data.name)[0],
annotations=dm_anno,
image=dm_image,
attributes={'frame': frame_data.frame})
dm_items.append(dm_item)
self._items = dm_items
def __init__(self, jid, data):
self.__threshold = data["threshold"]
self.__max_distance = data["maxDistance"]
self.__frame_boxes = {}
db_job = Job.objects.select_related('segment__task').get(pk=jid)
db_segment = db_job.segment
db_task = db_segment.task
self.__frame_iter = itertools.islice(
FrameProvider(db_task.data).get_frames(
FrameProvider.Quality.ORIGINAL),
db_segment.start_frame,
db_segment.stop_frame + 1,
)
self.__stop_frame = db_segment.stop_frame
for frame in range(db_segment.start_frame, db_segment.stop_frame + 1):
self.__frame_boxes[frame] = [
box for box in data["boxes"] if box["frame"] == frame
]
IE_PLUGINS_PATH = os.getenv('IE_PLUGINS_PATH', None)
REID_MODEL_DIR = os.getenv('REID_MODEL_DIR', None)
if not IE_PLUGINS_PATH:
raise Exception(
"Environment variable 'IE_PLUGINS_PATH' isn't defined")
if not REID_MODEL_DIR:
raise Exception(
"Environment variable 'REID_MODEL_DIR' isn't defined")
REID_XML = os.path.join(REID_MODEL_DIR, "reid.xml")
REID_BIN = os.path.join(REID_MODEL_DIR, "reid.bin")
self.__plugin = IEPlugin(device="CPU", plugin_dirs=[IE_PLUGINS_PATH])
network = IENetwork.from_ir(model=REID_XML, weights=REID_BIN)
self.__input_blob_name = next(iter(network.inputs))
self.__output_blob_name = next(iter(network.outputs))
self.__input_height, self.__input_width = network.inputs[
self.__input_blob_name].shape[-2:]
self.__executable_network = self.__plugin.load(network=network)
del network
def _import_from_task(self, user):
self._project = Project.generate(self._project_dir, config={
'project_name': self._db_task.name,
'plugins_dir': _FORMATS_DIR,
})
self._project.add_source('task_%s_images' % self._db_task.id, {
'format': _TASK_IMAGES_EXTRACTOR,
})
self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR,
lambda url: CvatImagesExtractor(url,
FrameProvider(self._db_task.data)))
self._project.add_source('task_%s_anno' % self._db_task.id, {
'format': _TASK_ANNO_EXTRACTOR,
})
self._project.env.extractors.register(_TASK_ANNO_EXTRACTOR,
lambda url: CvatTaskExtractor(url,
db_task=self._db_task, user=user))
self._init_dataset()
def data(self, request, pk):
if request.method == 'POST':
db_task = self.get_object(
) # call check_object_permissions as well
serializer = DataSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
db_data = serializer.save()
db_task.data = db_data
db_task.save()
data = {k: v for k, v in serializer.data.items()}
data['use_zip_chunks'] = serializer.validated_data[
'use_zip_chunks']
data['use_cache'] = serializer.validated_data['use_cache']
if data['use_cache']:
db_task.data.storage_method = StorageMethodChoice.CACHE
db_task.data.save(update_fields=['storage_method'])
# if the value of stop_frame is 0, then inside the function we cannot know
# the value specified by the user or it's default value from the database
if 'stop_frame' not in serializer.validated_data:
data['stop_frame'] = None
task.create(db_task.id, data)
return Response(serializer.data, status=status.HTTP_202_ACCEPTED)
else:
data_type = request.query_params.get('type', None)
data_id = request.query_params.get('number', None)
data_quality = request.query_params.get('quality', 'compressed')
possible_data_type_values = ('chunk', 'frame', 'preview')
possible_quality_values = ('compressed', 'original')
if not data_type or data_type not in possible_data_type_values:
return Response(
data='data type not specified or has wrong value',
status=status.HTTP_400_BAD_REQUEST)
elif data_type == 'chunk' or data_type == 'frame':
if not data_id:
return Response(data='number not specified',
status=status.HTTP_400_BAD_REQUEST)
elif data_quality not in possible_quality_values:
return Response(data='wrong quality value',
status=status.HTTP_400_BAD_REQUEST)
try:
db_task = self.get_object()
db_data = db_task.data
frame_provider = FrameProvider(db_task.data)
if data_type == 'chunk':
data_id = int(data_id)
data_quality = FrameProvider.Quality.COMPRESSED \
if data_quality == 'compressed' else FrameProvider.Quality.ORIGINAL
#TODO: av.FFmpegError processing
if settings.USE_CACHE and db_data.storage_method == StorageMethodChoice.CACHE:
buff, mime_type = frame_provider.get_chunk(
data_id, data_quality)
return HttpResponse(buff.getvalue(),
content_type=mime_type)
# Follow symbol links if the chunk is a link on a real image otherwise
# mimetype detection inside sendfile will work incorrectly.
path = os.path.realpath(
frame_provider.get_chunk(data_id, data_quality))
return sendfile(request, path)
elif data_type == 'frame':
data_id = int(data_id)
data_quality = FrameProvider.Quality.COMPRESSED \
if data_quality == 'compressed' else FrameProvider.Quality.ORIGINAL
buf, mime = frame_provider.get_frame(data_id, data_quality)
return HttpResponse(buf.getvalue(), content_type=mime)
elif data_type == 'preview':
return sendfile(request, frame_provider.get_preview())
else:
return Response(
data='unknown data type {}.'.format(data_type),
status=status.HTTP_400_BAD_REQUEST)
except APIException as e:
return Response(data=e.default_detail, status=e.status_code)
except Exception as e:
msg = 'cannot get requested data type: {}, number: {}, quality: {}'.format(
data_type, data_id, data_quality)
slogger.task[pk].error(msg, exc_info=True)
return Response(data=msg + '\n' + str(e),
status=status.HTTP_400_BAD_REQUEST)
def _load(self):
self._project = Project.load(self._project_dir)
self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR,
lambda url: CvatImagesExtractor(url,
FrameProvider(self._db_task.data)))
def handle(self, db_data, frame, points):
# Lazy initialization
if not self._plugin:
self._plugin = make_plugin_or_core()
self._network = make_network(
os.path.join(_DEXTR_MODEL_DIR, 'dextr.xml'),
os.path.join(_DEXTR_MODEL_DIR, 'dextr.bin'))
self._input_blob = next(iter(self._network.inputs))
self._output_blob = next(iter(self._network.outputs))
if getattr(self._plugin, 'load_network', False):
self._exec_network = self._plugin.load_network(
self._network, 'CPU')
else:
self._exec_network = self._plugin.load(network=self._network)
frame_provider = FrameProvider(db_data)
image = frame_provider.get_frame(frame,
frame_provider.Quality.ORIGINAL)
image = PIL.Image.open(image[0])
numpy_image = np.array(image)
points = np.asarray([[int(p["x"]), int(p["y"])] for p in points],
dtype=int)
bounding_box = (max(min(points[:, 0]) - _DEXTR_PADDING,
0), max(min(points[:, 1]) - _DEXTR_PADDING, 0),
min(
max(points[:, 0]) + _DEXTR_PADDING,
numpy_image.shape[1] - 1),
min(
max(points[:, 1]) + _DEXTR_PADDING,
numpy_image.shape[0] - 1))
# Prepare an image
numpy_cropped = np.array(image.crop(bounding_box))
resized = cv2.resize(numpy_cropped, (_DEXTR_SIZE, _DEXTR_SIZE),
interpolation=cv2.INTER_CUBIC).astype(np.float32)
# Make a heatmap
points = points - [min(points[:, 0]),
min(points[:, 1])
] + [_DEXTR_PADDING, _DEXTR_PADDING]
points = (points * [
_DEXTR_SIZE / numpy_cropped.shape[1],
_DEXTR_SIZE / numpy_cropped.shape[0]
]).astype(int)
heatmap = np.zeros(shape=resized.shape[:2], dtype=np.float64)
for point in points:
gaussian_x_axis = np.arange(0, _DEXTR_SIZE, 1, float) - point[0]
gaussian_y_axis = np.arange(0, _DEXTR_SIZE, 1,
float)[:, np.newaxis] - point[1]
gaussian = np.exp(
-4 * np.log(2) *
((gaussian_x_axis**2 + gaussian_y_axis**2) / 100)).astype(
np.float64)
heatmap = np.maximum(heatmap, gaussian)
cv2.normalize(heatmap, heatmap, 0, 255, cv2.NORM_MINMAX)
# Concat an image and a heatmap
input_dextr = np.concatenate(
(resized, heatmap[:, :, np.newaxis].astype(resized.dtype)), axis=2)
input_dextr = input_dextr.transpose((2, 0, 1))
pred = self._exec_network.infer(
inputs={self._input_blob: input_dextr[np.newaxis, ...]})[
self._output_blob][0, 0, :, :]
pred = cv2.resize(pred,
tuple(reversed(numpy_cropped.shape[:2])),
interpolation=cv2.INTER_CUBIC)
result = np.zeros(numpy_image.shape[:2])
result[bounding_box[1]:bounding_box[1] + pred.shape[0],
bounding_box[0]:bounding_box[0] +
pred.shape[1]] = pred > _DEXTR_TRESHOLD
# Convert a mask to a polygon
result = np.array(result, dtype=np.uint8)
cv2.normalize(result, result, 0, 255, cv2.NORM_MINMAX)
contours = None
if int(cv2.__version__.split('.')[0]) > 3:
contours = cv2.findContours(result, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_TC89_KCOS)[0]
else:
contours = cv2.findContours(result, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_TC89_KCOS)[1]
contours = max(contours, key=lambda arr: arr.size)
if contours.shape.count(1):
contours = np.squeeze(contours)
if contours.size < 3 * 2:
raise Exception(
'Less then three point have been detected. Can not build a polygon.'
)
result = ""
for point in contours:
result += "{},{} ".format(int(point[0]), int(point[1]))
result = result[:-1]
return result
