class VideoDatasetManifestReader(FragmentMediaReader):
def __init__(self, manifest_path, **kwargs):
self.source_path = kwargs.pop('source_path')
super().__init__(**kwargs)
self._manifest = VideoManifestManager(manifest_path)
self._manifest.init_index()
def _get_nearest_left_key_frame(self):
if self._start_chunk_frame_number >= \
self._manifest[len(self._manifest) - 1].get('number'):
left_border = len(self._manifest) - 1
else:
left_border = 0
delta = len(self._manifest)
while delta:
step = delta // 2
cur_position = left_border + step
if self._manifest[cur_position].get(
'number') < self._start_chunk_frame_number:
cur_position += 1
left_border = cur_position
delta -= step + 1
else:
delta = step
if self._manifest[cur_position].get(
'number') > self._start_chunk_frame_number:
left_border -= 1
frame_number = self._manifest[left_border].get('number')
timestamp = self._manifest[left_border].get('pts')
return frame_number, timestamp
def __iter__(self):
start_decode_frame_number, start_decode_timestamp = self._get_nearest_left_key_frame(
)
with closing(av.open(self.source_path, mode='r')) as container:
video_stream = next(stream for stream in container.streams
if stream.type == 'video')
video_stream.thread_type = 'AUTO'
container.seek(offset=start_decode_timestamp, stream=video_stream)
frame_number = start_decode_frame_number - 1
for packet in container.demux(video_stream):
for frame in packet.decode():
frame_number += 1
if frame_number in self._frame_range:
if video_stream.metadata.get('rotate'):
frame = av.VideoFrame().from_ndarray(
rotate_image(
frame.to_ndarray(format='bgr24'),
360 - int(container.streams.video[0].
metadata.get('rotate'))),
format='bgr24')
yield frame
elif frame_number < self._frame_range[-1]:
continue
else:
return
def migrate2meta(apps, shema_editor):
logger = get_logger(MIGRATION_NAME, MIGRATION_LOG)
query_set = _get_query_set(apps)
for db_data in query_set:
try:
upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT,
db_data.id)
logger.info('Migrate data({}), folder - {}'.format(
db_data.id, upload_dir))
meta_path = os.path.join(upload_dir, "meta_info.txt")
if os.path.exists(os.path.join(upload_dir, 'manifest.jsonl')):
os.remove(os.path.join(upload_dir, 'manifest.jsonl'))
logger.info('A manifest file has been deleted')
if os.path.exists(os.path.join(upload_dir, 'index.json')):
os.remove(os.path.join(upload_dir, 'index.json'))
logger.info('A manifest index file has been deleted')
data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT
if hasattr(db_data, 'video'):
if os.path.exists(meta_path):
logger.info('A meta_info.txt already exists')
continue
media_file = os.path.join(data_dir, db_data.video.path)
logger.info('Preparing of the video meta has begun')
manifest = VideoManifestManager(manifest_path=upload_dir)
manifest.link(media_file=media_file, force=True)
manifest.init_index()
with open(meta_path, "w") as meta_file:
for idx, pts, _ in manifest.reader:
meta_file.write(f"{idx} {pts}\n")
else:
name_format = "dummy_{}.txt"
sources = [
db_image.path
for db_image in db_data.images.all().order_by('frame')
]
counter = itertools.count()
logger.info('Preparing of the dummy chunks has begun')
for idx, img_paths in itertools.groupby(
sources,
lambda x: next(counter) // db_data.chunk_size):
if os.path.exists(
os.path.join(upload_dir, name_format.format(idx))):
logger.info(
name_format.format(idx) + " already exists")
continue
with open(
os.path.join(upload_dir, name_format.format(idx)),
"w") as dummy_chunk:
dummy_chunk.writelines(
[f"{img_path}\n" for img_path in img_paths])
logger.info('Succesfull migration for the data({})'.format(
db_data.id))
except Exception as ex:
logger.error(str(ex))
def migrate2manifest(apps, shema_editor):
logger = get_logger(MIGRATION_NAME, MIGRATION_LOG)
logger.info(
'The data migration has been started for creating manifest`s files')
query_set = _get_query_set(apps)
logger.info('Need to update {} data objects'.format(len(query_set)))
for db_data in query_set:
try:
upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT,
db_data.id)
logger.info('Migrate data({}), folder - {}'.format(
db_data.id, upload_dir))
if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')):
os.remove(os.path.join(upload_dir, 'meta_info.txt'))
logger.info(
'{}/meta_info.txt has been deleted'.format(upload_dir))
else:
for path in glob.glob(f'{upload_dir}/dummy_*.txt'):
os.remove(path)
logger.info(f"{path} has been deleted")
# it's necessary for case with long data migration
if os.path.exists(os.path.join(upload_dir, 'manifest.jsonl')):
logger.info('Manifest file already exists')
continue
data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT
if hasattr(db_data, 'video'):
media_file = os.path.join(data_dir, db_data.video.path)
manifest = VideoManifestManager(manifest_path=upload_dir)
logger.info(
'Preparing of the video meta information has begun')
meta_info = manifest.prepare_meta(media_file=media_file,
force=True)
logger.info('Manifest creating has begun')
manifest.create(meta_info)
logger.info('Index creating has begun')
manifest.init_index()
else:
manifest = ImageManifestManager(manifest_path=upload_dir)
sources = []
if db_data.storage == StorageChoice.LOCAL:
for (root, _, files) in os.walk(data_dir):
sources.extend([
os.path.join(root, f) for f in files
if get_mime(f) == 'image'
])
sources.sort()
# using share, this means that we can not explicitly restore the entire data structure
else:
sources = [
os.path.join(data_dir, db_image.path)
for db_image in db_data.images.all().order_by('frame')
]
if any(
list(
filter(
lambda x: x.dimension == DimensionType.DIM_3D,
db_data.tasks.all()))):
logger.info(
'Preparing of images 3d meta information has begun')
content = []
for source in sources:
name, ext = os.path.splitext(
os.path.relpath(source, upload_dir))
content.append({'name': name, 'extension': ext})
else:
logger.info(
'Preparing of 2d images meta information has begun')
meta_info = manifest.prepare_meta(sources=sources,
data_dir=data_dir)
content = meta_info.content
if db_data.storage == StorageChoice.SHARE:
def _get_frame_step(str_):
match = search("step\s*=\s*([1-9]\d*)", str_)
return int(match.group(1)) if match else 1
logger.info(
'Data is located on the share, metadata update has been started'
)
step = _get_frame_step(db_data.frame_filter)
start = db_data.start_frame
stop = db_data.stop_frame + 1
images_range = range(start, stop, step)
result_content = []
for i in range(stop):
item = content.pop(0) if i in images_range else dict()
result_content.append(item)
content = result_content
logger.info('Manifest creating has begun')
manifest.create(content)
logger.info('Index creating has begun')
manifest.init_index()
logger.info('Succesfull migration for the data({})'.format(
db_data.id))
except Exception as ex:
logger.error(str(ex))
def migrate_data(apps, shema_editor):
Data = apps.get_model("engine", "Data")
query_set = Data.objects.filter(storage_method=StorageMethodChoice.CACHE)
for db_data in query_set:
try:
upload_dir = '{}/{}/raw'.format(settings.MEDIA_DATA_ROOT,
db_data.id)
if os.path.exists(os.path.join(upload_dir, 'meta_info.txt')):
os.remove(os.path.join(upload_dir, 'meta_info.txt'))
else:
for path in glob.glob(f'{upload_dir}/dummy_*.txt'):
os.remove(path)
# it's necessary for case with long data migration
if os.path.exists(os.path.join(upload_dir, 'manifest.jsonl')):
continue
data_dir = upload_dir if db_data.storage == StorageChoice.LOCAL else settings.SHARE_ROOT
if hasattr(db_data, 'video'):
media_file = os.path.join(data_dir, db_data.video.path)
manifest = VideoManifestManager(manifest_path=upload_dir)
meta_info = manifest.prepare_meta(media_file=media_file)
manifest.create(meta_info)
manifest.init_index()
else:
manifest = ImageManifestManager(manifest_path=upload_dir)
sources = []
if db_data.storage == StorageChoice.LOCAL:
for (root, _, files) in os.walk(data_dir):
sources.extend([os.path.join(root, f) for f in files])
sources.sort()
# using share, this means that we can not explicitly restore the entire data structure
else:
sources = [
os.path.join(data_dir, db_image.path)
for db_image in db_data.images.all().order_by('frame')
]
if any(
list(
filter(
lambda x: x.dimension == DimensionType.DIM_3D,
db_data.tasks.all()))):
content = []
for source in sources:
name, ext = os.path.splitext(
os.path.relpath(source, upload_dir))
content.append({'name': name, 'extension': ext})
else:
meta_info = manifest.prepare_meta(sources=sources,
data_dir=data_dir)
content = meta_info.content
if db_data.storage == StorageChoice.SHARE:
def _get_frame_step(str_):
match = search("step\s*=\s*([1-9]\d*)", str_)
return int(match.group(1)) if match else 1
step = _get_frame_step(db_data.frame_filter)
start = db_data.start_frame
stop = db_data.stop_frame + 1
images_range = range(start, stop, step)
result_content = []
for i in range(stop):
item = content.pop(0) if i in images_range else dict()
result_content.append(item)
content = result_content
manifest.create(content)
manifest.init_index()
except Exception as ex:
print(str(ex))