def update(self, instance, validated_data):
credentials = Credentials()
credentials.convert_from_db({
'type': instance.credentials_type,
'value': instance.credentials,
})
tmp = {k:v for k,v in validated_data.items() if k in {'key', 'secret_key', 'account_name', 'session_token', 'key_file_path', 'credentials_type'}}
credentials.mapping_with_new_values(tmp)
instance.credentials = credentials.convert_to_db()
instance.credentials_type = validated_data.get('credentials_type', instance.credentials_type)
instance.resource = validated_data.get('resource', instance.resource)
instance.display_name = validated_data.get('display_name', instance.display_name)
instance.description = validated_data.get('description', instance.description)
instance.specific_attributes = validated_data.get('specific_attributes', instance.specific_attributes)
# check cloud storage existing
details = {
'resource': instance.resource,
'credentials': credentials,
'specific_attributes': parse_specific_attributes(instance.specific_attributes)
}
storage = get_cloud_storage_instance(cloud_provider=instance.provider_type, **details)
storage_status = storage.get_status()
if storage_status == Status.AVAILABLE:
new_manifest_names = set(i.get('filename') for i in validated_data.get('manifests', []))
previos_manifest_names = set(i.filename for i in instance.manifests.all())
delta_to_delete = tuple(previos_manifest_names - new_manifest_names)
delta_to_create = tuple(new_manifest_names - previos_manifest_names)
if delta_to_delete:
instance.manifests.filter(filename__in=delta_to_delete).delete()
if delta_to_create:
# check manifest files existing
for manifest in delta_to_create:
file_status = storage.get_file_status(manifest)
if file_status == Status.NOT_FOUND:
raise serializers.ValidationError({
'manifests': "The '{}' file does not exist on '{}' cloud storage"
.format(manifest, storage.name)
})
elif file_status == Status.FORBIDDEN:
raise serializers.ValidationError({
'manifests': "The '{}' file does not available on '{}' cloud storage. Access denied" \
.format(manifest.get('filename'), storage.name)
})
manifest_instances = [models.Manifest(filename=f, cloud_storage=instance) for f in delta_to_create]
models.Manifest.objects.bulk_create(manifest_instances)
instance.save()
return instance
elif storage_status == Status.FORBIDDEN:
field = 'credentials'
message = 'Cannot update resource {} with specified credentials. Access forbidden.'.format(storage.name)
else:
field = 'recource'
message = 'The resource {} not found. It may have been deleted.'.format(storage.name)
slogger.glob.error(message)
raise serializers.ValidationError({field: message})
def create(self, validated_data):
provider_type = validated_data.get('provider_type')
should_be_created = validated_data.pop('should_be_created', None)
credentials = Credentials(
account_name=validated_data.pop('account_name', ''),
key=validated_data.pop('key', ''),
secret_key=validated_data.pop('secret_key', ''),
session_token=validated_data.pop('session_token', ''),
key_file_path=validated_data.pop('key_file_path', ''),
credentials_type = validated_data.get('credentials_type')
)
details = {
'resource': validated_data.get('resource'),
'credentials': credentials,
'specific_attributes': parse_specific_attributes(validated_data.get('specific_attributes', ''))
}
storage = get_cloud_storage_instance(cloud_provider=provider_type, **details)
if should_be_created:
try:
storage.create()
except Exception as ex:
slogger.glob.warning("Failed with creating storage\n{}".format(str(ex)))
raise
storage_status = storage.get_status()
if storage_status == Status.AVAILABLE:
manifests = validated_data.pop('manifests')
# check manifest files availability
for manifest in manifests:
file_status = storage.get_file_status(manifest.get('filename'))
if file_status == Status.NOT_FOUND:
raise serializers.ValidationError({
'manifests': "The '{}' file does not exist on '{}' cloud storage" \
.format(manifest.get('filename'), storage.name)
})
elif file_status == Status.FORBIDDEN:
raise serializers.ValidationError({
'manifests': "The '{}' file does not available on '{}' cloud storage. Access denied" \
.format(manifest.get('filename'), storage.name)
})
db_storage = models.CloudStorage.objects.create(
credentials=credentials.convert_to_db(),
**validated_data
)
db_storage.save()
manifest_file_instances = [models.Manifest(**manifest, cloud_storage=db_storage) for manifest in manifests]
models.Manifest.objects.bulk_create(manifest_file_instances)
cloud_storage_path = db_storage.get_storage_dirname()
if os.path.isdir(cloud_storage_path):
shutil.rmtree(cloud_storage_path)
os.makedirs(db_storage.get_storage_logs_dirname(), exist_ok=True)
return db_storage
elif storage_status == Status.FORBIDDEN:
field = 'credentials'
message = 'Cannot create resource {} with specified credentials. Access forbidden.'.format(storage.name)
else:
field = 'recource'
message = 'The resource {} not found. It may have been deleted.'.format(storage.name)
slogger.glob.error(message)
raise serializers.ValidationError({field: message})
def prepare_chunk_buff(self, db_data, quality, chunk_number):
from cvat.apps.engine.frame_provider import FrameProvider # TODO: remove circular dependency
writer_classes = {
FrameProvider.Quality.COMPRESSED:
Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type
== DataChoice.VIDEO else ZipCompressedChunkWriter,
FrameProvider.Quality.ORIGINAL:
Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO
else ZipChunkWriter,
}
image_quality = 100 if writer_classes[quality] in [
Mpeg4ChunkWriter, ZipChunkWriter
] else db_data.image_quality
mime_type = 'video/mp4' if writer_classes[quality] in [
Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter
] else 'application/zip'
kwargs = {}
if self._dimension == DimensionType.DIM_3D:
kwargs["dimension"] = DimensionType.DIM_3D
writer = writer_classes[quality](image_quality, **kwargs)
images = []
buff = BytesIO()
upload_dir = {
StorageChoice.LOCAL: db_data.get_upload_dirname(),
StorageChoice.SHARE: settings.SHARE_ROOT,
StorageChoice.CLOUD_STORAGE: db_data.get_upload_dirname(),
}[db_data.storage]
if hasattr(db_data, 'video'):
source_path = os.path.join(upload_dir, db_data.video.path)
reader = VideoDatasetManifestReader(
manifest_path=db_data.get_manifest_path(),
source_path=source_path,
chunk_number=chunk_number,
chunk_size=db_data.chunk_size,
start=db_data.start_frame,
stop=db_data.stop_frame,
step=db_data.get_frame_step())
for frame in reader:
images.append((frame, source_path, None))
else:
reader = ImageDatasetManifestReader(
manifest_path=db_data.get_manifest_path(),
chunk_number=chunk_number,
chunk_size=db_data.chunk_size,
start=db_data.start_frame,
stop=db_data.stop_frame,
step=db_data.get_frame_step())
if db_data.storage == StorageChoice.CLOUD_STORAGE:
db_cloud_storage = db_data.cloud_storage
assert db_cloud_storage, 'Cloud storage instance was deleted'
credentials = Credentials()
credentials.convert_from_db({
'type':
db_cloud_storage.credentials_type,
'value':
db_cloud_storage.credentials,
})
details = {
'resource':
db_cloud_storage.resource,
'credentials':
credentials,
'specific_attributes':
db_cloud_storage.get_specific_attributes()
}
try:
cloud_storage_instance = get_cloud_storage_instance(
cloud_provider=db_cloud_storage.provider_type,
**details)
cloud_storage_instance.initialize_content()
for item in reader:
file_name = f"{item['name']}{item['extension']}"
if file_name not in cloud_storage_instance:
raise Exception(
'{} file was not found on a {} storage'.format(
file_name, cloud_storage_instance.name))
with NamedTemporaryFile(mode='w+b',
prefix='cvat',
suffix=file_name.replace(
os.path.sep, '#'),
delete=False) as temp_file:
source_path = temp_file.name
buf = cloud_storage_instance.download_fileobj(
file_name)
temp_file.write(buf.getvalue())
checksum = item.get('checksum', None)
if not checksum:
slogger.cloud_storage[
db_cloud_storage.id].warning(
'A manifest file does not contain checksum for image {}'
.format(item.get('name')))
if checksum and not md5_hash(
source_path) == checksum:
slogger.cloud_storage[
db_cloud_storage.id].warning(
'Hash sums of files {} do not match'.
format(file_name))
images.append((source_path, source_path, None))
except Exception as ex:
storage_status = cloud_storage_instance.get_status()
if storage_status == Status.FORBIDDEN:
msg = 'The resource {} is no longer available. Access forbidden.'.format(
cloud_storage_instance.name)
elif storage_status == Status.NOT_FOUND:
msg = 'The resource {} not found. It may have been deleted.'.format(
cloud_storage_instance.name)
else:
# check status of last file
file_status = cloud_storage_instance.get_file_status(
file_name)
if file_status == Status.NOT_FOUND:
raise Exception(
"'{}' not found on the cloud storage '{}'".
format(file_name, cloud_storage_instance.name))
elif file_status == Status.FORBIDDEN:
raise Exception(
"Access to the file '{}' on the '{}' cloud storage is denied"
.format(file_name,
cloud_storage_instance.name))
msg = str(ex)
raise Exception(msg)
else:
for item in reader:
source_path = os.path.join(
upload_dir, f"{item['name']}{item['extension']}")
images.append((source_path, source_path, None))
writer.save_as_chunk(images, buff)
buff.seek(0)
if db_data.storage == StorageChoice.CLOUD_STORAGE:
images = [image[0] for image in images if os.path.exists(image[0])]
for image_path in images:
os.remove(image_path)
return buff, mime_type