def clean_record(cls, uuid, recid=None):
"""Clean record."""
logging.debug('Clean record {1}:{0}'.format(uuid, recid or 'deposit'))
record_bucket = RecordsBuckets.query.filter(
RecordsBuckets.record_id == uuid).one_or_none()
PersistentIdentifier.query.filter(
PersistentIdentifier.object_uuid == uuid).delete()
RecordsBuckets.query.filter(RecordsBuckets.record_id == uuid).delete()
RecordMetadata.query.filter(RecordMetadata.id == uuid).delete()
if recid:
RecordIdentifier.query.filter(
RecordIdentifier.recid == recid).delete()
files = []
if record_bucket:
bucket = as_bucket(record_bucket.bucket_id)
record_bucket.bucket.locked = False
# Make files writable
for obj in bucket.objects:
files.append(obj.file.id)
obj.file.writable = True
db.session.add(obj.file)
bucket.remove()
db.session.commit()
cls.clean_files(files)
def check_files(video):
bucket = CDSRecordDumpLoader._get_bucket(record=video)
files = [
dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)
]
for file_ in files:
assert as_bucket(file_['bucket_id']) is not None
assert 'checksum' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert FileInstance.query.filter_by(
id=file_['file_id']) is not None
assert 'key' in file_
assert 'links' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert 'media_type' in file_
assert 'tags' in file_
# check extracted metadata
master_video = CDSVideosFilesIterator.get_master_video_file(video)
assert any([
key in master_video['tags']
for key in ExtractMetadataTask._all_keys
])
assert any([
key in video['_cds']['extracted_metadata']
for key in ExtractMetadataTask._all_keys
])
def _resolve_files(cls, deposit, record):
"""Create files."""
# build deposit files
bucket = as_bucket(deposit['_buckets']['deposit'])
# build objects/tags from marc21 metadata
for file_ in record.get('_files', []):
cls._resolve_file(deposit=deposit, bucket=bucket, file_=file_)
# attach the master tag to the proper dependent files
cls._resolve_master_tag(deposit=deposit)
if Video.get_record_schema() == record['$schema']:
# probe metadata from video
cls._resolve_extracted_metadata(deposit=deposit, record=record)
# update tag 'timestamp'
cls._update_timestamp(deposit=deposit)
# build a partial files dump
cls._resolve_dumps(record=deposit)
# snapshot them to record bucket
snapshot = bucket.snapshot(lock=True)
db.session.add(RecordsBuckets(
record_id=record.id, bucket_id=snapshot.id
))
if Video.get_record_schema() == record['$schema']:
# create smil file
cls._resolve_dumps(record=record)
cls._resolve_smil(record=record)
# update tag 'master'
cls._update_tag_master(record=record)
def _create_gif(cls, bucket, frames, output_dir, master_id):
"""Generate a gif image."""
gif_filename = 'frames.gif'
bucket = as_bucket(bucket)
images = []
for f in frames:
image = Image.open(file_opener_xrootd(f, 'rb'))
# Convert image for better quality
im = image.convert('RGB').convert(
'P', palette=Image.ADAPTIVE, colors=255
)
images.append(im)
gif_image = create_gif_from_frames(images)
gif_fullpath = os.path.join(output_dir, gif_filename)
gif_image.save(gif_fullpath, save_all=True)
cls._create_object(bucket=bucket, key=gif_filename,
stream=open(gif_fullpath, 'rb'),
size=os.path.getsize(gif_fullpath),
media_type='image', context_type='frames-preview',
master_id=master_id)
def _resolve_files(cls, deposit, record):
"""Create files."""
cls._clean_file_list(record)
logging.info('Moving files from DFS.')
bucket = as_bucket(deposit['_buckets']['deposit'])
if Video.get_record_schema() == record['$schema']:
has_master = cls._resolve_master_file(record, bucket)
if has_master:
cls._create_or_update_frames(record=record)
# build objects/tags from marc21 metadata
for file_ in record.get('_files', []):
if file_['tags']['context_type'] != 'master':
cls._resolve_file(bucket=bucket, file_=file_)
# attach the master tag to the proper dependent files
cls._resolve_master_tag(deposit=deposit)
# probe metadata from video
cls._resolve_extracted_metadata(deposit=deposit, record=record)
# update tag 'timestamp'
cls._update_timestamp(deposit=deposit)
# build a partial files dump
cls._resolve_dumps(record=deposit)
# create gif
cls._create_gif(video=deposit)
cls._resolve_dumps(record=deposit)
# snapshot them to record bucket
snapshot = bucket.snapshot(lock=True)
db.session.add(RecordsBuckets(
record_id=record.id, bucket_id=snapshot.id
))
cls._resolve_dumps(record=record)
if Video.get_record_schema() == record['$schema'] and has_master:
# update tag 'master'
cls._update_tag_master(record=record)
# create an empty smil file
cls._resolve_smil(record=record)
# create the full smil file
cls._resolve_dumps(record=record)
cls._resolve_smil(record=record)
def _bucket(self):
"""Get the bucket object."""
return as_bucket(self['_buckets']['deposit'])