def clean_record(cls, uuid, recid=None): """Clean record.""" logging.debug('Clean record {1}:{0}'.format(uuid, recid or 'deposit')) record_bucket = RecordsBuckets.query.filter( RecordsBuckets.record_id == uuid).one_or_none() PersistentIdentifier.query.filter( PersistentIdentifier.object_uuid == uuid).delete() RecordsBuckets.query.filter(RecordsBuckets.record_id == uuid).delete() RecordMetadata.query.filter(RecordMetadata.id == uuid).delete() if recid: RecordIdentifier.query.filter( RecordIdentifier.recid == recid).delete() files = [] if record_bucket: bucket = as_bucket(record_bucket.bucket_id) record_bucket.bucket.locked = False # Make files writable for obj in bucket.objects: files.append(obj.file.id) obj.file.writable = True db.session.add(obj.file) bucket.remove() db.session.commit() cls.clean_files(files)
def check_files(video): bucket = CDSRecordDumpLoader._get_bucket(record=video) files = [ dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) ] for file_ in files: assert as_bucket(file_['bucket_id']) is not None assert 'checksum' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert FileInstance.query.filter_by( id=file_['file_id']) is not None assert 'key' in file_ assert 'links' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert 'media_type' in file_ assert 'tags' in file_ # check extracted metadata master_video = CDSVideosFilesIterator.get_master_video_file(video) assert any([ key in master_video['tags'] for key in ExtractMetadataTask._all_keys ]) assert any([ key in video['_cds']['extracted_metadata'] for key in ExtractMetadataTask._all_keys ])
def _resolve_files(cls, deposit, record): """Create files.""" # build deposit files bucket = as_bucket(deposit['_buckets']['deposit']) # build objects/tags from marc21 metadata for file_ in record.get('_files', []): cls._resolve_file(deposit=deposit, bucket=bucket, file_=file_) # attach the master tag to the proper dependent files cls._resolve_master_tag(deposit=deposit) if Video.get_record_schema() == record['$schema']: # probe metadata from video cls._resolve_extracted_metadata(deposit=deposit, record=record) # update tag 'timestamp' cls._update_timestamp(deposit=deposit) # build a partial files dump cls._resolve_dumps(record=deposit) # snapshot them to record bucket snapshot = bucket.snapshot(lock=True) db.session.add(RecordsBuckets( record_id=record.id, bucket_id=snapshot.id )) if Video.get_record_schema() == record['$schema']: # create smil file cls._resolve_dumps(record=record) cls._resolve_smil(record=record) # update tag 'master' cls._update_tag_master(record=record)
def _create_gif(cls, bucket, frames, output_dir, master_id): """Generate a gif image.""" gif_filename = 'frames.gif' bucket = as_bucket(bucket) images = [] for f in frames: image = Image.open(file_opener_xrootd(f, 'rb')) # Convert image for better quality im = image.convert('RGB').convert( 'P', palette=Image.ADAPTIVE, colors=255 ) images.append(im) gif_image = create_gif_from_frames(images) gif_fullpath = os.path.join(output_dir, gif_filename) gif_image.save(gif_fullpath, save_all=True) cls._create_object(bucket=bucket, key=gif_filename, stream=open(gif_fullpath, 'rb'), size=os.path.getsize(gif_fullpath), media_type='image', context_type='frames-preview', master_id=master_id)
def _resolve_files(cls, deposit, record): """Create files.""" cls._clean_file_list(record) logging.info('Moving files from DFS.') bucket = as_bucket(deposit['_buckets']['deposit']) if Video.get_record_schema() == record['$schema']: has_master = cls._resolve_master_file(record, bucket) if has_master: cls._create_or_update_frames(record=record) # build objects/tags from marc21 metadata for file_ in record.get('_files', []): if file_['tags']['context_type'] != 'master': cls._resolve_file(bucket=bucket, file_=file_) # attach the master tag to the proper dependent files cls._resolve_master_tag(deposit=deposit) # probe metadata from video cls._resolve_extracted_metadata(deposit=deposit, record=record) # update tag 'timestamp' cls._update_timestamp(deposit=deposit) # build a partial files dump cls._resolve_dumps(record=deposit) # create gif cls._create_gif(video=deposit) cls._resolve_dumps(record=deposit) # snapshot them to record bucket snapshot = bucket.snapshot(lock=True) db.session.add(RecordsBuckets( record_id=record.id, bucket_id=snapshot.id )) cls._resolve_dumps(record=record) if Video.get_record_schema() == record['$schema'] and has_master: # update tag 'master' cls._update_tag_master(record=record) # create an empty smil file cls._resolve_smil(record=record) # create the full smil file cls._resolve_dumps(record=record) cls._resolve_smil(record=record)
def _bucket(self): """Get the bucket object.""" return as_bucket(self['_buckets']['deposit'])