def test_retry_run_extracted_metadata(app): """Test retry is working properly.""" with mock.patch.object( ExtractMetadataTask, 'create_metadata_tags', side_effect=Exception): with pytest.raises(Exception): CDSRecordDumpLoader._run_extracted_metadata(master={}, retry=1)
def get(key, record): bucket = CDSRecordDumpLoader._get_bucket(record=record) files = [ dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) ] return [file_[key] for file_ in files]
def check_files(video): bucket = CDSRecordDumpLoader._get_bucket(record=video) files = [ dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) ] for file_ in files: assert as_bucket(file_['bucket_id']) is not None assert 'checksum' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert FileInstance.query.filter_by( id=file_['file_id']) is not None assert 'key' in file_ assert 'links' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert 'media_type' in file_ assert 'tags' in file_ # check extracted metadata master_video = CDSVideosFilesIterator.get_master_video_file(video) assert any([ key in master_video['tags'] for key in ExtractMetadataTask._all_keys ]) assert any([ key in video['_cds']['extracted_metadata'] for key in ExtractMetadataTask._all_keys ])
def check_tag_master(record): bucket = CDSRecordDumpLoader._get_bucket(record=record) master = CDSVideosFilesIterator.get_master_video_file(record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) if obj.get_tags().get('master')] assert all([file_['tags']['master'] == master['version_id'] for file_ in files])
def test_mingrate_pids(app, location, datadir): """Test migrate pids.""" data = load_json(datadir, 'cds_records_demo_1_project.json') dump = CDSRecordDump(data=data[0]) record = CDSRecordDumpLoader.create(dump=dump) pids = [pid.pid_value for pid in PersistentIdentifier.query.filter_by(object_uuid=record.id)] expected = sorted(['2093596', 'CERN-MOVIE-2012-193']) assert sorted(pids) == expected
def test_subformat_creation_if_missing(api_app, location, datadir, es, users): """Test subformat creation if missing.""" # [[ migrate the video ]] migration_streams = get_migration_streams(datadir=datadir) data = load_json(datadir, 'cds_records_demo_1_video.json') dump = CDSRecordDump(data=data[0]) with mock.patch.object(DataCiteProvider, 'register'), \ mock.patch.object(CDSRecordDumpLoader, '_create_frame', side_effect=get_frames), \ mock.patch.object(ExtractFramesTask, '_create_gif'), \ mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'), \ mock.patch.object( CDSRecordDumpLoader, '_get_migration_file_stream_and_size', side_effect=migration_streams): video = CDSRecordDumpLoader.create(dump=dump) db.session.commit() with mock.patch.object(TranscodeVideoTask, 'run') as mock_transcode: deposit = deposit_video_resolver(video['_deposit']['id']) deposit_id = deposit.id # simulate the missing of a subformat del deposit['_files'][0]['subformat'][0] assert len(deposit['_files'][0]['subformat']) == 4 # recreate 240p format CDSRecordDumpLoader._create_missing_subformats( record=video, deposit=deposit) db.session.commit() # check subformats deposit = Video.get_record(deposit_id) rec_video = record_resolver.resolve(video['recid'])[1] # rec_video = record_resolver.resolve(video['recid'])[1] assert len(deposit['_files'][0]['subformat']) == 5 assert len(rec_video['_files'][0]['subformat']) == 5 # check if transcoding is called properly assert mock_transcode.called is True [(_, call_args)] = mock_transcode.call_args_list assert call_args == {'preset_quality': '240p'}
def test_migrate_record(frames_required, api_app, location, datadir, es, users): """Test migrate date.""" # [[ migrate the project ]] data = load_json(datadir, 'cds_records_demo_1_project.json') dump = CDSRecordDump(data=data[0]) project = CDSRecordDumpLoader.create(dump=dump) p_id = project.id assert project['$schema'] == Project.get_record_schema() assert project['publication_date'] == '2016-01-05' assert 'license' not in project assert 'copyright' not in project assert project['_cds'] == { "state": { "file_transcode": "SUCCESS", "file_video_extract_frames": "SUCCESS", "file_video_metadata_extraction": "SUCCESS" }, 'modified_by': users[0], } # check project deposit deposit_project_uuid = PersistentIdentifier.query.filter_by( pid_type='depid', object_type='rec').one().object_uuid deposit_project = Record.get_record(deposit_project_uuid) assert Project._schema in deposit_project['$schema'] assert project.revision_id == deposit_project[ '_deposit']['pid']['revision_id'] assert deposit_project['_deposit']['created_by'] == 1 assert deposit_project['_deposit']['owners'] == [1] assert deposit_project['_files'] == [] # [[ migrate the video ]] data = load_json(datadir, 'cds_records_demo_1_video.json') dump = CDSRecordDump(data=data[0]) db.session.commit() def check_symlinks(video): symlinks_creator = SymlinksCreator() files = list(symlinks_creator._get_list_files(record=video)) assert len(files) == 1 for file_ in files: path = symlinks_creator._build_link_path( symlinks_creator._symlinks_location, video, file_['key']) assert os.path.lexists(path) def check_gif(video, mock_gif): # called only once for deposit (_, _, mock_args) = mock_gif.mock_calls[0] # check gif record video = CDSRecord(dict(video), video.model) # check gif deposit deposit = deposit_video_resolver(video['_deposit']['id']) master_video = CDSVideosFilesIterator.get_master_video_file(deposit) assert mock_args['master_id'] == master_video['version_id'] assert str(deposit.files.bucket.id) == mock_args['bucket'] # assert mock_args['bucket'].id == deposit.files.bucket.id assert len(mock_args['frames']) == 10 assert 'output_dir' in mock_args migration_streams = get_migration_streams(datadir=datadir) with mock.patch.object(DataCiteProvider, 'register'), \ mock.patch.object(CDSRecordDumpLoader, '_create_frame', side_effect=get_frames), \ mock.patch.object(CDSRecordDumpLoader, '_get_minimum_frames', return_value=frames_required) as mock_frames, \ mock.patch.object( ExtractFramesTask, '_create_gif') as mock_gif, \ mock.patch.object( CDSRecordDumpLoader, '_get_migration_file_stream_and_size', side_effect=migration_streams), \ mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'): video = CDSRecordDumpLoader.create(dump=dump) assert mock_frames.called is True db.session.add(video.model) video_id = video.id # check smil file smil_obj = ObjectVersion.query.filter_by( key='CERN-MOVIE-2012-193-001.smil', is_head=True).one() storage = smil_obj.file.storage() assert '<video src' in storage.open().read().decode('utf-8') # check video symlinks check_symlinks(video) # check gif check_gif(video, mock_gif) # check project project = Record.get_record(p_id) assert project['videos'] == [ {'$ref': 'https://cds.cern.ch/api/record/1495143'} ] assert video['$schema'] == Video.get_record_schema() assert video['date'] == '2012-11-21' # metadata data assert video['publication_date'] == '2017-07-13' # creation date (DB) assert video['_project_id'] == '2093596' assert video['license'] == [{ 'license': 'CERN', 'url': 'http://copyright.web.cern.ch', }] assert video['copyright'] == { 'holder': 'CERN', 'year': '2012', 'url': 'http://copyright.web.cern.ch', } assert video['description'] == '' assert 'doi' in video assert video['_cds']['state'] == { "file_transcode": "SUCCESS", "file_video_extract_frames": "SUCCESS", "file_video_metadata_extraction": "SUCCESS" } assert 'extracted_metadata' in video['_cds'] def check_files(video): bucket = CDSRecordDumpLoader._get_bucket(record=video) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket)] for file_ in files: assert as_bucket(file_['bucket_id']) is not None assert 'checksum' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert FileInstance.query.filter_by( id=file_['file_id']) is not None assert 'key' in file_ assert 'links' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert 'media_type' in file_ assert 'tags' in file_ # check extracted metadata master_video = CDSVideosFilesIterator.get_master_video_file(video) assert any([key in master_video['tags'] for key in ExtractMetadataTask._all_keys]) assert any([key in video['_cds']['extracted_metadata'] for key in ExtractMetadataTask._all_keys]) def check_buckets(record, deposit): def get(key, record): bucket = CDSRecordDumpLoader._get_bucket(record=record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket)] return [file_[key] for file_ in files] def check(record, deposit, file_key, different=None): values_record = set(get(file_key, record)) values_deposit = set(get(file_key, deposit)) difference = len(values_record - values_deposit) assert different == difference def check_tag_master(record): bucket = CDSRecordDumpLoader._get_bucket(record=record) master = CDSVideosFilesIterator.get_master_video_file(record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) if obj.get_tags().get('master')] assert all([file_['tags']['master'] == master['version_id'] for file_ in files]) # 1 bucket record != 1 bucket deposit check(record, deposit, 'bucket_id', 1) # all file_id are the same except the smil file (only in record) check(record, deposit, 'file_id', 1) check(record, deposit, 'key', 1) # 18 object_version record != 17 object_version deposit check(record, deposit, 'version_id', 18) # check tag 'master' where is pointing check_tag_master(record) check_tag_master(deposit) def check_first_level_files(record): [master] = [file_ for file_ in deposit_video['_files'] if file_['context_type'] == 'master'] assert len(master['subformat']) == 5 assert len(master['frame']) == 10 # TODO assert len(master['playlist']) == ?? assert len([file_ for file_ in deposit_video['_files'] if file_['context_type'] == 'master']) == 1 duration = float(record['_cds']['extracted_metadata']['duration']) for frame in master['frame']: assert float(frame['tags']['timestamp']) < duration assert float(frame['tags']['timestamp']) > 0 # check tag 'preset_quality' pqs = [form['tags']['preset_quality'] for form in master['subformat']] assert sorted(pqs) == sorted(['1080p', '240p', '360p', '480p', '720p']) # check tag 'display_aspect_ratio' dar = set([form['tags']['display_aspect_ratio'] for form in master['subformat']]) assert dar == {'16:9'} def check_pids(record): """Check pids.""" assert record['report_number'][0] == 'CERN-VIDEO-2012-193-001' assert PersistentIdentifier.query.filter_by( pid_value='CERN-VIDEO-2012-193-001').count() == 1 assert PersistentIdentifier.query.filter_by( pid_value='CERN-MOVIE-2012-193-001').count() == 1 db.session.commit() # check video deposit deposit_video_uuid = PersistentIdentifier.query.filter( PersistentIdentifier.pid_type == 'depid', PersistentIdentifier.object_uuid != str(deposit_project_uuid), PersistentIdentifier.object_type == 'rec' ).one().object_uuid deposit_video = Video.get_record(str(deposit_video_uuid)) assert Video._schema in deposit_video['$schema'] video = Record.get_record(video_id) assert video.revision_id == deposit_video[ '_deposit']['pid']['revision_id'] assert deposit_video['_deposit']['created_by'] == users[0] assert deposit_video['_deposit']['owners'] == [users[0]] assert deposit_video['_project_id'] == '2093596' assert len(video['_files']) == 2 assert len(deposit_video['_files']) == 2 check_files(video) check_files(deposit_video) check_buckets(video, deposit_video) check_first_level_files(video) check_first_level_files(deposit_video) check_pids(video) # try to edit video deposit_video = deposit_video_resolver(deposit_video['_deposit']['id']) deposit_video = deposit_video.edit() # try to edit project deposit_project = deposit_project_resolver( deposit_project['_deposit']['id']) deposit_project = deposit_project.edit() login_user(User.query.filter_by(id=users[0]).first()) deposit_video['title']['title'] = 'test' deposit_video = deposit_video.publish() _, record_video = deposit_video.fetch_published() assert record_video['title']['title'] == 'test'
def test_migrate_record(app, location, datadir, es): """Test migrate date.""" # create the project data = load_json(datadir, 'cds_records_demo_1_project.json') dump = CDSRecordDump(data=data[0]) project = CDSRecordDumpLoader.create(dump=dump) p_id = project.id date = '2015-11-13' assert project['$schema'] == Project.get_record_schema() assert project['date'] == date assert project['publication_date'] == date assert 'license' not in project assert 'copyright' not in project assert project['_cds'] == { "state": { "file_transcode": "SUCCESS", "file_video_extract_frames": "SUCCESS", "file_video_metadata_extraction": "SUCCESS" }, 'modified_by': None, } # check project deposit deposit_project_uuid = PersistentIdentifier.query.filter_by( pid_type='depid', object_type='rec').one().object_uuid deposit_project = Record.get_record(deposit_project_uuid) assert Project._schema in deposit_project['$schema'] assert project.revision_id == deposit_project[ '_deposit']['pid']['revision_id'] assert deposit_project['_deposit']['created_by'] == -1 assert deposit_project['_deposit']['owners'] == [-1] assert deposit_project['_files'] == [] # create the video data = load_json(datadir, 'cds_records_demo_1_video.json') dump = CDSRecordDump(data=data[0]) def load_video(*args, **kwargs): return open(join(datadir, 'test.mp4'), 'rb') with mock.patch.object(DataCiteProvider, 'register') as mock_datacite, \ mock.patch.object( CDSRecordDumpLoader, '_get_migration_file_stream', return_value=load_video()): video = CDSRecordDumpLoader.create(dump=dump) # assert mock_datacite.called is True project = Record.get_record(p_id) assert project['videos'] == [ {'$ref': 'https://cds.cern.ch/api/record/1495143'} ] assert video['$schema'] == Video.get_record_schema() date = '2012-11-20' assert video['date'] == date assert video['publication_date'] == date assert video['_project_id'] == '2093596' assert video['license'] == [{ 'license': 'CERN', 'url': 'http://copyright.web.cern.ch', }] assert video['copyright'] == { 'holder': 'CERN', 'year': '2012', 'url': 'http://copyright.web.cern.ch', } assert video['description'] == '' assert 'doi' in video assert video['_cds']['state'] == { "file_transcode": "SUCCESS", "file_video_extract_frames": "SUCCESS", "file_video_metadata_extraction": "SUCCESS" } assert 'extracted_metadata' in video['_cds'] def check_files(video): bucket = CDSRecordDumpLoader._get_bucket(record=video) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket)] for file_ in files: assert as_bucket(file_['bucket_id']) is not None assert 'checksum' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert FileInstance.query.filter_by( id=file_['file_id']) is not None assert 'key' in file_ assert 'links' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert 'media_type' in file_ assert 'tags' in file_ # check extracted metadata master_video = CDSVideosFilesIterator.get_master_video_file(video) assert any([key in master_video['tags'] for key in ExtractMetadataTask._all_keys]) assert any([key in video['_cds']['extracted_metadata'] for key in ExtractMetadataTask._all_keys]) def check_buckets(record, deposit): def get(key, record): bucket = CDSRecordDumpLoader._get_bucket(record=record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket)] return [file_[key] for file_ in files] def check(record, deposit, file_key, different=None): values_record = set(get(file_key, record)) values_deposit = set(get(file_key, deposit)) difference = len(values_record - values_deposit) assert different == difference def check_tag_master(record): bucket = CDSRecordDumpLoader._get_bucket(record=record) master = CDSVideosFilesIterator.get_master_video_file(record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) if obj.get_tags().get('master')] assert all([file_['tags']['master'] == master['version_id'] for file_ in files]) # 1 bucket record != 1 bucket deposit check(record, deposit, 'bucket_id', 1) # all file_id are the same except the smil file (only in record) check(record, deposit, 'file_id', 1) check(record, deposit, 'key', 1) # 18 object_version record != 17 object_version deposit check(record, deposit, 'version_id', 18) # check tag 'master' where is pointing check_tag_master(record) check_tag_master(deposit) def check_first_level_files(record): [master] = [file_ for file_ in deposit_video['_files'] if file_['context_type'] == 'master'] assert len(master['subformat']) == 5 assert len(master['frame']) == 10 # TODO assert len(master['playlist']) == ?? assert len([file_ for file_ in deposit_video['_files'] if file_['context_type'] == 'master']) == 1 duration = float(record['_cds']['extracted_metadata']['duration']) for frame in master['frame']: assert float(frame['tags']['timestamp']) < duration assert float(frame['tags']['timestamp']) > 0 # check video deposit deposit_video_uuid = PersistentIdentifier.query.filter( PersistentIdentifier.pid_type == 'depid', PersistentIdentifier.object_uuid != str(deposit_project_uuid), PersistentIdentifier.object_type == 'rec' ).one().object_uuid deposit_video = Video.get_record(str(deposit_video_uuid)) assert Video._schema in deposit_video['$schema'] assert video.revision_id == deposit_video[ '_deposit']['pid']['revision_id'] assert deposit_video['_deposit']['created_by'] == -1 assert deposit_video['_deposit']['owners'] == [-1] assert len(video['_files']) == 2 assert len(deposit_video['_files']) == 2 check_files(video) check_files(deposit_video) check_buckets(video, deposit_video) check_first_level_files(video) check_first_level_files(deposit_video) # try to edit video deposit_video = deposit_video_resolver(deposit_video['_deposit']['id']) deposit_video = deposit_video.edit() # try to edit project deposit_project = deposit_project_resolver( deposit_project['_deposit']['id']) deposit_project = deposit_project.edit() # try to publish again the video deposit_video['title']['title'] = 'test' deposit_video = deposit_video.publish() _, record_video = deposit_video.fetch_published() assert record_video['title']['title'] == 'test'