def check(project_status, video_1_status, video_2_status): project = deposit_project_resolver(project_depid) video_1 = deposit_video_resolver(video_1_depid) video_2 = deposit_video_resolver(video_2_depid) assert project.status == project_status assert video_1.status == video_1_status assert video_2.status == video_2_status
def test_project_publish_with_workflow(api_app, users, api_project, es): """Test publish a project with a workflow.""" project, video_1, video_2 = api_project prepare_videos_for_publish([video_1, video_2]) project_depid = project['_deposit']['id'] project_id = str(project.id) video_1_depid = video_1['_deposit']['id'] video_1_id = str(video_1.id) video_2_depid = video_2['_deposit']['id'] receiver_id = 'test_project_publish_with_workflow' workflow_receiver_video_failing(api_app, db, video_1, receiver_id=receiver_id) headers = [('Content-Type', 'application/json')] payload = json.dumps(dict(somekey='somevalue')) with mock.patch('invenio_indexer.tasks.index_record.delay') \ as mock_indexer, \ api_app.test_request_context(headers=headers, data=payload): event = Event.create(receiver_id=receiver_id) db.session.add(event) event.process() # check video and project are indexed assert mock_indexer.called is True ids = get_indexed_records_from_mock(mock_indexer) assert video_1_id == ids[0] assert project_id == ids[1] db.session.commit() # check tasks status is propagated to video and project video_1 = deposit_video_resolver(video_1_depid) expected = {u'add': u'SUCCESS', u'failing': u'FAILURE'} assert video_1['_cds']['state'] == expected assert video_1.project['_cds']['state'] == expected events = get_deposit_events(deposit_id=video_1_depid) assert len(events) == 1 def check(project_status, video_1_status, video_2_status): project = deposit_project_resolver(project_depid) video_1 = deposit_video_resolver(video_1_depid) video_2 = deposit_video_resolver(video_2_depid) assert project.status == project_status assert video_1.status == video_1_status assert video_2.status == video_2_status check('draft', 'draft', 'draft') login_user(User.query.get(users[0])) video_2 = deposit_video_resolver(video_2_depid) video_2.publish() check('draft', 'draft', 'published') project = deposit_project_resolver(project_depid) project.publish() check('published', 'published', 'published')
def test_deposit_vtt_tags(api_app, db, api_project, users): """Test VTT tag generation.""" project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] # insert a master file inside the video add_master_to_video( video_deposit=video_1, filename='test.mp4', stream=BytesIO(b'1234'), video_duration="15" ) # try to insert a new vtt object obj = ObjectVersion.create( video_1._bucket, key="test_fr.vtt", stream=BytesIO(b'hello')) # publish the video prepare_videos_for_publish([video_1]) video_1 = deposit_video_resolver(video_1_depid) login_user(User.query.get(users[0])) video_1 = video_1.publish() # check tags check_object_tags(obj, video_1, content_type='vtt', media_type='subtitle', context_type='subtitle', language='fr') # edit the video video_1 = video_1.edit() # try to delete the old vtt file and substitute with a new one video_1 = deposit_video_resolver(video_1_depid) ObjectVersion.delete(bucket=video_1._bucket, key=obj.key) obj2 = ObjectVersion.create( video_1._bucket, key="test_en.vtt", stream=BytesIO(b'hello')) # publish again the video video_1 = video_1.publish() # check tags check_object_tags(obj2, video_1, content_type='vtt', media_type='subtitle', context_type='subtitle', language='en') # edit a re-published video video_1 = video_1.edit() # add a new vtt file obj3 = ObjectVersion.create( video_1._bucket, key="test_it.vtt", stream=BytesIO(b'hello')) # publish again the video video_1 = video_1.publish() # check tags check_object_tags(obj3, video_1, content_type='vtt', media_type='subtitle', context_type='subtitle', language='it')
def test_deposit_poster_tags(api_app, db, api_project, users): """Test poster tag generation.""" project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] master_video_filename = 'test.mp4' poster_filename = 'poster.jpg' poster_filename2 = 'poster.png' # insert a master file inside the video add_master_to_video(video_deposit=video_1, filename=master_video_filename, stream=BytesIO(b'1234'), video_duration='15') # try to insert a new vtt object obj = ObjectVersion.create(video_1._bucket, key=poster_filename, stream=BytesIO(b'hello')) # publish the video prepare_videos_for_publish([video_1]) video_1 = deposit_video_resolver(video_1_depid) login_user(User.query.get(users[0])) video_1 = video_1.publish() # check tags check_object_tags(obj, video_1, content_type='jpg', context_type='poster', media_type='image') # edit the video video_1 = video_1.edit() # try to delete the old poster frame and substitute with a new one video_1 = deposit_video_resolver(video_1_depid) ObjectVersion.delete(bucket=video_1._bucket, key=obj.key) obj2 = ObjectVersion.create(video_1._bucket, key=poster_filename2, stream=BytesIO(b'hello')) # publish again the video video_1 = video_1.publish() # check tags check_object_tags(obj2, video_1, content_type='png', context_type='poster', media_type='image')
def test_video_delete_with_workflow(api_app, users, api_project, webhooks, es): """Test publish a project with a workflow.""" project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] receiver_id = 'test_video_delete_with_workflow' workflow_receiver_video_failing(api_app, db, video_1, receiver_id=receiver_id) mock_delete = MagicMock(return_value=None) current_webhooks.receivers[receiver_id].delete = mock_delete headers = [('Content-Type', 'application/json')] payload = json.dumps(dict(somekey='somevalue')) with api_app.test_request_context(headers=headers, data=payload): event = Event.create(receiver_id=receiver_id) db.session.add(event) event.process() db.session.commit() video_1 = deposit_video_resolver(video_1_depid) video_1.delete() assert mock_delete.called is True
def test_sync_records_with_deposits(app, db, location, users, project_deposit_metadata, video_deposit_metadata): """Test sync records with deposits task.""" # create a project project = Project.create(project_deposit_metadata) project_deposit_metadata['report_number'] = ['123'] # create new video video_deposit_metadata['_project_id'] = project['_deposit']['id'] deposit = Video.create(video_deposit_metadata) depid = deposit['_deposit']['id'] # insert objects inside the deposit ObjectVersion.create(deposit.files.bucket, "obj_1").set_location("mylocation1", 1, "mychecksum1") ObjectVersion.create(deposit.files.bucket, "obj_2").set_location("mylocation2", 1, "mychecksum2") ObjectVersion.create(deposit.files.bucket, "obj_3").set_location("mylocation3", 1, "mychecksum3") obj_4 = ObjectVersion.create(deposit.files.bucket, "obj_4").set_location("mylocation4", 1, "mychecksum4") # publish login_user(User.query.get(users[0])) prepare_videos_for_publish([deposit]) deposit = deposit.publish() _, record = deposit.fetch_published() assert deposit.is_published() is True # add a new object ObjectVersion.create(deposit.files.bucket, "obj_new").set_location("mylocation_new", 1, "mychecksum") # modify obj_1 ObjectVersion.create(deposit.files.bucket, "obj_new").set_location("mylocation2.1", 1, "mychecksum2.1") # delete obj_3 ObjectVersion.delete(deposit.files.bucket, "obj_3") # remove obj_4 obj_4.remove() # check video and record files = ['obj_1', 'obj_2', 'obj_3', 'obj_4'] edited_files = ['obj_1', 'obj_2', 'obj_3', 'obj_new'] check_deposit_record_files(deposit, edited_files, record, files) # try to sync deposit and record sync_records_with_deposit_files.s(deposit_id=depid).apply_async() # get deposit and record deposit = deposit_video_resolver(depid) _, record = deposit.fetch_published() assert deposit.is_published() is True # check that record and deposit are sync re_edited_files = edited_files + ['obj_4'] check_deposit_record_files(deposit, edited_files, record, re_edited_files)
def _resolve_deposit(id_type, id_value): """Return the deposit video.""" depid = id_value if id_type == 'recid': _, record = record_resolver.resolve(id_value) depid = record['_deposit']['id'] return deposit_video_resolver(depid), depid
def test_record_video_links(datacite_mock, api_app, es, api_project, users, json_headers): """Test record video links.""" (project, video_1, video_2) = api_project user = User.query.filter_by(id=users[0]).first() prepare_videos_for_publish([video_1, video_2]) vid = video_1['_deposit']['id'] pid = project['_deposit']['id'] with api_app.test_client() as client: login_user_via_session(client, user) # publish video url = url_for('invenio_deposit_rest.video_actions', pid_value=vid, action='publish') assert client.post(url).status_code == 202 rec_pid, rec_video = deposit_video_resolver(vid).fetch_published() # get a record video (with no published project) url = url_for('invenio_records_rest.recid_item', pid_value=rec_pid.pid_value, _external=True) res = client.get(url, headers=json_headers) assert res.status_code == 200 # check video record data = json.loads(res.data.decode('utf-8')) assert data['links'] == {'self': url} # publish the project url = url_for('invenio_deposit_rest.project_actions', pid_value=pid, action='publish') assert client.post(url).status_code == 202 rec_pid_proj, rec_proj = video_1.project.fetch_published() # get a record video (with published project) url = url_for('invenio_records_rest.recid_item', pid_value=rec_pid.pid_value, _external=True) res = client.get(url, headers=json_headers) assert res.status_code == 200 # check video record data = json.loads(res.data.decode('utf-8')) url_api_prj = 'http://localhost/record/3' url_prj = 'http://localhost/record/3' url_prj_edit = 'http://localhost/deposit/project/{0}'.format(pid) assert data['links'] == { 'self': url, 'project': url_api_prj, 'project_html': url_prj, 'project_edit': url_prj_edit, }
def test_video_publish_registering_the_datacite_not_local( datacite_mock, api_app, users, location, cds_jsonresolver, json_headers, json_partial_project_headers, json_partial_video_headers, deposit_metadata, video_deposit_metadata, project_deposit_metadata, keyword_1, keyword_2): """Test video publish registering the datacite not local.""" # test: enable datacite registration api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True with api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) project_deposit_metadata['keywords'] = [copy.deepcopy(keyword_2)] # [[ CREATE NEW PROJECT ]] project_dict = _create_new_project(client, json_partial_project_headers, project_deposit_metadata) assert project_dict['metadata']['keywords'][0] == keyword_2 project_depid = project_dict['metadata']['_deposit']['id'] project = deposit_project_resolver(project_depid) assert project['keywords'] == [{ '$ref': keyword_2.ref, 'name': keyword_2['name'], }] # [[ ADD A NEW VIDEO_1 ]] video_metadata = copy.deepcopy(video_deposit_metadata) video_metadata['keywords'] = [copy.deepcopy(keyword_1)] video_metadata.update( _project_id=project_dict['metadata']['_deposit']['id']) res = client.post(url_for('invenio_deposit_rest.video_list'), data=json.dumps(video_metadata), headers=json_partial_video_headers) assert res.status_code == 201 video_1_dict = json.loads(res.data.decode('utf-8')) assert video_1_dict['metadata']['keywords'][0] == keyword_1 video_1_depid = video_1_dict['metadata']['_deposit']['id'] video_1 = deposit_video_resolver(video_1_depid) assert video_1['keywords'] == [{ '$ref': keyword_1.ref, 'name': keyword_1['name'], }] video_1['doi'] = '10.1123/doi' prepare_videos_for_publish([video_1]) # [[ PUBLISH VIDEO ]] video_1.publish() datacite_register.s(pid_value='123', record_uuid=str(video_1.id)).apply() assert datacite_mock.called is False
def check_gif(video, mock_gif): # called only once for deposit (_, _, mock_args) = mock_gif.mock_calls[0] # check gif record video = CDSRecord(dict(video), video.model) # check gif deposit deposit = deposit_video_resolver(video['_deposit']['id']) master_video = CDSVideosFilesIterator.get_master_video_file(deposit) assert mock_args['master_id'] == master_video['version_id'] assert str(deposit.files.bucket.id) == mock_args['bucket'] # assert mock_args['bucket'].id == deposit.files.bucket.id assert len(mock_args['frames']) == 10 assert 'output_dir' in mock_args
def test_video_publish_with_no_type(api_project): """Test video publish with no type.""" (project, video_1, video_2) = api_project prepare_videos_for_publish([video_1, video_2]) video_1_depid = video_1['_deposit']['id'] # test: no type in project del project['type'] assert 'category' in project project.commit() db.session.commit() video_1 = deposit_video_resolver(video_1_depid) with pytest.raises(ValidationError): video_1.publish()
def test_video_publish_with_category_and_type(api_project): """Test video publish with category and type.""" (project, video_1, video_2) = api_project prepare_videos_for_publish([video_1, video_2]) video_1_depid = video_1['_deposit']['id'] # test with category + type assert 'type' in project assert 'category' in project project.commit() db.session.commit() video_1 = deposit_video_resolver(video_1_depid) video_1.publish() assert video_1['_deposit']['status'] == 'published'
def extract_frames(recid, depid): """Re-trigger the extract frames task.""" if not recid and not depid: raise ClickException('Missing option "--recid" or "--depid"') if recid: _, record = record_resolver.resolve(recid) depid = record['_deposit']['id'] video_deposit = deposit_video_resolver(depid) master = CDSVideosFilesIterator.get_master_video_file(video_deposit) ExtractFramesTask().si(version_id=master['version_id'], deposit_id=depid).apply_async()
def test_boolean_fields_are_indexed(api_app, es, api_project, users, json_headers): """Test boolean fields (i.e. featured and vr) are indexed.""" (project, video_1, video_2) = api_project with api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) # [[ PUBLISH THE PROJECT ]] prepare_videos_for_publish([video_1, video_2]) client.post( url_for('invenio_deposit_rest.project_actions', pid_value=project['_deposit']['id'], action='publish'), headers=json_headers) RecordIndexer().process_bulk_queue() sleep(2) video_1_record = deposit_video_resolver(video_1['_deposit']['id']) video_2_record = deposit_video_resolver(video_2['_deposit']['id']) def assert_get_video(query, video_record): url = url_for('invenio_records_rest.recid_list', q=query) res = client.get(url, headers=json_headers) assert res.status_code == 200 data = json.loads(res.data.decode('utf-8')) assert len(data['hits']['hits']) == 1 assert data['hits']['hits'][0]['id'] == video_record['recid'] # Featured assert_get_video('featured:true', video_1_record) # Not featured assert_get_video('featured:false', video_2_record) # VR assert_get_video('vr:true', video_1_record) # Not VR assert_get_video('vr:false', video_2_record)
def test_video_publish_registering_the_datacite( datacite_mock, api_app, users, location, cds_jsonresolver, json_headers, json_partial_project_headers, json_partial_video_headers, deposit_metadata, video_deposit_metadata, project_deposit_metadata): """Test video publish registering the datacite.""" # test: enable datacite registration api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True with api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) # [[ CREATE NEW PROJECT ]] project_dict = _create_new_project(client, json_partial_project_headers, project_deposit_metadata) # [[ ADD A NEW EMPTY VIDEO_1 ]] video_1_dict = _add_video_info_to_project(client, json_partial_video_headers, project_dict, video_deposit_metadata) video_1_depid = video_1_dict['metadata']['_deposit']['id'] video_1 = deposit_video_resolver(video_1_depid) prepare_videos_for_publish([video_1]) # [[ PUBLISH VIDEO ]] video_1.publish() # [[ REGISTER DATACITE ]] datacite_register_after_publish(sender=api_app, action='publish', deposit=video_1) assert datacite_mock.called is True assert datacite_mock().metadata_post.call_count == 1 datacite_mock().doi_post.assert_called_once_with( '10.0000/cds.1', 'https://cds.cern.ch/record/1') # [[ UPDATE DATACITE ]] datacite_register_after_publish(sender=api_app, action='publish', deposit=video_1) assert datacite_mock.called is True assert datacite_mock().metadata_post.call_count == 2 datacite_mock().doi_post.assert_called_with( '10.0000/cds.1', 'https://cds.cern.ch/record/1')
def run(self, preset_quality, sleep_time=5, *args, **kwargs): super(MaintenanceTranscodeVideoTask, self).run(preset_quality=preset_quality, sleep_time=sleep_time, *args, **kwargs) logger.debug("Updating deposit and record") # get deposit and record dep_video = deposit_video_resolver(self.deposit_id) rec_video = record_resolver.resolve(dep_video['recid'])[1] # sync deposit --> record dep_video._sync_record_files(record=rec_video) dep_video.commit() rec_video.commit() db.session.commit()
def test_deposit_smil_tag_generation(api_app, db, api_project, users): """Test AVCWorkflow receiver.""" def check_smil(video): _, record = video.fetch_published() master = CDSVideosFilesIterator.get_master_video_file(record) playlist = master['playlist'] assert playlist[0]['key'] == '{}.smil'. \ format(record['report_number'][0]) assert playlist[0]['content_type'] == 'smil' assert playlist[0]['context_type'] == 'playlist' assert playlist[0]['media_type'] == 'text' assert playlist[0]['tags']['master'] == master['version_id'] # check bucket dump is done correctly master_video = CDSVideosFilesIterator.get_master_video_file(video) assert master_video['version_id'] != master['version_id'] project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] # insert a master file inside the video add_master_to_video(video_deposit=video_1, filename='test.mp4', stream=BytesIO(b'1234'), video_duration="15s") # publish the video prepare_videos_for_publish([video_1]) video_1 = deposit_video_resolver(video_1_depid) login_user(User.query.get(users[0])) video_1 = video_1.publish() # check smil check_smil(video_1) # edit the video video_1 = video_1.edit() # publish again the video video_1 = video_1.publish() # check smil check_smil(video_1)
def test_avc_workflow_delete(api_app, db, api_project, users, access_token, json_headers, mock_sorenson, online_video, webhooks, checker): """Test AVCWorkflow receiver REST API.""" project, video_1, video_2 = api_project video_1_id = video_1.id video_1_depid = video_1['_deposit']['id'] master_key = 'test.mp4' with api_app.test_request_context(): url = url_for('invenio_webhooks.event_list', receiver_id='avc', access_token=access_token) with api_app.test_client() as client, \ mock.patch('invenio_sse.ext._SSEState.publish'), \ mock.patch('invenio_indexer.api.RecordIndexer.bulk_index'): sse_channel = 'mychannel' payload = dict( uri=online_video, deposit_id=video_1_depid, key=master_key, sse_channel=sse_channel, sleep_time=0, ) resp = client.post(url, headers=json_headers, data=json.dumps(payload)) assert resp.status_code == 201 data = json.loads(resp.data.decode('utf-8')) # check extracted metadata is there record = RecordMetadata.query.get(video_1_id) assert 'extracted_metadata' in record.json['_cds'] assert ObjectVersion.query.count() == get_object_count() assert ObjectVersionTag.query.count() == get_tag_count() event_id = data['tags']['_event_id'] video_1_id = str(deposit_video_resolver(video_1_depid).id) ### checker(api_app, event_id, access_token, json_headers, data, video_1_id, video_1_depid, users)
def test_video_name_after_publish(api_app, db, api_project, users): project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] master_video_filename = 'test.mp4' # insert a master file inside the video add_master_to_video(video_deposit=video_1, filename=master_video_filename, stream=BytesIO(b'1234'), video_duration='15') # publish the video prepare_videos_for_publish([video_1]) video_1 = deposit_video_resolver(video_1_depid) login_user(User.query.get(users[0])) video_1 = video_1.publish() _, record = video_1.fetch_published() master = CDSVideosFilesIterator.get_master_video_file(record) assert master['key'] == '{}.mp4'.format(record['report_number'][0])
def test_project_edit(app, project_published, users): """Test project edit.""" (project, video_1, video_2) = project_published assert project.status == 'published' assert video_1.status == 'published' assert video_2.status == 'published' # Edit project (change project title) new_project = project.edit() assert new_project.status == 'draft' new_project.update(title={'title': 'My project'}) # Edit videos inside project (change video titles) videos = [ deposit_video_resolver(record_video_resolver(id_)['_deposit']['id']) for id_ in new_project.video_ids ] login_user(User.query.get(users[0])) assert len(videos) == 2 for i, video in enumerate(videos): # video = Video.get_record(video.id) assert video['_deposit']['status'] == 'published' new_video = video.edit() assert new_video['_deposit']['status'] == 'draft' new_video.update(title={'title': 'Video {}'.format(i + 1)}) new_video.publish() # Publish all changes new_project.publish() # Check that everything is published videos = [record_video_resolver(id_) for id_ in new_project.video_ids] assert new_project['_deposit']['status'] == 'published' assert all(video['_deposit']['status'] == 'published' for video in videos) # Check that all titles where properly changed assert new_project['title']['title'] == 'My project' assert videos[0]['title']['title'] in ['Video 1', 'Video 2'] assert videos[1]['title']['title'] in ['Video 1', 'Video 2'] assert videos[0]['title']['title'] != videos[1]['title']['title']
def test_video_publish_registering_the_datacite_if_fail( datacite_mock, api_app, users, location, cds_jsonresolver, json_headers, json_partial_project_headers, json_partial_video_headers, deposit_metadata, video_deposit_metadata, project_deposit_metadata): """Test video publish registering the datacite.""" # test: enable datacite registration api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True with api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) # [[ CREATE NEW PROJECT ]] project_dict = _create_new_project(client, json_partial_project_headers, project_deposit_metadata) # [[ ADD A NEW EMPTY VIDEO_1 ]] video_1_dict = _add_video_info_to_project(client, json_partial_video_headers, project_dict, video_deposit_metadata) video_1_depid = video_1_dict['metadata']['_deposit']['id'] video_1 = deposit_video_resolver(video_1_depid) prepare_videos_for_publish([video_1]) # [[ PUBLISH VIDEO ]] video_1.publish() db.session.commit() with mock.patch('invenio_records.api.Record.get_record', side_effect=[Exception, video_1], return_value=video_1): with pytest.raises(Retry): datacite_register.s(pid_value='1', record_uuid=str(video_1.id)).apply() assert datacite_mock.called is True assert datacite_mock().metadata_post.call_count == 1 datacite_mock().doi_post.assert_called_once_with( '10.0000/cds.1', 'https://cds.cern.ch/record/1') assert datacite_mock.call_count == 3
def test_subformat_creation_if_missing(api_app, location, datadir, es, users): """Test subformat creation if missing.""" # [[ migrate the video ]] migration_streams = get_migration_streams(datadir=datadir) data = load_json(datadir, 'cds_records_demo_1_video.json') dump = CDSRecordDump(data=data[0]) with mock.patch.object(DataCiteProvider, 'register'), \ mock.patch.object(CDSRecordDumpLoader, '_create_frame', side_effect=get_frames), \ mock.patch.object(ExtractFramesTask, '_create_gif'), \ mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'), \ mock.patch.object( CDSRecordDumpLoader, '_get_migration_file_stream_and_size', side_effect=migration_streams): video = CDSRecordDumpLoader.create(dump=dump) db.session.commit() with mock.patch.object(TranscodeVideoTask, 'run') as mock_transcode: deposit = deposit_video_resolver(video['_deposit']['id']) deposit_id = deposit.id # simulate the missing of a subformat del deposit['_files'][0]['subformat'][0] assert len(deposit['_files'][0]['subformat']) == 4 # recreate 240p format CDSRecordDumpLoader._create_missing_subformats( record=video, deposit=deposit) db.session.commit() # check subformats deposit = Video.get_record(deposit_id) rec_video = record_resolver.resolve(video['recid'])[1] # rec_video = record_resolver.resolve(video['recid'])[1] assert len(deposit['_files'][0]['subformat']) == 5 assert len(rec_video['_files'][0]['subformat']) == 5 # check if transcoding is called properly assert mock_transcode.called is True [(_, call_args)] = mock_transcode.call_args_list assert call_args == {'preset_quality': '240p'}
def test_video_publish_edit_publish_again( datacite_mock, es, api_app, users, location, cds_jsonresolver, json_headers, json_partial_project_headers, json_partial_video_headers, deposit_metadata, video_deposit_metadata, project_deposit_metadata): """Test video publish registering the datacite not local.""" # test: enable datacite registration api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True api_app.config['PIDSTORE_DATACITE_DOI_PREFIX'] = '10.5072' with api_app.test_request_context(): with api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) # [[ CREATE A NEW PROJECT ]] project_dict = _create_new_project(client, json_partial_project_headers, project_deposit_metadata) # [[ ADD A NEW EMPTY VIDEO_1 ]] video_1_dict = _add_video_info_to_project( client, json_partial_video_headers, project_dict, video_deposit_metadata) video_1_depid = video_1_dict['metadata']['_deposit']['id'] video_1 = deposit_video_resolver(video_1_depid) prepare_videos_for_publish([video_1]) # [[ PUBLISH VIDEO ]] _deposit_publish(client, json_headers, video_1['_deposit']['id']) datacite_register.s(pid_value='123', record_uuid=str(video_1.id)).apply() # [[ EDIT VIDEO ]] _deposit_edit(client, json_headers, video_1['_deposit']['id']) # [[ MODIFY DOI -> SAVE ]] video_1 = deposit_video_resolver(video_1_depid) video_1_dict = copy.deepcopy(video_1) # old_doi = video_1_dict['doi'] video_1_dict['doi'] = '10.1123/doi' del video_1_dict['_files'] res = client.put(url_for('invenio_deposit_rest.video_item', pid_value=video_1['_deposit']['id']), data=json.dumps(video_1_dict), headers=json_headers) # check returned value assert res.status_code == 400 data = json.loads(res.data.decode('utf-8')) assert data['errors'] == [{ "field": "doi", "message": "The DOI cannot be changed." }] video_1 = deposit_video_resolver(video_1_depid) # video_1['doi'] = old_doi video_1_dict = copy.deepcopy(video_1) del video_1_dict['_files'] # try to modify preserved fields video_1_dict['recid'] = 12323233 video_1_dict['report_number'] = ['fuuu barrrr'] video_1_dict['publication_date'] = '2000-12-03' video_1_dict['_project_id'] = '1234567' # do the call res = client.put(url_for('invenio_deposit_rest.video_item', pid_value=video_1['_deposit']['id']), data=json.dumps(video_1_dict), headers=json_headers) # check returned value assert res.status_code == 200 # check preserved fields video_1_new = deposit_video_resolver(video_1_depid) assert video_1_new['recid'] == video_1['recid'] assert video_1_new['report_number'] == video_1['report_number'] assert video_1_new['publication_date'] == video_1[ 'publication_date'] assert video_1_new['_project_id'] == video_1['_project_id'] # [[ PUBLISH VIDEO ]] _deposit_publish(client, json_headers, video_1['_deposit']['id']) # check indexed record RecordIndexer().process_bulk_queue() sleep(2) res = client.get( url_for('invenio_records_rest.recid_list', headers=json_headers)) assert res.status_code == 200 data = json.loads(res.data.decode('utf-8')) for hit in data['hits']['hits']: assert isinstance(hit['id'], int)
def test_download_receiver(api_app, db, api_project, access_token, webhooks, json_headers): """Test downloader receiver.""" project, video_1, video_2 = api_project video_1_depid = video_1['_deposit']['id'] video_1_id = str(video_1.id) project_id = str(project.id) with api_app.test_request_context(): url = url_for('invenio_webhooks.event_list', receiver_id='downloader', access_token=access_token) with mock.patch('requests.get') as mock_request, \ mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \ mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \ as mock_indexer, \ api_app.test_client() as client: sse_channel = 'mychannel' mock_sse.return_value = None file_size = 1024 mock_request.return_value = type( 'Response', (object, ), { 'raw': BytesIO(b'\x00' * file_size), 'headers': { 'Content-Length': file_size } }) payload = dict(uri='http://example.com/test.pdf', deposit_id=video_1_depid, key='test.pdf', sse_channel=sse_channel) resp = client.post(url, headers=json_headers, data=json.dumps(payload)) assert resp.status_code == 201 data = json.loads(resp.data.decode('utf-8')) assert '_tasks' in data assert data['tags']['uri_origin'] == 'http://example.com/test.pdf' assert data['key'] == 'test.pdf' assert 'version_id' in data assert 'links' in data # TODO decide with links are needed assert all( [link in data['links'] for link in ['self', 'version', 'cancel']]) assert ObjectVersion.query.count() == 1 obj = ObjectVersion.query.first() tags = obj.get_tags() assert tags['_event_id'] == data['tags']['_event_id'] assert obj.key == data['key'] assert str(obj.version_id) == data['version_id'] assert obj.file assert obj.file.size == file_size # check sse is called assert mock_sse.called def set_data(state, message, size, total, percentage, type_): return { 'state': state, 'meta': { 'message': message, 'payload': { 'event_id': str(tags['_event_id']), 'key': u'test.pdf', 'tags': { u'uri_origin': u'http://example.com/test.pdf', u'_event_id': str(tags['_event_id']), u'context_type': u'master', }, 'deposit_id': video_1_depid, 'percentage': percentage, 'version_id': str(obj.version_id), 'size': size, 'total': total, 'sse_channel': sse_channel, 'type': type_ } } } assert mock_sse.call_count == 7 mock_sse.assert_any_call(data=set_data( states.STARTED, 'Downloading {} of {}'.format(file_size, file_size), file_size, file_size, 100, 'file_download'), channel=u'mychannel', type_='file_download') mock_sse.assert_any_call(data=set_data(states.SUCCESS, str(obj.version_id), file_size, file_size, 100, 'file_download'), channel=u'mychannel', type_='file_download') deposit = deposit_video_resolver(video_1_depid) mock_sse.assert_any_call( channel='mychannel', data={ 'state': states.SUCCESS, 'meta': { 'payload': { 'event_id': str(tags['_event_id']), 'deposit_id': video_1_depid, 'deposit': deposit, } } }, type_='update_deposit', ) # check ElasticSearch is called ids = set(get_indexed_records_from_mock(mock_indexer)) assert video_1_id in ids assert project_id in ids assert deposit['_cds']['state'] == {u'file_download': states.SUCCESS} # Test cleaning! url = '{0}?access_token={1}'.format(data['links']['cancel'], access_token) with mock.patch('requests.get') as mock_request, \ mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \ mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \ as mock_indexer, \ api_app.test_client() as client: resp = client.delete(url, headers=json_headers) assert resp.status_code == 201 assert ObjectVersion.query.count() == 2 bucket = Bucket.query.first() assert bucket.size == 0 assert mock_sse.called is False assert mock_indexer.called is False
def get_video_record(depid): deposit = deposit_video_resolver(depid) return Video.get_record(deposit.fetch_published()[1].id)
def test_video_events_on_workflow(webhooks, api_app, db, api_project, bucket, access_token, json_headers): """Test deposit events.""" (project, video_1, video_2) = api_project project_depid = project['_deposit']['id'] video_1_depid = video_1['_deposit']['id'] db.session.add(bucket) # registering receiver receiver_id = 'test_video_events_on_workflow' workflow_receiver_video_failing(api_app, db, video_1, receiver_id=receiver_id) with api_app.test_request_context(): url = url_for('invenio_webhooks.event_list', receiver_id=receiver_id, access_token=access_token) with api_app.test_client() as client: # run workflow resp = client.post(url, headers=json_headers, data=json.dumps({})) assert resp.status_code == 500 # run again workflow resp = client.post(url, headers=json_headers, data=json.dumps({})) assert resp.status_code == 500 # resolve deposit and events deposit = deposit_video_resolver(video_1_depid) events = get_deposit_events(deposit['_deposit']['id']) # check events assert len(events) == 2 assert events[0].payload['deposit_id'] == video_1_depid assert events[1].payload['deposit_id'] == video_1_depid # check computed status status = get_tasks_status_by_task(events) assert status['add'] == states.SUCCESS assert status['failing'] == states.FAILURE # check every task for every event for event in events: result = event.receiver._deserialize_result(event) assert result.parent.status == states.SUCCESS assert result.children[0].status == states.FAILURE assert result.children[1].status == states.SUCCESS # check if the states are inside the deposit res = client.get(url_for('invenio_deposit_rest.video_item', pid_value=video_1_depid, access_token=access_token), headers=json_headers) assert res.status_code == 200 data = json.loads(res.data.decode('utf-8'))['metadata'] assert data['_cds']['state']['add'] == states.SUCCESS assert data['_cds']['state']['failing'] == states.FAILURE # run indexer RecordIndexer().process_bulk_queue() sleep(2) # check elasticsearch video_1 state resp = client.get(url_for('invenio_deposit_rest.video_list', q='_deposit.id:{0}'.format(video_1_depid), access_token=access_token), headers=json_headers) assert resp.status_code == 200 data = json.loads(resp.data.decode('utf-8')) status = data['hits']['hits'][0]['metadata']['_cds']['state'] assert status['add'] == states.SUCCESS assert status['failing'] == states.FAILURE # check elasticsearch project state resp = client.get(url_for('invenio_deposit_rest.video_list', q='_deposit.id:{0}'.format(project_depid), access_token=access_token), headers=json_headers) assert resp.status_code == 200 data = json.loads(resp.data.decode('utf-8')) status = data['hits']['hits'][0]['metadata']['_cds']['state'] assert status['add'] == states.SUCCESS assert status['failing'] == states.FAILURE
def test_video_events_on_download_create(api_app, webhooks, db, api_project, access_token, json_headers): """Test deposit events.""" (project, video_1, video_2) = api_project video_1_depid = video_1['_deposit']['id'] project_id = str(project.id) video_1_id = str(video_1.id) bucket_id = video_1._bucket.id with api_app.test_request_context(): url = url_for('invenio_webhooks.event_list', receiver_id='downloader', access_token=access_token) with mock.patch('requests.get') as mock_request, \ mock.patch('invenio_indexer.tasks.index_record.delay') \ as mock_indexer, \ api_app.test_client() as client: file_size = 1024 * 1024 mock_request.return_value = type( 'Response', (object, ), { 'raw': BytesIO(b'\x00' * file_size), 'headers': { 'Content-Length': file_size } }) payload = dict(uri='http://example.com/test.pdf', bucket_id=str(bucket_id), deposit_id=video_1_depid, key='test.pdf') resp = client.post(url, headers=json_headers, data=json.dumps(payload)) assert resp.status_code == 201 file_size = 1024 * 1024 * 6 mock_request.return_value = type( 'Response', (object, ), { 'raw': BytesIO(b'\x00' * file_size), 'headers': { 'Content-Length': file_size } }) resp = client.post(url, headers=json_headers, data=json.dumps(payload)) assert resp.status_code == 201 deposit = deposit_video_resolver(video_1_depid) events = get_deposit_events(deposit['_deposit']['id']) assert len(events) == 2 assert events[0].payload['deposit_id'] == video_1_depid assert events[1].payload['deposit_id'] == video_1_depid status = get_tasks_status_by_task(events) assert status == {'file_download': states.SUCCESS} # check if the states are inside the deposit res = client.get(url_for('invenio_deposit_rest.video_item', pid_value=video_1_depid, access_token=access_token), headers=json_headers) assert res.status_code == 200 data = json.loads(res.data.decode('utf-8'))['metadata'] assert data['_cds']['state']['file_download'] == states.SUCCESS assert deposit._get_files_dump() == data['_files'] # check the record is inside the indexer queue ids = set(get_indexed_records_from_mock(mock_indexer)) assert len(ids) == 2 assert video_1_id in ids assert project_id in ids
def test_video_events_on_download_check_index(api_app, webhooks, db, api_project, access_token, json_headers, users): """Test deposit events.""" (project, video_1, video_2) = api_project prepare_videos_for_publish([video_1, video_2]) project_depid = project['_deposit']['id'] video_1_depid = video_1['_deposit']['id'] bucket_id = video_1._bucket.id with api_app.test_request_context(): url = url_for('invenio_webhooks.event_list', receiver_id='downloader', access_token=access_token) with mock.patch('requests.get') as mock_request, \ api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) file_size = 1024 * 1024 mock_request.return_value = type( 'Response', (object, ), { 'raw': BytesIO(b'\x00' * file_size), 'headers': { 'Content-Length': file_size } }) payload = dict(uri='http://example.com/test.pdf', bucket_id=str(bucket_id), deposit_id=video_1_depid, key='test.pdf') resp = client.post(url, headers=json_headers, data=json.dumps(payload)) assert resp.status_code == 201 # run indexer RecordIndexer().process_bulk_queue() sleep(2) deposit = deposit_video_resolver(video_1_depid) file_dumps = deposit._get_files_dump() assert len(file_dumps) == 1 def search_record(url): res = client.get(url, headers=json_headers) assert res.status_code == 200 data = json.loads( res.data.decode('utf-8'))['hits']['hits'][0]['metadata'] return data # check if the tasks states and files are inside elasticsearch # -> check video url_video_deposit = url_for('invenio_deposit_rest.video_list', q='_deposit.id:{0}'.format(video_1_depid), access_token=access_token) data = search_record(url_video_deposit) assert data['_cds']['state']['file_download'] == states.SUCCESS assert file_dumps == data['_files'] # -> check project url_project_deposit = url_for( 'invenio_deposit_rest.project_list', q='_deposit.id:{0}'.format(project_depid), access_token=access_token) search_record(url_project_deposit) assert data['_cds']['state']['file_download'] == states.SUCCESS # [[ EDIT VIDEO ]] deposit = deposit_video_resolver(video_1_depid) video_edited = deepcopy(deposit) del video_edited['_files'] del video_edited['_cds']['state'] reset_oauth2() res = client.put(url_for('invenio_deposit_rest.video_item', pid_value=video_1_depid), data=json.dumps(video_edited), headers=json_headers) assert res.status_code == 200 # check if the tasks states and files are inside elasticsearch # -> check video data = search_record(url_video_deposit) assert data['_cds']['state']['file_download'] == states.SUCCESS assert file_dumps == data['_files'] # -> check project url_project_deposit = url_for( 'invenio_deposit_rest.project_list', q='_deposit.id:{0}'.format(project_depid), access_token=access_token) search_record(url_project_deposit) assert data['_cds']['state']['file_download'] == states.SUCCESS # [[ PUBLISH THE PROJECT ]] reset_oauth2() res = client.post(url_for( 'invenio_deposit_rest.project_actions', pid_value=project['_deposit']['id'], action='publish', ), headers=json_headers) assert res.status_code == 202 # run indexer RecordIndexer().process_bulk_queue() sleep(2) deposit = deposit_video_resolver(video_1_depid) # check if the files are inside elasticsearch # -> check video deposit data = search_record(url_video_deposit) assert data['_cds']['state']['file_download'] == states.SUCCESS assert file_dumps == data['_files'] # check video record pid, record = deposit.fetch_published() url = url_for('invenio_records_rest.recid_list', q='_deposit.pid.value:{0}'.format(pid.pid_value)) data = search_record(url) assert record['_files'] == data['_files']
def test_simple_workflow( api_app, db, es, users, location, cds_jsonresolver, data_file_1, data_file_2, json_headers, json_partial_project_headers, json_partial_video_headers, deposit_metadata, project_deposit_metadata, video_deposit_metadata): """Test project simple workflow.""" def check_connection(videos, project): """check project <---> video connection.""" assert all({"$ref": video.ref} in project['videos'] for video in videos) assert len(videos) == len(project['videos']) project_schema = ('https://cdslabs.cern.ch/schemas/' 'deposits/records/videos/project/project-v1.0.0.json') video_schema = ('https://cdslabs.cern.ch/schemas/' 'deposits/records/videos/video/video-v1.0.0.json') with api_app.test_client() as client: login_user_via_session(client, email=User.query.get(users[0]).email) # [[ CREATE NEW PROJECT ]] res = client.post( url_for('invenio_deposit_rest.project_list'), data=json.dumps(project_deposit_metadata), headers=json_partial_project_headers) # check returned value assert res.status_code == 201 project_dict = json.loads(res.data.decode('utf-8')) assert project_dict['metadata']['videos'] == [] assert project_dict['metadata']['title']['title'] == 'my project' assert project_dict['links']['bucket'].startswith( 'http://localhost/files/') assert all(link.startswith('http://localhost/deposits/project') for (key, link) in project_dict['links'].items() if key not in ['html', 'bucket']) # check database project_id = project_dict['metadata']['_deposit']['id'] project = project_resolver.resolve(project_id)[1] assert project['$schema'] == project_schema # [[ ADD A NEW EMPTY VIDEO_1 ]] video_metadata = deepcopy(video_deposit_metadata) video_metadata.update( _project_id=project_dict['metadata']['_deposit']['id']) res = client.post( url_for('invenio_deposit_rest.video_list'), data=json.dumps(video_metadata), headers=json_partial_video_headers) # check returned value assert res.status_code == 201 video_1_dict = json.loads(res.data.decode('utf-8')) assert video_1_dict['metadata']['_project_id'] == project_id assert all(link.startswith('http://localhost/deposits/video') for (key, link) in video_1_dict['links'].items() if key not in ['html', 'bucket']) # check database: connection project <---> videos video_ids = [ video_1_dict['metadata']['_deposit']['id'] ] [video_1] = deposit_videos_resolver(video_ids) check_connection( [video_1], project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] ) assert video_1['$schema'] == video_schema # [[ GET THE VIDEO 1 ]] res = client.get( video_1_dict['links']['self'], headers=json_headers) # check returned value assert res.status_code == 200 video_1_dict = json.loads(res.data.decode('utf-8')) assert video_1_dict['metadata']['_files'] == [] # [[ ADD A NEW EMPTY VIDEO_2 ]] video_metadata = deepcopy(video_deposit_metadata) video_metadata.update( _project_id=project_dict['metadata']['_deposit']['id']) res = client.post( url_for('invenio_deposit_rest.video_list'), data=json.dumps(video_metadata), headers=json_partial_video_headers) # check returned value assert res.status_code == 201 video_2_dict = json.loads(res.data.decode('utf-8')) assert video_2_dict['metadata']['_project_id'] == project_id assert all(link.startswith('http://localhost/deposits/video') for (key, link) in video_2_dict['links'].items() if key not in ['html', 'bucket']) # check database: connection project <---> videos video_ids = [ video_1_dict['metadata']['_deposit']['id'], video_2_dict['metadata']['_deposit']['id'] ] [video_1, video_2] = deposit_videos_resolver(video_ids) check_connection( [video_1, video_2], project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] ) assert video_2['$schema'] == video_schema # [[ ADD A FILE INSIDE VIDEO_1 ]] res = client.post( url_for('invenio_deposit_rest.video_files', pid_value=video_1_dict['metadata']['_deposit']['id']), data=data_file_1, content_type='multipart/form-data') # check returned value assert res.status_code == 201 file_1 = json.loads(res.data.decode('utf-8')) assert file_1['checksum'] == 'md5:eb88ae1e3666e6fe96a33ea72aab630e' assert file_1['filesize'] == 24 assert file_1['filename'] == 'test.json' assert file_1['id'] # check database: connection project <---> videos video_1 = deposit_video_resolver( video_1_dict['metadata']['_deposit']['id']) assert video_1['_files'][0]['key'] == 'test.json' video_ids = [ video_1_dict['metadata']['_deposit']['id'], video_2_dict['metadata']['_deposit']['id'] ] check_connection( deposit_videos_resolver(video_ids), project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] ) # [[ GET THE VIDEO 1 ]] res = client.get(video_1_dict['links']['self'], headers=json_headers) # check video metadata assert res.status_code == 200 video_1_dict = json.loads(res.data.decode('utf-8')) assert len(video_1_dict['metadata']['_files']) == 1 myfile = video_1_dict['metadata']['_files'][0] assert myfile['links']['self'].startswith('/api/files/') assert myfile['checksum'] == 'md5:eb88ae1e3666e6fe96a33ea72aab630e' assert myfile['completed'] is True assert 'version_id' in myfile assert myfile['key'] == 'test.json' assert myfile['size'] == 24 # [[ PUBLISH VIDEO_1 ]] # Not need to send _files del video_1_dict['metadata']['_files'] # Prepare video for publishing prepare_videos_for_publish([video_1, video_2]) res = client.post( url_for('invenio_deposit_rest.video_actions', pid_value=video_1['_deposit']['id'], action='publish'), headers=json_headers) # check returned value assert res.status_code == 202 video_1_dict = json.loads(res.data.decode('utf-8')) assert video_1_dict['metadata']['_deposit']['status'] == 'published' assert video_1_dict['metadata']['recid'] == 1 assert video_1_dict['metadata']['_project_id'] == project_id # check database: connection project <---> videos video_ids = [ video_1_dict['metadata']['_deposit']['id'], video_2_dict['metadata']['_deposit']['id'] ] check_connection( deposit_videos_resolver(video_ids), project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] ) # [[ ADD A VIDEO INSIDE VIDEO_2 ]] res = client.post( url_for('invenio_deposit_rest.video_files', pid_value=video_2_dict['metadata']['_deposit']['id']), data=data_file_2, content_type='multipart/form-data') # check returned value assert res.status_code == 201 file_2 = json.loads(res.data.decode('utf-8')) assert file_2['checksum'] == 'md5:95405c14852500dcbb6dbfd9e27a3594' assert file_2['filesize'] == 26 assert file_2['filename'] == 'test2.json' # check connection project <---> videos video_2 = deposit_video_resolver( video_2_dict['metadata']['_deposit']['id']) assert video_2['_files'][0]['key'] == 'test2.json' video_ids = [ video_1_dict['metadata']['_deposit']['id'], video_2_dict['metadata']['_deposit']['id'] ] check_connection( deposit_videos_resolver(video_ids), project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] ) # [[ PUBLISH THE PROJECT ]] res = client.post( url_for('invenio_deposit_rest.project_actions', pid_value=project['_deposit']['id'], action='publish'), headers=json_headers) def get_video_record(depid): deposit = deposit_video_resolver(depid) return Video.get_record(deposit.fetch_published()[1].id) video_1 = get_video_record(video_1_dict['metadata']['_deposit']['id']) video_2 = get_video_record(video_2_dict['metadata']['_deposit']['id']) record_videos = [video_1, video_2] # check returned value assert res.status_code == 202 project_dict = json.loads(res.data.decode('utf-8')) assert project_dict['metadata']['_deposit']['status'] == 'published' assert project_dict['metadata']['recid'] == 3 assert project_dict['metadata']['videos'][0] == record_videos[0] assert project_dict['metadata']['videos'][1] == record_videos[1] # check database: connection project <---> videos check_connection( record_videos, project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] ) # check indexed record RecordIndexer().process_bulk_queue() sleep(2) res = client.get(url_for('invenio_records_rest.recid_list', headers=json_headers)) assert res.status_code == 200 data = json.loads(res.data.decode('utf-8')) for hit in data['hits']['hits']: assert isinstance(hit['id'], int) # [[ EDIT THE PROJECT ]] res = client.post( url_for('invenio_deposit_rest.project_actions', pid_value=project_dict['metadata']['_deposit']['id'], action='edit'), headers=json_headers) # check returned value assert res.status_code == 201 project_dict = json.loads(res.data.decode('utf-8')) assert project_dict['metadata']['_deposit']['status'] == 'draft' # check database project = project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] assert project['_deposit']['status'] == 'draft' # [[ MODIFY PROJECT ]] project_before = project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] project_dict['metadata']['title']['title'] = 'new project title' # Not need to send _files del project_dict['metadata']['_files'] # try to modify preserved fields project_dict['metadata']['recid'] = 12323233 project_dict['metadata']['report_number'][0] = 'fuuu barrrr' project_dict['metadata']['publication_date'] = '2000-12-03' # do the call res = client.put( url_for('invenio_deposit_rest.project_item', pid_value=project_dict['metadata']['_deposit']['id']), data=json.dumps(project_dict['metadata']), headers=json_headers) # check returned value assert res.status_code == 200 project_dict = json.loads(res.data.decode('utf-8')) assert project_dict['metadata']['title']['title'] ==\ 'new project title' assert all(link.startswith('http://localhost/deposits/project') for (key, link) in project_dict['links'].items() if key not in ['html', 'bucket']) video_1 = get_video_record(video_1_dict['metadata']['_deposit']['id']) video_2 = get_video_record(video_2_dict['metadata']['_deposit']['id']) assert video_1 == project_dict['metadata']['videos'][0] assert video_2 == project_dict['metadata']['videos'][1] # check database project = project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] assert project['title']['title'] == 'new project title' # check preserved fields assert project_before['recid'] == project['recid'] assert project_before['report_number'] == project['report_number'] assert project_before[ 'publication_date'] == project['publication_date'] # [[ DISCARD PROJECT ]] res = client.post( url_for('invenio_deposit_rest.project_actions', pid_value=project_dict['metadata']['_deposit']['id'], action='discard'), headers=json_headers) # check returned value assert res.status_code == 201 project_dict = json.loads(res.data.decode('utf-8')) assert project_dict['metadata']['title']['title'] == 'my project' # check database project = project_resolver.resolve( project_dict['metadata']['_deposit']['id'])[1] assert project['title']['title'] == 'my project'
def test_migrate_record(frames_required, api_app, location, datadir, es, users): """Test migrate date.""" # [[ migrate the project ]] data = load_json(datadir, 'cds_records_demo_1_project.json') dump = CDSRecordDump(data=data[0]) project = CDSRecordDumpLoader.create(dump=dump) p_id = project.id assert project['$schema'] == Project.get_record_schema() assert project['publication_date'] == '2016-01-05' assert 'license' not in project assert 'copyright' not in project assert project['_cds'] == { "state": { "file_transcode": "SUCCESS", "file_video_extract_frames": "SUCCESS", "file_video_metadata_extraction": "SUCCESS" }, 'modified_by': users[0], } # check project deposit deposit_project_uuid = PersistentIdentifier.query.filter_by( pid_type='depid', object_type='rec').one().object_uuid deposit_project = Record.get_record(deposit_project_uuid) assert Project._schema in deposit_project['$schema'] assert project.revision_id == deposit_project[ '_deposit']['pid']['revision_id'] assert deposit_project['_deposit']['created_by'] == 1 assert deposit_project['_deposit']['owners'] == [1] assert deposit_project['_files'] == [] # [[ migrate the video ]] data = load_json(datadir, 'cds_records_demo_1_video.json') dump = CDSRecordDump(data=data[0]) db.session.commit() def check_symlinks(video): symlinks_creator = SymlinksCreator() files = list(symlinks_creator._get_list_files(record=video)) assert len(files) == 1 for file_ in files: path = symlinks_creator._build_link_path( symlinks_creator._symlinks_location, video, file_['key']) assert os.path.lexists(path) def check_gif(video, mock_gif): # called only once for deposit (_, _, mock_args) = mock_gif.mock_calls[0] # check gif record video = CDSRecord(dict(video), video.model) # check gif deposit deposit = deposit_video_resolver(video['_deposit']['id']) master_video = CDSVideosFilesIterator.get_master_video_file(deposit) assert mock_args['master_id'] == master_video['version_id'] assert str(deposit.files.bucket.id) == mock_args['bucket'] # assert mock_args['bucket'].id == deposit.files.bucket.id assert len(mock_args['frames']) == 10 assert 'output_dir' in mock_args migration_streams = get_migration_streams(datadir=datadir) with mock.patch.object(DataCiteProvider, 'register'), \ mock.patch.object(CDSRecordDumpLoader, '_create_frame', side_effect=get_frames), \ mock.patch.object(CDSRecordDumpLoader, '_get_minimum_frames', return_value=frames_required) as mock_frames, \ mock.patch.object( ExtractFramesTask, '_create_gif') as mock_gif, \ mock.patch.object( CDSRecordDumpLoader, '_get_migration_file_stream_and_size', side_effect=migration_streams), \ mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'): video = CDSRecordDumpLoader.create(dump=dump) assert mock_frames.called is True db.session.add(video.model) video_id = video.id # check smil file smil_obj = ObjectVersion.query.filter_by( key='CERN-MOVIE-2012-193-001.smil', is_head=True).one() storage = smil_obj.file.storage() assert '<video src' in storage.open().read().decode('utf-8') # check video symlinks check_symlinks(video) # check gif check_gif(video, mock_gif) # check project project = Record.get_record(p_id) assert project['videos'] == [ {'$ref': 'https://cds.cern.ch/api/record/1495143'} ] assert video['$schema'] == Video.get_record_schema() assert video['date'] == '2012-11-21' # metadata data assert video['publication_date'] == '2017-07-13' # creation date (DB) assert video['_project_id'] == '2093596' assert video['license'] == [{ 'license': 'CERN', 'url': 'http://copyright.web.cern.ch', }] assert video['copyright'] == { 'holder': 'CERN', 'year': '2012', 'url': 'http://copyright.web.cern.ch', } assert video['description'] == '' assert 'doi' in video assert video['_cds']['state'] == { "file_transcode": "SUCCESS", "file_video_extract_frames": "SUCCESS", "file_video_metadata_extraction": "SUCCESS" } assert 'extracted_metadata' in video['_cds'] def check_files(video): bucket = CDSRecordDumpLoader._get_bucket(record=video) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket)] for file_ in files: assert as_bucket(file_['bucket_id']) is not None assert 'checksum' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert FileInstance.query.filter_by( id=file_['file_id']) is not None assert 'key' in file_ assert 'links' in file_ assert 'content_type' in file_ assert 'context_type' in file_ assert 'media_type' in file_ assert 'tags' in file_ # check extracted metadata master_video = CDSVideosFilesIterator.get_master_video_file(video) assert any([key in master_video['tags'] for key in ExtractMetadataTask._all_keys]) assert any([key in video['_cds']['extracted_metadata'] for key in ExtractMetadataTask._all_keys]) def check_buckets(record, deposit): def get(key, record): bucket = CDSRecordDumpLoader._get_bucket(record=record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket)] return [file_[key] for file_ in files] def check(record, deposit, file_key, different=None): values_record = set(get(file_key, record)) values_deposit = set(get(file_key, deposit)) difference = len(values_record - values_deposit) assert different == difference def check_tag_master(record): bucket = CDSRecordDumpLoader._get_bucket(record=record) master = CDSVideosFilesIterator.get_master_video_file(record) files = [dump_object(obj) for obj in ObjectVersion.get_by_bucket(bucket=bucket) if obj.get_tags().get('master')] assert all([file_['tags']['master'] == master['version_id'] for file_ in files]) # 1 bucket record != 1 bucket deposit check(record, deposit, 'bucket_id', 1) # all file_id are the same except the smil file (only in record) check(record, deposit, 'file_id', 1) check(record, deposit, 'key', 1) # 18 object_version record != 17 object_version deposit check(record, deposit, 'version_id', 18) # check tag 'master' where is pointing check_tag_master(record) check_tag_master(deposit) def check_first_level_files(record): [master] = [file_ for file_ in deposit_video['_files'] if file_['context_type'] == 'master'] assert len(master['subformat']) == 5 assert len(master['frame']) == 10 # TODO assert len(master['playlist']) == ?? assert len([file_ for file_ in deposit_video['_files'] if file_['context_type'] == 'master']) == 1 duration = float(record['_cds']['extracted_metadata']['duration']) for frame in master['frame']: assert float(frame['tags']['timestamp']) < duration assert float(frame['tags']['timestamp']) > 0 # check tag 'preset_quality' pqs = [form['tags']['preset_quality'] for form in master['subformat']] assert sorted(pqs) == sorted(['1080p', '240p', '360p', '480p', '720p']) # check tag 'display_aspect_ratio' dar = set([form['tags']['display_aspect_ratio'] for form in master['subformat']]) assert dar == {'16:9'} def check_pids(record): """Check pids.""" assert record['report_number'][0] == 'CERN-VIDEO-2012-193-001' assert PersistentIdentifier.query.filter_by( pid_value='CERN-VIDEO-2012-193-001').count() == 1 assert PersistentIdentifier.query.filter_by( pid_value='CERN-MOVIE-2012-193-001').count() == 1 db.session.commit() # check video deposit deposit_video_uuid = PersistentIdentifier.query.filter( PersistentIdentifier.pid_type == 'depid', PersistentIdentifier.object_uuid != str(deposit_project_uuid), PersistentIdentifier.object_type == 'rec' ).one().object_uuid deposit_video = Video.get_record(str(deposit_video_uuid)) assert Video._schema in deposit_video['$schema'] video = Record.get_record(video_id) assert video.revision_id == deposit_video[ '_deposit']['pid']['revision_id'] assert deposit_video['_deposit']['created_by'] == users[0] assert deposit_video['_deposit']['owners'] == [users[0]] assert deposit_video['_project_id'] == '2093596' assert len(video['_files']) == 2 assert len(deposit_video['_files']) == 2 check_files(video) check_files(deposit_video) check_buckets(video, deposit_video) check_first_level_files(video) check_first_level_files(deposit_video) check_pids(video) # try to edit video deposit_video = deposit_video_resolver(deposit_video['_deposit']['id']) deposit_video = deposit_video.edit() # try to edit project deposit_project = deposit_project_resolver( deposit_project['_deposit']['id']) deposit_project = deposit_project.edit() login_user(User.query.filter_by(id=users[0]).first()) deposit_video['title']['title'] = 'test' deposit_video = deposit_video.publish() _, record_video = deposit_video.fetch_published() assert record_video['title']['title'] == 'test'