def check(project_status, video_1_status, video_2_status):
project = deposit_project_resolver(project_depid)
video_1 = deposit_video_resolver(video_1_depid)
video_2 = deposit_video_resolver(video_2_depid)
assert project.status == project_status
assert video_1.status == video_1_status
assert video_2.status == video_2_status
def test_project_publish_with_workflow(api_app, users, api_project, es):
"""Test publish a project with a workflow."""
project, video_1, video_2 = api_project
prepare_videos_for_publish([video_1, video_2])
project_depid = project['_deposit']['id']
project_id = str(project.id)
video_1_depid = video_1['_deposit']['id']
video_1_id = str(video_1.id)
video_2_depid = video_2['_deposit']['id']
receiver_id = 'test_project_publish_with_workflow'
workflow_receiver_video_failing(api_app,
db,
video_1,
receiver_id=receiver_id)
headers = [('Content-Type', 'application/json')]
payload = json.dumps(dict(somekey='somevalue'))
with mock.patch('invenio_indexer.tasks.index_record.delay') \
as mock_indexer, \
api_app.test_request_context(headers=headers, data=payload):
event = Event.create(receiver_id=receiver_id)
db.session.add(event)
event.process()
# check video and project are indexed
assert mock_indexer.called is True
ids = get_indexed_records_from_mock(mock_indexer)
assert video_1_id == ids[0]
assert project_id == ids[1]
db.session.commit()
# check tasks status is propagated to video and project
video_1 = deposit_video_resolver(video_1_depid)
expected = {u'add': u'SUCCESS', u'failing': u'FAILURE'}
assert video_1['_cds']['state'] == expected
assert video_1.project['_cds']['state'] == expected
events = get_deposit_events(deposit_id=video_1_depid)
assert len(events) == 1
def check(project_status, video_1_status, video_2_status):
project = deposit_project_resolver(project_depid)
video_1 = deposit_video_resolver(video_1_depid)
video_2 = deposit_video_resolver(video_2_depid)
assert project.status == project_status
assert video_1.status == video_1_status
assert video_2.status == video_2_status
check('draft', 'draft', 'draft')
login_user(User.query.get(users[0]))
video_2 = deposit_video_resolver(video_2_depid)
video_2.publish()
check('draft', 'draft', 'published')
project = deposit_project_resolver(project_depid)
project.publish()
check('published', 'published', 'published')
def test_deposit_vtt_tags(api_app, db, api_project, users):
"""Test VTT tag generation."""
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
# insert a master file inside the video
add_master_to_video(
video_deposit=video_1,
filename='test.mp4',
stream=BytesIO(b'1234'), video_duration="15"
)
# try to insert a new vtt object
obj = ObjectVersion.create(
video_1._bucket, key="test_fr.vtt",
stream=BytesIO(b'hello'))
# publish the video
prepare_videos_for_publish([video_1])
video_1 = deposit_video_resolver(video_1_depid)
login_user(User.query.get(users[0]))
video_1 = video_1.publish()
# check tags
check_object_tags(obj, video_1, content_type='vtt', media_type='subtitle',
context_type='subtitle', language='fr')
# edit the video
video_1 = video_1.edit()
# try to delete the old vtt file and substitute with a new one
video_1 = deposit_video_resolver(video_1_depid)
ObjectVersion.delete(bucket=video_1._bucket, key=obj.key)
obj2 = ObjectVersion.create(
video_1._bucket, key="test_en.vtt", stream=BytesIO(b'hello'))
# publish again the video
video_1 = video_1.publish()
# check tags
check_object_tags(obj2, video_1, content_type='vtt', media_type='subtitle',
context_type='subtitle', language='en')
# edit a re-published video
video_1 = video_1.edit()
# add a new vtt file
obj3 = ObjectVersion.create(
video_1._bucket, key="test_it.vtt", stream=BytesIO(b'hello'))
# publish again the video
video_1 = video_1.publish()
# check tags
check_object_tags(obj3, video_1, content_type='vtt', media_type='subtitle',
context_type='subtitle', language='it')
def test_deposit_poster_tags(api_app, db, api_project, users):
"""Test poster tag generation."""
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
master_video_filename = 'test.mp4'
poster_filename = 'poster.jpg'
poster_filename2 = 'poster.png'
# insert a master file inside the video
add_master_to_video(video_deposit=video_1,
filename=master_video_filename,
stream=BytesIO(b'1234'),
video_duration='15')
# try to insert a new vtt object
obj = ObjectVersion.create(video_1._bucket,
key=poster_filename,
stream=BytesIO(b'hello'))
# publish the video
prepare_videos_for_publish([video_1])
video_1 = deposit_video_resolver(video_1_depid)
login_user(User.query.get(users[0]))
video_1 = video_1.publish()
# check tags
check_object_tags(obj,
video_1,
content_type='jpg',
context_type='poster',
media_type='image')
# edit the video
video_1 = video_1.edit()
# try to delete the old poster frame and substitute with a new one
video_1 = deposit_video_resolver(video_1_depid)
ObjectVersion.delete(bucket=video_1._bucket, key=obj.key)
obj2 = ObjectVersion.create(video_1._bucket,
key=poster_filename2,
stream=BytesIO(b'hello'))
# publish again the video
video_1 = video_1.publish()
# check tags
check_object_tags(obj2,
video_1,
content_type='png',
context_type='poster',
media_type='image')
def test_video_delete_with_workflow(api_app, users, api_project, webhooks, es):
"""Test publish a project with a workflow."""
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
receiver_id = 'test_video_delete_with_workflow'
workflow_receiver_video_failing(api_app,
db,
video_1,
receiver_id=receiver_id)
mock_delete = MagicMock(return_value=None)
current_webhooks.receivers[receiver_id].delete = mock_delete
headers = [('Content-Type', 'application/json')]
payload = json.dumps(dict(somekey='somevalue'))
with api_app.test_request_context(headers=headers, data=payload):
event = Event.create(receiver_id=receiver_id)
db.session.add(event)
event.process()
db.session.commit()
video_1 = deposit_video_resolver(video_1_depid)
video_1.delete()
assert mock_delete.called is True
def test_sync_records_with_deposits(app, db, location, users,
project_deposit_metadata,
video_deposit_metadata):
"""Test sync records with deposits task."""
# create a project
project = Project.create(project_deposit_metadata)
project_deposit_metadata['report_number'] = ['123']
# create new video
video_deposit_metadata['_project_id'] = project['_deposit']['id']
deposit = Video.create(video_deposit_metadata)
depid = deposit['_deposit']['id']
# insert objects inside the deposit
ObjectVersion.create(deposit.files.bucket,
"obj_1").set_location("mylocation1", 1, "mychecksum1")
ObjectVersion.create(deposit.files.bucket,
"obj_2").set_location("mylocation2", 1, "mychecksum2")
ObjectVersion.create(deposit.files.bucket,
"obj_3").set_location("mylocation3", 1, "mychecksum3")
obj_4 = ObjectVersion.create(deposit.files.bucket,
"obj_4").set_location("mylocation4", 1,
"mychecksum4")
# publish
login_user(User.query.get(users[0]))
prepare_videos_for_publish([deposit])
deposit = deposit.publish()
_, record = deposit.fetch_published()
assert deposit.is_published() is True
# add a new object
ObjectVersion.create(deposit.files.bucket,
"obj_new").set_location("mylocation_new", 1,
"mychecksum")
# modify obj_1
ObjectVersion.create(deposit.files.bucket,
"obj_new").set_location("mylocation2.1", 1,
"mychecksum2.1")
# delete obj_3
ObjectVersion.delete(deposit.files.bucket, "obj_3")
# remove obj_4
obj_4.remove()
# check video and record
files = ['obj_1', 'obj_2', 'obj_3', 'obj_4']
edited_files = ['obj_1', 'obj_2', 'obj_3', 'obj_new']
check_deposit_record_files(deposit, edited_files, record, files)
# try to sync deposit and record
sync_records_with_deposit_files.s(deposit_id=depid).apply_async()
# get deposit and record
deposit = deposit_video_resolver(depid)
_, record = deposit.fetch_published()
assert deposit.is_published() is True
# check that record and deposit are sync
re_edited_files = edited_files + ['obj_4']
check_deposit_record_files(deposit, edited_files, record, re_edited_files)
def _resolve_deposit(id_type, id_value):
"""Return the deposit video."""
depid = id_value
if id_type == 'recid':
_, record = record_resolver.resolve(id_value)
depid = record['_deposit']['id']
return deposit_video_resolver(depid), depid
def test_record_video_links(datacite_mock, api_app, es, api_project, users,
json_headers):
"""Test record video links."""
(project, video_1, video_2) = api_project
user = User.query.filter_by(id=users[0]).first()
prepare_videos_for_publish([video_1, video_2])
vid = video_1['_deposit']['id']
pid = project['_deposit']['id']
with api_app.test_client() as client:
login_user_via_session(client, user)
# publish video
url = url_for('invenio_deposit_rest.video_actions',
pid_value=vid,
action='publish')
assert client.post(url).status_code == 202
rec_pid, rec_video = deposit_video_resolver(vid).fetch_published()
# get a record video (with no published project)
url = url_for('invenio_records_rest.recid_item',
pid_value=rec_pid.pid_value,
_external=True)
res = client.get(url, headers=json_headers)
assert res.status_code == 200
# check video record
data = json.loads(res.data.decode('utf-8'))
assert data['links'] == {'self': url}
# publish the project
url = url_for('invenio_deposit_rest.project_actions',
pid_value=pid,
action='publish')
assert client.post(url).status_code == 202
rec_pid_proj, rec_proj = video_1.project.fetch_published()
# get a record video (with published project)
url = url_for('invenio_records_rest.recid_item',
pid_value=rec_pid.pid_value,
_external=True)
res = client.get(url, headers=json_headers)
assert res.status_code == 200
# check video record
data = json.loads(res.data.decode('utf-8'))
url_api_prj = 'http://localhost/record/3'
url_prj = 'http://localhost/record/3'
url_prj_edit = 'http://localhost/deposit/project/{0}'.format(pid)
assert data['links'] == {
'self': url,
'project': url_api_prj,
'project_html': url_prj,
'project_edit': url_prj_edit,
}
def test_video_publish_registering_the_datacite_not_local(
datacite_mock, api_app, users, location, cds_jsonresolver,
json_headers, json_partial_project_headers, json_partial_video_headers,
deposit_metadata, video_deposit_metadata, project_deposit_metadata,
keyword_1, keyword_2):
"""Test video publish registering the datacite not local."""
# test: enable datacite registration
api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
project_deposit_metadata['keywords'] = [copy.deepcopy(keyword_2)]
# [[ CREATE NEW PROJECT ]]
project_dict = _create_new_project(client,
json_partial_project_headers,
project_deposit_metadata)
assert project_dict['metadata']['keywords'][0] == keyword_2
project_depid = project_dict['metadata']['_deposit']['id']
project = deposit_project_resolver(project_depid)
assert project['keywords'] == [{
'$ref': keyword_2.ref,
'name': keyword_2['name'],
}]
# [[ ADD A NEW VIDEO_1 ]]
video_metadata = copy.deepcopy(video_deposit_metadata)
video_metadata['keywords'] = [copy.deepcopy(keyword_1)]
video_metadata.update(
_project_id=project_dict['metadata']['_deposit']['id'])
res = client.post(url_for('invenio_deposit_rest.video_list'),
data=json.dumps(video_metadata),
headers=json_partial_video_headers)
assert res.status_code == 201
video_1_dict = json.loads(res.data.decode('utf-8'))
assert video_1_dict['metadata']['keywords'][0] == keyword_1
video_1_depid = video_1_dict['metadata']['_deposit']['id']
video_1 = deposit_video_resolver(video_1_depid)
assert video_1['keywords'] == [{
'$ref': keyword_1.ref,
'name': keyword_1['name'],
}]
video_1['doi'] = '10.1123/doi'
prepare_videos_for_publish([video_1])
# [[ PUBLISH VIDEO ]]
video_1.publish()
datacite_register.s(pid_value='123',
record_uuid=str(video_1.id)).apply()
assert datacite_mock.called is False
def check_gif(video, mock_gif):
# called only once for deposit
(_, _, mock_args) = mock_gif.mock_calls[0]
# check gif record
video = CDSRecord(dict(video), video.model)
# check gif deposit
deposit = deposit_video_resolver(video['_deposit']['id'])
master_video = CDSVideosFilesIterator.get_master_video_file(deposit)
assert mock_args['master_id'] == master_video['version_id']
assert str(deposit.files.bucket.id) == mock_args['bucket']
# assert mock_args['bucket'].id == deposit.files.bucket.id
assert len(mock_args['frames']) == 10
assert 'output_dir' in mock_args
def test_video_publish_with_no_type(api_project):
"""Test video publish with no type."""
(project, video_1, video_2) = api_project
prepare_videos_for_publish([video_1, video_2])
video_1_depid = video_1['_deposit']['id']
# test: no type in project
del project['type']
assert 'category' in project
project.commit()
db.session.commit()
video_1 = deposit_video_resolver(video_1_depid)
with pytest.raises(ValidationError):
video_1.publish()
def test_video_publish_with_category_and_type(api_project):
"""Test video publish with category and type."""
(project, video_1, video_2) = api_project
prepare_videos_for_publish([video_1, video_2])
video_1_depid = video_1['_deposit']['id']
# test with category + type
assert 'type' in project
assert 'category' in project
project.commit()
db.session.commit()
video_1 = deposit_video_resolver(video_1_depid)
video_1.publish()
assert video_1['_deposit']['status'] == 'published'
def extract_frames(recid, depid):
"""Re-trigger the extract frames task."""
if not recid and not depid:
raise ClickException('Missing option "--recid" or "--depid"')
if recid:
_, record = record_resolver.resolve(recid)
depid = record['_deposit']['id']
video_deposit = deposit_video_resolver(depid)
master = CDSVideosFilesIterator.get_master_video_file(video_deposit)
ExtractFramesTask().si(version_id=master['version_id'],
deposit_id=depid).apply_async()
def test_boolean_fields_are_indexed(api_app, es, api_project, users,
json_headers):
"""Test boolean fields (i.e. featured and vr) are indexed."""
(project, video_1, video_2) = api_project
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
# [[ PUBLISH THE PROJECT ]]
prepare_videos_for_publish([video_1, video_2])
client.post(
url_for('invenio_deposit_rest.project_actions',
pid_value=project['_deposit']['id'], action='publish'),
headers=json_headers)
RecordIndexer().process_bulk_queue()
sleep(2)
video_1_record = deposit_video_resolver(video_1['_deposit']['id'])
video_2_record = deposit_video_resolver(video_2['_deposit']['id'])
def assert_get_video(query, video_record):
url = url_for('invenio_records_rest.recid_list', q=query)
res = client.get(url, headers=json_headers)
assert res.status_code == 200
data = json.loads(res.data.decode('utf-8'))
assert len(data['hits']['hits']) == 1
assert data['hits']['hits'][0]['id'] == video_record['recid']
# Featured
assert_get_video('featured:true', video_1_record)
# Not featured
assert_get_video('featured:false', video_2_record)
# VR
assert_get_video('vr:true', video_1_record)
# Not VR
assert_get_video('vr:false', video_2_record)
def test_video_publish_registering_the_datacite(
datacite_mock, api_app, users, location, cds_jsonresolver,
json_headers, json_partial_project_headers, json_partial_video_headers,
deposit_metadata, video_deposit_metadata, project_deposit_metadata):
"""Test video publish registering the datacite."""
# test: enable datacite registration
api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
# [[ CREATE NEW PROJECT ]]
project_dict = _create_new_project(client,
json_partial_project_headers,
project_deposit_metadata)
# [[ ADD A NEW EMPTY VIDEO_1 ]]
video_1_dict = _add_video_info_to_project(client,
json_partial_video_headers,
project_dict,
video_deposit_metadata)
video_1_depid = video_1_dict['metadata']['_deposit']['id']
video_1 = deposit_video_resolver(video_1_depid)
prepare_videos_for_publish([video_1])
# [[ PUBLISH VIDEO ]]
video_1.publish()
# [[ REGISTER DATACITE ]]
datacite_register_after_publish(sender=api_app,
action='publish',
deposit=video_1)
assert datacite_mock.called is True
assert datacite_mock().metadata_post.call_count == 1
datacite_mock().doi_post.assert_called_once_with(
'10.0000/cds.1', 'https://cds.cern.ch/record/1')
# [[ UPDATE DATACITE ]]
datacite_register_after_publish(sender=api_app,
action='publish',
deposit=video_1)
assert datacite_mock.called is True
assert datacite_mock().metadata_post.call_count == 2
datacite_mock().doi_post.assert_called_with(
'10.0000/cds.1', 'https://cds.cern.ch/record/1')
def run(self, preset_quality, sleep_time=5, *args, **kwargs):
super(MaintenanceTranscodeVideoTask,
self).run(preset_quality=preset_quality,
sleep_time=sleep_time,
*args,
**kwargs)
logger.debug("Updating deposit and record")
# get deposit and record
dep_video = deposit_video_resolver(self.deposit_id)
rec_video = record_resolver.resolve(dep_video['recid'])[1]
# sync deposit --> record
dep_video._sync_record_files(record=rec_video)
dep_video.commit()
rec_video.commit()
db.session.commit()
def test_deposit_smil_tag_generation(api_app, db, api_project, users):
"""Test AVCWorkflow receiver."""
def check_smil(video):
_, record = video.fetch_published()
master = CDSVideosFilesIterator.get_master_video_file(record)
playlist = master['playlist']
assert playlist[0]['key'] == '{}.smil'. \
format(record['report_number'][0])
assert playlist[0]['content_type'] == 'smil'
assert playlist[0]['context_type'] == 'playlist'
assert playlist[0]['media_type'] == 'text'
assert playlist[0]['tags']['master'] == master['version_id']
# check bucket dump is done correctly
master_video = CDSVideosFilesIterator.get_master_video_file(video)
assert master_video['version_id'] != master['version_id']
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
# insert a master file inside the video
add_master_to_video(video_deposit=video_1,
filename='test.mp4',
stream=BytesIO(b'1234'),
video_duration="15s")
# publish the video
prepare_videos_for_publish([video_1])
video_1 = deposit_video_resolver(video_1_depid)
login_user(User.query.get(users[0]))
video_1 = video_1.publish()
# check smil
check_smil(video_1)
# edit the video
video_1 = video_1.edit()
# publish again the video
video_1 = video_1.publish()
# check smil
check_smil(video_1)
def test_avc_workflow_delete(api_app, db, api_project, users, access_token,
json_headers, mock_sorenson, online_video,
webhooks, checker):
"""Test AVCWorkflow receiver REST API."""
project, video_1, video_2 = api_project
video_1_id = video_1.id
video_1_depid = video_1['_deposit']['id']
master_key = 'test.mp4'
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_list',
receiver_id='avc',
access_token=access_token)
with api_app.test_client() as client, \
mock.patch('invenio_sse.ext._SSEState.publish'), \
mock.patch('invenio_indexer.api.RecordIndexer.bulk_index'):
sse_channel = 'mychannel'
payload = dict(
uri=online_video,
deposit_id=video_1_depid,
key=master_key,
sse_channel=sse_channel,
sleep_time=0,
)
resp = client.post(url, headers=json_headers, data=json.dumps(payload))
assert resp.status_code == 201
data = json.loads(resp.data.decode('utf-8'))
# check extracted metadata is there
record = RecordMetadata.query.get(video_1_id)
assert 'extracted_metadata' in record.json['_cds']
assert ObjectVersion.query.count() == get_object_count()
assert ObjectVersionTag.query.count() == get_tag_count()
event_id = data['tags']['_event_id']
video_1_id = str(deposit_video_resolver(video_1_depid).id)
###
checker(api_app, event_id, access_token, json_headers, data, video_1_id,
video_1_depid, users)
def test_video_name_after_publish(api_app, db, api_project, users):
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
master_video_filename = 'test.mp4'
# insert a master file inside the video
add_master_to_video(video_deposit=video_1,
filename=master_video_filename,
stream=BytesIO(b'1234'),
video_duration='15')
# publish the video
prepare_videos_for_publish([video_1])
video_1 = deposit_video_resolver(video_1_depid)
login_user(User.query.get(users[0]))
video_1 = video_1.publish()
_, record = video_1.fetch_published()
master = CDSVideosFilesIterator.get_master_video_file(record)
assert master['key'] == '{}.mp4'.format(record['report_number'][0])
def test_project_edit(app, project_published, users):
"""Test project edit."""
(project, video_1, video_2) = project_published
assert project.status == 'published'
assert video_1.status == 'published'
assert video_2.status == 'published'
# Edit project (change project title)
new_project = project.edit()
assert new_project.status == 'draft'
new_project.update(title={'title': 'My project'})
# Edit videos inside project (change video titles)
videos = [
deposit_video_resolver(record_video_resolver(id_)['_deposit']['id'])
for id_ in new_project.video_ids
]
login_user(User.query.get(users[0]))
assert len(videos) == 2
for i, video in enumerate(videos):
# video = Video.get_record(video.id)
assert video['_deposit']['status'] == 'published'
new_video = video.edit()
assert new_video['_deposit']['status'] == 'draft'
new_video.update(title={'title': 'Video {}'.format(i + 1)})
new_video.publish()
# Publish all changes
new_project.publish()
# Check that everything is published
videos = [record_video_resolver(id_) for id_ in new_project.video_ids]
assert new_project['_deposit']['status'] == 'published'
assert all(video['_deposit']['status'] == 'published' for video in videos)
# Check that all titles where properly changed
assert new_project['title']['title'] == 'My project'
assert videos[0]['title']['title'] in ['Video 1', 'Video 2']
assert videos[1]['title']['title'] in ['Video 1', 'Video 2']
assert videos[0]['title']['title'] != videos[1]['title']['title']
def test_video_publish_registering_the_datacite_if_fail(
datacite_mock, api_app, users, location, cds_jsonresolver,
json_headers, json_partial_project_headers, json_partial_video_headers,
deposit_metadata, video_deposit_metadata, project_deposit_metadata):
"""Test video publish registering the datacite."""
# test: enable datacite registration
api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
# [[ CREATE NEW PROJECT ]]
project_dict = _create_new_project(client,
json_partial_project_headers,
project_deposit_metadata)
# [[ ADD A NEW EMPTY VIDEO_1 ]]
video_1_dict = _add_video_info_to_project(client,
json_partial_video_headers,
project_dict,
video_deposit_metadata)
video_1_depid = video_1_dict['metadata']['_deposit']['id']
video_1 = deposit_video_resolver(video_1_depid)
prepare_videos_for_publish([video_1])
# [[ PUBLISH VIDEO ]]
video_1.publish()
db.session.commit()
with mock.patch('invenio_records.api.Record.get_record',
side_effect=[Exception, video_1],
return_value=video_1):
with pytest.raises(Retry):
datacite_register.s(pid_value='1',
record_uuid=str(video_1.id)).apply()
assert datacite_mock.called is True
assert datacite_mock().metadata_post.call_count == 1
datacite_mock().doi_post.assert_called_once_with(
'10.0000/cds.1', 'https://cds.cern.ch/record/1')
assert datacite_mock.call_count == 3
def test_subformat_creation_if_missing(api_app, location, datadir, es, users):
"""Test subformat creation if missing."""
# [[ migrate the video ]]
migration_streams = get_migration_streams(datadir=datadir)
data = load_json(datadir, 'cds_records_demo_1_video.json')
dump = CDSRecordDump(data=data[0])
with mock.patch.object(DataCiteProvider, 'register'), \
mock.patch.object(CDSRecordDumpLoader, '_create_frame',
side_effect=get_frames), \
mock.patch.object(ExtractFramesTask, '_create_gif'), \
mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'), \
mock.patch.object(
CDSRecordDumpLoader, '_get_migration_file_stream_and_size',
side_effect=migration_streams):
video = CDSRecordDumpLoader.create(dump=dump)
db.session.commit()
with mock.patch.object(TranscodeVideoTask, 'run') as mock_transcode:
deposit = deposit_video_resolver(video['_deposit']['id'])
deposit_id = deposit.id
# simulate the missing of a subformat
del deposit['_files'][0]['subformat'][0]
assert len(deposit['_files'][0]['subformat']) == 4
# recreate 240p format
CDSRecordDumpLoader._create_missing_subformats(
record=video, deposit=deposit)
db.session.commit()
# check subformats
deposit = Video.get_record(deposit_id)
rec_video = record_resolver.resolve(video['recid'])[1]
# rec_video = record_resolver.resolve(video['recid'])[1]
assert len(deposit['_files'][0]['subformat']) == 5
assert len(rec_video['_files'][0]['subformat']) == 5
# check if transcoding is called properly
assert mock_transcode.called is True
[(_, call_args)] = mock_transcode.call_args_list
assert call_args == {'preset_quality': '240p'}
def test_video_publish_edit_publish_again(
datacite_mock, es, api_app, users, location, cds_jsonresolver,
json_headers, json_partial_project_headers, json_partial_video_headers,
deposit_metadata, video_deposit_metadata, project_deposit_metadata):
"""Test video publish registering the datacite not local."""
# test: enable datacite registration
api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True
api_app.config['PIDSTORE_DATACITE_DOI_PREFIX'] = '10.5072'
with api_app.test_request_context():
with api_app.test_client() as client:
login_user_via_session(client,
email=User.query.get(users[0]).email)
# [[ CREATE A NEW PROJECT ]]
project_dict = _create_new_project(client,
json_partial_project_headers,
project_deposit_metadata)
# [[ ADD A NEW EMPTY VIDEO_1 ]]
video_1_dict = _add_video_info_to_project(
client, json_partial_video_headers, project_dict,
video_deposit_metadata)
video_1_depid = video_1_dict['metadata']['_deposit']['id']
video_1 = deposit_video_resolver(video_1_depid)
prepare_videos_for_publish([video_1])
# [[ PUBLISH VIDEO ]]
_deposit_publish(client, json_headers, video_1['_deposit']['id'])
datacite_register.s(pid_value='123',
record_uuid=str(video_1.id)).apply()
# [[ EDIT VIDEO ]]
_deposit_edit(client, json_headers, video_1['_deposit']['id'])
# [[ MODIFY DOI -> SAVE ]]
video_1 = deposit_video_resolver(video_1_depid)
video_1_dict = copy.deepcopy(video_1)
# old_doi = video_1_dict['doi']
video_1_dict['doi'] = '10.1123/doi'
del video_1_dict['_files']
res = client.put(url_for('invenio_deposit_rest.video_item',
pid_value=video_1['_deposit']['id']),
data=json.dumps(video_1_dict),
headers=json_headers)
# check returned value
assert res.status_code == 400
data = json.loads(res.data.decode('utf-8'))
assert data['errors'] == [{
"field": "doi",
"message": "The DOI cannot be changed."
}]
video_1 = deposit_video_resolver(video_1_depid)
# video_1['doi'] = old_doi
video_1_dict = copy.deepcopy(video_1)
del video_1_dict['_files']
# try to modify preserved fields
video_1_dict['recid'] = 12323233
video_1_dict['report_number'] = ['fuuu barrrr']
video_1_dict['publication_date'] = '2000-12-03'
video_1_dict['_project_id'] = '1234567'
# do the call
res = client.put(url_for('invenio_deposit_rest.video_item',
pid_value=video_1['_deposit']['id']),
data=json.dumps(video_1_dict),
headers=json_headers)
# check returned value
assert res.status_code == 200
# check preserved fields
video_1_new = deposit_video_resolver(video_1_depid)
assert video_1_new['recid'] == video_1['recid']
assert video_1_new['report_number'] == video_1['report_number']
assert video_1_new['publication_date'] == video_1[
'publication_date']
assert video_1_new['_project_id'] == video_1['_project_id']
# [[ PUBLISH VIDEO ]]
_deposit_publish(client, json_headers, video_1['_deposit']['id'])
# check indexed record
RecordIndexer().process_bulk_queue()
sleep(2)
res = client.get(
url_for('invenio_records_rest.recid_list',
headers=json_headers))
assert res.status_code == 200
data = json.loads(res.data.decode('utf-8'))
for hit in data['hits']['hits']:
assert isinstance(hit['id'], int)
def test_download_receiver(api_app, db, api_project, access_token, webhooks,
json_headers):
"""Test downloader receiver."""
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
video_1_id = str(video_1.id)
project_id = str(project.id)
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_list',
receiver_id='downloader',
access_token=access_token)
with mock.patch('requests.get') as mock_request, \
mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \
mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \
as mock_indexer, \
api_app.test_client() as client:
sse_channel = 'mychannel'
mock_sse.return_value = None
file_size = 1024
mock_request.return_value = type(
'Response', (object, ), {
'raw': BytesIO(b'\x00' * file_size),
'headers': {
'Content-Length': file_size
}
})
payload = dict(uri='http://example.com/test.pdf',
deposit_id=video_1_depid,
key='test.pdf',
sse_channel=sse_channel)
resp = client.post(url, headers=json_headers, data=json.dumps(payload))
assert resp.status_code == 201
data = json.loads(resp.data.decode('utf-8'))
assert '_tasks' in data
assert data['tags']['uri_origin'] == 'http://example.com/test.pdf'
assert data['key'] == 'test.pdf'
assert 'version_id' in data
assert 'links' in data # TODO decide with links are needed
assert all(
[link in data['links'] for link in ['self', 'version', 'cancel']])
assert ObjectVersion.query.count() == 1
obj = ObjectVersion.query.first()
tags = obj.get_tags()
assert tags['_event_id'] == data['tags']['_event_id']
assert obj.key == data['key']
assert str(obj.version_id) == data['version_id']
assert obj.file
assert obj.file.size == file_size
# check sse is called
assert mock_sse.called
def set_data(state, message, size, total, percentage, type_):
return {
'state': state,
'meta': {
'message': message,
'payload': {
'event_id': str(tags['_event_id']),
'key': u'test.pdf',
'tags': {
u'uri_origin': u'http://example.com/test.pdf',
u'_event_id': str(tags['_event_id']),
u'context_type': u'master',
},
'deposit_id': video_1_depid,
'percentage': percentage,
'version_id': str(obj.version_id),
'size': size,
'total': total,
'sse_channel': sse_channel,
'type': type_
}
}
}
assert mock_sse.call_count == 7
mock_sse.assert_any_call(data=set_data(
states.STARTED,
'Downloading {} of {}'.format(file_size, file_size), file_size,
file_size, 100, 'file_download'),
channel=u'mychannel',
type_='file_download')
mock_sse.assert_any_call(data=set_data(states.SUCCESS,
str(obj.version_id), file_size,
file_size, 100,
'file_download'),
channel=u'mychannel',
type_='file_download')
deposit = deposit_video_resolver(video_1_depid)
mock_sse.assert_any_call(
channel='mychannel',
data={
'state': states.SUCCESS,
'meta': {
'payload': {
'event_id': str(tags['_event_id']),
'deposit_id': video_1_depid,
'deposit': deposit,
}
}
},
type_='update_deposit',
)
# check ElasticSearch is called
ids = set(get_indexed_records_from_mock(mock_indexer))
assert video_1_id in ids
assert project_id in ids
assert deposit['_cds']['state'] == {u'file_download': states.SUCCESS}
# Test cleaning!
url = '{0}?access_token={1}'.format(data['links']['cancel'], access_token)
with mock.patch('requests.get') as mock_request, \
mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \
mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \
as mock_indexer, \
api_app.test_client() as client:
resp = client.delete(url, headers=json_headers)
assert resp.status_code == 201
assert ObjectVersion.query.count() == 2
bucket = Bucket.query.first()
assert bucket.size == 0
assert mock_sse.called is False
assert mock_indexer.called is False
def get_video_record(depid):
deposit = deposit_video_resolver(depid)
return Video.get_record(deposit.fetch_published()[1].id)
def test_video_events_on_workflow(webhooks, api_app, db, api_project, bucket,
access_token, json_headers):
"""Test deposit events."""
(project, video_1, video_2) = api_project
project_depid = project['_deposit']['id']
video_1_depid = video_1['_deposit']['id']
db.session.add(bucket)
# registering receiver
receiver_id = 'test_video_events_on_workflow'
workflow_receiver_video_failing(api_app,
db,
video_1,
receiver_id=receiver_id)
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_list',
receiver_id=receiver_id,
access_token=access_token)
with api_app.test_client() as client:
# run workflow
resp = client.post(url, headers=json_headers, data=json.dumps({}))
assert resp.status_code == 500
# run again workflow
resp = client.post(url, headers=json_headers, data=json.dumps({}))
assert resp.status_code == 500
# resolve deposit and events
deposit = deposit_video_resolver(video_1_depid)
events = get_deposit_events(deposit['_deposit']['id'])
# check events
assert len(events) == 2
assert events[0].payload['deposit_id'] == video_1_depid
assert events[1].payload['deposit_id'] == video_1_depid
# check computed status
status = get_tasks_status_by_task(events)
assert status['add'] == states.SUCCESS
assert status['failing'] == states.FAILURE
# check every task for every event
for event in events:
result = event.receiver._deserialize_result(event)
assert result.parent.status == states.SUCCESS
assert result.children[0].status == states.FAILURE
assert result.children[1].status == states.SUCCESS
# check if the states are inside the deposit
res = client.get(url_for('invenio_deposit_rest.video_item',
pid_value=video_1_depid,
access_token=access_token),
headers=json_headers)
assert res.status_code == 200
data = json.loads(res.data.decode('utf-8'))['metadata']
assert data['_cds']['state']['add'] == states.SUCCESS
assert data['_cds']['state']['failing'] == states.FAILURE
# run indexer
RecordIndexer().process_bulk_queue()
sleep(2)
# check elasticsearch video_1 state
resp = client.get(url_for('invenio_deposit_rest.video_list',
q='_deposit.id:{0}'.format(video_1_depid),
access_token=access_token),
headers=json_headers)
assert resp.status_code == 200
data = json.loads(resp.data.decode('utf-8'))
status = data['hits']['hits'][0]['metadata']['_cds']['state']
assert status['add'] == states.SUCCESS
assert status['failing'] == states.FAILURE
# check elasticsearch project state
resp = client.get(url_for('invenio_deposit_rest.video_list',
q='_deposit.id:{0}'.format(project_depid),
access_token=access_token),
headers=json_headers)
assert resp.status_code == 200
data = json.loads(resp.data.decode('utf-8'))
status = data['hits']['hits'][0]['metadata']['_cds']['state']
assert status['add'] == states.SUCCESS
assert status['failing'] == states.FAILURE
def test_video_events_on_download_create(api_app, webhooks, db, api_project,
access_token, json_headers):
"""Test deposit events."""
(project, video_1, video_2) = api_project
video_1_depid = video_1['_deposit']['id']
project_id = str(project.id)
video_1_id = str(video_1.id)
bucket_id = video_1._bucket.id
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_list',
receiver_id='downloader',
access_token=access_token)
with mock.patch('requests.get') as mock_request, \
mock.patch('invenio_indexer.tasks.index_record.delay') \
as mock_indexer, \
api_app.test_client() as client:
file_size = 1024 * 1024
mock_request.return_value = type(
'Response', (object, ), {
'raw': BytesIO(b'\x00' * file_size),
'headers': {
'Content-Length': file_size
}
})
payload = dict(uri='http://example.com/test.pdf',
bucket_id=str(bucket_id),
deposit_id=video_1_depid,
key='test.pdf')
resp = client.post(url, headers=json_headers, data=json.dumps(payload))
assert resp.status_code == 201
file_size = 1024 * 1024 * 6
mock_request.return_value = type(
'Response', (object, ), {
'raw': BytesIO(b'\x00' * file_size),
'headers': {
'Content-Length': file_size
}
})
resp = client.post(url, headers=json_headers, data=json.dumps(payload))
assert resp.status_code == 201
deposit = deposit_video_resolver(video_1_depid)
events = get_deposit_events(deposit['_deposit']['id'])
assert len(events) == 2
assert events[0].payload['deposit_id'] == video_1_depid
assert events[1].payload['deposit_id'] == video_1_depid
status = get_tasks_status_by_task(events)
assert status == {'file_download': states.SUCCESS}
# check if the states are inside the deposit
res = client.get(url_for('invenio_deposit_rest.video_item',
pid_value=video_1_depid,
access_token=access_token),
headers=json_headers)
assert res.status_code == 200
data = json.loads(res.data.decode('utf-8'))['metadata']
assert data['_cds']['state']['file_download'] == states.SUCCESS
assert deposit._get_files_dump() == data['_files']
# check the record is inside the indexer queue
ids = set(get_indexed_records_from_mock(mock_indexer))
assert len(ids) == 2
assert video_1_id in ids
assert project_id in ids
def test_video_events_on_download_check_index(api_app, webhooks, db,
api_project, access_token,
json_headers, users):
"""Test deposit events."""
(project, video_1, video_2) = api_project
prepare_videos_for_publish([video_1, video_2])
project_depid = project['_deposit']['id']
video_1_depid = video_1['_deposit']['id']
bucket_id = video_1._bucket.id
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_list',
receiver_id='downloader',
access_token=access_token)
with mock.patch('requests.get') as mock_request, \
api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
file_size = 1024 * 1024
mock_request.return_value = type(
'Response', (object, ), {
'raw': BytesIO(b'\x00' * file_size),
'headers': {
'Content-Length': file_size
}
})
payload = dict(uri='http://example.com/test.pdf',
bucket_id=str(bucket_id),
deposit_id=video_1_depid,
key='test.pdf')
resp = client.post(url, headers=json_headers, data=json.dumps(payload))
assert resp.status_code == 201
# run indexer
RecordIndexer().process_bulk_queue()
sleep(2)
deposit = deposit_video_resolver(video_1_depid)
file_dumps = deposit._get_files_dump()
assert len(file_dumps) == 1
def search_record(url):
res = client.get(url, headers=json_headers)
assert res.status_code == 200
data = json.loads(
res.data.decode('utf-8'))['hits']['hits'][0]['metadata']
return data
# check if the tasks states and files are inside elasticsearch
# -> check video
url_video_deposit = url_for('invenio_deposit_rest.video_list',
q='_deposit.id:{0}'.format(video_1_depid),
access_token=access_token)
data = search_record(url_video_deposit)
assert data['_cds']['state']['file_download'] == states.SUCCESS
assert file_dumps == data['_files']
# -> check project
url_project_deposit = url_for(
'invenio_deposit_rest.project_list',
q='_deposit.id:{0}'.format(project_depid),
access_token=access_token)
search_record(url_project_deposit)
assert data['_cds']['state']['file_download'] == states.SUCCESS
# [[ EDIT VIDEO ]]
deposit = deposit_video_resolver(video_1_depid)
video_edited = deepcopy(deposit)
del video_edited['_files']
del video_edited['_cds']['state']
reset_oauth2()
res = client.put(url_for('invenio_deposit_rest.video_item',
pid_value=video_1_depid),
data=json.dumps(video_edited),
headers=json_headers)
assert res.status_code == 200
# check if the tasks states and files are inside elasticsearch
# -> check video
data = search_record(url_video_deposit)
assert data['_cds']['state']['file_download'] == states.SUCCESS
assert file_dumps == data['_files']
# -> check project
url_project_deposit = url_for(
'invenio_deposit_rest.project_list',
q='_deposit.id:{0}'.format(project_depid),
access_token=access_token)
search_record(url_project_deposit)
assert data['_cds']['state']['file_download'] == states.SUCCESS
# [[ PUBLISH THE PROJECT ]]
reset_oauth2()
res = client.post(url_for(
'invenio_deposit_rest.project_actions',
pid_value=project['_deposit']['id'],
action='publish',
),
headers=json_headers)
assert res.status_code == 202
# run indexer
RecordIndexer().process_bulk_queue()
sleep(2)
deposit = deposit_video_resolver(video_1_depid)
# check if the files are inside elasticsearch
# -> check video deposit
data = search_record(url_video_deposit)
assert data['_cds']['state']['file_download'] == states.SUCCESS
assert file_dumps == data['_files']
# check video record
pid, record = deposit.fetch_published()
url = url_for('invenio_records_rest.recid_list',
q='_deposit.pid.value:{0}'.format(pid.pid_value))
data = search_record(url)
assert record['_files'] == data['_files']
def test_simple_workflow(
api_app, db, es, users, location, cds_jsonresolver,
data_file_1, data_file_2,
json_headers, json_partial_project_headers, json_partial_video_headers,
deposit_metadata, project_deposit_metadata, video_deposit_metadata):
"""Test project simple workflow."""
def check_connection(videos, project):
"""check project <---> video connection."""
assert all({"$ref": video.ref} in project['videos']
for video in videos)
assert len(videos) == len(project['videos'])
project_schema = ('https://cdslabs.cern.ch/schemas/'
'deposits/records/videos/project/project-v1.0.0.json')
video_schema = ('https://cdslabs.cern.ch/schemas/'
'deposits/records/videos/video/video-v1.0.0.json')
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
# [[ CREATE NEW PROJECT ]]
res = client.post(
url_for('invenio_deposit_rest.project_list'),
data=json.dumps(project_deposit_metadata),
headers=json_partial_project_headers)
# check returned value
assert res.status_code == 201
project_dict = json.loads(res.data.decode('utf-8'))
assert project_dict['metadata']['videos'] == []
assert project_dict['metadata']['title']['title'] == 'my project'
assert project_dict['links']['bucket'].startswith(
'http://localhost/files/')
assert all(link.startswith('http://localhost/deposits/project')
for (key, link) in project_dict['links'].items()
if key not in ['html', 'bucket'])
# check database
project_id = project_dict['metadata']['_deposit']['id']
project = project_resolver.resolve(project_id)[1]
assert project['$schema'] == project_schema
# [[ ADD A NEW EMPTY VIDEO_1 ]]
video_metadata = deepcopy(video_deposit_metadata)
video_metadata.update(
_project_id=project_dict['metadata']['_deposit']['id'])
res = client.post(
url_for('invenio_deposit_rest.video_list'),
data=json.dumps(video_metadata),
headers=json_partial_video_headers)
# check returned value
assert res.status_code == 201
video_1_dict = json.loads(res.data.decode('utf-8'))
assert video_1_dict['metadata']['_project_id'] == project_id
assert all(link.startswith('http://localhost/deposits/video')
for (key, link) in video_1_dict['links'].items()
if key not in ['html', 'bucket'])
# check database: connection project <---> videos
video_ids = [
video_1_dict['metadata']['_deposit']['id']
]
[video_1] = deposit_videos_resolver(video_ids)
check_connection(
[video_1],
project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
)
assert video_1['$schema'] == video_schema
# [[ GET THE VIDEO 1 ]]
res = client.get(
video_1_dict['links']['self'],
headers=json_headers)
# check returned value
assert res.status_code == 200
video_1_dict = json.loads(res.data.decode('utf-8'))
assert video_1_dict['metadata']['_files'] == []
# [[ ADD A NEW EMPTY VIDEO_2 ]]
video_metadata = deepcopy(video_deposit_metadata)
video_metadata.update(
_project_id=project_dict['metadata']['_deposit']['id'])
res = client.post(
url_for('invenio_deposit_rest.video_list'),
data=json.dumps(video_metadata),
headers=json_partial_video_headers)
# check returned value
assert res.status_code == 201
video_2_dict = json.loads(res.data.decode('utf-8'))
assert video_2_dict['metadata']['_project_id'] == project_id
assert all(link.startswith('http://localhost/deposits/video')
for (key, link) in video_2_dict['links'].items()
if key not in ['html', 'bucket'])
# check database: connection project <---> videos
video_ids = [
video_1_dict['metadata']['_deposit']['id'],
video_2_dict['metadata']['_deposit']['id']
]
[video_1, video_2] = deposit_videos_resolver(video_ids)
check_connection(
[video_1, video_2],
project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
)
assert video_2['$schema'] == video_schema
# [[ ADD A FILE INSIDE VIDEO_1 ]]
res = client.post(
url_for('invenio_deposit_rest.video_files',
pid_value=video_1_dict['metadata']['_deposit']['id']),
data=data_file_1, content_type='multipart/form-data')
# check returned value
assert res.status_code == 201
file_1 = json.loads(res.data.decode('utf-8'))
assert file_1['checksum'] == 'md5:eb88ae1e3666e6fe96a33ea72aab630e'
assert file_1['filesize'] == 24
assert file_1['filename'] == 'test.json'
assert file_1['id']
# check database: connection project <---> videos
video_1 = deposit_video_resolver(
video_1_dict['metadata']['_deposit']['id'])
assert video_1['_files'][0]['key'] == 'test.json'
video_ids = [
video_1_dict['metadata']['_deposit']['id'],
video_2_dict['metadata']['_deposit']['id']
]
check_connection(
deposit_videos_resolver(video_ids),
project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
)
# [[ GET THE VIDEO 1 ]]
res = client.get(video_1_dict['links']['self'], headers=json_headers)
# check video metadata
assert res.status_code == 200
video_1_dict = json.loads(res.data.decode('utf-8'))
assert len(video_1_dict['metadata']['_files']) == 1
myfile = video_1_dict['metadata']['_files'][0]
assert myfile['links']['self'].startswith('/api/files/')
assert myfile['checksum'] == 'md5:eb88ae1e3666e6fe96a33ea72aab630e'
assert myfile['completed'] is True
assert 'version_id' in myfile
assert myfile['key'] == 'test.json'
assert myfile['size'] == 24
# [[ PUBLISH VIDEO_1 ]]
# Not need to send _files
del video_1_dict['metadata']['_files']
# Prepare video for publishing
prepare_videos_for_publish([video_1, video_2])
res = client.post(
url_for('invenio_deposit_rest.video_actions',
pid_value=video_1['_deposit']['id'], action='publish'),
headers=json_headers)
# check returned value
assert res.status_code == 202
video_1_dict = json.loads(res.data.decode('utf-8'))
assert video_1_dict['metadata']['_deposit']['status'] == 'published'
assert video_1_dict['metadata']['recid'] == 1
assert video_1_dict['metadata']['_project_id'] == project_id
# check database: connection project <---> videos
video_ids = [
video_1_dict['metadata']['_deposit']['id'],
video_2_dict['metadata']['_deposit']['id']
]
check_connection(
deposit_videos_resolver(video_ids),
project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
)
# [[ ADD A VIDEO INSIDE VIDEO_2 ]]
res = client.post(
url_for('invenio_deposit_rest.video_files',
pid_value=video_2_dict['metadata']['_deposit']['id']),
data=data_file_2, content_type='multipart/form-data')
# check returned value
assert res.status_code == 201
file_2 = json.loads(res.data.decode('utf-8'))
assert file_2['checksum'] == 'md5:95405c14852500dcbb6dbfd9e27a3594'
assert file_2['filesize'] == 26
assert file_2['filename'] == 'test2.json'
# check connection project <---> videos
video_2 = deposit_video_resolver(
video_2_dict['metadata']['_deposit']['id'])
assert video_2['_files'][0]['key'] == 'test2.json'
video_ids = [
video_1_dict['metadata']['_deposit']['id'],
video_2_dict['metadata']['_deposit']['id']
]
check_connection(
deposit_videos_resolver(video_ids),
project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
)
# [[ PUBLISH THE PROJECT ]]
res = client.post(
url_for('invenio_deposit_rest.project_actions',
pid_value=project['_deposit']['id'], action='publish'),
headers=json_headers)
def get_video_record(depid):
deposit = deposit_video_resolver(depid)
return Video.get_record(deposit.fetch_published()[1].id)
video_1 = get_video_record(video_1_dict['metadata']['_deposit']['id'])
video_2 = get_video_record(video_2_dict['metadata']['_deposit']['id'])
record_videos = [video_1, video_2]
# check returned value
assert res.status_code == 202
project_dict = json.loads(res.data.decode('utf-8'))
assert project_dict['metadata']['_deposit']['status'] == 'published'
assert project_dict['metadata']['recid'] == 3
assert project_dict['metadata']['videos'][0] == record_videos[0]
assert project_dict['metadata']['videos'][1] == record_videos[1]
# check database: connection project <---> videos
check_connection(
record_videos,
project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
)
# check indexed record
RecordIndexer().process_bulk_queue()
sleep(2)
res = client.get(url_for('invenio_records_rest.recid_list',
headers=json_headers))
assert res.status_code == 200
data = json.loads(res.data.decode('utf-8'))
for hit in data['hits']['hits']:
assert isinstance(hit['id'], int)
# [[ EDIT THE PROJECT ]]
res = client.post(
url_for('invenio_deposit_rest.project_actions',
pid_value=project_dict['metadata']['_deposit']['id'],
action='edit'),
headers=json_headers)
# check returned value
assert res.status_code == 201
project_dict = json.loads(res.data.decode('utf-8'))
assert project_dict['metadata']['_deposit']['status'] == 'draft'
# check database
project = project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
assert project['_deposit']['status'] == 'draft'
# [[ MODIFY PROJECT ]]
project_before = project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
project_dict['metadata']['title']['title'] = 'new project title'
# Not need to send _files
del project_dict['metadata']['_files']
# try to modify preserved fields
project_dict['metadata']['recid'] = 12323233
project_dict['metadata']['report_number'][0] = 'fuuu barrrr'
project_dict['metadata']['publication_date'] = '2000-12-03'
# do the call
res = client.put(
url_for('invenio_deposit_rest.project_item',
pid_value=project_dict['metadata']['_deposit']['id']),
data=json.dumps(project_dict['metadata']),
headers=json_headers)
# check returned value
assert res.status_code == 200
project_dict = json.loads(res.data.decode('utf-8'))
assert project_dict['metadata']['title']['title'] ==\
'new project title'
assert all(link.startswith('http://localhost/deposits/project')
for (key, link) in project_dict['links'].items()
if key not in ['html', 'bucket'])
video_1 = get_video_record(video_1_dict['metadata']['_deposit']['id'])
video_2 = get_video_record(video_2_dict['metadata']['_deposit']['id'])
assert video_1 == project_dict['metadata']['videos'][0]
assert video_2 == project_dict['metadata']['videos'][1]
# check database
project = project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
assert project['title']['title'] == 'new project title'
# check preserved fields
assert project_before['recid'] == project['recid']
assert project_before['report_number'] == project['report_number']
assert project_before[
'publication_date'] == project['publication_date']
# [[ DISCARD PROJECT ]]
res = client.post(
url_for('invenio_deposit_rest.project_actions',
pid_value=project_dict['metadata']['_deposit']['id'],
action='discard'),
headers=json_headers)
# check returned value
assert res.status_code == 201
project_dict = json.loads(res.data.decode('utf-8'))
assert project_dict['metadata']['title']['title'] == 'my project'
# check database
project = project_resolver.resolve(
project_dict['metadata']['_deposit']['id'])[1]
assert project['title']['title'] == 'my project'
def test_migrate_record(frames_required, api_app, location, datadir, es,
users):
"""Test migrate date."""
# [[ migrate the project ]]
data = load_json(datadir, 'cds_records_demo_1_project.json')
dump = CDSRecordDump(data=data[0])
project = CDSRecordDumpLoader.create(dump=dump)
p_id = project.id
assert project['$schema'] == Project.get_record_schema()
assert project['publication_date'] == '2016-01-05'
assert 'license' not in project
assert 'copyright' not in project
assert project['_cds'] == {
"state": {
"file_transcode": "SUCCESS",
"file_video_extract_frames": "SUCCESS",
"file_video_metadata_extraction": "SUCCESS"
},
'modified_by': users[0],
}
# check project deposit
deposit_project_uuid = PersistentIdentifier.query.filter_by(
pid_type='depid', object_type='rec').one().object_uuid
deposit_project = Record.get_record(deposit_project_uuid)
assert Project._schema in deposit_project['$schema']
assert project.revision_id == deposit_project[
'_deposit']['pid']['revision_id']
assert deposit_project['_deposit']['created_by'] == 1
assert deposit_project['_deposit']['owners'] == [1]
assert deposit_project['_files'] == []
# [[ migrate the video ]]
data = load_json(datadir, 'cds_records_demo_1_video.json')
dump = CDSRecordDump(data=data[0])
db.session.commit()
def check_symlinks(video):
symlinks_creator = SymlinksCreator()
files = list(symlinks_creator._get_list_files(record=video))
assert len(files) == 1
for file_ in files:
path = symlinks_creator._build_link_path(
symlinks_creator._symlinks_location, video, file_['key'])
assert os.path.lexists(path)
def check_gif(video, mock_gif):
# called only once for deposit
(_, _, mock_args) = mock_gif.mock_calls[0]
# check gif record
video = CDSRecord(dict(video), video.model)
# check gif deposit
deposit = deposit_video_resolver(video['_deposit']['id'])
master_video = CDSVideosFilesIterator.get_master_video_file(deposit)
assert mock_args['master_id'] == master_video['version_id']
assert str(deposit.files.bucket.id) == mock_args['bucket']
# assert mock_args['bucket'].id == deposit.files.bucket.id
assert len(mock_args['frames']) == 10
assert 'output_dir' in mock_args
migration_streams = get_migration_streams(datadir=datadir)
with mock.patch.object(DataCiteProvider, 'register'), \
mock.patch.object(CDSRecordDumpLoader, '_create_frame',
side_effect=get_frames), \
mock.patch.object(CDSRecordDumpLoader, '_get_minimum_frames',
return_value=frames_required) as mock_frames, \
mock.patch.object(
ExtractFramesTask, '_create_gif') as mock_gif, \
mock.patch.object(
CDSRecordDumpLoader, '_get_migration_file_stream_and_size',
side_effect=migration_streams), \
mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'):
video = CDSRecordDumpLoader.create(dump=dump)
assert mock_frames.called is True
db.session.add(video.model)
video_id = video.id
# check smil file
smil_obj = ObjectVersion.query.filter_by(
key='CERN-MOVIE-2012-193-001.smil', is_head=True).one()
storage = smil_obj.file.storage()
assert '<video src' in storage.open().read().decode('utf-8')
# check video symlinks
check_symlinks(video)
# check gif
check_gif(video, mock_gif)
# check project
project = Record.get_record(p_id)
assert project['videos'] == [
{'$ref': 'https://cds.cern.ch/api/record/1495143'}
]
assert video['$schema'] == Video.get_record_schema()
assert video['date'] == '2012-11-21' # metadata data
assert video['publication_date'] == '2017-07-13' # creation date (DB)
assert video['_project_id'] == '2093596'
assert video['license'] == [{
'license': 'CERN',
'url': 'http://copyright.web.cern.ch',
}]
assert video['copyright'] == {
'holder': 'CERN',
'year': '2012',
'url': 'http://copyright.web.cern.ch',
}
assert video['description'] == ''
assert 'doi' in video
assert video['_cds']['state'] == {
"file_transcode": "SUCCESS",
"file_video_extract_frames": "SUCCESS",
"file_video_metadata_extraction": "SUCCESS"
}
assert 'extracted_metadata' in video['_cds']
def check_files(video):
bucket = CDSRecordDumpLoader._get_bucket(record=video)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)]
for file_ in files:
assert as_bucket(file_['bucket_id']) is not None
assert 'checksum' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert FileInstance.query.filter_by(
id=file_['file_id']) is not None
assert 'key' in file_
assert 'links' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert 'media_type' in file_
assert 'tags' in file_
# check extracted metadata
master_video = CDSVideosFilesIterator.get_master_video_file(video)
assert any([key in master_video['tags']
for key in ExtractMetadataTask._all_keys])
assert any([key in video['_cds']['extracted_metadata']
for key in ExtractMetadataTask._all_keys])
def check_buckets(record, deposit):
def get(key, record):
bucket = CDSRecordDumpLoader._get_bucket(record=record)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)]
return [file_[key] for file_ in files]
def check(record, deposit, file_key, different=None):
values_record = set(get(file_key, record))
values_deposit = set(get(file_key, deposit))
difference = len(values_record - values_deposit)
assert different == difference
def check_tag_master(record):
bucket = CDSRecordDumpLoader._get_bucket(record=record)
master = CDSVideosFilesIterator.get_master_video_file(record)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)
if obj.get_tags().get('master')]
assert all([file_['tags']['master'] == master['version_id']
for file_ in files])
# 1 bucket record != 1 bucket deposit
check(record, deposit, 'bucket_id', 1)
# all file_id are the same except the smil file (only in record)
check(record, deposit, 'file_id', 1)
check(record, deposit, 'key', 1)
# 18 object_version record != 17 object_version deposit
check(record, deposit, 'version_id', 18)
# check tag 'master' where is pointing
check_tag_master(record)
check_tag_master(deposit)
def check_first_level_files(record):
[master] = [file_ for file_ in deposit_video['_files']
if file_['context_type'] == 'master']
assert len(master['subformat']) == 5
assert len(master['frame']) == 10
# TODO assert len(master['playlist']) == ??
assert len([file_ for file_ in deposit_video['_files']
if file_['context_type'] == 'master']) == 1
duration = float(record['_cds']['extracted_metadata']['duration'])
for frame in master['frame']:
assert float(frame['tags']['timestamp']) < duration
assert float(frame['tags']['timestamp']) > 0
# check tag 'preset_quality'
pqs = [form['tags']['preset_quality'] for form in master['subformat']]
assert sorted(pqs) == sorted(['1080p', '240p', '360p', '480p', '720p'])
# check tag 'display_aspect_ratio'
dar = set([form['tags']['display_aspect_ratio']
for form in master['subformat']])
assert dar == {'16:9'}
def check_pids(record):
"""Check pids."""
assert record['report_number'][0] == 'CERN-VIDEO-2012-193-001'
assert PersistentIdentifier.query.filter_by(
pid_value='CERN-VIDEO-2012-193-001').count() == 1
assert PersistentIdentifier.query.filter_by(
pid_value='CERN-MOVIE-2012-193-001').count() == 1
db.session.commit()
# check video deposit
deposit_video_uuid = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == 'depid',
PersistentIdentifier.object_uuid != str(deposit_project_uuid),
PersistentIdentifier.object_type == 'rec'
).one().object_uuid
deposit_video = Video.get_record(str(deposit_video_uuid))
assert Video._schema in deposit_video['$schema']
video = Record.get_record(video_id)
assert video.revision_id == deposit_video[
'_deposit']['pid']['revision_id']
assert deposit_video['_deposit']['created_by'] == users[0]
assert deposit_video['_deposit']['owners'] == [users[0]]
assert deposit_video['_project_id'] == '2093596'
assert len(video['_files']) == 2
assert len(deposit_video['_files']) == 2
check_files(video)
check_files(deposit_video)
check_buckets(video, deposit_video)
check_first_level_files(video)
check_first_level_files(deposit_video)
check_pids(video)
# try to edit video
deposit_video = deposit_video_resolver(deposit_video['_deposit']['id'])
deposit_video = deposit_video.edit()
# try to edit project
deposit_project = deposit_project_resolver(
deposit_project['_deposit']['id'])
deposit_project = deposit_project.edit()
login_user(User.query.filter_by(id=users[0]).first())
deposit_video['title']['title'] = 'test'
deposit_video = deposit_video.publish()
_, record_video = deposit_video.fetch_published()
assert record_video['title']['title'] == 'test'
