def check(project_status, video_1_status, video_2_status):
project = deposit_project_resolver(project_depid)
video_1 = deposit_video_resolver(video_1_depid)
video_2 = deposit_video_resolver(video_2_depid)
assert project.status == project_status
assert video_1.status == video_1_status
assert video_2.status == video_2_status
def test_project_publish_with_workflow(api_app, users, api_project, es):
"""Test publish a project with a workflow."""
project, video_1, video_2 = api_project
prepare_videos_for_publish([video_1, video_2])
project_depid = project['_deposit']['id']
project_id = str(project.id)
video_1_depid = video_1['_deposit']['id']
video_1_id = str(video_1.id)
video_2_depid = video_2['_deposit']['id']
receiver_id = 'test_project_publish_with_workflow'
workflow_receiver_video_failing(api_app,
db,
video_1,
receiver_id=receiver_id)
headers = [('Content-Type', 'application/json')]
payload = json.dumps(dict(somekey='somevalue'))
with mock.patch('invenio_indexer.tasks.index_record.delay') \
as mock_indexer, \
api_app.test_request_context(headers=headers, data=payload):
event = Event.create(receiver_id=receiver_id)
db.session.add(event)
event.process()
# check video and project are indexed
assert mock_indexer.called is True
ids = get_indexed_records_from_mock(mock_indexer)
assert video_1_id == ids[0]
assert project_id == ids[1]
db.session.commit()
# check tasks status is propagated to video and project
video_1 = deposit_video_resolver(video_1_depid)
expected = {u'add': u'SUCCESS', u'failing': u'FAILURE'}
assert video_1['_cds']['state'] == expected
assert video_1.project['_cds']['state'] == expected
events = get_deposit_events(deposit_id=video_1_depid)
assert len(events) == 1
def check(project_status, video_1_status, video_2_status):
project = deposit_project_resolver(project_depid)
video_1 = deposit_video_resolver(video_1_depid)
video_2 = deposit_video_resolver(video_2_depid)
assert project.status == project_status
assert video_1.status == video_1_status
assert video_2.status == video_2_status
check('draft', 'draft', 'draft')
login_user(User.query.get(users[0]))
video_2 = deposit_video_resolver(video_2_depid)
video_2.publish()
check('draft', 'draft', 'published')
project = deposit_project_resolver(project_depid)
project.publish()
check('published', 'published', 'published')
def test_project_deposit(es, location, deposit_metadata):
"""Test CDS deposit creation."""
deposit = Project.create(deposit_metadata)
id_ = deposit.id
db.session.expire_all()
deposit = Project.get_record(id_)
assert deposit['_cds']['state'] == {}
assert deposit_project_resolver(deposit['_deposit']['id']) is not None
assert '_buckets' in deposit
def test_video_publish_registering_the_datacite_not_local(
datacite_mock, api_app, users, location, cds_jsonresolver,
json_headers, json_partial_project_headers, json_partial_video_headers,
deposit_metadata, video_deposit_metadata, project_deposit_metadata,
keyword_1, keyword_2):
"""Test video publish registering the datacite not local."""
# test: enable datacite registration
api_app.config['DEPOSIT_DATACITE_MINTING_ENABLED'] = True
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
project_deposit_metadata['keywords'] = [copy.deepcopy(keyword_2)]
# [[ CREATE NEW PROJECT ]]
project_dict = _create_new_project(client,
json_partial_project_headers,
project_deposit_metadata)
assert project_dict['metadata']['keywords'][0] == keyword_2
project_depid = project_dict['metadata']['_deposit']['id']
project = deposit_project_resolver(project_depid)
assert project['keywords'] == [{
'$ref': keyword_2.ref,
'name': keyword_2['name'],
}]
# [[ ADD A NEW VIDEO_1 ]]
video_metadata = copy.deepcopy(video_deposit_metadata)
video_metadata['keywords'] = [copy.deepcopy(keyword_1)]
video_metadata.update(
_project_id=project_dict['metadata']['_deposit']['id'])
res = client.post(url_for('invenio_deposit_rest.video_list'),
data=json.dumps(video_metadata),
headers=json_partial_video_headers)
assert res.status_code == 201
video_1_dict = json.loads(res.data.decode('utf-8'))
assert video_1_dict['metadata']['keywords'][0] == keyword_1
video_1_depid = video_1_dict['metadata']['_deposit']['id']
video_1 = deposit_video_resolver(video_1_depid)
assert video_1['keywords'] == [{
'$ref': keyword_1.ref,
'name': keyword_1['name'],
}]
video_1['doi'] = '10.1123/doi'
prepare_videos_for_publish([video_1])
# [[ PUBLISH VIDEO ]]
video_1.publish()
datacite_register.s(pid_value='123',
record_uuid=str(video_1.id)).apply()
assert datacite_mock.called is False
def test_transcode_and_undo(db, cds_depid, mock_sorenson):
"""Test TranscodeVideoTask task."""
def get_bucket_keys():
return [o.key for o in list(ObjectVersion.get_by_bucket(bucket))]
bucket = deposit_project_resolver(cds_depid).files.bucket
filesize = 1024
filename = 'test.mp4'
preset_quality = '480p'
new_filename = '{0}.mp4'.format(preset_quality)
obj = ObjectVersion.create(bucket,
key=filename,
stream=BytesIO(b'\x00' * filesize))
ObjectVersionTag.create(obj, 'display_aspect_ratio', '16:9')
obj_id = str(obj.version_id)
db.session.commit()
assert get_bucket_keys() == [filename]
assert bucket.size == filesize
task_s = TranscodeVideoTask().s(version_id=obj_id,
preset_quality=preset_quality,
sleep_time=0)
# Transcode
task_s.delay(deposit_id=cds_depid)
db.session.add(bucket)
keys = get_bucket_keys()
assert len(keys) == 2
assert filename in keys
assert new_filename in keys
assert bucket.size == 2 * filesize
# Undo
TranscodeVideoTask().clean(version_id=obj_id,
preset_quality=preset_quality)
db.session.add(bucket)
keys = get_bucket_keys()
assert len(keys) == 1
assert filename in keys
assert new_filename not in keys
# file size doesn't change
assert bucket.size == 2 * filesize
def test_transcode_2tasks_delete1(db, cds_depid, mock_sorenson):
"""Test TranscodeVideoTask task when run 2 task and delete 1."""
def get_bucket_keys():
return [o.key for o in list(ObjectVersion.get_by_bucket(bucket))]
bucket = deposit_project_resolver(cds_depid).files.bucket
filesize = 1024
filename = 'test.mp4'
preset_qualities = ['480p', '720p']
new_filenames = ['{0}.mp4'.format(p) for p in preset_qualities]
(version_id, [task_s1,
task_s2]) = transcode_task(bucket=bucket,
filesize=filesize,
filename=filename,
preset_qualities=preset_qualities)
assert get_bucket_keys() == [filename]
assert bucket.size == filesize
# Transcode
task_s1.delay(deposit_id=cds_depid)
task_s2.delay(deposit_id=cds_depid)
db.session.add(bucket)
keys = get_bucket_keys()
assert len(keys) == 3
assert new_filenames[0] in keys
assert new_filenames[1] in keys
assert filename in keys
assert bucket.size == (3 * filesize)
# Undo
TranscodeVideoTask().clean(version_id=version_id,
preset_quality=preset_qualities[0])
db.session.add(bucket)
keys = get_bucket_keys()
assert len(keys) == 2
assert new_filenames[0] not in keys
assert new_filenames[1] in keys
assert filename in keys
assert bucket.size == (3 * filesize)
def test_migrate_record(frames_required, api_app, location, datadir, es,
users):
"""Test migrate date."""
# [[ migrate the project ]]
data = load_json(datadir, 'cds_records_demo_1_project.json')
dump = CDSRecordDump(data=data[0])
project = CDSRecordDumpLoader.create(dump=dump)
p_id = project.id
assert project['$schema'] == Project.get_record_schema()
assert project['publication_date'] == '2016-01-05'
assert 'license' not in project
assert 'copyright' not in project
assert project['_cds'] == {
"state": {
"file_transcode": "SUCCESS",
"file_video_extract_frames": "SUCCESS",
"file_video_metadata_extraction": "SUCCESS"
},
'modified_by': users[0],
}
# check project deposit
deposit_project_uuid = PersistentIdentifier.query.filter_by(
pid_type='depid', object_type='rec').one().object_uuid
deposit_project = Record.get_record(deposit_project_uuid)
assert Project._schema in deposit_project['$schema']
assert project.revision_id == deposit_project[
'_deposit']['pid']['revision_id']
assert deposit_project['_deposit']['created_by'] == 1
assert deposit_project['_deposit']['owners'] == [1]
assert deposit_project['_files'] == []
# [[ migrate the video ]]
data = load_json(datadir, 'cds_records_demo_1_video.json')
dump = CDSRecordDump(data=data[0])
db.session.commit()
def check_symlinks(video):
symlinks_creator = SymlinksCreator()
files = list(symlinks_creator._get_list_files(record=video))
assert len(files) == 1
for file_ in files:
path = symlinks_creator._build_link_path(
symlinks_creator._symlinks_location, video, file_['key'])
assert os.path.lexists(path)
def check_gif(video, mock_gif):
# called only once for deposit
(_, _, mock_args) = mock_gif.mock_calls[0]
# check gif record
video = CDSRecord(dict(video), video.model)
# check gif deposit
deposit = deposit_video_resolver(video['_deposit']['id'])
master_video = CDSVideosFilesIterator.get_master_video_file(deposit)
assert mock_args['master_id'] == master_video['version_id']
assert str(deposit.files.bucket.id) == mock_args['bucket']
# assert mock_args['bucket'].id == deposit.files.bucket.id
assert len(mock_args['frames']) == 10
assert 'output_dir' in mock_args
migration_streams = get_migration_streams(datadir=datadir)
with mock.patch.object(DataCiteProvider, 'register'), \
mock.patch.object(CDSRecordDumpLoader, '_create_frame',
side_effect=get_frames), \
mock.patch.object(CDSRecordDumpLoader, '_get_minimum_frames',
return_value=frames_required) as mock_frames, \
mock.patch.object(
ExtractFramesTask, '_create_gif') as mock_gif, \
mock.patch.object(
CDSRecordDumpLoader, '_get_migration_file_stream_and_size',
side_effect=migration_streams), \
mock.patch.object(CDSRecordDumpLoader, '_clean_file_list'):
video = CDSRecordDumpLoader.create(dump=dump)
assert mock_frames.called is True
db.session.add(video.model)
video_id = video.id
# check smil file
smil_obj = ObjectVersion.query.filter_by(
key='CERN-MOVIE-2012-193-001.smil', is_head=True).one()
storage = smil_obj.file.storage()
assert '<video src' in storage.open().read().decode('utf-8')
# check video symlinks
check_symlinks(video)
# check gif
check_gif(video, mock_gif)
# check project
project = Record.get_record(p_id)
assert project['videos'] == [
{'$ref': 'https://cds.cern.ch/api/record/1495143'}
]
assert video['$schema'] == Video.get_record_schema()
assert video['date'] == '2012-11-21' # metadata data
assert video['publication_date'] == '2017-07-13' # creation date (DB)
assert video['_project_id'] == '2093596'
assert video['license'] == [{
'license': 'CERN',
'url': 'http://copyright.web.cern.ch',
}]
assert video['copyright'] == {
'holder': 'CERN',
'year': '2012',
'url': 'http://copyright.web.cern.ch',
}
assert video['description'] == ''
assert 'doi' in video
assert video['_cds']['state'] == {
"file_transcode": "SUCCESS",
"file_video_extract_frames": "SUCCESS",
"file_video_metadata_extraction": "SUCCESS"
}
assert 'extracted_metadata' in video['_cds']
def check_files(video):
bucket = CDSRecordDumpLoader._get_bucket(record=video)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)]
for file_ in files:
assert as_bucket(file_['bucket_id']) is not None
assert 'checksum' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert FileInstance.query.filter_by(
id=file_['file_id']) is not None
assert 'key' in file_
assert 'links' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert 'media_type' in file_
assert 'tags' in file_
# check extracted metadata
master_video = CDSVideosFilesIterator.get_master_video_file(video)
assert any([key in master_video['tags']
for key in ExtractMetadataTask._all_keys])
assert any([key in video['_cds']['extracted_metadata']
for key in ExtractMetadataTask._all_keys])
def check_buckets(record, deposit):
def get(key, record):
bucket = CDSRecordDumpLoader._get_bucket(record=record)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)]
return [file_[key] for file_ in files]
def check(record, deposit, file_key, different=None):
values_record = set(get(file_key, record))
values_deposit = set(get(file_key, deposit))
difference = len(values_record - values_deposit)
assert different == difference
def check_tag_master(record):
bucket = CDSRecordDumpLoader._get_bucket(record=record)
master = CDSVideosFilesIterator.get_master_video_file(record)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)
if obj.get_tags().get('master')]
assert all([file_['tags']['master'] == master['version_id']
for file_ in files])
# 1 bucket record != 1 bucket deposit
check(record, deposit, 'bucket_id', 1)
# all file_id are the same except the smil file (only in record)
check(record, deposit, 'file_id', 1)
check(record, deposit, 'key', 1)
# 18 object_version record != 17 object_version deposit
check(record, deposit, 'version_id', 18)
# check tag 'master' where is pointing
check_tag_master(record)
check_tag_master(deposit)
def check_first_level_files(record):
[master] = [file_ for file_ in deposit_video['_files']
if file_['context_type'] == 'master']
assert len(master['subformat']) == 5
assert len(master['frame']) == 10
# TODO assert len(master['playlist']) == ??
assert len([file_ for file_ in deposit_video['_files']
if file_['context_type'] == 'master']) == 1
duration = float(record['_cds']['extracted_metadata']['duration'])
for frame in master['frame']:
assert float(frame['tags']['timestamp']) < duration
assert float(frame['tags']['timestamp']) > 0
# check tag 'preset_quality'
pqs = [form['tags']['preset_quality'] for form in master['subformat']]
assert sorted(pqs) == sorted(['1080p', '240p', '360p', '480p', '720p'])
# check tag 'display_aspect_ratio'
dar = set([form['tags']['display_aspect_ratio']
for form in master['subformat']])
assert dar == {'16:9'}
def check_pids(record):
"""Check pids."""
assert record['report_number'][0] == 'CERN-VIDEO-2012-193-001'
assert PersistentIdentifier.query.filter_by(
pid_value='CERN-VIDEO-2012-193-001').count() == 1
assert PersistentIdentifier.query.filter_by(
pid_value='CERN-MOVIE-2012-193-001').count() == 1
db.session.commit()
# check video deposit
deposit_video_uuid = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == 'depid',
PersistentIdentifier.object_uuid != str(deposit_project_uuid),
PersistentIdentifier.object_type == 'rec'
).one().object_uuid
deposit_video = Video.get_record(str(deposit_video_uuid))
assert Video._schema in deposit_video['$schema']
video = Record.get_record(video_id)
assert video.revision_id == deposit_video[
'_deposit']['pid']['revision_id']
assert deposit_video['_deposit']['created_by'] == users[0]
assert deposit_video['_deposit']['owners'] == [users[0]]
assert deposit_video['_project_id'] == '2093596'
assert len(video['_files']) == 2
assert len(deposit_video['_files']) == 2
check_files(video)
check_files(deposit_video)
check_buckets(video, deposit_video)
check_first_level_files(video)
check_first_level_files(deposit_video)
check_pids(video)
# try to edit video
deposit_video = deposit_video_resolver(deposit_video['_deposit']['id'])
deposit_video = deposit_video.edit()
# try to edit project
deposit_project = deposit_project_resolver(
deposit_project['_deposit']['id'])
deposit_project = deposit_project.edit()
login_user(User.query.filter_by(id=users[0]).first())
deposit_video['title']['title'] = 'test'
deposit_video = deposit_video.publish()
_, record_video = deposit_video.fetch_published()
assert record_video['title']['title'] == 'test'
def test_avc_workflow_receiver_pass(api_app, db, api_project, access_token,
json_headers, mock_sorenson, online_video,
webhooks, users):
"""Test AVCWorkflow receiver."""
project, video_1, video_2 = api_project
video_1_depid = video_1['_deposit']['id']
video_1_id = str(video_1.id)
project_id = str(project.id)
bucket_id = video_1['_buckets']['deposit']
video_size = 5510872
master_key = 'test.mp4'
slave_keys = [
'slave_{0}.mp4'.format(quality) for quality in get_presets_applied()
if quality != '1024p'
]
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_list',
receiver_id='avc',
access_token=access_token)
with api_app.test_client() as client, \
mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \
mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \
as mock_indexer:
sse_channel = 'mychannel'
payload = dict(
uri=online_video,
deposit_id=video_1_depid,
key=master_key,
sse_channel=sse_channel,
sleep_time=0,
)
resp = client.post(url, headers=json_headers, data=json.dumps(payload))
assert resp.status_code == 201
data = json.loads(resp.data.decode('utf-8'))
assert '_tasks' in data
assert data['tags']['uri_origin'] == online_video
assert data['key'] == master_key
assert 'version_id' in data
assert data.get('presets') == get_available_preset_qualities()
assert 'links' in data # TODO decide with links are needed
assert ObjectVersion.query.count() == get_object_count()
# Master file
master = ObjectVersion.get(bucket_id, master_key)
tags = master.get_tags()
assert tags['_event_id'] == data['tags']['_event_id']
assert master.key == master_key
assert str(master.version_id) == data['version_id']
assert master.file
assert master.file.size == video_size
# Check metadata tags
metadata_keys = [
'duration', 'bit_rate', 'size', 'avg_frame_rate', 'codec_name',
'codec_long_name', 'width', 'height', 'nb_frames',
'display_aspect_ratio', 'color_range'
]
assert all([key in tags for key in metadata_keys])
# Check metadata patch
recid = PersistentIdentifier.get('depid', video_1_depid).object_uuid
record = Record.get_record(recid)
assert 'extracted_metadata' in record['_cds']
assert all([
key in str(record['_cds']['extracted_metadata'])
for key in metadata_keys
])
# Check slaves
for slave_key in slave_keys:
slave = ObjectVersion.get(bucket_id, slave_key)
tags = slave.get_tags()
assert slave.key == slave_key
assert '_sorenson_job_id' in tags
assert tags['_sorenson_job_id'] == '1234'
assert 'master' in tags
assert tags['master'] == str(master.version_id)
assert master.file
assert master.file.size == video_size
video = deposit_video_resolver(video_1_depid)
events = get_deposit_events(video['_deposit']['id'])
# check deposit tasks status
tasks_status = get_tasks_status_by_task(events)
assert len(tasks_status) == 4
assert 'file_download' in tasks_status
assert 'file_transcode' in tasks_status
assert 'file_video_extract_frames' in tasks_status
assert 'file_video_metadata_extraction' in tasks_status
# check single status
collector = CollectInfoTasks()
iterate_events_results(events=events, fun=collector)
info = list(collector)
presets = get_presets_applied().keys()
assert info[0][0] == 'file_download'
assert info[0][1].status == states.SUCCESS
assert info[1][0] == 'file_video_metadata_extraction'
assert info[1][1].status == states.SUCCESS
assert info[2][0] == 'file_video_extract_frames'
assert info[2][1].status == states.SUCCESS
for i in info[3:]:
assert i[0] == 'file_transcode'
if i[1].status == states.SUCCESS:
assert i[1].result['payload']['preset_quality'] in presets
# check tags
assert ObjectVersionTag.query.count() == get_tag_count()
# check sse is called
assert mock_sse.called
messages = [
(sse_channel, states.STARTED, 'file_download'),
(sse_channel, states.SUCCESS, 'file_download'),
(sse_channel, states.SUCCESS, 'file_video_metadata_extraction'),
(sse_channel, states.STARTED, 'file_transcode'),
(sse_channel, states.SUCCESS, 'file_transcode'),
(sse_channel, states.REVOKED, 'file_transcode'), # ResolutionError
(sse_channel, states.STARTED, 'file_video_extract_frames'),
(sse_channel, states.SUCCESS, 'file_video_extract_frames'),
(sse_channel, states.SUCCESS, 'update_deposit'),
]
call_args = []
for (_, kwargs) in mock_sse.call_args_list:
type_ = kwargs['type_']
state = kwargs['data']['state']
channel = kwargs['channel']
tuple_ = (channel, state, type_)
if tuple_ not in call_args:
call_args.append(tuple_)
assert len(call_args) == len(messages)
for message in messages:
assert message in call_args
deposit = deposit_video_resolver(video_1_depid)
def filter_events(call_args):
_, x = call_args
return x['type_'] == 'update_deposit'
list_kwargs = list(filter(filter_events, mock_sse.call_args_list))
assert len(list_kwargs) == 12
_, kwargs = list_kwargs[10]
assert kwargs['type_'] == 'update_deposit'
assert kwargs['channel'] == 'mychannel'
assert kwargs['data']['state'] == states.SUCCESS
assert kwargs['data']['meta']['payload'] == {
'deposit_id': deposit['_deposit']['id'],
'event_id': data['tags']['_event_id'],
'deposit': deposit,
}
# check ElasticSearch is called
ids = set(get_indexed_records_from_mock(mock_indexer))
assert video_1_id in ids
assert project_id in ids
assert deposit['_cds']['state'] == {
'file_download': states.SUCCESS,
'file_video_metadata_extraction': states.SUCCESS,
'file_video_extract_frames': states.SUCCESS,
'file_transcode': states.SUCCESS,
}
# check feedback from anoymous user
event_id = data['tags']['_event_id']
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_feedback_item',
event_id=event_id,
receiver_id='avc')
with api_app.test_client() as client:
resp = client.get(url, headers=json_headers)
assert resp.status_code == 401
# check feedback from owner
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_feedback_item',
event_id=event_id,
receiver_id='avc')
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[0]).email)
resp = client.get(url, headers=json_headers)
assert resp.status_code == 200
# check feedback from another user without access
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_feedback_item',
event_id=event_id,
receiver_id='avc')
with api_app.test_client() as client:
login_user_via_session(client, email=User.query.get(users[1]).email)
resp = client.get(url, headers=json_headers)
assert resp.status_code == 403
# check feedback from another user with access
user_2 = User.query.get(users[1])
user_2_id = str(user_2.id)
user_2_email = user_2.email
project = deposit_project_resolver(project['_deposit']['id'])
project['_access'] = {'update': [user_2_email]}
project = project.commit()
with api_app.test_request_context():
url = url_for('invenio_webhooks.event_feedback_item',
event_id=event_id,
receiver_id='avc')
with api_app.test_client() as client:
@identity_loaded.connect
def load_email(sender, identity):
if current_user.get_id() == user_2_id:
identity.provides.update([UserNeed(user_2_email)])
login_user_via_session(client, email=user_2_email)
resp = client.get(url, headers=json_headers)
assert resp.status_code == 200
# Test cleaning!
url = '{0}?access_token={1}'.format(data['links']['cancel'], access_token)
with mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \
mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \
as mock_indexer, \
api_app.test_client() as client:
resp = client.delete(url, headers=json_headers)
assert resp.status_code == 201
# check that object versions and tags are deleted
# (Create + Delete) * Num Objs
assert ObjectVersion.query.count() == 2 * get_object_count()
# Tags connected with the old version
assert ObjectVersionTag.query.count() == get_tag_count()
bucket = Bucket.query.first()
# and bucket is empty
assert bucket.size == 0
record = RecordMetadata.query.filter_by(id=video_1_id).one()
# check metadata patch are deleted
assert 'extracted_metadata' not in record.json['_cds']
# check the corresponding Event persisted after cleaning
assert len(get_deposit_events(record.json['_deposit']['id'])) == 0
assert len(
get_deposit_events(record.json['_deposit']['id'],
_deleted=True)) == 1
# check no SSE message and reindexing is fired
assert mock_sse.called is False
assert mock_indexer.called is False
def test_project_publish_with_workflow(api_app, users, api_project, es):
"""Test publish a project with a workflow."""
project, video_1, video_2 = api_project
prepare_videos_for_publish([video_1, video_2])
project_depid = project['_deposit']['id']
project_id = str(project.id)
video_1_depid = video_1['_deposit']['id']
video_1_id = str(video_1.id)
video_2_depid = video_2['_deposit']['id']
sse_channel = 'mychannel'
receiver_id = 'test_project_publish_with_workflow'
workflow_receiver_video_failing(api_app,
db,
video_1,
receiver_id=receiver_id,
sse_channel=sse_channel)
headers = [('Content-Type', 'application/json')]
payload = json.dumps(dict(somekey='somevalue'))
with mock.patch('invenio_sse.ext._SSEState.publish') as mock_sse, \
mock.patch('invenio_indexer.api.RecordIndexer.bulk_index') \
as mock_indexer, \
api_app.test_request_context(headers=headers, data=payload):
event = Event.create(receiver_id=receiver_id)
db.session.add(event)
event.process()
# check messages are sent to the sse channel
assert mock_sse.called is True
args = list(mock_sse.mock_calls[0])[2]
assert args['channel'] == sse_channel
assert args['type_'] == 'update_deposit'
assert args['data']['meta']['payload']['deposit_id'] == video_1_depid
args = list(mock_sse.mock_calls[1])[2]
assert args['channel'] == sse_channel
assert args['type_'] == 'update_deposit'
assert args['data']['meta']['payload']['deposit_id'] == project_depid
# check video and project are indexed
assert mock_indexer.called is True
ids = get_indexed_records_from_mock(mock_indexer)
assert video_1_id == ids[0]
assert project_id == ids[1]
db.session.commit()
# check tasks status is propagated to video and project
video_1 = deposit_video_resolver(video_1_depid)
expected = {u'add': u'SUCCESS', u'failing': u'FAILURE'}
assert video_1['_cds']['state'] == expected
assert video_1.project['_cds']['state'] == expected
events = get_deposit_events(deposit_id=video_1_depid)
assert len(events) == 1
def check(project_status, video_1_status, video_2_status):
project = deposit_project_resolver(project_depid)
video_1 = deposit_video_resolver(video_1_depid)
video_2 = deposit_video_resolver(video_2_depid)
assert project.status == project_status
assert video_1.status == video_1_status
assert video_2.status == video_2_status
check('draft', 'draft', 'draft')
video_2 = deposit_video_resolver(video_2_depid)
video_2.publish()
check('draft', 'draft', 'published')
project = deposit_project_resolver(project_depid)
project.publish()
check('published', 'published', 'published')
def test_migrate_record(app, location, datadir, es):
"""Test migrate date."""
# create the project
data = load_json(datadir, 'cds_records_demo_1_project.json')
dump = CDSRecordDump(data=data[0])
project = CDSRecordDumpLoader.create(dump=dump)
p_id = project.id
date = '2015-11-13'
assert project['$schema'] == Project.get_record_schema()
assert project['date'] == date
assert project['publication_date'] == date
assert 'license' not in project
assert 'copyright' not in project
assert project['_cds'] == {
"state": {
"file_transcode": "SUCCESS",
"file_video_extract_frames": "SUCCESS",
"file_video_metadata_extraction": "SUCCESS"
},
'modified_by': None,
}
# check project deposit
deposit_project_uuid = PersistentIdentifier.query.filter_by(
pid_type='depid', object_type='rec').one().object_uuid
deposit_project = Record.get_record(deposit_project_uuid)
assert Project._schema in deposit_project['$schema']
assert project.revision_id == deposit_project[
'_deposit']['pid']['revision_id']
assert deposit_project['_deposit']['created_by'] == -1
assert deposit_project['_deposit']['owners'] == [-1]
assert deposit_project['_files'] == []
# create the video
data = load_json(datadir, 'cds_records_demo_1_video.json')
dump = CDSRecordDump(data=data[0])
def load_video(*args, **kwargs):
return open(join(datadir, 'test.mp4'), 'rb')
with mock.patch.object(DataCiteProvider, 'register') as mock_datacite, \
mock.patch.object(
CDSRecordDumpLoader, '_get_migration_file_stream',
return_value=load_video()):
video = CDSRecordDumpLoader.create(dump=dump)
# assert mock_datacite.called is True
project = Record.get_record(p_id)
assert project['videos'] == [
{'$ref': 'https://cds.cern.ch/api/record/1495143'}
]
assert video['$schema'] == Video.get_record_schema()
date = '2012-11-20'
assert video['date'] == date
assert video['publication_date'] == date
assert video['_project_id'] == '2093596'
assert video['license'] == [{
'license': 'CERN',
'url': 'http://copyright.web.cern.ch',
}]
assert video['copyright'] == {
'holder': 'CERN',
'year': '2012',
'url': 'http://copyright.web.cern.ch',
}
assert video['description'] == ''
assert 'doi' in video
assert video['_cds']['state'] == {
"file_transcode": "SUCCESS",
"file_video_extract_frames": "SUCCESS",
"file_video_metadata_extraction": "SUCCESS"
}
assert 'extracted_metadata' in video['_cds']
def check_files(video):
bucket = CDSRecordDumpLoader._get_bucket(record=video)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)]
for file_ in files:
assert as_bucket(file_['bucket_id']) is not None
assert 'checksum' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert FileInstance.query.filter_by(
id=file_['file_id']) is not None
assert 'key' in file_
assert 'links' in file_
assert 'content_type' in file_
assert 'context_type' in file_
assert 'media_type' in file_
assert 'tags' in file_
# check extracted metadata
master_video = CDSVideosFilesIterator.get_master_video_file(video)
assert any([key in master_video['tags']
for key in ExtractMetadataTask._all_keys])
assert any([key in video['_cds']['extracted_metadata']
for key in ExtractMetadataTask._all_keys])
def check_buckets(record, deposit):
def get(key, record):
bucket = CDSRecordDumpLoader._get_bucket(record=record)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)]
return [file_[key] for file_ in files]
def check(record, deposit, file_key, different=None):
values_record = set(get(file_key, record))
values_deposit = set(get(file_key, deposit))
difference = len(values_record - values_deposit)
assert different == difference
def check_tag_master(record):
bucket = CDSRecordDumpLoader._get_bucket(record=record)
master = CDSVideosFilesIterator.get_master_video_file(record)
files = [dump_object(obj)
for obj in ObjectVersion.get_by_bucket(bucket=bucket)
if obj.get_tags().get('master')]
assert all([file_['tags']['master'] == master['version_id']
for file_ in files])
# 1 bucket record != 1 bucket deposit
check(record, deposit, 'bucket_id', 1)
# all file_id are the same except the smil file (only in record)
check(record, deposit, 'file_id', 1)
check(record, deposit, 'key', 1)
# 18 object_version record != 17 object_version deposit
check(record, deposit, 'version_id', 18)
# check tag 'master' where is pointing
check_tag_master(record)
check_tag_master(deposit)
def check_first_level_files(record):
[master] = [file_ for file_ in deposit_video['_files']
if file_['context_type'] == 'master']
assert len(master['subformat']) == 5
assert len(master['frame']) == 10
# TODO assert len(master['playlist']) == ??
assert len([file_ for file_ in deposit_video['_files']
if file_['context_type'] == 'master']) == 1
duration = float(record['_cds']['extracted_metadata']['duration'])
for frame in master['frame']:
assert float(frame['tags']['timestamp']) < duration
assert float(frame['tags']['timestamp']) > 0
# check video deposit
deposit_video_uuid = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == 'depid',
PersistentIdentifier.object_uuid != str(deposit_project_uuid),
PersistentIdentifier.object_type == 'rec'
).one().object_uuid
deposit_video = Video.get_record(str(deposit_video_uuid))
assert Video._schema in deposit_video['$schema']
assert video.revision_id == deposit_video[
'_deposit']['pid']['revision_id']
assert deposit_video['_deposit']['created_by'] == -1
assert deposit_video['_deposit']['owners'] == [-1]
assert len(video['_files']) == 2
assert len(deposit_video['_files']) == 2
check_files(video)
check_files(deposit_video)
check_buckets(video, deposit_video)
check_first_level_files(video)
check_first_level_files(deposit_video)
# try to edit video
deposit_video = deposit_video_resolver(deposit_video['_deposit']['id'])
deposit_video = deposit_video.edit()
# try to edit project
deposit_project = deposit_project_resolver(
deposit_project['_deposit']['id'])
deposit_project = deposit_project.edit()
# try to publish again the video
deposit_video['title']['title'] = 'test'
deposit_video = deposit_video.publish()
_, record_video = deposit_video.fetch_published()
assert record_video['title']['title'] == 'test'