def test_do_activity_bad_files(self, fake_emit, fake_session,
fake_storage_context, fake_glencoe_metadata,
fake_store_jpgs, fake_list_files_from_cdn):
# Given
activity_data = test_activity_data.data_example_before_publish
fake_storage_context.return_value = FakeStorageContext()
fake_session.return_value = FakeSession(
test_activity_data.session_example)
fake_glencoe_metadata.return_value = test_activity_data.glencoe_metadata
self.copyglencoestillimages.logger = MagicMock()
fake_list_files_from_cdn.return_value = test_activity_data.cdn_folder_files
fake_store_jpgs.return_value = test_activity_data.jpgs_added_in_cdn
# When
result = self.copyglencoestillimages.do_activity(activity_data)
# Then
self.assertEqual(
result, self.copyglencoestillimages.ACTIVITY_PERMANENT_FAILURE)
fake_emit.assert_called_with(
settings_mock, activity_data["article_id"],
activity_data["version"], activity_data["run"],
self.copyglencoestillimages.pretty_name, "error",
"Not all still images .jpg have a video with the same name " +
"missing videos file names: ['elife-12620-media1', 'elife-12620-media2']"
+ " Please check them against CDN files. Article: 00353")
def test_do_activity_bad_response_glencoe_404(self, request_mock, fake_emit_monitor, fake_session,
fake_storage_context, fake_get_xml_file_name, fake_sleep):
session_example = {
'version': '1',
'article_id': '7777777701234',
'run': '1ee54f9a-cb28-4c8e-8232-4b317cf4beda',
'expanded_folder': '7777777701234.1/1ee54f9a-cb28-4c8e-8232-4b317cf4beda',
'update_date': '2012-12-13T00:00:00Z',
'file_name': 'elife-7777777701234-vor-v1.zip',
'filename_last_element': 'elife-7777777701234-vor-r1.zip',
'eif_location': '7777777701234.1/cf9c7e86-7355-4bb4-b48e-0bc284221251/elife-7777777701234-v1.json'
}
request_mock.return_value = FakeResponse(404, None)
fake_session.return_value = FakeSession(session_example)
fake_storage_context.return_value = FakeStorageContext()
fake_get_xml_file_name.return_value = "anything.xml"
self.verifyglencoe.logger = MagicMock()
result = self.verifyglencoe.do_activity(test_data.ExpandArticle_data)
fake_emit_monitor.assert_called_with(settings_mock,
session_example["article_id"],
session_example["version"],
session_example["run"],
self.verifyglencoe.pretty_name,
"error",
"Glencoe video is not available for article 7777777701234; message: "
"article has no videos - url requested: 10.7554/eLife.01234")
self.assertEqual(result, self.verifyglencoe.ACTIVITY_TEMPORARY_FAILURE)
def test_do_activity(self, mock_storage_context, mock_session,
mock_get_tmp_dir):
mock_storage_context.return_value = FakeStorageContext()
mock_session.return_value = FakeSession(testdata.session_example)
mock_get_tmp_dir.return_value = classes_mock.fake_get_tmp_dir(
testdata.ExpandArticle_path)
self.expandarticle.emit_monitor_event = mock.MagicMock()
self.expandarticle.set_monitor_property = mock.MagicMock()
self.expandarticle.logger = mock.MagicMock()
success = self.expandarticle.do_activity(testdata.ExpandArticle_data)
self.assertEqual(True, success)
# Check destination folder as a list
files = sorted(os.listdir(testdata.ExpandArticle_files_dest_folder))
# self.assertListEqual(testdata.ExpandArticle_files_dest_expected, files)
index = 0
for file in files:
self.assertEqual(
testdata.ExpandArticle_files_dest_bytes_expected[index]
['name'], file)
statinfo = os.stat(testdata.ExpandArticle_files_dest_folder + '/' +
file)
self.assertEqual(
testdata.ExpandArticle_files_dest_bytes_expected[index]
['bytes'], statinfo.st_size)
index += 1
def test_activity(self, fake_session_mock, fake_s3_mock, fake_key_mock,
mock_sqs_message, mock_sqs_connect):
directory = TempDirectory()
for testdata in self.do_activity_passes:
fake_session_mock.return_value = FakeSession(
test_data.PreparePost_session_example(testdata["update_date"]))
mock_sqs_connect.return_value = FakeSQSConn(directory)
mock_sqs_message.return_value = FakeSQSMessage(directory)
fake_s3_mock.return_value = FakeS3Connection()
self.activity_PreparePostEIF.logger = mock.MagicMock()
self.activity_PreparePostEIF.set_monitor_property = mock.MagicMock(
)
self.activity_PreparePostEIF.emit_monitor_event = mock.MagicMock()
success = self.activity_PreparePostEIF.do_activity(
test_data.PreparePostEIF_data)
fake_sqs_queue = FakeSQSQueue(directory)
data_written_in_test_queue = fake_sqs_queue.read(
test_data.PreparePostEIF_test_dir)
self.assertEqual(True, success)
self.assertEqual(json.dumps(testdata["message"]),
data_written_in_test_queue)
output_json = json.loads(
directory.read(test_data.PreparePostEIF_test_dir))
expected = testdata["expected"]
self.assertDictEqual(output_json, expected)
def test_do_activity(self, mock_session, mock_get_file_infos,
mock_store_in_cdn, mock_get_file_pointer):
mock_session.return_value = FakeSession(testdata.session_example)
mock_get_file_infos.return_value = self.fake_get_file_infos()
mock_store_in_cdn.side_effect = self.load_to_cdn
mock_get_file_pointer.side_effect = self.fake_get_file_pointer
self.resizeimages.emit_monitor_event = mock.MagicMock()
self.resizeimages.logger = mock.MagicMock()
success = self.resizeimages.do_activity(testdata.ResizeImages_data)
self.assertEqual('ActivitySuccess', success)
formats = self.resizeimages.get_formats('Figure')
prefix = self.image_prefix #this is the name of the file (without extension) that is an image inside files_source folder
for format_spec_name in formats:
if format_spec_name != 'Original':
format_spec = formats[format_spec_name]
suffix = format_spec['suffix']
width = format_spec['width']
fname = 'tests/test_cdn/' + prefix + suffix + '.' + format_spec[
'format']
real_width, real_height = self.get_image_dimensions(fname)
self.assertEqual(width, real_width)
def test_get_version_silent_corrections_version_in_zip(
self, fake_lookup_functions, fake_session):
fake_session.return_value = FakeSession({})
self.versionlookup.emit_monitor_event = MagicMock()
result = self.versionlookup.do_activity(
data("article_next_version", "elife-00353-vor-v1.zip"))
fake_lookup_functions.assert_not_called()
self.assertEqual(self.versionlookup.ACTIVITY_SUCCESS, result)
def test_activity_permanent_failure(self, fake_emit, fake_storage_context,
fake_session):
fake_storage_context.side_effect = Exception("An error occurred")
fake_session.return_value = FakeSession(
test_activity_data.session_example)
result = self.depositassets.do_activity(activity_data)
self.assertEqual(self.depositassets.ACTIVITY_PERMANENT_FAILURE, result)
def test_activity_success(self, fake_emit, fake_storage_context,
fake_session):
fake_storage_context.return_value = FakeStorageContext()
fake_session.return_value = FakeSession(
test_activity_data.session_example)
result = self.depositassets.do_activity(activity_data)
self.assertEqual(self.depositassets.ACTIVITY_SUCCESS, result)
def test_do_activity_invalid_status(self, session_example,
mock_storage_context, mock_session):
mock_storage_context.return_value = FakeStorageContext()
mock_session.return_value = FakeSession(session_example)
self.expandarticle.logger = mock.MagicMock()
success = self.expandarticle.do_activity(
testdata.ExpandArticle_data_invalid_status)
self.assertEqual(self.expandarticle.ACTIVITY_PERMANENT_FAILURE,
success)
def test_activity_success(self, fake_emit, fake_storage_context,
fake_session, fake_gen_images):
fake_storage_context.return_value = FakeStorageContext()
fake_session.return_value = FakeSession(
test_activity_data.session_example)
activity_data = test_activity_data.data_example_before_publish
result = self.convertimagestojpg.do_activity(activity_data)
self.assertEqual(self.convertimagestojpg.ACTIVITY_SUCCESS, result)
def test_get_version_normal_process_version_not_in_zip(
self, fake_lookup_functions, fake_session):
fake_session.return_value = FakeSession({})
fake_lookup_functions.return_value = "1"
self.versionlookup.emit_monitor_event = MagicMock()
result = self.versionlookup.do_activity(
data("article_next_version", "elife-00353-vor-r1.zip"))
fake_lookup_functions.assert_called_with(
provider.lax_provider.article_next_version, '00353', settings_mock)
self.assertEqual(self.versionlookup.ACTIVITY_SUCCESS, result)
def test_do_activity_success_POA(self, fake_emit, fake_session):
# Given
activity_data = test_activity_data.data_example_before_publish
session_POA = test_activity_data.session_example.copy()
session_POA['file_name'] = 'elife-00353-poa-v1.zip'
fake_session.return_value = FakeSession(session_POA)
# When
result = self.copyglencoestillimages.do_activity(activity_data)
# Then
self.assertEqual(self.copyglencoestillimages.ACTIVITY_SUCCESS, result)
def test_activity_failure(self, fake_emit, fake_storage_context,
fake_session):
fake_storage_context.side_effect = Exception("An error occurred")
fake_session.return_value = FakeSession(
test_activity_data.session_example)
activity_data = test_activity_data.data_example_before_publish
result = self.convertimagestojpg.do_activity(activity_data)
self.assertEqual(self.convertimagestojpg.ACTIVITY_PERMANENT_FAILURE,
result)
def test_do_activity_success(self, fake_retrieve_endpoints_check,
fake_session, fake_storage_context):
# Given
data = test_data.data_example_before_publish
fake_retrieve_endpoints_check.return_value = [(True, "test.path")]
fake_session.return_value = FakeSession(test_data.session_example)
fake_storage_context.return_value = FakeStorageContext()
self.verifyimageserver.emit_monitor_event = MagicMock()
self.verifyimageserver.logger = MagicMock()
# When
result = self.verifyimageserver.do_activity(data)
# Then
self.assertEqual(result, self.verifyimageserver.ACTIVITY_SUCCESS)
def test_do_activity_invalid_articleid(self, mock_storage_context,
mock_session):
mock_storage_context.return_value = FakeStorageContext()
mock_session.return_value = FakeSession(testdata.session_example)
self.expandarticle.logger = mock.MagicMock()
self.expandarticle.emit_monitor_event = mock.MagicMock()
self.expandarticle.set_monitor_property = mock.MagicMock()
success = self.expandarticle.do_activity(
testdata.ExpandArticle_data_invalid_article)
self.assertEqual(self.expandarticle.ACTIVITY_PERMANENT_FAILURE,
success)
def test_error_no_eif_location(self, data, fake_session,
fake_emit_monitor_event):
fake_session.return_value = FakeSession({})
self.seteifpublish.logger = MagicMock()
result = self.seteifpublish.do_activity(data)
fake_emit_monitor_event.assert_called_with(
settings_mock, data['article_id'], data['version'], data['run'],
self.seteifpublish.pretty_name, "error",
"eif_location not available")
self.assertEqual(result, self.seteifpublish.ACTIVITY_PERMANENT_FAILURE)
def test_error_fetch_eif(self, data, fake_session, fake_get_eif,
fake_emit_monitor_event):
fake_session.return_value = FakeSession(test_data.session_example)
fake_get_eif.side_effect = Exception('Value error')
self.seteifpublish.logger = MagicMock()
result = self.seteifpublish.do_activity(data)
fake_emit_monitor_event.assert_called_with(
settings_mock, data['article_id'], data['version'], data['run'],
self.seteifpublish.pretty_name, "error",
"Could not fetch/load EIF data. Error details: Value error")
self.assertEqual(result, self.seteifpublish.ACTIVITY_PERMANENT_FAILURE)
def test_do_activity_error(self, fake_retrieve_endpoints_check,
fake_session, fake_storage_context):
# Given
data = test_data.data_example_before_publish
fake_retrieve_endpoints_check.side_effect = Exception("Error!")
fake_session.return_value = FakeSession(test_data.session_example)
fake_storage_context.return_value = FakeStorageContext()
self.verifyimageserver.emit_monitor_event = MagicMock()
self.verifyimageserver.logger = MagicMock()
# When
result = self.verifyimageserver.do_activity(data)
# Then
self.assertEqual(result,
self.verifyimageserver.ACTIVITY_PERMANENT_FAILURE)
def test_get_version_date_silent_corrections(self,
fake_date_lookup_function,
fake_session):
fake_session_obj = FakeSession(testdata.data_example_before_publish)
fake_session.return_value = fake_session_obj
fake_date_lookup_function.return_value = '2015-11-30T00:00:00Z'
self.versiondatelookup.emit_monitor_event = MagicMock()
result = self.versiondatelookup.do_activity(testdata.raw_data_activity)
fake_date_lookup_function.assert_called_with('00353', "1",
settings_mock)
self.assertEqual(self.versiondatelookup.ACTIVITY_SUCCESS, result)
self.assertEqual(fake_session_obj.session_dict["update_date"],
'2015-11-30T00:00:00Z')
def test_get_version_silent_corrections_version_date_in_zip(
self, fake_lookup_functions, fake_session):
data_rep = testdata.data_example_before_publish.copy()
data_rep[
'filename_last_element'] = "elife-00353-vor-v1-20121213000000.zip"
fake_session_obj = FakeSession(data_rep)
fake_session.return_value = fake_session_obj
self.versiondatelookup.emit_monitor_event = MagicMock()
result = self.versiondatelookup.do_activity(testdata.raw_data_activity)
fake_lookup_functions.assert_not_called()
self.assertEqual(self.versiondatelookup.ACTIVITY_SUCCESS, result)
self.assertEqual(fake_session_obj.session_dict["update_date"],
'2012-12-13T00:00:00Z')
def test_do_activity(self, data, fake_session, fake_get_eif,
fake_storage_context, fake_emit_monitor_event):
fake_session.return_value = FakeSession(test_data.session_example)
fake_get_eif.return_value = test_data.json_output_parameter_example
fake_storage_context.return_value = FakeStorageContext()
self.seteifpublish.logger = MagicMock()
result = self.seteifpublish.do_activity(data)
fake_emit_monitor_event.assert_called_with(
settings_mock, data['article_id'], data['version'], data['run'],
self.seteifpublish.pretty_name, "end",
"Finished to set EIF to publish")
self.assertEqual(result, self.seteifpublish.ACTIVITY_SUCCESS)
def test_do_activity_success_no_videos_for_article(
self, fake_emit, fake_session, fake_storage_context,
fake_glencoe_metadata, fake_store_jpgs, fake_list_files_from_cdn):
# Given
activity_data = test_activity_data.data_example_before_publish
fake_storage_context.return_value = FakeStorageContext()
fake_session.return_value = FakeSession(
test_activity_data.session_example)
fake_glencoe_metadata.side_effect = AssertionError(
"article has no videos - url requested: ...")
# When
result = self.copyglencoestillimages.do_activity(activity_data)
# Then
self.assertEqual(self.copyglencoestillimages.ACTIVITY_SUCCESS, result)
def test_do_activity_no_version_error(self, session_example, mock_session, fake_emit_monitor_event):
#given
session_example = session_example.copy()
del session_example['version']
mock_session.return_value = FakeSession(session_example)
data = test_data.ApplyVersionNumber_data_no_renaming
#when
result = self.applyversionnumber.do_activity(data)
#then
fake_emit_monitor_event.assert_called_with(settings_mock, session_example['article_id'], None, data['run'],
self.applyversionnumber.pretty_name, "error",
"Error in applying version number to files for " +
session_example['article_id'] +
" message: No version available")
self.assertEqual(result, self.applyversionnumber.ACTIVITY_PERMANENT_FAILURE)
def test_get_version_date_silent_corrections_error(
self, fake_date_lookup_function, fake_session):
fake_session_obj = FakeSession(testdata.data_example_before_publish)
fake_session.return_value = fake_session_obj
fake_date_lookup_function.side_effect = Exception(
"Error in article_publication_date_by_version: "
"Version date not found. Status: 500")
self.versiondatelookup.emit_monitor_event = MagicMock()
self.versiondatelookup.logger = MagicMock()
result = self.versiondatelookup.do_activity(testdata.raw_data_activity)
fake_date_lookup_function.assert_called_with('00353', "1",
settings_mock)
self.assertRaises(Exception, fake_date_lookup_function)
self.assertEqual(self.versiondatelookup.ACTIVITY_PERMANENT_FAILURE,
result)
def test_do_activity(self, fake_session_mock, fake_s3_mock, fake_key_mock, fake_get_article_xml_key, fake_add_update_date_to_json):
directory = TempDirectory()
#preparing Mocks
fake_session_mock.return_value = FakeSession(data.session_example)
fake_s3_mock.return_value = FakeS3Connection()
fake_key_mock.return_value = FakeKey(directory, data.bucket_dest_file_name)
fake_get_article_xml_key.return_value = FakeKey(directory), data.bucket_origin_file_name
fake_add_update_date_to_json.return_value = data.json_output_return_example_string
self.jats.emit_monitor_event = mock.MagicMock()
self.jats.set_dashboard_properties = mock.MagicMock()
success = self.jats.do_activity(testdata.ExpandArticle_data)
self.assertEqual(success, True)
output_json = json.loads(directory.read("test_dest.json"))
expected = data.json_output_return_example
self.assertDictEqual(output_json, expected)
def test_error_update_eif(self, data, fake_session, fake_get_eif,
fake_storage_context, fake_emit_monitor_event):
fake_session.return_value = FakeSession(test_data.session_example)
fake_get_eif.return_value = None
fake_storage_context.return_value = FakeStorageContext()
self.seteifpublish.logger = MagicMock()
result = self.seteifpublish.do_activity(data)
fake_emit_monitor_event.assert_called_with(
settings_mock, data['article_id'], data['version'], data['run'],
self.seteifpublish.pretty_name, "error",
"There is something wrong with EIF data and/or we could not upload "
"it. Error details: 'NoneType' object does not support item "
"assignment")
self.assertEqual(result, self.seteifpublish.ACTIVITY_PERMANENT_FAILURE)
def test_do_activity_bad_response_glencoe_500(self, request_mock, fake_emit_monitor, fake_session,
fake_storage_context, fake_get_xml_file_name, fake_sleep):
request_mock.return_value = FakeResponse(500, None)
fake_session.return_value = FakeSession(test_data.session_example)
fake_storage_context.return_value = FakeStorageContext()
fake_get_xml_file_name.return_value = "anything.xml"
self.verifyglencoe.logger = MagicMock()
result = self.verifyglencoe.do_activity(test_data.ExpandArticle_data)
fake_emit_monitor.assert_called_with(settings_mock,
test_data.session_example["article_id"],
test_data.session_example["version"],
test_data.session_example["run"],
self.verifyglencoe.pretty_name,
"error",
'Glencoe video is not available for article 00353; message: '
'unhandled status code from Glencoe: 500 - '
'url requested: 10.7554/eLife.00353')
self.assertEqual(result, self.verifyglencoe.ACTIVITY_TEMPORARY_FAILURE)
def test_do_activity(self, fake_emit, fake_session, fake_storage_context,
fake_glencoe_metadata, fake_store_jpgs,
fake_list_files_from_cdn):
# Given
activity_data = test_activity_data.data_example_before_publish
fake_storage_context.return_value = FakeStorageContext()
fake_session.return_value = FakeSession(
test_activity_data.session_example)
fake_glencoe_metadata.return_value = test_activity_data.glencoe_metadata
fake_store_jpgs.return_value = test_activity_data.jpgs_added_in_cdn
fake_list_files_from_cdn.return_value = test_activity_data.cdn_folder_files + \
test_activity_data.jpgs_added_in_cdn
# When
result = self.copyglencoestillimages.do_activity(activity_data)
# Then
self.assertEqual(self.copyglencoestillimages.ACTIVITY_SUCCESS, result)
def test_do_acitvity_exception(self, fake_emit_monitor, fake_session, fake_storage_context, fake_get_xml_file_name,
fake_glencoe_check_validate_sources, fake_glencoe_check_metadata):
fake_session.return_value = FakeSession(test_data.session_example)
fake_storage_context.return_value = FakeStorageContext()
fake_get_xml_file_name.return_value = "anything.xml"
self.verifyglencoe.logger = MagicMock()
fake_glencoe_check_validate_sources.side_effect = Exception("Fake Time out")
result = self.verifyglencoe.do_activity(test_data.ExpandArticle_data)
fake_emit_monitor.assert_called_with(settings_mock,
test_data.session_example["article_id"],
test_data.session_example["version"],
test_data.session_example["run"],
self.verifyglencoe.pretty_name,
"error",
"An error occurred when checking for Glencoe video. Article 00353; message: "
"Fake Time out")
self.assertEqual(result, self.verifyglencoe.ACTIVITY_PERMANENT_FAILURE)
def test_do_activity_error(self, fake_emit, fake_session,
fake_storage_context, fake_glencoe_metadata,
fake_store_jpgs):
# Given
activity_data = test_activity_data.data_example_before_publish
fake_storage_context.return_value = FakeStorageContext()
fake_session.return_value = FakeSession(
test_activity_data.session_example)
fake_glencoe_metadata.return_value = test_activity_data.glencoe_metadata
fake_store_jpgs.side_effect = Exception("Something went wrong!")
# When
result = self.copyglencoestillimages.do_activity(activity_data)
# Then
self.assertEqual(
result, self.copyglencoestillimages.ACTIVITY_PERMANENT_FAILURE)
fake_emit.assert_called_with(
settings_mock, activity_data["article_id"],
activity_data["version"], activity_data["run"],
self.copyglencoestillimages.pretty_name, "error",
"An error occurred when checking/copying Glencoe still images. Article "
+ activity_data["article_id"] + "; message: Something went wrong!")