def test_index_words_by_strongs(self):
rc = factory.load(os.path.join(self.resources_dir, 'tw_rc'))
strongs_index = maptwtousfm3.indexWordByStrongs(rc)
expected = {
"G11610": ["abomination"],
"G60000": ["abel"],
"G94600": ["abomination", "test"],
"H62920": ["abomination", "test"],
"H34780": ["12tribesofisrael"],
"H81470": ["12tribesofisrael"],
"G35880": ["abel"],
"H84410": ["abomination", "test"],
"G09760": ["test"],
"G54430": ["12tribesofisrael"],
"G24740": ["12tribesofisrael"],
"H82630": ["test"],
"H82620": ["abomination", "test"],
"H76260": ["12tribesofisrael"],
"H88700": ["abomination", "test"],
"G14270": ["12tribesofisrael"],
"H82510": ["abomination", "test"],
"H01893": ["abel"]
}
assert_object_equals_file(
self, strongs_index,
os.path.join(self.resources_dir, 'expected_strongs_index.json'))
def test_signing_handler_s3(self, mock_reporter):
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
original_item = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertIn('signed', original_item)
self.assertFalse(original_item['signed'])
global_headers = HeaderReader([('last-modified',
'Fri, 03 Jun 2017 20:23:12 GMT'),
('content-length', 12345)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: global_headers)
result = signer.run()
self.assertTrue(result)
self.assertTrue(len(mock_s3._recent_uploads) > 0)
has_prod_uploads = False
for key in mock_s3._recent_uploads:
# assert prod uploads have signatures
if not key.startswith('temp/') and not key.endswith('.sig'):
has_prod_uploads = True
self.assertIn('{}.sig'.format(key), mock_s3._recent_uploads)
self.assertTrue(has_prod_uploads)
updated_item = mock_db.get_item({'repo_name': 'en_obs'}).copy()
assert_object_not_equals(self, updated_item, original_item)
assert_object_equals_file(
self, json.loads(updated_item['package']),
os.path.join(self.resources_dir,
'db/expected_signed_package.json'))
self.assertIn(
'Skipping chapter obs:01 missing url https://cdn.door43.org/en/obs/v4/32kbps/en_obs_01_32kbps.mp3',
mock_logger._messages)
self.assertTrue(updated_item['signed'])
def test_catalog_localization(self, mock_reporter):
mock_instance = MagicMock()
mock_reporter.return_value = mock_instance
state = self.run_with_db('localization.json')
response = state['response']
assert_object_equals_file(
self, response['catalog'],
os.path.join(self.resources_dir, 'v3_catalog_localization.json'))
mock_instance.commit.assert_called_once()
mock_instance.add_error.assert_not_called()
def test_webhook_with_missing_obs_data(self, mock_reporter,
mock_url_exists):
request_file = os.path.join(self.resources_dir,
'obs-missing-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockLogger = MockLogger()
mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/')
self.MockDynamodbHandler.data = None
self.MockS3Handler.reset()
urls = {
'https://git.door43.org/Door43-Catalog/ylb_obs/archive/f8a8d8d757e7ea287cf91b266963f8523bdbd5ad.zip':
'ylb_obs_missing_data.zip'
}
mock_download = lambda url, dest: mockDCS.download_file(
urls[url], dest)
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=self.MockS3Handler,
dynamodb_handler=self.MockDynamodbHandler,
download_handler=mock_download)
handler.run()
entry = self.MockDynamodbHandler.data
self.assertEqual(1, len(self.MockS3Handler.uploads))
self.assertIn('/ylb_obs.zip', self.MockS3Handler.uploads[0]['path'])
self.assertIn(
'temp/ylb_obs/{}/ylb/obs/v4.1/obs.zip'.format(entry['commit_id']),
self.MockS3Handler.uploads[0]['key'])
self.assertEqual('f8a8d8d757', entry['commit_id'])
self.assertEqual(False, entry['dirty'])
self.assertEqual('ylb', entry['language'])
self.assertEqual('2017-04-25T21:46:30+00:00', entry['timestamp'])
self.assertEqual(False, entry['signed'])
self.assertEqual('ylb_obs', entry['repo_name'])
assert_object_equals_file(
self, json.loads(entry['package']),
os.path.join(self.resources_dir,
'expected_obs_package_missing_data.json'))
def test_catalog_complex(self, mock_reporter):
"""
Tests multiple repositories sharing a single resource
and other complex situations
:return:
"""
mock_instance = MagicMock()
mock_reporter.return_value = mock_instance
state = self.run_with_db('complex.json')
assert_object_equals_file(
self, state['response']['catalog'],
os.path.join(self.resources_dir, 'v3_catalog_complex.json'))
mock_instance.commit.assert_called_once()
mock_instance.add_error.assert_not_called()
def test_catalog_mixed_valid_content(self, mock_reporter):
"""
Test with one valid and one invalid record
:return:
"""
state = self.run_with_db('mixed.json')
response = state['response']
self.assertTrue(response['success'])
self.assertIn('Uploaded new catalog', response['message'])
self.assertTrue(response['incomplete'])
# we expect the invalid record to be skipped
assert_object_equals_file(
self, response['catalog'],
os.path.join(self.resources_dir, 'v3_catalog_obs.json'))
def test_webhook_ulb(self, mock_reporter, mock_url_exists):
request_file = os.path.join(self.resources_dir, 'ulb-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockLogger = MockLogger()
mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/')
urls = {
'https://git.door43.org/Door43-Catalog/en_ulb/archive/2fbfd081f46487e48e49090a95c48d45e04e6bed.zip':
'en_ulb.zip'
}
mock_download = lambda url, dest: mockDCS.download_file(
urls[url], dest)
self.MockDynamodbHandler.data = None
self.MockS3Handler.reset()
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=self.MockS3Handler,
dynamodb_handler=self.MockDynamodbHandler,
download_handler=mock_download)
handler.run()
entry = self.MockDynamodbHandler.data
self.assertEqual(4,
len(self.MockS3Handler.uploads)) # books and bundle
self.assertIn('/en_ulb.zip', self.MockS3Handler.uploads[0]['path'])
self.assertEqual('2fbfd081f4', entry['commit_id'])
self.assertEqual(False, entry['dirty'])
self.assertEqual('en', entry['language'])
self.assertEqual('2017-05-02T22:52:04+00:00', entry['timestamp'])
self.assertEqual(False, entry['signed'])
self.assertEqual('en_ulb', entry['repo_name'])
self.assertIn(
'temp/en_ulb/{}/en/ulb/v7/ulb.zip'.format(entry['commit_id']),
self.MockS3Handler.uploads[0]['key'])
assert_object_equals_file(
self, json.loads(entry['package']),
os.path.join(self.resources_dir, 'expected_ulb_package.json'))
def test_catalog_versification_tq(self, mock_reporter):
"""
Tests processing versification for tQ (a help RC).
We are no longer processing versification.
:return:
"""
mock_instance = MagicMock()
mock_reporter.return_value = mock_instance
state = self.run_with_db('versification_tq.json')
response = state['response']
assert_object_equals_file(
self, response['catalog'],
os.path.join(self.resources_dir,
'v3_catalog_versification_tq.json'))
mock_instance.commit.assert_called_once()
def test_catalog_valid_obs_content(self, mock_reporter):
mock_instance = MagicMock()
mock_reporter.return_value = mock_instance
state = self.run_with_db('valid.json')
response = state['response']
mock_errors_db = state['mocks']['db']['errors']
mock_progress_db = state['mocks']['db']['progress']
self.assertTrue(response['success'])
self.assertFalse(response['incomplete'])
self.assertIn('Uploaded new catalog', response['message'])
assert_object_equals_file(
self, response['catalog'],
os.path.join(self.resources_dir, 'v3_catalog_obs.json'))
self.assertEqual(1, len(mock_progress_db._db))
mock_instance.add_error.assert_not_called()
mock_instance.commit.assert_called_once()
def test_catalog_versification_ulb(self, mock_reporter):
"""
Tests processing versification first then ulb.
It's important to test order of processing versification because it can take two code paths.
We are no longer processing versification. This checks that nothing happens.
:return:
"""
mock_instance = MagicMock()
mock_reporter.return_value = mock_instance
state = self.run_with_db('versification_ulb.json')
response = state['response']
assert_object_equals_file(
self, response['catalog'],
os.path.join(self.resources_dir,
'v3_catalog_versification_ulb.json'))
mock_instance.commit.assert_called_once()