def test_webhook_ulb_pull_request(self, mock_reporter, mock_url_exists):
request_file = os.path.join(self.resources_dir,
'ulb-pull-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockLogger = MockLogger()
mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/')
urls = {
'https://git.door43.org/Door43-Catalog/en_ulb/archive/2fbfd081f46487e48e49090a95c48d45e04e6bed.zip':
'en_ulb.zip'
}
mock_download = lambda url, dest: mockDCS.download_file(
urls[url], dest)
self.MockDynamodbHandler.data = None
self.MockS3Handler.reset()
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=self.MockS3Handler,
dynamodb_handler=self.MockDynamodbHandler,
download_handler=mock_download)
with self.assertRaises(Exception) as error_context:
handler.run()
self.assertIn('Skipping un-merged pull request',
str(error_context.exception))
entry = self.MockDynamodbHandler.data
self.assertEqual(0, len(self.MockS3Handler.uploads))
self.assertIsNone(entry)
def test_webhook_with_missing_obs_data(self, mock_reporter,
mock_url_exists):
request_file = os.path.join(self.resources_dir,
'obs-missing-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockLogger = MockLogger()
mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/')
self.MockDynamodbHandler.data = None
self.MockS3Handler.reset()
urls = {
'https://git.door43.org/Door43-Catalog/ylb_obs/archive/f8a8d8d757e7ea287cf91b266963f8523bdbd5ad.zip':
'ylb_obs_missing_data.zip'
}
mock_download = lambda url, dest: mockDCS.download_file(
urls[url], dest)
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=self.MockS3Handler,
dynamodb_handler=self.MockDynamodbHandler,
download_handler=mock_download)
handler.run()
entry = self.MockDynamodbHandler.data
self.assertEqual(1, len(self.MockS3Handler.uploads))
self.assertIn('/ylb_obs.zip', self.MockS3Handler.uploads[0]['path'])
self.assertIn(
'temp/ylb_obs/{}/ylb/obs/v4.1/obs.zip'.format(entry['commit_id']),
self.MockS3Handler.uploads[0]['key'])
self.assertEqual('f8a8d8d757', entry['commit_id'])
self.assertEqual(False, entry['dirty'])
self.assertEqual('ylb', entry['language'])
self.assertEqual('2017-04-25T21:46:30+00:00', entry['timestamp'])
self.assertEqual(False, entry['signed'])
self.assertEqual('ylb_obs', entry['repo_name'])
assert_object_equals_file(
self, json.loads(entry['package']),
os.path.join(self.resources_dir,
'expected_obs_package_missing_data.json'))
def test_signing_handler_already_signed(self, mock_reporter):
event = self.create_event()
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_signed.json'))
mock_logger = MockLogger()
original_record = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertFalse(original_record['signed'])
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
updated_record = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertTrue(updated_record['signed'])
def test_signing_handler_text_wrong_key(self, mock_reporter):
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
mock_logger = MockLogger()
# TRICKY: a wrong signing key will result in failed verification
self.mock_signer._fail_verification()
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
for f in mock_s3._recent_uploads:
# assert nothing was uploaded to production
self.assertTrue(f.startswith('temp/'))
self.assertFalse(f.endswith('.sig'))
self.assertIn('The signature was not successfully verified.',
mock_logger._messages)
def test_skip_signing_large_file(self, mock_reporter):
"""
Ensure that large files are not signed.
Because lambda functions have limited disk space.
:return:
"""
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_logger.warning = MagicMock()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.sign_given_url"],
"chapters": [],
"contributor": [
"Narrator: Steve Lossing", "Checker: Brad Harrington",
"Engineer: Brad Harrington"
],
"format":
"",
"modified":
"",
"quality":
"64kbps",
"signature":
"",
"size":
0,
"url":
"https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip"
}
mockHeaders = HeaderReader([('content-length',
SigningHandler.max_file_size + 1)])
signer = SigningHandler(event=event,
context=None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, None, None, format)
mock_logger.warning.assert_called_once_with(
'File is too large to sign https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip'
)
self.assertTrue(already_signed)
self.assertEqual(
'https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip.sig',
format['signature'])
self.assertTrue(newly_signed)
def test_webhook_ulb(self, mock_reporter, mock_url_exists):
request_file = os.path.join(self.resources_dir, 'ulb-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockLogger = MockLogger()
mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/')
urls = {
'https://git.door43.org/Door43-Catalog/en_ulb/archive/2fbfd081f46487e48e49090a95c48d45e04e6bed.zip':
'en_ulb.zip'
}
mock_download = lambda url, dest: mockDCS.download_file(
urls[url], dest)
self.MockDynamodbHandler.data = None
self.MockS3Handler.reset()
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=self.MockS3Handler,
dynamodb_handler=self.MockDynamodbHandler,
download_handler=mock_download)
handler.run()
entry = self.MockDynamodbHandler.data
self.assertEqual(4,
len(self.MockS3Handler.uploads)) # books and bundle
self.assertIn('/en_ulb.zip', self.MockS3Handler.uploads[0]['path'])
self.assertEqual('2fbfd081f4', entry['commit_id'])
self.assertEqual(False, entry['dirty'])
self.assertEqual('en', entry['language'])
self.assertEqual('2017-05-02T22:52:04+00:00', entry['timestamp'])
self.assertEqual(False, entry['signed'])
self.assertEqual('en_ulb', entry['repo_name'])
self.assertIn(
'temp/en_ulb/{}/en/ulb/v7/ulb.zip'.format(entry['commit_id']),
self.MockS3Handler.uploads[0]['key'])
assert_object_equals_file(
self, json.loads(entry['package']),
os.path.join(self.resources_dir, 'expected_ulb_package.json'))
def test_webhook_ulb_merged_pull_request(self, mock_reporter,
mock_url_exists):
request_file = os.path.join(self.resources_dir,
'ulb-merged-pull-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockLogger = MockLogger()
mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/')
urls = {
'https://git.door43.org/Door43-Catalog/ta_ulb/archive/0a7e25cd939f00086262fe94b9d25afc3b5dabd3.zip':
'ta_ulb.zip'
}
mock_download = lambda url, dest: mockDCS.download_file(
urls[url], dest)
self.MockDynamodbHandler.data = None
self.MockS3Handler.reset()
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=self.MockS3Handler,
dynamodb_handler=self.MockDynamodbHandler,
download_handler=mock_download)
handler.run()
entry = self.MockDynamodbHandler.data
self.assertEqual(4,
len(self.MockS3Handler.uploads)) # books and bundle
self.assertIn('/ta_ulb.zip', self.MockS3Handler.uploads[0]['path'])
self.assertEqual('0a7e25cd93', entry['commit_id'])
self.assertEqual(False, entry['dirty'])
self.assertEqual('ta', entry['language'])
self.assertEqual('2017-08-17T18:56:52.884140+00:00',
entry['timestamp'])
self.assertEqual(False, entry['signed'])
self.assertEqual('ta_ulb', entry['repo_name'])
self.assertIn(
'temp/ta_ulb/{}/ta/ulb/v3/ulb.zip'.format(entry['commit_id']),
self.MockS3Handler.uploads[0]['key'])
def test_signing_handler_s3(self, mock_reporter):
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
original_item = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertIn('signed', original_item)
self.assertFalse(original_item['signed'])
global_headers = HeaderReader([('last-modified',
'Fri, 03 Jun 2017 20:23:12 GMT'),
('content-length', 12345)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: global_headers)
result = signer.run()
self.assertTrue(result)
self.assertTrue(len(mock_s3._recent_uploads) > 0)
has_prod_uploads = False
for key in mock_s3._recent_uploads:
# assert prod uploads have signatures
if not key.startswith('temp/') and not key.endswith('.sig'):
has_prod_uploads = True
self.assertIn('{}.sig'.format(key), mock_s3._recent_uploads)
self.assertTrue(has_prod_uploads)
updated_item = mock_db.get_item({'repo_name': 'en_obs'}).copy()
assert_object_not_equals(self, updated_item, original_item)
assert_object_equals_file(
self, json.loads(updated_item['package']),
os.path.join(self.resources_dir,
'db/expected_signed_package.json'))
self.assertIn(
'Skipping chapter obs:01 missing url https://cdn.door43.org/en/obs/v4/32kbps/en_obs_01_32kbps.mp3',
mock_logger._messages)
self.assertTrue(updated_item['signed'])
def test_create_v2_catalog(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV2Api = MockAPI(os.path.join(self.resources_dir, 'v2_api'),
'https://test')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
converter.run()
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/uw/catalog.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/uw/obs/en/obs/v4/source.json')
assert_s3_equals_api_text(self, mockS3, mockV2Api,
'v2/uw/gen/en/udb/v7/gen.usfm')
assert_s3_equals_api_text(self, mockS3, mockV2Api,
'v2/uw/1ch/en/ulb/v7/1ch.usfm')
self.assertIn('v2/uw/obs/en/obs/v4/source.json.sig',
mockS3._recent_uploads)
self.assertIn('uw/txt/2/catalog.json', mockS3._recent_uploads)
self.assertIn(
'en_udb_1ch: media format "https://cdn.door43.org/en/udb/v9/1ch.pdf" does not match source version "7" and will be excluded.',
mockLogger._messages)
self.assertIn(
'en_obs_obs: media format "https://cdn.door43.org/en/obs/v999/129kbps/en_obs_129kbps.zip" does not match source version "4" and will be excluded.',
mockLogger._messages)
def make_handler_instance(self, progress_db):
"""
Generates a new handler instance for testing
:param progress_db:
:return:
"""
progress_db_file = os.path.join(self.resources_dir,
'progress_db/{}'.format(progress_db))
self.assertTrue(os.path.isfile(progress_db_file))
# now run the code
event = self.create_event()
mock_progress_db = MockDynamodbHandler()
mock_progress_db._load_db(progress_db_file)
mock_status_db = MockDynamodbHandler()
mock_errors_db = MockDynamodbHandler()
dbs = {
'd43-catalog-in-progress': mock_progress_db,
'd43-catalog-status': mock_status_db,
'd43-catalog-errors': mock_errors_db
}
mock_ses = MockSESHandler()
mock_s3 = MockS3Handler()
mock_checker = MockChecker()
mock_api = MockAPI(os.path.join(self.resources_dir), '/')
mock_ses_handler = lambda: mock_ses
mock_s3_handler = lambda bucket: mock_s3
mock_dbs_handler = lambda bucket: dbs[bucket]
mock_checker_handler = lambda: mock_checker
handler = CatalogHandler(event,
None,
s3_handler=mock_s3_handler,
dynamodb_handler=mock_dbs_handler,
ses_handler=mock_ses_handler,
consistency_checker=mock_checker_handler,
url_exists_handler=lambda url: True,
get_url_handler=mock_api.get_url)
return {
'handler': handler,
'event': event,
'mocks': {
'db': {
'progress': mock_progress_db,
'status': mock_status_db,
'errors': mock_errors_db
},
'api': mock_api,
'ses': mock_ses,
's3': mock_s3,
'checker': mock_checker
}
}
def test_broken_catalog(self, mock_reporter):
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'broken_api'),
'https://api.door43.org/')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json'))
mockLog = MockLogger()
mock_get_url = lambda url, catch_exception: mockV3Api.get_url(
url, catch_exception)
mock_download = lambda url, dest: mockV3Api.download_file(url, dest)
event = self.make_event()
converter = TsV2CatalogHandler(event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mock_get_url,
download_handler=mock_download,
url_exists_handler=lambda url: False)
result = converter.run()
self.assertFalse(result)
self.assertIn(
'Failed to load the catalog json: No JSON object could be decoded',
mockLog._messages)
def test_missing_catalog(self, mock_reporter):
mockV3Api = MockAPI(self.resources_dir, 'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://test-cdn.door43.org/')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json'))
mockLog = MockLogger()
event = self.make_event()
converter = TsV2CatalogHandler(
event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
url_exists_handler=lambda url: False)
result = converter.run()
self.assertFalse(result)
self.assertIn('https://api.door43.org/v3/catalog.json does not exist',
mockLog._messages)
def test_signing_small_file(self, mock_reporter):
"""
Ensure that small files are signed properly
:return:
"""
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.sign_given_url"],
"chapters": [],
"contributor": [
"Narrator: Steve Lossing", "Checker: Brad Harrington",
"Engineer: Brad Harrington"
],
"format":
"",
"modified":
"",
"quality":
"64kbps",
"signature":
"",
"size":
0,
"url":
"https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip"
}
mockHeaders = HeaderReader([('content-length', 123)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, None, None, format)
self.assertEqual(
'https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip.sig',
format['signature'])
self.assertNotIn(
'File is too large to sign https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip',
mock_logger._messages)
self.assertFalse(already_signed)
self.assertTrue(newly_signed)
def test_complete_status(self, mock_reporter):
mockV3Api = MockAPI(self.resources_dir, 'https://cdn.door43.org/')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir, 'complete_db.json'))
mockLog = MockLogger()
mock_get_url = lambda url, catch_exception: mockV3Api.get_url(
url, catch_exception)
mock_download = lambda url, dest: mockV3Api.download_file(url, dest)
event = self.make_event()
converter = TsV2CatalogHandler(event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mock_get_url,
download_handler=mock_download,
url_exists_handler=lambda url: True)
result = converter.run()
self.assertTrue(result)
self.assertEqual(0, len(mockS3._recent_uploads))
self.assertIn('Catalog already generated', mockLog._messages)
def test_signing_handler_text_no_records(self, mock_reporter):
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_s3 = MockS3Handler()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
handler = SigningHandler(event,
None,
logger=MockLogger(),
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = handler.run()
self.assertFalse(result)
def test_index_inconsistent_image_titles(self):
mockApi = MockAPI(self.resources_dir, 'https://example.com')
format = {
'format': 'type=book',
'url': 'https://example.com/en_obs.zip'
}
response = index_obs('en', 'obs', format, self.temp_dir,
mockApi.download_file)
chapters = response['chapters']
self.assertEqual(2, len(chapters))
self.assertEqual('01', chapters[0]['number'])
self.assertEqual(16, len(chapters[0]['frames']))
frames = chapters[0]['frames']
self.assertEqual('01-01', frames[0]['id'])
self.assertEqual('01-02', frames[1]['id'])
self.assertEqual('02', chapters[1]['number'])
self.assertEqual(12, len(chapters[1]['frames']))
def test_signing_handler_text_missing_file(self, mock_url_headers,
mock_reporter):
"""
Signing will continue to run even if a file is missing.
The missing file will just be ignored.
:return:
"""
mock_url_headers.return_value = HeaderReader([
('last-modified', 'Fri, 03 Jun 2017 20:23:12 GMT'),
('content-length', 12345)
])
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
mock_logger = MockLogger()
mock_s3 = MockS3Handler()
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
mock_instance.add_error.assert_called_once_with(
'The file "obs.zip" could not be downloaded: File not found for key: temp/en_obs/f8a8d8d757/en/obs/v4/obs.zip'
)
def test_signing_remote_3gp_file(self, mock_reporter):
"""
Ensure that small files are signed properly
:return:
"""
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'filedn'),
'https://filedn.com/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.sign_given_url"],
"chapters": [],
"contributor": [],
"format": "",
"modified": "",
"quality": "3GP",
"signature": "",
"size": 0,
"url": "https://filedn.com/XYZ/en/obs/v6/en_obs_v6_3GP.zip"
}
mockHeaders = HeaderReader([('content-length', 345)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, None, None, format)
self.assertEqual('', format['signature'])
self.assertEqual('application/zip; content=video/3gpp',
format['format'])
self.assertEqual(116, format['size'])
self.assertTrue(already_signed)
self.assertTrue(newly_signed)
def test_status_not_ready(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'not_ready_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
result = converter._get_status()
self.assertFalse(result)
def test_signing_obs_html(self, mock_reporter):
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.html_format"],
"format": "text/html",
"modified": "",
"signature": "",
"size": "",
# NOTE: this is not the actual url format used
"url": "https://cdn.door43.org/temp/en_obs/f8a8d8d757/en/obs.html"
}
dublin_core = {"identifier": "obs", "language": {"identifier": "en"}}
mockHeaders = HeaderReader([('content-length', 123)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, dublin_core, None, format)
self.assertEqual(
'https://cdn.door43.org/en/obs/v3/media/html/obs.html',
format['url'])
self.assertEqual(
'https://cdn.door43.org/en/obs/v3/media/html/obs.html.sig',
format['signature'])
self.assertFalse(already_signed)
self.assertTrue(newly_signed)
def test_status_ready_new_db(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
(status, source_status) = converter._get_status()
self.assertEqual('complete', source_status['state'])
self.assertEqual('in-progress', status['state'])
self.assertEqual(0, len(status['processed']))
def test_malformed_manifest(self, mock_reporter, mock_url_exists):
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
request_file = os.path.join(self.resources_dir,
'malformed-manifest.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
# deserialized object
request_json = json.loads(content)
mockS3 = MockS3Handler()
mockDb = MockDynamodbHandler()
mockLogger = MockLogger()
mockApi = MockAPI(os.path.join(self.resources_dir, 'git_api'),
'https://git.door43.org/')
handler = WebhookHandler(event=request_json,
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDb,
download_handler=mockApi.download_file)
with self.assertRaises(Exception) as error_context:
handler.run()
self.assertIn('manifest missing dublin_core key "type"',
str(error_context.exception))
self.assertFalse(os.path.isdir(handler.temp_dir))
mock_instance.add_error.assert_called_once_with(
'Bad Manifest: manifest missing dublin_core key "type"')
def test_webhook_versification(self, mock_reporter, mock_url_exists):
"""
We are not currently processing versification.
Therefore, we test that nothing happens.
:param mock_report_error:
:return:
"""
request_file = os.path.join(self.resources_dir,
'versification-request.json')
with codecs.open(request_file, 'r', encoding='utf-8') as in_file:
request_text = in_file.read()
# convert Windows line endings to Linux line endings
content = request_text.replace('\r\n', '\n')
request_json = json.loads(content)
urls = {
'https://git.door43.org/Door43-Catalog/versification/archive/c7e936e4dcc103560987c8475db69e292aa66dca.zip':
'versification.zip'
}
mockLogger = MockLogger()
mock_api = MockAPI(self.resources_dir, 'https://git.door43.org')
mock_db = MockDynamodbHandler()
mock_s3 = MockS3Handler()
handler = WebhookHandler(request_json,
context=None,
s3_handler=mock_s3,
logger=mockLogger,
dynamodb_handler=mock_db,
download_handler=lambda url, dest: mock_api.
download_file(urls[url], dest))
handler.run()
self.assertEqual(0, len(mock_s3._recent_uploads))
data = mock_db._last_inserted_item
self.assertEqual(None, data)
def test_signing_handler_invalid_manifest(self, mock_reporter):
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_db._load_db(os.path.join(self.resources_dir, 'db/invalid.json'))
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
mock_logger = MockLogger()
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
for f in mock_s3._recent_uploads:
# assert nothing was uploaded to production
self.assertTrue(f.startswith('temp/'))
self.assertFalse(f.endswith('.sig'))
# self.assertIn('Skipping unit-test. Bad Manifest: No JSON object could be decoded', mock_logger._messages)
mock_instance.add_error.assert_called_once_with(
'Skipping unit-test. Bad Manifest: No JSON object could be decoded'
)
def test_inprogress(self, mock_reporter):
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://test-cdn.door43.org/')
mockV2Api = MockAPI(os.path.join(self.resources_dir, 'ts_api'),
'https://test')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir,
'ready_inprogress_db.json'))
mockLog = MockLogger()
event = self.make_event()
converter = TsV2CatalogHandler(
event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
url_exists_handler=lambda url: True)
converter.run()
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/catalog.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/languages.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/en/resources.json')
self.assertNotIn('v2/ts/obs/en/obs/source.json',
mockS3._recent_uploads)
# assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/obs/source.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/en/notes.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/en/questions.json')
# we have frozen tw_cat
self.assertNotIn('v2/ts/obs/en/tw_cat.json', mockS3._recent_uploads)
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/languages.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/en/resources.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/en/ulb/v7/source.json')
self.assertNotIn('v2/ts/1ch/en/notes.json', mockS3._recent_uploads)
# assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/notes.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/en/questions.json')
self.assertNotIn('v2/ts/1ch/en/tw_cat.json', mockS3._recent_uploads)
# assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/tw_cat.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/bible/en/words.json')
# validate urls in generate catalogs match the generated output paths
root_url = '{}/'.format(
event['stage-variables']['cdn_url'].rstrip('/'))
catalog = json.loads(
read_file(mockS3._recent_uploads['v2/ts/catalog.json']))
url_err_msg = 'url in catalog does not match upload path: {}'
for project in catalog:
lang_catalog_path = project['lang_catalog'].replace(
root_url, '').split('?')[0]
self.assertIn(lang_catalog_path, mockS3._recent_uploads,
url_err_msg.format(lang_catalog_path))
lang_catalog = json.loads(
read_file(mockS3._recent_uploads[lang_catalog_path]))
for language in lang_catalog:
res_catalog_path = language['res_catalog'].replace(
root_url, '').split('?')[0]
self.assertIn(res_catalog_path, mockS3._recent_uploads,
url_err_msg.format(res_catalog_path))
res_catalog = json.loads(
read_file(mockS3._recent_uploads[res_catalog_path]))
for resource in res_catalog:
questions_path = resource['checking_questions'].replace(
root_url, '').split('?')[0]
# notes_path = resource['notes'].replace(root_url, '').split('?')[0]
# source_path = resource['source'].replace(root_url, '').split('?')[0]
terms_path = resource['terms'].replace(root_url,
'').split('?')[0]
# terms_map_path = resource['tw_cat'].replace(root_url, '').split('?')[0]
if questions_path:
self.assertIn(questions_path, mockS3._recent_uploads,
url_err_msg.format(questions_path))
# if notes_path:
# self.assertIn(notes_path, mockS3._uploads, url_err_msg.format(notes_path))
# if source_path:
# self.assertIn(source_path, mockS3._uploads, url_err_msg.format(source_path))
if terms_path:
self.assertIn(terms_path, mockS3._recent_uploads,
url_err_msg.format(terms_path))