def test_create_v2_catalog(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV2Api = MockAPI(os.path.join(self.resources_dir, 'v2_api'),
'https://test')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
converter.run()
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/uw/catalog.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/uw/obs/en/obs/v4/source.json')
assert_s3_equals_api_text(self, mockS3, mockV2Api,
'v2/uw/gen/en/udb/v7/gen.usfm')
assert_s3_equals_api_text(self, mockS3, mockV2Api,
'v2/uw/1ch/en/ulb/v7/1ch.usfm')
self.assertIn('v2/uw/obs/en/obs/v4/source.json.sig',
mockS3._recent_uploads)
self.assertIn('uw/txt/2/catalog.json', mockS3._recent_uploads)
self.assertIn(
'en_udb_1ch: media format "https://cdn.door43.org/en/udb/v9/1ch.pdf" does not match source version "7" and will be excluded.',
mockLogger._messages)
self.assertIn(
'en_obs_obs: media format "https://cdn.door43.org/en/obs/v999/129kbps/en_obs_129kbps.zip" does not match source version "4" and will be excluded.',
mockLogger._messages)
def test_status_not_ready(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'not_ready_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
result = converter._get_status()
self.assertFalse(result)
def test_status_ready_new_db(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
(status, source_status) = converter._get_status()
self.assertEqual('complete', source_status['state'])
self.assertEqual('in-progress', status['state'])
self.assertEqual(0, len(status['processed']))
def setUp(self):
self.mock_signer = MockSigner()
self.resources_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'resources')
self.temp_dir = tempfile.mkdtemp(prefix='signing_tests_')
self.s3keys = []
class TestSigningHandler(TestCase):
def setUp(self):
self.mock_signer = MockSigner()
self.resources_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'resources')
self.temp_dir = tempfile.mkdtemp(prefix='signing_tests_')
self.s3keys = []
def tearDown(self):
# clean up local temp files
if os.path.isdir(self.temp_dir):
shutil.rmtree(self.temp_dir, ignore_errors=True)
def create_db_item(self, commit_id=None):
item = load_json_object(
os.path.join(self.resources_dir, 'dynamodb_record.json'))[0]
if commit_id:
item['commit_id'] = commit_id
return item
def create_event(self):
event = {
'stage-variables': {
'cdn_bucket': 'cdn.door43.org',
'cdn_url': 'https://cdn.door43.org',
'from_email': '',
'to_email': '',
'version': '3',
'api_bucket': 'api.door43.org'
}
}
return event
def test_signing_handler_text_no_records(self, mock_reporter):
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_s3 = MockS3Handler()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
handler = SigningHandler(event,
None,
logger=MockLogger(),
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = handler.run()
self.assertFalse(result)
def test_signing_handler_invalid_manifest(self, mock_reporter):
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_db._load_db(os.path.join(self.resources_dir, 'db/invalid.json'))
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
mock_logger = MockLogger()
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
for f in mock_s3._recent_uploads:
# assert nothing was uploaded to production
self.assertTrue(f.startswith('temp/'))
self.assertFalse(f.endswith('.sig'))
# self.assertIn('Skipping unit-test. Bad Manifest: No JSON object could be decoded', mock_logger._messages)
mock_instance.add_error.assert_called_once_with(
'Skipping unit-test. Bad Manifest: No JSON object could be decoded'
)
@patch('libraries.lambda_handlers.signing_handler.url_headers')
def test_signing_handler_text_missing_file(self, mock_url_headers,
mock_reporter):
"""
Signing will continue to run even if a file is missing.
The missing file will just be ignored.
:return:
"""
mock_url_headers.return_value = HeaderReader([
('last-modified', 'Fri, 03 Jun 2017 20:23:12 GMT'),
('content-length', 12345)
])
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
mock_logger = MockLogger()
mock_s3 = MockS3Handler()
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
mock_instance.add_error.assert_called_once_with(
'The file "obs.zip" could not be downloaded: File not found for key: temp/en_obs/f8a8d8d757/en/obs/v4/obs.zip'
)
def test_signing_handler_text_wrong_key(self, mock_reporter):
event = self.create_event()
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
mock_logger = MockLogger()
# TRICKY: a wrong signing key will result in failed verification
self.mock_signer._fail_verification()
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
for f in mock_s3._recent_uploads:
# assert nothing was uploaded to production
self.assertTrue(f.startswith('temp/'))
self.assertFalse(f.endswith('.sig'))
self.assertIn('The signature was not successfully verified.',
mock_logger._messages)
def test_signing_handler_s3(self, mock_reporter):
mock_s3 = MockS3Handler()
mock_s3._load_path(os.path.join(self.resources_dir, 'cdn'))
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_unsigned.json'))
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
original_item = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertIn('signed', original_item)
self.assertFalse(original_item['signed'])
global_headers = HeaderReader([('last-modified',
'Fri, 03 Jun 2017 20:23:12 GMT'),
('content-length', 12345)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: global_headers)
result = signer.run()
self.assertTrue(result)
self.assertTrue(len(mock_s3._recent_uploads) > 0)
has_prod_uploads = False
for key in mock_s3._recent_uploads:
# assert prod uploads have signatures
if not key.startswith('temp/') and not key.endswith('.sig'):
has_prod_uploads = True
self.assertIn('{}.sig'.format(key), mock_s3._recent_uploads)
self.assertTrue(has_prod_uploads)
updated_item = mock_db.get_item({'repo_name': 'en_obs'}).copy()
assert_object_not_equals(self, updated_item, original_item)
assert_object_equals_file(
self, json.loads(updated_item['package']),
os.path.join(self.resources_dir,
'db/expected_signed_package.json'))
self.assertIn(
'Skipping chapter obs:01 missing url https://cdn.door43.org/en/obs/v4/32kbps/en_obs_01_32kbps.mp3',
mock_logger._messages)
self.assertTrue(updated_item['signed'])
def test_signing_handler_no_records(self, mock_reporter):
event = self.create_event()
mock_logger = MockLogger()
mock_db = MockDynamodbHandler()
mock_s3 = MockS3Handler()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertFalse(result)
self.assertIn('No items found for signing', mock_logger._messages)
def test_signing_handler_already_signed(self, mock_reporter):
event = self.create_event()
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_db._load_db(
os.path.join(self.resources_dir, 'db/valid_signed.json'))
mock_logger = MockLogger()
original_record = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertFalse(original_record['signed'])
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org')
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file)
result = signer.run()
self.assertTrue(result)
updated_record = mock_db.get_item({'repo_name': 'en_obs'}).copy()
self.assertTrue(updated_record['signed'])
def test_skip_signing_large_file(self, mock_reporter):
"""
Ensure that large files are not signed.
Because lambda functions have limited disk space.
:return:
"""
mock_instance = MagicMock()
mock_instance.add_error = MagicMock()
mock_reporter.return_value = mock_instance
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.sign_given_url"],
"chapters": [],
"contributor": [
"Narrator: Steve Lossing", "Checker: Brad Harrington",
"Engineer: Brad Harrington"
],
"format":
"",
"modified":
"",
"quality":
"64kbps",
"signature":
"",
"size":
0,
"url":
"https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip"
}
mockHeaders = HeaderReader([('content-length',
SigningHandler.max_file_size + 1)])
signer = SigningHandler(event=event,
context=None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, None, None, format)
mock_instance.add_error.assert_called_once_with(
'File is too large to sign https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip'
)
self.assertFalse(already_signed)
self.assertEqual('', format['signature'])
self.assertFalse(newly_signed)
def test_signing_small_file(self, mock_reporter):
"""
Ensure that small files are signed properly
:return:
"""
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.sign_given_url"],
"chapters": [],
"contributor": [
"Narrator: Steve Lossing", "Checker: Brad Harrington",
"Engineer: Brad Harrington"
],
"format":
"",
"modified":
"",
"quality":
"64kbps",
"signature":
"",
"size":
0,
"url":
"https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip"
}
mockHeaders = HeaderReader([('content-length', 123)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, None, None, format)
self.assertEqual(
'https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip.sig',
format['signature'])
self.assertNotIn(
'File is too large to sign https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip',
mock_logger._messages)
self.assertFalse(already_signed)
self.assertTrue(newly_signed)
def test_signing_obs_html(self, mock_reporter):
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'),
'https://cdn.door43.org/')
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
format = {
"build_rules": ["signing.html_format"],
"format": "text/html",
"modified": "",
"signature": "",
"size": "",
# NOTE: this is not the actual url format used
"url": "https://cdn.door43.org/temp/en_obs/f8a8d8d757/en/obs.html"
}
dublin_core = {"identifier": "obs", "language": {"identifier": "en"}}
mockHeaders = HeaderReader([('content-length', 123)])
signer = SigningHandler(event,
None,
logger=mock_logger,
signer=self.mock_signer,
s3_handler=mock_s3,
dynamodb_handler=mock_db,
url_exists_handler=mock_api.url_exists,
download_handler=mock_api.download_file,
url_headers_handler=lambda url: mockHeaders)
(already_signed,
newly_signed) = signer.process_format(item, dublin_core, None, format)
self.assertEqual(
'https://cdn.door43.org/en/obs/v3/media/html/obs.html',
format['url'])
self.assertEqual(
'https://cdn.door43.org/en/obs/v3/media/html/obs.html.sig',
format['signature'])
self.assertFalse(already_signed)
self.assertTrue(newly_signed)
@unittest.skipIf(is_travis(), 'Skipping test_everything on Travis CI.')
def test_manually_sign(self, mock_reporter):
"""
This is used to manually sign large media files.
You shouldn't actually run this test unless you want to use the signature
:return:
"""
return # This takes a long time so you don't usually want to do this
mock_s3 = MockS3Handler()
mock_db = MockDynamodbHandler()
mock_logger = MockLogger()
event = self.create_event()
item = {'repo_name': 'repo_name', 'commit_id': 'commitid'}
quality = '720p'
key = 'en/obs/v4/{0}/en_obs_{0}.zip'.format(quality)
format = {
"build_rules": ["signing.sign_given_url"],
"format": "",
"modified": "",
"signature": "",
"size": 0,
"url": "https://cdn.door43.org/{}".format(key)
}
pem_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../../functions/signing/uW-sk.enc')
signer = Signer(pem_file)
signing_handler = SigningHandler(event,
None,
logger=mock_logger,
s3_handler=mock_s3,
signer=signer,
dynamodb_handler=mock_db,
url_size_handler=lambda url: 1)
(already_signed, newly_signed) = signing_handler.process_format(
item, None, None, format)
self.assertTrue(newly_signed)
mock_s3.download_file(
'{}.sig'.format(key),
os.path.expanduser('~/{}.sig'.format(os.path.basename(key))))