def test_broken_catalog(self, mock_reporter): mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_cdn'), 'https://cdn.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'broken_api'), 'https://api.door43.org/') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json')) mockLog = MockLogger() mock_get_url = lambda url, catch_exception: mockV3Api.get_url( url, catch_exception) mock_download = lambda url, dest: mockV3Api.download_file(url, dest) event = self.make_event() converter = TsV2CatalogHandler(event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mock_get_url, download_handler=mock_download, url_exists_handler=lambda url: False) result = converter.run() self.assertFalse(result) self.assertIn( 'Failed to load the catalog json: No JSON object could be decoded', mockLog._messages)
def test_signing_handler_already_signed(self, mock_reporter): event = self.create_event() mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_signed.json')) mock_logger = MockLogger() original_record = mock_db.get_item({'repo_name': 'en_obs'}).copy() self.assertFalse(original_record['signed']) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) updated_record = mock_db.get_item({'repo_name': 'en_obs'}).copy() self.assertTrue(updated_record['signed'])
def test_missing_catalog(self, mock_reporter): mockV3Api = MockAPI(self.resources_dir, 'https://api.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://cdn.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://test-cdn.door43.org/') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json')) mockLog = MockLogger() event = self.make_event() converter = TsV2CatalogHandler( event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, url_exists_handler=lambda url: False) result = converter.run() self.assertFalse(result) self.assertIn('https://api.door43.org/v3/catalog.json does not exist', mockLog._messages)
def test_signing_handler_text_wrong_key(self, mock_reporter): event = self.create_event() mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_unsigned.json')) mock_s3 = MockS3Handler() mock_s3._load_path(os.path.join(self.resources_dir, 'cdn')) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') mock_logger = MockLogger() # TRICKY: a wrong signing key will result in failed verification self.mock_signer._fail_verification() signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) for f in mock_s3._recent_uploads: # assert nothing was uploaded to production self.assertTrue(f.startswith('temp/')) self.assertFalse(f.endswith('.sig')) self.assertIn('The signature was not successfully verified.', mock_logger._messages)
def test_signing_handler_s3(self, mock_reporter): mock_s3 = MockS3Handler() mock_s3._load_path(os.path.join(self.resources_dir, 'cdn')) mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_unsigned.json')) mock_logger = MockLogger() mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org/') event = self.create_event() original_item = mock_db.get_item({'repo_name': 'en_obs'}).copy() self.assertIn('signed', original_item) self.assertFalse(original_item['signed']) global_headers = HeaderReader([('last-modified', 'Fri, 03 Jun 2017 20:23:12 GMT'), ('content-length', 12345)]) signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file, url_headers_handler=lambda url: global_headers) result = signer.run() self.assertTrue(result) self.assertTrue(len(mock_s3._recent_uploads) > 0) has_prod_uploads = False for key in mock_s3._recent_uploads: # assert prod uploads have signatures if not key.startswith('temp/') and not key.endswith('.sig'): has_prod_uploads = True self.assertIn('{}.sig'.format(key), mock_s3._recent_uploads) self.assertTrue(has_prod_uploads) updated_item = mock_db.get_item({'repo_name': 'en_obs'}).copy() assert_object_not_equals(self, updated_item, original_item) assert_object_equals_file( self, json.loads(updated_item['package']), os.path.join(self.resources_dir, 'db/expected_signed_package.json')) self.assertIn( 'Skipping chapter obs:01 missing url https://cdn.door43.org/en/obs/v4/32kbps/en_obs_01_32kbps.mp3', mock_logger._messages) self.assertTrue(updated_item['signed'])
def make_handler_instance(self, progress_db): """ Generates a new handler instance for testing :param progress_db: :return: """ progress_db_file = os.path.join(self.resources_dir, 'progress_db/{}'.format(progress_db)) self.assertTrue(os.path.isfile(progress_db_file)) # now run the code event = self.create_event() mock_progress_db = MockDynamodbHandler() mock_progress_db._load_db(progress_db_file) mock_status_db = MockDynamodbHandler() mock_errors_db = MockDynamodbHandler() dbs = { 'd43-catalog-in-progress': mock_progress_db, 'd43-catalog-status': mock_status_db, 'd43-catalog-errors': mock_errors_db } mock_ses = MockSESHandler() mock_s3 = MockS3Handler() mock_checker = MockChecker() mock_api = MockAPI(os.path.join(self.resources_dir), '/') mock_ses_handler = lambda: mock_ses mock_s3_handler = lambda bucket: mock_s3 mock_dbs_handler = lambda bucket: dbs[bucket] mock_checker_handler = lambda: mock_checker handler = CatalogHandler(event, None, s3_handler=mock_s3_handler, dynamodb_handler=mock_dbs_handler, ses_handler=mock_ses_handler, consistency_checker=mock_checker_handler, url_exists_handler=lambda url: True, get_url_handler=mock_api.get_url) return { 'handler': handler, 'event': event, 'mocks': { 'db': { 'progress': mock_progress_db, 'status': mock_status_db, 'errors': mock_errors_db }, 'api': mock_api, 'ses': mock_ses, 's3': mock_s3, 'checker': mock_checker } }
def test_create_v2_catalog(self, mock_reporter): mockDB = MockDynamodbHandler() mockDB._load_db( os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json')) mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://cdn.door43.org/') mockV2Api = MockAPI(os.path.join(self.resources_dir, 'v2_api'), 'https://test') mockS3 = MockS3Handler('uw_bucket') mockSigner = MockSigner() mockLogger = MockLogger() converter = UwV2CatalogHandler( event=self._make_event(), context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDB, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, signing_handler=mockSigner) converter.run() assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/uw/catalog.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/uw/obs/en/obs/v4/source.json') assert_s3_equals_api_text(self, mockS3, mockV2Api, 'v2/uw/gen/en/udb/v7/gen.usfm') assert_s3_equals_api_text(self, mockS3, mockV2Api, 'v2/uw/1ch/en/ulb/v7/1ch.usfm') self.assertIn('v2/uw/obs/en/obs/v4/source.json.sig', mockS3._recent_uploads) self.assertIn('uw/txt/2/catalog.json', mockS3._recent_uploads) self.assertIn( 'en_udb_1ch: media format "https://cdn.door43.org/en/udb/v9/1ch.pdf" does not match source version "7" and will be excluded.', mockLogger._messages) self.assertIn( 'en_obs_obs: media format "https://cdn.door43.org/en/obs/v999/129kbps/en_obs_129kbps.zip" does not match source version "4" and will be excluded.', mockLogger._messages)
def test_signing_handler_text_missing_file(self, mock_url_headers, mock_reporter): """ Signing will continue to run even if a file is missing. The missing file will just be ignored. :return: """ mock_url_headers.return_value = HeaderReader([ ('last-modified', 'Fri, 03 Jun 2017 20:23:12 GMT'), ('content-length', 12345) ]) mock_instance = MagicMock() mock_instance.add_error = MagicMock() mock_reporter.return_value = mock_instance event = self.create_event() mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_unsigned.json')) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') mock_logger = MockLogger() mock_s3 = MockS3Handler() signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) mock_instance.add_error.assert_called_once_with( 'The file "obs.zip" could not be downloaded: File not found for key: temp/en_obs/f8a8d8d757/en/obs/v4/obs.zip' )
def test_status_not_ready(self, mock_reporter): mockDB = MockDynamodbHandler() mockDB._load_db( os.path.join(TestUwV2Catalog.resources_dir, 'not_ready_db.json')) mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockS3 = MockS3Handler('uw_bucket') mockSigner = MockSigner() mockLogger = MockLogger() converter = UwV2CatalogHandler( event=self._make_event(), context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDB, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, signing_handler=mockSigner) result = converter._get_status() self.assertFalse(result)
def test_status_ready_new_db(self, mock_reporter): mockDB = MockDynamodbHandler() mockDB._load_db( os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json')) mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockS3 = MockS3Handler('uw_bucket') mockSigner = MockSigner() mockLogger = MockLogger() converter = UwV2CatalogHandler( event=self._make_event(), context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDB, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, signing_handler=mockSigner) (status, source_status) = converter._get_status() self.assertEqual('complete', source_status['state']) self.assertEqual('in-progress', status['state']) self.assertEqual(0, len(status['processed']))
def test_complete_status(self, mock_reporter): mockV3Api = MockAPI(self.resources_dir, 'https://cdn.door43.org/') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'complete_db.json')) mockLog = MockLogger() mock_get_url = lambda url, catch_exception: mockV3Api.get_url( url, catch_exception) mock_download = lambda url, dest: mockV3Api.download_file(url, dest) event = self.make_event() converter = TsV2CatalogHandler(event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mock_get_url, download_handler=mock_download, url_exists_handler=lambda url: True) result = converter.run() self.assertTrue(result) self.assertEqual(0, len(mockS3._recent_uploads)) self.assertIn('Catalog already generated', mockLog._messages)
def test_signing_handler_invalid_manifest(self, mock_reporter): mock_instance = MagicMock() mock_instance.add_error = MagicMock() mock_reporter.return_value = mock_instance event = self.create_event() mock_db = MockDynamodbHandler() mock_db._load_db(os.path.join(self.resources_dir, 'db/invalid.json')) mock_s3 = MockS3Handler() mock_s3._load_path(os.path.join(self.resources_dir, 'cdn')) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') mock_logger = MockLogger() signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) for f in mock_s3._recent_uploads: # assert nothing was uploaded to production self.assertTrue(f.startswith('temp/')) self.assertFalse(f.endswith('.sig')) # self.assertIn('Skipping unit-test. Bad Manifest: No JSON object could be decoded', mock_logger._messages) mock_instance.add_error.assert_called_once_with( 'Skipping unit-test. Bad Manifest: No JSON object could be decoded' )
def test_inprogress(self, mock_reporter): mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://test-cdn.door43.org/') mockV2Api = MockAPI(os.path.join(self.resources_dir, 'ts_api'), 'https://test') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'ready_inprogress_db.json')) mockLog = MockLogger() event = self.make_event() converter = TsV2CatalogHandler( event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, url_exists_handler=lambda url: True) converter.run() assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/catalog.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/languages.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/resources.json') self.assertNotIn('v2/ts/obs/en/obs/source.json', mockS3._recent_uploads) # assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/obs/source.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/notes.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/questions.json') # we have frozen tw_cat self.assertNotIn('v2/ts/obs/en/tw_cat.json', mockS3._recent_uploads) assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/languages.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/resources.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/ulb/v7/source.json') self.assertNotIn('v2/ts/1ch/en/notes.json', mockS3._recent_uploads) # assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/notes.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/questions.json') self.assertNotIn('v2/ts/1ch/en/tw_cat.json', mockS3._recent_uploads) # assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/tw_cat.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/bible/en/words.json') # validate urls in generate catalogs match the generated output paths root_url = '{}/'.format( event['stage-variables']['cdn_url'].rstrip('/')) catalog = json.loads( read_file(mockS3._recent_uploads['v2/ts/catalog.json'])) url_err_msg = 'url in catalog does not match upload path: {}' for project in catalog: lang_catalog_path = project['lang_catalog'].replace( root_url, '').split('?')[0] self.assertIn(lang_catalog_path, mockS3._recent_uploads, url_err_msg.format(lang_catalog_path)) lang_catalog = json.loads( read_file(mockS3._recent_uploads[lang_catalog_path])) for language in lang_catalog: res_catalog_path = language['res_catalog'].replace( root_url, '').split('?')[0] self.assertIn(res_catalog_path, mockS3._recent_uploads, url_err_msg.format(res_catalog_path)) res_catalog = json.loads( read_file(mockS3._recent_uploads[res_catalog_path])) for resource in res_catalog: questions_path = resource['checking_questions'].replace( root_url, '').split('?')[0] # notes_path = resource['notes'].replace(root_url, '').split('?')[0] # source_path = resource['source'].replace(root_url, '').split('?')[0] terms_path = resource['terms'].replace(root_url, '').split('?')[0] # terms_map_path = resource['tw_cat'].replace(root_url, '').split('?')[0] if questions_path: self.assertIn(questions_path, mockS3._recent_uploads, url_err_msg.format(questions_path)) # if notes_path: # self.assertIn(notes_path, mockS3._uploads, url_err_msg.format(notes_path)) # if source_path: # self.assertIn(source_path, mockS3._uploads, url_err_msg.format(source_path)) if terms_path: self.assertIn(terms_path, mockS3._recent_uploads, url_err_msg.format(terms_path))