def test_skip_signing_large_file(self, mock_reporter): """ Ensure that large files are not signed. Because lambda functions have limited disk space. :return: """ mock_instance = MagicMock() mock_instance.add_error = MagicMock() mock_reporter.return_value = mock_instance mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_logger = MockLogger() mock_logger.warning = MagicMock() mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org/') event = self.create_event() item = {'repo_name': 'repo_name', 'commit_id': 'commitid'} format = { "build_rules": ["signing.sign_given_url"], "chapters": [], "contributor": [ "Narrator: Steve Lossing", "Checker: Brad Harrington", "Engineer: Brad Harrington" ], "format": "", "modified": "", "quality": "64kbps", "signature": "", "size": 0, "url": "https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip" } mockHeaders = HeaderReader([('content-length', SigningHandler.max_file_size + 1)]) signer = SigningHandler(event=event, context=None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file, url_headers_handler=lambda url: mockHeaders) (already_signed, newly_signed) = signer.process_format(item, None, None, format) mock_logger.warning.assert_called_once_with( 'File is too large to sign https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip' ) self.assertTrue(already_signed) self.assertEqual( 'https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip.sig', format['signature']) self.assertTrue(newly_signed)
def test_make_hook_payload(self, mock_url_exists, mock_reporter): mock_url_exists.return_value = True event = self.create_event() mockDb = MockDynamodbHandler() self.MockGogsClient.MockGogsApi.branch = TestFork.create_branch( "branch") mockLog = MockLogger() handler = ForkHandler(event=event, context=None, logger=mockLog, gogs_client=self.MockGogsClient, dynamodb_handler=mockDb) repo = TestFork.create_repo("en_obs") payload = handler.make_hook_payload(repo) self.assertIn('body-json', payload) self.assertIn('stage-variables', payload) self.assertEqual(1, len(payload['body-json']['commits'])) self.assertEqual(repo.name, payload['body-json']['repository']['name']) TestFork.mock_download = os.path.join(TestFork.resources_dir, 'en_obs.zip') s3Handler = MockS3Handler('test') mockLogger = MockLogger() dbHandler = MockDynamodbHandler() webhook_handler = WebhookHandler( event=payload, context=None, logger=mockLogger, s3_handler=s3Handler, dynamodb_handler=dbHandler, download_handler=TestFork.mock_download_file) webhook_handler.run()
def test_webook_with_invalid_data(self, mock_reporter, mock_url_exists): request_file = os.path.join(self.resources_dir, 'missing-manifest.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) self.MockDynamodbHandler.data = None self.MockS3Handler.reset() mockLogger = MockLogger() handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=self.MockS3Handler, dynamodb_handler=self.MockDynamodbHandler) with self.assertRaises(Exception) as error_context: handler.run() self.assertIn('does not have a manifest.yaml file', str(error_context.exception)) self.assertFalse(os.path.isdir(handler.temp_dir))
def test_get_dirty_repos(self, mock_reporter): event = self.create_event() # mock data self.MockGogsClient.MockGogsApi.repos = [] self.MockGogsClient.MockGogsApi.repos.append( TestFork.create_repo("hmr-obs")) self.MockGogsClient.MockGogsApi.repos.append( TestFork.create_repo("en-obs")) self.MockGogsClient.MockGogsApi.repos.append( TestFork.create_repo("es-obs")) mockBoto = self.MockBotoClient() dirty_record = TestFork.create_db_item("hmr-obs") dirty_record['dirty'] = True mockDb = MockDynamodbHandler() mockDb.insert_item(dirty_record) mockDb.insert_item(TestFork.create_db_item("pt-br-obs")) mockLog = MockLogger() handler = ForkHandler(event=event, context=None, logger=mockLog, gogs_client=self.MockGogsClient, dynamodb_handler=mockDb, boto_handler=mockBoto) repos = handler.get_new_repos() self.assertEqual(3, len(repos)) for repo in repos: self.assertNotIn(repo.full_name, ['Door43-Catalog/pt-br-obs'])
def test_missing_catalog(self, mock_reporter): mockV3Api = MockAPI(self.resources_dir, 'https://api.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://cdn.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://test-cdn.door43.org/') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json')) mockLog = MockLogger() event = self.make_event() converter = TsV2CatalogHandler( event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, url_exists_handler=lambda url: False) result = converter.run() self.assertFalse(result) self.assertIn('https://api.door43.org/v3/catalog.json does not exist', mockLog._messages)
def test_signing_handler_already_signed(self, mock_reporter): event = self.create_event() mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_signed.json')) mock_logger = MockLogger() original_record = mock_db.get_item({'repo_name': 'en_obs'}).copy() self.assertFalse(original_record['signed']) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) updated_record = mock_db.get_item({'repo_name': 'en_obs'}).copy() self.assertTrue(updated_record['signed'])
def test_signing_handler_text_wrong_key(self, mock_reporter): event = self.create_event() mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_unsigned.json')) mock_s3 = MockS3Handler() mock_s3._load_path(os.path.join(self.resources_dir, 'cdn')) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') mock_logger = MockLogger() # TRICKY: a wrong signing key will result in failed verification self.mock_signer._fail_verification() signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) for f in mock_s3._recent_uploads: # assert nothing was uploaded to production self.assertTrue(f.startswith('temp/')) self.assertFalse(f.endswith('.sig')) self.assertIn('The signature was not successfully verified.', mock_logger._messages)
def test_broken_catalog(self, mock_reporter): mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_cdn'), 'https://cdn.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'broken_api'), 'https://api.door43.org/') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json')) mockLog = MockLogger() mock_get_url = lambda url, catch_exception: mockV3Api.get_url( url, catch_exception) mock_download = lambda url, dest: mockV3Api.download_file(url, dest) event = self.make_event() converter = TsV2CatalogHandler(event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mock_get_url, download_handler=mock_download, url_exists_handler=lambda url: False) result = converter.run() self.assertFalse(result) self.assertIn( 'Failed to load the catalog json: No JSON object could be decoded', mockLog._messages)
def test_webhook_ulb_pull_request(self, mock_reporter, mock_url_exists): request_file = os.path.join(self.resources_dir, 'ulb-pull-request.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) mockLogger = MockLogger() mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/') urls = { 'https://git.door43.org/Door43-Catalog/en_ulb/archive/2fbfd081f46487e48e49090a95c48d45e04e6bed.zip': 'en_ulb.zip' } mock_download = lambda url, dest: mockDCS.download_file( urls[url], dest) self.MockDynamodbHandler.data = None self.MockS3Handler.reset() handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=self.MockS3Handler, dynamodb_handler=self.MockDynamodbHandler, download_handler=mock_download) with self.assertRaises(Exception) as error_context: handler.run() self.assertIn('Skipping un-merged pull request', str(error_context.exception)) entry = self.MockDynamodbHandler.data self.assertEqual(0, len(self.MockS3Handler.uploads)) self.assertIsNone(entry)
def test_signing_handler_s3(self, mock_reporter): mock_s3 = MockS3Handler() mock_s3._load_path(os.path.join(self.resources_dir, 'cdn')) mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_unsigned.json')) mock_logger = MockLogger() mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org/') event = self.create_event() original_item = mock_db.get_item({'repo_name': 'en_obs'}).copy() self.assertIn('signed', original_item) self.assertFalse(original_item['signed']) global_headers = HeaderReader([('last-modified', 'Fri, 03 Jun 2017 20:23:12 GMT'), ('content-length', 12345)]) signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file, url_headers_handler=lambda url: global_headers) result = signer.run() self.assertTrue(result) self.assertTrue(len(mock_s3._recent_uploads) > 0) has_prod_uploads = False for key in mock_s3._recent_uploads: # assert prod uploads have signatures if not key.startswith('temp/') and not key.endswith('.sig'): has_prod_uploads = True self.assertIn('{}.sig'.format(key), mock_s3._recent_uploads) self.assertTrue(has_prod_uploads) updated_item = mock_db.get_item({'repo_name': 'en_obs'}).copy() assert_object_not_equals(self, updated_item, original_item) assert_object_equals_file( self, json.loads(updated_item['package']), os.path.join(self.resources_dir, 'db/expected_signed_package.json')) self.assertIn( 'Skipping chapter obs:01 missing url https://cdn.door43.org/en/obs/v4/32kbps/en_obs_01_32kbps.mp3', mock_logger._messages) self.assertTrue(updated_item['signed'])
def test_stage_prefix_prod(self, mock_reporter): event = self.create_event() mockDb = MockDynamodbHandler() self.MockGogsClient.MockGogsApi.branch = TestFork.create_branch("branch") mockLog = MockLogger() handler = ForkHandler(event=event, context=None, logger=mockLog, gitea_client=self.MockGogsClient, dynamodb_handler=mockDb) self.assertEqual('', handler.stage_prefix())
def test_signing_small_file(self, mock_reporter): """ Ensure that small files are signed properly :return: """ mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_logger = MockLogger() mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org/') event = self.create_event() item = {'repo_name': 'repo_name', 'commit_id': 'commitid'} format = { "build_rules": ["signing.sign_given_url"], "chapters": [], "contributor": [ "Narrator: Steve Lossing", "Checker: Brad Harrington", "Engineer: Brad Harrington" ], "format": "", "modified": "", "quality": "64kbps", "signature": "", "size": 0, "url": "https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip" } mockHeaders = HeaderReader([('content-length', 123)]) signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file, url_headers_handler=lambda url: mockHeaders) (already_signed, newly_signed) = signer.process_format(item, None, None, format) self.assertEqual( 'https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip.sig', format['signature']) self.assertNotIn( 'File is too large to sign https://cdn.door43.org/en/obs/v4/64kbps/en_obs_64kbps.zip', mock_logger._messages) self.assertFalse(already_signed) self.assertTrue(newly_signed)
def test_trigger_hook_no_repos(self, mock_reporter): event = self.create_event() mockDb = MockDynamodbHandler() self.MockGogsClient.MockGogsApi.branch = TestFork.create_branch("branch") mockLog = MockLogger() handler = ForkHandler(event=event, context=None, logger=mockLog, gitea_client=self.MockGogsClient, dynamodb_handler=mockDb) mockClient = self.MockBotoClient() handler._trigger_webhook(mockClient, []) self.assertIn('No new repositories found', mockLog._messages)
def test_webhook_with_missing_obs_data(self, mock_reporter, mock_url_exists): request_file = os.path.join(self.resources_dir, 'obs-missing-request.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) mockLogger = MockLogger() mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/') self.MockDynamodbHandler.data = None self.MockS3Handler.reset() urls = { 'https://git.door43.org/Door43-Catalog/ylb_obs/archive/f8a8d8d757e7ea287cf91b266963f8523bdbd5ad.zip': 'ylb_obs_missing_data.zip' } mock_download = lambda url, dest: mockDCS.download_file( urls[url], dest) handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=self.MockS3Handler, dynamodb_handler=self.MockDynamodbHandler, download_handler=mock_download) handler.run() entry = self.MockDynamodbHandler.data self.assertEqual(1, len(self.MockS3Handler.uploads)) self.assertIn('/ylb_obs.zip', self.MockS3Handler.uploads[0]['path']) self.assertIn( 'temp/ylb_obs/{}/ylb/obs/v4.1/obs.zip'.format(entry['commit_id']), self.MockS3Handler.uploads[0]['key']) self.assertEqual('f8a8d8d757', entry['commit_id']) self.assertEqual(False, entry['dirty']) self.assertEqual('ylb', entry['language']) self.assertEqual('2017-04-25T21:46:30+00:00', entry['timestamp']) self.assertEqual(False, entry['signed']) self.assertEqual('ylb_obs', entry['repo_name']) assert_object_equals_file( self, json.loads(entry['package']), os.path.join(self.resources_dir, 'expected_obs_package_missing_data.json'))
def test_webhook_ulb(self, mock_reporter, mock_url_exists): request_file = os.path.join(self.resources_dir, 'ulb-request.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) mockLogger = MockLogger() mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/') urls = { 'https://git.door43.org/Door43-Catalog/en_ulb/archive/2fbfd081f46487e48e49090a95c48d45e04e6bed.zip': 'en_ulb.zip' } mock_download = lambda url, dest: mockDCS.download_file( urls[url], dest) self.MockDynamodbHandler.data = None self.MockS3Handler.reset() handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=self.MockS3Handler, dynamodb_handler=self.MockDynamodbHandler, download_handler=mock_download) handler.run() entry = self.MockDynamodbHandler.data self.assertEqual(4, len(self.MockS3Handler.uploads)) # books and bundle self.assertIn('/en_ulb.zip', self.MockS3Handler.uploads[0]['path']) self.assertEqual('2fbfd081f4', entry['commit_id']) self.assertEqual(False, entry['dirty']) self.assertEqual('en', entry['language']) self.assertEqual('2017-05-02T22:52:04+00:00', entry['timestamp']) self.assertEqual(False, entry['signed']) self.assertEqual('en_ulb', entry['repo_name']) self.assertIn( 'temp/en_ulb/{}/en/ulb/v7/ulb.zip'.format(entry['commit_id']), self.MockS3Handler.uploads[0]['key']) assert_object_equals_file( self, json.loads(entry['package']), os.path.join(self.resources_dir, 'expected_ulb_package.json'))
def test_trigger_hook_with_repos(self, mock_reporter): event = self.create_event() mockDb = MockDynamodbHandler() self.MockGogsClient.MockGogsApi.branch = TestFork.create_branch("branch") mockLog = MockLogger() handler = ForkHandler(event=event, context=None, logger=mockLog, gitea_client=self.MockGogsClient, dynamodb_handler=mockDb) mockClient = self.MockBotoClient() mockRepo = self.MockGogsRepo() mockRepo.full_name = 'my_repo' handler._trigger_webhook(mockClient, [mockRepo]) self.assertIn('Simulating Webhook for my_repo', mockLog._messages)
def test_convert_invalid_links(self, mock_reporter): mockLog = MockLogger() content = """ some text [[rc://tl/ta/vol]] some more text """ convert_rc_links(content, mockLog) self.assertEqual(1, len(mockLog._messages)) longer_content = """ some text [[rc://tl/ta/vol/hi]] some more text """ convert_rc_links(longer_content, mockLog) self.assertEqual(2, len(mockLog._messages))
def test_signing_handler_text_no_records(self, mock_reporter): event = self.create_event() mock_db = MockDynamodbHandler() mock_s3 = MockS3Handler() mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') handler = SigningHandler(event, None, logger=MockLogger(), signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = handler.run() self.assertFalse(result)
def test_webhook_ulb_merged_pull_request(self, mock_reporter, mock_url_exists): request_file = os.path.join(self.resources_dir, 'ulb-merged-pull-request.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) mockLogger = MockLogger() mockDCS = MockAPI(self.resources_dir, 'https://git.door43.org/') urls = { 'https://git.door43.org/Door43-Catalog/ta_ulb/archive/0a7e25cd939f00086262fe94b9d25afc3b5dabd3.zip': 'ta_ulb.zip' } mock_download = lambda url, dest: mockDCS.download_file( urls[url], dest) self.MockDynamodbHandler.data = None self.MockS3Handler.reset() handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=self.MockS3Handler, dynamodb_handler=self.MockDynamodbHandler, download_handler=mock_download) handler.run() entry = self.MockDynamodbHandler.data self.assertEqual(4, len(self.MockS3Handler.uploads)) # books and bundle self.assertIn('/ta_ulb.zip', self.MockS3Handler.uploads[0]['path']) self.assertEqual('0a7e25cd93', entry['commit_id']) self.assertEqual(False, entry['dirty']) self.assertEqual('ta', entry['language']) self.assertEqual('2017-08-17T18:56:52.884140+00:00', entry['timestamp']) self.assertEqual(False, entry['signed']) self.assertEqual('ta_ulb', entry['repo_name']) self.assertIn( 'temp/ta_ulb/{}/ta/ulb/v3/ulb.zip'.format(entry['commit_id']), self.MockS3Handler.uploads[0]['key'])
def test_create_v2_catalog(self, mock_reporter): mockDB = MockDynamodbHandler() mockDB._load_db( os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json')) mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'), 'https://cdn.door43.org/') mockV2Api = MockAPI(os.path.join(self.resources_dir, 'v2_api'), 'https://test') mockS3 = MockS3Handler('uw_bucket') mockSigner = MockSigner() mockLogger = MockLogger() converter = UwV2CatalogHandler( event=self._make_event(), context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDB, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, signing_handler=mockSigner) converter.run() assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/uw/catalog.json') assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/uw/obs/en/obs/v4/source.json') assert_s3_equals_api_text(self, mockS3, mockV2Api, 'v2/uw/gen/en/udb/v7/gen.usfm') assert_s3_equals_api_text(self, mockS3, mockV2Api, 'v2/uw/1ch/en/ulb/v7/1ch.usfm') self.assertIn('v2/uw/obs/en/obs/v4/source.json.sig', mockS3._recent_uploads) self.assertIn('uw/txt/2/catalog.json', mockS3._recent_uploads) self.assertIn( 'en_udb_1ch: media format "https://cdn.door43.org/en/udb/v9/1ch.pdf" does not match source version "7" and will be excluded.', mockLogger._messages) self.assertIn( 'en_obs_obs: media format "https://cdn.door43.org/en/obs/v999/129kbps/en_obs_129kbps.zip" does not match source version "4" and will be excluded.', mockLogger._messages)
def test_webhook_localization(self, mock_reporter, mock_url_exists): request_file = os.path.join(self.resources_dir, 'localization-request.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) mockLogger = MockLogger() self.MockDynamodbHandler.data = None self.MockS3Handler.reset() handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=self.MockS3Handler, dynamodb_handler=self.MockDynamodbHandler) handler.run()
def test_signing_handler_text_missing_file(self, mock_url_headers, mock_reporter): """ Signing will continue to run even if a file is missing. The missing file will just be ignored. :return: """ mock_url_headers.return_value = HeaderReader([ ('last-modified', 'Fri, 03 Jun 2017 20:23:12 GMT'), ('content-length', 12345) ]) mock_instance = MagicMock() mock_instance.add_error = MagicMock() mock_reporter.return_value = mock_instance event = self.create_event() mock_db = MockDynamodbHandler() mock_db._load_db( os.path.join(self.resources_dir, 'db/valid_unsigned.json')) mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org') mock_logger = MockLogger() mock_s3 = MockS3Handler() signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file) result = signer.run() self.assertTrue(result) mock_instance.add_error.assert_called_once_with( 'The file "obs.zip" could not be downloaded: File not found for key: temp/en_obs/f8a8d8d757/en/obs/v4/obs.zip' )
def test_manually_sign(self, mock_reporter): """ This is used to manually sign large media files. You shouldn't actually run this test unless you want to use the signature :return: """ return # This takes a long time so you don't usually want to do this mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_logger = MockLogger() event = self.create_event() item = {'repo_name': 'repo_name', 'commit_id': 'commitid'} quality = '720p' key = 'en/obs/v4/{0}/en_obs_{0}.zip'.format(quality) format = { "build_rules": ["signing.sign_given_url"], "format": "", "modified": "", "signature": "", "size": 0, "url": "https://cdn.door43.org/{}".format(key) } pem_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../functions/signing/uW-sk.enc') signer = Signer(pem_file) signing_handler = SigningHandler(event, None, logger=mock_logger, s3_handler=mock_s3, signer=signer, dynamodb_handler=mock_db, url_size_handler=lambda url: 1) (already_signed, newly_signed) = signing_handler.process_format( item, None, None, format) self.assertTrue(newly_signed) mock_s3.download_file( '{}.sig'.format(key), os.path.expanduser('~/{}.sig'.format(os.path.basename(key))))
def test_signing_remote_3gp_file(self, mock_reporter): """ Ensure that small files are signed properly :return: """ mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_logger = MockLogger() mock_api = MockAPI(os.path.join(self.resources_dir, 'filedn'), 'https://filedn.com/') event = self.create_event() item = {'repo_name': 'repo_name', 'commit_id': 'commitid'} format = { "build_rules": ["signing.sign_given_url"], "chapters": [], "contributor": [], "format": "", "modified": "", "quality": "3GP", "signature": "", "size": 0, "url": "https://filedn.com/XYZ/en/obs/v6/en_obs_v6_3GP.zip" } mockHeaders = HeaderReader([('content-length', 345)]) signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file, url_headers_handler=lambda url: mockHeaders) (already_signed, newly_signed) = signer.process_format(item, None, None, format) self.assertEqual('', format['signature']) self.assertEqual('application/zip; content=video/3gpp', format['format']) self.assertEqual(116, format['size']) self.assertTrue(already_signed) self.assertTrue(newly_signed)
def test_status_not_ready(self, mock_reporter): mockDB = MockDynamodbHandler() mockDB._load_db( os.path.join(TestUwV2Catalog.resources_dir, 'not_ready_db.json')) mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockS3 = MockS3Handler('uw_bucket') mockSigner = MockSigner() mockLogger = MockLogger() converter = UwV2CatalogHandler( event=self._make_event(), context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDB, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, signing_handler=mockSigner) result = converter._get_status() self.assertFalse(result)
def test_signing_obs_html(self, mock_reporter): mock_s3 = MockS3Handler() mock_db = MockDynamodbHandler() mock_logger = MockLogger() mock_api = MockAPI(os.path.join(self.resources_dir, 'cdn'), 'https://cdn.door43.org/') event = self.create_event() item = {'repo_name': 'repo_name', 'commit_id': 'commitid'} format = { "build_rules": ["signing.html_format"], "format": "text/html", "modified": "", "signature": "", "size": "", # NOTE: this is not the actual url format used "url": "https://cdn.door43.org/temp/en_obs/f8a8d8d757/en/obs.html" } dublin_core = {"identifier": "obs", "language": {"identifier": "en"}} mockHeaders = HeaderReader([('content-length', 123)]) signer = SigningHandler(event, None, logger=mock_logger, signer=self.mock_signer, s3_handler=mock_s3, dynamodb_handler=mock_db, url_exists_handler=mock_api.url_exists, download_handler=mock_api.download_file, url_headers_handler=lambda url: mockHeaders) (already_signed, newly_signed) = signer.process_format(item, dublin_core, None, format) self.assertEqual( 'https://cdn.door43.org/en/obs/v3/media/html/obs.html', format['url']) self.assertEqual( 'https://cdn.door43.org/en/obs/v3/media/html/obs.html.sig', format['signature']) self.assertFalse(already_signed) self.assertTrue(newly_signed)
def test_status_ready_new_db(self, mock_reporter): mockDB = MockDynamodbHandler() mockDB._load_db( os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json')) mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'), 'https://api.door43.org/') mockS3 = MockS3Handler('uw_bucket') mockSigner = MockSigner() mockLogger = MockLogger() converter = UwV2CatalogHandler( event=self._make_event(), context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDB, url_handler=mockV3Api.get_url, download_handler=mockV3Api.download_file, signing_handler=mockSigner) (status, source_status) = converter._get_status() self.assertEqual('complete', source_status['state']) self.assertEqual('in-progress', status['state']) self.assertEqual(0, len(status['processed']))
def test_complete_status(self, mock_reporter): mockV3Api = MockAPI(self.resources_dir, 'https://cdn.door43.org/') mockS3 = MockS3Handler('ts_bucket') mockDb = MockDynamodbHandler() mockDb._load_db( os.path.join(TestTsV2Catalog.resources_dir, 'complete_db.json')) mockLog = MockLogger() mock_get_url = lambda url, catch_exception: mockV3Api.get_url( url, catch_exception) mock_download = lambda url, dest: mockV3Api.download_file(url, dest) event = self.make_event() converter = TsV2CatalogHandler(event=event, context=None, logger=mockLog, s3_handler=mockS3, dynamodb_handler=mockDb, url_handler=mock_get_url, download_handler=mock_download, url_exists_handler=lambda url: True) result = converter.run() self.assertTrue(result) self.assertEqual(0, len(mockS3._recent_uploads)) self.assertIn('Catalog already generated', mockLog._messages)
def test_malformed_manifest(self, mock_reporter, mock_url_exists): mock_instance = MagicMock() mock_instance.add_error = MagicMock() mock_reporter.return_value = mock_instance request_file = os.path.join(self.resources_dir, 'malformed-manifest.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') # deserialized object request_json = json.loads(content) mockS3 = MockS3Handler() mockDb = MockDynamodbHandler() mockLogger = MockLogger() mockApi = MockAPI(os.path.join(self.resources_dir, 'git_api'), 'https://git.door43.org/') handler = WebhookHandler(event=request_json, context=None, logger=mockLogger, s3_handler=mockS3, dynamodb_handler=mockDb, download_handler=mockApi.download_file) with self.assertRaises(Exception) as error_context: handler.run() self.assertIn('manifest missing dublin_core key "type"', str(error_context.exception)) self.assertFalse(os.path.isdir(handler.temp_dir)) mock_instance.add_error.assert_called_once_with( 'Bad Manifest: manifest missing dublin_core key "type"')
def test_webhook_versification(self, mock_reporter, mock_url_exists): """ We are not currently processing versification. Therefore, we test that nothing happens. :param mock_report_error: :return: """ request_file = os.path.join(self.resources_dir, 'versification-request.json') with codecs.open(request_file, 'r', encoding='utf-8') as in_file: request_text = in_file.read() # convert Windows line endings to Linux line endings content = request_text.replace('\r\n', '\n') request_json = json.loads(content) urls = { 'https://git.door43.org/Door43-Catalog/versification/archive/c7e936e4dcc103560987c8475db69e292aa66dca.zip': 'versification.zip' } mockLogger = MockLogger() mock_api = MockAPI(self.resources_dir, 'https://git.door43.org') mock_db = MockDynamodbHandler() mock_s3 = MockS3Handler() handler = WebhookHandler(request_json, context=None, s3_handler=mock_s3, logger=mockLogger, dynamodb_handler=mock_db, download_handler=lambda url, dest: mock_api. download_file(urls[url], dest)) handler.run() self.assertEqual(0, len(mock_s3._recent_uploads)) data = mock_db._last_inserted_item self.assertEqual(None, data)