def test_missing_catalog(self, mock_reporter):
mockV3Api = MockAPI(self.resources_dir, 'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://test-cdn.door43.org/')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json'))
mockLog = MockLogger()
event = self.make_event()
converter = TsV2CatalogHandler(
event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
url_exists_handler=lambda url: False)
result = converter.run()
self.assertFalse(result)
self.assertIn('https://api.door43.org/v3/catalog.json does not exist',
mockLog._messages)
def test_broken_catalog(self, mock_reporter):
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'broken_api'),
'https://api.door43.org/')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir, 'ready_new_db.json'))
mockLog = MockLogger()
mock_get_url = lambda url, catch_exception: mockV3Api.get_url(
url, catch_exception)
mock_download = lambda url, dest: mockV3Api.download_file(url, dest)
event = self.make_event()
converter = TsV2CatalogHandler(event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mock_get_url,
download_handler=mock_download,
url_exists_handler=lambda url: False)
result = converter.run()
self.assertFalse(result)
self.assertIn(
'Failed to load the catalog json: No JSON object could be decoded',
mockLog._messages)
def test_create_v2_catalog(self, mock_reporter):
mockDB = MockDynamodbHandler()
mockDB._load_db(
os.path.join(TestUwV2Catalog.resources_dir, 'ready_new_db.json'))
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://cdn.door43.org/')
mockV2Api = MockAPI(os.path.join(self.resources_dir, 'v2_api'),
'https://test')
mockS3 = MockS3Handler('uw_bucket')
mockSigner = MockSigner()
mockLogger = MockLogger()
converter = UwV2CatalogHandler(
event=self._make_event(),
context=None,
logger=mockLogger,
s3_handler=mockS3,
dynamodb_handler=mockDB,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
signing_handler=mockSigner)
converter.run()
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/uw/catalog.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/uw/obs/en/obs/v4/source.json')
assert_s3_equals_api_text(self, mockS3, mockV2Api,
'v2/uw/gen/en/udb/v7/gen.usfm')
assert_s3_equals_api_text(self, mockS3, mockV2Api,
'v2/uw/1ch/en/ulb/v7/1ch.usfm')
self.assertIn('v2/uw/obs/en/obs/v4/source.json.sig',
mockS3._recent_uploads)
self.assertIn('uw/txt/2/catalog.json', mockS3._recent_uploads)
self.assertIn(
'en_udb_1ch: media format "https://cdn.door43.org/en/udb/v9/1ch.pdf" does not match source version "7" and will be excluded.',
mockLogger._messages)
self.assertIn(
'en_obs_obs: media format "https://cdn.door43.org/en/obs/v999/129kbps/en_obs_129kbps.zip" does not match source version "4" and will be excluded.',
mockLogger._messages)
def test_inprogress(self, mock_reporter):
mockV3Api = MockAPI(os.path.join(self.resources_dir, 'v3_api'),
'https://api.door43.org/')
mockV3Api.add_host(os.path.join(self.resources_dir, 'v3_cdn'),
'https://test-cdn.door43.org/')
mockV2Api = MockAPI(os.path.join(self.resources_dir, 'ts_api'),
'https://test')
mockS3 = MockS3Handler('ts_bucket')
mockDb = MockDynamodbHandler()
mockDb._load_db(
os.path.join(TestTsV2Catalog.resources_dir,
'ready_inprogress_db.json'))
mockLog = MockLogger()
event = self.make_event()
converter = TsV2CatalogHandler(
event=event,
context=None,
logger=mockLog,
s3_handler=mockS3,
dynamodb_handler=mockDb,
url_handler=mockV3Api.get_url,
download_handler=mockV3Api.download_file,
url_exists_handler=lambda url: True)
converter.run()
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/catalog.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/languages.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/en/resources.json')
self.assertNotIn('v2/ts/obs/en/obs/source.json',
mockS3._recent_uploads)
# assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/obs/en/obs/source.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/en/notes.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/obs/en/questions.json')
# we have frozen tw_cat
self.assertNotIn('v2/ts/obs/en/tw_cat.json', mockS3._recent_uploads)
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/languages.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/en/resources.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/en/ulb/v7/source.json')
self.assertNotIn('v2/ts/1ch/en/notes.json', mockS3._recent_uploads)
# assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/notes.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/1ch/en/questions.json')
self.assertNotIn('v2/ts/1ch/en/tw_cat.json', mockS3._recent_uploads)
# assert_s3_equals_api_json(self, mockS3, mockV2Api, 'v2/ts/1ch/en/tw_cat.json')
assert_s3_equals_api_json(self, mockS3, mockV2Api,
'v2/ts/bible/en/words.json')
# validate urls in generate catalogs match the generated output paths
root_url = '{}/'.format(
event['stage-variables']['cdn_url'].rstrip('/'))
catalog = json.loads(
read_file(mockS3._recent_uploads['v2/ts/catalog.json']))
url_err_msg = 'url in catalog does not match upload path: {}'
for project in catalog:
lang_catalog_path = project['lang_catalog'].replace(
root_url, '').split('?')[0]
self.assertIn(lang_catalog_path, mockS3._recent_uploads,
url_err_msg.format(lang_catalog_path))
lang_catalog = json.loads(
read_file(mockS3._recent_uploads[lang_catalog_path]))
for language in lang_catalog:
res_catalog_path = language['res_catalog'].replace(
root_url, '').split('?')[0]
self.assertIn(res_catalog_path, mockS3._recent_uploads,
url_err_msg.format(res_catalog_path))
res_catalog = json.loads(
read_file(mockS3._recent_uploads[res_catalog_path]))
for resource in res_catalog:
questions_path = resource['checking_questions'].replace(
root_url, '').split('?')[0]
# notes_path = resource['notes'].replace(root_url, '').split('?')[0]
# source_path = resource['source'].replace(root_url, '').split('?')[0]
terms_path = resource['terms'].replace(root_url,
'').split('?')[0]
# terms_map_path = resource['tw_cat'].replace(root_url, '').split('?')[0]
if questions_path:
self.assertIn(questions_path, mockS3._recent_uploads,
url_err_msg.format(questions_path))
# if notes_path:
# self.assertIn(notes_path, mockS3._uploads, url_err_msg.format(notes_path))
# if source_path:
# self.assertIn(source_path, mockS3._uploads, url_err_msg.format(source_path))
if terms_path:
self.assertIn(terms_path, mockS3._recent_uploads,
url_err_msg.format(terms_path))