def __init__(self, *args, **kwargs):
super(TnTemplater, self).__init__(*args, **kwargs)
index = file_utils.load_json_object(os.path.join(self.source_dir, 'index.json'))
if index:
self.titles = index['titles']
self.chapters = index['chapters']
self.book_codes = index['book_codes']
def test_callbackMultpleJob_build_error(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.lint_callback_data['s3_results_key'] = self.results_key + '/2'
self.lint_callback_data['identifier'] = '1234567890/4/2/03-LEV.usfm'
self.unzip_resource_files("en_ulb.zip", convert_finished=False)
build_log_path = self.get_source_path('build_log.json')
build_log = file_utils.load_json_object(build_log_path)
build_log['errors'].append('convert error')
build_log['success'] = False
build_log['status'] = 'errors'
file_utils.write_file(build_log_path, build_log)
self.finish_convert(self.source_folder)
self.expected_error_count = 1
self.expected_success = False
self.expected_status = "errors"
self.expected_log_count = 36
self.expected_multipart = None
linter_cb = self.mock_client_linter_callback()
# when
results = linter_cb.process_callback()
# then
self.validate_results(results, linter_cb)
def __init__(self, *args, **kwargs):
super(TwTemplater, self).__init__(*args, **kwargs)
index = file_utils.load_json_object(
os.path.join(self.source_dir, 'index.json'))
if index:
self.titles = index['titles']
self.chapters = index['chapters']
def test_callbackMultpleJob_build_error(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.lint_callback_data['s3_results_key'] = self.results_key + '/2'
self.lint_callback_data['identifier'] = '1234567890/4/2/03-LEV.usfm'
self.unzip_resource_files("en_ulb.zip", convert_finished=False)
build_log_path = self.get_source_path('build_log.json')
build_log = file_utils.load_json_object(build_log_path)
build_log['errors'].append('convert error')
build_log['success'] = False
build_log['status'] = 'errors'
file_utils.write_file(build_log_path, build_log)
self.finish_convert(self.source_folder)
self.expected_error_count = 1
self.expected_success = False
self.expected_status = "errors"
self.expected_log_count = 36
self.expected_multipart = None
linter_cb = self.mock_client_linter_callback()
# when
results = linter_cb.process_callback()
# then
self.validate_results(results, linter_cb)
def test_listOfTempLanguageNames(self):
lang_names_file = os.path.join(self.resources_dir, "templanguages.json")
lang_names = file_utils.load_json_object(lang_names_file)
vc = PageMetrics()
success = True
msg = ""
for lang_name in lang_names:
# given
code = lang_name["lc"]
language_name = lang_name["ln"]
expected_language_code = code.lower()
self.lang_url = "https://live.door43.org/pt-BR/"
# when
lang_code = vc.validate_language_code(code)
# then
if lang_code != expected_language_code:
msg = "FAILURE: For language '{0}', expected code '{2}' but got '{1}'".format(language_name, lang_code,
expected_language_code)
App.logger.debug(msg)
success = False
if not success:
self.assertTrue(success, msg)
def test_ceb_psa_text_ulb_L3(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'ceb_psa_text_ulb_L3.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'ceb_psa_text_ulb_l3')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'manifest.json'))
self.assertEqual(rc.resource.identifier, json['resource']['id'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format']))
self.assertEqual(rc.resource.file_ext, json['format'])
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.projects[0].chapters()
idx = 1
for chapter in chapters:
if chapter.isnumeric():
self.assertEqual(int(chapter), idx)
idx += 1
self.assertEqual(len(chapters), 151)
chunks = rc.projects[0].chunks('01')
self.assertEqual(len(chunks), 5)
def mock_s3_tn_project(self, part):
zip_file = os.path.join(self.resources_dir, 'converted_projects',
'en_tn_converted.zip')
out_dir = os.path.join(self.temp_dir, 'en_tn_converted')
unzip(zip_file, out_dir)
src_dir = os.path.join(out_dir, 'en_tn_converted')
self.project_files = [
f for f in os.listdir(src_dir)
if os.path.isfile(os.path.join(src_dir, f))
]
self.project_key = 'u/door43/en_tn/12345678'
build_log = file_utils.load_json_object(
os.path.join(src_dir, 'build_log.json'))
build_log['part'] = part
file_utils.write_file(os.path.join(src_dir, 'build_log.json'),
build_log)
App.cdn_s3_handler().upload_file(
os.path.join(src_dir, 'build_log.json'),
'{0}/{1}/build_log.json'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(
os.path.join(src_dir, 'index.json'),
'{0}/{1}/index.json'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(
os.path.join(src_dir, 'build_log.json'),
'{0}/{1}/finished'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(
os.path.join(src_dir, '01-GEN.html'),
'{0}/{1}/01-GEN.html'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'),
'u/door43/en_tq/project.json')
App.door43_s3_handler().upload_file(
os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def register_modules(self):
script_dir = os.path.dirname(__file__)
module_jsons_path = os.path.join(script_dir, '..', '..', 'functions', '*', 'module.json')
module_jsons = glob(module_jsons_path)
manager = TxManager()
for module_json_file in module_jsons:
module_json = load_json_object(module_json_file)
manager.register_module(module_json)
def register_modules(self):
script_dir = os.path.dirname(__file__)
module_jsons_path = os.path.join(script_dir, '..', '..', 'functions',
'*', 'module.json')
module_jsons = glob(module_jsons_path)
manager = TxManager()
for module_json_file in module_jsons:
module_json = load_json_object(module_json_file)
manager.register_module(module_json)
def get_manifest_from_dir(self):
if not self.path or not os.path.isdir(self.path):
return get_manifest_from_repo_name(self.repo_name)
manifest = load_yaml_object(os.path.join(self.path, 'manifest.yaml'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'manifest.json'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'package.json'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'project.json'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'meta.json'))
if manifest:
return manifest
return get_manifest_from_repo_name(self.repo_name)
def get_manifest_from_dir(self):
if not self.path or not os.path.isdir(self.path):
return get_manifest_from_repo_name(self.repo_name)
manifest = load_yaml_object(os.path.join(self.path, 'manifest.yaml'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'manifest.json'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'package.json'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'project.json'))
if manifest:
return manifest
manifest = load_json_object(os.path.join(self.path, 'meta.json'))
if manifest:
return manifest
return get_manifest_from_repo_name(self.repo_name)
def test_load_json_object(self):
d = {
"one": 1,
"two": 2,
"child": {
"three": 3
}
}
_, self.tmp_file = tempfile.mkstemp()
with open(self.tmp_file, "w") as tmpf:
json.dump(d, tmpf)
self.assertEqual(file_utils.load_json_object(self.tmp_file), d)
def test_callbackMultpleJob_first_merged(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.unzip_resource_files("en_ulb.zip", convert_finished=True)
self.lint_callback_data['s3_results_key'] = self.results_key + '/0'
self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm'
build_log_path = self.get_source_path('build_log.json')
build_log = file_utils.load_json_object(build_log_path)
lint_log_path = self.get_source_path(file_name='lint_log.json')
lint_log = file_utils.load_json_object(lint_log_path)
build_log['log'] += lint_log['log']
merged_log_path = self.get_source_path(file_name='merged.json')
file_utils.write_file(merged_log_path, build_log)
self.expected_log_count = 36
self.expected_multipart = True
linter_cb = self.mock_client_linter_callback()
# when
results = linter_cb.process_callback()
# then
self.validate_results(results, linter_cb)
def test_callbackMultpleJob_first_merged(self):
# given
self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a'
self.unzip_resource_files("en_ulb.zip", convert_finished=True)
self.lint_callback_data['s3_results_key'] = self.results_key + '/0'
self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm'
build_log_path = self.get_source_path('build_log.json')
build_log = file_utils.load_json_object(build_log_path)
lint_log_path = self.get_source_path(file_name='lint_log.json')
lint_log = file_utils.load_json_object(lint_log_path)
build_log['log'] += lint_log['log']
merged_log_path = self.get_source_path(file_name='merged.json')
file_utils.write_file(merged_log_path, build_log)
self.expected_log_count = 36
self.expected_multipart = True
linter_cb = self.mock_client_linter_callback()
# when
results = linter_cb.process_callback()
# then
self.validate_results(results, linter_cb)
def test_en_obs_package_json(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs-package-json.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en-obs')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'package.json'))
self.assertEqual(rc.resource.identifier, json['resource']['slug'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, json['content_mime_type'])
self.assertEqual(rc.resource.file_ext, 'md')
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.issued, json['resource']['status']['pub_date'])
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 2)
chunks = rc.project().chunks('_back')
self.assertEqual(chunks, ['back-matter.md'])
def test_bible_from_tx_pre_rc(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'id_mat_text_ulb-ts.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'id_mat_text_ulb-ts')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'manifest.json'))
self.assertEqual(rc.resource.identifier, json['resource']['id'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format']))
self.assertEqual(rc.resource.file_ext, json['format'])
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 29)
chunks = rc.projects[0].chunks('01')
self.assertEqual(len(chunks), 11)
def test_callbackSimpleJob_build_error(self):
# given
self.unzip_resource_files("id_mat_ulb.zip",convert_finished=False)
build_log_path = self.get_source_path('build_log.json')
build_log = file_utils.load_json_object(build_log_path)
build_log['errors'].append('convert error')
build_log['success'] = False
build_log['status'] = 'errors'
file_utils.write_file(build_log_path, build_log)
self.finish_convert(self.source_folder)
self.expected_log_count = 9
self.expected_error_count = 1
self.expected_success = False
self.expected_status = "errors"
linter_cb = self.mock_client_linter_callback()
# when
results = linter_cb.process_callback()
# then
self.validate_results_and_log(results, linter_cb, self.expected_success, self.expected_status)
def mock_s3_tn_project(self, part):
zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip')
out_dir = os.path.join(self.temp_dir, 'en_tn_converted')
unzip(zip_file, out_dir)
src_dir = os.path.join(out_dir, 'en_tn_converted')
self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))]
self.project_key = 'u/door43/en_tn/12345678'
build_log = file_utils.load_json_object(os.path.join(src_dir, 'build_log.json'))
build_log['part'] = part
file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log)
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'),
'{0}/{1}/build_log.json'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'index.json'),
'{0}/{1}/index.json'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'),
'{0}/{1}/finished'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, '01-GEN.html'),
'{0}/{1}/01-GEN.html'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'),
'u/door43/en_tq/project.json')
App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def test_callbackSimpleJob_build_error(self):
# given
self.unzip_resource_files("id_mat_ulb.zip", convert_finished=False)
build_log_path = self.get_source_path('build_log.json')
build_log = file_utils.load_json_object(build_log_path)
build_log['errors'].append('convert error')
build_log['success'] = False
build_log['status'] = 'errors'
file_utils.write_file(build_log_path, build_log)
self.finish_convert(self.source_folder)
self.expected_log_count = 9
self.expected_error_count = 1
self.expected_success = False
self.expected_status = "errors"
linter_cb = self.mock_client_linter_callback()
# when
results = linter_cb.process_callback()
# then
self.validate_results_and_log(results, linter_cb,
self.expected_success,
self.expected_status)
def mock_cdn_get_json(self, s3_key):
source_path = os.path.join(self.source_folder, s3_key)
json_data = file_utils.load_json_object(source_path)
if not json_data:
json_data = {}
return json_data
def test_load_json_object(self):
d = {"one": 1, "two": 2, "child": {"three": 3}}
_, self.tmp_file = tempfile.mkstemp()
with open(self.tmp_file, "w") as tmpf:
json.dump(d, tmpf)
self.assertEqual(file_utils.load_json_object(self.tmp_file), d)
def mock_cdn_get_json(self, s3_key):
source_path = os.path.join(self.source_folder, s3_key)
json_data = file_utils.load_json_object(source_path)
if not json_data:
json_data = {}
return json_data
