def __init__(self, *args, **kwargs): super(TnTemplater, self).__init__(*args, **kwargs) index = file_utils.load_json_object(os.path.join(self.source_dir, 'index.json')) if index: self.titles = index['titles'] self.chapters = index['chapters'] self.book_codes = index['book_codes']
def test_callbackMultpleJob_build_error(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/2' self.lint_callback_data['identifier'] = '1234567890/4/2/03-LEV.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') build_log = file_utils.load_json_object(build_log_path) build_log['errors'].append('convert error') build_log['success'] = False build_log['status'] = 'errors' file_utils.write_file(build_log_path, build_log) self.finish_convert(self.source_folder) self.expected_error_count = 1 self.expected_success = False self.expected_status = "errors" self.expected_log_count = 36 self.expected_multipart = None linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results(results, linter_cb)
def __init__(self, *args, **kwargs): super(TwTemplater, self).__init__(*args, **kwargs) index = file_utils.load_json_object( os.path.join(self.source_dir, 'index.json')) if index: self.titles = index['titles'] self.chapters = index['chapters']
def test_callbackMultpleJob_build_error(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.lint_callback_data['s3_results_key'] = self.results_key + '/2' self.lint_callback_data['identifier'] = '1234567890/4/2/03-LEV.usfm' self.unzip_resource_files("en_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') build_log = file_utils.load_json_object(build_log_path) build_log['errors'].append('convert error') build_log['success'] = False build_log['status'] = 'errors' file_utils.write_file(build_log_path, build_log) self.finish_convert(self.source_folder) self.expected_error_count = 1 self.expected_success = False self.expected_status = "errors" self.expected_log_count = 36 self.expected_multipart = None linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results(results, linter_cb)
def test_listOfTempLanguageNames(self): lang_names_file = os.path.join(self.resources_dir, "templanguages.json") lang_names = file_utils.load_json_object(lang_names_file) vc = PageMetrics() success = True msg = "" for lang_name in lang_names: # given code = lang_name["lc"] language_name = lang_name["ln"] expected_language_code = code.lower() self.lang_url = "https://live.door43.org/pt-BR/" # when lang_code = vc.validate_language_code(code) # then if lang_code != expected_language_code: msg = "FAILURE: For language '{0}', expected code '{2}' but got '{1}'".format(language_name, lang_code, expected_language_code) App.logger.debug(msg) success = False if not success: self.assertTrue(success, msg)
def test_ceb_psa_text_ulb_L3(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'ceb_psa_text_ulb_L3.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'ceb_psa_text_ulb_l3') rc = RC(directory=repo_dir) rc.as_dict() json = load_json_object(os.path.join(repo_dir, 'manifest.json')) self.assertEqual(rc.resource.identifier, json['resource']['id']) self.assertEqual(rc.resource.type, 'book') self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format'])) self.assertEqual(rc.resource.file_ext, json['format']) self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d")) chapters = rc.projects[0].chapters() idx = 1 for chapter in chapters: if chapter.isnumeric(): self.assertEqual(int(chapter), idx) idx += 1 self.assertEqual(len(chapters), 151) chunks = rc.projects[0].chunks('01') self.assertEqual(len(chunks), 5)
def mock_s3_tn_project(self, part): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip') out_dir = os.path.join(self.temp_dir, 'en_tn_converted') unzip(zip_file, out_dir) src_dir = os.path.join(out_dir, 'en_tn_converted') self.project_files = [ f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f)) ] self.project_key = 'u/door43/en_tn/12345678' build_log = file_utils.load_json_object( os.path.join(src_dir, 'build_log.json')) build_log['part'] = part file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log) App.cdn_s3_handler().upload_file( os.path.join(src_dir, 'build_log.json'), '{0}/{1}/build_log.json'.format(self.project_key, part)) App.cdn_s3_handler().upload_file( os.path.join(src_dir, 'index.json'), '{0}/{1}/index.json'.format(self.project_key, part)) App.cdn_s3_handler().upload_file( os.path.join(src_dir, 'build_log.json'), '{0}/{1}/finished'.format(self.project_key, part)) App.cdn_s3_handler().upload_file( os.path.join(src_dir, '01-GEN.html'), '{0}/{1}/01-GEN.html'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'), 'u/door43/en_tq/project.json') App.door43_s3_handler().upload_file( os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def register_modules(self): script_dir = os.path.dirname(__file__) module_jsons_path = os.path.join(script_dir, '..', '..', 'functions', '*', 'module.json') module_jsons = glob(module_jsons_path) manager = TxManager() for module_json_file in module_jsons: module_json = load_json_object(module_json_file) manager.register_module(module_json)
def register_modules(self): script_dir = os.path.dirname(__file__) module_jsons_path = os.path.join(script_dir, '..', '..', 'functions', '*', 'module.json') module_jsons = glob(module_jsons_path) manager = TxManager() for module_json_file in module_jsons: module_json = load_json_object(module_json_file) manager.register_module(module_json)
def get_manifest_from_dir(self): if not self.path or not os.path.isdir(self.path): return get_manifest_from_repo_name(self.repo_name) manifest = load_yaml_object(os.path.join(self.path, 'manifest.yaml')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'manifest.json')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'package.json')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'project.json')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'meta.json')) if manifest: return manifest return get_manifest_from_repo_name(self.repo_name)
def get_manifest_from_dir(self): if not self.path or not os.path.isdir(self.path): return get_manifest_from_repo_name(self.repo_name) manifest = load_yaml_object(os.path.join(self.path, 'manifest.yaml')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'manifest.json')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'package.json')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'project.json')) if manifest: return manifest manifest = load_json_object(os.path.join(self.path, 'meta.json')) if manifest: return manifest return get_manifest_from_repo_name(self.repo_name)
def test_load_json_object(self): d = { "one": 1, "two": 2, "child": { "three": 3 } } _, self.tmp_file = tempfile.mkstemp() with open(self.tmp_file, "w") as tmpf: json.dump(d, tmpf) self.assertEqual(file_utils.load_json_object(self.tmp_file), d)
def test_callbackMultpleJob_first_merged(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.unzip_resource_files("en_ulb.zip", convert_finished=True) self.lint_callback_data['s3_results_key'] = self.results_key + '/0' self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm' build_log_path = self.get_source_path('build_log.json') build_log = file_utils.load_json_object(build_log_path) lint_log_path = self.get_source_path(file_name='lint_log.json') lint_log = file_utils.load_json_object(lint_log_path) build_log['log'] += lint_log['log'] merged_log_path = self.get_source_path(file_name='merged.json') file_utils.write_file(merged_log_path, build_log) self.expected_log_count = 36 self.expected_multipart = True linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results(results, linter_cb)
def test_callbackMultpleJob_first_merged(self): # given self.results_key = 'u/tx-manager-test-data/en-ulb/22f3d09f7a' self.unzip_resource_files("en_ulb.zip", convert_finished=True) self.lint_callback_data['s3_results_key'] = self.results_key + '/0' self.lint_callback_data['identifier'] = '1234567890/4/0/01-GEN.usfm' build_log_path = self.get_source_path('build_log.json') build_log = file_utils.load_json_object(build_log_path) lint_log_path = self.get_source_path(file_name='lint_log.json') lint_log = file_utils.load_json_object(lint_log_path) build_log['log'] += lint_log['log'] merged_log_path = self.get_source_path(file_name='merged.json') file_utils.write_file(merged_log_path, build_log) self.expected_log_count = 36 self.expected_multipart = True linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results(results, linter_cb)
def test_en_obs_package_json(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-obs-package-json.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'en-obs') rc = RC(directory=repo_dir) rc.as_dict() json = load_json_object(os.path.join(repo_dir, 'package.json')) self.assertEqual(rc.resource.identifier, json['resource']['slug']) self.assertEqual(rc.resource.type, 'book') self.assertEqual(rc.resource.format, json['content_mime_type']) self.assertEqual(rc.resource.file_ext, 'md') self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.issued, json['resource']['status']['pub_date']) chapters = rc.projects[0].chapters() self.assertEqual(len(chapters), 2) chunks = rc.project().chunks('_back') self.assertEqual(chunks, ['back-matter.md'])
def test_bible_from_tx_pre_rc(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'id_mat_text_ulb-ts.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'id_mat_text_ulb-ts') rc = RC(directory=repo_dir) rc.as_dict() json = load_json_object(os.path.join(repo_dir, 'manifest.json')) self.assertEqual(rc.resource.identifier, json['resource']['id']) self.assertEqual(rc.resource.type, 'book') self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format'])) self.assertEqual(rc.resource.file_ext, json['format']) self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d")) chapters = rc.projects[0].chapters() self.assertEqual(len(chapters), 29) chunks = rc.projects[0].chunks('01') self.assertEqual(len(chunks), 11)
def test_callbackSimpleJob_build_error(self): # given self.unzip_resource_files("id_mat_ulb.zip",convert_finished=False) build_log_path = self.get_source_path('build_log.json') build_log = file_utils.load_json_object(build_log_path) build_log['errors'].append('convert error') build_log['success'] = False build_log['status'] = 'errors' file_utils.write_file(build_log_path, build_log) self.finish_convert(self.source_folder) self.expected_log_count = 9 self.expected_error_count = 1 self.expected_success = False self.expected_status = "errors" linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results_and_log(results, linter_cb, self.expected_success, self.expected_status)
def mock_s3_tn_project(self, part): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip') out_dir = os.path.join(self.temp_dir, 'en_tn_converted') unzip(zip_file, out_dir) src_dir = os.path.join(out_dir, 'en_tn_converted') self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))] self.project_key = 'u/door43/en_tn/12345678' build_log = file_utils.load_json_object(os.path.join(src_dir, 'build_log.json')) build_log['part'] = part file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'), '{0}/{1}/build_log.json'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'index.json'), '{0}/{1}/index.json'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'), '{0}/{1}/finished'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, '01-GEN.html'), '{0}/{1}/01-GEN.html'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'), 'u/door43/en_tq/project.json') App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_callbackSimpleJob_build_error(self): # given self.unzip_resource_files("id_mat_ulb.zip", convert_finished=False) build_log_path = self.get_source_path('build_log.json') build_log = file_utils.load_json_object(build_log_path) build_log['errors'].append('convert error') build_log['success'] = False build_log['status'] = 'errors' file_utils.write_file(build_log_path, build_log) self.finish_convert(self.source_folder) self.expected_log_count = 9 self.expected_error_count = 1 self.expected_success = False self.expected_status = "errors" linter_cb = self.mock_client_linter_callback() # when results = linter_cb.process_callback() # then self.validate_results_and_log(results, linter_cb, self.expected_success, self.expected_status)
def mock_cdn_get_json(self, s3_key): source_path = os.path.join(self.source_folder, s3_key) json_data = file_utils.load_json_object(source_path) if not json_data: json_data = {} return json_data
def test_load_json_object(self): d = {"one": 1, "two": 2, "child": {"three": 3}} _, self.tmp_file = tempfile.mkstemp() with open(self.tmp_file, "w") as tmpf: json.dump(d, tmpf) self.assertEqual(file_utils.load_json_object(self.tmp_file), d)
def mock_cdn_get_json(self, s3_key): source_path = os.path.join(self.source_folder, s3_key) json_data = file_utils.load_json_object(source_path) if not json_data: json_data = {} return json_data