def test_tq(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tq.zip' # when self.doTransformTq(file_name) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['manifest.yaml', 'index.json'] for book in BOOK_NUMBERS: html_file = '{0}-{1}.html'.format(BOOK_NUMBERS[book], book.upper()) files_to_verify.append(html_file) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def download_and_check_zip_file(self, handler, expected_output_files, extension, key, file_type, success, chapter_count=-1, file_ext=""): zip_path = os.path.join(self.temp_dir, file_type + ".zip") handler.download_file(key, zip_path) temp_sub_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix=file_type + "_") unzip(zip_path, temp_sub_dir) check_list = [] if chapter_count <= 0: for file_name in expected_output_files: check_list.append(file_name + "." + extension) else: check_list = ['{0:0>2}.{1}'.format(i, file_ext) for i in range(1, chapter_count + 1)] is_first = True for file_name in check_list: output_file_path = os.path.join(temp_sub_dir, file_name) App.logger.debug("checking preconvert zip for: " + output_file_path) self.assertTrue(os.path.exists(output_file_path), "missing file: " + file_name) if is_first: self.print_file(file_name, output_file_path) is_first = False manifest_json = os.path.join(temp_sub_dir, "manifest.json") json_exists = os.path.exists(manifest_json) if not success and json_exists: # print out for troubleshooting self.print_file("manifest.json", manifest_json) manifest_yaml = os.path.join(temp_sub_dir, "manifest.yaml") yaml_exists = os.path.exists(manifest_yaml) if not success and yaml_exists: # print out for troubleshooting self.print_file("manifest.yaml", manifest_yaml) self.assertTrue(json_exists or yaml_exists, "missing manifest file")
def test_convert_only_jas(self): """Runs the converter and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'eight_bible_books.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir) source_url = 'http://test.com/preconvert/22f3d09f7a.zip?convert_only=60-JAS.usfm' with closing(Usfm2HtmlConverter(source_url, 'udb', out_zip_file)) as tx: tx.input_zip_file = zip_file results = tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(results) self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir) unzip(out_zip_file, self.out_dir) files_to_verify = ['60-JAS.html'] for file_to_verify in files_to_verify: file_name = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_name), 'UDB HTML file not found: {0}'.format(file_name)) files_to_not_verify = ['61-1PE.html', '62-2PE.html', '63-1JN.html', '64-2JN.html', '65-3JN.html', '66-JUD.html', '67-REV.html'] for file_to_verify in files_to_not_verify: file_name = os.path.join(self.out_dir, file_to_verify) self.assertFalse(os.path.isfile(file_name), 'UDB HTML file not found: {0}'.format(file_name)) self.assertEqual(tx.source, source_url.split('?')[0])
def unzip_resource_files(self, resource_file_name): self.source_zip = os.path.join(self.resources_dir, "conversion_callback", resource_file_name) self.source_folder = tempfile.mkdtemp(dir=self.temp_dir, prefix='sources_') unzip(self.source_zip, self.source_folder) source_subfolder = os.path.join(self.source_folder, resource_file_name.split('.')[0]) results_subfolder = os.path.join(self.source_folder, self.results_key) file_utils.copy_tree(source_subfolder, results_subfolder)
def test_tn_part(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tn.zip' part = '01-GEN.md' # when self.doTransformTn(file_name, part=part) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['01-GEN.html', 'manifest.yaml', 'index.json'] for dir in BOOK_NUMBERS: book = '{0}-{1}'.format(BOOK_NUMBERS[dir], dir.upper()) file = '{0}.html'.format(book) file_path = os.path.join(self.out_dir, file) if file not in files_to_verify: self.assertFalse(os.path.isfile(file_path), 'file should not be converted: {0}'.format(file)) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def mock_s3_bible_project(self, test_file_name, project_key, multi_part=False): converted_proj_dir = os.path.join(self.resources_dir, 'converted_projects') test_file_base = test_file_name.split('.zip')[0] zip_file = os.path.join(converted_proj_dir, test_file_name) out_dir = os.path.join(self.temp_dir, test_file_base) unzip(zip_file, out_dir) project_dir = os.path.join(out_dir, test_file_base) + os.path.sep self.project_files = file_utils.get_files(out_dir) self.project_key = project_key for filename in self.project_files: sub_path = filename.split(project_dir)[1].replace(os.path.sep, '/') # Make sure it is a bucket path App.cdn_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, sub_path)) if multi_part: # copy files from cdn to door43 base_name = os.path.basename(filename) if '.html' in base_name: with codecs.open(filename, 'r', 'utf-8-sig') as f: soup = BeautifulSoup(f, 'html.parser') # add nav tag new_tag = soup.new_tag('div', id='right-sidebar') soup.body.append(new_tag) html = unicode(soup) file_utils.write_file(filename, html.encode('ascii', 'xmlcharrefreplace')) App.door43_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, base_name)) # u, user, repo = project_key App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_multiple_projects(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-ta-multiple-projects.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'en_ta') rc = RC(directory=repo_dir) rc.as_dict() yaml = load_yaml_object(os.path.join(repo_dir, 'manifest.yaml')) self.assertEqual(rc.resource.identifier, yaml['dublin_core']['identifier']) self.assertEqual(rc.resource.type, yaml['dublin_core']['type']) self.assertEqual(rc.resource.format, yaml['dublin_core']['format']) self.assertEqual(rc.resource.file_ext, 'md') self.assertEqual(rc.resource.conformsto, yaml['dublin_core']['conformsto']) self.assertEqual(rc.resource.modified, yaml['dublin_core']['modified']) self.assertEqual(len(rc.project_ids), 4) self.assertEqual(rc.project_count, 4) chapters = rc.project('checking').chapters() self.assertEqual(len(chapters), 44) chunks = rc.project('checking').chunks('level1') self.assertEqual(chunks, ['01.md', 'sub-title.md', 'title.md']) self.assertTrue('acceptable' in rc.project('checking').config()) self.assertTrue('title' in rc.project('checking').toc()) self.assertTrue(rc.project('checking').toc()['title'], 'Table of Contents')
def download_repo(self, commit_url, repo_dir): """ Downloads and unzips a git repository from Github or git.door43.org :param str|unicode commit_url: The URL of the repository to download :param str|unicode repo_dir: The directory where the downloaded file should be unzipped :return: None """ repo_zip_url = commit_url.replace('commit', 'archive') + '.zip' repo_zip_file = os.path.join(self.base_temp_dir, repo_zip_url.rpartition(os.path.sep)[2]) try: App.logger.debug('Downloading {0}...'.format(repo_zip_url)) # if the file already exists, remove it, we want a fresh copy if os.path.isfile(repo_zip_file): os.remove(repo_zip_file) download_file(repo_zip_url, repo_zip_file) finally: App.logger.debug('finished.') try: App.logger.debug('Unzipping {0}...'.format(repo_zip_file)) unzip(repo_zip_file, repo_dir) finally: App.logger.debug('finished.') # clean up the downloaded zip file if os.path.isfile(repo_zip_file): os.remove(repo_zip_file)
def test_ceb_psa_text_ulb_L3(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'ceb_psa_text_ulb_L3.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'ceb_psa_text_ulb_l3') rc = RC(directory=repo_dir) rc.as_dict() json = load_json_object(os.path.join(repo_dir, 'manifest.json')) self.assertEqual(rc.resource.identifier, json['resource']['id']) self.assertEqual(rc.resource.type, 'book') self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format'])) self.assertEqual(rc.resource.file_ext, json['format']) self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d")) chapters = rc.projects[0].chapters() idx = 1 for chapter in chapters: if chapter.isnumeric(): self.assertEqual(int(chapter), idx) idx += 1 self.assertEqual(len(chapters), 151) chunks = rc.projects[0].chunks('01') self.assertEqual(len(chunks), 5)
def unzip_archive(self): App.logger.debug("Unzipping {0} to {1}".format(self.source_zip_file, self.temp_dir)) unzip(self.source_zip_file, self.temp_dir) dirs = [d for d in os.listdir(self.temp_dir) if os.path.isdir(os.path.join(self.temp_dir, d))] if len(dirs): self.source_dir = os.path.join(self.temp_dir, dirs[0]) else: self.source_dir = self.temp_dir
def unzip_converted_files(self, converted_zip_file): unzip_dir = tempfile.mkdtemp(prefix='unzip_', dir=self.temp_dir) try: App.logger.debug('Unzipping {0}...'.format(converted_zip_file)) unzip(converted_zip_file, unzip_dir) finally: App.logger.debug('finished.') return unzip_dir
def test_lint_warnings(self, mock_invoke): obs_zip_file = os.path.join(self.resources_dir, 'obs_linter', 'en-obs.zip') unzip(obs_zip_file, self.temp_dir) source_dir = os.path.join(self.temp_dir, 'en-obs') mock_invoke.return_value = {} expected_warnings = True linter = ObsLinter(source_dir=source_dir) linter.run() self.verify_results(expected_warnings, linter)
def mock_s3_obs_project(self): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en-obs-complete.zip') out_dir = os.path.join(self.temp_dir, 'en-obs-complete') unzip(zip_file, out_dir) project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678') self.project_files = [f for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f))] self.project_key = 'door43/en-obs/12345678' for filename in self.project_files: App.cdn_s3_handler().upload_file(os.path.join(project_dir, filename), 'u/{0}/{1}'.format(self.project_key, filename)) App.cdn_s3_handler().upload_file(os.path.join(out_dir, 'door43', 'en-obs', 'project.json'), 'u/door43/en-obs/project.json')
def extractFiles(cls, file_name, repo_name): file_path = os.path.join(TestBiblePreprocessor.resources_dir, file_name) # 1) unzip the repo files temp_dir = tempfile.mkdtemp(prefix='repo_') unzip(file_path, temp_dir) repo_dir = os.path.join(temp_dir, repo_name) if not os.path.isdir(repo_dir): repo_dir = temp_dir # 2) Get the resource container rc = RC(repo_dir) return rc, repo_dir, temp_dir
def test_unzip(self): self.tmp_dir = tempfile.mkdtemp() zip_file = os.path.join(self.tmp_dir, 'foo.zip') _, self.tmp_file = tempfile.mkstemp() with open(self.tmp_file, "w") as tmpf: tmpf.write("hello world") with zipfile.ZipFile(zip_file, "w") as zf: zf.write(self.tmp_file, os.path.basename(self.tmp_file)) file_utils.unzip(zip_file, self.tmp_dir) with open(os.path.join(self.tmp_dir, os.path.basename(self.tmp_file))) as outf: self.assertEqual(outf.read(), "hello world")
def mock_s3_tw_project(self): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tw_converted.zip') out_dir = os.path.join(self.temp_dir, 'en_tw_converted') unzip(zip_file, out_dir) self.src_dir = src_dir = os.path.join(out_dir, 'en_tw_converted') self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))] self.project_key = 'u/door43/en_tw/12345678' for filename in self.project_files: App.cdn_s3_handler().upload_file(os.path.join(src_dir, filename), '{0}/{1}'.format(self.project_key, filename)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'), 'u/door43/en_tw/project.json') App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_en_obs_manifest_yaml(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-obs-manifest-yaml.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'en_obs') rc = RC(directory=repo_dir, repo_name='en_obs') rc_dic = rc.as_dict() yaml = load_yaml_object(os.path.join(repo_dir, 'manifest.yaml')) self.assertDictEqual(yaml, rc_dic) chapters = rc.projects[0].chapters() self.assertEqual(len(chapters), 2) chunks = rc.project().chunks('front') self.assertEqual(chunks, ['intro.md', 'title.md'])
def test_ta(self): """ Runs the converter and verifies the output """ file_name = 'en_ta.zip' self.doTransformTa(file_name) self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='ta_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['checking.html', 'checking-toc.yaml', 'intro.html', 'intro-toc.yaml', 'process.html', 'process-toc.yaml', 'translate.html', 'translate-toc.yaml'] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def test_matt_complete_with_backslash(self): """ Runs the converter and verifies the output """ zip_file = os.path.join(self.resources_dir, 'kpb_mat_text_udb.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir) with closing(Usfm2HtmlConverter('', 'udb', out_zip_file)) as tx: tx.input_zip_file = zip_file results = tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(results) self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir) unzip(out_zip_file, self.out_dir) files_to_verify = ['41-MAT.html'] self.verify_files(files_to_verify)
def test_bible_no_manifest(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'bible-no-manifest.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'en_ulb') rc = RC(directory=repo_dir) rc.as_dict() self.assertEqual(rc.resource.identifier, 'ulb') self.assertEqual(rc.resource.type, 'bundle') self.assertEqual(rc.resource.format, 'text/usfm') self.assertEqual(rc.resource.file_ext, 'usfm') self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d")) chapters = rc.project().chapters() self.assertEqual(len(chapters), 0) self.assertEqual(len(rc.project().usfm_files()), 8)
def test_php_illegal_url(self): """ Runs the converter and verifies the output """ # test with the English OBS zip_file = os.path.join(self.resources_dir, '51-PHP.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir) with closing(Usfm2HtmlConverter(' ', 'udb', out_zip_file)) as tx: tx.input_zip_file = zip_file results = tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(results) self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir) unzip(out_zip_file, self.out_dir) files_to_verify = ['51-PHP.html'] self.verify_files(files_to_verify)
def test_bible_from_tx_pre_rc(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'id_mat_text_ulb-ts.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'id_mat_text_ulb-ts') rc = RC(directory=repo_dir) rc.as_dict() json = load_json_object(os.path.join(repo_dir, 'manifest.json')) self.assertEqual(rc.resource.identifier, json['resource']['id']) self.assertEqual(rc.resource.type, 'book') self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format'])) self.assertEqual(rc.resource.file_ext, json['format']) self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d")) chapters = rc.projects[0].chapters() self.assertEqual(len(chapters), 29) chunks = rc.projects[0].chunks('01') self.assertEqual(len(chunks), 11)
def test_en_obs_package_json(self): """ Populates the ResourceContainer object and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-obs-package-json.zip') self.out_dir = tempfile.mkdtemp(prefix='repo_') unzip(zip_file, self.out_dir) repo_dir = os.path.join(self.out_dir, 'en-obs') rc = RC(directory=repo_dir) rc.as_dict() json = load_json_object(os.path.join(repo_dir, 'package.json')) self.assertEqual(rc.resource.identifier, json['resource']['slug']) self.assertEqual(rc.resource.type, 'book') self.assertEqual(rc.resource.format, json['content_mime_type']) self.assertEqual(rc.resource.file_ext, 'md') self.assertEqual(rc.resource.conformsto, 'pre-rc') self.assertEqual(rc.resource.issued, json['resource']['status']['pub_date']) chapters = rc.projects[0].chapters() self.assertEqual(len(chapters), 2) chunks = rc.project().chunks('_back') self.assertEqual(chunks, ['back-matter.md'])
def test_ta(self): """ Runs the converter and verifies the output """ file_name = 'en_ta.zip' self.doTransformTa(file_name) self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='ta_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = [ 'checking.html', 'checking-toc.yaml', 'intro.html', 'intro-toc.yaml', 'process.html', 'process-toc.yaml', 'translate.html', 'translate-toc.yaml' ] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'.format(file_to_verify))
def mock_s3_obs_project(self): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en-obs-complete.zip') out_dir = os.path.join(self.temp_dir, 'en-obs-complete') unzip(zip_file, out_dir) project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678') self.project_files = [ f for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f)) ] self.project_key = 'u/door43/en-obs/12345678' for filename in self.project_files: App.cdn_s3_handler().upload_file( os.path.join(project_dir, filename), '{0}/{1}'.format(self.project_key, filename)) App.cdn_s3_handler().upload_file( os.path.join(out_dir, 'door43', 'en-obs', 'project.json'), 'u/door43/en-obs/project.json') App.door43_s3_handler().upload_file( os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_run(self): """Runs the converter and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-obs.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp(prefix="en-obs", suffix=".zip") with closing(Md2HtmlConverter('', 'obs', out_zip_file)) as tx: tx.input_zip_file = zip_file tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.out_dir = tempfile.mkdtemp(prefix='obs_') unzip(out_zip_file, self.out_dir) remove(out_zip_file) files_to_verify = [] for i in range(1, 51): files_to_verify.append(str(i).zfill(2) + '.html') for file_to_verify in files_to_verify: file_name = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_name), 'OBS HTML file not found: {0}'.format(file_name))
def mock_s3_tn_project(self, part): zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip') out_dir = os.path.join(self.temp_dir, 'en_tn_converted') unzip(zip_file, out_dir) src_dir = os.path.join(out_dir, 'en_tn_converted') self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))] self.project_key = 'u/door43/en_tn/12345678' build_log = file_utils.load_json_object(os.path.join(src_dir, 'build_log.json')) build_log['part'] = part file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'), '{0}/{1}/build_log.json'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'index.json'), '{0}/{1}/index.json'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'), '{0}/{1}/finished'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, '01-GEN.html'), '{0}/{1}/01-GEN.html'.format(self.project_key, part)) App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'), 'u/door43/en_tq/project.json') App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_tw(self): """ Runs the converter and verifies the output """ # given file_name = 'en_tw.zip' # when self.doTransformTw(file_name) # then self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='tw_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = ['index.json', 'kt.html', 'names.html', 'other.html', 'config.yaml', 'manifest.yaml'] for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_path), 'file not found: {0}' .format(file_to_verify))
def mock_s3_bible_project(self, test_file_name, project_key, multi_part=False): converted_proj_dir = os.path.join(self.resources_dir, 'converted_projects') test_file_base = test_file_name.split('.zip')[0] zip_file = os.path.join(converted_proj_dir, test_file_name) out_dir = os.path.join(self.temp_dir, test_file_base) unzip(zip_file, out_dir) project_dir = os.path.join(out_dir, test_file_base) + os.path.sep self.project_files = file_utils.get_files(out_dir) self.project_key = project_key for filename in self.project_files: sub_path = filename.split(project_dir)[1].replace( os.path.sep, '/') # Make sure it is a bucket path App.cdn_s3_handler().upload_file( filename, '{0}/{1}'.format(project_key, sub_path)) if multi_part: # copy files from cdn to door43 base_name = os.path.basename(filename) if '.html' in base_name: with codecs.open(filename, 'r', 'utf-8-sig') as f: soup = BeautifulSoup(f, 'html.parser') # add nav tag new_tag = soup.new_tag('div', id='right-sidebar') soup.body.append(new_tag) html = unicode(soup) file_utils.write_file( filename, html.encode('ascii', 'xmlcharrefreplace')) App.door43_s3_handler().upload_file( filename, '{0}/{1}'.format(project_key, base_name)) # u, user, repo = project_key App.door43_s3_handler().upload_file( os.path.join(self.resources_dir, 'templates', 'project-page.html'), 'templates/project-page.html')
def test_run(self): """Runs the converter and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'eight_bible_books.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir) with closing(Usfm2HtmlConverter('', 'udb', out_zip_file)) as tx: tx.input_zip_file = zip_file results = tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(results) self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir) unzip(out_zip_file, self.out_dir) files_to_verify = [ '60-JAS.html', '61-1PE.html', '62-2PE.html', '63-1JN.html', '64-2JN.html', '65-3JN.html', '66-JUD.html', '67-REV.html' ] for file_to_verify in files_to_verify: file_name = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_name), 'UDB HTML file not found: {0}'.format(file_name))
def test_run(self): """Runs the converter and verifies the output.""" # test with the English OBS zip_file = os.path.join(self.resources_dir, 'en-obs.zip') zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file) out_zip_file = tempfile.mktemp(prefix="en-obs", suffix=".zip") with closing(Md2HtmlConverter('', 'obs', out_zip_file)) as tx: tx.input_zip_file = zip_file tx.run() # verify the output self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.") self.out_dir = tempfile.mkdtemp(prefix='obs_') unzip(out_zip_file, self.out_dir) remove(out_zip_file) files_to_verify = [] for i in range(1, 51): files_to_verify.append(str(i).zfill(2) + '.html') for file_to_verify in files_to_verify: file_name = os.path.join(self.out_dir, file_to_verify) self.assertTrue(os.path.isfile(file_name), 'OBS HTML file not found: {0}'.format(file_name))
def verifyTransform(self, tx, missing_chapters=None): if not missing_chapters: missing_chapters = [] self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.") self.assertIsNotNone(self.return_val, "There was no return value.") self.out_dir = tempfile.mkdtemp(prefix='obs_') unzip(self.out_zip_file, self.out_dir) remove(self.out_zip_file) files_to_verify = [] files_missing = [] for i in range(1, 51): file_name = str(i).zfill(2) + '.html' if not i in missing_chapters: files_to_verify.append(file_name) else: files_missing.append(file_name) for file_to_verify in files_to_verify: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNotNone(contents, 'OBS HTML body contents not found: {0}'.format(os.path.basename(file_path))) for file_to_verify in files_missing: file_path = os.path.join(self.out_dir, file_to_verify) contents = self.getContents(file_path) self.assertIsNone(contents, 'OBS HTML body contents present, but should not be: {0}'.format(os.path.basename(file_path))) self.assertEqual(self.return_val['success'], self.expected_success, "Mismatch in for success boolean") self.assertEqual(len(self.return_val['info']) == 0, self.expected_info_empty, "Mismatch in expected info empty") for warning in self.return_val['warnings']: App.logger.debug("Warning: " + warning) for error in self.return_val['errors']: App.logger.debug("Error: " + error) self.assertEqual(len(self.return_val['warnings']), self.expected_warnings, "Mismatch in expected warnings") self.assertEqual(len(self.return_val['errors']), self.expected_errors, "Mismatch in expected errors")
def unzip_resource(self, zip_name): zip_file = os.path.join(self.resources_dir, zip_name) out_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix='linter_test_') unzip(zip_file, out_dir) return out_dir
def setUp(self): """Runs before each test.""" self.temp_dir = tempfile.mkdtemp(prefix='tmp_obs_') unzip(self.obs_zip_file, self.temp_dir) self.repo_dir = os.path.join(self.temp_dir, 'es_obs')
def test_lint_no_warnings(self, mock_invoke): unzip(self.obs_zip_file, self.temp_dir) mock_invoke.return_value = {} expected_warnings = False linter = self.run_linter() self.verify_results(expected_warnings, linter)
def unzip_resource(self, zip_name): zip_file = os.path.join(self.resources_dir, zip_name) out_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix='linter_test_') unzip(zip_file, out_dir) return out_dir
def extractZipFiles(self, test_folder_name): file_path = os.path.join(self.resources_dir, test_folder_name) self.temp_dir = tempfile.mkdtemp(prefix='repo_') unzip(file_path, self.temp_dir) return self.temp_dir
def test_lint_no_warnings(self, mock_invoke): unzip(self.obs_zip_file, self.temp_dir) mock_invoke.return_value = {} expected_warnings = False linter = self.run_linter() self.verify_results(expected_warnings, linter)
def setUp(self): """Runs before each test.""" self.temp_dir = tempfile.mkdtemp(prefix='tmp_obs_') unzip(self.obs_zip_file, self.temp_dir) self.repo_dir = os.path.join(self.temp_dir, 'es_obs')