def test_tq(self):
"""
Runs the converter and verifies the output
"""
# given
file_name = 'en_tq.zip'
# when
self.doTransformTq(file_name)
# then
self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='tw_')
unzip(self.out_zip_file, self.out_dir)
remove(self.out_zip_file)
files_to_verify = ['manifest.yaml', 'index.json']
for book in BOOK_NUMBERS:
html_file = '{0}-{1}.html'.format(BOOK_NUMBERS[book], book.upper())
files_to_verify.append(html_file)
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'
.format(file_to_verify))
def download_and_check_zip_file(self, handler, expected_output_files, extension, key, file_type, success,
chapter_count=-1, file_ext=""):
zip_path = os.path.join(self.temp_dir, file_type + ".zip")
handler.download_file(key, zip_path)
temp_sub_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix=file_type + "_")
unzip(zip_path, temp_sub_dir)
check_list = []
if chapter_count <= 0:
for file_name in expected_output_files:
check_list.append(file_name + "." + extension)
else:
check_list = ['{0:0>2}.{1}'.format(i, file_ext) for i in range(1, chapter_count + 1)]
is_first = True
for file_name in check_list:
output_file_path = os.path.join(temp_sub_dir, file_name)
App.logger.debug("checking preconvert zip for: " + output_file_path)
self.assertTrue(os.path.exists(output_file_path), "missing file: " + file_name)
if is_first:
self.print_file(file_name, output_file_path)
is_first = False
manifest_json = os.path.join(temp_sub_dir, "manifest.json")
json_exists = os.path.exists(manifest_json)
if not success and json_exists: # print out for troubleshooting
self.print_file("manifest.json", manifest_json)
manifest_yaml = os.path.join(temp_sub_dir, "manifest.yaml")
yaml_exists = os.path.exists(manifest_yaml)
if not success and yaml_exists: # print out for troubleshooting
self.print_file("manifest.yaml", manifest_yaml)
self.assertTrue(json_exists or yaml_exists, "missing manifest file")
def test_convert_only_jas(self):
"""Runs the converter and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'eight_bible_books.zip')
zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir)
source_url = 'http://test.com/preconvert/22f3d09f7a.zip?convert_only=60-JAS.usfm'
with closing(Usfm2HtmlConverter(source_url, 'udb', out_zip_file)) as tx:
tx.input_zip_file = zip_file
results = tx.run()
# verify the output
self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(results)
self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir)
unzip(out_zip_file, self.out_dir)
files_to_verify = ['60-JAS.html']
for file_to_verify in files_to_verify:
file_name = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_name), 'UDB HTML file not found: {0}'.format(file_name))
files_to_not_verify = ['61-1PE.html', '62-2PE.html', '63-1JN.html', '64-2JN.html', '65-3JN.html',
'66-JUD.html', '67-REV.html']
for file_to_verify in files_to_not_verify:
file_name = os.path.join(self.out_dir, file_to_verify)
self.assertFalse(os.path.isfile(file_name), 'UDB HTML file not found: {0}'.format(file_name))
self.assertEqual(tx.source, source_url.split('?')[0])
def unzip_resource_files(self, resource_file_name):
self.source_zip = os.path.join(self.resources_dir, "conversion_callback", resource_file_name)
self.source_folder = tempfile.mkdtemp(dir=self.temp_dir, prefix='sources_')
unzip(self.source_zip, self.source_folder)
source_subfolder = os.path.join(self.source_folder, resource_file_name.split('.')[0])
results_subfolder = os.path.join(self.source_folder, self.results_key)
file_utils.copy_tree(source_subfolder, results_subfolder)
def test_tn_part(self):
"""
Runs the converter and verifies the output
"""
# given
file_name = 'en_tn.zip'
part = '01-GEN.md'
# when
self.doTransformTn(file_name, part=part)
# then
self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='tw_')
unzip(self.out_zip_file, self.out_dir)
remove(self.out_zip_file)
files_to_verify = ['01-GEN.html', 'manifest.yaml', 'index.json']
for dir in BOOK_NUMBERS:
book = '{0}-{1}'.format(BOOK_NUMBERS[dir], dir.upper())
file = '{0}.html'.format(book)
file_path = os.path.join(self.out_dir, file)
if file not in files_to_verify:
self.assertFalse(os.path.isfile(file_path), 'file should not be converted: {0}'.format(file))
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'
.format(file_to_verify))
def mock_s3_bible_project(self, test_file_name, project_key, multi_part=False):
converted_proj_dir = os.path.join(self.resources_dir, 'converted_projects')
test_file_base = test_file_name.split('.zip')[0]
zip_file = os.path.join(converted_proj_dir, test_file_name)
out_dir = os.path.join(self.temp_dir, test_file_base)
unzip(zip_file, out_dir)
project_dir = os.path.join(out_dir, test_file_base) + os.path.sep
self.project_files = file_utils.get_files(out_dir)
self.project_key = project_key
for filename in self.project_files:
sub_path = filename.split(project_dir)[1].replace(os.path.sep, '/') # Make sure it is a bucket path
App.cdn_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, sub_path))
if multi_part: # copy files from cdn to door43
base_name = os.path.basename(filename)
if '.html' in base_name:
with codecs.open(filename, 'r', 'utf-8-sig') as f:
soup = BeautifulSoup(f, 'html.parser')
# add nav tag
new_tag = soup.new_tag('div', id='right-sidebar')
soup.body.append(new_tag)
html = unicode(soup)
file_utils.write_file(filename, html.encode('ascii', 'xmlcharrefreplace'))
App.door43_s3_handler().upload_file(filename, '{0}/{1}'.format(project_key, base_name))
# u, user, repo = project_key
App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def test_multiple_projects(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-ta-multiple-projects.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en_ta')
rc = RC(directory=repo_dir)
rc.as_dict()
yaml = load_yaml_object(os.path.join(repo_dir, 'manifest.yaml'))
self.assertEqual(rc.resource.identifier, yaml['dublin_core']['identifier'])
self.assertEqual(rc.resource.type, yaml['dublin_core']['type'])
self.assertEqual(rc.resource.format, yaml['dublin_core']['format'])
self.assertEqual(rc.resource.file_ext, 'md')
self.assertEqual(rc.resource.conformsto, yaml['dublin_core']['conformsto'])
self.assertEqual(rc.resource.modified, yaml['dublin_core']['modified'])
self.assertEqual(len(rc.project_ids), 4)
self.assertEqual(rc.project_count, 4)
chapters = rc.project('checking').chapters()
self.assertEqual(len(chapters), 44)
chunks = rc.project('checking').chunks('level1')
self.assertEqual(chunks, ['01.md', 'sub-title.md', 'title.md'])
self.assertTrue('acceptable' in rc.project('checking').config())
self.assertTrue('title' in rc.project('checking').toc())
self.assertTrue(rc.project('checking').toc()['title'], 'Table of Contents')
def download_repo(self, commit_url, repo_dir):
"""
Downloads and unzips a git repository from Github or git.door43.org
:param str|unicode commit_url: The URL of the repository to download
:param str|unicode repo_dir: The directory where the downloaded file should be unzipped
:return: None
"""
repo_zip_url = commit_url.replace('commit', 'archive') + '.zip'
repo_zip_file = os.path.join(self.base_temp_dir, repo_zip_url.rpartition(os.path.sep)[2])
try:
App.logger.debug('Downloading {0}...'.format(repo_zip_url))
# if the file already exists, remove it, we want a fresh copy
if os.path.isfile(repo_zip_file):
os.remove(repo_zip_file)
download_file(repo_zip_url, repo_zip_file)
finally:
App.logger.debug('finished.')
try:
App.logger.debug('Unzipping {0}...'.format(repo_zip_file))
unzip(repo_zip_file, repo_dir)
finally:
App.logger.debug('finished.')
# clean up the downloaded zip file
if os.path.isfile(repo_zip_file):
os.remove(repo_zip_file)
def test_ceb_psa_text_ulb_L3(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'ceb_psa_text_ulb_L3.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'ceb_psa_text_ulb_l3')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'manifest.json'))
self.assertEqual(rc.resource.identifier, json['resource']['id'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format']))
self.assertEqual(rc.resource.file_ext, json['format'])
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.projects[0].chapters()
idx = 1
for chapter in chapters:
if chapter.isnumeric():
self.assertEqual(int(chapter), idx)
idx += 1
self.assertEqual(len(chapters), 151)
chunks = rc.projects[0].chunks('01')
self.assertEqual(len(chunks), 5)
def unzip_archive(self):
App.logger.debug("Unzipping {0} to {1}".format(self.source_zip_file, self.temp_dir))
unzip(self.source_zip_file, self.temp_dir)
dirs = [d for d in os.listdir(self.temp_dir) if os.path.isdir(os.path.join(self.temp_dir, d))]
if len(dirs):
self.source_dir = os.path.join(self.temp_dir, dirs[0])
else:
self.source_dir = self.temp_dir
def unzip_converted_files(self, converted_zip_file):
unzip_dir = tempfile.mkdtemp(prefix='unzip_', dir=self.temp_dir)
try:
App.logger.debug('Unzipping {0}...'.format(converted_zip_file))
unzip(converted_zip_file, unzip_dir)
finally:
App.logger.debug('finished.')
return unzip_dir
def test_lint_warnings(self, mock_invoke):
obs_zip_file = os.path.join(self.resources_dir, 'obs_linter', 'en-obs.zip')
unzip(obs_zip_file, self.temp_dir)
source_dir = os.path.join(self.temp_dir, 'en-obs')
mock_invoke.return_value = {}
expected_warnings = True
linter = ObsLinter(source_dir=source_dir)
linter.run()
self.verify_results(expected_warnings, linter)
def mock_s3_obs_project(self):
zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en-obs-complete.zip')
out_dir = os.path.join(self.temp_dir, 'en-obs-complete')
unzip(zip_file, out_dir)
project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678')
self.project_files = [f for f in os.listdir(project_dir) if os.path.isfile(os.path.join(project_dir, f))]
self.project_key = 'door43/en-obs/12345678'
for filename in self.project_files:
App.cdn_s3_handler().upload_file(os.path.join(project_dir, filename), 'u/{0}/{1}'.format(self.project_key, filename))
App.cdn_s3_handler().upload_file(os.path.join(out_dir, 'door43', 'en-obs', 'project.json'), 'u/door43/en-obs/project.json')
def extractFiles(cls, file_name, repo_name):
file_path = os.path.join(TestBiblePreprocessor.resources_dir, file_name)
# 1) unzip the repo files
temp_dir = tempfile.mkdtemp(prefix='repo_')
unzip(file_path, temp_dir)
repo_dir = os.path.join(temp_dir, repo_name)
if not os.path.isdir(repo_dir):
repo_dir = temp_dir
# 2) Get the resource container
rc = RC(repo_dir)
return rc, repo_dir, temp_dir
def test_unzip(self):
self.tmp_dir = tempfile.mkdtemp()
zip_file = os.path.join(self.tmp_dir, 'foo.zip')
_, self.tmp_file = tempfile.mkstemp()
with open(self.tmp_file, "w") as tmpf:
tmpf.write("hello world")
with zipfile.ZipFile(zip_file, "w") as zf:
zf.write(self.tmp_file, os.path.basename(self.tmp_file))
file_utils.unzip(zip_file, self.tmp_dir)
with open(os.path.join(self.tmp_dir, os.path.basename(self.tmp_file))) as outf:
self.assertEqual(outf.read(), "hello world")
def mock_s3_tw_project(self):
zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tw_converted.zip')
out_dir = os.path.join(self.temp_dir, 'en_tw_converted')
unzip(zip_file, out_dir)
self.src_dir = src_dir = os.path.join(out_dir, 'en_tw_converted')
self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))]
self.project_key = 'u/door43/en_tw/12345678'
for filename in self.project_files:
App.cdn_s3_handler().upload_file(os.path.join(src_dir, filename), '{0}/{1}'.format(self.project_key,
filename))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'),
'u/door43/en_tw/project.json')
App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def test_en_obs_manifest_yaml(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs-manifest-yaml.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en_obs')
rc = RC(directory=repo_dir, repo_name='en_obs')
rc_dic = rc.as_dict()
yaml = load_yaml_object(os.path.join(repo_dir, 'manifest.yaml'))
self.assertDictEqual(yaml, rc_dic)
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 2)
chunks = rc.project().chunks('front')
self.assertEqual(chunks, ['intro.md', 'title.md'])
def test_ta(self):
"""
Runs the converter and verifies the output
"""
file_name = 'en_ta.zip'
self.doTransformTa(file_name)
self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='ta_')
unzip(self.out_zip_file, self.out_dir)
remove(self.out_zip_file)
files_to_verify = ['checking.html', 'checking-toc.yaml', 'intro.html', 'intro-toc.yaml',
'process.html', 'process-toc.yaml', 'translate.html', 'translate-toc.yaml']
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'
.format(file_to_verify))
def test_matt_complete_with_backslash(self):
"""
Runs the converter and verifies the output
"""
zip_file = os.path.join(self.resources_dir, 'kpb_mat_text_udb.zip')
zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir)
with closing(Usfm2HtmlConverter('', 'udb', out_zip_file)) as tx:
tx.input_zip_file = zip_file
results = tx.run()
# verify the output
self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(results)
self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir)
unzip(out_zip_file, self.out_dir)
files_to_verify = ['41-MAT.html']
self.verify_files(files_to_verify)
def test_bible_no_manifest(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'bible-no-manifest.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en_ulb')
rc = RC(directory=repo_dir)
rc.as_dict()
self.assertEqual(rc.resource.identifier, 'ulb')
self.assertEqual(rc.resource.type, 'bundle')
self.assertEqual(rc.resource.format, 'text/usfm')
self.assertEqual(rc.resource.file_ext, 'usfm')
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.project().chapters()
self.assertEqual(len(chapters), 0)
self.assertEqual(len(rc.project().usfm_files()), 8)
def test_php_illegal_url(self):
"""
Runs the converter and verifies the output
"""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, '51-PHP.zip')
zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir)
with closing(Usfm2HtmlConverter(' ', 'udb', out_zip_file)) as tx:
tx.input_zip_file = zip_file
results = tx.run()
# verify the output
self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(results)
self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir)
unzip(out_zip_file, self.out_dir)
files_to_verify = ['51-PHP.html']
self.verify_files(files_to_verify)
def test_bible_from_tx_pre_rc(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'id_mat_text_ulb-ts.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'id_mat_text_ulb-ts')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'manifest.json'))
self.assertEqual(rc.resource.identifier, json['resource']['id'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, 'text/{0}'.format(json['format']))
self.assertEqual(rc.resource.file_ext, json['format'])
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.modified, datetime.utcnow().strftime("%Y-%m-%d"))
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 29)
chunks = rc.projects[0].chunks('01')
self.assertEqual(len(chunks), 11)
def test_en_obs_package_json(self):
""" Populates the ResourceContainer object and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs-package-json.zip')
self.out_dir = tempfile.mkdtemp(prefix='repo_')
unzip(zip_file, self.out_dir)
repo_dir = os.path.join(self.out_dir, 'en-obs')
rc = RC(directory=repo_dir)
rc.as_dict()
json = load_json_object(os.path.join(repo_dir, 'package.json'))
self.assertEqual(rc.resource.identifier, json['resource']['slug'])
self.assertEqual(rc.resource.type, 'book')
self.assertEqual(rc.resource.format, json['content_mime_type'])
self.assertEqual(rc.resource.file_ext, 'md')
self.assertEqual(rc.resource.conformsto, 'pre-rc')
self.assertEqual(rc.resource.issued, json['resource']['status']['pub_date'])
chapters = rc.projects[0].chapters()
self.assertEqual(len(chapters), 2)
chunks = rc.project().chunks('_back')
self.assertEqual(chunks, ['back-matter.md'])
def test_ta(self):
"""
Runs the converter and verifies the output
"""
file_name = 'en_ta.zip'
self.doTransformTa(file_name)
self.assertTrue(os.path.isfile(self.out_zip_file),
"There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='ta_')
unzip(self.out_zip_file, self.out_dir)
remove(self.out_zip_file)
files_to_verify = [
'checking.html', 'checking-toc.yaml', 'intro.html',
'intro-toc.yaml', 'process.html', 'process-toc.yaml',
'translate.html', 'translate-toc.yaml'
]
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_path),
'file not found: {0}'.format(file_to_verify))
def mock_s3_obs_project(self):
zip_file = os.path.join(self.resources_dir, 'converted_projects',
'en-obs-complete.zip')
out_dir = os.path.join(self.temp_dir, 'en-obs-complete')
unzip(zip_file, out_dir)
project_dir = os.path.join(out_dir, 'door43', 'en-obs', '12345678')
self.project_files = [
f for f in os.listdir(project_dir)
if os.path.isfile(os.path.join(project_dir, f))
]
self.project_key = 'u/door43/en-obs/12345678'
for filename in self.project_files:
App.cdn_s3_handler().upload_file(
os.path.join(project_dir, filename),
'{0}/{1}'.format(self.project_key, filename))
App.cdn_s3_handler().upload_file(
os.path.join(out_dir, 'door43', 'en-obs', 'project.json'),
'u/door43/en-obs/project.json')
App.door43_s3_handler().upload_file(
os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def test_run(self):
"""Runs the converter and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs.zip')
zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
out_zip_file = tempfile.mktemp(prefix="en-obs", suffix=".zip")
with closing(Md2HtmlConverter('', 'obs', out_zip_file)) as tx:
tx.input_zip_file = zip_file
tx.run()
# verify the output
self.assertTrue(os.path.isfile(out_zip_file), "There was no output zip file produced.")
self.out_dir = tempfile.mkdtemp(prefix='obs_')
unzip(out_zip_file, self.out_dir)
remove(out_zip_file)
files_to_verify = []
for i in range(1, 51):
files_to_verify.append(str(i).zfill(2) + '.html')
for file_to_verify in files_to_verify:
file_name = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_name), 'OBS HTML file not found: {0}'.format(file_name))
def mock_s3_tn_project(self, part):
zip_file = os.path.join(self.resources_dir, 'converted_projects', 'en_tn_converted.zip')
out_dir = os.path.join(self.temp_dir, 'en_tn_converted')
unzip(zip_file, out_dir)
src_dir = os.path.join(out_dir, 'en_tn_converted')
self.project_files = [f for f in os.listdir(src_dir) if os.path.isfile(os.path.join(src_dir, f))]
self.project_key = 'u/door43/en_tn/12345678'
build_log = file_utils.load_json_object(os.path.join(src_dir, 'build_log.json'))
build_log['part'] = part
file_utils.write_file(os.path.join(src_dir, 'build_log.json'), build_log)
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'),
'{0}/{1}/build_log.json'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'index.json'),
'{0}/{1}/index.json'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'build_log.json'),
'{0}/{1}/finished'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, '01-GEN.html'),
'{0}/{1}/01-GEN.html'.format(self.project_key, part))
App.cdn_s3_handler().upload_file(os.path.join(src_dir, 'project.json'),
'u/door43/en_tq/project.json')
App.door43_s3_handler().upload_file(os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def test_tw(self):
"""
Runs the converter and verifies the output
"""
# given
file_name = 'en_tw.zip'
# when
self.doTransformTw(file_name)
# then
self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='tw_')
unzip(self.out_zip_file, self.out_dir)
remove(self.out_zip_file)
files_to_verify = ['index.json', 'kt.html', 'names.html', 'other.html', 'config.yaml', 'manifest.yaml']
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_path), 'file not found: {0}'
.format(file_to_verify))
def mock_s3_bible_project(self,
test_file_name,
project_key,
multi_part=False):
converted_proj_dir = os.path.join(self.resources_dir,
'converted_projects')
test_file_base = test_file_name.split('.zip')[0]
zip_file = os.path.join(converted_proj_dir, test_file_name)
out_dir = os.path.join(self.temp_dir, test_file_base)
unzip(zip_file, out_dir)
project_dir = os.path.join(out_dir, test_file_base) + os.path.sep
self.project_files = file_utils.get_files(out_dir)
self.project_key = project_key
for filename in self.project_files:
sub_path = filename.split(project_dir)[1].replace(
os.path.sep, '/') # Make sure it is a bucket path
App.cdn_s3_handler().upload_file(
filename, '{0}/{1}'.format(project_key, sub_path))
if multi_part: # copy files from cdn to door43
base_name = os.path.basename(filename)
if '.html' in base_name:
with codecs.open(filename, 'r', 'utf-8-sig') as f:
soup = BeautifulSoup(f, 'html.parser')
# add nav tag
new_tag = soup.new_tag('div', id='right-sidebar')
soup.body.append(new_tag)
html = unicode(soup)
file_utils.write_file(
filename, html.encode('ascii', 'xmlcharrefreplace'))
App.door43_s3_handler().upload_file(
filename, '{0}/{1}'.format(project_key, base_name))
# u, user, repo = project_key
App.door43_s3_handler().upload_file(
os.path.join(self.resources_dir, 'templates', 'project-page.html'),
'templates/project-page.html')
def test_run(self):
"""Runs the converter and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'eight_bible_books.zip')
zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
out_zip_file = tempfile.mktemp('.zip', dir=self.temp_dir)
with closing(Usfm2HtmlConverter('', 'udb', out_zip_file)) as tx:
tx.input_zip_file = zip_file
results = tx.run()
# verify the output
self.assertTrue(os.path.isfile(out_zip_file),
"There was no output zip file produced.")
self.assertIsNotNone(results)
self.out_dir = tempfile.mkdtemp(prefix='udb_', dir=self.temp_dir)
unzip(out_zip_file, self.out_dir)
files_to_verify = [
'60-JAS.html', '61-1PE.html', '62-2PE.html', '63-1JN.html',
'64-2JN.html', '65-3JN.html', '66-JUD.html', '67-REV.html'
]
for file_to_verify in files_to_verify:
file_name = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_name),
'UDB HTML file not found: {0}'.format(file_name))
def test_run(self):
"""Runs the converter and verifies the output."""
# test with the English OBS
zip_file = os.path.join(self.resources_dir, 'en-obs.zip')
zip_file = self.make_duplicate_zip_that_can_be_deleted(zip_file)
out_zip_file = tempfile.mktemp(prefix="en-obs", suffix=".zip")
with closing(Md2HtmlConverter('', 'obs', out_zip_file)) as tx:
tx.input_zip_file = zip_file
tx.run()
# verify the output
self.assertTrue(os.path.isfile(out_zip_file),
"There was no output zip file produced.")
self.out_dir = tempfile.mkdtemp(prefix='obs_')
unzip(out_zip_file, self.out_dir)
remove(out_zip_file)
files_to_verify = []
for i in range(1, 51):
files_to_verify.append(str(i).zfill(2) + '.html')
for file_to_verify in files_to_verify:
file_name = os.path.join(self.out_dir, file_to_verify)
self.assertTrue(os.path.isfile(file_name),
'OBS HTML file not found: {0}'.format(file_name))
def verifyTransform(self, tx, missing_chapters=None):
if not missing_chapters:
missing_chapters = []
self.assertTrue(os.path.isfile(self.out_zip_file), "There was no output zip file produced.")
self.assertIsNotNone(self.return_val, "There was no return value.")
self.out_dir = tempfile.mkdtemp(prefix='obs_')
unzip(self.out_zip_file, self.out_dir)
remove(self.out_zip_file)
files_to_verify = []
files_missing = []
for i in range(1, 51):
file_name = str(i).zfill(2) + '.html'
if not i in missing_chapters:
files_to_verify.append(file_name)
else:
files_missing.append(file_name)
for file_to_verify in files_to_verify:
file_path = os.path.join(self.out_dir, file_to_verify)
contents = self.getContents(file_path)
self.assertIsNotNone(contents, 'OBS HTML body contents not found: {0}'.format(os.path.basename(file_path)))
for file_to_verify in files_missing:
file_path = os.path.join(self.out_dir, file_to_verify)
contents = self.getContents(file_path)
self.assertIsNone(contents, 'OBS HTML body contents present, but should not be: {0}'.format(os.path.basename(file_path)))
self.assertEqual(self.return_val['success'], self.expected_success, "Mismatch in for success boolean")
self.assertEqual(len(self.return_val['info']) == 0, self.expected_info_empty, "Mismatch in expected info empty")
for warning in self.return_val['warnings']:
App.logger.debug("Warning: " + warning)
for error in self.return_val['errors']:
App.logger.debug("Error: " + error)
self.assertEqual(len(self.return_val['warnings']), self.expected_warnings, "Mismatch in expected warnings")
self.assertEqual(len(self.return_val['errors']), self.expected_errors, "Mismatch in expected errors")
def unzip_resource(self, zip_name):
zip_file = os.path.join(self.resources_dir, zip_name)
out_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix='linter_test_')
unzip(zip_file, out_dir)
return out_dir
def setUp(self):
"""Runs before each test."""
self.temp_dir = tempfile.mkdtemp(prefix='tmp_obs_')
unzip(self.obs_zip_file, self.temp_dir)
self.repo_dir = os.path.join(self.temp_dir, 'es_obs')
def test_lint_no_warnings(self, mock_invoke):
unzip(self.obs_zip_file, self.temp_dir)
mock_invoke.return_value = {}
expected_warnings = False
linter = self.run_linter()
self.verify_results(expected_warnings, linter)
def unzip_resource(self, zip_name):
zip_file = os.path.join(self.resources_dir, zip_name)
out_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix='linter_test_')
unzip(zip_file, out_dir)
return out_dir
def extractZipFiles(self, test_folder_name):
file_path = os.path.join(self.resources_dir, test_folder_name)
self.temp_dir = tempfile.mkdtemp(prefix='repo_')
unzip(file_path, self.temp_dir)
return self.temp_dir
def test_lint_no_warnings(self, mock_invoke):
unzip(self.obs_zip_file, self.temp_dir)
mock_invoke.return_value = {}
expected_warnings = False
linter = self.run_linter()
self.verify_results(expected_warnings, linter)
def setUp(self):
"""Runs before each test."""
self.temp_dir = tempfile.mkdtemp(prefix='tmp_obs_')
unzip(self.obs_zip_file, self.temp_dir)
self.repo_dir = os.path.join(self.temp_dir, 'es_obs')