def get_chunk_html(self, resource, chapter, verse):
# print("html: {0}-{3}-{1}-{2}".format(resource, chapter, verse, self.book_id))
path = os.path.join(self.working_dir, 'usfm_chunks', 'usfm-{0}-{1}-{2}-{3}-{4}'.
format(self.lang_code, resource, self.book_id, chapter, verse))
filename_base = '{0}-{1}-{2}-{3}'.format(resource, self.book_id, chapter, verse)
html_file = os.path.join(path, '{0}.html'.format(filename_base))
usfm_file = os.path.join(path, '{0}.usfm'.format(filename_base))
if os.path.isfile(html_file):
return read_file(html_file)
if not os.path.exists(path):
os.makedirs(path)
chunk = self.usfm_chunks[resource][str(chapter)][str(verse)]['usfm']
usfm = self.usfm_chunks[resource]['header']
if '\\c' not in chunk:
usfm += '\n\n\\c {0}\n'.format(chapter)
usfm += chunk
write_file(usfm_file, usfm)
UsfmTransform.buildSingleHtml(path, path, filename_base)
html = read_file(os.path.join(path, filename_base+'.html'))
soup = BeautifulSoup(html, 'html.parser')
header = soup.find('h1')
if header:
header.decompose()
chapter = soup.find('h2')
if chapter:
chapter.decompose()
html = ''.join(['%s' % x for x in soup.body.contents])
write_file(html_file, html)
return html
def get_chunk_html(self, resource, chapter, verse):
# print("html: {0}-{3}-{1}-{2}".format(resource, chapter, verse, self.book_id))
path = os.path.join(
self.working_dir, 'usfm_chunks',
'usfm-{0}-{1}-{2}-{3}-{4}'.format(self.lang_code, resource,
self.book_id, chapter, verse))
filename_base = '{0}-{1}-{2}-{3}'.format(resource, self.book_id,
chapter, verse)
html_file = os.path.join(path, '{0}.html'.format(filename_base))
usfm_file = os.path.join(path, '{0}.usfm'.format(filename_base))
if os.path.isfile(html_file):
return read_file(html_file)
if not os.path.exists(path):
os.makedirs(path)
chunk = self.usfm_chunks[resource][str(chapter)][str(verse)]['usfm']
usfm = self.usfm_chunks[resource]['header']
if '\\c' not in chunk:
usfm += '\n\n\\c {0}\n'.format(chapter)
usfm += chunk
write_file(usfm_file, usfm)
UsfmTransform.buildSingleHtml(path, path, filename_base)
html = read_file(os.path.join(path, filename_base + '.html'))
soup = BeautifulSoup(html, 'html.parser')
header = soup.find('h1')
if header:
header.decompose()
chapter = soup.find('h2')
if chapter:
chapter.decompose()
html = ''.join(['%s' % x for x in soup.body.contents])
write_file(html_file, html)
return html
def build_usx(usfm_dir, usx_dir):
"""
Builds the usx from usfm after performing some custom processing
:param usfm_dir:
:param usx_dir:
:return:
"""
# strip word data
files = os.listdir(usfm_dir)
usfm2_dir = tempfile.mkdtemp(prefix='usfm2')
try:
for name in files:
if name == '.DS_Store':
continue
f = os.path.join(usfm_dir, name)
usfm3 = read_file(f)
usfm2 = usfm3_to_usfm2(usfm3)
out_f = os.path.join(usfm2_dir, name)
write_file(out_f, usfm2)
UsfmTransform.buildUSX(usfm2_dir, usx_dir, '', True)
finally:
try:
shutil.rmtree(usfm2_dir)
finally:
pass
def get_chunk_html(self, resource, chapter, verse):
# print("html: {0}-{3}-{1}-{2}".format(resource, chapter, verse, self.book_id))
path = tempfile.mkdtemp(dir=self.working_dir, prefix='usfm-{0}-{1}-{2}-{3}-{4}_'.
format(self.lang_code, resource, self.book_id, chapter, verse))
filename_base = '{0}-{1}-{2}-{3}'.format(resource, self.book_id, chapter, verse)
try:
chunk = self.usfm_chunks[resource][chapter][verse]['usfm']
except KeyError:
chunk = u''
usfm = self.usfm_chunks[resource]['header']
if '\\c' not in chunk:
usfm += '\n\n\\c {0}\n'.format(chapter)
usfm += chunk
write_file(os.path.join(path, filename_base+'.usfm'), usfm)
UsfmTransform.buildSingleHtml(path, path, filename_base)
html = read_file(os.path.join(path, filename_base+'.html'))
shutil.rmtree(path, ignore_errors=True)
soup = BeautifulSoup(html, 'html.parser')
header = soup.find('h1')
if header:
header.decompose()
chapter = soup.find('h2')
if chapter:
chapter.decompose()
html = ''.join(['%s' % x for x in soup.body.contents])
return html
def main(langcode, ver, books, format, outfile):
sys.stdout = codecs.getwriter('utf8')(sys.stdout);
# Get the JSON
catalog = json.load(urllib2.urlopen(CatalogJSON))
bible=None
for item in catalog['cat']:
if item['slug'] == 'bible':
bible = item
break
lang=None
for language in bible['langs']:
if language['lc'] == langcode:
lang=language
break
if lang is None:
print "The language code {0} is not found in the catalog at {1}. Exiting...".format(langcode, CatalogJSON)
sys.exit(1)
bible=None
for version in lang['vers']:
if version['slug'] == ver:
bible=version
break
if bible is None:
print "The Bible version {0} for language {1} is not found in the catalog at {2}. Exiting...".format(ver, langcode, CatalogJSON)
sys.exit(1)
sources = []
for source in bible['toc']:
if books is None or source['slug'] in books:
sources += [source['src']]
if not sources:
print "No sources were found for langage {0} of version {1} in {2}. Exiting...".format(langcode, ver, CatalogJSON)
sys.exit(1)
tmpdir = tempfile.mkdtemp(prefix='uwb-{0}-{1}-'.format(ver, langcode))
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
os.makedirs(tmpdir+"/sources")
for source in sources:
f = urllib2.urlopen(source)
with open(tmpdir+"/sources/"+os.path.basename(source), "wb") as local_file:
local_file.write(f.read())
if format == 'html':
UsfmTransform.buildSingleHtml(tmpdir+"/sources", tmpdir, "bible")
shutil.copyfile(tmpdir+'/bible.html', outfile);
if format == 'tex':
UsfmTransform.buildConTeXt(tmpdir+"/sources", tmpdir, "bible")
shutil.copyfile(tmpdir+'/working/tex/bible.tex', outfile);
def test_quote_indents_of_large_book(self):
usfm_dir = os.path.join(self.resources_dir, 'usfm_projects', 'numbers')
html_dir = os.path.join(self.temp_dir, 'quote_indents_of_large_book')
os.mkdir(html_dir)
UsfmTransform.buildSingleHtml(usfm_dir, html_dir, 'bible')
html_file = os.path.join(html_dir, 'bible.html')
self.assertTrue(os.path.exists(html_file))
with codecs.open(html_file, 'r', 'utf-8-sig') as f:
converted_html = f.read()
soup = BeautifulSoup(converted_html, 'html.parser')
indent_count = len(soup.select("p[class^=indent-]"))
self.assertEqual(indent_count, 190)
def test_quote_indents_of_large_book(self):
usfm_dir = os.path.join(self.resources_dir, 'usfm_projects', 'numbers')
html_dir = os.path.join(self.temp_dir, 'quote_indents_of_large_book')
os.mkdir(html_dir)
UsfmTransform.buildSingleHtml(usfm_dir, html_dir, 'bible')
html_file = os.path.join(html_dir, 'bible.html')
self.assertTrue(os.path.exists(html_file))
with codecs.open(html_file, 'r', 'utf-8-sig') as f:
converted_html = f.read()
soup = BeautifulSoup(converted_html, 'html.parser')
indent_count = len(soup.select("p[class^=indent-]"))
self.assertEqual(indent_count, 190)
def test_quote_indents_of_large_book_with_illegal_usfm(self):
usfm_dir = os.path.join(self.resources_dir, 'usfm_projects', 'matthew')
html_dir = os.path.join(self.temp_dir, 'quote_indents_of_large_book')
os.mkdir(html_dir)
UsfmTransform.buildSingleHtml(usfm_dir, html_dir, 'bible')
html_file = os.path.join(html_dir, 'bible.html')
self.assertTrue(os.path.exists(html_file))
with codecs.open(html_file, 'r', 'utf-8-sig') as f:
converted_html = f.read()
soup = BeautifulSoup(converted_html, 'html.parser')
verse_count = len(soup.select("span[class^=v-num]"))
self.assertEqual(verse_count, 315)
nbsp = converted_html.find('\xa0')
self.assertTrue(nbsp < 0, "'\\xa0' should not be in text")
def test_quote_indents_of_large_book_with_illegal_usfm(self):
usfm_dir = os.path.join(self.resources_dir, 'usfm_projects', 'matthew')
html_dir = os.path.join(self.temp_dir, 'quote_indents_of_large_book')
os.mkdir(html_dir)
UsfmTransform.buildSingleHtml(usfm_dir, html_dir, 'bible')
html_file = os.path.join(html_dir, 'bible.html')
self.assertTrue(os.path.exists(html_file))
with codecs.open(html_file, 'r', 'utf-8-sig') as f:
converted_html = f.read()
soup = BeautifulSoup(converted_html, 'html.parser')
verse_count = len(soup.select("span[class^=v-num]"))
self.assertEqual(verse_count, 315)
nbsp = converted_html.find('\xa0')
self.assertTrue(nbsp < 0,"'\\xa0' should not be in text")
def build_usx(usfm_dir, usx_dir):
"""
Builds the usx from usfm after performing some custom processing
:param usfm_dir:
:param usx_dir:
:return:
"""
# strip word data
files = os.listdir(usfm_dir)
for name in files:
f = os.path.join(usfm_dir, name)
usfm = read_file(f)
write_file(f, convert_chunk_markers(strip_word_data(usfm)))
UsfmTransform.buildUSX(usfm_dir, usx_dir, '', True)
def build_usx(usfm_dir, usx_dir, logger=None):
"""
Builds the usx from usfm after performing some custom processing
:param usfm_dir:
:param usx_dir:
:return:
"""
# strip word data
files = os.listdir(usfm_dir)
for name in files:
f = os.path.join(usfm_dir, name)
usfm = read_file(f)
write_file(f, remove_unknown_markers(convert_chunk_markers(strip_word_data(usfm))))
if logger:
logger.debug("Actual USX conversion into {}".format(usx_dir))
UsfmTransform.buildUSX(usfm_dir, usx_dir, '', True)
def main(source, lang_code, resource_id, books, outfile):
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
tmpdir = tempfile.mkdtemp(prefix='uwb-{0}-{1}-'.format(resource_id, lang_code))
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
usfm_dir = tmpdir + "/usfm"
if not books:
shutil.copytree(source, usfm_dir)
else:
os.makedirs(usfm_dir)
for book in books:
source_file = '{0}/{1}-{2}.usfm'.format(source, BOOK_NUMBERS[book.lower()], book.upper())
if not os.path.isfile(source_file):
raise IOError('File not found: {}'.format(source_file))
shutil.copy(source_file, usfm_dir)
UsfmTransform.buildSingleHtml(usfm_dir, tmpdir, "bible")
shutil.copyfile(tmpdir+'/bible.html', outfile)
def run(self):
today = ''.join(str(datetime.date.today()).rsplit('-')[0:3])
dirs = []
if self.source:
dirs.append(self.source)
else:
for source_dir in api_publish.source_dirs:
udb_dir = [
os.path.join(source_dir, x) for x in os.listdir(source_dir)
]
dirs += udb_dir
for d in dirs:
ver, lang = d.rsplit('/', 1)[1].split('-', 1)
self.temp_dir = '/tmp/{0}-{1}'.format(ver, lang)
if os.path.isdir(self.temp_dir):
shutil.rmtree(self.temp_dir)
UsfmTransform.buildUSX(d, self.temp_dir, '', True)
print("#### Chunking...")
for f in os.listdir(self.temp_dir):
# use utf-8-sig to remove the byte order mark
with codecs.open(os.path.join(self.temp_dir, f),
'r',
encoding='utf-8-sig') as in_file:
usx = in_file.readlines()
slug = f.split('.')[0].lower()
print(' ({0})'.format(slug.upper()))
book = self.parse(usx)
payload = {'chapters': book, 'date_modified': today}
write_file(
os.path.join(api_publish.api_v2, slug, lang, ver,
'source.json'), payload)
chunks = self.get_chunks(book)
write_file(
os.path.join(api_publish.api_v2, slug, lang, ver,
'chunks.json'), chunks)
def get_bible_body_html(self):
usfm_conversion_path = os.path.join(
self.working_dir,
'{0}_{1}_usfm_conversion'.format(self.lang_code, self.resource_id),
'{0}'.format(self.book_id))
filename_base = '{0}-{1}'.format(self.book_number,
self.book_id.upper())
html_file = os.path.join(usfm_conversion_path,
'{0}.html'.format(filename_base))
if not os.path.exists(usfm_conversion_path):
os.makedirs(usfm_conversion_path)
if True or not os.path.exists(html_file):
usfm_files = glob(os.path.join(self.bible_dir, '*.usfm'))
for usfm_file in usfm_files:
usfm_file_base = os.path.basename(usfm_file)
if filename_base in usfm_file_base or (
self.book_id == 'nt'
and int(usfm_file_base.split('-')[0]) > 40):
usfm3 = read_file(usfm_file)
usfm2 = usfm3_to_usfm2(usfm3)
write_file(
os.path.join(usfm_conversion_path, usfm_file_base),
usfm2)
UsfmTransform.buildSingleHtml(usfm_conversion_path,
usfm_conversion_path, filename_base)
html = read_file(html_file)
soup = BeautifulSoup(html, 'html.parser')
html = unicode(soup.find('body'))
html = re.sub(r' +(<span id="ref-fn-)',
r'\1',
html,
flags=re.MULTILINE)
html = re.sub(r'(</b></sup></span>) +',
r'\1',
html,
flags=re.MULTILINE)
html = re.sub(r' +(</i>)', r'\1', html, flags=re.MULTILINE)
return html
def main(source, lang_code, resource_id, books, outfile):
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
tmpdir = tempfile.mkdtemp(
prefix='uwb-{0}-{1}-'.format(resource_id, lang_code))
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
usfm_dir = tmpdir + "/usfm"
if not books:
shutil.copytree(source, usfm_dir)
else:
os.makedirs(usfm_dir)
for book in books:
source_file = '{0}/{1}-{2}.usfm'.format(source,
BOOK_NUMBERS[book.lower()],
book.upper())
if not os.path.isfile(source_file):
raise IOError('File not found: {}'.format(source_file))
shutil.copy(source_file, usfm_dir)
UsfmTransform.buildSingleHtml(usfm_dir, tmpdir, "bible")
shutil.copyfile(tmpdir + '/bible.html', outfile)
def get_bible_body_html(self):
usfm_conversion_path = os.path.join(self.working_dir, '{0}_{1}_usfm_conversion'.format(self.lang_code, self.resource_id), '{0}'.
format(self.book_id))
filename_base = '{0}-{1}'.format(self.book_number, self.book_id.upper())
html_file = os.path.join(usfm_conversion_path, '{0}.html'.format(filename_base))
if not os.path.exists(usfm_conversion_path):
os.makedirs(usfm_conversion_path)
if True or not os.path.exists(html_file):
usfm_files = glob(os.path.join(self.bible_dir, '*.usfm'))
for usfm_file in usfm_files:
usfm_file_base = os.path.basename(usfm_file)
if filename_base in usfm_file_base or (self.book_id == 'nt' and int(usfm_file_base.split('-')[0]) > 40):
usfm3 = read_file(usfm_file)
usfm2 = usfm3_to_usfm2(usfm3)
write_file(os.path.join(usfm_conversion_path, usfm_file_base), usfm2)
UsfmTransform.buildSingleHtml(usfm_conversion_path, usfm_conversion_path, filename_base)
html = read_file(html_file)
soup = BeautifulSoup(html, 'html.parser')
html = unicode(soup.find('body'))
html = re.sub(r' +(<span id="ref-fn-)', r'\1', html, flags=re.MULTILINE)
html = re.sub(r'(</b></sup></span>) +', r'\1', html, flags=re.MULTILINE)
html = re.sub(r' +(</i>)', r'\1', html, flags=re.MULTILINE)
return html
def main(langcode, ver, books, format, outfile):
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
# Get the JSON
catalog = json.load(urllib2.urlopen(CatalogJSON))
bible = None
for item in catalog['cat']:
if item['slug'] == 'bible':
bible = item
break
lang = None
for language in bible['langs']:
if language['lc'] == langcode:
lang = language
break
if lang is None:
print "The language code {0} is not found in the catalog at {1}. Exiting...".format(
langcode, CatalogJSON)
sys.exit(1)
bible = None
for version in lang['vers']:
if version['slug'] == ver:
bible = version
break
if bible is None:
print "The Bible version {0} for language {1} is not found in the catalog at {2}. Exiting...".format(
ver, langcode, CatalogJSON)
sys.exit(1)
sources = []
for source in bible['toc']:
if books is None or source['slug'] in books:
sources += [source['src']]
if not sources:
print "No sources were found for langage {0} of version {1} in {2}. Exiting...".format(
langcode, ver, CatalogJSON)
sys.exit(1)
tmpdir = tempfile.mkdtemp(prefix='uwb-{0}-{1}-'.format(ver, langcode))
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
os.makedirs(tmpdir + "/sources")
for source in sources:
f = urllib2.urlopen(source)
with open(tmpdir + "/sources/" + os.path.basename(source),
"wb") as local_file:
local_file.write(f.read())
if format == 'html':
UsfmTransform.buildSingleHtml(tmpdir + "/sources", tmpdir, "bible")
shutil.copyfile(tmpdir + '/bible.html', outfile)
if format == 'tex':
UsfmTransform.buildConTeXt(tmpdir + "/sources", tmpdir, "bible")
shutil.copyfile(tmpdir + '/working/tex/bible.tex', outfile)
def convert(self):
App.logger.debug('Processing the Bible USFM files')
# find the first directory that has usfm files.
files = get_files(directory=self.files_dir,
exclude=self.EXCLUDED_FILES)
convert_only_list = self.check_for_exclusive_convert()
current_dir = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(current_dir, 'templates',
'template.html')) as template_file:
template_html = template_file.read()
for filename in files:
if filename.endswith('.usfm'):
base_name = os.path.basename(filename)
if convert_only_list and (
base_name not in convert_only_list
): # see if this is a file we are to convert
continue
msg = 'Converting Bible USFM file: {0}'.format(base_name)
self.log.info(msg)
App.logger.debug(msg)
# Covert the USFM file
scratch_dir = tempfile.mkdtemp(prefix='scratch_')
copyfile(filename,
os.path.join(scratch_dir, os.path.basename(filename)))
filebase = os.path.splitext(os.path.basename(filename))[0]
UsfmTransform.buildSingleHtml(scratch_dir, scratch_dir,
filebase)
html_filename = filebase + ".html"
with codecs.open(os.path.join(scratch_dir, html_filename), 'r',
'utf-8-sig') as html_file:
converted_html = html_file.read()
template_soup = BeautifulSoup(template_html, 'html.parser')
template_soup.head.title.string = self.resource.upper()
converted_soup = BeautifulSoup(converted_html, 'html.parser')
content_div = template_soup.find('div', id='content')
content_div.clear()
if converted_soup and converted_soup.body:
content_div.append(converted_soup.body)
content_div.body.unwrap()
else:
content_div.append(
'<div class="error">ERROR! NOT CONVERTED!</div>')
output_file = os.path.join(self.output_dir, html_filename)
write_file(output_file, unicode(template_soup))
self.log.info('Converted {0} to {1}.'.format(
os.path.basename(filename),
os.path.basename(html_filename)))
remove_tree(scratch_dir)
else:
# Directly copy over files that are not USFM files
try:
output_file = os.path.join(self.output_dir,
os.path.basename(filename))
if not os.path.exists(output_file):
copyfile(filename, output_file)
except:
pass
self.log.info('Finished processing Bible USFM files.')
return True
def asset_content(
resource_lookup_dto: model.ResourceLookupDto,
resource_dir: str,
output_dir: str = settings.output_dir(),
) -> str:
"""
Gather and parse USFM content into HTML content and return the
HTML content.
"""
usfm_content_files: list[str] = []
txt_content_files: list[str] = []
# We don't need a manifest file to find resource assets
# on disk. We just use globbing and then filter
# down the list found to only include those
# files that match the resource code, i.e., book, being requested.
# This frees us from some of the brittleness of using manifests
# to find files. Some resources do not provide a manifest
# anyway.
usfm_content_files = glob("{}**/*.usfm".format(resource_dir))
if not usfm_content_files:
# USFM files sometimes have txt suffix instead of usfm
txt_content_files = glob("{}**/*.txt".format(resource_dir))
# Sometimes the txt USFM files live at another location
if not txt_content_files:
txt_content_files = glob("{}**/**/*.txt".format(resource_dir))
# If desired, in the case where a manifest must be consulted
# to determine if the file is considered usable, i.e.,
# 'complete' or 'finished', that can also be done by comparing
# the filtered file(s) against the manifest's 'finished' list
# to see if it can be used. Such logic could live
# approximately here if desired.
content_files: list[str] = []
if usfm_content_files:
# Only use the content files that match the resource_code
# in the resource request.
content_files = [
usfm_content_file for usfm_content_file in usfm_content_files
if resource_lookup_dto.resource_code.lower() in str(
usfm_content_file).lower()
]
elif txt_content_files:
# Only use the content files that match the resource_code.
content_files = [
txt_content_file for txt_content_file in txt_content_files
if resource_lookup_dto.resource_code.lower() in str(
txt_content_file).lower()
]
html_content = ""
if content_files:
# Some languages, like ndh-x-chindali, provide their USFM files in
# a git repo rather than as standalone USFM files. A USFM git repo can
# have each USFM chapter in a separate directory and each verse in a
# separate file in that directory. However, the parser expects one USFM
# file per book, therefore we need to concatenate the book's USFM files
# into one USFM file.
if len(content_files) > 1:
# Make the temp file our only content file.
content_files = [
attempt_asset_content_rescue(resource_dir, resource_lookup_dto)
]
logger.debug("content_files: %s", content_files)
resource_filename_ = resource_filename(
resource_lookup_dto.lang_code,
resource_lookup_dto.resource_type,
resource_lookup_dto.resource_code,
)
# Convert the USFM to HTML and store in file. USFM-Tools books.py can
# raise MalformedUsfmError when the following code is called. The
# document_generator module will catch that error but continue with
# other resource requests in the same document request.
UsfmTransform.buildSingleHtmlFromFile(
pathlib.Path(content_files[0]),
output_dir,
resource_filename_,
)
# Read the HTML file into _content.
html_file = "{}.html".format(
os.path.join(output_dir, resource_filename_))
assert os.path.exists(html_file)
html_content = file_utils.read_file(html_file)
return html_content