def get_scripture_with_sn_quotes(self, bible_id, chapter, verse, rc,
scripture):
if not scripture:
scripture = self.get_plain_scripture(bible_id, chapter, verse)
footnotes_split = re.compile('<div class="footnotes">',
flags=re.IGNORECASE | re.MULTILINE)
verses_and_footnotes = re.split(footnotes_split, scripture, maxsplit=1)
scripture = verses_and_footnotes[0]
footnote = ''
if len(verses_and_footnotes) == 2:
footnote = f'<div class="footnotes">{verses_and_footnotes[1]}'
if verse in self.sn_book_data[chapter]:
sn_notes = self.sn_book_data[chapter][verse]
else:
sn_notes = []
orig_scripture = scripture
for sn_note_idx, sn_note in enumerate(sn_notes):
occurrence = 1
if RepresentsInt(
sn_note['Occurrence']) and int(sn_note['Occurrence']) > 0:
occurrence = int(sn_note['Occurrence'])
gl_quote_phrase = [[{
'word': sn_note['GLQuote'],
'occurrence': occurrence
}]]
phrase = sn_note['alignments'][bible_id]
if not phrase:
phrase = gl_quote_phrase
if flatten_alignment(phrase).lower() in QUOTES_TO_IGNORE:
continue
split = ''
if len(phrase) > 1:
split = ' split'
tag = f'<span class="highlight phrase phrase-{sn_note_idx+1}{split}">'
marked_verse_html = html_tools.mark_phrases_in_html(scripture,
phrase,
tag=tag)
if not marked_verse_html:
fix = None
if flatten_alignment(phrase).lower() not in QUOTES_TO_IGNORE:
if sn_note['GLQuote']:
marked_with_gl_quote = html_tools.mark_phrases_in_html(
scripture, gl_quote_phrase)
if marked_with_gl_quote:
fix = sn_note['GLQuote']
self.add_bad_highlight(rc, orig_scripture, sn_note['rc'],
sn_note['GLQuote'], fix)
else:
scripture = marked_verse_html
scripture += footnote
return scripture
def get_aligned_text(self, bible_id, context_id):
if not context_id or 'quote' not in context_id or not context_id['quote'] or 'reference' not in context_id or \
'chapter' not in context_id['reference'] or 'verse' not in context_id['reference']:
return None
chapter = str(context_id['reference']['chapter'])
verse = str(context_id['reference']['verse'])
verse_objects = self.get_verse_objects(bible_id, chapter, verse)
if not verse_objects:
return None
alignment = get_alignment(verse_objects, context_id['quote'],
context_id['occurrence'])
if not alignment:
title = f'{self.project_title} {chapter}:{verse}'
aligned_text_rc_link = f'rc://{self.lang_code}/{bible_id}/bible/{self.project_id}/{self.pad(chapter)}/{str(verse).zfill(3)}'
aligned_text_rc = self.create_rc(aligned_text_rc_link, title=title)
quote_string = context_id[
'quoteString'] if 'quoteString' in context_id else flatten_alignment(
context_id['quote'])
if int(self.book_number) > 40 or self.project_id.lower(
) == 'rut' or self.project_id.lower() == 'jon':
message = f'''OL ({self.ol_lang_code.upper()}) quote not found in {bible_id.upper()} alignment:
VERSE: {self.project_title} {chapter}:{verse}
RC: {context_id['rc']}
QUOTE: {quote_string}
{bible_id.upper()}: {self.verse_usfm[bible_id][chapter][verse]['usfm']}
{self.ol_bible_id.upper()}: {self.verse_usfm[self.ol_bible_id][chapter][verse]['usfm']}
'''
self.add_error_message(aligned_text_rc, context_id['rc'],
message)
return alignment
def get_scripture_with_tw_words(self, bible_id, chapter, verse, rc=None):
scripture = self.get_plain_scripture(bible_id, chapter, verse)
footnotes_split = re.compile('<div class="footnotes">',
flags=re.IGNORECASE | re.MULTILINE)
verses_and_footnotes = re.split(footnotes_split, scripture, maxsplit=1)
scripture = verses_and_footnotes[0]
footnote = ''
if len(verses_and_footnotes) == 2:
footnote = f'<div class="footnotes">{verses_and_footnotes[1]}'
orig_scripture = scripture
if chapter not in self.tw_words_data or verse not in self.tw_words_data[chapter] or \
not self.tw_words_data[chapter][verse]:
return scripture
phrases = self.tw_words_data[chapter][verse]
for group_data_idx, group_data in enumerate(phrases):
tw_rc = group_data['contextId']['rc']
split = ''
if len(group_data):
split = ' split'
tag = f'<a href="{tw_rc}" class="tw-phrase tw-phrase-{group_data_idx + 1}{split}">'
alignment = group_data['alignments'][bible_id]
if alignment:
marked_verse_html = html_tools.mark_phrases_in_html(scripture,
alignment,
tag=tag)
if not marked_verse_html:
if rc:
self.add_bad_highlight(rc, orig_scripture, tw_rc,
flatten_alignment(group_data))
else:
scripture = marked_verse_html
scripture += footnote
return scripture
def get_tw_html_list(self, bible_id, chapter, verse):
if chapter not in self.tw_words_data or verse not in self.tw_words_data[chapter] or \
not self.tw_words_data[chapter][verse]:
return ''
phrases = self.tw_words_data[chapter][verse]
links = []
for group_data_idx, group_data in enumerate(phrases):
tw_rc = group_data['contextId']['rc']
occurrence = group_data['contextId']['occurrence']
occurrence_text = ''
if occurrence > 1:
occurrence_text = f' ({occurrence})'
alignment = group_data['alignments'][bible_id]
if alignment:
title = flatten_alignment(alignment)
else:
title = f'[[{group_data["contextId"]["rc"]}]] (not aligned)'
links.append(
f'<a href="{tw_rc}" class="tw-phrase tw-phrase-{group_data_idx + 1}">{title}</a>{occurrence_text}'
)
tw_html = f'''
<h3>{self.resources['tw'].simple_title} - {bible_id.upper()}</h3>
<ul class="tw-list">
<li>{'</li><li>'.join(links)}</li>
</ul>
'''
return tw_html
def get_tw_html_list(self, bible_id, chapter, verse, scripture=''):
if chapter not in self.tw_words_data or verse not in self.tw_words_data[chapter] or \
not self.tw_words_data[chapter][verse]:
return ''
group_datas = self.tw_words_data[chapter][verse]
for group_data_idx, group_data in enumerate(group_datas):
alignment = group_data['alignments'][bible_id]
if alignment:
title = flatten_alignment(alignment)
else:
title = f'[[{group_data["contextId"]["rc"]}]]'
group_datas[group_data_idx]['title'] = title
rc_pattern = 'rc://[/A-Za-z0-9*_-]+'
rc_order = re.findall(rc_pattern, scripture)
group_datas.sort(key=lambda x: str(
rc_order.index(x['contextId']['rc'])
if x['contextId']['rc'] in rc_order else x['title']))
links = []
for group_data_idx, group_data in enumerate(group_datas):
tw_rc = group_data['contextId']['rc']
occurrence = group_data['contextId']['occurrence']
occurrence_text = ''
if occurrence > 1:
occurrence_text = f' ({occurrence})'
title = group_data['title']
links.append(
f'<a href="{tw_rc}" class="tw-phrase tw-phrase-{group_data_idx + 1}">{title}</a>{occurrence_text}'
)
tw_html = f'''
<h3>{self.resources['tw'].simple_title} - {bible_id.upper()}</h3>
<ul class="tw-list">
<li>{'</li><li>'.join(links)}</li>
</ul>
'''
return tw_html
def get_tn_checking_html(self):
self.populate_tn_groups_data()
self.populate_tn_book_data()
by_rc_cat_group = {}
for chapter in self.tn_book_data:
for verse in self.tn_book_data[chapter]:
for data in self.tn_book_data[chapter][verse]:
if data['contextId']:
rc_link = data['contextId']['rc']
parts = rc_link[5:].split('/')
category = parts[3]
group = parts[4]
if category not in by_rc_cat_group:
by_rc_cat_group[category] = {}
if group not in by_rc_cat_group[category]:
by_rc_cat_group[category][group] = []
by_rc_cat_group[category][group].append(data)
tn_html = f'''
<section id="{self.lang_code}-{self.name}-{self.project_id}" class="{self.name}">
<article id="{self.lang_code}-{self.name}-{self.project_id}-cover" class="resource-title-page">
<img src="{self.main_resource.logo_url}" class="logo" alt="UTN">
<h1 class="section-header">{self.title}</h1>
<h2 class="section-header">{self.project_title}</h2>
</article>
'''
categories = sorted(by_rc_cat_group.keys())
for category in categories:
groups = sorted(by_rc_cat_group[category].keys())
for group in groups:
tn_rc_link = f'rc://{self.lang_code}/tn/help/{category}/{group}'
tn_rc = self.add_rc(tn_rc_link, title=f'{group} ({category})')
tn_html += f'''
<article id="{tn_rc.article_id}">
<h3 class="section-header">Support Reference: [[{tn_rc.rc_link}]]</h3>
<table width="100%">
<tr>
<th style="width:1px;padding:0 !important"></th>
<th>Verse</th>
<th>{self.ult_id.upper()} Alignment</th>
<th>{self.ult_id.upper()} Text</th>
<th>{self.ust_id.upper()} Alignment</th>
<th>{self.ust_id.upper()} Text</th>
<th>{self.ol_bible_id.upper()} Quote</th>
<th>{self.ol_bible_id.upper()} Text</th>
</tr>
'''
for group_data in by_rc_cat_group[category][group]:
context_id = group_data['contextId']
context_id['rc'] = tn_rc.rc_link
chapter = str(context_id['reference']['chapter'])
verse = str(context_id['reference']['verse'])
group_data['scripture'] = {}
for bible_id in [self.ult_id, self.ust_id]:
alignment = group_data['alignments'][bible_id]
if not alignment:
group_data['alignments'][
bible_id] = '<div style="color: red">NONE</div>'
else:
group_data['alignments'][
bible_id] = flatten_alignment(alignment)
scripture = self.get_plain_scripture(
bible_id, chapter, verse)
marked_html = None
if alignment:
marked_html = mark_phrases_in_html(
scripture, alignment)
if marked_html:
group_data['scripture'][bible_id] = marked_html
else:
group_data['scripture'][
bible_id] = f'<div style="color: red">{scripture}</div>'
scripture = self.get_plain_scripture(
self.ol_bible_id, chapter, verse)
ol_alignment = split_string_into_alignment(
group_data['OrigQuote'])
marked_html = mark_phrases_in_html(scripture, ol_alignment)
if marked_html:
group_data['scripture'][self.ol_bible_id] = marked_html
else:
marked_html = mark_phrases_in_html(scripture,
ol_alignment,
break_on_word=False)
if marked_html:
scripture = marked_html
group_data['scripture'][
self.
ol_bible_id] = f'<div style="color: red">{scripture}</div>'
tn_html += f'''
<tr id="{group_data['rc'].article_id}">
<td style="width:1px;padding:0 !important"><a id="{group_data['ID']}"></a><a href="#{group_data['rc'].article_id}"><i class="fa fa-link"></i></a></td>
<td>
{chapter}:{verse}
(<a href="https://git.door43.org/unfoldingWord/{self.lang_code}_tn/src/branch/master/{self.lang_code}_tn_{self.book_number}-{self.project_id.upper()}.tsv#L{group_data['row']}" target="tn-repo">{group_data['ID']}</a>)
</td>
<td>
{group_data['alignments'][self.ult_id]}
</td>
<td>
{group_data['scripture'][self.ult_id]}
</td>
<td>
{group_data['alignments'][self.ust_id]}
</td>
<td>
{group_data['scripture'][self.ust_id]}
</td>
<td style="direction: {'rtl' if self.ol_lang_code == 'hbo' else 'ltr'}">
{group_data['OrigQuote']}
</td>
<td style="direction: {'rtl' if self.ol_lang_code == 'hbo' else 'ltr'}">
{group_data['scripture'][self.ol_bible_id]}
</td>
</tr>
'''
tn_html += '''
</table>
</article>
'''
tn_html += '''
</section>
'''
self.logger.info('Done generating TN Checking HTML.')
return tn_html
def populate_tn_book_data(self):
book_file = os.path.join(
self.main_resource.repo_dir,
f'{self.lang_code}_tn_{self.book_number}-{self.project_id.upper()}.tsv'
)
if not os.path.isfile(book_file):
return
book_data = OrderedDict()
reader = self.unicode_csv_reader(open(book_file))
header = next(reader)
row_count = 1
for row in reader:
row_count += 1
verse_data = {
'contextId': None,
'row': row_count,
'alignments': {
self.ult_id: None,
self.ust_id: None
}
}
found = False
for idx, field in enumerate(header):
field = field.strip()
if idx >= len(row):
self.logger.error(f'ERROR: {book_file} is malformed')
found = False
break
else:
found = True
verse_data[field] = row[idx]
if not found:
break
chapter = verse_data['Chapter'].lstrip('0')
verse = verse_data['Verse'].lstrip('0')
if verse_data['Occurrence']:
occurrence = int(verse_data['Occurrence'])
else:
occurrence = 1
tn_rc_link = f'rc://{self.lang_code}/tn/help/{self.project_id}/{self.pad(chapter)}/{verse.zfill(3)}/{verse_data["ID"]}'
tn_title = f'{verse_data["GLQuote"]} (not aligned)'
if verse_data['OrigQuote']:
context_id = None
if chapter in self.tn_groups_data and verse in self.tn_groups_data[chapter] and \
self.tn_groups_data[chapter][verse]:
for c_id in self.tn_groups_data[chapter][verse]:
if c_id['quoteString'] == verse_data[
'OrigQuote'] and c_id[
'occurrence'] == occurrence:
context_id = c_id
break
if not context_id and chapter.isdigit() and verse.isdigit():
context_id = {
'reference': {
'chapter': int(chapter),
'verse': int(verse)
},
'rc':
f'rc://{self.lang_code}/tn/help///{self.project_id}/{self.pad(chapter)}/{verse.zfill(3)}',
'quote': verse_data['OrigQuote'],
'occurrence': int(verse_data['Occurrence']),
'quoteString': verse_data['OrigQuote']
}
if context_id:
context_id['rc'] += f'/{verse_data["ID"]}'
verse_data['contextId'] = context_id
verse_data['alignments'] = {
self.ult_id:
self.get_aligned_text(self.ult_id, context_id),
self.ust_id:
self.get_aligned_text(self.ust_id, context_id)
}
if verse_data['alignments'][self.ult_id]:
tn_title = flatten_alignment(verse_data['alignments'][
self.ult_id]) + f' ({self.ult_id.upper()})'
else:
tn_title = f'{verse_data["GLQuote"]} ({self.ult_id.upper()} not aligned)'
tn_title += '; '
if verse_data['alignments'][self.ust_id]:
tn_title += flatten_alignment(verse_data['alignments'][
self.ust_id]) + f' ({self.ust_id.upper()})'
else:
tn_title = f'{verse_data["GLQuote"]} ({self.ust_id.upper()} not aligned)'
tn_rc = self.create_rc(tn_rc_link, title=tn_title)
verse_data['title'] = tn_title
verse_data['rc'] = tn_rc
if chapter not in book_data:
book_data[chapter] = OrderedDict()
if verse not in book_data[chapter]:
book_data[chapter][verse] = []
book_data[str(chapter)][str(verse)].append(verse_data)
self.tn_book_data = book_data
def populate_sn_book_data(self):
book_filename = f'{self.lang_code}_{self.main_resource.resource_name}_{self.book_number}-{self.project_id.upper()}.tsv'
book_filepath = os.path.join(self.main_resource.repo_dir,
book_filename)
if not os.path.isfile(book_filepath):
return
book_data = OrderedDict()
reader = self.unicode_csv_reader(open(book_filepath))
header = next(reader)
row_count = 1
for row in reader:
row_count += 1
verse_data = {
'contextId': None,
'row': row_count,
'alignments': {
self.ult_id: None,
self.ust_id: None
}
}
found = False
for idx, field in enumerate(header):
field = field.strip()
if idx >= len(row):
self.logger.error(
f'ERROR: {book_filepath} is malformed at row {row_count}: {row}'
)
self.add_error_message(
self.create_rc(f'{book_filename}#{row_count}'),
f'Line {row_count}', f'Malformed row: {row}')
found = False
break
else:
found = True
verse_data[field] = row[idx]
if not found:
continue
chapter = verse_data['Chapter'].lstrip('0')
verse = verse_data['Verse'].lstrip('0')
if verse_data['Occurrence']:
occurrence = int(verse_data['Occurrence'])
else:
occurrence = 1
sn_rc_link = f'rc://{self.lang_code}/{self.main_resource.resource_name}/help/{self.project_id}/{self.pad(chapter)}/{verse.zfill(3)}/{verse_data["ID"]}'
sn_title = f'{verse_data["GLQuote"]}'
if verse_data['OrigQuote']:
context_id = None
if not context_id and chapter.isdigit() and verse.isdigit():
context_id = {
'reference': {
'chapter': int(chapter),
'verse': int(verse)
},
'rc':
f'rc://{self.lang_code}/{self.main_resource.resource_name}/help///{self.project_id}/{self.pad(chapter)}/{verse.zfill(3)}',
'quote': verse_data['OrigQuote'],
'occurrence': occurrence,
'quoteString': verse_data['OrigQuote']
}
if context_id:
context_id['rc'] += f'/{verse_data["ID"]}'
context_id['quoteString'] = verse_data['OrigQuote']
verse_data['contextId'] = context_id
verse_data['alignments'] = {
self.ult_id:
self.get_aligned_text(self.ult_id, context_id),
self.ust_id:
self.get_aligned_text(self.ust_id, context_id)
}
if verse_data['alignments'][self.ult_id]:
sn_title = flatten_alignment(verse_data['alignments'][
self.ult_id]) + f' ({self.ult_id.upper()})'
if verse_data['alignments'][self.ust_id]:
sn_title += '<br/>' + flatten_alignment(
verse_data['alignments'][
self.ust_id]) + f' ({self.ust_id.upper()})'
else:
sn_title = f'{verse_data["GLQuote"]}'
sn_rc = self.create_rc(sn_rc_link, title=sn_title)
verse_data['title'] = sn_title
verse_data['rc'] = sn_rc
if chapter not in book_data:
book_data[chapter] = OrderedDict()
if verse not in book_data[chapter]:
book_data[chapter][verse] = []
book_data[str(chapter)][str(verse)].append(verse_data)
self.sn_book_data = book_data
def get_tw_checking_html(self):
tw_html = f'''
<section id="{self.lang_code}-{self.name}-{self.project_id}" class="{self.name}">
<article id="{self.lang_code}-{self.name}-{self.project_id}-cover" class="resource-title-page">
<img src="{self.main_resource.logo_url}" class="logo" alt="UTW">
<h1 class="section-header">{self.title}</h1>
<h2 class="section-header">{self.project_title}</h2>
</article>
'''
tw_path = os.path.join(self.resources_dir, self.ol_lang_code,
'translationHelps/translationWords')
if not tw_path:
self.logger.error(f'{tw_path} not found!')
exit(1)
tw_version_path = get_latest_version_path(tw_path)
if not tw_version_path:
self.logger.error(f'No versions found in {tw_path}!')
exit(1)
groups = get_child_directories(tw_version_path)
for group in groups:
files_path = os.path.join(tw_version_path,
f'{group}/groups/{self.project_id}',
'*.json')
files = glob(files_path)
for file in files:
base = os.path.splitext(os.path.basename(file))[0]
tw_rc_link = f'rc://{self.lang_code}/tw/dict/bible/{group}/{base}'
tw_rc = self.add_rc(tw_rc_link, title=base)
self.get_tw_article_html(tw_rc)
tw_html += f'''
<article id="{tw_rc.article_id}">
<h3 class="section-header">[[{tw_rc.rc_link}]]</h3>
<table width="100%">
<tr>
<th style="width:1px;padding:0 !important"></th>
<th>Verse</th>
<th>{self.ult_id.upper()} Alignment</th>
<th>{self.ult_id.upper()} Text</th>
<th>{self.ust_id.upper()} Alignment</th>
<th>{self.ust_id.upper()} Text</th>
<th>{self.ol_bible_id.upper()} Quote</th>
<th>{self.ol_bible_id.upper()} Text</th>
</tr>
'''
tw_group_data = load_json_object(file)
for group_data in tw_group_data:
context_id = group_data['contextId']
context_id['rc'] = tw_rc.rc_link
chapter = str(context_id['reference']['chapter'])
verse = str(context_id['reference']['verse'])
context_id['scripture'] = {}
context_id['alignments'] = {}
for bible_id in [self.ult_id, self.ust_id]:
alignment = self.get_aligned_text(
bible_id, group_data['contextId'])
if alignment:
context_id['alignments'][
bible_id] = flatten_alignment(alignment)
else:
context_id['alignments'][
bible_id] = '<div style="color: red">NONE</div>'
scripture = self.get_plain_scripture(
bible_id, chapter, verse)
marked_html = None
if alignment:
marked_html = mark_phrases_in_html(
scripture, alignment)
if marked_html:
context_id['scripture'][bible_id] = marked_html
else:
context_id['scripture'][
bible_id] = f'<div style="color: red">{scripture}</div>'
scripture = self.get_plain_scripture(
self.ol_bible_id, chapter, verse)
ol_alignment = context_id['quote']
if isinstance(ol_alignment, str):
ol_alignment = split_string_into_alignment(
ol_alignment)
if not isinstance(ol_alignment[0], list):
ol_alignment = convert_single_dimensional_quote_to_multidimensional(
ol_alignment)
marked_html = mark_phrases_in_html(scripture, ol_alignment)
if marked_html:
context_id['scripture'][self.ol_bible_id] = marked_html
else:
context_id['scripture'][
self.
ol_bible_id] = f'<div style="color: red">{scripture}</div>'
tw_html += f'''
<tr id="{tw_rc.article_id}-{chapter}-{verse}">
<td style="width:1px;padding:0 !important"><a href="#{tw_rc.article_id}-{chapter}-{verse}"><i class="fa fa-link"></i></td>
<td>
{chapter}:{verse}
</td>
<td>
{context_id['alignments'][self.ult_id]}
</td>
<td>
{context_id['scripture'][self.ult_id]}
</td>
<td>
{context_id['alignments'][self.ust_id]}
</td>
<td>
{context_id['scripture'][self.ust_id]}
</td>
<td style="direction: {'rtl' if self.ol_lang_code == 'hbo' else 'ltr'}">
{flatten_alignment(ol_alignment)}
</td>
<td style="direction: {'rtl' if self.ol_lang_code == 'hbo' else 'ltr'}">
{context_id['scripture'][self.ol_bible_id]}
</td>
</tr>
'''
tw_html += '''
</table>
</article>
'''
tw_html += '''
</section>
'''
self.logger.info('Done generating TW Checking HTML.')
return tw_html