def update_report(self):
all_msg = []
for name, allowed in self.pkg_validations.allowed_to_update.items():
msg = []
if self.pkg_validations.allowed_to_update[name] is False:
msg.append(html_reports.p_message(self.pkg_validations.evaluate_changes[name] + ': ' + _('{item} is not allowed to be updated.').format(item=name)))
# allowed to update
order_change = self.pkg_validations.changed_orders.get(name)
if order_change is not None:
# order change
if order_change[1] in self.pkg_validations.rejected_order_change.keys():
# order change is rejected
msg.append(html_reports.p_message(validation_status.STATUS_FATAL_ERROR + ': ' + _('{new_order} is being assign to more than one file: {files}').format(new_order=order_change[1], files=', '.join(self.pkg_validations.rejected_order_change[order_change[1]]))))
else:
# order change is acceptable
msg.append(html_reports.p_message(validation_status.STATUS_WARNING + ': ' + _('order changed: {old} => {new}').format(old=order_change[0], new=order_change[1])))
if len(self.pkg_validations.relaxed_comparison[name]) > 0:
msg.append(html_reports.p_message(self.pkg_validations.evaluate_changes[name] + ': ' + _('{item} contains too many differences. It seems {item} in the package is very different from the one previously published.').format(item=name)))
if len(self.pkg_validations.exact_comparison[name]) > 0:
#self.evaluate_changes[name]
msg.append(html_reports.tag('h5', _('Previously registered:')))
for label, differences in self.pkg_validations.exact_comparison[name]:
msg.append(html_reports.tag('p', differences[1]))
msg.append(html_reports.tag('h5', _('Update:')))
for label, differences in self.pkg_validations.exact_comparison[name]:
msg.append(html_reports.tag('p', differences[0]))
if len(msg) > 0:
all_msg.append(html_reports.tag('h4', name))
all_msg.append('<div class="issue-problem">')
all_msg.append(''.join(msg))
all_msg.append('</div>')
return ''.join(all_msg)
def affiliations(self):
r = html_reports.tag('p', 'Affiliations:', 'label')
for item in self.article.affiliations:
r += html_reports.tag('p', html_reports.format_html_data(item.xml))
th, w, data = self.affiliations_sheet_data()
r += html_reports.sheet(th, data)
return r
def table_of_contents_data_with_lang(self):
r = ''
for lang in sorted(self.article.title_abstract_kwd_languages):
label = html_reports.tag('smaller', attributes.LANGUAGES.get(lang, _('unknown')) + ' [' + lang + ']')
r += '<h4>' + label + '</h4>'
r += '<p>' + '; '.join([k.text for k in self.article.abstracts_by_lang.get(lang, [])]) + '</p>'
r += html_reports.tag('h5', '; '.join([k.text for k in self.article.keywords_by_lang.get(lang, [])]))
return r
def files_and_href(self):
r = ''
r += html_reports.tag('h4', _('Files in the package'))
th, data = self.package_files()
r += html_reports.sheet(th, data, table_style='validation')
r += html_reports.tag('h4', '@href')
th, data = self.hrefs_sheet_data()
r += html_reports.sheet(th, data, table_style='validation')
return r
def content(self):
error_messages = []
if self.rejected_order_changes + self.update_report != '':
error_messages.append(html_reports.tag('h2', _('Order validations')))
error_messages.append(self.rejected_order_changes)
if self.update_report != '':
error_messages.append(html_reports.tag('h3', _('Changes detected')))
error_messages.append(self.update_report)
return ''.join(error_messages)
def footnotes(self):
r = ''
for item in self.article.article_fn_list:
scope, fn_xml = item
r += html_reports.tag('p', scope, 'label')
r += html_reports.tag('p', fn_xml)
if len(r) > 0:
r = html_reports.tag('p', 'foot notes:', 'label') + r
return r
def rejected_order_changes(self):
error_messages = []
if len(self.pkg_validations.rejected_order_change) > 0:
error_messages.append(html_reports.tag('h3', _('rejected orders')))
error_messages.append('<div class="issue-problem">')
error_messages.append(html_reports.p_message(validation_status.STATUS_FATAL_ERROR + ': ' + _('It is not allowed to use same order for different articles.')))
for order, items in self.pkg_validations.rejected_order_change.items():
error_messages.append(html_reports.tag('p', html_reports.format_html_data({order:items})))
error_messages.append('</div>')
return ''.join(error_messages)
def table_tables(self):
r = '<!-- no tables -->'
if len(self.article.tables) > 0:
r = html_reports.tag('p', 'Tables:', 'label')
for t in self.article.tables:
if t.table:
table_data = ''
table_data += html_reports.display_labeled_value('label', t.label, 'label')
table_data += html_reports.tag('div', t.table, 'element-table')
r += html_reports.tag('div', table_data, 'block')
return r
def display_article_data_in_toc(_article):
r = ''
style = 'excluded' if _article.is_ex_aop else None
#status = validation_status.STATUS_INFO + ': ' + _('This article is an ex-aop article. ') + _('Order of ex-aop is reserved, it is not allowed to reuse it for other article. ') if _article.is_ex_aop else ''
#r += html_reports.p_message(status)
r += html_reports.tag('p', _article.toc_section, 'toc-section')
r += html_reports.tag('p', _article.article_type, 'article-type')
r += display_article_metadata(_article, '; ')
return html_reports.tag('div', r, style)
def old_embedded_pdf_items(self, page_id='', width='400px', height='400px'):
items = []
pdf = self.xml_path + '/' + self.xml_name + '.pdf'
if os.path.isfile(pdf):
#items.append('<object id="' + pdf_id + '" data="file://' + pdf + '" width="100%" height="100%"><param name="view" value="Fit" /></object>')
items.append(html_reports.tag('p', html_reports.display_embedded_object(pdf, os.path.basename(pdf), page_id + self.xml_name, width, height)))
for lang in self.article.trans_languages:
pdf = self.xml_path + '/' + self.xml_name + '-' + lang + '.pdf'
if os.path.isfile(pdf):
items.append(html_reports.tag('p', html_reports.display_embedded_object(pdf, os.path.basename(pdf), page_id + self.xml_name + '_' + lang, width, height)))
return ''.join(items)
def article_history(articles):
r = []
for status, article in articles:
text = []
text.append(html_reports.tag('h4', _(status)))
text.append(html_reports.display_label_value(_('name'), article.xml_name, 'p'))
if article.is_ex_aop:
text.append(html_reports.display_label_value(_('status'), 'ex-aop', 'p'))
text.append(html_reports.display_label_value('order', article.order, 'p'))
if article.creation_date_display is not None:
text.append(html_reports.display_label_value(_('creation date'), article.creation_date_display, 'p'))
text.append(html_reports.display_label_value(_('last update date'), article.last_update_display, 'p'))
r.append(html_reports.tag('div', ''.join(text), 'hist-' + status))
return ''.join(r)
def display_author(author):
text = ''
if isinstance(author, PersonAuthor):
if author.surname is not None:
text += html_reports.tag('span', author.surname, 'surname')
if author.suffix is not None:
text += ' ' + html_reports.tag('span', author.suffix, 'suffix')
if author.fname is not None:
text += ', ' + html_reports.tag('span', author.fname, 'name')
else:
text += html_reports.tag('span', author.collab, 'collab')
return text
def display_article_metadata(_article, sep='<br/>'):
r = ''
r += html_reports.tag('p', html_reports.tag('strong', _article.xml_name), 'doi')
r += html_reports.tag('p', _article.publisher_article_id, 'doi')
r += html_reports.tag('p', html_reports.tag('strong', _article.order), 'fpage')
r += html_reports.tag('p', html_reports.tag('strong', _article.title), 'article-title')
r += html_reports.tag('p', display_authors(_article.article_contrib_items, sep))
return r
def table_of_contents_data(self):
r = ''
r += html_reports.tag('p', self.article.toc_section, 'toc-section')
r += html_reports.tag('p', self.article.article_type, 'article-type')
r += html_reports.tag('p', html_reports.tag('strong', self.article.pages), 'fpage')
r += html_reports.tag('p', self.article.doi, 'doi')
r += html_reports.tag('p', html_reports.tag('strong', self.article.title), 'article-title')
a = []
for item in authors_list(self.article.article_contrib_items):
a.append(html_reports.tag('span', item))
r += html_reports.tag('p', '; '.join(a))
return r
def conclusion_message(self):
text = ''.join(self.error_messages)
app_site = converter_env.web_app_site if converter_env.web_app_site is not None else _('scielo web site')
status = ''
result = _('be updated/published on {app_site}').format(app_site=app_site)
reason = ''
update = True
if self.xc_status == 'rejected':
update = False
status = validation_status.STATUS_BLOCKING_ERROR
if self.articles_merger.total_to_convert > 0:
if self.total_not_converted > 0:
reason = _('because it is not complete ({value} were not converted).').format(value=str(self.total_not_converted) + '/' + str(self.articles_merger.total_to_convert))
else:
reason = _('unknown')
else:
reason = _('because there are blocking errors in the package.')
elif self.xc_status == 'ignored':
update = False
reason = _('because no document was changed.')
elif self.xc_status == 'accepted':
status = validation_status.STATUS_WARNING
reason = _(' even though there are some fatal errors. Note: These errors must be fixed in order to have good quality of bibliometric indicators and services.')
elif self.xc_status == 'approved':
status = validation_status.STATUS_OK
reason = ''
else:
status = validation_status.STATUS_FATAL_ERROR
reason = _('because there are blocking errors in the package.')
action = ' not'
if update:
action = ''
text = _('{status}: {issueid} will{action} {result} {reason}.').format(status=status, issueid=self.acron_issue_label, result=result, reason=reason, action=action)
text = html_reports.tag('h2', _('Summary report')) + html_reports.p_message(_('converted') + ': ' + str(self.total_converted) + '/' + str(self.articles_merger.total_to_convert), False) + html_reports.p_message(text, False)
return text
def aop_report(self, status, status_items):
if status_items is None:
return ''
r = ''
if len(status_items) > 0:
labels = []
widths = {}
if status == 'aop':
labels = [_('issue')]
widths = {_('issue'): '5'}
labels.extend([_('filename'), 'order', _('article')])
widths.update({_('filename'): '5', 'order': '2', _('article'): '88'})
report_items = []
for item in status_items:
issueid = None
article = None
if status == 'aop':
issueid, name, article = item
else:
name = item
article = self.articles_merger.merged_articles.get(name)
if article is not None:
if not article.is_ex_aop:
values = []
if issueid is not None:
values.append(issueid)
values.append(name)
values.append(article.order)
values.append(article.title)
report_items.append(pkg_validations.label_values(labels, values))
r = html_reports.tag('h3', _(status)) + html_reports.sheet(labels, report_items, table_style='reports-sheet', html_cell_content=[_('article')], widths=widths)
return r
def link_to_pdf_items(self):
items = []
for item in self.article.related_files:
items.append(html_reports.tag('p', html_reports.link(
item,
os.path.basename(item), window=('800', '400'))))
return ''.join(items)
def display_article_data_to_compare(_article):
r = ''
style = 'excluded' if _article.is_ex_aop else None
status = validation_status.STATUS_INFO + ': ' + _('This article is an ex-aop article. ') + _('Order of ex-aop is reserved, it is not allowed to reuse it for other article. ') if _article.is_ex_aop else ''
r += html_reports.p_message(status)
r += display_article_metadata(_article, '<br/>')
return html_reports.tag('div', r, style)
def conclusion_message(total, converted, not_converted, xc_status, acron_issue_label):
app_site = converter_env.web_app_site if converter_env.web_app_site is not None else _('scielo web site')
status = ''
action = ''
result = _('be updated/published on ') + app_site
reason = ''
if xc_status == 'rejected':
action = _(' not')
status = validation_status.STATUS_FATAL_ERROR
if total > 0:
if not_converted > 0:
reason = _('because it is not complete (') + str(not_converted) + '/' + str(total) + _(' were not converted).')
else:
reason = _('unknown')
else:
reason = _('because there are blocking errors in the package.')
elif xc_status == 'ignored':
action = _(' not')
reason = _('because no document was changed.')
elif xc_status == 'accepted':
status = validation_status.STATUS_WARNING
reason = _(' even though there are some fatal errors. Note: These errors must be fixed in order to have good quality of bibliometric indicators and services.')
elif xc_status == 'approved':
status = validation_status.STATUS_OK
reason = ''
else:
status = validation_status.STATUS_FATAL_ERROR
reason = _('because there are blocking errors in the package.')
text = status + ': ' + acron_issue_label + _(' will') + action + ' ' + result + ' ' + reason
text = html_reports.tag('h2', _('Summary report')) + html_reports.p_message(_('converted') + ': ' + str(converted) + '/' + str(total), False) + html_reports.p_message(text, False)
return text
def embedded_pdf_items(self, page_id='', width='400px', height='400px'):
items = []
for item in self.article.related_files:
items.append(html_reports.tag('p', html_reports.display_embedded_object(
item,
os.path.basename(item), page_id + item, width, height)))
return ''.join(items)
def conversion_report(self):
#resulting_orders
labels = [_('article'), _('registered') + '/' + _('before conversion'), _('package'), _('executed actions'), _('achieved results')]
widths = {_('article'): '20', _('registered') + '/' + _('before conversion'): '20', _('package'): '20', _('executed actions'): '20', _('achieved results'): '20'}
#print(self.articles_merger.history_items)
for status, status_items in self.aop_status.items():
for status_data in status_items:
if status != 'aop':
name = status_data
article = self.articles_merger.xc_articles[name]
self.articles_merger.history_items[name].append((status, article))
for status, names in self.conversion_status.items():
for name in names:
self.articles_merger.history_items[name].append((status, self.articles_merger.xc_articles[name]))
history = sorted([(hist[0][1].order, xml_name) for xml_name, hist in self.articles_merger.history_items.items()])
history = [(xml_name, self.articles_merger.history_items[xml_name]) for order, xml_name in history]
items = []
for xml_name, hist in history:
values = []
values.append(article_reports.display_article_data_in_toc(hist[-1][1]))
values.append(article_reports.article_history([item for item in hist if item[0] == 'registered article']))
values.append(article_reports.article_history([item for item in hist if item[0] == 'package']))
values.append(article_reports.article_history([item for item in hist if not item[0] in ['registered article', 'package', 'rejected', 'converted', 'not converted']]))
values.append(article_reports.article_history([item for item in hist if item[0] in ['rejected', 'converted', 'not converted']]))
items.append(pkg_validations.label_values(labels, values))
return html_reports.tag('h3', _('Conversion steps')) + html_reports.sheet(labels, items, html_cell_content=[_('article'), _('registered') + '/' + _('before conversion'), _('package'), _('executed actions'), _('achieved results')], widths=widths)
def link_to_pdf_and_xml_files(self):
items = []
for item in self.article.related_files:
location = '{PDF_PATH}' if item.endswith('.pdf') else '{XML_PATH}'
items.append(html_reports.tag('p', html_reports.link(
location + item,
item, window=('1000', '400'))))
return ''.join(items)
def final_status_report(self):
actions_result_labels = {'delete': 'deleted', 'update': 'updated', 'add': 'added', '-': '-', 'skip-update': 'skipped', 'order changed': 'order changed', 'fail': 'update/add failed', 'block-update': 'update blocked'}
orders = sorted(list(set([article.order for article in self.previous_registered_articles.values()] + [article.order for article in self.db.registered_articles.values()] + [article.order if article.tree is not None else 'None' for article in self.pkg.articles.values()])))
sorted_previous_registered = pkg_reports.articles_sorted_by_order(self.previous_registered_articles)
sorted_registered = pkg_reports.articles_sorted_by_order(self.db.registered_articles)
sorted_package = pkg_reports.articles_sorted_by_order(self.pkg.articles)
items = []
for order in orders:
if order in sorted_registered.keys():
# documento na base
for article in sorted_registered[order]:
action = self.actions[article.xml_name]
result = actions_result_labels[action]
_notes = ''
if action == 'update':
if article.last_update_display is None:
result = 'error'
elif self.previous_registered_articles.get(article.xml_name).last_update_display == article.last_update_display:
result = 'error'
name = article.xml_name
if name in self.changed_orders.keys():
previous_order, new_order = self.changed_orders[name]
_notes = previous_order + '=>' + new_order
if result == 'error':
_notes = validation_status.STATUS_ERROR + ': ' + _('Unable to replace ') + _notes
labels, values = complete_issue_items_row(article, action, result, 'registered', _notes, True)
items.append(pkg_reports.label_values(labels, values))
elif order in sorted_package.keys():
# documento no pacote mas nao na base
for article in sorted_package[order]:
action = self.actions[article.xml_name]
name = article.xml_name
_notes = ''
if name in self.changed_orders.keys():
previous_order, new_order = self.changed_orders[name]
_notes = previous_order + '=>' + new_order
_notes = validation_status.STATUS_ERROR + ': ' + _('Unable to replace ') + _notes
labels, values = complete_issue_items_row(article, action, 'error', 'package', _notes, True)
items.append(pkg_reports.label_values(labels, values))
elif order in sorted_previous_registered.keys():
# documento anteriormente na base
for article in sorted_previous_registered[order]:
action = 'delete'
name = article.xml_name
_notes = ''
if name in self.changed_orders.keys():
previous_order, new_order = self.changed_orders[name]
_notes = 'deleted ' + previous_order + '=> new: ' + new_order
labels, values = complete_issue_items_row(article, '?', 'deleted', 'excluded', _notes, True)
items.append(pkg_reports.label_values(labels, values))
after_conversion_report = html_reports.tag('h4', _('Documents status in the package/database - after conversion'))
after_conversion_report += html_reports.sheet(labels, items, 'dbstatus', 'result')
return after_conversion_report
def references_stats(self):
r = html_reports.tag('h2', 'references')
sheet_data = []
for ref_type, q in self.article.refstats.items():
row = {}
row['element-citation/@publication-type'] = ref_type
row['quantity'] = q
sheet_data.append(row)
r += html_reports.sheet(['element-citation/@publication-type', 'quantity'], sheet_data)
return r
def display_formulas(self):
labels = ['id', 'code', 'graphic', 'xml']
formulas_data = []
for formula_data in self.article.formulas_data:
if formula_data['graphic'] is not None:
formula_data['graphic'] = html_reports.link('{IMG_PATH}/' + formula_data['graphic'], html_reports.image('{IMG_PATH}/' + formula_data['graphic']))
if formula_data['code'] is not None:
formula_data['code'] = formula_data['code'].replace('mml:', '')
formulas_data.append(formula_data)
return html_reports.tag('h1', _('Equations')) + html_reports.sheet(labels, formulas_data, html_cell_content=['code'])
def article_data_and_validations_report(journal, article, new_name, package_path, images_generation_report_filename, is_db_generation, is_sgml_generation):
if article.tree is None:
sheet_data = None
article_display_report = None
article_validation_report = None
content = validation_status.STATUS_FATAL_ERROR + ': ' + _('Unable to get data of ') + new_name + '.'
else:
article_validation = article_validations.ArticleContentValidation(journal, article, is_db_generation, False)
sheet_data = ArticleSheetData(article, article_validation)
article_display_report = ArticleDisplayReport(article, sheet_data, package_path, new_name)
article_validation_report = ArticleValidationReport(article_validation)
content = []
img_report_content = ''
if os.path.isfile(images_generation_report_filename):
img_report_content = fs_utils.read_file(images_generation_report_filename)
if len(img_report_content) > 0:
content.append(html_reports.tag('h1', _('ATTENTION'), 'warning'))
content.append(html_reports.tag('h1', _('New report: Images Report at the bottom'), 'warning'))
if is_sgml_generation:
content.append(article_display_report.issue_header)
content.append(article_display_report.article_front)
content.append(article_validation_report.validations(display_all_message_types=False))
content.append(article_display_report.table_tables)
content.append(article_display_report.article_body)
content.append(article_display_report.article_back)
else:
content.append(article_validation_report.validations(display_all_message_types=False))
content.append(article_display_report.table_tables)
content.append(sheet_data.files_and_href(package_path))
if len(img_report_content) > 0:
content.append(img_report_content)
content = html_reports.join_texts(content)
return content
def id_and_tag_list(self):
sheet_data = []
t_header = ['@id', 'tag']
for item in self.article.elements_which_has_id_attribute:
row = {}
row['@id'] = item.attrib.get('id')
row['tag'] = item.tag
sheet_data.append(row)
r = html_reports.tag('h2', 'elements and @id:')
r += html_reports.sheet(t_header, sheet_data)
return r
def validations_table(results):
r = ''
if results is not None:
rows = []
for result in results:
result = list(result)
if len(result) == 3:
result.append('')
label, status, msg, xml = result
rows.append({'label': sps_help(label), 'status': status, 'message': msg, 'xml': xml, _('why it is not a valid message?'): ' '})
r = html_reports.tag('div', html_reports.sheet(['label', 'status', 'message', 'xml', _('why it is not a valid message?')], rows, table_style='validation'))
return r
def references(self, display_all):
rows = ''
found_errors = []
for ref, ref_result in self.article_validation.references:
if not display_all:
found_errors = [res[1] for res in ref_result if res[1] in [validation_status.STATUS_WARNING, validation_status.STATUS_ERROR, validation_status.STATUS_FATAL_ERROR]]
ref_result = [res for res in ref_result if res[1] != validation_status.STATUS_OK]
if len(found_errors) > 0:
rows += html_reports.tag('h3', _('Reference {id}').format(id=ref.id))
rows += validations_table(ref_result)
return rows
def references(self, display_all):
rows = ''
found_errors = []
for ref, ref_result in self.article_validation.references:
if not display_all:
found_errors = [status for label, status, msg in ref_result if status in [validation_status.STATUS_WARNING, validation_status.STATUS_ERROR, validation_status.STATUS_FATAL_ERROR]]
ref_result = [(label, status, msg) for label, status, msg in ref_result if status != validation_status.STATUS_OK]
if len(found_errors) > 0:
rows += html_reports.tag('h3', 'Reference ' + ref.id)
rows += validations_table(ref_result)
return rows
