Пример #1
0
def convert_package(src_path):
    xc_process_logger = fs_utils.ProcessLogger()

    scilista_items = []
    is_xml_generation = False

    scielo_dtd_files = xml_versions.DTDFiles('scielo', converter_env.version)

    pkg_name = os.path.basename(src_path)[:-4]

    if not os.path.isdir('./../log'):
        os.makedirs('./../log')
    log_package = './../log/' + datetime.now().isoformat().replace(':', '_') + os.path.basename(pkg_name)

    fs_utils.append_file(log_package, 'preparing')
    tmp_report_path, wrk_path, scielo_pkg_path, tmp_result_path = package_paths_preparation(src_path)
    final_result_path = tmp_result_path
    final_report_path = tmp_report_path

    fs_utils.append_file(log_package, 'normalized_package')
    articles, articles_work_area = normalized_package(src_path, tmp_report_path, wrk_path, scielo_pkg_path, converter_env.version)

    #, converter_env.is_windows
    doi_services = article_validations.DOI_Services()

    articles_pkg = pkg_validations.ArticlesPackage(scielo_pkg_path, articles, is_xml_generation)

    articles_data = pkg_validations.ArticlesData()
    articles_data.setup(articles_pkg, xc_models.JournalsManager(), db_manager=converter_env.db_manager)

    articles_set_validations = pkg_validations.ArticlesSetValidations(articles_pkg, articles_data, xc_process_logger)
    articles_set_validations.validate(doi_services, scielo_dtd_files, articles_work_area)

    conversion = ArticlesConversion(articles_set_validations, articles_data.articles_db_manager, not converter_env.is_windows)
    conversion.final_result_path = final_result_path
    conversion.final_report_path = final_report_path
    scilista_items = conversion.convert()

    reports = pkg_validations.ReportsMaker(articles_set_validations, None, conversion, display_report=converter_env.is_windows)

    reports.processing_result_location = conversion.final_result_path
    report_location = conversion.final_report_path + '/xml_converter.html'
    reports.save_report(conversion.final_report_path, 'xml_converter.html', _('XML Conversion (XML to Database)'))

    if not converter_env.is_windows:
        format_reports_for_web(conversion.final_report_path, scielo_pkg_path, conversion.acron_issue_label.replace(' ', '/'))
    if tmp_result_path != final_result_path:
        fs_utils.delete_file_or_folder(tmp_result_path)
    os.unlink(log_package)
    return (scilista_items, conversion.xc_status, reports.validations.statistics_display(), report_location)
Пример #2
0
def convert_package(src_path):
    xc_process_logger = fs_utils.ProcessLogger()

    scilista_items = []
    is_xml_generation = False

    pkg_name = os.path.basename(src_path)[:-4]

    if not os.path.isdir('./../log'):
        os.makedirs('./../log')
    log_package = './../log/' + datetime.now().isoformat().replace(':', '_') + os.path.basename(pkg_name)

    fs_utils.append_file(log_package, 'preparing')
    tmp_result_path = src_path + '_xc'

    fs_utils.append_file(log_package, 'normalized_package')
    xml_files = sorted([src_path + '/' + f for f in os.listdir(src_path) if f.endswith('.xml') and not 'incorrect' in f])
    pkg_maker = xpmaker.PackageMaker(xml_files, tmp_result_path, 'acron', converter_env.version, is_db_generation=True)
    pkg_maker.make_sps_package()

    #, converter_env.is_windows
    doi_services = article_validations.DOI_Services()

    articles_pkg = pkg_validations.ArticlesPackage(pkg_maker.scielo_pkg_path, pkg_maker.article_items, is_xml_generation)

    articles_data = pkg_validations.ArticlesData()
    articles_data.setup(articles_pkg, xc_models.JournalsManager(), db_manager=converter_env.db_manager)

    articles_set_validations = pkg_validations.ArticlesSetValidations(articles_pkg, articles_data, xc_process_logger)
    articles_set_validations.validate(doi_services, pkg_maker.scielo_dtd_files, pkg_maker.article_work_area_items)

    conversion = ArticlesConversion(articles_set_validations, articles_data.articles_db_manager, not converter_env.is_windows)
    scilista_items = conversion.convert()

    #reports.validations.statistics_display()
    #conversion.statistics_display
    #report_location
    #conversion.report_location

    if tmp_result_path != conversion.results_path:
        fs_utils.delete_file_or_folder(tmp_result_path)
    os.unlink(log_package)
    return (scilista_items, conversion.xc_status, conversion.statistics_display, conversion.report_location)
Пример #3
0
def run_cmd(cmd, log_filename=None):
    print(cmd)
    if log_filename is not None:
        fs_utils.append_file(log_filename, datetime.now().isoformat() + ' ' + cmd)
    try:
        os.system(cmd)
        if log_filename is not None:
            fs_utils.append_file(log_filename, 'done')
    except:
        if log_filename is not None:
            fs_utils.append_file(log_filename, 'failure')
Пример #4
0
def convert_package(src_path):
    xc_conclusion_msg = ''
    pkg_xml_fatal_errors = 0
    xc_results_report = ''
    aop_results_report = ''
    before_conversion_report = ''
    after_conversion_report = ''
    registered_scilista_item = None
    report_components = {}
    scilista_items = []
    xc_status = 'not processed'
    is_db_generation = True
    converted = 0
    not_converted = 0
    total = 0

    dtd_files = xml_versions.DTDFiles('scielo', converter_env.version)

    pkg_name = os.path.basename(src_path)[:-4]

    if not os.path.isdir('./../log'):
        os.makedirs('./../log')
    log_package = './../log/' + datetime.now().isoformat().replace(':', '_') + os.path.basename(pkg_name)

    fs_utils.append_file(log_package, 'preparing')
    tmp_report_path, wrk_path, pkg_path, tmp_result_path = package_paths_preparation(src_path)
    final_result_path = tmp_result_path
    final_report_path = tmp_report_path

    fs_utils.append_file(log_package, 'normalized_package')
    pkg_articles, doc_file_info_items = normalized_package(src_path, tmp_report_path, wrk_path, pkg_path, converter_env.version)

    pkg = pkg_reports.PkgArticles(pkg_articles, pkg_path)

    journals_list = xc_models.JournalsList()
    journal = journals_list.get_journal(pkg.pkg_p_issn, pkg.pkg_e_issn, pkg.pkg_journal_title)

    fs_utils.append_file(log_package, 'identify_issue')
    issue_error_msg = pkg.identify_issue(converter_env.db_manager, pkg_name)
    #FIXME
    issue = None

    fs_utils.append_file(log_package, 'pkg.xml_list()')
    report_components['xml-files'] = pkg.xml_list()

    scilista_items.append(pkg.acron_issue_label)
    if issue_error_msg is not None:
        xc_status = 'rejected'
        report_components['issue-report'] = issue_error_msg
    else:

        fs_utils.append_file(log_package, 'db_article')
        db_article = xc_models.ArticleDB(converter_env.db_manager.db_isis, pkg.issue_files, xc_models.AopManager(converter_env.db_manager.db_isis, pkg.issue_files.journal_files))

        conversion = Conversion(pkg, db_article)

        fs_utils.append_file(log_package, 'conversion.evaluate_pkg_and_registered_items')
        conversion.evaluate_pkg_and_registered_items(converter_env.skip_identical_xml)

        pkg_validator = pkg_reports.ArticlesPkgReport(tmp_report_path, pkg, journal, issue, conversion.previous_registered_articles, is_db_generation)
        pkg_validator.xc_validations = conversion.xc_validations

        fs_utils.append_file(log_package, 'pkg_validator.overview_report()')
        report_components['pkg_overview'] = pkg_validator.overview_report()

        fs_utils.append_file(log_package, 'pkg_validator.references_overview_report()')
        report_components['pkg_overview'] += pkg_validator.references_overview_report()

        fs_utils.append_file(log_package, 'pkg_validator.sources_overview_report()')
        report_components['references'] = pkg_validator.sources_overview_report()

        fs_utils.append_file(log_package, 'pkg_validator.issue_report')
        report_components['issue-report'] = pkg_validator.issue_report

        conversion.blocking_errors = pkg_validator.blocking_errors

        fs_utils.append_file(log_package, 'conversion.initial_status_report')
        before_conversion_report = conversion.initial_status_report()

        if conversion.blocking_errors == 0:

            fs_utils.append_file(log_package, 'pkg_validator.validate_articles_pkg_xml_and_data')

            pkg_validator.validate_articles_pkg_xml_and_data(doc_file_info_items, dtd_files, False, conversion.selected_articles.keys())

            pkg_xml_fatal_errors = pkg_validator.pkg_xml_structure_validations.fatal_errors + pkg_validator.pkg_xml_content_validations.fatal_errors

            fs_utils.append_file(log_package, 'pkg_validator.detail_report')
            report_components['detail-report'] = pkg_validator.detail_report()

            fs_utils.append_file(log_package, 'conversion.convert_articles')
            registered_scilista_item = conversion.convert_articles(pkg_validator)

            fs_utils.append_file(log_package, 'conversion.pkg_xc_validations.report')
            report_components['conversion-report'] = conversion.pkg_xc_validations.report()
            if conversion.pkg_xc_validations.fatal_errors == 0:
                after_conversion_report = conversion.final_status_report()

            fs_utils.append_file(log_package, 'Conversion results')
            xc_results_report = report_status(_('Conversion results'), conversion.conversion_status, 'conversion')

            fs_utils.append_file(log_package, 'AOP status')
            aop_results_report = report_status(_('AOP status'), conversion.db.aop_manager.aop_sorted_by_status, 'aop-block')
            if len(aop_results_report) == 0:
                aop_results_report = _('this journal has no aop.')

            final_report_path = pkg.issue_files.base_reports_path
            final_result_path = pkg.issue_files.issue_path

            if registered_scilista_item is not None:
                fs_utils.append_file(log_package, 'pkg.issue_files.copy_files_to_local_web_app()')
                pkg.issue_files.copy_files_to_local_web_app()

            fs_utils.append_file(log_package, 'xc_status = get_xc_status()')
            xc_status = get_xc_status(registered_scilista_item, conversion.pkg_xc_validations.fatal_errors, pkg_xml_fatal_errors, conversion.blocking_errors)

            if conversion.db.aop_manager.aop_sorted_by_status.get('aop scilista item to update') is not None:
                for item in conversion.db.aop_manager.aop_sorted_by_status.get('aop scilista item to update'):
                    scilista_items.append(item)

            total = len(conversion.selected_articles) if conversion.selected_articles is not None else 0
            converted = len(conversion.conversion_status.get('converted', [])) if conversion.conversion_status.get('converted', []) is not None else 0
            not_converted = len(conversion.conversion_status.get('not converted', [])) if conversion.conversion_status.get('not converted', []) is not None else 0

        fs_utils.append_file(log_package, 'conversion.conclusion()')
        xc_conclusion_msg = ''.join([html_reports.p_message(item) for item in conversion.error_messages])
        xc_conclusion_msg += conclusion_message(total, converted, not_converted, xc_status, pkg.acron_issue_label)
        if len(after_conversion_report) == 0:
            after_conversion_report = xc_conclusion_msg

    if converter_env.is_windows:
        fs_utils.append_file(log_package, 'pkg_reports.processing_result_location')
        report_components['xml-files'] += pkg_reports.processing_result_location(final_result_path)

    report_components['db-overview'] = before_conversion_report + after_conversion_report
    report_components['summary-report'] = xc_conclusion_msg + xc_results_report + aop_results_report

    fs_utils.append_file(log_package, 'pkg_reports.format_complete_report')

    xc_validations = pkg_reports.format_complete_report(report_components)
    content = xc_validations.message
    if tmp_report_path in content:
        fs_utils.append_file(log_package, 'content.replace(tmp_report_path, final_report_path)')
        content = content.replace(tmp_report_path, final_report_path)

    report_location = final_report_path + '/xml_converter.html'
    pkg_reports.save_report(report_location, [_('XML Conversion (XML to Database)'), pkg.acron_issue_label], content)

    if not converter_env.is_windows:
        fs_utils.append_file(log_package, 'format_reports_for_web')
        format_reports_for_web(final_report_path, pkg_path, pkg.acron_issue_label.replace(' ', '/'))

    if tmp_result_path != final_result_path:
        fs_utils.delete_file_or_folder(tmp_result_path)

    fs_utils.append_file(log_package, 'antes de return - convert_package')

    os.unlink(log_package)
    return (scilista_items, xc_status, xc_validations.statistics_message(), report_location)
Пример #5
0
 def register(self, downloaded, destination):
     fs_utils.append_file(self.filename, '\t'.join([utils.now()[0], utils.now()[1], downloaded, destination]))