def generate_report(cls, results: List[speedwagon.tasks.Result],
**user_args: Union[str, bool]) -> Optional[str]:
results_sorted = sorted(results, key=lambda x: x.source.__name__)
_result_grouped: Iterator[Tuple[
Any, Iterator[speedwagon.tasks.Result]]] = itertools.groupby(
results_sorted, lambda x: x.source)
results_grouped = {
key: [i.data for i in group]
for key, group in _result_grouped
}
manifest_report = results_grouped[HathiManifestGenerationTask][0]
error_results: List[hathi_result.Result] = []
error_results += cls._get_result(results_grouped,
HathiCheckMissingPackageFilesTask)
error_results += cls._get_result(results_grouped,
HathiCheckMissingComponentsTask)
error_results += cls._get_result(results_grouped,
ValidateExtraSubdirectoriesTask)
error_results += cls._get_result(results_grouped,
ValidateChecksumsTask)
error_results += cls._get_result(results_grouped, ValidateMarcTask)
error_results += cls._get_result(results_grouped, ValidateYMLTask)
error_report = hathi_reporter.get_report_as_str(error_results, 70)
# ########################### Warnings ###########################
warning_results: List[hathi_result.Result] = []
warning_results += cls._get_result(results_grouped,
PackageNamingConventionTask)
warning_report = hathi_reporter.get_report_as_str(warning_results, 70)
report = f"\n" \
f"Report:\n" \
f"{manifest_report}\n" \
f"\n"
if error_report:
report = f"{report}\n" \
f"\n" \
f"Errors:\n" \
f"{error_report}\n"
if warning_results:
report = f"{report}\n" \
f"\n" \
f"Warnings:\n" \
f"{warning_report}\n"
return report
def build_report(self) -> str:
# ############################ Errors ############################
error_report = self.generate_error_report()
# ########################### Warnings ###########################
warning_results: List[hathi_result.Result] = []
warning_results += self._get_result(self.results,
PackageNamingConventionTask)
warning_report = hathi_reporter.get_report_as_str(
warning_results, self.line_length)
report_lines: List[str] = [
"",
"Report:",
]
if HathiManifestGenerationTask in self.results:
report_lines += [
typing.cast(str, self.results[HathiManifestGenerationTask][0]),
""
]
if error_report:
report_lines += ["", "Errors:", error_report, ""]
if warning_results:
report_lines += ["", "Warnings:", warning_report, ""]
return "\n".join(report_lines)
def generate_error_report(self) -> str:
error_results: List[hathi_result.Result] = []
for task in self._tasks_performed:
error_results += self._get_result(self.results, task)
return hathi_reporter.get_report_as_str(error_results,
self.line_length)
def _script(self):
errors = []
path = self.args["Path"].value
for i, pkg in enumerate(package.get_dirs(path)):
if self._abort_flag.is_set():
self.logger.warning("Aborted")
self.announce(SignalTypes.FAILED, "Script was aborted")
break
self.logger.info("{}) Checking {}".format(i + 1, pkg))
errors += process.process_directory(pkg)
else:
self.logger.info("Checking Finished")
# report_str = report.get_report_as_str(errors)
self.announce(SignalTypes.SUCCESS,
report.get_report_as_str(errors, 37))
console_reporter = report.Reporter(report.LogReporter(self.logger))
console_reporter.report(report.get_report_as_str(errors, width=80))
def main():
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
parser = get_parser()
args = parser.parse_args()
configure_logging.configure_logger(debug_mode=args.debug,
log_file=args.log_debug)
errors = []
batch_manifest_builder = manifest.PackageManifestDirector()
for pkg in package.get_dirs(args.path):
logger.info("Creating a manifest for {}".format(pkg))
package_builder = batch_manifest_builder.add_package(pkg)
for root, dirs, files in os.walk(pkg):
for file_name in files:
package_builder.add_file(file_name)
logger.info("Checking {}".format(pkg))
# Validate missing files
logger.debug("Looking for missing package files in {}".format(pkg))
missing_files_errors = process.run_validation(
validator.ValidateMissingFiles(path=pkg))
if not missing_files_errors:
logger.info("Found no missing package files in {}".format(pkg))
else:
for error in missing_files_errors:
logger.info(error.message)
errors.append(error)
# Look for missing components
extensions = [".txt", ".jp2"]
if args.check_ocr:
extensions.append(".xml")
logger.debug("Looking for missing component files in {}".format(pkg))
missing_files_errors = process.run_validation(
validator.ValidateComponents(pkg, "^\d{8}$", *extensions))
if not missing_files_errors:
logger.info("Found no missing component files in {}".format(pkg))
else:
for error in missing_files_errors:
logger.info(error.message)
errors.append(error)
# exit()
# Validate extra subdirectories
logger.debug("Looking for extra subdirectories in {}".format(pkg))
extra_subdirectories_errors = process.run_validation(
validator.ValidateExtraSubdirectories(path=pkg))
if not extra_subdirectories_errors:
pass
else:
for error in extra_subdirectories_errors:
errors.append(error)
# Validate Checksums
checksum_report = os.path.join(pkg, "checksum.md5")
checksum_report_errors = process.run_validation(
validator.ValidateChecksumReport(pkg, checksum_report))
if not checksum_report_errors:
logger.info("All checksums in {} successfully validated".format(
checksum_report))
else:
for error in checksum_report_errors:
errors.append(error)
# Validate Marc
marc_file = os.path.join(pkg, "marc.xml")
marc_errors = process.run_validation(validator.ValidateMarc(marc_file))
if not marc_errors:
logger.info("{} successfully validated".format(marc_file))
else:
for error in marc_errors:
errors.append(error)
# Validate YML
yml_file = os.path.join(pkg, "meta.yml")
meta_yml_errors = process.run_validation(
validator.ValidateMetaYML(yaml_file=yml_file,
path=pkg,
required_page_data=True))
if not meta_yml_errors:
logger.info("{} successfully validated".format(yml_file))
else:
for error in meta_yml_errors:
errors.append(error)
#
# Validate ocr files
if args.check_ocr:
ocr_errors = process.run_validation(
validator.ValidateOCRFiles(path=pkg))
if not ocr_errors:
logger.info("No validation errors found in ".format(pkg))
else:
for error in ocr_errors:
errors.append(error)
batch_manifest = batch_manifest_builder.build_manifest()
manifest_report = manifest.get_report_as_str(batch_manifest, width=80)
console_reporter2 = report.Reporter(report.ConsoleReporter())
validation_report = report.get_report_as_str(errors)
console_reporter2.report(manifest_report)
console_reporter2.report(validation_report)
if args.report_name:
file_reporter = report.Reporter(
report.FileOutputReporter(args.report_name))
file_reporter.report(validation_report)
def step_impl(context):
"""
Args:
context (behave.runner.Context):
"""
context.string_report = report.get_report_as_str(context.summary, width=80)
