def _identify(ctx, src, failfast, verbose):
"""Identify files for import.
Walk the SRC list of files or directories and report each file
identified by one of the configured importers. When verbose
output is requested, also print the account name associated to the
document by the importer.
"""
log = utils.logger(verbose)
errors = exceptions.ExceptionsTrap(log)
for filename in _walk(src, log):
with errors:
importer = identify.identify(ctx.importers, filename)
if not importer:
log('') # Newline.
continue
# Signal processing of this document.
log(' ...', nl=False)
# When verbose output is requested, get the associated account.
account = importer.file_account(
cache.get_file(filename)) if verbose else None
log(' OK', fg='green')
log(f' {importer.name():}')
log(f' {account:}', 1)
if failfast and errors:
break
if errors:
sys.exit(1)
def _extract(ctx, src, output, existing, reverse, failfast, quiet):
"""Extract transactions from documents.
Walk the SRC list of files or directories and extract the ledger
entries from each file identified by one of the configured
importers. The entries are written to the specified output file
or to the standard output in Beancount ledger format in sections
associated to the source document.
"""
verbosity = -quiet
log = utils.logger(verbosity, err=True)
errors = exceptions.ExceptionsTrap(log)
# Load the ledger, if one is specified.
existing_entries = loader.load_file(existing)[0] if existing else []
extracted = []
for filename in _walk(src, log):
with errors:
importer = identify.identify(ctx.importers, filename)
if not importer:
log('') # Newline.
continue
# Signal processing of this document.
log(' ...', nl=False)
# Extract entries.
entries = extract.extract_from_file(importer, filename,
existing_entries)
extracted.append((filename, entries))
log(' OK', fg='green')
if failfast and errors:
break
# Invoke hooks.
hooks = [extract.find_duplicate_entries
] if ctx.hooks is None else ctx.hooks
for func in hooks:
extracted = func(extracted, existing_entries)
# Reverse sort order, if requested.
if reverse:
for filename, entries in extracted:
entries.reverse()
# Serialize entries.
extract.print_extracted_entries(extracted, output)
if errors:
sys.exit(1)
def _run(ctx,
documents: List[str],
expected: str,
verbose: int,
quiet: int,
generate: bool = False,
failfast: bool = False,
force: bool = False):
"""Implement the test and generate commands."""
assert len(ctx.importers) == 1
importer = ctx.importers[0]
verbosity = verbose - quiet
log = utils.logger(verbosity)
failures = 0
for doc in utils.walk(documents):
if doc.endswith('.beancount'):
continue
# Unless verbose mode is enabled, do not output a newline so
# the test result is printed on the same line as the test
# document filename.
log(f'* {doc}', nl=verbosity > 0)
# Compute the path to the expected output file.
expected_filename = f"{doc}.beancount"
if expected:
expected_filename = path.join(expected,
path.basename(expected_filename))
# Run the importer's identify() method.
if importer.identify(doc):
account, date, name, entries = run_importer(importer, doc)
log(f' {expected_filename}', 1)
if account is None:
failures += 1
log(' ERROR', fg='red')
log(' ValueError: account() should not return None')
continue
log(
' {}/{:%Y-%m-%d}-{}'.format(account.replace(":", "/"), date
or utils.getmdate(doc), name
or path.basename(doc)), 1)
if generate:
try:
write_expected_file(expected_filename,
account,
date,
name,
entries,
force=force)
except FileExistsError as ex:
failures += 1
log(' ERROR', fg='red')
log(' FileExistsError: {}'.format(ex.filename))
continue
log(' OK', fg='green')
continue
try:
diff = compare_expected(expected_filename, account, date, name,
entries)
except FileNotFoundError:
# The importer has positively identified a document
# for which there is no expecred output file.
failures += 1
log(' ERROR', fg='red')
log(' ExpectedOutputFileNotFound')
continue
if diff:
# Test failure. Log an error.
failures += 1
log(' ERROR', fg='red')
if verbosity >= 0:
sys.stdout.writelines(diff)
sys.stdout.write(os.linesep)
continue
log(' PASSED', fg='green')
elif path.exists(expected_filename):
# The importer has not identified a document it should have.
failures += 1
log(' ERROR', fg='red')
log(' DocumentNotIdentified')
else:
# Ignore files that are not positively identified by the
# importer and for which there is no expected output file.
log(' IGNORED')
if failfast and failures:
break
if failures:
sys.exit(1)
def test_logger(self):
logger = utils.logger()
self.assertIsInstance(logger, types.FunctionType)
logger = utils.logger(logging.INFO, err=True)
self.assertIsInstance(logger, types.FunctionType)
def _archive(ctx, src, destination, dry_run, overwrite, failfast):
"""Archive documents.
Walk the SRC list of files or directories and move each file
identified by one of the configured importers in a directory
hierarchy mirroring the structure of the accounts associated to
the documents and with a file name composed by the document date
and document name returned by the importer.
Documents are moved to their filing location only when no errors
are encountered processing all the input files. Documents in the
destination directory are not overwritten, unless the --force
option is used. When the directory hierarchy root is not
specified with the --destination DIR options, it is assumed to be
directory in which the ingest script is located.
"""
# If the output directory is not specified, move the files at the
# root where the import script is located. Providing this default
# seems better than using a required option.
if destination is None:
import __main__
destination = os.path.dirname(os.path.abspath(__main__.__file__))
log = utils.logger()
errors = exceptions.ExceptionsTrap(log)
renames = []
for filename in _walk(src, log):
with errors:
importer = identify.identify(ctx.importers, filename)
if not importer:
log('') # Newline.
continue
# Signal processing of this document.
log(' ...', nl=False)
destpath = archive.filepath(importer, filename)
# Prepend destination directory path.
destpath = os.path.join(destination, destpath)
# Check for destination filename collisions.
collisions = [src for src, dst in renames if dst == destpath]
if collisions:
raise exceptions.Error('Collision in destination file path.',
destpath)
# Check if the destination file already exists.
if not overwrite and os.path.exists(destpath):
raise exceptions.Error('Destination file already exists.',
destpath)
renames.append((filename, destpath))
log(' OK', fg='green')
log(f' {destpath:}')
if failfast and errors:
break
# If there are any errors, stop here.
if errors:
log('# Errors detected: documents will not be filed.')
sys.exit(1)
if not dry_run:
for src, dst in renames:
archive.move(src, dst)
