def test_categorizer_two_arguments(self, filename):
"""\
Date,Amount,Payee,Description
6/2/2020,30.00,"Payee here","Description"
7/2/2020,-25.00,"Supermarket","Groceries"
"""
file = cache.get_file(filename)
def categorizer(txn, row):
txn = txn._replace(payee=row[2])
txn.meta['source'] = pformat(row)
return txn
importer = csv.Importer(
{
Col.DATE: 'Date',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount'
},
'Assets:Bank',
'EUR', ('Date,Amount,Payee,Description'),
categorizer=categorizer,
institution='foobar')
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2020-06-02 * "Payee here" "Description"
source: "['6/2/2020', '30.00', 'Supermarket', 'Groceries']"
Assets:Bank 30.00 EUR
2020-07-02 * "Supermarket" "Groceries"
source: "['7/2/2020', '-25.00', 'Supermarket', 'Groceries']"
Assets:Bank -25.00 EUR
""", entries)
def _identify(ctx, src, failfast, verbose):
"""Identify files for import.
Walk the SRC list of files or directories and report each file
identified by one of the configured importers. When verbose
output is requested, also print the account name associated to the
document by the importer.
"""
log = utils.logger(verbose)
errors = exceptions.ExceptionsTrap(log)
for filename in _walk(src, log):
with errors:
importer = identify.identify(ctx.importers, filename)
if not importer:
log('') # Newline.
continue
# Signal processing of this document.
log(' ...', nl=False)
# When verbose output is requested, get the associated account.
account = importer.file_account(
cache.get_file(filename)) if verbose else None
log(' OK', fg='green')
log(f' {importer.name():}')
log(f' {account:}', 1)
if failfast and errors:
break
if errors:
sys.exit(1)
def filename(self, filepath):
filename = self.importer.file_name(cache.get_file(filepath))
# The current implementation of the archive command does not
# modify the filename returned by the importer. This preserves
# backward compatibility with the old implementation of the
# command.
return misc_utils.idify(filename) if filename else None
def test_date_formats(self, filename):
"""\
Posting,Description,Amount
11/7/2016,A,2
12/7/2016,B,3
13/7/2016,C,4
"""
file = cache.get_file(filename)
importer = csv.Importer(
{
Col.DATE: 'Posting',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount'
},
'Assets:Bank',
'EUR', [],
dateutil_kwds={'dayfirst': True})
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2016-07-11 * "A"
Assets:Bank 2 EUR
2016-07-12 * "B"
Assets:Bank 3 EUR
2016-07-13 * "C"
Assets:Bank 4 EUR
""", entries)
def test_categorizer_one_argument(self, filename):
"""\
Date,Amount,Payee,Description
6/2/2020,30.00,"Payee here","Description"
7/2/2020,-25.00,"Supermarket","Groceries"
"""
file = cache.get_file(filename)
def categorizer(txn):
if txn.narration == "Groceries":
txn.postings.append(
data.Posting("Expenses:Groceries",
-txn.postings[0].units,
None, None, None, None))
return txn
importer = csvimp.Importer({Col.DATE: 'Date',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount'},
'Assets:Bank',
'EUR',
('Date,Amount,Payee,Description'),
categorizer=categorizer,
institution='foobar')
entries = importer.extract(file)
self.assertEqualEntries(r"""
2020-06-02 * "Description"
Assets:Bank 30.00 EUR
2020-07-02 * "Groceries"
Assets:Bank -25.00 EUR
Expenses:Groceries 25.00 EUR
""", entries)
def extract_from_file(importer, filename, existing_entries):
"""Import entries from a document.
Args:
importer: The importer instance to handle the document.
filename: Filesystem path to the document.
existing_entries: Existing entries.
Returns:
The list of imported entries.
"""
file = cache.get_file(filename)
# Support calling without the existing entries.
kwargs = {}
if 'existing_entries' in inspect.signature(importer.extract).parameters:
kwargs['existing_entries'] = existing_entries
entries = importer.extract(file, **kwargs)
if entries is None:
entries = []
# Make sure the newly imported entries are sorted; don't trust the importer.
entries.sort(key=data.entry_sortkey)
# Ensure that the entries are typed correctly.
for entry in entries:
data.sanity_check_types(entry)
return entries
def test_zero_balance_produces_assertion(self, filename):
"""\
Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #,
DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,0,,
"""
file = cache.get_file(filename)
importer = csv.Importer(
{
Col.DATE: 'Posting Date',
Col.NARRATION1: 'Description',
Col.NARRATION2: 'Check or Slip #',
Col.AMOUNT: 'Amount',
Col.BALANCE: 'Balance',
Col.DRCR: 'Details'
},
'Assets:Bank',
'USD', ('Details,Posting Date,"Description",Amount,'
'Type,Balance,Check or Slip #,'),
institution='chafe')
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2016-03-18 * "Payment to Chafe card ending in 1234 03/18"
Assets:Bank -2680.89 USD
2016-03-19 balance Assets:Bank 0 USD
""", entries)
def test_newliens(self):
content = textwrap.dedent("""\
Date,Description,Amount
2020-07-03,A,2
2020-07-03,B,3
""")
importer = csv.Importer(
{
Col.DATE: 'Date',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount'
}, 'Assets:Bank', 'EUR', [])
for nl in '\n', '\r\n', '\r':
with tempfile.NamedTemporaryFile('w') as temp:
temp.write(content.replace('\n', nl))
temp.flush()
entries = importer.extract(cache.get_file(temp.name))
self.assertEqualEntries(
"""
2020-07-03 * "A"
Assets:Bank 2 EUR
2020-07-03 * "B"
Assets:Bank 3 EUR
""", entries)
def test_zero_balance_assertion_is_added_with_currency_field(
self, filename):
"""\
Posting Date,"Description",Amount,Currency,Balance
3/18/2016,"1st Payment in GBP",-1.00,GBP,-1
3/18/2016,"1st Payment in PLN",-1,PLN,-1
3/18/2016,"1st Payment in ZAR",-1.0,ZAR,-1
3/19/2016,"2nd Payment in GBP",-2,GBP,-3
3/19/2016,"2nd Payment in Main Currency",-2.00,,-3
3/20/2016,"3rd Payment in GBP",-3,GBP,-6
3/21/2016,"4th Payment in GBP",6,GBP,0
"""
file = cache.get_file(filename)
importer = csv.Importer(
{
Col.DATE: 'Posting Date',
Col.NARRATION1: 'Description',
Col.AMOUNT: 'Amount',
Col.CURRENCY: 'Currency',
Col.BALANCE: 'Balance'
}, 'Assets:Bank', 'PLN')
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2016-03-18 * "1st Payment in GBP"
Assets:Bank -1.00 GBP
2016-03-18 * "1st Payment in PLN"
Assets:Bank -1 PLN
2016-03-18 * "1st Payment in ZAR"
Assets:Bank -1.0 ZAR
2016-03-19 * "2nd Payment in GBP"
Assets:Bank -2 GBP
2016-03-19 * "2nd Payment in Main Currency"
Assets:Bank -2.00 PLN
2016-03-20 * "3rd Payment in GBP"
Assets:Bank -3 GBP
2016-03-21 * "4th Payment in GBP"
Assets:Bank 6 GBP
2016-03-20 balance Assets:Bank -3 PLN
2016-03-19 balance Assets:Bank -1 ZAR
2016-03-22 balance Assets:Bank 0 GBP
""", entries)
def extract_from_file(filename, importer,
existing_entries=None,
min_date=None,
allow_none_for_tags_and_links=False):
"""Import entries from file 'filename' with the given matches,
Also cross-check against a list of provided 'existing_entries' entries,
de-duplicating and possibly auto-categorizing.
Args:
filename: The name of the file to import.
importer: An importer object that matched the file.
existing_entries: A list of existing entries parsed from a ledger, used to
detect duplicates and automatically complete or categorize transactions.
min_date: A date before which entries should be ignored. This is useful
when an account has a valid check/assert; we could just ignore whatever
comes before, if desired.
allow_none_for_tags_and_links: A boolean, whether to allow plugins to
generate Transaction objects with None as value for the 'tags' or 'links'
attributes.
Returns:
A list of new imported entries.
Raises:
Exception: If there is an error in the importer's extract() method.
"""
# Extract the entries.
file = cache.get_file(filename)
# Note: Let the exception through on purpose. This makes developing
# importers much easier by rendering the details of the exceptions.
#
# Note: For legacy support, support calling without the existing entries.
kwargs = {}
if 'existing_entries' in inspect.signature(importer.extract).parameters:
kwargs['existing_entries'] = existing_entries
new_entries = importer.extract(file, **kwargs)
if not new_entries:
return []
# Make sure the newly imported entries are sorted; don't trust the importer.
new_entries.sort(key=data.entry_sortkey)
# Ensure that the entries are typed correctly.
for entry in new_entries:
data.sanity_check_types(entry, allow_none_for_tags_and_links)
# Filter out entries with dates before 'min_date'.
if min_date:
new_entries = list(itertools.dropwhile(lambda x: x.date < min_date,
new_entries))
return new_entries
def test_column_types(self, filename):
# pylint: disable=line-too-long
"""\
Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #,
DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,,
CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,,
DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,,
DEBIT,3/6/2016,"ATM WITHDRAWAL 001234 03/8888 DELANC",-60.00,ATM,3650.02,,
CREDIT,3/5/2016,"CA STATE NYSTTAXRFD PPD ID: 1111111111",110.00,ACH_CREDIT,3710.02,,
DEBIT,3/4/2016,"BOOGLE WALLET US000NEI9T WEB ID: C234567890",-1300.00,ACH_DEBIT,3600.02,,
"""
file = cache.get_file(filename)
importer = csv.Importer(
{
Col.DATE: 'Posting Date',
Col.NARRATION1: 'Description',
Col.NARRATION2: 'Check or Slip #',
Col.AMOUNT: 'Amount',
Col.BALANCE: 'Balance',
Col.DRCR: 'Details'
},
'Assets:Bank',
'USD', ('Details,Posting Date,"Description",Amount,'
'Type,Balance,Check or Slip #,'),
institution='chafe')
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2016-03-18 * "Payment to Chafe card ending in 1234 03/18"
Assets:Bank -2680.89 USD
2016-03-15 * "EMPLOYER INC DIRECT DEP PPD ID: 1111111111"
Assets:Bank 2590.73 USD
2016-03-14 * "INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789"
Assets:Bank -150.00 USD
2016-03-06 * "ATM WITHDRAWAL 001234 03/8888 DELANC"
Assets:Bank -60.00 USD
2016-03-05 * "CA STATE NYSTTAXRFD PPD ID: 1111111111"
Assets:Bank 110.00 USD
2016-03-04 * "BOOGLE WALLET US000NEI9T WEB ID: C234567890"
Assets:Bank -1300.00 USD
2016-03-19 balance Assets:Bank 3409.86 USD
""", entries)
def test_currency_and_balances_where_there_are_multiple_currency_transactions(
self, filename):
"""\
Posting Date,"Description",Amount,Currency,Balance
3/18/2016,"1st Payment in GBP",-1.00,GBP,-1
3/18/2016,"1st Payment in PLN",-1,PLN,-1
3/18/2016,"1st Payment in ZAR",-1.0,ZAR,-1
3/19/2016,"2nd Payment in GBP",-2,GBP,-3
3/19/2016,"2nd Payment in Main Currency",-2.00,,-3
3/20/2016,"3rd Payment in GBP",-3,GBP,-6
"""
file = cache.get_file(filename)
importer = csv.Importer(
{
Col.DATE: 'Posting Date',
Col.NARRATION1: 'Description',
Col.AMOUNT: 'Amount',
Col.CURRENCY: 'Currency',
Col.BALANCE: 'Balance'
}, 'Assets:Bank', 'PLN')
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2016-03-18 * "1st Payment in GBP"
Assets:Bank -1.00 GBP
2016-03-18 * "1st Payment in PLN"
Assets:Bank -1 PLN
2016-03-18 * "1st Payment in ZAR"
Assets:Bank -1.0 ZAR
2016-03-19 * "2nd Payment in GBP"
Assets:Bank -2 GBP
2016-03-19 * "2nd Payment in Main Currency"
Assets:Bank -2.00 PLN
2016-03-20 * "3rd Payment in GBP"
Assets:Bank -3 GBP
2016-03-20 balance Assets:Bank -3 PLN
2016-03-19 balance Assets:Bank -1 ZAR
2016-03-21 balance Assets:Bank -6 GBP
""", entries)
def identify(importers_list, files_or_directories):
"""Run the identification loop.
Args:
importers_list: A list of importer instances.
files_or_directories: A list of strings, files or directories.
"""
logfile = sys.stdout
for filename, importers in find_imports(importers_list, files_or_directories,
logfile=logfile):
file = cache.get_file(filename)
for importer in importers:
logfile.write('Importer: {}\n'.format(importer.name() if importer else '-'))
logfile.write('Account: {}\n'.format(importer.file_account(file)))
logfile.write('\n')
def test_explict_encoding_utf8(self, filename):
"""\
Posting,Description,Amount
2020/08/08,🍏,2
"""
file = cache.get_file(filename)
importer = csvimp.Importer({Col.DATE: 'Posting',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount'},
'Assets:Bank', 'EUR', [],
encoding='utf-8')
entries = importer.extract(file)
self.assertEqualEntries(r"""
2020-08-08 * "🍏"
Assets:Bank 2 EUR
""", entries)
def filepath(importer, filepath: str) -> str:
"""Compute filing path for a document.
The path mirrors the structure of the accounts associated to the
documents and with a file name composed by the document date and
document name returned by the importer.
Args:
importer: The importer instance to handle the document.
filepath: Filesystem path to the document.
Returns:
Filing tree location where to store the document.
Raises:
beangulp.exceptions.Error: The canonical file name returned by
the importer contains a path separator charachter or seems to
contain a date.
"""
file = cache.get_file(filepath)
# Get the account corresponding to the file.
account = importer.file_account(file)
filename = importer.file_name(file) or os.path.basename(file.name)
date = importer.file_date(file) or utils.getmdate(file.name)
# The returned filename cannot contain the file path separator character.
if os.sep in filename:
raise Error("The filename contains path separator character.")
if re.match(r'\d\d\d\d-\d\d-\d\d', filename):
raise Error("The filename contains what looks like a date.")
# Remove whitespace and other funny characters from the filename.
# TODO(dnicolodi): This should probably be importer responsibility.
filename = misc_utils.idify(filename)
# Prepend account directory and date prefix.
filename = os.path.join(account.replace(':', os.sep),
f'{date:%Y-%m-%d}.{filename:}')
return filename
def find_imports(importer_config, files_or_directories, logfile=None):
"""Given an importer configuration, search for files that can be imported in the
list of files or directories, run the signature checks on them and return a list
of (filename, importers), where 'importers' is a list of importers that matched
the file.
Args:
importer_config: a list of importer instances that define the config.
files_or_directories: a list of files of directories to walk recursively and
hunt for files to import.
logfile: A file object to write log entries to, or None, in which case no log is
written out.
Yields:
Triples of filename found, textified contents of the file, and list of
importers matching this file.
"""
# Iterate over all files found; accumulate the entries by identification.
for filename in file_utils.find_files(files_or_directories):
if logfile is not None:
logfile.write(SECTION.format(filename))
logfile.write('\n')
# Skip files that are simply too large.
size = path.getsize(filename)
if size > FILE_TOO_LARGE_THRESHOLD:
logging.warning("File too large: '{}' ({} bytes); skipping.".format(
filename, size))
continue
# For each of the sources the user has declared, identify which
# match the text.
file = cache.get_file(filename)
matching_importers = []
for importer in importer_config:
try:
matched = importer.identify(file)
if matched:
matching_importers.append(importer)
except Exception as exc:
logging.exception("Importer %s.identify() raised an unexpected error: %s",
importer.name(), exc)
yield (filename, matching_importers)
def test_links(self, filename):
"""\
Date,Description,Amount,Link
2020-07-03,A,2,
2020-07-03,B,3,123
"""
file = cache.get_file(filename)
importer = csvimp.Importer({Col.DATE: 'Date',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount',
Col.REFERENCE_ID: 'Link'},
'Assets:Bank', 'EUR', [])
entries = importer.extract(file)
self.assertEqualEntries(r"""
2020-07-03 * "A"
Assets:Bank 2 EUR
2020-07-03 * "B" ^123
Assets:Bank 3 EUR
""", entries)
def test_tags(self, filename):
"""\
Date,Description,Amount,Tag
2020-07-03,A,2,
2020-07-03,B,3,foo
"""
file = cache.get_file(filename)
importer = csvimp.Importer({Col.DATE: 'Date',
Col.NARRATION: 'Description',
Col.AMOUNT: 'Amount',
Col.TAG: 'Tag'},
'Assets:Bank', 'EUR', [])
entries = importer.extract(file)
self.assertEqualEntries(r"""
2020-07-03 * "A"
Assets:Bank 2 EUR
2020-07-03 * "B" #foo
Assets:Bank 3 EUR
""", entries)
def test_main_currency_should_be_used_when_no_currency_is_specified(
self, filename):
"""\
Posting Date,"Description",Amount,Currency,Balance
3/18/2016,"1st Payment",-1.00,,-1
3/18/2016,"2nd Payment",-1.0,,-2
3/19/2016,"3rd Payment",-2,,-4
3/20/2016,"4th Payment",-3,,-7
"""
file = cache.get_file(filename)
importer = csv.Importer(
{
Col.DATE: 'Posting Date',
Col.NARRATION1: 'Description',
Col.AMOUNT: 'Amount',
Col.CURRENCY: 'Currency',
Col.BALANCE: 'Balance'
}, 'Assets:Bank', 'PLN')
entries = importer.extract(file)
self.assertEqualEntries(
r"""
2016-03-18 * "1st Payment"
Assets:Bank -1.00 PLN
2016-03-18 * "2nd Payment"
Assets:Bank -1.0 PLN
2016-03-19 * "3rd Payment"
Assets:Bank -2 PLN
2016-03-20 * "4th Payment"
Assets:Bank -3 PLN
2016-03-21 balance Assets:Bank -7 PLN
""", entries)
def identify(importers, filepath: str):
"""Identify the correct importer to handle a document.
Args:
importers: List of importer instances.
filepath: Filesystem path to the document.
Returns:
The correct importer to handle the document or None if no importer
matches the document.
Raises:
beangulp.exceptions.Error: More than one importer matched the file.
"""
file = cache.get_file(filepath)
match = [importer for importer in importers if importer.identify(file)]
if len(match) > 1:
match = [importer.name() for importer in match]
raise Error('Document identified by more than one importer.', *match)
return match[0] if match else None
def extract(self, filepath, existing):
p = inspect.signature(self.importer.extract).parameters
if len(p) > 1:
return self.importer.extract(cache.get_file(filepath), existing)
return self.importer.extract(cache.get_file(filepath))
def date(self, filepath):
return self.importer.file_date(cache.get_file(filepath))
def _run(ctx,
documents: List[str],
expected: str,
verbose: int,
quiet: int,
generate: bool = False,
failfast: bool = False,
force: bool = False):
"""Implement the test and generate commands."""
assert len(ctx.importers) == 1
importer = ctx.importers[0]
verbosity = verbose - quiet
log = utils.logger(verbosity)
failures = 0
for doc in utils.walk(documents):
if doc.endswith('.beancount'):
continue
# Unless verbose mode is enabled, do not output a newline so
# the test result is printed on the same line as the test
# document filename.
log(f'* {doc}', nl=verbosity > 0)
# Compute the path to the expected output file.
expected_filename = f"{doc}.beancount"
if expected:
expected_filename = path.join(expected,
path.basename(expected_filename))
# Use the in-memory cache.
# TODO(blais): This will get replaced by an on-disk cache.
cached_file = cache.get_file(path.abspath(doc))
# Run the importer's identify() method.
if importer.identify(cached_file):
account, date, name, entries = run_importer(importer, cached_file)
log(f' {expected_filename}', 1)
if account is None:
failures += 1
log(' ERROR', fg='red')
log(' ValueError: account() should not return None')
continue
log(
' {}/{:%Y-%m-%d}-{}'.format(account.replace(":", "/"), date
or utils.getmdate(doc), name
or path.basename(doc)), 1)
if generate:
try:
write_expected_file(expected_filename,
account,
date,
name,
entries,
force=force)
except FileExistsError as ex:
failures += 1
log(' ERROR', fg='red')
log(' FileExistsError: {}'.format(ex.filename))
continue
log(' OK', fg='green')
continue
try:
diff = compare_expected(expected_filename, account, date, name,
entries)
except FileNotFoundError:
# The importer has positively identified a document
# for which there is no expecred output file.
failures += 1
log(' ERROR', fg='red')
log(' ExpectedOutputFileNotFound')
continue
if diff:
# Test failure. Log an error.
failures += 1
log(' ERROR', fg='red')
if verbosity >= 0:
sys.stdout.writelines(diff)
sys.stdout.write(os.linesep)
continue
log(' PASSED', fg='green')
elif path.exists(expected_filename):
# The importer has not identified a document it should have.
failures += 1
log(' ERROR', fg='red')
log(' DocumentNotIdentified')
else:
# Ignore files that are not positively identified by the
# importer and for which there is no expected output file.
log(' IGNORED')
if failfast and failures:
break
if failures:
sys.exit(1)
def account(self, filepath):
return self.importer.file_account(cache.get_file(filepath))
def identify(self, filepath):
return self.importer.identify(cache.get_file(filepath))
def account(self, filepath):
return self.filing.file_account(cache.get_file(filepath))
def date(self, filepath):
return self.base.file_date(cache.get_file(filepath))
def extract(self, filepath, existing=None):
account = self.account(filepath)
return self.base._do_extract(cache.get_file(filepath), account, existing)
