def test_extract_from_file__raises_exception(self):
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(
side_effect=ValueError("Unexpected error!"))
with self.assertRaises(ValueError):
extract.extract_from_file('/tmp/blabla.ofx', imp, [])
def test_extract_from_file_ensure_sanity(self):
entries, errors, options = parser.parse_string('''
1970-01-01 * "Test"
Assets:Tests 1.00 USD
''')
# Break something.
entries[-1] = entries[-1]._replace(narration=42)
importer = mock.MagicMock(wraps=self.importer)
importer.extract.return_value = entries
with self.assertRaises(AssertionError):
extract.extract_from_file(importer, path.abspath('test.csv'), [])
def test_extract_from_file__ensure_sanity(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
# Break something.
entries[-1] = entries[-1]._replace(narration=42)
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
with self.assertRaises(AssertionError):
extract.extract_from_file('blabla.ofx', imp)
def test_extract_from_file__min_date(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-03 * "C"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
new_entries = extract.extract_from_file('/tmp/blabla.ofx',
imp,
min_date=datetime.date(
2016, 2, 2))
self.assertEqual(2, len(new_entries))
self.assertEqual(
[datetime.date(2016, 2, 2),
datetime.date(2016, 2, 3)], [entry.date for entry in new_entries])
def test_extract_from_file__explicitly_marked_duplicates_entries(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
entries[1].meta[extract.DUPLICATE_META] = True
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
new_entries, dup_entries = extract.extract_from_file(
'/tmp/blabla.ofx', imp, [])
self.assertEqual(1, len(dup_entries))
self.assertEqual(
[datetime.date(2016, 2, 1),
datetime.date(2016, 2, 2)], [entry.date for entry in new_entries])
self.assertEqual([datetime.date(2016, 2, 2)],
[entry.date for entry in dup_entries])
def run_importer(
importer: Importer, document: str
) -> Tuple[data.Account, Optional[datetime.date], Optional[str], data.Entries]:
"""Run the various importer methods on the given cached file."""
account = importer.account(document)
date = importer.date(document)
name = importer.filename(document)
entries = extract.extract_from_file(importer, document, [])
return account, date, name, entries
def run_importer(
importer: ImporterProtocol, cfile: cache._FileMemo
) -> Tuple[data.Account, Optional[datetime.date], Optional[str], data.Entries]:
"""Run the various importer methods on the given cached file."""
account = importer.file_account(cfile)
date = importer.file_date(cfile)
name = importer.file_name(cfile)
entries = extract.extract_from_file(importer, cfile.name, None)
return account, date, name, entries
def _extract(ctx, src, output, existing, reverse, failfast, quiet):
"""Extract transactions from documents.
Walk the SRC list of files or directories and extract the ledger
entries from each file identified by one of the configured
importers. The entries are written to the specified output file
or to the standard output in Beancount ledger format in sections
associated to the source document.
"""
verbosity = -quiet
log = utils.logger(verbosity, err=True)
errors = exceptions.ExceptionsTrap(log)
# Load the ledger, if one is specified.
existing_entries = loader.load_file(existing)[0] if existing else []
extracted = []
for filename in _walk(src, log):
with errors:
importer = identify.identify(ctx.importers, filename)
if not importer:
log('') # Newline.
continue
# Signal processing of this document.
log(' ...', nl=False)
# Extract entries.
entries = extract.extract_from_file(importer, filename,
existing_entries)
extracted.append((filename, entries))
log(' OK', fg='green')
if failfast and errors:
break
# Invoke hooks.
hooks = [extract.find_duplicate_entries
] if ctx.hooks is None else ctx.hooks
for func in hooks:
extracted = func(extracted, existing_entries)
# Reverse sort order, if requested.
if reverse:
for filename, entries in extracted:
entries.reverse()
# Serialize entries.
extract.print_extracted_entries(extracted, output)
if errors:
sys.exit(1)
def test_extract_from_file(self):
entries, errors, options = parser.parse_string(
textwrap.dedent('''
1970-01-03 * "Test"
Assets:Tests 1.00 USD
1970-01-01 * "Test"
Assets:Tests 1.00 USD
1970-01-02 * "Test"
Assets:Tests 1.00 USD
'''))
importer = mock.MagicMock(wraps=self.importer)
importer.extract.return_value = entries
entries = extract.extract_from_file(importer, path.abspath('test.csv'),
[])
dates = [entry.date for entry in entries]
self.assertSequenceEqual(dates, sorted(dates))
def test_extract_from_file__existing_entries(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-03 * "C"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-04 * "D"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=[entries[1], entries[3]])
new_entries = extract.extract_from_file('/tmp/blabla.ofx', imp,
entries)
self.assertEqual(2, len(new_entries))
self.assertEqual(
[datetime.date(2016, 2, 2),
datetime.date(2016, 2, 4)], [entry.date for entry in new_entries])
# Check that the entries have also been marked.
marked_entries = [
entry for entry in new_entries
if extract.DUPLICATE_META in entry.meta
]
self.assertEqual(new_entries, marked_entries)
def test_extract_from_file__ensure_sorted(self):
entries, _, __ = loader.load_string("""
2016-02-03 * "C"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
new_entries = extract.extract_from_file('/tmp/blabla.ofx', imp)
self.assertEqual(3, len(entries))
self.assertTrue(
misc_utils.is_sorted(new_entries, key=lambda entry: entry.date))
def test_extract_from_file_no_entries(self):
entries = extract.extract_from_file(self.importer,
path.abspath('test.csv'), [])
self.assertEqual(entries, [])
def test_extract_from_file__empty(self):
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=[])
new_entries = extract.extract_from_file('/tmp/blabla.ofx', imp)
self.assertEqual([], new_entries)