def test_extract_from_file__raises_exception(self):
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(
side_effect=ValueError("Unexpected error!"))
with self.assertRaises(ValueError):
extract.extract_from_file('/tmp/blabla.ofx', imp, [])
def test_extract_from_file__ensure_sanity(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
# Break something.
entries[-1] = entries[-1]._replace(narration=42)
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
with self.assertRaises(AssertionError):
extract.extract_from_file('blabla.ofx', imp)
def test_expect_extract(self, filename, msg):
"""Extract entries from a test file and compare against expected output.
If an expected file (as <filename>.extract) is not present, we issue a
warning. Missing expected files can be written out by removing them
before running the tests.
Args:
filename: A string, the name of the file to import using self.importer.
Raises:
AssertionError: If the contents differ from the expected file.
"""
# Import the file.
entries = extract.extract_from_file(filename, self.importer, None,
None)
# Render the entries to a string.
oss = io.StringIO()
printer.print_entries(entries, file=oss)
string = oss.getvalue()
expect_filename = '{}.extract'.format(filename)
if path.exists(expect_filename):
expect_string = open(expect_filename, encoding='utf-8').read()
self.assertEqual(expect_string.strip(), string.strip())
else:
# Write out the expected file for review.
open(expect_filename, 'w', encoding='utf-8').write(string)
self.skipTest("Expected file not present; generating '{}'".format(
expect_filename))
def extract(self, filename: str, importer_name: str):
"""Extract entries from filename with the specified importer.
Args:
filename: The full path to a file.
importer_name: The name of an importer that matched the file.
Returns:
A list of new imported entries.
"""
if (not filename or not importer_name or not self.config
or not self.module_path):
return []
if os.stat(self.module_path).st_mtime_ns > self.mtime:
self.load_file()
new_entries = extract.extract_from_file(
filename,
self.importers.get(importer_name),
existing_entries=self.ledger.all_entries,
)
new_entries = extract.find_duplicate_entries(
[(filename, new_entries)], self.ledger.all_entries)[0][1]
return new_entries
def extract(self, filename, importer_name):
"""Extract entries from filename with the specified importer.
Args:
filename: The full path to a file.
importer_name: The name of an importer that matched the file.
Returns:
A list of new imported entries.
"""
if not filename or not importer_name or not self.config:
return []
if os.stat(self.module_path).st_mtime_ns > self.mtime:
self.load_file()
new_entries = extract.extract_from_file(
filename,
self.importers.get(importer_name),
existing_entries=self.ledger.all_entries,
)
if isinstance(new_entries, tuple):
new_entries, _ = new_entries
return new_entries
def test_extract_from_file__existing_entries(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-03 * "C"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-04 * "D"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=[entries[1], entries[3]])
new_entries = extract.extract_from_file(
'/tmp/blabla.ofx', imp, entries)
self.assertEqual(2, len(new_entries))
self.assertEqual([datetime.date(2016, 2, 2), datetime.date(2016, 2, 4)],
[entry.date for entry in new_entries])
# Check that the entries have also been marked.
marked_entries = [entry
for entry in new_entries
if extract.DUPLICATE_META in entry.meta]
self.assertEqual(new_entries, marked_entries)
def test_extract_from_file__min_date(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-03 * "C"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
new_entries = extract.extract_from_file('/tmp/blabla.ofx',
imp,
min_date=datetime.date(
2016, 2, 2))
self.assertEqual(2, len(new_entries))
self.assertEqual(
[datetime.date(2016, 2, 2),
datetime.date(2016, 2, 3)], [entry.date for entry in new_entries])
def test_extract_from_file__explicitly_marked_duplicates_entries(self):
entries, _, __ = loader.load_string("""
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
entries[1].meta[extract.DUPLICATE_META] = True
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
new_entries, dup_entries = extract.extract_from_file(
'/tmp/blabla.ofx', imp, [])
self.assertEqual(1, len(dup_entries))
self.assertEqual(
[datetime.date(2016, 2, 1),
datetime.date(2016, 2, 2)], [entry.date for entry in new_entries])
self.assertEqual([datetime.date(2016, 2, 2)],
[entry.date for entry in dup_entries])
def extract(self, filename: str, importer_name: str) -> Entries:
"""Extract entries from filename with the specified importer.
Args:
filename: The full path to a file.
importer_name: The name of an importer that matched the file.
Returns:
A list of new imported entries.
"""
if (not filename or not importer_name or not self.config
or not self.module_path):
return []
if (self.mtime is None
or os.stat(self.module_path).st_mtime_ns > self.mtime):
self.load_file()
new_entries = extract.extract_from_file(
filename,
self.importers.get(importer_name),
existing_entries=self.ledger.all_entries,
)
new_entries_list: List[Tuple[str, Entries]] = [(filename, new_entries)]
for hook_fn in self.hooks:
new_entries_list = hook_fn(new_entries_list,
self.ledger.all_entries)
return new_entries_list[0][1]
def test_extract_from_file__ensure_sorted(self):
entries, _, __ = loader.load_string("""
2016-02-03 * "C"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-01 * "A"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
2016-02-02 * "B"
Assets:Account1 10.00 USD
Assets:Account2 -10.00 USD
""")
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=entries)
new_entries, dup_entries = extract.extract_from_file(
'/tmp/blabla.ofx', imp)
self.assertEqual(3, len(entries))
self.assertTrue(
misc_utils.is_sorted(new_entries, key=lambda entry: entry.date))
self.assertEqual([], dup_entries)
def test_extract(self, importer, file, pytestconfig):
"""Extract entries from a test file and compare against expected output."""
entries = extract.extract_from_file(file.name, importer, None, None)
oss = io.StringIO()
printer.print_entries(entries, file=oss)
string = oss.getvalue()
compare_contents_or_generate(string, '{}.extract'.format(file.name),
pytestconfig.getoption("generate", False))
def test_extract_from_file__empty(self):
imp = mock.MagicMock()
imp.identify = mock.MagicMock(return_value=True)
imp.extract = mock.MagicMock(return_value=[])
new_entries, dup_entries = extract.extract_from_file(
'/tmp/blabla.ofx', imp)
self.assertEqual([], new_entries)
self.assertEqual([], dup_entries)
def extract(self, filename, importer_name):
"""Extract entries from filename with the specified importer.
Args:
filename: The full path to a file.
importer_name: The name of an importer that matched the file.
Returns:
A list of new imported entries.
"""
if not filename or not importer_name:
return []
new_entries, _ = extract.extract_from_file(
filename,
self.importers.get(importer_name),
existing_entries=self.ledger.all_entries)
return new_entries
def test_extract_as_expected():
importer = get_importer("examples/wechat.import")
fs = normpath(
abspath("tests/fixtures/wechat/微信支付账单(20200830-20200906).csv"))
extracted = extract_from_file(fs, importer)
assert len(extracted) == 7, "should extract 17 entries from file"
