def hierarchy(self, account_name, begin=None, end=None):
"""An account tree."""
if begin:
tree = Tree(iter_entry_dates(self.ledger.entries, begin, end))
else:
tree = self.ledger.root_tree
return tree.get(account_name).serialise(end)
def find_similar_entries(entries,
source_entries,
comparator=None,
window_days=2,
filter_type=Transaction):
'''Same as beancount's similar.find_similar_entries function, but with the
ability to filter any type of entries'''
window_head = datetime.timedelta(days=window_days)
window_tail = datetime.timedelta(days=window_days + 1)
if comparator is None:
comparator = SimilarityComparator()
# For each of the new entries, look at entries at a nearby date.
duplicates = []
for entry in filter_ledger(entries, filter_type):
filtered_entries = list(
filter_ledger(
data.iter_entry_dates(source_entries, entry.date - window_head,
entry.date + window_tail), filter_type))
for source_entry in filtered_entries:
if comparator(entry, source_entry):
duplicates.append((entry, source_entry))
break
return duplicates
def interval_balances(self, interval, account_name, accumulate=False):
"""Balances by interval.
Arguments:
interval: An interval.
account_name: An account name.
accumulate: A boolean, ``True`` if the balances for an interval
should include all entries up to the end of the interval.
Returns:
A list of RealAccount instances for all the intervals.
"""
min_accounts = [
account for account in
self.accounts.keys()
if account.startswith(account_name)]
interval_tuples = list(
reversed(list(pairwise(self.interval_ends(interval))))
)
interval_balances = [
realization.realize(list(iter_entry_dates(
self.entries,
datetime.date.min if accumulate else begin_date,
end_date)), min_accounts)
for begin_date, end_date in interval_tuples]
return interval_balances, interval_tuples
def hierarchy(self, account_name, begin=None, end=None):
"""An account tree."""
if begin:
tree = Tree(iter_entry_dates(self.ledger.entries, begin, end))
else:
tree = self.ledger.root_tree
return _serialize_account_node(tree.get(account_name), end)
def interval_totals(
self,
interval: Interval,
accounts: Union[str, Tuple[str]],
conversion: str,
):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
"""
price_map = self.ledger.price_map
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
"date":
begin,
"balance":
cost_or_value(inventory, conversion, price_map, end - ONE_DAY),
"budgets":
self.ledger.budgets.calculate_children(accounts, begin, end),
}
def interval_balances(self, interval, account_name, accumulate=False):
"""Balances by interval.
Arguments:
interval: An interval.
account_name: An account name.
accumulate: A boolean, ``True`` if the balances for an interval
should include all entries up to the end of the interval.
Returns:
A list of RealAccount instances for all the intervals.
"""
min_accounts = [
account for account in self.accounts.keys()
if account.startswith(account_name)
]
interval_tuples = list(
reversed(list(pairwise(self.interval_ends(interval)))))
interval_balances = [
realization.realize(
list(
iter_entry_dates(
self.entries,
datetime.date.min if accumulate else begin_date,
end_date,
)),
min_accounts,
) for begin_date, end_date in interval_tuples
]
return interval_balances, interval_tuples
def hierarchy(self, account_name, begin=None, end=None):
"""An account tree."""
if begin:
entries = iter_entry_dates(self.ledger.entries, begin, end)
tree = Tree(entries)
else:
tree = self.ledger.root_tree
return _serialize_account_node(tree.get(account_name), end)
def interval_totals(
self,
filtered: FilteredLedger,
interval: Interval,
accounts: str | tuple[str],
conversion: str,
invert: bool = False,
) -> Generator[DateAndBalanceWithBudget, None, None]:
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
invert: invert all numbers.
"""
# pylint: disable=too-many-locals
price_map = self.ledger.price_map
for begin, end in pairwise(filtered.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(filtered.entries, begin, end)
account_inventories = {}
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
if posting.account not in account_inventories:
account_inventories[posting.account] = Inventory()
account_inventories[posting.account].add_position(
posting)
inventory.add_position(posting)
balance = cost_or_value(inventory, conversion, price_map,
end - ONE_DAY)
account_balances = {}
for account, acct_value in account_inventories.items():
account_balances[account] = cost_or_value(
acct_value,
conversion,
price_map,
end - ONE_DAY,
)
budgets = {}
if isinstance(accounts, str):
budgets = self.ledger.budgets.calculate_children(
accounts, begin, end)
if invert:
# pylint: disable=invalid-unary-operand-type
balance = -balance
budgets = {k: -v for k, v in budgets.items()}
account_balances = {k: -v for k, v in account_balances.items()}
yield DateAndBalanceWithBudget(
begin,
balance,
account_balances,
budgets,
)
def hierarchy(self, account_name, begin=None, end=None):
"""An account tree."""
if begin:
entries = iter_entry_dates(self.ledger.entries, begin, end)
root_account = realization.realize(entries)
else:
root_account = self.ledger.root_account
return _serialize_real_account(
realization.get_or_create(root_account, account_name), end)
def hierarchy(
self,
account_name: str,
begin: Optional[datetime.date] = None,
end: Optional[datetime.date] = None,
):
"""An account tree."""
if begin is not None:
tree = Tree(iter_entry_dates(self.ledger.entries, begin, end))
else:
tree = self.ledger.root_tree
return tree.get(account_name).serialise(end)
def hierarchy(
self,
account_name: str,
conversion: str,
begin: Optional[date] = None,
end: Optional[date] = None,
) -> SerialisedTreeNode:
"""An account tree."""
if begin is not None and end is not None:
tree = Tree(iter_entry_dates(self.ledger.entries, begin, end))
else:
tree = self.ledger.root_tree
return tree.get(account_name).serialise(conversion, self.ledger,
end - ONE_DAY if end else None)
def interval_balances(self, interval, account_name, accumulate=False):
"""accumulate is False for /changes and True for /balances"""
min_accounts = [account for account in self.all_accounts if account.startswith(account_name)]
interval_tuples = list(reversed(self._interval_tuples(interval)))
interval_balances = [
realization.realize(
list(iter_entry_dates(self.entries, self.date_first if accumulate else begin_date, end_date)),
min_accounts,
)
for begin_date, end_date in interval_tuples
]
return interval_balances, interval_tuples
def interval_balances(self, interval, account_name, accumulate=False):
"""accumulate is False for /changes and True for /balances"""
min_accounts = [account
for account in self.all_accounts
if account.startswith(account_name)]
interval_tuples = list(reversed(self._interval_tuples(interval)))
interval_balances = [
realization.realize(list(iter_entry_dates(
self.entries,
self.date_first if accumulate else begin_date,
end_date)), min_accounts)
for begin_date, end_date in interval_tuples]
return interval_balances, interval_tuples
def hierarchy(
self,
filtered: FilteredLedger,
account_name: str,
conversion: str,
begin: date | None = None,
end: date | None = None,
) -> SerialisedTreeNode:
"""An account tree."""
if begin is not None and end is not None:
tree = Tree(iter_entry_dates(filtered.entries, begin, end))
else:
tree = filtered.root_tree
return tree.get(account_name).serialise(conversion,
self.ledger.price_map,
end - ONE_DAY if end else None)
def _real_account(account_name, entries, begin_date=None, end_date=None, min_accounts=None):
"""
Returns the realization.RealAccount instances for account_name, and
their entries clamped by the optional begin_date and end_date.
Warning: For efficiency, the returned result does not include any added
postings to account for balances at 'begin_date'.
:return: realization.RealAccount instances
"""
if begin_date:
entries = list(iter_entry_dates(entries, begin_date, end_date))
if not min_accounts:
min_accounts = [account_name]
return realization.get(realization.realize(entries, min_accounts), account_name)
def _real_account(self, account_name, entries, begin_date=None,
end_date=None, min_accounts=None):
"""
Returns the realization.RealAccount instances for account_name, and
their entries clamped by the optional begin_date and end_date.
Warning: For efficiency, the returned result does not include any added
postings to account for balances at 'begin_date'.
:return: realization.RealAccount instances
"""
if begin_date:
entries = list(iter_entry_dates(entries, begin_date, end_date))
if not min_accounts:
min_accounts = [account_name]
return realization.get(realization.realize(entries, min_accounts),
account_name)
def interval_totals(
self,
interval: Interval,
accounts: Union[str, Tuple[str]],
conversion: str,
invert: bool = False,
) -> Generator[DateAndBalanceWithBudget, None, None]:
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
"""
price_map = self.ledger.price_map
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
balance = cost_or_value(
inventory, conversion, price_map, end - ONE_DAY
)
budgets = {}
if isinstance(accounts, str):
budgets = self.ledger.budgets.calculate_children(
accounts, begin, end
)
if invert:
# pylint: disable=invalid-unary-operand-type
balance = -balance
budgets = {k: -v for k, v in budgets.items()}
yield {
"date": begin,
"balance": balance,
"budgets": budgets,
}
def find_similar_entries(entries,
source_entries,
comparator=None,
window_days=2):
"""Find which entries from a list are potential duplicates of a set.
Note: If there are multiple entries from 'source_entries' matching an entry
in 'entries', only the first match is returned. Note that this function
could in theory decide to merge some of the imported entries with each
other.
Args:
entries: The list of entries to classify as duplicate or note.
source_entries: The list of entries against which to match. This is the
previous, or existing set of entries to compare against. This may be null
or empty.
comparator: A functor used to establish the similarity of two entries.
window_days: The number of days (inclusive) before or after to scan the
entries to classify against.
Returns:
A list of pairs of entries (entry, source_entry) where entry is from
'entries' and is deemed to be a duplicate of source_entry, from
'source_entries'.
"""
window_head = datetime.timedelta(days=window_days)
window_tail = datetime.timedelta(days=window_days + 1)
if comparator is None:
comparator = SimilarityComparator()
# For each of the new entries, look at existing entries at a nearby date.
duplicates = []
if source_entries is not None:
for entry in data.filter_txns(entries):
for source_entry in data.filter_txns(
data.iter_entry_dates(source_entries,
entry.date - window_head,
entry.date + window_tail)):
if comparator(entry, source_entry):
duplicates.append((entry, source_entry))
break
return duplicates
def interval_balances(
self,
filtered: FilteredLedger,
interval: date.Interval,
account_name: str,
accumulate: bool = False,
) -> tuple[list[realization.RealAccount], list[tuple[datetime.date,
datetime.date]], ]:
"""Balances by interval.
Arguments:
filtered: The currently filtered ledger.
interval: An interval.
account_name: An account name.
accumulate: A boolean, ``True`` if the balances for an interval
should include all entries up to the end of the interval.
Returns:
A list of RealAccount instances for all the intervals.
"""
min_accounts = [
account for account in self.accounts.keys()
if account.startswith(account_name)
]
interval_tuples = list(
reversed(list(pairwise(filtered.interval_ends(interval)))))
interval_balances = [
realization.realize(
list(
iter_entry_dates(
filtered.entries,
datetime.date.min if accumulate else begin_date,
end_date,
)),
min_accounts,
) for begin_date, end_date in interval_tuples
]
return interval_balances, interval_tuples
def portfolio_accounts(self, begin=None, end=None):
"""An account tree based on matching regex patterns."""
if begin:
tree = Tree(iter_entry_dates(self.ledger.entries, begin, end))
else:
tree = self.ledger.root_tree
portfolios = []
for option in self.config:
opt_key = option[0]
if opt_key == "account_name_pattern":
portfolio = self._account_name_pattern(tree, end, option[1])
elif opt_key == "account_open_metadata_pattern":
portfolio = self._account_metadata_pattern(
tree, end, option[1][0], option[1][1])
else:
raise FavaAPIException("Portfolio List: Invalid option.")
portfolios.append(portfolio)
return portfolios
def interval_totals(self, interval, accounts):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: A string for the interval.
accounts: A single account (str) or a tuple of accounts.
"""
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in filter_type(entries, Transaction):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
'begin_date': begin,
'totals': _inventory_cost_or_value(inventory, end),
'budgets':
self.ledger.budgets.calculate(accounts[0], begin, end),
}
def interval_totals(self, interval, accounts):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
"""
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = CounterInventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in filter_type(entries, Transaction):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
'date': begin,
'balance': cost_or_value(inventory, end),
'budgets':
self.ledger.budgets.calculate_children(accounts, begin, end),
}
def interval_totals(self, interval, accounts):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
"""
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = CounterInventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in filter_type(entries, Transaction):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
"date": begin,
"balance": cost_or_value(inventory, end),
"budgets": self.ledger.budgets.calculate_children(
accounts, begin, end
),
}
def portfolio_accounts(self, begin=None, end=None):
"""An account tree based on matching regex patterns."""
portfolios = []
try:
self.load_report()
if begin:
tree = Tree(iter_entry_dates(self.ledger.entries, begin, end))
else:
tree = self.ledger.root_tree
for option in self.config:
opt_key = option[0]
if opt_key == "account_name_pattern":
portfolio = self._account_name_pattern(tree, end, option[1])
elif opt_key == "account_open_metadata_pattern":
portfolio = self._account_metadata_pattern(
tree, end, option[1][0], option[1][1]
)
else:
exception = FavaAPIException("Classy Portfolio: Invalid option.")
raise (exception)
portfolio = (
portfolio[0], # title
portfolio[1], # subtitle
(
insert_rowspans(portfolio[2][0], portfolio[2][1], True),
portfolio[2][1],
), # portfolio data
)
portfolios.append(portfolio)
except Exception as exc:
traceback.print_exc(file=sys.stdout)
return portfolios
def _real_account(account_name, entries, begin_date, end_date):
if begin_date:
entries = list(iter_entry_dates(entries, begin_date, end_date))
return realization.get_or_create(realization.realize(entries),
account_name)
def test_iter_entry_dates(self):
prototype = data.Transaction(data.new_metadata("misc", 200), None, '*',
None, "", None, None, [])
dates = [
datetime.date(2016, 1, 10),
datetime.date(2016, 1, 11),
datetime.date(2016, 1, 14),
datetime.date(2016, 1, 15),
datetime.date(2016, 1, 16),
datetime.date(2016, 1, 20),
datetime.date(2016, 1, 22)
]
entries = [prototype._replace(date=date) for date in dates]
# Same date, present.
self.assertEqual([], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 14), datetime.date(
2016, 1, 14))
])
# Same date, absent.
self.assertEqual([], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 12), datetime.date(
2016, 1, 12))
])
# Both dates exist.
self.assertEqual(
[datetime.date(2016, 1, 11),
datetime.date(2016, 1, 14)], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 11),
datetime.date(2016, 1, 15))
])
# First doesn't exist.
self.assertEqual([datetime.date(2016, 1, 14)], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 12), datetime.date(
2016, 1, 15))
])
# Second doesn't exist.
self.assertEqual([datetime.date(2016, 1, 11)], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 11), datetime.date(
2016, 1, 13))
])
# Neither exist.
self.assertEqual([
datetime.date(2016, 1, 14),
datetime.date(2016, 1, 15),
datetime.date(2016, 1, 16)
], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 13), datetime.date(
2016, 1, 17))
])
# Before.
self.assertEqual([datetime.date(2016, 1, 10)], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 5), datetime.date(2016, 1, 11))
])
# After.
self.assertEqual([datetime.date(2016, 1, 22)], [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 21), datetime.date(
2016, 1, 30))
])
# Around.
self.assertEqual(dates, [
entry.date for entry in data.iter_entry_dates(
entries, datetime.date(2016, 1, 2), datetime.date(2016, 1, 30))
])
def deduplicate(new_entries: List[Directive],
existing_entries: List[Directive],
window_days: int = 10) -> List[Directive]:
"""De-duplicate entries.
A new non-transaction entry is considered connected to an existing entry
iff they are identical.
A new transaction is considered connected to an existing transaction iff:
* the new transaction is no earlier than the existing transaction.
* the new transaction is at most {window_days} days later than the existing
transaction.
* guess_transaction_duplicated returns True.
If a new entry doesn't have any connection, it's considered non-duplicated.
For each strongly connected subgraph, if all new entries are matched, all
of them are considered duplicated. Otherwise, all of them are considered
possibly duplicated.
Returns new entries where:
* Non-duplicated entries are preserved.
* Duplicated entries are removed.
* Possibly-duplicated entries are marked with DUPLICATE_META.
"""
window_head = datetime.timedelta(days=window_days)
window_tail = datetime.timedelta(days=1)
matcher = _Matcher()
for new_entry in new_entries:
if isinstance(new_entry, Transaction):
for existing_entry in filter_txns(
iter_entry_dates(existing_entries,
new_entry.date - window_head,
new_entry.date + window_tail)):
if guess_transaction_duplicated(new_entry, existing_entry):
matcher.add_edge(id(new_entry), id(existing_entry))
else:
for existing_entry in iter_entry_dates(
existing_entries, new_entry.date,
new_entry.date + window_tail):
if new_entry == existing_entry:
matcher.add_edge(id(new_entry), id(existing_entry))
duplicates = set()
possibly_duplicates = set()
matches = matcher.matches()
for subgraph in matcher.subgraphs():
n = len([True for node in subgraph if node in matches])
if n == len(subgraph): # duplicated
duplicates.update(node[1] for node in subgraph if node[0])
elif n: # possibly duplicated
possibly_duplicates.update(node[1] for node in subgraph if node[0])
ret = []
for new_entry in new_entries:
if id(new_entry) in duplicates:
continue
elif id(new_entry) in possibly_duplicates and hasattr(
new_entry, 'meta'):
meta = copy.deepcopy(new_entry.meta)
meta[DUPLICATE_META] = True
ret.append(new_entry._replace(meta=meta))
else:
ret.append(new_entry)
return ret
def _real_account(account_name, entries, begin_date, end_date):
if begin_date:
entries = list(iter_entry_dates(entries, begin_date, end_date))
return realization.get_or_create(realization.realize(entries),
account_name)
def _build_graph(
entries_by_file: Dict[str, List[Directive]],
links: Iterable[Link],
logger: ErrorLogger) -> Dict[int, List[Tuple[Transaction, str]]]:
day = timedelta(days=1)
edges = defaultdict(list) # id(txn) -> [(complement txn, account)]
for link in links:
if not link.valid():
continue
entries = entries_by_file.get(link.filename, [])
complement_entries = entries_by_file.get(link.complement_filename, [])
for entry in filter_txns(entries):
expected_complement_feature = _transaction_feature(
entry, link.account, True)
if not expected_complement_feature[1]:
continue
# find complement txn and check duplicated complement
complement_txn = None
complement_duplicated = False
for complement_entry in filter_txns(iter_entry_dates(
complement_entries, entry.date, entry.date + day)):
complement_feature = _transaction_feature(
complement_entry, link.complement_account, False)
if complement_feature == expected_complement_feature:
if not complement_txn:
complement_txn = complement_entry
else:
complement_duplicated = True
break
# check duplicated
duplicated = False
found = False
feature = _transaction_feature(entry, link.account, False)
for entry2 in filter_txns(iter_entry_dates(
entries, entry.date, entry.date + day)):
feature2 = _transaction_feature(entry2, link.account, False)
if feature2 == feature:
if found:
duplicated = True
break
found = True
if not complement_txn:
logger.log_error(UnresolvedLinkError(
entry.meta,
f'No complement transaction found for link {link}',
entry,
))
elif complement_duplicated:
logger.log_error(UnresolvedLinkError(
entry.meta,
f'Multiple complement transactions found for link {link}',
entry,
))
elif duplicated:
logger.log_error(UnresolvedLinkError(
complement_txn.meta,
f'Multiple complement transactions found for link {link}',
complement_txn,
))
else:
edges[id(entry)].append((complement_txn, link.account))
edges[id(complement_txn)].append((entry, link.complement_account))
for complement_txn in filter_txns(complement_entries):
complement_feature = _transaction_feature(
complement_txn, link.complement_account, False)
if complement_feature[1] and id(complement_txn) not in edges:
logger.log_error(UnresolvedLinkError(
complement_txn.meta,
f'No complement transaction found for link {link}',
complement_txn,
))
return edges
