def inventory_at_dates(transactions, dates, posting_predicate):
"""Generator that yields the aggregate inventory at the specified dates.
The inventory for a date includes all matching postings PRIOR to it.
Args:
transactions: list of transactions, sorted by date.
dates: iterator of dates
posting_predicate: predicate with the Transaction and Posting to
decide whether to include the posting in the inventory.
"""
if not transactions:
return
iterator = iter(transactions)
txn = next(iterator)
inventory = Inventory()
for date in dates:
while txn.date < date:
for posting in txn.postings:
if posting_predicate(posting):
inventory.add_position(posting)
try:
txn = next(iterator)
except StopIteration:
break
yield inventory.get_positions()
def balance_check(entries, options_map):
errors = []
tracking_accounts = set()
for entry in entries:
if isinstance(entry, Open):
if entry.meta.get('tracking', False):
tracking_accounts.add(entry.account)
asum = Inventory()
bsum = Inventory()
for entry in filter_txns(entries):
for posting in entry.postings:
if posting.account in tracking_accounts:
continue
components = posting.account.split(':')
if components[0] in ('Assets', 'Liabilities'):
asum.add_position(posting)
elif components[0] in ('Income', 'Expenses'):
bsum.add_position(posting)
csum = asum.reduce(convert.get_weight) + bsum.reduce(convert.get_weight)
if not csum.is_small(interpolate.infer_tolerances({}, options_map)):
errors.append(
BudgetBalanceError(
{
'filename': '<budget_balance_check>',
'lineno': 0
},
f"On-budget accounts and budget total do not match: {asum} vs {-bsum}",
None))
return entries, errors
def net_worth(self, interval):
"""Compute net worth.
Args:
interval: A string for the interval.
Returns:
A list of dicts for all ends of the given interval containing the
net worth (Assets + Liabilities) separately converted to all
operating currencies.
"""
transactions = (entry for entry in self.ledger.entries
if (isinstance(entry, Transaction)
and entry.flag != flags.FLAG_UNREALIZED))
types = (
self.ledger.options["name_assets"],
self.ledger.options["name_liabilities"],
)
txn = next(transactions, None)
inventory = Inventory()
for end_date_exclusive in self.ledger.interval_ends(interval):
end_date_inclusive = end_date_exclusive - datetime.timedelta(
days=1)
while txn and txn.date < end_date_exclusive:
for posting in txn.postings:
if posting.account.startswith(types):
inventory.add_position(posting)
txn = next(transactions, None)
yield {
"date": end_date_exclusive,
"balance": cost_or_value(inventory, end_date_inclusive),
}
def measure_balance_total(entries):
# For timing only.
with utils.log_time('balance_all', logging.info): # 89ms... okay.
balance = Inventory()
for entry in utils.filter_type(entries, data.Transaction):
for posting in entry.postings:
balance.add_position(posting.position)
def inventory_at_dates(transactions, dates, posting_predicate):
"""Generator that yields the aggregate inventory at the specified dates.
The inventory for a specified date includes all matching postings PRIOR to
it.
:param transactions: list of transactions, sorted by date.
:param dates: iterator of dates
:param posting_predicate: predicate with the Transaction and Posting to
decide whether to include the posting in the inventory.
"""
index = 0
length = len(transactions)
# inventory maps lot to amount
inventory = Inventory()
prev_date = None
for date in dates:
assert prev_date is None or date > prev_date
prev_date = date
while index < length and transactions[index].date < date:
entry = transactions[index]
index += 1
for posting in entry.postings:
if posting_predicate(posting):
inventory.add_position(posting)
yield inventory
def interval_totals(
self,
interval: Interval,
accounts: Union[str, Tuple[str]],
conversion: str,
):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
"""
price_map = self.ledger.price_map
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
"date":
begin,
"balance":
cost_or_value(inventory, conversion, price_map, end - ONE_DAY),
"budgets":
self.ledger.budgets.calculate_children(accounts, begin, end),
}
def process_account_entries(entries: data.Entries, options_map: data.Options,
account: Account) -> AccountData:
"""Process a single account."""
logging.info("Processing account: %s", account)
# Extract the relevant transactions.
transactions = transactions_for_account(entries, account)
if not transactions:
logging.warning("No transactions for %s; skipping.", account)
return transactions, None, None
# Categorize the set of accounts encountered in the filtered transactions.
seen_accounts = {
posting.account
for entry in transactions for posting in entry.postings
}
atypes = options.get_account_types(options_map)
catmap = categorize_accounts(account, seen_accounts, atypes)
# Process each of the transactions, adding derived values as metadata.
cash_flows = []
balance = Inventory()
decorated_transactions = []
for entry in transactions:
# Update the total position in the asset we're interested in.
positions = []
for posting in entry.postings:
category = catmap[posting.account]
if category is Cat.ASSET:
balance.add_position(posting)
positions.append(posting)
# Compute the signature of the transaction.
entry = copy_and_normalize(entry)
signature = compute_transaction_signature(catmap, entry)
entry.meta["signature"] = signature
entry.meta["description"] = KNOWN_SIGNATURES[signature]
# Compute the cash flows associated with the transaction.
flows = produce_cash_flows(entry)
entry.meta['cash_flows'] = flows
cash_flows.extend(
flow._replace(balance=copy.deepcopy(balance)) for flow in flows)
decorated_transactions.append(entry)
currency = accountlib.leaf(account)
cost_currencies = set(cf.amount.currency for cf in cash_flows)
assert len(cost_currencies) == 1, str(cost_currencies)
cost_currency = cost_currencies.pop()
commodity_map = getters.get_commodity_directives(entries)
comm = commodity_map[currency]
return AccountData(account, currency, cost_currency, comm, cash_flows,
decorated_transactions, catmap)
def interval_totals(
self,
filtered: FilteredLedger,
interval: Interval,
accounts: str | tuple[str],
conversion: str,
invert: bool = False,
) -> Generator[DateAndBalanceWithBudget, None, None]:
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
invert: invert all numbers.
"""
# pylint: disable=too-many-locals
price_map = self.ledger.price_map
for begin, end in pairwise(filtered.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(filtered.entries, begin, end)
account_inventories = {}
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
if posting.account not in account_inventories:
account_inventories[posting.account] = Inventory()
account_inventories[posting.account].add_position(
posting)
inventory.add_position(posting)
balance = cost_or_value(inventory, conversion, price_map,
end - ONE_DAY)
account_balances = {}
for account, acct_value in account_inventories.items():
account_balances[account] = cost_or_value(
acct_value,
conversion,
price_map,
end - ONE_DAY,
)
budgets = {}
if isinstance(accounts, str):
budgets = self.ledger.budgets.calculate_children(
accounts, begin, end)
if invert:
# pylint: disable=invalid-unary-operand-type
balance = -balance
budgets = {k: -v for k, v in budgets.items()}
account_balances = {k: -v for k, v in account_balances.items()}
yield DateAndBalanceWithBudget(
begin,
balance,
account_balances,
budgets,
)
def compute_balance_at(transactions: data.Entries,
date: Optional[Date] = None) -> Inventory:
"""Compute the balance at a specific date."""
balance = Inventory()
for entry in transactions:
if date is not None and entry.date >= date:
break
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
balance.add_position(posting)
return balance
def get_inventory(self, account, date):
inventory = Inventory()
for entry in self.entries:
if date is not None and entry.date > date:
break
if not isinstance(entry, Transaction):
continue
for posting in entry.postings:
if posting.account != account:
continue
inventory.add_position(get_position(posting))
return inventory
def SumPostings(postings: Iterable[Posting]) -> Inventory:
"""Aggregate all the positions for the given postings."""
acc = Inventory()
for elem in postings:
# Note: 'elem' can be a Posting or an Inventory (the intermediate result
# from a subcombiner).
if isinstance(elem, Inventory):
acc.add_inventory(elem)
else:
assert isinstance(elem, Posting)
acc.add_position(elem)
return acc
def compute_portfolio_values(
price_map: prices.PriceMap,
transactions: data.Entries) -> Tuple[List[Date], List[float]]:
"""Compute a serie of portfolio values over time."""
# Infer the list of required prices.
currency_pairs = set()
for entry in transactions:
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
if posting.cost:
currency_pairs.add(
(posting.units.currency, posting.cost.currency))
first = lambda x: x[0]
price_dates = sorted(itertools.chain(
((date, None) for pair in currency_pairs
for date, _ in prices.get_all_prices(price_map, pair)),
((entry.date, entry) for entry in transactions)),
key=first)
# Iterate computing the balance.
value_dates = []
value_values = []
balance = Inventory()
for date, group in itertools.groupby(price_dates, key=first):
# Update balances.
for _, entry in group:
if entry is None:
continue
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
balance.add_position(posting)
# Convert to market value.
value_balance = balance.reduce(convert.get_value, price_map, date)
cost_balance = value_balance.reduce(convert.convert_position, "USD",
price_map)
pos = cost_balance.get_only_position()
value = pos.units.number if pos else ZERO
# Add one data point.
value_dates.append(date)
value_values.append(value)
return value_dates, value_values
def net_worth(
self, interval: Interval, conversion: str
) -> Generator[DateAndBalance, None, None]:
"""Compute net worth.
Args:
interval: A string for the interval.
conversion: The conversion to use.
Returns:
A list of dicts for all ends of the given interval containing the
net worth (Assets + Liabilities) separately converted to all
operating currencies.
"""
transactions = (
entry
for entry in self.ledger.entries
if (
isinstance(entry, Transaction)
and entry.flag != FLAG_UNREALIZED
)
)
types = (
self.ledger.options["name_assets"],
self.ledger.options["name_liabilities"],
)
txn = next(transactions, None)
inventory = Inventory()
price_map = self.ledger.price_map
for end_date in self.ledger.interval_ends(interval):
while txn and txn.date < end_date:
for posting in txn.postings:
if posting.account.startswith(types):
inventory.add_position(posting)
txn = next(transactions, None)
yield {
"date": end_date,
"balance": cost_or_value(
inventory, conversion, price_map, end_date - ONE_DAY
),
}
def tracking(entries, options_map):
account_types = get_account_types(options_map)
income_tracking = set()
expense_tracking = set()
errors = []
new_entries = []
for entry in entries:
new_entry = None
if isinstance(entry, Open) and entry.meta.get('tracking', False):
if is_account_type(account_types.expenses, entry.account):
expense_tracking.add(entry.account)
elif is_account_type(account_types.income, entry.account):
income_tracking.add(entry.account)
elif isinstance(entry, Transaction):
new_postings = []
tracking_balance = Inventory()
for posting in entry.postings:
if 'tracking' in posting.meta:
new_acct = posting.meta['tracking']
new_posting = posting._replace(account=new_acct, meta=None)
new_postings.append(new_posting)
tracking_balance.add_position(posting)
if new_postings:
for position in -tracking_balance:
if position.units.number < 0 and len(income_tracking) == 1:
posting_acct, = income_tracking
elif position.units.number > 0 and len(
expense_tracking) == 1:
posting_acct, = expense_tracking
else:
continue
new_posting = Posting(posting_acct, position.units,
position.cost, None, None, None)
new_postings.append(new_posting)
link_id = 'tracking-' + compare.hash_entry(entry)
new_links = entry.links | set([link_id])
entry = entry._replace(links=new_links)
new_entry = entry._replace(postings=new_postings)
new_entries.append(entry)
if new_entry:
new_entries.append(new_entry)
return new_entries, errors
def interval_totals(
self,
interval: Interval,
accounts: Union[str, Tuple[str]],
conversion: str,
invert: bool = False,
) -> Generator[DateAndBalanceWithBudget, None, None]:
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
"""
price_map = self.ledger.price_map
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
balance = cost_or_value(
inventory, conversion, price_map, end - ONE_DAY
)
budgets = {}
if isinstance(accounts, str):
budgets = self.ledger.budgets.calculate_children(
accounts, begin, end
)
if invert:
# pylint: disable=invalid-unary-operand-type
balance = -balance
budgets = {k: -v for k, v in budgets.items()}
yield {
"date": begin,
"balance": balance,
"budgets": budgets,
}
def interval_totals(self, interval, accounts):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: A string for the interval.
accounts: A single account (str) or a tuple of accounts.
"""
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in filter_type(entries, Transaction):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
'begin_date': begin,
'totals': _inventory_cost_or_value(inventory, end),
'budgets':
self.ledger.budgets.calculate(accounts[0], begin, end),
}
def merge_postings(account: Account, postings: List[Posting], meta_name: Union[str, None]) -> List[Posting]:
"""
Merges postings with an equal account name and takes meta of the first one.
If `meta_name` is provided, then in a way of metaset combine meta values whose keys equal to `meta_name`.
Args:
postings: a list of postings.
account: an account name whose postings should be merged.
meta_name: a key for meta values that should be combined in a way of metaset.
Example:
Returns:
A list of postings.
"""
grouped_postings = []
share_postings = []
share_balance = Inventory()
meta = dict()
for posting in postings:
if posting.account == account:
share_postings.append(posting)
share_balance.add_position(posting)
if len(meta) == 0:
meta = posting.meta
elif meta_name is not None:
for mark in metaset.get(posting.meta, meta_name):
share_postings[0] = share_postings[0]._replace(
meta=metaset.add(share_postings[0].meta, meta_name, mark)
)
else:
grouped_postings.append(posting)
if share_postings:
for pos in share_balance:
grouped_postings.append(Posting(account, pos.units, pos.cost, None, None, share_postings[0].meta))
return grouped_postings
def net_worth(self, interval):
"""Compute net worth.
Args:
interval: A string for the interval.
Returns:
A list of dicts for all ends of the given interval containing the
net worth (Assets + Liabilities) separately converted to all
operating currencies.
"""
transactions = (entry for entry in self.ledger.entries
if (isinstance(entry, Transaction)
and entry.flag != flags.FLAG_UNREALIZED))
types = (self.ledger.options['name_assets'],
self.ledger.options['name_liabilities'])
txn = next(transactions, None)
inventory = Inventory()
for date in self.ledger.interval_ends(interval):
while txn and txn.date < date:
for posting in filter(lambda p: p.account.startswith(types),
txn.postings):
inventory.add_position(posting)
txn = next(transactions, None)
yield {
'date': date,
'balance': {
currency:
inventory.reduce(convert.convert_position, currency,
self.ledger.price_map,
date).get_currency_units(currency).number
for currency in self.ledger.options['operating_currency']
}
}
def inventory_at_dates(transactions, dates, posting_predicate):
"""Generator that yields the aggregate inventory at the specified dates.
The inventory for a date includes all matching postings PRIOR to it.
Args:
transactions: list of transactions, sorted by date.
dates: iterator of dates
posting_predicate: predicate with the Transaction and Posting to
decide whether to include the posting in the inventory.
"""
iterator = iter(transactions)
txn = next(iterator, None)
inventory = Inventory()
for date in dates:
while txn and txn.date < date:
for posting in txn.postings:
if posting_predicate(posting):
inventory.add_position(posting)
txn = next(iterator, None)
yield inventory.get_positions()
def compute_entries_balance(entries, prefix=None, date=None):
"""Compute the balance of all postings of a list of entries.
Sum up all the positions in all the postings of all the transactions in the
list of entries and return an inventory of it.
Args:
entries: A list of directives.
prefix: If specified, a prefix string to restrict by account name. Only
postings with an account that starts with this prefix will be summed up.
date: A datetime.date instance at which to stop adding up the balance.
The date is exclusive.
Returns:
An instance of Inventory.
"""
total_balance = Inventory()
for entry in entries:
if not (date is None or entry.date < date):
break
if isinstance(entry, Transaction):
for posting in entry.postings:
if prefix is None or posting.account.startswith(prefix):
total_balance.add_position(posting)
return total_balance
def test_add_position(self):
inv = Inventory()
for pos in self.POSITIONS_ALL_KINDS:
inv.add_position(pos)
self.assertEqual(Inventory(self.POSITIONS_ALL_KINDS), inv)
new_inventory = Inventory()
do_round = True
for position in inventory:
# units_digits = get_digits(position.units.number)
# cost_digits = get_digits(position.cost.number)
units_digits = 4
cost_digits = 4
new_units_number = position.units.number * new_to_old_ratio
new_cost_number = position.cost.number / new_to_old_ratio
if do_round:
new_units_number = round(new_units_number, units_digits)
new_cost_number = round(new_cost_number, cost_digits)
new_position = Position(units=Amount(new_units_number,
new_currency),
cost=Cost(number=new_cost_number,
currency=position.cost.currency,
date=position.cost.date,
label=position.cost.label))
new_inventory.add_position(new_position)
print(' %s %s' % (args.transfer_to, new_position))
print(' %s %s' % (args.account, -position))
print('New units: ', new_inventory.reduce(get_units))
print('New cost: ', new_inventory.reduce(get_cost))
elif args.transfer_to:
for position in inventory:
print(' %s %s' % (args.transfer_to, position))
print(' %s %s' % (args.account, -position))
else:
for position in inventory:
print(' %s %s' % (args.account, position))
def sum_income(tx: Transaction) -> Inventory:
total = Inventory()
for posting in tx.postings:
if posting.account.split(":")[0] == "Income":
total.add_position(posting)
return total
def sum_expenses(tx: Transaction) -> Inventory:
total = Inventory()
for posting in tx.postings:
if posting.account.split(":")[0] == "Expenses":
total.add_position(posting)
return total
def process_account_entries(entries: data.Entries, config: InvestmentConfig,
investment: Investment,
check_explicit_flows: bool) -> AccountData:
"""Process a single account."""
account = investment.asset_account
logging.info("Processing account: %s", account)
# Extract the relevant transactions.
transactions = extract_transactions_for_account(entries, investment)
if not transactions:
logging.warning("No transactions for %s; skipping.", account)
return None
# Categorize the set of accounts encountered in the filtered transactions.
seen_accounts = {
posting.account
for entry in transactions for posting in entry.postings
}
catmap = categorize_accounts(config, investment, seen_accounts)
# Process each of the transactions, adding derived values as metadata.
cash_flows = []
balance = Inventory()
decorated_transactions = []
for entry in transactions:
# Compute the signature of the transaction.
entry = categorize_entry(catmap, entry)
signature = compute_transaction_signature(entry)
entry.meta["signature"] = signature
# TODO(blais): Cache balance in every transaction to speed up
# computation? Do this later.
if False:
# Update the total position in the asset we're interested in.
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
balance.add_position(posting)
# Compute the cash flows associated with the transaction.
flows_general = produce_cash_flows_general(entry, account)
if check_explicit_flows:
# Attempt the explicit method.
flows_explicit = produce_cash_flows_explicit(entry, account)
if flows_explicit != flows_general:
print(
"Differences found between general and explicit methods:")
print("Explicit handlers:")
for flow in flows_explicit:
print(" ", flow)
print("General handler:")
for flow in flows_general:
print(" ", flow)
raise ValueError(
"Differences found between general and explicit methods:")
cash_flows.extend(flows_general)
decorated_transactions.append(entry)
cost_currencies = set(cf.amount.currency for cf in cash_flows)
#assert len(cost_currencies) == 1, str(cost_currencies)
cost_currency = cost_currencies.pop() if cost_currencies else None
currency = investment.currency
commodity_map = getters.get_commodity_directives(entries)
comm = commodity_map[currency] if currency else None
open_close_map = getters.get_account_open_close(entries)
opn, cls = open_close_map[account]
# Compute the final balance.
balance = compute_balance_at(decorated_transactions)
return AccountData(account, currency, cost_currency, comm, opn, cls,
cash_flows, decorated_transactions, balance, catmap)
