def balance_check(entries, options_map):
errors = []
tracking_accounts = set()
for entry in entries:
if isinstance(entry, Open):
if entry.meta.get('tracking', False):
tracking_accounts.add(entry.account)
asum = Inventory()
bsum = Inventory()
for entry in filter_txns(entries):
for posting in entry.postings:
if posting.account in tracking_accounts:
continue
components = posting.account.split(':')
if components[0] in ('Assets', 'Liabilities'):
asum.add_position(posting)
elif components[0] in ('Income', 'Expenses'):
bsum.add_position(posting)
csum = asum.reduce(convert.get_weight) + bsum.reduce(convert.get_weight)
if not csum.is_small(interpolate.infer_tolerances({}, options_map)):
errors.append(
BudgetBalanceError(
{
'filename': '<budget_balance_check>',
'lineno': 0
},
f"On-budget accounts and budget total do not match: {asum} vs {-bsum}",
None))
return entries, errors
def test_add_amount__withlots(self):
# Testing the strict case where everything matches, with only a cost.
inv = Inventory()
inv.add_amount(A('50 HOOL'), Cost(D('700'), 'USD', None, None))
self.checkAmount(inv, '50', 'HOOL')
inv.add_amount(A('-40 HOOL'), Cost(D('700'), 'USD', None, None))
self.checkAmount(inv, '10', 'HOOL')
position_, _ = inv.add_amount(A('-12 HOOL'),
Cost(D('700'), 'USD', None, None))
self.assertTrue(next(iter(inv)).is_negative_at_cost())
# Testing the strict case where everything matches, a cost and a lot-date.
inv = Inventory()
inv.add_amount(A('50 HOOL'),
Cost(D('700'), 'USD', date(2000, 1, 1), None))
self.checkAmount(inv, '50', 'HOOL')
inv.add_amount(A('-40 HOOL'),
Cost(D('700'), 'USD', date(2000, 1, 1), None))
self.checkAmount(inv, '10', 'HOOL')
position_, _ = inv.add_amount(
A('-12 HOOL'), Cost(D('700'), 'USD', date(2000, 1, 1), None))
self.assertTrue(next(iter(inv)).is_negative_at_cost())
def interval_totals(
self,
filtered: FilteredLedger,
interval: Interval,
accounts: str | tuple[str],
conversion: str,
invert: bool = False,
) -> Generator[DateAndBalanceWithBudget, None, None]:
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
invert: invert all numbers.
"""
# pylint: disable=too-many-locals
price_map = self.ledger.price_map
for begin, end in pairwise(filtered.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(filtered.entries, begin, end)
account_inventories = {}
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
if posting.account not in account_inventories:
account_inventories[posting.account] = Inventory()
account_inventories[posting.account].add_position(
posting)
inventory.add_position(posting)
balance = cost_or_value(inventory, conversion, price_map,
end - ONE_DAY)
account_balances = {}
for account, acct_value in account_inventories.items():
account_balances[account] = cost_or_value(
acct_value,
conversion,
price_map,
end - ONE_DAY,
)
budgets = {}
if isinstance(accounts, str):
budgets = self.ledger.budgets.calculate_children(
accounts, begin, end)
if invert:
# pylint: disable=invalid-unary-operand-type
balance = -balance
budgets = {k: -v for k, v in budgets.items()}
account_balances = {k: -v for k, v in account_balances.items()}
yield DateAndBalanceWithBudget(
begin,
balance,
account_balances,
budgets,
)
def test_sum_inventories(self):
inv1 = Inventory()
inv1.add_amount(A('10 USD'))
inv2 = Inventory()
inv2.add_amount(A('20 CAD'))
inv2.add_amount(A('55 HOOL'))
_ = inv1 + inv2
def find_loose_cash(accapi, options):
"""Find uninvested cash in specified accounts"""
currencies_pattern = find_cash_commodities(accapi, options)
sql = """
SELECT account AS account,
sum(position) AS position
WHERE account ~ '{accounts_pattern}'
AND not account ~ '{accounts_exclude_pattern}'
AND currency ~ '{currencies_pattern}'
GROUP BY account
ORDER BY sum(position) DESC
""".format(
accounts_pattern=options.get('accounts_pattern', '^Assets'),
accounts_exclude_pattern=options.get('accounts_exclude_pattern',
'^ $'), # TODO
currencies_pattern=currencies_pattern,
)
rtypes, rrows = accapi.query_func(sql)
if not rtypes:
return [], {}, [[]]
rrows = [r for r in rrows if r.position != Inventory()]
footer = libinvestor.build_table_footer(rtypes, rrows, accapi)
return rtypes, rrows, None, footer
def inventory_at_dates(transactions, dates, posting_predicate):
"""Generator that yields the aggregate inventory at the specified dates.
The inventory for a specified date includes all matching postings PRIOR to
it.
:param transactions: list of transactions, sorted by date.
:param dates: iterator of dates
:param posting_predicate: predicate with the Transaction and Posting to
decide whether to include the posting in the inventory.
"""
index = 0
length = len(transactions)
# inventory maps lot to amount
inventory = Inventory()
prev_date = None
for date in dates:
assert prev_date is None or date > prev_date
prev_date = date
while index < length and transactions[index].date < date:
entry = transactions[index]
index += 1
for posting in entry.postings:
if posting_predicate(posting):
inventory.add_position(posting)
yield inventory
def interval_totals(
self,
interval: Interval,
accounts: Union[str, Tuple[str]],
conversion: str,
):
"""Renders totals for account (or accounts) in the intervals.
Args:
interval: An interval.
accounts: A single account (str) or a tuple of accounts.
conversion: The conversion to use.
"""
price_map = self.ledger.price_map
for begin, end in pairwise(self.ledger.interval_ends(interval)):
inventory = Inventory()
entries = iter_entry_dates(self.ledger.entries, begin, end)
for entry in (e for e in entries if isinstance(e, Transaction)):
for posting in entry.postings:
if posting.account.startswith(accounts):
inventory.add_position(posting)
yield {
"date":
begin,
"balance":
cost_or_value(inventory, conversion, price_map, end - ONE_DAY),
"budgets":
self.ledger.budgets.calculate_children(accounts, begin, end),
}
def test_add_amount__allow_negative(self):
inv = Inventory()
# Test adding positions of different types.
position_, _ = inv.add_amount(A('-11 USD'))
self.assertIsNone(position_)
position_, _ = inv.add_amount(A('-11 USD'),
Cost(D('1.10'), 'CAD', None, None))
self.assertIsNone(position_)
position_, _ = inv.add_amount(A('-11 USD'),
Cost(D('1.10'), 'CAD', date(2012, 1, 1), None))
self.assertIsNone(position_)
# Check for reductions.
invlist = list(inv)
self.assertTrue(invlist[1].is_negative_at_cost())
self.assertTrue(invlist[2].is_negative_at_cost())
inv.add_amount(A('-11 USD'), Cost(D('1.10'), 'CAD', None, None))
inv.add_amount(A('-11 USD'), Cost(D('1.10'), 'CAD', date(2012, 1, 1), None))
self.assertEqual(3, len(inv))
# Test adding to a position that does exist.
inv = I('10 USD, 10 USD {1.10 CAD}, 10 USD {1.10 CAD, 2012-01-01}')
position_, _ = inv.add_amount(A('-11 USD'))
self.assertEqual(position_, position.from_string('10 USD'))
position_, _ = inv.add_amount(A('-11 USD'),
Cost(D('1.10'), 'CAD', None, None))
self.assertEqual(position_, position.from_string('10 USD {1.10 CAD}'))
position_, _ = inv.add_amount(A('-11 USD'),
Cost(D('1.10'), 'CAD', date(2012, 1, 1), None))
self.assertEqual(position_, position.from_string('10 USD {1.10 CAD, 2012-01-01}'))
def measure_balance_total(entries):
# For timing only.
with utils.log_time('balance_all', logging.info): # 89ms... okay.
balance = Inventory()
for entry in utils.filter_type(entries, data.Transaction):
for posting in entry.postings:
balance.add_position(posting.position)
def _total_balance(self, names, begin_date, end_date):
totals = [realization.compute_balance(
_real_account(account_name, self.ledger.entries, begin_date,
end_date))
for account_name in names]
return _serialize_inventory(sum(totals, Inventory()),
at_cost=True)
def net_worth(self, interval):
"""Compute net worth.
Args:
interval: A string for the interval.
Returns:
A list of dicts for all ends of the given interval containing the
net worth (Assets + Liabilities) separately converted to all
operating currencies.
"""
transactions = (entry for entry in self.ledger.entries
if (isinstance(entry, Transaction)
and entry.flag != flags.FLAG_UNREALIZED))
types = (
self.ledger.options["name_assets"],
self.ledger.options["name_liabilities"],
)
txn = next(transactions, None)
inventory = Inventory()
for end_date_exclusive in self.ledger.interval_ends(interval):
end_date_inclusive = end_date_exclusive - datetime.timedelta(
days=1)
while txn and txn.date < end_date_exclusive:
for posting in txn.postings:
if posting.account.startswith(types):
inventory.add_position(posting)
txn = next(transactions, None)
yield {
"date": end_date_exclusive,
"balance": cost_or_value(inventory, end_date_inclusive),
}
def process_account_entries(entries: data.Entries, options_map: data.Options,
account: Account) -> AccountData:
"""Process a single account."""
logging.info("Processing account: %s", account)
# Extract the relevant transactions.
transactions = transactions_for_account(entries, account)
if not transactions:
logging.warning("No transactions for %s; skipping.", account)
return transactions, None, None
# Categorize the set of accounts encountered in the filtered transactions.
seen_accounts = {
posting.account
for entry in transactions for posting in entry.postings
}
atypes = options.get_account_types(options_map)
catmap = categorize_accounts(account, seen_accounts, atypes)
# Process each of the transactions, adding derived values as metadata.
cash_flows = []
balance = Inventory()
decorated_transactions = []
for entry in transactions:
# Update the total position in the asset we're interested in.
positions = []
for posting in entry.postings:
category = catmap[posting.account]
if category is Cat.ASSET:
balance.add_position(posting)
positions.append(posting)
# Compute the signature of the transaction.
entry = copy_and_normalize(entry)
signature = compute_transaction_signature(catmap, entry)
entry.meta["signature"] = signature
entry.meta["description"] = KNOWN_SIGNATURES[signature]
# Compute the cash flows associated with the transaction.
flows = produce_cash_flows(entry)
entry.meta['cash_flows'] = flows
cash_flows.extend(
flow._replace(balance=copy.deepcopy(balance)) for flow in flows)
decorated_transactions.append(entry)
currency = accountlib.leaf(account)
cost_currencies = set(cf.amount.currency for cf in cash_flows)
assert len(cost_currencies) == 1, str(cost_currencies)
cost_currency = cost_currencies.pop()
commodity_map = getters.get_commodity_directives(entries)
comm = commodity_map[currency]
return AccountData(account, currency, cost_currency, comm, cash_flows,
decorated_transactions, catmap)
def test_op_neg(self):
inv = Inventory()
inv.add_amount(A('10 USD'))
ninv = -inv
self.checkAmount(ninv, '-10', 'USD')
pinv = I('1.50 JPY, 1.51 USD, 1.52 CAD')
ninv = I('-1.50 JPY, -1.51 USD, -1.52 CAD')
self.assertEqual(pinv, -ninv)
def test_currencies(self):
inv = Inventory()
self.assertEqual(set(), inv.currencies())
inv = I('40 USD {1.01 CAD}, 40 USD')
self.assertEqual({'USD'}, inv.currencies())
inv = I('40 AAPL {1.01 USD}, 10 HOOL {2.02 USD}')
self.assertEqual({'AAPL', 'HOOL'}, inv.currencies())
def test_currency_pairs(self):
inv = Inventory()
self.assertEqual(set(), inv.currency_pairs())
inv = I('40 USD {1.01 CAD}, 40 USD')
self.assertEqual(set([('USD', 'CAD'), ('USD', None)]), inv.currency_pairs())
inv = I('40 AAPL {1.01 USD}, 10 HOOL {2.02 USD}')
self.assertEqual(set([('AAPL', 'USD'), ('HOOL', 'USD')]), inv.currency_pairs())
def test_add_amount__multi_currency(self):
inv = Inventory()
inv.add_amount(A('100 USD'))
inv.add_amount(A('100 CAD'))
self.checkAmount(inv, '100', 'USD')
self.checkAmount(inv, '100', 'CAD')
inv.add_amount(A('25 USD'))
self.checkAmount(inv, '125', 'USD')
self.checkAmount(inv, '100', 'CAD')
def find_balance_before(cash_flows: List[CashFlow],
date: Date) -> Tuple[Inventory, int]:
"""Return the balance just before the given date in the sorted list of cash flows."""
balance = Inventory()
for index, flow in enumerate(cash_flows):
if flow.date >= date:
break
balance = flow.balance
else:
index = len(cash_flows)
return balance, index
def test_get_position(self):
inv = Inventory(self.POSITIONS_ALL_KINDS)
self.assertEqual(
position.from_string('40.50 USD'),
inv.get_position(Lot('USD', None, None)))
self.assertEqual(
position.from_string('40.50 USD {1.10 CAD}'),
inv.get_position(Lot('USD', A('1.10 CAD'), None)))
self.assertEqual(
position.from_string('40.50 USD {1.10 CAD, 2012-01-01}'),
inv.get_position(Lot('USD', A('1.10 CAD'), date(2012, 1, 1))))
def compute_balance_at(transactions: data.Entries,
date: Optional[Date] = None) -> Inventory:
"""Compute the balance at a specific date."""
balance = Inventory()
for entry in transactions:
if date is not None and entry.date >= date:
break
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
balance.add_position(posting)
return balance
def test_ctor_empty_len(self):
# Test regular constructor.
inv = Inventory()
self.assertTrue(inv.is_empty())
self.assertEqual(0, len(inv))
inv = Inventory([P('100.00 USD'), P('101.00 USD')])
self.assertFalse(inv.is_empty())
self.assertEqual(1, len(inv))
inv = Inventory([P('100.00 USD'), P('100.00 CAD')])
self.assertFalse(inv.is_empty())
self.assertEqual(2, len(inv))
inv = Inventory()
self.assertEqual(0, len(inv))
inv.add_amount(A('100 USD'))
self.assertEqual(1, len(inv))
inv.add_amount(A('100 CAD'))
self.assertEqual(2, len(inv))
def test_copy(self):
inv = Inventory()
inv.add_amount(A('100.00 USD'))
self.checkAmount(inv, '100', 'USD')
# Test copying.
inv2 = copy.copy(inv)
inv2.add_amount(A('50.00 USD'))
self.checkAmount(inv2, '150', 'USD')
# Check that the original object is not modified.
self.checkAmount(inv, '100', 'USD')
def get_inventory(self, account, date):
inventory = Inventory()
for entry in self.entries:
if date is not None and entry.date > date:
break
if not isinstance(entry, Transaction):
continue
for posting in entry.postings:
if posting.account != account:
continue
inventory.add_position(get_position(posting))
return inventory
def SumPostings(postings: Iterable[Posting]) -> Inventory:
"""Aggregate all the positions for the given postings."""
acc = Inventory()
for elem in postings:
# Note: 'elem' can be a Posting or an Inventory (the intermediate result
# from a subcombiner).
if isinstance(elem, Inventory):
acc.add_inventory(elem)
else:
assert isinstance(elem, Posting)
acc.add_position(elem)
return acc
def test_add_amount__booking(self):
inv = Inventory()
_, booking = inv.add_amount(A('100.00 USD'))
self.assertEqual(Booking.CREATED, booking)
_, booking = inv.add_amount(A('20.00 USD'))
self.assertEqual(Booking.AUGMENTED, booking)
_, booking = inv.add_amount(A('-20 USD'))
self.assertEqual(Booking.REDUCED, booking)
_, booking = inv.add_amount(A('-100 USD'))
self.assertEqual(Booking.REDUCED, booking)
def recently_sold_at_loss(accapi, options):
"""Looking back 30 days for sales that caused losses. These were likely to have been TLH (but not
necessarily so. This tells us what NOT to buy in order to avoid wash sales."""
operating_currencies = accapi.get_operating_currencies_regex()
wash_pattern = options.get('wash_pattern', '')
account_field = options.get('account_field', 'LEAF(account)')
wash_pattern_sql = 'AND account ~ "{}"'.format(
wash_pattern) if wash_pattern else ''
sql = '''
SELECT
date as sale_date,
DATE_ADD(date, 30) as until,
currency,
NEG(SUM(COST(position))) as basis,
NEG(SUM(CONVERT(position, cost_currency, date))) as proceeds
WHERE
date >= DATE_ADD(TODAY(), -30)
AND number < 0
AND not currency ~ "{operating_currencies}"
GROUP BY sale_date,until,currency
'''.format(**locals())
rtypes, rrows = accapi.query_func(sql)
if not rtypes:
return [], []
# filter out losses
retrow_types = rtypes + [('loss', Inventory)]
RetRow = collections.namedtuple('RetRow', [i[0] for i in retrow_types])
return_rows = []
for row in rrows:
loss = Inventory(row.proceeds)
loss.add_inventory(-(row.basis))
if loss != Inventory() and val(loss) < 0:
return_rows.append(RetRow(*row, loss))
footer = build_table_footer(retrow_types, return_rows, accapi)
return retrow_types, return_rows, None, footer
def compute_balance_by_type(real_accounts, date):
"""Compute the total balance for each account type, evaluated at the given
date. Returns a tuple with an inventor for each accoutn type."""
balances = {typename: Inventory()
for typename in AccountTypeData._fields}
for real_account in real_accounts:
if real_account.postings:
typename = account_type(real_account.fullname)
assert False, "FIXME: This doesn't work anymore; we need to use the entries in order to compute the balance at a specific date."
balance = realization.find_balance(real_account, date)
balances[typename] += balance
return AccountTypeData(**balances)
def test_units1(self):
inv = Inventory()
self.assertEqual(inv.units(), Inventory.from_string(''))
inv = Inventory.from_string('40.50 JPY, 40.51 USD {1.01 CAD}, 40.52 CAD')
self.assertEqual(inv.units(),
Inventory.from_string('40.50 JPY, 40.51 USD, 40.52 CAD'))
# Check that the same units coalesce.
inv = Inventory.from_string('2 HOOL {400 USD}, 3 HOOL {410 USD}')
self.assertEqual(inv.units(), Inventory.from_string('5 HOOL'))
inv = Inventory.from_string('2 HOOL {400 USD}, -3 HOOL {410 USD}')
self.assertEqual(inv.units(), Inventory.from_string('-1 HOOL'))
def test_units1(self):
inv = Inventory()
self.assertEqual(inv.reduce(convert.get_units), I(''))
inv = I('40.50 JPY, 40.51 USD {1.01 CAD}, 40.52 CAD')
self.assertEqual(inv.reduce(convert.get_units),
I('40.50 JPY, 40.51 USD, 40.52 CAD'))
# Check that the same units coalesce.
inv = I('2 HOOL {400 USD}, 3 HOOL {410 USD}')
self.assertEqual(inv.reduce(convert.get_units), I('5 HOOL'))
inv = I('2 HOOL {400 USD}, -3 HOOL {410 USD}')
self.assertEqual(inv.reduce(convert.get_units), I('-1 HOOL'))
def compute_portfolio_values(
price_map: prices.PriceMap,
transactions: data.Entries) -> Tuple[List[Date], List[float]]:
"""Compute a serie of portfolio values over time."""
# Infer the list of required prices.
currency_pairs = set()
for entry in transactions:
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
if posting.cost:
currency_pairs.add(
(posting.units.currency, posting.cost.currency))
first = lambda x: x[0]
price_dates = sorted(itertools.chain(
((date, None) for pair in currency_pairs
for date, _ in prices.get_all_prices(price_map, pair)),
((entry.date, entry) for entry in transactions)),
key=first)
# Iterate computing the balance.
value_dates = []
value_values = []
balance = Inventory()
for date, group in itertools.groupby(price_dates, key=first):
# Update balances.
for _, entry in group:
if entry is None:
continue
for posting in entry.postings:
if posting.meta["category"] is Cat.ASSET:
balance.add_position(posting)
# Convert to market value.
value_balance = balance.reduce(convert.get_value, price_map, date)
cost_balance = value_balance.reduce(convert.convert_position, "USD",
price_map)
pos = cost_balance.get_only_position()
value = pos.units.number if pos else ZERO
# Add one data point.
value_dates.append(date)
value_values.append(value)
return value_dates, value_values
def tracking(entries, options_map):
account_types = get_account_types(options_map)
income_tracking = set()
expense_tracking = set()
errors = []
new_entries = []
for entry in entries:
new_entry = None
if isinstance(entry, Open) and entry.meta.get('tracking', False):
if is_account_type(account_types.expenses, entry.account):
expense_tracking.add(entry.account)
elif is_account_type(account_types.income, entry.account):
income_tracking.add(entry.account)
elif isinstance(entry, Transaction):
new_postings = []
tracking_balance = Inventory()
for posting in entry.postings:
if 'tracking' in posting.meta:
new_acct = posting.meta['tracking']
new_posting = posting._replace(account=new_acct, meta=None)
new_postings.append(new_posting)
tracking_balance.add_position(posting)
if new_postings:
for position in -tracking_balance:
if position.units.number < 0 and len(income_tracking) == 1:
posting_acct, = income_tracking
elif position.units.number > 0 and len(
expense_tracking) == 1:
posting_acct, = expense_tracking
else:
continue
new_posting = Posting(posting_acct, position.units,
position.cost, None, None, None)
new_postings.append(new_posting)
link_id = 'tracking-' + compare.hash_entry(entry)
new_links = entry.links | set([link_id])
entry = entry._replace(links=new_links)
new_entry = entry._replace(postings=new_postings)
new_entries.append(entry)
if new_entry:
new_entries.append(new_entry)
return new_entries, errors
