def fill_residuals(entries: List[Directive],
options: Dict[str, Any]) -> List[Directive]:
ret = []
for entry in entries:
if not isinstance(entry, Transaction):
ret.append(entry)
continue
# Create residual postings for each party and add up all postings into real accounts
# If the original transaction was not balanced we book residuals to error account
# so we don't accidentially make it looks balanced for a subaccount in some cases.
original_residual = interpolate.compute_residual(entry.postings)
tolerances = interpolate.infer_tolerances(entry.postings, options)
if original_residual.is_small(tolerances):
subaccount_tmpl = '[Residuals]:[{}]'
else:
subaccount_tmpl = '[Error]:[{}]'
postings = entry.postings[:]
postings_by_party = defaultdict(list)
for posting in entry.postings:
party = posting.account.rsplit(':', 1)[1].strip('[]')
postings_by_party[party].append(posting)
for party, party_postings in postings_by_party.items():
residual = interpolate.compute_residual(party_postings)
# Use square brackets in account name to avoid collision with actual accounts
subaccount = subaccount_tmpl.format(party)
postings += utils.get_residual_postings(residual, subaccount)
ret.append(entry._replace(postings=postings))
return ret
def test_fill_residual_posting(self, entries, _, __):
"""
2001-01-01 open Assets:Account1
2001-01-01 open Assets:Other
2014-01-01 *
Assets:Account1 100.00 USD
Assets:Other -100.00 USD
2014-01-02 *
Assets:Account1 100.00 USD
Assets:Other -100.00 USD
2014-01-03 *
Assets:Account1 100.00 USD
Assets:Other -100.0000001 USD
2014-01-04 *
Assets:Account1 100.00 USD
Assets:Other -112.69 CAD @ 0.8875 USD
"""
account = 'Equity:Rounding'
entries = [entry for entry in entries if isinstance(entry, data.Transaction)]
for index in 0, 1:
entry = interpolate.fill_residual_posting(entries[index], account)
self.assertEqualEntries([entries[index]], [entry])
residual = interpolate.compute_residual(entry.postings)
self.assertTrue(residual.is_empty())
entry = interpolate.fill_residual_posting(entries[2], account)
self.assertEqualEntries("""
2014-01-03 *
Assets:Account1 100.00 USD
Assets:Other -100.0000001 USD
Equity:Rounding 0.0000001 USD
""", [entry])
residual = interpolate.compute_residual(entry.postings)
# Note: The residual calcualtion ignores postings inserted by the
# rounding account.
self.assertFalse(residual.is_empty())
self.assertEqual(inventory.from_string('-0.0000001 USD'), residual)
entry = interpolate.fill_residual_posting(entries[3], account)
self.assertEqualEntries("""
2014-01-04 *
Assets:Account1 100.00 USD
Assets:Other -112.69 CAD @ 0.8875 USD
Equity:Rounding 0.012375 USD
""", [entry])
residual = interpolate.compute_residual(entry.postings)
# Same as above.
self.assertFalse(residual.is_empty())
self.assertEqual(inventory.from_string('-0.012375 USD'), residual)
def apply_txn(self, txn: Transaction) -> typing.List[Member]:
bc_txn = txn.beancount_txn
# Ensure that the transaction balances
residual = bcinterp.compute_residual(bc_txn.postings)
tolerances = bcinterp.infer_tolerances(bc_txn.postings, self.bc_options_map)
assert residual.is_small(tolerances), "Imbalanced transaction generated"
# add the transaction to the ledger
while True:
try:
with self.git_transaction():
beancount.parser.printer.print_entry(bc_txn, file=self.instance_ledger)
self.instance_ledger.flush()
self.add_file(self.instance_ledger_name)
except subprocess.SubprocessError:
self.pull_changes()
else:
break
changed_members = {}
# Once it's durable, apply it to the live state
for posting in bc_txn.postings:
if posting.account in self.accounts_raw:
member = self.accounts_raw[posting.account]
member.balance.add_amount(posting.units)
changed_members[member.internal_name] = member
return list(changed_members.values())
def validate_check_transaction_balances(entries, options_map):
"""Check again that all transaction postings balance, as users may have
transformed transactions.
Args:
entries: A list of directives.
unused_options_map: An options map.
Returns:
A list of new errors, if any were found.
"""
# Note: this is a bit slow; we could limit our checks to the original
# transactions by using the hash function in the loader.
errors = []
for entry in entries:
if isinstance(entry, Transaction):
# IMPORTANT: This validation is _crucial_ and cannot be skipped.
# This is where we actually detect and warn on unbalancing
# transactions. This _must_ come after the user routines, because
# unbalancing input is legal, as those types of transactions may be
# "fixed up" by a user-plugin. In other words, we want to allow
# users to input unbalancing transactions as long as the final
# transactions objects that appear on the stream (after processing
# the plugins) are balanced. See {9e6c14b51a59}.
#
# Detect complete sets of postings that have residual balance;
residual = interpolate.compute_residual(entry.postings)
tolerances = interpolate.infer_tolerances(entry.postings, options_map)
if not residual.is_small(tolerances):
errors.append(
ValidationError(entry.meta,
"Transaction does not balance: {}".format(residual),
entry))
return errors
def simple_interpolation(entries, options_map):
"""Run a local interpolation on a list of incomplete entries from the parser.
Note: this does not take previous positions into account.
!WARNING!!! This destructively modifies some of the Transaction entries directly.
Args:
incomplete_entries: A list of directives, with some postings possibly left
with incomplete amounts as produced by the parser.
options_map: An options dict as produced by the parser.
Returns:
A pair of
entries: A list of interpolated entries with all their postings completed.
errors: New errors produced during interpolation.
"""
entries_with_lots, errors = convert_lot_specs_to_lots(entries, options_map)
for entry in entries_with_lots:
if not isinstance(entry, Transaction):
continue
# Balance incomplete auto-postings and set the parent link to this
# entry as well.
balance_errors = interpolate.balance_incomplete_postings(entry, options_map)
if balance_errors:
errors.extend(balance_errors)
# Check that the balance actually is empty.
if __sanity_checks__:
residual = interpolate.compute_residual(entry.postings)
tolerances = interpolate.infer_tolerances(entry.postings, options_map)
assert residual.is_small(tolerances, options_map['default_tolerance']), (
"Invalid residual {}".format(residual))
return entries_with_lots, errors
def test_compute_residual(self):
# Try with two accounts.
residual = interpolate.compute_residual([
P(None, "Assets:Bank:Checking", "105.50", "USD"),
P(None, "Assets:Bank:Checking", "-194.50", "USD"),
])
self.assertEqual(inventory.from_string("-89 USD"), residual.units())
# Try with more accounts.
residual = interpolate.compute_residual([
P(None, "Assets:Bank:Checking", "105.50", "USD"),
P(None, "Assets:Bank:Checking", "-194.50", "USD"),
P(None, "Assets:Bank:Investing", "5", "AAPL"),
P(None, "Assets:Bank:Savings", "89.00", "USD"),
])
self.assertEqual(inventory.from_string("5 AAPL"), residual.units())
def interpolate_group(postings, balances, currency, tolerances):
"""Interpolate missing numbers in the set of postings.
Args:
postings: A list of Posting instances.
balances: A dict of account to its ante-inventory.
currency: The weight currency of this group, used for reporting errors.
tolerances: A dict of currency to tolerance values.
Returns:
A tuple of
postings: A list of new posting instances.
errors: A list of errors generated during interpolation.
interpolated: A boolean, true if we did have to interpolate.
In the case of an error, this returns the original list of postings, which
is still incomplete. If an error is returned, you should probably skip the
transaction altogether, or just not include the postings in it. (An
alternative behaviour would be to return only the list of valid postings,
but that would likely result in an unbalanced transaction. We do it this
way by choice.)
"""
errors = []
# Figure out which type of amount is missing, by creating a list of
# incomplete postings and which type of units is missing.
incomplete = []
for index, posting in enumerate(postings):
units = posting.units
cost = posting.cost
price = posting.price
# Identify incomplete parts of the Posting components.
if units.number is MISSING:
incomplete.append((MissingType.UNITS, index))
if isinstance(cost, CostSpec):
if cost and cost.number_per is MISSING:
incomplete.append((MissingType.COST_PER, index))
if cost and cost.number_total is MISSING:
incomplete.append((MissingType.COST_TOTAL, index))
else:
# Check that a resolved instance of Cost never needs interpolation.
#
# Note that in theory we could support the interpolation of regular
# per-unit costs in these if we wanted to; but because they're all
# reducing postings that have been booked earlier, those never need
# to be interpolated.
if cost is not None:
assert isinstance(cost.number, Decimal), (
"Internal error: cost has no number: {}; on postings: {}".
format(cost, postings))
if price and price.number is MISSING:
incomplete.append((MissingType.PRICE, index))
# The replacement posting for the incomplete posting of this group.
new_posting = None
if len(incomplete) == 0:
# If there are no missing numbers, just convert the CostSpec to Cost and
# return that.
out_postings = [
convert_costspec_to_cost(posting) for posting in postings
]
elif len(incomplete) > 1:
# If there is more than a single value to be interpolated, generate an
# error and return no postings.
_, posting_index = incomplete[0]
errors.append(
InterpolationError(
postings[posting_index].meta,
"Too many missing numbers for currency group '{}'".format(
currency), None))
out_postings = []
else:
# If there is a single missing number, calculate it and fill it in here.
missing, index = incomplete[0]
incomplete_posting = postings[index]
# Convert augmenting postings' costs from CostSpec to corresponding Cost
# instances, except for the incomplete posting.
new_postings = [(posting if posting is incomplete_posting else
convert_costspec_to_cost(posting))
for posting in postings]
# Compute the balance of the other postings.
residual = interpolate.compute_residual(
posting for posting in new_postings
if posting is not incomplete_posting)
assert len(
residual) < 2, "Internal error in grouping postings by currencies."
if not residual.is_empty():
respos = next(iter(residual))
assert respos.cost is None, (
"Internal error; cost appears in weight calculation.")
assert respos.units.currency == currency, (
"Internal error; residual different than currency group.")
weight = -respos.units.number
weight_currency = respos.units.currency
else:
weight = ZERO
weight_currency = currency
if missing == MissingType.UNITS:
units = incomplete_posting.units
cost = incomplete_posting.cost
if cost:
# Handle the special case where we only have total cost.
if cost.number_per == ZERO:
errors.append(
InterpolationError(
incomplete_posting.meta,
"Cannot infer per-unit cost only from total",
None))
return postings, errors, True
assert cost.currency == weight_currency, (
"Internal error; residual currency different than missing currency."
)
cost_total = cost.number_total or ZERO
units_number = (weight - cost_total) / cost.number_per
elif incomplete_posting.price:
assert incomplete_posting.price.currency == weight_currency, (
"Internal error; residual currency different than missing currency."
)
units_number = weight / incomplete_posting.price.number
else:
assert units.currency == weight_currency, (
"Internal error; residual currency different than missing currency."
)
units_number = weight
# Quantize the interpolated units if necessary.
units_number = interpolate.quantize_with_tolerance(
tolerances, units.currency, units_number)
if weight != ZERO:
new_pos = Position(Amount(units_number, units.currency), cost)
new_posting = incomplete_posting._replace(units=new_pos.units,
cost=new_pos.cost)
else:
new_posting = None
elif missing == MissingType.COST_PER:
units = incomplete_posting.units
cost = incomplete_posting.cost
assert cost.currency == weight_currency, (
"Internal error; residual currency different than missing currency."
)
if units.number != ZERO:
number_per = (weight -
(cost.number_total or ZERO)) / units.number
new_cost = cost._replace(number_per=number_per)
new_pos = Position(units, new_cost)
new_posting = incomplete_posting._replace(units=new_pos.units,
cost=new_pos.cost)
else:
new_posting = None
elif missing == MissingType.COST_TOTAL:
units = incomplete_posting.units
cost = incomplete_posting.cost
assert cost.currency == weight_currency, (
"Internal error; residual currency different than missing currency."
)
number_total = (weight - cost.number_per * units.number)
new_cost = cost._replace(number_total=number_total)
new_pos = Position(units, new_cost)
new_posting = incomplete_posting._replace(units=new_pos.units,
cost=new_pos.cost)
elif missing == MissingType.PRICE:
units = incomplete_posting.units
cost = incomplete_posting.cost
if cost is not None:
errors.append(
InterpolationError(
incomplete_posting.meta,
"Cannot infer price for postings with units held at cost",
None))
return postings, errors, True
else:
price = incomplete_posting.price
assert price.currency == weight_currency, (
"Internal error; residual currency different than missing currency."
)
new_price_number = abs(weight / units.number)
new_posting = incomplete_posting._replace(
price=Amount(new_price_number, price.currency))
else:
assert False, "Internal error; Invalid missing type."
# Replace the number in the posting.
if new_posting is not None:
# Set meta-data on the new posting to indicate it was interpolated.
if new_posting.meta is None:
new_posting = new_posting._replace(meta={})
new_posting.meta[interpolate.AUTOMATIC_META] = True
# Convert augmenting posting costs from CostSpec to a corresponding
# Cost instance.
new_postings[index] = convert_costspec_to_cost(new_posting)
else:
del new_postings[index]
out_postings = new_postings
assert all(not isinstance(posting.cost, CostSpec)
for posting in out_postings)
# Check that units are non-zero and that no cost remains negative; issue an
# error if this is the case.
for posting in out_postings:
if posting.cost is None:
continue
# If there is a cost, we don't allow either a cost value of zero,
# nor a zero number of units. Note that we allow a price of zero as
# the only special case allowed (for conversion entries), but never
# for costs.
if posting.units.number == ZERO:
errors.append(
InterpolationError(
posting.meta, 'Amount is zero: "{}"'.format(posting.units),
None))
if posting.cost.number is not None and posting.cost.number < ZERO:
errors.append(
InterpolationError(
posting.meta,
'Cost is negative: "{}"'.format(posting.cost), None))
return out_postings, errors, (new_posting is not None)
def render_entry_context(entries, options_map, entry):
"""Render the context before and after a particular transaction is applied.
Args:
entries: A list of directives.
options_map: A dict of options, as produced by the parser.
entry: The entry instance which should be rendered. (Note that this object is
expected to be in the set of entries, not just structurally equal.)
Returns:
A multiline string of text, which consists of the context before the
transaction is applied, the transaction itself, and the context after it
is applied. You can just print that, it is in form that is intended to be
consumed by the user.
"""
oss = io.StringIO()
meta = entry.meta
print("Hash:{}".format(compare.hash_entry(entry)), file=oss)
print("Location: {}:{}".format(meta["filename"], meta["lineno"]), file=oss)
# Get the list of accounts sorted by the order in which they appear in the
# closest entry.
order = {}
if isinstance(entry, data.Transaction):
order = {
posting.account: index
for index, posting in enumerate(entry.postings)
}
accounts = sorted(getters.get_entry_accounts(entry),
key=lambda account: order.get(account, 10000))
# Accumulate the balances of these accounts up to the entry.
balance_before, balance_after = interpolate.compute_entry_context(
entries, entry)
# Create a format line for printing the contents of account balances.
max_account_width = max(map(len, accounts)) if accounts else 1
position_line = '{{:1}} {{:{width}}} {{:>49}}'.format(
width=max_account_width)
# Print the context before.
print(file=oss)
print("------------ Balances before transaction", file=oss)
print(file=oss)
before_hashes = set()
for account in accounts:
positions = balance_before[account].get_positions()
for position in positions:
before_hashes.add((account, hash(position)))
print(position_line.format('', account, str(position)), file=oss)
if not positions:
print(position_line.format('', account, ''), file=oss)
print(file=oss)
# Print the entry itself.
print(file=oss)
print("------------ Transaction", file=oss)
print(file=oss)
dcontext = options_map['dcontext']
printer.print_entry(entry, dcontext, render_weights=True, file=oss)
if isinstance(entry, data.Transaction):
print(file=oss)
# Print residuals.
residual = interpolate.compute_residual(entry.postings)
if not residual.is_empty():
# Note: We render the residual at maximum precision, for debugging.
print('Residual: {}'.format(residual), file=oss)
# Dump the tolerances used.
tolerances = interpolate.infer_tolerances(entry.postings, options_map)
if tolerances:
print('Tolerances: {}'.format(', '.join(
'{}={}'.format(key, value)
for key, value in sorted(tolerances.items()))),
file=oss)
# Compute the total cost basis.
cost_basis = inventory.Inventory(pos for pos in entry.postings
if pos.cost is not None).reduce(
convert.get_cost)
if not cost_basis.is_empty():
print('Basis: {}'.format(cost_basis), file=oss)
# Print the context after.
print(file=oss)
print("------------ Balances after transaction", file=oss)
print(file=oss)
for account in accounts:
positions = balance_after[account].get_positions()
for position in positions:
changed = (account, hash(position)) not in before_hashes
print(position_line.format('*' if changed else '', account,
str(position)),
file=oss)
if not positions:
print(position_line.format('', account, ''), file=oss)
print(file=oss)
return oss.getvalue()
def render_entry_context(entries, options_map, entry, parsed_entry=None):
"""Render the context before and after a particular transaction is applied.
Args:
entries: A list of directives.
options_map: A dict of options, as produced by the parser.
entry: The entry instance which should be rendered. (Note that this object is
expected to be in the set of entries, not just structurally equal.)
parsed_entry: An optional incomplete, parsed but not booked nor interpolated
entry. If this is provided, this is used for inspecting the list of prior
accounts and it is also rendered.
Returns:
A multiline string of text, which consists of the context before the
transaction is applied, the transaction itself, and the context after it
is applied. You can just print that, it is in form that is intended to be
consumed by the user.
"""
oss = io.StringIO()
pr = functools.partial(print, file=oss)
header = "** {} --------------------------------"
meta = entry.meta
pr(header.format("Transaction Id"))
pr()
pr("Hash:{}".format(compare.hash_entry(entry)))
pr("Location: {}:{}".format(meta["filename"], meta["lineno"]))
pr()
pr()
# Get the list of accounts sorted by the order in which they appear in the
# closest entry.
order = {}
if parsed_entry is None:
parsed_entry = entry
if isinstance(parsed_entry, data.Transaction):
order = {
posting.account: index
for index, posting in enumerate(parsed_entry.postings)
}
accounts = sorted(getters.get_entry_accounts(parsed_entry),
key=lambda account: order.get(account, 10000))
# Accumulate the balances of these accounts up to the entry.
balance_before, balance_after = interpolate.compute_entry_context(
entries, entry, additional_accounts=accounts)
# Create a format line for printing the contents of account balances.
max_account_width = max(map(len, accounts)) if accounts else 1
position_line = '{{:1}} {{:{width}}} {{:>49}}'.format(
width=max_account_width)
# Print the context before.
pr(header.format("Balances before transaction"))
pr()
before_hashes = set()
average_costs = {}
for account in accounts:
balance = balance_before[account]
pc_balances = balance.split()
for currency, pc_balance in pc_balances.items():
if len(pc_balance) > 1:
average_costs[account] = pc_balance.average()
positions = balance.get_positions()
for position in positions:
before_hashes.add((account, hash(position)))
pr(position_line.format('', account, str(position)))
if not positions:
pr(position_line.format('', account, ''))
pr()
pr()
# Print average cost per account, if relevant.
if average_costs:
pr(header.format("Average Costs"))
pr()
for account, average_cost in sorted(average_costs.items()):
for position in average_cost:
pr(position_line.format('', account, str(position)))
pr()
pr()
# Print the entry itself.
dcontext = options_map['dcontext']
pr(header.format("Unbooked Transaction"))
pr()
if parsed_entry:
printer.print_entry(parsed_entry,
dcontext,
render_weights=True,
file=oss)
pr()
pr(header.format("Transaction"))
pr()
printer.print_entry(entry, dcontext, render_weights=True, file=oss)
pr()
if isinstance(entry, data.Transaction):
pr(header.format("Residual and Tolerances"))
pr()
# Print residuals.
residual = interpolate.compute_residual(entry.postings)
if not residual.is_empty():
# Note: We render the residual at maximum precision, for debugging.
pr('Residual: {}'.format(residual))
# Dump the tolerances used.
tolerances = interpolate.infer_tolerances(entry.postings, options_map)
if tolerances:
pr('Tolerances: {}'.format(', '.join(
'{}={}'.format(key, value)
for key, value in sorted(tolerances.items()))))
# Compute the total cost basis.
cost_basis = inventory.Inventory(pos for pos in entry.postings
if pos.cost is not None).reduce(
convert.get_cost)
if not cost_basis.is_empty():
pr('Basis: {}'.format(cost_basis))
pr()
pr()
# Print the context after.
pr(header.format("Balances after transaction"))
pr()
for account in accounts:
positions = balance_after[account].get_positions()
for position in positions:
changed = (account, hash(position)) not in before_hashes
print(position_line.format('*' if changed else '', account,
str(position)),
file=oss)
if not positions:
pr(position_line.format('', account, ''))
pr()
return oss.getvalue()
def book(entries, options_map, unused_booking_methods):
"""Run a local interpolation on a list of incomplete entries from the parser.
Note: this does not take previous positions into account.
!WARNING!!! This destructively modifies some of the Transaction entries directly.
Args:
incomplete_entries: A list of directives, with some postings possibly left
with incomplete amounts as produced by the parser.
options_map: An options dict as produced by the parser.
unused_booking_methods: (Unused.)
Returns:
A pair of
entries: A list of interpolated entries with all their postings completed.
errors: New errors produced during interpolation.
"""
# Perform simple booking, that is convert the CostSpec instances to Cost,
# not even looking at inventory contents.
entries_with_lots, errors = convert_lot_specs_to_lots(entries)
new_entries = []
for entry in entries_with_lots:
if isinstance(entry, Transaction):
# Check that incompleteness may only occur at the posting level; reject
# otherwise. These cases are handled by the FULL booking algorithm but
# not so well by the SIMPLE algorithm. The new parsing is more liberal
# than this old code can handle, so explicitly reject cases where it
# would fail.
skip = False
for posting in entry.postings:
units = posting.units
if units is not MISSING:
if units.number is MISSING or units.currency is MISSING:
errors.append(
SimpleBookingError(
entry.meta,
"Missing number or currency on units not handled",
None))
skip = True
break
price = posting.price
if price is not None:
if price.number is MISSING or price.currency is MISSING:
errors.append(
SimpleBookingError(
entry.meta,
"Missing number or currency on price not handled",
None))
skip = True
break
if skip:
continue
# Balance incomplete auto-postings and set the parent link to this
# entry as well.
balance_errors = balance_incomplete_postings(entry, options_map)
if balance_errors:
errors.extend(balance_errors)
# Check that the balance actually is empty.
if __sanity_checks__:
residual = interpolate.compute_residual(entry.postings)
tolerances = interpolate.infer_tolerances(
entry.postings, options_map)
assert residual.is_small(
tolerances), "Invalid residual {}".format(residual)
new_entries.append(entry)
return new_entries, errors
