def holding_to_position(holding):
"""Convert the holding to a position.
Args:
holding: An instance of Holding.
Returns:
An instance of Position.
"""
return position.Position(
amount.Amount(holding.number, holding.currency),
(position.Cost(holding.cost_number, holding.cost_currency, None, None)
if holding.cost_number else None))
def test_holding_to_posting(self):
test_holding = holdings.Holding(
'Assets:US:Checking', D('100'), 'MSFT', D('54.34'), 'USD',
D('5434.00'), D('6000.00'), D('60'), datetime.date(2012, 5, 2))
posting = holdings.holding_to_posting(test_holding)
self.assertTrue(isinstance(posting, data.Posting))
expected_position = position.from_string('100 MSFT {54.34 USD}')
self.assertEqual(expected_position, position.Position(posting.units, posting.cost))
expected_price = A('60.00 USD')
self.assertEqual(expected_price, posting.price)
def augment_inventory(pending_lots, posting, eindex):
"""Add the lots from the given posting to the running inventory.
Args:
pending_lots: A list of pending ([number], Posting, Transaction) to be matched.
The number is modified in-place, destructively.
posting: The posting whose position is to be added.
eindex: The index of the parent transaction housing this posting.
Returns:
A new posting with cost basis inserted to be added to a transformed transaction.
"""
lot = posting.position.lot._replace(cost=copy.copy(posting.price))
number = posting.position.number
pos = position.Position(lot, number)
new_posting = posting._replace(position=pos)
pending_lots.append(([number], new_posting, eindex))
return new_posting
def add_postings(entry, amount_, neg_account, pos_account, flag):
"""Insert positive and negative postings of a position in an entry.
Args:
entry: A Transaction instance.
amount_: An Amount instance to create the position, with positive number.
neg_account: An account for the posting with the negative amount.
pos_account: An account for the posting with the positive amount.
flag: A string, that is to be set as flag for the new postings.
Returns:
A new, modified entry.
"""
pos = position.Position(position.Lot(amount_.currency, None, None),
amount_.number)
return entry._replace(postings=entry.postings + [
data.Posting(neg_account, pos.get_negative(), None, flag, None),
data.Posting(pos_account, pos, None, flag, None),
])
def booking_method_AVERAGE(entry, posting, matches):
"""AVERAGE booking method implementation."""
booked_reductions = []
booked_matches = []
# If there is more than a single match we need to ultimately merge the
# postings. Also, if the reducing posting provides a specific cost, we
# need to update the cost basis as well. Both of these cases are carried
# out by removing all the matches and readding them later on.
if len(matches) == 1 and (
not isinstance(posting.cost.number_per, Decimal)
and not isinstance(posting.cost.number_total, Decimal)):
# There is no cost. Just reduce the one leg. This should be the
# normal case if we always merge augmentations and the user lets
# Beancount deal with the cost.
match = matches[0]
sign = -1 if posting.units.number < ZERO else 1
number = min(abs(match.units.number), abs(posting.units.number))
match_units = Amount(number * sign, match.units.currency)
booked_reductions.append(
posting._replace(units=match_units, cost=match.cost))
insufficient = (match_units.number != posting.units.number)
return booked_reductions, [match], [], insufficient
else:
# Merge the matching postings to a single one.
merged_units = inventory.Inventory()
merged_cost = inventory.Inventory()
for match in matches:
merged_units.add_amount(match.units)
merged_cost.add_amount(convert.get_weight(match))
if len(merged_units) != 1 or len(merged_cost) != 1:
errors.append(
AmbiguousMatchError(
entry.meta,
'Cannot merge positions in multiple currencies: {}'.format(
', '.join(
position.to_string(match_posting)
for match_posting in matches)), entry))
return [], [], errors, False
else:
if (isinstance(posting.cost.number_per, Decimal)
or isinstance(posting.cost.number_total, Decimal)):
errors.append(
AmbiguousMatchError(
entry.meta,
"Explicit cost reductions aren't supported yet: {}".
format(position.to_string(posting)), entry))
return [], [], errors, False
else:
# Insert postings to remove all the matches.
booked_reductions.extend(
posting._replace(units=-match.units,
cost=match.cost,
flag=flags.FLAG_MERGING)
for match in matches)
units = merged_units.get_only_position().units
dates = {match.cost.date for match in matches}
cost_units = merged_cost.get_only_position().units
cost = Cost(cost_units.number / units.number,
cost_units.currency,
list(dates)[0] if len(dates) == 1 else None, None)
# Insert a posting to refill those with a replacement match.
booked_reductions.append(
posting._replace(units=units,
cost=cost,
flag=flags.FLAG_MERGING))
# Now, match the reducing request against this lot.
booked_reductions.append(
posting._replace(units=posting.units, cost=cost))
insufficient = abs(posting.units.number) > abs(units.number)
return booked_reductions, [
position.Position(posting.units, cost)
], [], insufficient
def add_unrealized_gains(entries, options_map, subaccount=None):
"""Insert entries for unrealized capital gains.
This function inserts entries that represent unrealized gains, at the end of
the available history. It returns a new list of entries, with the new gains
inserted. It replaces the account type with an entry in an income account.
Optionally, it can book the gain in a subaccount of the original and income
accounts.
Args:
entries: A list of data directives.
options_map: A dict of options, that confirms to beancount.parser.options.
subaccount: A string, and optional the name of a subaccount to create
under an account to book the unrealized gain. If this is left to its
default value, the gain is booked directly in the same account.
Returns:
A list of entries, which includes the new unrealized capital gains entries
at the end, and a list of errors. The new list of entries is still sorted.
"""
errors = []
meta = data.new_metadata('<unrealized_gains>', 0)
account_types = options.get_account_types(options_map)
# Assert the subaccount name is in valid format.
if subaccount:
validation_account = account.join(account_types.assets, subaccount)
if not account.is_valid(validation_account):
errors.append(
UnrealizedError(
meta, "Invalid subaccount name: '{}'".format(subaccount),
None))
return entries, errors
if not entries:
return (entries, errors)
# Group positions by (account, cost, cost_currency).
price_map = prices.build_price_map(entries)
holdings_list = holdings.get_final_holdings(entries, price_map=price_map)
# Group positions by (account, cost, cost_currency).
holdings_list = holdings.aggregate_holdings_by(
holdings_list, lambda h: (h.account, h.currency, h.cost_currency))
# Get the latest prices from the entries.
price_map = prices.build_price_map(entries)
# Create transactions to account for each position.
new_entries = []
latest_date = entries[-1].date
for index, holding in enumerate(holdings_list):
if (holding.currency == holding.cost_currency
or holding.cost_currency is None):
continue
# Note: since we're only considering positions held at cost, the
# transaction that created the position *must* have created at least one
# price point for that commodity, so we never expect for a price not to
# be available, which is reasonable.
if holding.price_number is None:
# An entry without a price might indicate that this is a holding
# resulting from leaked cost basis. {0ed05c502e63, b/16}
if holding.number:
errors.append(
UnrealizedError(
meta,
"A valid price for {h.currency}/{h.cost_currency} "
"could not be found".format(h=holding), None))
continue
# Compute the PnL; if there is no profit or loss, we create a
# corresponding entry anyway.
pnl = holding.market_value - holding.book_value
if holding.number == ZERO:
# If the number of units sum to zero, the holdings should have been
# zero.
errors.append(
UnrealizedError(
meta,
"Number of units of {} in {} in holdings sum to zero "
"for account {} and should not".format(
holding.currency, holding.cost_currency,
holding.account), None))
continue
# Compute the name of the accounts and add the requested subaccount name
# if requested.
asset_account = holding.account
income_account = account.join(account_types.income,
account.sans_root(holding.account))
if subaccount:
asset_account = account.join(asset_account, subaccount)
income_account = account.join(income_account, subaccount)
# Create a new transaction to account for this difference in gain.
gain_loss_str = "gain" if pnl > ZERO else "loss"
narration = (
"Unrealized {} for {h.number} units of {h.currency} "
"(price: {h.price_number:.4f} {h.cost_currency} as of {h.price_date}, "
"average cost: {h.cost_number:.4f} {h.cost_currency})").format(
gain_loss_str, h=holding)
entry = data.Transaction(
data.new_metadata(meta["filename"],
lineno=1000 + index), latest_date,
flags.FLAG_UNREALIZED, None, narration, None, None, [])
# Book this as income, converting the account name to be the same, but as income.
# Note: this is a rather convenient but arbitraty choice--maybe it would be best to
# let the user decide to what account to book it, but I don't a nice way to let the
# user specify this.
#
# Note: we never set a price because we don't want these to end up in Conversions.
entry.postings.extend([
data.Posting(
asset_account,
position.Position(
position.Lot(holding.cost_currency, None, None), pnl),
None, None, None),
data.Posting(
income_account,
position.Position(
position.Lot(holding.cost_currency, None, None), -pnl),
None, None, None)
])
new_entries.append(entry)
# Ensure that the accounts we're going to use to book the postings exist, by
# creating open entries for those that we generated that weren't already
# existing accounts.
new_accounts = {
posting.account
for entry in new_entries for posting in entry.postings
}
open_entries = getters.get_account_open_close(entries)
new_open_entries = []
for account_ in sorted(new_accounts):
if account_ not in open_entries:
meta = data.new_metadata(meta["filename"], index)
open_entry = data.Open(meta, latest_date, account_, None, None)
new_open_entries.append(open_entry)
return (entries + new_open_entries + new_entries, errors)
def pad(entries, options_map):
"""Insert transaction entries for to fulfill a subsequent balance check.
Synthesize and insert Transaction entries right after Pad entries in order
to fulfill checks in the padded accounts. Returns a new list of entries.
Note that this doesn't pad across parent-child relationships, it is a very
simple kind of pad. (I have found this to be sufficient in practice, and
simpler to implement and understand.)
Furthermore, this pads for a single currency only, that is, balance checks
are specified only for one currency at a time, and pads will only be
inserted for those currencies.
Args:
entries: A list of directives.
options_map: A parser options dict.
Returns:
A new list of directives, with Pad entries inserte, and a list of new
errors produced.
"""
pad_errors = []
# Find all the pad entries and group them by account.
pads = list(misc_utils.filter_type(entries, data.Pad))
pad_dict = misc_utils.groupby(lambda x: x.account, pads)
# Partially realize the postings, so we can iterate them by account.
by_account = realization.postings_by_account(entries)
# A dict of pad -> list of entries to be inserted.
new_entries = {id(pad): [] for pad in pads}
# Process each account that has a padding group.
for account_, pad_list in sorted(pad_dict.items()):
# Last encountered / currency active pad entry.
active_pad = None
# Gather all the postings for the account and its children.
postings = []
is_child = account.parent_matcher(account_)
for item_account, item_postings in by_account.items():
if is_child(item_account):
postings.extend(item_postings)
postings.sort(key=data.posting_sortkey)
# A set of currencies already padded so far in this account.
padded_lots = set()
pad_balance = inventory.Inventory()
for entry in postings:
assert not isinstance(entry, data.Posting)
if isinstance(entry, data.TxnPosting):
# This is a transaction; update the running balance for this
# account.
pad_balance.add_position(entry.posting.position)
elif isinstance(entry, data.Pad):
if entry.account == account_:
# Mark this newly encountered pad as active and allow all lots
# to be padded heretofore.
active_pad = entry
padded_lots = set()
elif isinstance(entry, data.Balance):
check_amount = entry.amount
# Compare the current balance amount to the expected one from
# the check entry. IMPORTANT: You need to understand that this
# does not check a single position, but rather checks that the
# total amount for a particular currency (which itself is
# distinct from the cost).
balance_amount = pad_balance.get_units(check_amount.currency)
diff_amount = amount.amount_sub(balance_amount, check_amount)
# Use the specified tolerance or automatically infer it.
tolerance = balance.get_tolerance(entry, options_map)
if abs(diff_amount.number) > tolerance:
# The check fails; we need to pad.
# Pad only if pad entry is active and we haven't already
# padded that lot since it was last encountered.
if active_pad and (check_amount.currency
not in padded_lots):
# Note: we decide that it's an error to try to pad
# positions at cost; we check here that all the existing
# positions with that currency have no cost.
positions = [
pos for pos in pad_balance.get_positions()
if pos.lot.currency == check_amount.currency
]
for position_ in positions:
if position_.lot.cost is not None:
pad_errors.append(
PadError(entry.meta, (
"Attempt to pad an entry with cost for "
"balance: {}".format(pad_balance)),
active_pad))
# Thus our padding lot is without cost by default.
lot = position.Lot(check_amount.currency, None, None)
diff_position = position.Position(
lot, check_amount.number - balance_amount.number)
# Synthesize a new transaction entry for the difference.
narration = ('(Padding inserted for Balance of {} for '
'difference {})').format(
check_amount, diff_position)
new_entry = data.Transaction(active_pad.meta.copy(),
active_pad.date,
flags.FLAG_PADDING, None,
narration, None, None, [])
new_entry.postings.append(
data.Posting(active_pad.account, diff_position,
None, None, None))
new_entry.postings.append(
data.Posting(active_pad.source_account,
-diff_position, None, None, None))
# Save it for later insertion after the active pad.
new_entries[id(active_pad)].append(new_entry)
# Fixup the running balance.
position_, _ = pad_balance.add_position(diff_position)
if position_.is_negative_at_cost():
raise ValueError(
"Position held at cost goes negative: {}".
format(position_))
# Mark this lot as padded. Further checks should not pad this lot.
padded_lots.add(check_amount.currency)
# Insert the newly created entries right after the pad entries that created them.
padded_entries = []
for entry in entries:
padded_entries.append(entry)
if isinstance(entry, data.Pad):
entry_list = new_entries[id(entry)]
if entry_list:
padded_entries.extend(entry_list)
else:
# Generate errors on unused pad entries.
pad_errors.append(
PadError(entry.meta, "Unused Pad entry", entry))
return padded_entries, pad_errors
def __call__(self, context):
posting = context.posting
return position.Position(posting.units, posting.cost)
def book_price_conversions(entries, assets_account, income_account):
"""Rewrite transactions to insert cost basis according to a booking method.
See module docstring for full details.
Args:
entries: A list of entry instances.
assets_account: An account string, the name of the account to process.
income_account: An account string, the name of the account to use for booking
realized profit/loss.
Returns:
A tuple of
entries: A list of new, modified entries.
errors: A list of errors generated by this plugin.
matches: A list of (number, augmenting-posting, reducing-postings) for all
matched lots.
"""
# Pairs of (Position, Transaction) instances used to match augmenting
# entries with reducing ones.
pending_lots = []
# A list of pairs of matching (augmenting, reducing) postings.
all_matches = []
new_entries = []
errors = []
for eindex, entry in enumerate(entries):
# Figure out if this transaction has postings in Bitcoins without a cost.
# The purpose of this plugin is to fixup those.
if isinstance(entry, data.Transaction) and any(
is_matching(posting, assets_account)
for posting in entry.postings):
# Segregate the reducing lots, augmenting lots and other lots.
augmenting, reducing, other = [], [], []
for pindex, posting in enumerate(entry.postings):
if is_matching(posting, assets_account):
out = augmenting if posting.position.number >= ZERO else reducing
else:
out = other
out.append(posting)
# We will create a replacement list of postings with costs filled
# in, possibly more than the original list, to account for the
# different lots.
new_postings = []
# Convert all the augmenting postings to cost basis.
for posting in augmenting:
new_postings.append(
augment_inventory(pending_lots, posting, eindex))
# Then process reducing postings.
if reducing:
# Process all the reducing postings, booking them to matching lots.
pnl = inventory.Inventory()
for posting in reducing:
rpostings, matches, posting_pnl, new_errors = (
reduce_inventory(pending_lots, posting, eindex))
new_postings.extend(rpostings)
all_matches.extend(matches)
errors.extend(new_errors)
pnl.add_amount(
amount.Amount(posting_pnl, posting.price.currency))
# If some reducing lots were seen in this transaction, insert an
# Income leg to absorb the P/L. We need to do this for each currency
# which incurred P/L.
if not pnl.is_empty():
for pos in pnl:
meta = data.new_metadata('<book_conversions>', 0)
new_postings.append(
data.Posting(
income_account,
position.Position(
position.Lot(pos.lot.currency, None, None),
-pos.number), None, None, meta))
# Third, add back all the other unrelated legs in.
for posting in other:
new_postings.append(posting)
# Create a replacement entry.
entry = entry._replace(postings=new_postings)
new_entries.append(entry)
# Add matching metadata to all matching postings.
mod_matches = link_entries_with_metadata(new_entries, all_matches)
# Resolve the indexes to their possibly modified Transaction instances.
matches = [(data.TxnPosting(new_entries[aug_index], aug_posting),
data.TxnPosting(new_entries[red_index], red_posting))
for (aug_index, aug_posting), (red_index,
red_posting) in mod_matches]
return new_entries, errors, matches
def reduce_inventory(pending_lots, posting, eindex):
"""Match a reducing posting against a list of lots (using FIFO order).
Args:
pending_lots: A list of pending ([number], Posting, Transaction) to be matched.
The number is modified in-place, destructively.
posting: The posting whose position is to be added.
eindex: The index of the parent transaction housing this posting.
Returns:
A tuple of
postings: A list of new Posting instances corresponding to the given
posting, that were booked to the current list of lots.
matches: A list of pairs of (augmenting-posting, reducing-posting).
pnl: A Decimal, the P/L incurred in reducing these lots.
errors: A list of new errors generated in reducing these lots.
"""
new_postings = []
matches = []
pnl = ZERO
errors = []
match_number = -posting.position.number
match_currency = posting.position.lot.currency
cost_currency = posting.price.currency
while match_number != ZERO:
# Find the first lot with matching currency.
for fnumber, fposting, findex in pending_lots:
fposition = fposting.position
if (fposition.lot.currency == match_currency and fposition.lot.cost
and fposition.lot.cost.currency == cost_currency):
assert fnumber[0] > ZERO, "Internal error, zero lot"
break
else:
errors.append(
BookConversionError(
posting.meta,
"Could not match position {}".format(posting), None))
break
# Reduce the pending lots.
number = min(match_number, fnumber[0])
cost = fposition.lot.cost
match_number -= number
fnumber[0] -= number
if fnumber[0] == ZERO:
pending_lots.pop(0)
# Add a corresponding posting.
pos = position.Position(
posting.position.lot._replace(cost=copy.copy(cost)), -number)
rposting = posting._replace(position=pos)
new_postings.append(rposting)
# Update the P/L.
pnl += number * (posting.price.number - pos.lot.cost.number)
# Add to the list of matches.
matches.append(((findex, fposting), (eindex, rposting)))
return new_postings, matches, pnl, errors
def split_currency_conversions(entry):
"""If the transcation has a mix of conversion at cost and a
currency conversion, split the transction into two transactions: one
that applies the currency conversion in the same account, and one
that uses the other currency without conversion.
This is required because Ledger does not appear to be able to grok a
transaction like this one:
2014-11-02 * "Buy some stock with foreign currency funds"
Assets:CA:Investment:HOOL 5 HOOL {520.0 USD}
Expenses:Commissions 9.95 USD
Assets:CA:Investment:Cash -2939.46 CAD @ 0.8879 USD
HISTORICAL NOTE: Adding a price directive on the first posting above makes
Ledger accept the transaction. So we will not split the transaction here
now. However, since Ledger's treatment of this type of conflict is subject
to revision (See http://bugs.ledger-cli.org/show_bug.cgi?id=630), we will
keep this code around, it might become useful eventually. See
https://groups.google.com/d/msg/ledger-cli/35hA0Dvhom0/WX8gY_5kHy0J for
details of the discussion.
Args:
entry: An instance of Transaction.
Returns:
A pair of
converted: boolean, true if a conversion was made.
entries: A list of the original entry if converted was False,
or a list of the split converted entries if True.
"""
assert isinstance(entry, data.Transaction)
(postings_simple, postings_at_price,
postings_at_cost) = postings_by_type(entry)
converted = postings_at_cost and postings_at_price
if converted:
# Generate a new entry for each currency conversion.
new_entries = []
replacement_postings = []
for posting_orig in postings_at_price:
weight = interpolate.get_posting_weight(posting_orig)
simple_position = position.Position(
position.Lot(weight.currency, None, None), weight.number)
posting_pos = data.Posting(posting_orig.account, simple_position,
None, None, None)
posting_neg = data.Posting(posting_orig.account, -simple_position,
None, None, None)
currency_entry = entry._replace(
postings=[posting_orig, posting_neg],
narration=entry.narration + ' (Currency conversion)')
new_entries.append(currency_entry)
replacement_postings.append(posting_pos)
converted_entry = entry._replace(postings=(postings_at_cost +
postings_simple +
replacement_postings))
new_entries.append(converted_entry)
else:
new_entries = [entry]
return converted, new_entries
