def account_for_symbol(self, statement, ib_symbol):
symbol = self.clean_symbol(ib_symbol)
root_account = self.accounts['root']
if self.prefix_account_id:
root_account = account.join(root_account, statement.accountId)
return account.join(root_account, symbol)
def test_account_join(self):
account_name = account.join("Expenses", "Toys", "Computer")
self.assertEqual("Expenses:Toys:Computer", account_name)
account_name = account.join("Expenses")
self.assertEqual("Expenses", account_name)
account_name = account.join()
self.assertEqual("", account_name)
def get_current_accounts(options):
"""Return account names for the current earnings and conversion accounts.
Args:
options: a dict of ledger options.
Returns:
A tuple of 2 account objects, one for booking current earnings, and one
for current conversions.
"""
equity = options['name_equity']
account_current_earnings = account.join(
equity, options['account_current_earnings'])
account_current_conversions = account.join(
equity, options['account_current_conversions'])
return (account_current_earnings, account_current_conversions)
def render_mini_balances(entries, options_map, conversion=None, price_map=None):
"""Render a treeified list of the balances for the given transactions.
Args:
entries: A list of selected transactions to render.
options_map: The parsed options.
conversion: Conversion method string, None, 'value' or 'cost'.
price_map: A price map from the original entries. If this isn't provided,
the inventories are rendered directly. If it is, their contents are
converted to market value.
"""
# Render linked entries (in date order) as errors (for Emacs).
errors = [RenderError(entry.meta, '', entry)
for entry in entries]
printer.print_errors(errors)
# Print out balances.
real_root = realization.realize(entries)
dformat = options_map['dcontext'].build(alignment=Align.DOT, reserved=2)
# TODO(blais): I always want to be able to convert at cost. We need
# arguments capability.
#
# TODO(blais): Ideally this conversion inserts a new transactions to
# 'Unrealized' to account for the difference between cost and market value.
# Insert one and update the realization. Add an update() method to the
# realization, given a transaction.
acctypes = options.get_account_types(options_map)
if conversion == 'value':
assert price_map is not None
# Warning: Mutate the inventories in-place, converting them to market
# value.
balance_diff = inventory.Inventory()
for real_account in realization.iter_children(real_root):
balance_cost = real_account.balance.reduce(convert.get_cost)
balance_value = real_account.balance.reduce(convert.get_value, price_map)
real_account.balance = balance_value
balance_diff.add_inventory(balance_cost)
balance_diff.add_inventory(-balance_value)
if not balance_diff.is_empty():
account_unrealized = account.join(acctypes.income,
options_map["account_unrealized_gains"])
unrealized = realization.get_or_create(real_root, account_unrealized)
unrealized.balance.add_inventory(balance_diff)
elif conversion == 'cost':
for real_account in realization.iter_children(real_root):
real_account.balance = real_account.balance.reduce(convert.get_cost)
realization.dump_balances(real_root, dformat, file=sys.stdout)
# Print out net income change.
net_income = inventory.Inventory()
for real_node in realization.iter_children(real_root):
if account_types.is_income_statement_account(real_node.account, acctypes):
net_income.add_inventory(real_node.balance)
print()
print('Net Income: {}'.format(-net_income))
def get_previous_accounts(options):
"""Return account names for the previous earnings, balances and conversion accounts.
Args:
options: a dict of ledger options.
Returns:
A tuple of 3 account objects, for booking previous earnings,
previous balances, and previous conversions.
"""
equity = options['name_equity']
account_previous_earnings = account.join(
equity, options['account_previous_earnings'])
account_previous_balances = account.join(
equity, options['account_previous_balances'])
account_previous_conversions = account.join(
equity, options['account_previous_conversions'])
return (account_previous_earnings, account_previous_balances,
account_previous_conversions)
def get_unrealized_account(options):
"""Return the full account name for the unrealized account.
Args:
options: a dict of ledger options.
Returns:
A tuple of 2 account objects, one for booking current earnings, and one
for current conversions.
"""
income = options['name_income']
return account.join(income, options['account_unrealized_gains'])
def get_neutralizing_postings(curmap, base_account, new_accounts):
"""Process an entry.
Args:
curmap: A dict of currency to a list of Postings of this transaction.
base_account: A string, the root account name to insert.
new_accounts: A set, a mutable accumulator of new account names.
Returns:
A modified entry, with new postings inserted to rebalance currency trading
accounts.
"""
new_postings = []
for currency, postings in curmap.items():
# Compute the per-currency balance.
inv = inventory.Inventory()
for posting in postings:
inv.add_amount(convert.get_cost(posting))
if inv.is_empty():
new_postings.extend(postings)
continue
# Re-insert original postings and remove price conversions.
#
# Note: This may cause problems if the implicit_prices plugin is
# configured to run after this one, or if you need the price annotations
# for some scripting or serious work.
#
# FIXME: We need to handle these important cases (they're not frivolous,
# this is a prototype), probably by inserting some exceptions with
# collaborating code in the booking (e.g. insert some metadata that
# disables price conversions on those postings).
#
# FIXME(2): Ouch! Some of the residual seeps through here, where there
# are more than a single currency block. This needs fixing too. You can
# easily mitigate some of this to some extent, by excluding transactions
# which don't have any price conversion in them.
for pos in postings:
if pos.price is not None:
pos = pos._replace(price=None)
new_postings.append(pos)
# Insert the currency trading accounts postings.
amount = inv.get_only_position().units
acc = account.join(base_account, currency)
new_accounts.add(acc)
new_postings.append(Posting(acc, -amount, None, None, None, None))
return new_postings
def balance_incomplete_postings(entry, options_map):
"""Balance an entry with incomplete postings, modifying the
empty postings on the entry itself. This sets the parent of
all the postings to this entry. Futhermore, it stores the dict
of inferred tolerances as metadata.
WARNING: This destructively modifies entry itself!
Args:
entry: An instance of a valid directive. This entry is modified by
having new postings inserted to it.
options_map: A dict of options, as produced by the parser.
Returns:
A list of errors, or None, if none occurred.
"""
# No postings... nothing to do.
if not entry.postings:
return None
# Get the list of corrected postings.
(postings, unused_inserted, errors, residual,
tolerances) = get_incomplete_postings(entry, options_map)
# If we need to accumulate rounding error to accumulate the residual, add
# suitable postings here.
if not residual.is_empty():
rounding_subaccount = options_map["account_rounding"]
if rounding_subaccount:
account_rounding = account.join(options_map['name_equity'],
rounding_subaccount)
rounding_postings = get_residual_postings(residual,
account_rounding)
postings.extend(rounding_postings)
# If we could make this faster to avoid the unnecessary copying, it would
# make parsing substantially faster.
entry.postings.clear()
for posting in postings:
entry.postings.append(posting)
if entry.meta is None:
entry.meta = {}
entry.meta['__tolerances__'] = tolerances
return errors or None
def get_or_create(real_account, account_name):
"""Fetch the subaccount name from the real_account node.
Args:
real_account: An instance of RealAccount, the parent node to look for
children of, or create under.
account_name: A string, the name of the direct or indirect child leaf
to get or create.
Returns:
A RealAccount instance for the child, or the default value, if the child
is not found.
"""
if not isinstance(account_name, str):
raise ValueError
components = account.split(account_name)
path = []
for component in components:
path.append(component)
real_child = real_account.get(component, None)
if real_child is None:
real_child = RealAccount(account.join(*path))
real_account[component] = real_child
real_account = real_child
return real_account
def split_expenses(entries, options_map, config):
"""Split postings according to expenses (see module docstring for details).
Args:
entries: A list of directives. We're interested only in the Transaction instances.
unused_options_map: A parser options dict.
config: The plugin configuration string.
Returns:
A list of entries, with potentially more accounts and potentially more
postings with smaller amounts.
"""
# Validate and sanitize configuration.
if isinstance(config, str):
members = config.split()
elif isinstance(config, (tuple, list)):
members = config
else:
raise RuntimeError(
"Invalid plugin configuration: configuration for split_expenses "
"should be a string or a sequence.")
acctypes = options.get_account_types(options_map)
def is_expense_account(account):
return account_types.get_account_type(account) == acctypes.expenses
# A predicate to quickly identify if an account contains the name of a
# member.
is_individual_account = re.compile('|'.join(map(re.escape,
members))).search
# Existing and previously unseen accounts.
new_accounts = set()
# Filter the entries and transform transactions.
new_entries = []
for entry in entries:
if isinstance(entry, data.Transaction):
new_postings = []
for posting in entry.postings:
if (is_expense_account(posting.account)
and not is_individual_account(posting.account)):
# Split this posting into multiple postings.
split_units = amount.Amount(
posting.units.number / len(members),
posting.units.currency)
for member in members:
# Mark the account as new if never seen before.
subaccount = account.join(posting.account, member)
new_accounts.add(subaccount)
# Ensure the modified postings are marked as
# automatically calculated, so that the resulting
# calculated amounts aren't used to affect inferred
# tolerances.
meta = posting.meta.copy() if posting.meta else {}
meta[interpolate.AUTOMATIC_META] = True
# Add a new posting for each member, to a new account
# with the name of this member.
new_postings.append(
posting._replace(meta=meta,
account=subaccount,
units=split_units,
cost=posting.cost))
else:
new_postings.append(posting)
# Modify the entry in-place, replace its postings.
entry = entry._replace(postings=new_postings)
new_entries.append(entry)
# Create Open directives for new subaccounts if necessary.
oc_map = getters.get_account_open_close(entries)
open_date = entries[0].date
meta = data.new_metadata('<split_expenses>', 0)
open_entries = []
for new_account in new_accounts:
if new_account not in oc_map:
entry = data.Open(meta, open_date, new_account, None, None)
open_entries.append(entry)
return open_entries + new_entries, []
def add_unrealized_gains_at_date(entries, unrealized_entries, income_account_type,
price_map, date, meta, subaccount):
"""Insert/remove entries for unrealized capital gains
This function takes a list of entries and a date and creates a set of unrealized gains
transactions, negating previous unrealized gains transactions within the same account.
Args:
entries: A list of data directives.
unrealized_entries: A list of previously generated unrealized transactions.
income_account_type: The income account type.
price_map: A price map returned by prices.build_price_map.
date: The effective date to generate the unrealized transactions for.
meta: meta.
subaccount: A string, and optional the name of a subaccount to create
under an account to book the unrealized gain. If this is left to its
default value, the gain is booked directly in the same account.
Returns:
A list of newly created unrealized transactions and a list of errors.
"""
errors = []
entries_truncated = summarize.truncate(entries, date + ONEDAY)
holdings_list = holdings.get_final_holdings(entries_truncated, price_map=price_map, date=date)
# Group positions by (account, cost, cost_currency).
holdings_list = holdings.aggregate_holdings_by(
holdings_list, lambda h: (h.account, h.currency, h.cost_currency))
holdings_with_currencies = set()
# Create transactions to account for each position.
new_entries = []
for index, holding in enumerate(holdings_list):
if (holding.currency == holding.cost_currency or
holding.cost_currency is None):
continue
# Note: since we're only considering positions held at cost, the
# transaction that created the position *must* have created at least one
# price point for that commodity, so we never expect for a price not to
# be available, which is reasonable.
if holding.price_number is None:
# An entry without a price might indicate that this is a holding
# resulting from leaked cost basis. {0ed05c502e63, b/16}
if holding.number:
errors.append(
UnrealizedError(meta,
"A valid price for {h.currency}/{h.cost_currency} "
"could not be found".format(h=holding), None))
continue
# Compute the PnL; if there is no profit or loss, we create a
# corresponding entry anyway.
pnl = holding.market_value - holding.book_value
if holding.number == ZERO:
# If the number of units sum to zero, the holdings should have been
# zero.
errors.append(
UnrealizedError(
meta,
"Number of units of {} in {} in holdings sum to zero "
"for account {} and should not".format(
holding.currency, holding.cost_currency, holding.account),
None))
continue
# Compute the name of the accounts and add the requested subaccount name
# if requested.
asset_account = holding.account
income_account = account.join(income_account_type,
account.sans_root(holding.account))
if subaccount:
asset_account = account.join(asset_account, subaccount)
income_account = account.join(income_account, subaccount)
holdings_with_currencies.add((holding.account, holding.cost_currency, holding.currency))
# Find the previous unrealized gain entry to negate and decide if we
# should create a new posting.
latest_unrealized_entry = find_previous_unrealized_transaction(unrealized_entries, asset_account, holding.cost_currency, holding.currency)
# Don't create a new transaction if our last one hasn't changed.
if (latest_unrealized_entry and
pnl == latest_unrealized_entry.postings[0].units.number):
continue
# Don't bother creating a blank unrealized transaction if none existed
if pnl == ZERO and not latest_unrealized_entry:
continue
relative_pnl = pnl
if latest_unrealized_entry:
relative_pnl = pnl - latest_unrealized_entry.postings[0].units.number
# Create a new transaction to account for this difference in gain.
gain_loss_str = "gain" if relative_pnl > ZERO else "loss"
narration = ("Unrealized {} for {h.number} units of {h.currency} "
"(price: {h.price_number:.4f} {h.cost_currency} as of {h.price_date}, "
"average cost: {h.cost_number:.4f} {h.cost_currency})").format(
gain_loss_str, h=holding)
entry = data.Transaction(data.new_metadata(meta["filename"], lineno=1000 + index,
kvlist={'prev_currency': holding.currency}), date,
flags.FLAG_UNREALIZED, None, narration, set(), set(), [])
# Book this as income, converting the account name to be the same, but as income.
# Note: this is a rather convenient but arbitraty choice--maybe it would be best to
# let the user decide to what account to book it, but I don't a nice way to let the
# user specify this.
#
# Note: we never set a price because we don't want these to end up in Conversions.
entry.postings.extend([
data.Posting(
asset_account,
amount.Amount(pnl, holding.cost_currency),
None,
None,
None,
None),
data.Posting(
income_account,
amount.Amount(-pnl, holding.cost_currency),
None,
None,
None,
None)
])
if latest_unrealized_entry:
for posting in latest_unrealized_entry.postings[:2]:
entry.postings.append(
data.Posting(
posting.account,
-posting.units,
None,
None,
None,
None))
new_entries.append(entry)
return new_entries, holdings_with_currencies, errors
def extract(self, file):
# Open the CSV file and create directives.
entries = []
index = 0
for index, row in enumerate(csv.DictReader(open(file.name))):
meta = data.new_metadata(file.name, index)
date = parse(row['DATE']).date()
rtype = row['TYPE']
link = "ut{0[REF #]}".format(row)
desc = "({0[TYPE]}) {0[DESCRIPTION]}".format(row)
units = amount.Amount(D(row['AMOUNT']), self.currency)
fees = amount.Amount(D(row['FEES']), self.currency)
other = amount.add(units, fees)
if rtype == 'XFER':
assert fees.number == ZERO
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET, {link},
[
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(self.account_external, -other, None, None,
None, None),
])
elif rtype == 'DIV':
assert fees.number == ZERO
# Extract the instrument name from its description.
match = re.search(r'~([A-Z]+)$', row['DESCRIPTION'])
if not match:
logging.error("Missing instrument name in '%s'",
row['DESCRIPTION'])
continue
instrument = match.group(1)
account_dividends = self.account_dividends.format(instrument)
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET, {link},
[
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(account_dividends, -other, None, None,
None, None),
])
elif rtype in ('BUY', 'SELL'):
# Extract the instrument name, number of units, and price from
# the description. That's just what we're provided with (this is
# actually realistic of some data from some institutions, you
# have to figure out a way in your parser).
match = re.search(r'\+([A-Z]+)\b +([0-9.]+)\b +@([0-9.]+)',
row['DESCRIPTION'])
if not match:
logging.error("Missing purchase infos in '%s'",
row['DESCRIPTION'])
continue
instrument = match.group(1)
account_inst = account.join(self.account_root, instrument)
units_inst = amount.Amount(D(match.group(2)), instrument)
rate = D(match.group(3))
if rtype == 'BUY':
cost = position.Cost(rate, self.currency, None, None)
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET,
{link}, [
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(self.account_fees, fees, None, None,
None, None),
data.Posting(account_inst, units_inst, cost, None,
None, None),
])
elif rtype == 'SELL':
# Extract the lot. In practice this information not be there
# and you will have to identify the lots manually by editing
# the resulting output. You can leave the cost.number slot
# set to None if you like.
match = re.search(r'\(LOT ([0-9.]+)\)', row['DESCRIPTION'])
if not match:
logging.error("Missing cost basis in '%s'",
row['DESCRIPTION'])
continue
cost_number = D(match.group(1))
cost = position.Cost(cost_number, self.currency, None,
None)
price = amount.Amount(rate, self.currency)
account_gains = self.account_gains.format(instrument)
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET,
{link}, [
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(self.account_fees, fees, None, None,
None, None),
data.Posting(account_inst, units_inst, cost, price,
None, None),
data.Posting(account_gains, None, None, None, None,
None),
])
else:
logging.error("Unknown row type: %s; skipping", rtype)
continue
entries.append(txn)
# Insert a final balance check.
if index:
entries.append(
data.Balance(meta, date + datetime.timedelta(days=1),
self.account_cash,
amount.Amount(D(row['BALANCE']),
self.currency), None, None))
return entries
def main():
position_account = "Liabilities:Crypto:Bitfinex:Positions"
margin_account = "Liabilities:Crypto:Bitfinex:Positions"
income_account = "Income:Crypto:Bitfinex:Realized"
fee_account = "Expenses:Trading:Bitfinex:Fees:Trading"
positions: typing.Dict[str, Decimal] = defaultdict(Decimal)
balances: typing.Dict[str, datetime.date] = defaultdict(
lambda: datetime.date(2000, 1, 1))
records = list(records_from_string(csvdata))
records.sort(key=lambda r: (r.date, r.commodity, -r.quantity))
entries = []
for record in records:
# Now, we need to Handle the Incoming Account, Only if our Absolute
# Position is decreasing:
current_position = positions[record.commodity]
new_position = current_position + record.quantity
positions[record.commodity] = new_position
# This is the account who's position we need to track for our lots
commodity_account = account.join(position_account, record.commodity)
if balances[commodity_account] != record.date:
entries.append(
data.Balance(date=record.date,
account=commodity_account,
amount=Amount(current_position, record.commodity),
tolerance=Q,
diff_amount=None,
meta=data.new_metadata("", 0)))
balances[commodity_account] = record.date
entry = data.Transaction(date=record.date,
narration=str(record),
payee="",
tags={"margin"},
links=set(),
flag='*',
meta=data.new_metadata(
"",
0,
kvlist=dict(position=str(new_position),
order_id=record.order_id)),
postings=[])
# The Commodity Adjustment
# Assets:Crypto:Bitfinex:Positions:BTC 200 BTC {} @ 6950 USD
posting = data.Posting(account=commodity_account,
units=data.Amount(record.quantity,
record.commodity),
cost=empty_cost_spec,
price=Amount(record.price, record.currency),
flag=None,
meta=data.new_metadata("", 0))
entry.postings.append(posting)
# The Currency Account Adjustment (based on Cost?)
# ; Liabilities:Crypto:Bitfinex:Borrowed:USD -1,390,102.19 USD
data.create_simple_posting(entry=entry,
account=account.join(
margin_account, record.currency),
number=-record.amount,
currency=record.currency)
if record.fee:
# We add Two fee Records
# ; Liabilities:Crypto:Bitfinex:Borrowed:USD -839.747894 USD
data.create_simple_posting(entry=entry,
account=account.join(
margin_account, record.currency),
number=record.fee,
currency=record.currency)
data.create_simple_posting(entry=entry,
account=fee_account,
number=-record.fee,
currency=record.currency)
if abs(new_position) < abs(current_position):
# Add an Income Account Entry
data.create_simple_posting(entry=entry,
account=income_account,
number=None,
currency=None)
entries.append(entry)
printer.print_entries(entries)
def add_unrealized_gains(entries, options_map, subaccount=None):
"""Insert entries for unrealized capital gains.
This function inserts entries that represent unrealized gains, at the end of
the available history. It returns a new list of entries, with the new gains
inserted. It replaces the account type with an entry in an income account.
Optionally, it can book the gain in a subaccount of the original and income
accounts.
Args:
entries: A list of data directives.
options_map: A dict of options, that confirms to beancount.parser.options.
subaccount: A string, and optional the name of a subaccount to create
under an account to book the unrealized gain. If this is left to its
default value, the gain is booked directly in the same account.
Returns:
A list of entries, which includes the new unrealized capital gains entries
at the end, and a list of errors. The new list of entries is still sorted.
"""
errors = []
meta = data.new_metadata('<unrealized_gains>', 0)
account_types = options.get_account_types(options_map)
# Assert the subaccount name is in valid format.
if subaccount:
validation_account = account.join(account_types.assets, subaccount)
if not account.is_valid(validation_account):
errors.append(
UnrealizedError(meta,
"Invalid subaccount name: '{}'".format(subaccount),
None))
return entries, errors
if not entries:
return (entries, errors)
# Group positions by (account, cost, cost_currency).
price_map = prices.build_price_map(entries)
new_entries = []
# Start at the first month after our first transaction
date = date_utils.next_month(entries[0].date)
last_month = date_utils.next_month(entries[-1].date)
last_holdings_with_currencies = None
while date <= last_month:
date_entries, holdings_with_currencies, date_errors = add_unrealized_gains_at_date(
entries, new_entries, account_types.income, price_map, date, meta,
subaccount)
new_entries.extend(date_entries)
errors.extend(date_errors)
if last_holdings_with_currencies:
for account_, cost_currency, currency in last_holdings_with_currencies - holdings_with_currencies:
# Create a negation transaction specifically to mark that all gains have been realized
if subaccount:
account_ = account.join(account_, subaccount)
latest_unrealized_entry = find_previous_unrealized_transaction(new_entries, account_, cost_currency, currency)
if not latest_unrealized_entry:
continue
entry = data.Transaction(data.new_metadata(meta["filename"], lineno=999,
kvlist={'prev_currency': currency}), date,
flags.FLAG_UNREALIZED, None, 'Clear unrealized gains/losses of {}'.format(currency), set(), set(), [])
# Negate the previous transaction because of unrealized gains are now 0
for posting in latest_unrealized_entry.postings[:2]:
entry.postings.append(
data.Posting(
posting.account,
-posting.units,
None,
None,
None,
None))
new_entries.append(entry)
last_holdings_with_currencies = holdings_with_currencies
date = date_utils.next_month(date)
# Ensure that the accounts we're going to use to book the postings exist, by
# creating open entries for those that we generated that weren't already
# existing accounts.
new_accounts = {posting.account
for entry in new_entries
for posting in entry.postings}
open_entries = getters.get_account_open_close(entries)
new_open_entries = []
for index, account_ in enumerate(sorted(new_accounts)):
if account_ not in open_entries:
meta = data.new_metadata(meta["filename"], index)
open_entry = data.Open(meta, new_entries[0].date, account_, None, None)
new_open_entries.append(open_entry)
return (entries + new_open_entries + new_entries, errors)
def get_invalid_account(self):
"""See base class."""
return account.join(self.options['name_equity'], 'InvalidAccountName')
def categorize_accounts(account: Account, accounts: Set[Account],
atypes: tuple) -> Dict[Account, Cat]:
"""Categorize the type of accounts for a particular stock. Our purpose is to
make the types of postings generic, so they can be categorized and handled
generically later on.
The patterns used in this file depend on the particular choices of account
names in my chart of accounts, and for others to use this, this needs to be
specialized somewhat.
"""
accpath = accountlib.join(*accountlib.split(account)[1:-1])
currency = accountlib.leaf(account)
accounts = set(accounts)
catmap = {}
def move(acc, category):
if acc in accounts:
accounts.remove(acc)
catmap[acc] = category
# The account itself.
move(account, Cat.ASSET)
# The adjacent cash account.
move(accountlib.join(atypes.assets, accpath, "Cash"), Cat.CASH)
# The adjacent P/L and interest account.
move(accountlib.join(atypes.income, accpath, "PnL"), Cat.PNL)
move(accountlib.join(atypes.income, accountlib.parent(accpath), "PnL"),
Cat.PNL)
move(accountlib.join(atypes.income, accpath, "Interest"), Cat.INTEREST)
# The associated dividend account.
move(accountlib.join(atypes.income, accpath, currency, "Dividend"),
Cat.DIVIDEND)
# Rounding error account.
move("Equity:RoundingError", Cat.ROUNDING)
move("Expenses:Losses", Cat.ROUNDING)
move("Income:US:MSSB:RoundingVariance", Cat.ROUNDING)
for acc in list(accounts):
# Employer match and corresponding tracking accounts in IRAUSD.
if re.match(
"({}|{}):.*:Match401k".format(atypes.assets, atypes.expenses),
acc):
move(acc, Cat.TRACKING)
elif re.search(r"\b(Vested|Unvested)", acc):
move(acc, Cat.TRACKING)
# Dividends for other stocks.
elif re.search(r":Dividends?$", acc):
move(acc, Cat.OTHERDIVIDEND)
# Direct contribution from employer.
elif re.match("{}:.*:Match401k$".format(atypes.income), acc):
move(acc, Cat.CASH)
elif re.search(":GSURefund$", acc):
move(acc, Cat.CASH)
# Expenses accounts.
elif acc.startswith(atypes.expenses):
move(acc, Cat.EXPENSES)
elif acc.endswith(":Commissions"): # Income..Commissions
move(acc, Cat.EXPENSES)
# Currency accounts.
elif acc.startswith("Equity:CurrencyAccounts"):
move(acc, Cat.CONVERSIONS)
# Other cash or checking accounts.
elif acc.endswith(":Cash"):
move(acc, Cat.CASH)
elif acc.endswith(":Checking"):
move(acc, Cat.CASH)
elif acc.endswith("Receivable"):
move(acc, Cat.CASH)
# Other stock.
elif re.match("{}:[A-Z_.]+$".format(accountlib.parent(acc)), acc):
move(acc, Cat.OTHERASSET)
else:
print("ERROR: Unknown account: {}".format(acc))
move(acc, Cat.UNKNOWN)
return catmap
def _reverse_parents(a):
chain = acctops.split(a)
for i in range(len(chain)):
yield acctops.join(*chain[:i + 1])
def add_unrealized_gains_at_date(entries, unrealized_entries,
income_account_type, price_map, date, meta,
subaccount):
"""Insert/remove entries for unrealized capital gains
This function takes a list of entries and a date and creates a set of unrealized gains
transactions, negating previous unrealized gains transactions within the same account.
Args:
entries: A list of data directives.
unrealized_entries: A list of previously generated unrealized transactions.
income_account_type: The income account type.
price_map: A price map returned by prices.build_price_map.
date: The effective date to generate the unrealized transactions for.
meta: meta.
subaccount: A string, and optional the name of a subaccount to create
under an account to book the unrealized gain. If this is left to its
default value, the gain is booked directly in the same account.
Returns:
A list of newly created unrealized transactions and a list of errors.
"""
errors = []
entries_truncated = summarize.truncate(entries, date + ONEDAY)
holdings_list = holdings.get_final_holdings(entries_truncated,
price_map=price_map,
date=date)
# Group positions by (account, cost, cost_currency).
holdings_list = holdings.aggregate_holdings_by(
holdings_list, lambda h: (h.account, h.currency, h.cost_currency))
holdings_with_currencies = set()
# Create transactions to account for each position.
new_entries = []
for index, holding in enumerate(holdings_list):
if (holding.currency == holding.cost_currency
or holding.cost_currency is None):
continue
# Note: since we're only considering positions held at cost, the
# transaction that created the position *must* have created at least one
# price point for that commodity, so we never expect for a price not to
# be available, which is reasonable.
if holding.price_number is None:
# An entry without a price might indicate that this is a holding
# resulting from leaked cost basis. {0ed05c502e63, b/16}
if holding.number:
errors.append(
UnrealizedError(
meta,
"A valid price for {h.currency}/{h.cost_currency} "
"could not be found".format(h=holding), None))
continue
# Compute the PnL; if there is no profit or loss, we create a
# corresponding entry anyway.
pnl = holding.market_value - holding.book_value
if holding.number == ZERO:
# If the number of units sum to zero, the holdings should have been
# zero.
errors.append(
UnrealizedError(
meta,
"Number of units of {} in {} in holdings sum to zero "
"for account {} and should not".format(
holding.currency, holding.cost_currency,
holding.account), None))
continue
# Compute the name of the accounts and add the requested subaccount name
# if requested.
asset_account = holding.account
income_account = account.join(income_account_type,
account.sans_root(holding.account))
if subaccount:
asset_account = account.join(asset_account, subaccount)
income_account = account.join(income_account, subaccount)
holdings_with_currencies.add(
(holding.account, holding.cost_currency, holding.currency))
# Find the previous unrealized gain entry to negate and decide if we
# should create a new posting.
latest_unrealized_entry = find_previous_unrealized_transaction(
unrealized_entries, asset_account, holding.cost_currency,
holding.currency)
# Don't create a new transaction if our last one hasn't changed.
if (latest_unrealized_entry
and pnl == latest_unrealized_entry.postings[0].units.number):
continue
# Don't bother creating a blank unrealized transaction if none existed
if pnl == ZERO and not latest_unrealized_entry:
continue
relative_pnl = pnl
if latest_unrealized_entry:
relative_pnl = pnl - latest_unrealized_entry.postings[
0].units.number
# Create a new transaction to account for this difference in gain.
gain_loss_str = "gain" if relative_pnl > ZERO else "loss"
narration = (
"Unrealized {} for {h.number} units of {h.currency} "
"(price: {h.price_number:.4f} {h.cost_currency} as of {h.price_date}, "
"average cost: {h.cost_number:.4f} {h.cost_currency})").format(
gain_loss_str, h=holding)
entry = data.Transaction(
data.new_metadata(meta["filename"],
lineno=1000 + index,
kvlist={'prev_currency': holding.currency}),
date, flags.FLAG_UNREALIZED, None, narration, set(), set(), [])
# Book this as income, converting the account name to be the same, but as income.
# Note: this is a rather convenient but arbitraty choice--maybe it would be best to
# let the user decide to what account to book it, but I don't a nice way to let the
# user specify this.
#
# Note: we never set a price because we don't want these to end up in Conversions.
entry.postings.extend([
data.Posting(asset_account,
amount.Amount(pnl, holding.cost_currency), None, None,
None, None),
data.Posting(income_account,
amount.Amount(-pnl, holding.cost_currency), None,
None, None, None)
])
if latest_unrealized_entry:
for posting in latest_unrealized_entry.postings[:2]:
entry.postings.append(
data.Posting(posting.account, -posting.units, None, None,
None, None))
new_entries.append(entry)
return new_entries, holdings_with_currencies, errors
def add_unrealized_gains(entries, options_map, subaccount=None):
"""Insert entries for unrealized capital gains.
This function inserts entries that represent unrealized gains, at the end of
the available history. It returns a new list of entries, with the new gains
inserted. It replaces the account type with an entry in an income account.
Optionally, it can book the gain in a subaccount of the original and income
accounts.
Args:
entries: A list of data directives.
options_map: A dict of options, that confirms to beancount.parser.options.
subaccount: A string, and optional the name of a subaccount to create
under an account to book the unrealized gain. If this is left to its
default value, the gain is booked directly in the same account.
Returns:
A list of entries, which includes the new unrealized capital gains entries
at the end, and a list of errors. The new list of entries is still sorted.
"""
errors = []
meta = data.new_metadata('<unrealized_gains>', 0)
account_types = options.get_account_types(options_map)
# Assert the subaccount name is in valid format.
if subaccount:
validation_account = account.join(account_types.assets, subaccount)
if not account.is_valid(validation_account):
errors.append(
UnrealizedError(
meta, "Invalid subaccount name: '{}'".format(subaccount),
None))
return entries, errors
if not entries:
return (entries, errors)
# Group positions by (account, cost, cost_currency).
price_map = prices.build_price_map(entries)
new_entries = []
# Start at the first month after our first transaction
date = date_utils.next_month(entries[0].date)
last_month = date_utils.next_month(entries[-1].date)
last_holdings_with_currencies = None
while date <= last_month:
date_entries, holdings_with_currencies, date_errors = add_unrealized_gains_at_date(
entries, new_entries, account_types.income, price_map, date, meta,
subaccount)
new_entries.extend(date_entries)
errors.extend(date_errors)
if last_holdings_with_currencies:
for account_, cost_currency, currency in last_holdings_with_currencies - holdings_with_currencies:
# Create a negation transaction specifically to mark that all gains have been realized
if subaccount:
account_ = account.join(account_, subaccount)
latest_unrealized_entry = find_previous_unrealized_transaction(
new_entries, account_, cost_currency, currency)
if not latest_unrealized_entry:
continue
entry = data.Transaction(
data.new_metadata(meta["filename"],
lineno=999,
kvlist={'prev_currency': currency}),
date, flags.FLAG_UNREALIZED, None,
'Clear unrealized gains/losses of {}'.format(currency),
set(), set(), [])
# Negate the previous transaction because of unrealized gains are now 0
for posting in latest_unrealized_entry.postings[:2]:
entry.postings.append(
data.Posting(posting.account, -posting.units, None,
None, None, None))
new_entries.append(entry)
last_holdings_with_currencies = holdings_with_currencies
date = date_utils.next_month(date)
# Ensure that the accounts we're going to use to book the postings exist, by
# creating open entries for those that we generated that weren't already
# existing accounts.
new_accounts = {
posting.account
for entry in new_entries for posting in entry.postings
}
open_entries = getters.get_account_open_close(entries)
new_open_entries = []
for index, account_ in enumerate(sorted(new_accounts)):
if account_ not in open_entries:
meta = data.new_metadata(meta["filename"], index)
open_entry = data.Open(meta, new_entries[0].date, account_, None,
None)
new_open_entries.append(open_entry)
return (entries + new_open_entries + new_entries, errors)
def add_unrealized_gains(entries, options_map, subaccount=None):
"""Insert entries for unrealized capital gains.
This function inserts entries that represent unrealized gains, at the end of
the available history. It returns a new list of entries, with the new gains
inserted. It replaces the account type with an entry in an income account.
Optionally, it can book the gain in a subaccount of the original and income
accounts.
Args:
entries: A list of data directives.
options_map: A dict of options, that confirms to beancount.parser.options.
subaccount: A string, and optional the name of a subaccount to create
under an account to book the unrealized gain. If this is left to its
default value, the gain is booked directly in the same account.
Returns:
A list of entries, which includes the new unrealized capital gains entries
at the end, and a list of errors. The new list of entries is still sorted.
"""
errors = []
meta = data.new_metadata('<unrealized_gains>', 0)
account_types = options.get_account_types(options_map)
# Assert the subaccount name is in valid format.
if subaccount:
validation_account = account.join(account_types.assets, subaccount)
if not account.is_valid(validation_account):
errors.append(
UnrealizedError(meta,
"Invalid subaccount name: '{}'".format(subaccount),
None))
return entries, errors
if not entries:
return (entries, errors)
# Group positions by (account, cost, cost_currency).
price_map = prices.build_price_map(entries)
holdings_list = holdings.get_final_holdings(entries, price_map=price_map)
# Group positions by (account, cost, cost_currency).
holdings_list = holdings.aggregate_holdings_by(
holdings_list, lambda h: (h.account, h.currency, h.cost_currency))
# Get the latest prices from the entries.
price_map = prices.build_price_map(entries)
# Create transactions to account for each position.
new_entries = []
latest_date = entries[-1].date
for index, holding in enumerate(holdings_list):
if (holding.currency == holding.cost_currency or
holding.cost_currency is None):
continue
# Note: since we're only considering positions held at cost, the
# transaction that created the position *must* have created at least one
# price point for that commodity, so we never expect for a price not to
# be available, which is reasonable.
if holding.price_number is None:
# An entry without a price might indicate that this is a holding
# resulting from leaked cost basis. {0ed05c502e63, b/16}
if holding.number:
errors.append(
UnrealizedError(meta,
"A valid price for {h.currency}/{h.cost_currency} "
"could not be found".format(h=holding), None))
continue
# Compute the PnL; if there is no profit or loss, we create a
# corresponding entry anyway.
pnl = holding.market_value - holding.book_value
if holding.number == ZERO:
# If the number of units sum to zero, the holdings should have been
# zero.
errors.append(
UnrealizedError(
meta,
"Number of units of {} in {} in holdings sum to zero "
"for account {} and should not".format(
holding.currency, holding.cost_currency, holding.account),
None))
continue
# Compute the name of the accounts and add the requested subaccount name
# if requested.
asset_account = holding.account
income_account = account.join(account_types.income,
account.sans_root(holding.account))
if subaccount:
asset_account = account.join(asset_account, subaccount)
income_account = account.join(income_account, subaccount)
# Create a new transaction to account for this difference in gain.
gain_loss_str = "gain" if pnl > ZERO else "loss"
narration = ("Unrealized {} for {h.number} units of {h.currency} "
"(price: {h.price_number:.4f} {h.cost_currency} as of {h.price_date}, "
"average cost: {h.cost_number:.4f} {h.cost_currency})").format(
gain_loss_str, h=holding)
entry = data.Transaction(data.new_metadata(meta["filename"], lineno=1000 + index),
latest_date, flags.FLAG_UNREALIZED,
None, narration, EMPTY_SET, EMPTY_SET, [])
# Book this as income, converting the account name to be the same, but as income.
# Note: this is a rather convenient but arbitrary choice--maybe it would be best to
# let the user decide to what account to book it, but I don't a nice way to let the
# user specify this.
#
# Note: we never set a price because we don't want these to end up in Conversions.
entry.postings.extend([
data.Posting(
asset_account,
amount.Amount(pnl, holding.cost_currency),
None,
None,
None,
None),
data.Posting(
income_account,
amount.Amount(-pnl, holding.cost_currency),
None,
None,
None,
None)
])
new_entries.append(entry)
# Ensure that the accounts we're going to use to book the postings exist, by
# creating open entries for those that we generated that weren't already
# existing accounts.
new_accounts = {posting.account
for entry in new_entries
for posting in entry.postings}
open_entries = getters.get_account_open_close(entries)
new_open_entries = []
for account_ in sorted(new_accounts):
if account_ not in open_entries:
meta = data.new_metadata(meta["filename"], index)
open_entry = data.Open(meta, latest_date, account_, None, None)
new_open_entries.append(open_entry)
return (entries + new_open_entries + new_entries, errors)
def make_records(*, transaction_details: TransactionDetailsForAddresses,
addresses: list[Address], transactions: list[Transactions],
currency: spotbit.CurrencyName) -> str:
assert transaction_details
assert addresses
assert transactions
assert currency
# FIXME I can't use the beancount api to create a data structure that I can then dump to a beancount file.
# Instead I must emit strings then load the strings to test for correctness.
# Ref. https://beancount.github.io/docs/beancount_language_syntax.html
# FINDOUT How to create a collection of entries and dump it to text file.
# - Create accounts
# - Create transactions
# - Add postings to transactions.
# - Dump the account to a file.
# Ref. realization.py
type = 'Assets'
country = ''
institution = ''
btc_account_name = 'BTC'
fiat_account_name = currency.value
subaccount_name = ''
from beancount.core import account
components = [
type.title(),
country.title(),
institution.title(), btc_account_name, subaccount_name
]
components = [c for c in components if c != '']
btc_account = account.join(*components)
assert account.is_valid(
btc_account), f'Account name is not valid. Got: {btc_account}'
# Test: Treat cash as a liability.
# TODO(nochiel) Store the exchange rate at each transaction date.
components = [
'liabilities'.title(), 'Cash', fiat_account_name, subaccount_name
]
components = [c for c in components if c != '']
fiat_account = account.join(*components)
assert account.is_valid(
fiat_account), f'Account name is not valid. Got: {fiat_account}'
# Loop through on-chain transactions and create transactions and relevant postings for each transaction.
def get_earliest_blocktime(
transactions: list[Transactions] = transactions) -> datetime:
assert transactions
result = datetime.now()
if transactions[0]:
result = datetime.fromtimestamp(
transactions[0][0]['status']['block_time'])
for transactions_for in transactions:
if transactions_for:
for transaction in transactions_for:
timestamp = datetime.fromtimestamp(
transaction['status']['block_time'])
result = timestamp if timestamp < result else result
return result
date_of_account_open = get_earliest_blocktime().date()
# Commodity directive
'''
1867-07-01 commodity CAD
name: "Canadian Dollar"
asset-class: "cash"
'''
btc_commodity_directive = ('2008-10-31 commodity BTC\n'
' name: "Bitcoin"\n'
' asset-class: "cryptocurrency"\n')
# Account directive
# e.g. YYYY-MM-DD open Account [ConstraintCurrency,...] ["BookingMethod"]
account_directives = [
f'{date_of_account_open} open {btc_account}\tBTC',
f'{date_of_account_open} open {fiat_account}\t{currency.value}',
]
transactions_by_hash = {
tx['txid']: tx
for for_address in transactions for tx in for_address
}
'''
transaction_details_by_hash = {detail.hash : detail
for details_for in transaction_details.values()
for detail in details_for}
n_inputs = 0
for d in transaction_details_by_hash.values():
if d.is_input: n_inputs += 1
logger.debug(f'Number of input txs: {n_inputs}')
logger.debug(f'Number of all txs: {len(transaction_details_by_hash.values())}')
'''
# TODO(nochiel) Order transactions by date. For each date record a btc price.
# e.g. 2015-04-30 price AAPL 125.15 USD
btc_price_directive = ''
# Transactions and entries. e.g.
'''
2014-05-05 * "Using my new credit card"
Liabilities:CreditCard:CapitalOne -37.45 USD
Expenses:Restaurant
2014-02-03 * "Initial deposit"
Assets:US:BofA:Checking 100 USD
Assets:Cash -100 USD
'''
# TODO Should I generate Expense accounts if there is an output address that's re-used?
# TODO Create an Asset:BTC and Asset:Cash account
# The Asset:Cash is equivalent to Asset:BTC but in USD exchange rates at the time of transaction.
# Because BTC is volatile, we should list transaction time.
class Payee:
def __init__(self, address: str, amount: int):
self.address = address
self.amount = amount # satoshis
Transaction = dict
def get_payees(transaction: Transaction, ) -> list[Payee]:
assert Transaction
result = []
outputs = transaction['vout']
result = [
Payee(address=output['scriptpubkey_address'],
amount=output['value']) for output in outputs
]
return result
assert transaction_details
# logger.debug(f'transaction_details: {transaction_details}')
transaction_directives = []
for address in addresses:
details = transaction_details[address]
# Post transactions in chronological order. Esplora gives us reverse-chronological order
details.reverse()
for detail in details:
# Create a beancount transaction for each transaction.
# Then add beancount transaction entries for each payee/output that is not one of the user's addresses.
''' E.g.
2014-07-11 * "Sold shares of S&P 500"
Assets:ETrade:IVV -10 IVV {183.07 USD} @ 197.90 USD
Assets:ETrade:Cash 1979.90 USD
Income:ETrade:CapitalGains
'''
meta = ''
date = detail.timestamp.date()
flag = '*'
payees = get_payees(transactions_by_hash[detail.hash])
if not detail.is_input:
payees_in_descriptor = filter(
lambda payee: payee.address in addresses, payees)
payees = list(payees_in_descriptor)
tags = []
links = []
# Should a payee posting use the output address as a subaccount?
# Each payee is a transaction
# If not is_input put our receiving transactions first.
for payee in payees:
transaction_directive = f'{date} * "{payee.address}" "Transaction hash: {detail.hash}"'
btc_payee_transaction_directive = f'\t{btc_account}\t{"-" if detail.is_input else ""}{payee.amount * 1e-8 : .8f} BTC'
transaction_fiat_amount = detail.twap * payee.amount * 1e-8
if not detail.is_input:
btc_payee_transaction_directive += f' {{{detail.twap : .2f} {currency.value} }}'
if detail.is_input:
btc_payee_transaction_directive += f' @ {detail.twap : .2f} {currency.value}\t'
fiat_payee_transaction_directive = (
f'\t{fiat_account}\t{"-" if not detail.is_input else ""}' +
f'{transaction_fiat_amount : .2f} {currency.value}\t')
payee_transaction_directive = btc_payee_transaction_directive
payee_transaction_directive += '\n'
payee_transaction_directive += fiat_payee_transaction_directive
transaction_directive += '\n'
transaction_directive += payee_transaction_directive
transaction_directive += '\n'
transaction_directives.append(transaction_directive)
document = ''
document = btc_commodity_directive
document += '\n'
document += str.join('\n', account_directives)
document += '\n\n'
document += str.join('\n', transaction_directives)
# TODO Validate document
from beancount import loader
_, errors, _ = loader.load_string(document)
if errors:
logger.error(
f'---{len(errors)} Errors in the generated beancount file---')
for error in errors:
logger.error(error)
return document