def test_get_latest_price(self, entries, _, __):
"""
2013-06-01 price USD 1.01 CAD
2013-06-09 price USD 1.09 CAD
2013-06-11 price USD 1.11 CAD
"""
price_map = prices.build_price_map(entries)
price_list = prices.get_latest_price(price_map, ('USD', 'CAD'))
expected = (datetime.date(2013, 6, 11), D('1.11'))
self.assertEqual(expected, price_list)
# Test not found.
result = prices.get_latest_price(price_map, ('EWJ', 'JPY'))
self.assertEqual((None, None), result)
def convert_to_currency(price_map, target_currency, holdings_list):
"""Convert the given list of holdings's fields to a common currency.
If the rate is not available to convert, leave the fields empty.
Args:
price_map: A price-map, as built by prices.build_price_map().
target_currency: The target common currency to convert amounts to.
holdings_list: A list of holdings.Holding instances.
Returns:
A modified list of holdings, with the 'extra' field set to the value in
'currency', or None, if it was not possible to convert.
"""
# A list of the fields we should convert.
convert_fields = ('cost_number', 'book_value', 'market_value', 'price_number')
new_holdings = []
for holding in holdings_list:
if holding.cost_currency == target_currency:
# The holding is already priced in the target currency; do nothing.
new_holding = holding
else:
if holding.cost_currency is None:
# There is no cost currency; make the holding priced in its own
# units. The price-map should yield a rate of 1.0 and everything
# else works out.
if holding.currency is None:
raise ValueError("Invalid currency '{}'".format(holding.currency))
holding = holding._replace(cost_currency=holding.currency)
# Fill in with book and market value as well.
if holding.book_value is None:
holding = holding._replace(book_value=holding.number)
if holding.market_value is None:
holding = holding._replace(market_value=holding.number)
assert holding.cost_currency, "Missing cost currency: {}".format(holding)
base_quote = (holding.cost_currency, target_currency)
# Get the conversion rate and replace the required numerical
# fields..
_, rate = prices.get_latest_price(price_map, base_quote)
if rate is not None:
new_holding = misc_utils.map_namedtuple_attributes(
convert_fields,
lambda number, r=rate: number if number is None else number * r,
holding)
# Ensure we set the new cost currency after conversion.
new_holding = new_holding._replace(cost_currency=target_currency)
else:
# Could not get the rate... clear every field and set the cost
# currency to None. This enough marks the holding conversion as
# a failure.
new_holding = misc_utils.map_namedtuple_attributes(
convert_fields, lambda number: None, holding)
new_holding = new_holding._replace(cost_currency=None)
new_holdings.append(new_holding)
return new_holdings
def _account_latest_price(self, node):
# Get latest price date
quote_price = list(node.balance.keys())[0]
if quote_price[1] is None:
latest_price = None
else:
base = quote_price[0]
currency = quote_price[1][1]
latest_price = prices.get_latest_price(g.ledger.price_map, (currency, base))
return latest_price
def get_rates_table(entries: data.Entries, currencies: Set[str],
main_currency: str) -> Table:
"""Enumerate all the exchange rates."""
price_map = prices.build_price_map(entries)
header = ['cost_currency', 'rate_file']
rows = []
for currency in currencies:
_, rate = prices.get_latest_price(price_map, (currency, main_currency))
if rate is None:
continue
rows.append([currency, rate.quantize(PRICE_Q)])
return Table(header, rows)
def get_prices_table(entries: data.Entries, main_currency: str) -> Table:
"""Enumerate all the prices seen."""
price_map = prices.build_price_map(entries)
header = ['currency', 'cost_currency', 'price_file']
rows = []
for base_quote in price_map.keys():
_, price = prices.get_latest_price(price_map, base_quote)
if price is None:
continue
base, quote = base_quote
rows.append([base, quote, price.quantize(PRICE_Q)])
return Table(header, rows)
def get_price_jobs_up_to_date(entries,
date_last=None,
inactive=False,
undeclared_source=None,
update_rate='weekday',
compress_days=1):
"""Get a list of trailing prices to fetch from a stream of entries.
The list of dates runs from the latest available price up to the latest date.
Args:
entries: list of Beancount entries
date_last: The date up to where to find prices to as an exclusive range end.
inactive: Include currencies with no balance at the given date. The default
is to only include those currencies which have a non-zero balance.
undeclared_source: A string, the name of the default source module to use to
pull prices for commodities without a price source metadata on their
Commodity directive declaration.
Returns:
A list of DatedPrice instances.
"""
price_map = prices.build_price_map(entries)
# Find the list of declared currencies, and from it build a mapping for
# tickers for each (base, quote) pair. This is the only place tickers
# appear.
declared_triples = find_currencies_declared(entries, date_last)
currency_map = {(base, quote): psources
for base, quote, psources in declared_triples}
# Compute the initial list of currencies to consider.
if undeclared_source:
# Use the full set of possible currencies.
cur_at_cost = find_prices.find_currencies_at_cost(entries)
cur_converted = find_prices.find_currencies_converted(
entries, date_last)
cur_priced = find_prices.find_currencies_priced(entries, date_last)
currencies = cur_at_cost | cur_converted | cur_priced
log_currency_list("Currency held at cost", cur_at_cost)
log_currency_list("Currency converted", cur_converted)
log_currency_list("Currency priced", cur_priced)
default_source = import_source(undeclared_source)
else:
# Use the currencies from the Commodity directives.
currencies = set(currency_map.keys())
default_source = None
log_currency_list("Currencies in primary list", currencies)
# By default, restrict to only the currencies with non-zero balances
# up to the given date.
# Also, find the earliest start date to fetch prices from.
# Look at both latest prices and start dates.
lifetimes_map = lifetimes.get_commodity_lifetimes(entries)
commodity_map = getters.get_commodity_directives(entries)
price_start_dates = {}
stale_currencies = set()
if inactive:
for base_quote in currencies:
if lifetimes_map[base_quote]:
# Use first date from lifetime
lifetimes_map[base_quote] = [(lifetimes_map[base_quote][0][0],
None)]
else:
# Insert never active commodities into lifetimes
# Start from date of currency directive
base, _ = base_quote
commodity_entry = commodity_map.get(base, None)
lifetimes_map[base_quote] = [(commodity_entry.date, None)]
else:
#Compress any lifetimes based on compress_days
lifetimes_map = lifetimes.compress_lifetimes_days(
lifetimes_map, compress_days)
#Trim lifetimes based on latest price dates.
for base_quote in lifetimes_map:
intervals = lifetimes_map[base_quote]
result = prices.get_latest_price(price_map, base_quote)
if (result is None or result[0] is None):
lifetimes_map[base_quote] = \
lifetimes.trim_intervals(intervals,
None,
date_last)
else:
latest_price_date = result[0]
date_first = latest_price_date + datetime.timedelta(days=1)
if date_first < date_last:
lifetimes_map[base_quote] = \
lifetimes.trim_intervals(intervals,
date_first,
date_last)
else:
# We don't need to update if we're already up to date.
lifetimes_map[base_quote] = []
# Remove currency pairs we can't fetch any prices for.
if not default_source:
keys = list(lifetimes_map.keys())
for key in keys:
if not currency_map.get(key, None):
del lifetimes_map[key]
# Create price jobs based on fetch rate
if update_rate == 'daily':
required_prices = lifetimes.required_daily_prices(lifetimes_map,
date_last,
weekdays_only=False)
elif update_rate == 'weekday':
required_prices = lifetimes.required_daily_prices(lifetimes_map,
date_last,
weekdays_only=True)
elif update_rate == 'weekly':
required_prices = lifetimes.required_weekly_prices(
lifetimes_map, date_last)
else:
raise ValueError('Invalid Update Rate')
jobs = []
# Build up the list of jobs to fetch prices for.
for key in required_prices:
date, base, quote = key
psources = currency_map.get((base, quote), None)
if not psources:
psources = [PriceSource(default_source, base, False)]
jobs.append(DatedPrice(base, quote, date, psources))
return sorted(jobs)
def export_holdings(entries,
options_map,
promiscuous,
aggregate_by_commodity=False):
"""Compute a list of holdings to export.
Holdings that are converted to cash equivalents will receive a currency of
"CASH:<currency>" where <currency> is the converted cash currency.
Args:
entries: A list of directives.
options_map: A dict of options as provided by the parser.
promiscuous: A boolean, true if we should output a promiscuious memo.
aggregate_by_commodity: A boolean, true if we should group the holdings by account.
Returns:
A pair of
exported: A list of ExportEntry tuples, one for each exported position.
converted: A list of ExportEntry tuples, one for each converted position.
These will contain multiple holdings.
holdings_ignored: A list of Holding instances that were ignored, either
because they were explicitly marked to be ignored, or because we could
not convert them to a money vehicle matching the holding's cost-currency.
"""
# Get the desired list of holdings.
holdings_list, price_map = holdings_reports.get_assets_holdings(
entries, options_map)
commodities_map = getters.get_commodity_map(entries)
dcontext = options_map['dcontext']
# Aggregate the holdings, if requested. Google Finance is notoriously
# finnicky and if you have many holdings this might help.
if aggregate_by_commodity:
holdings_list = holdings.aggregate_holdings_by(
holdings_list, lambda holding: holding.currency)
# Classify all the holdings for export.
action_holdings = classify_holdings_for_export(holdings_list,
commodities_map)
# The lists of exported and converted export entries, and the list of
# ignored holdings.
exported = []
converted = []
holdings_ignored = []
# Export the holdings with tickers individually.
for symbol, holding in action_holdings:
if symbol in ("CASH", "IGNORE"):
continue
if holding.cost_number is None:
assert holding.cost_currency in (None, holding.currency)
cost_number = holding.number
cost_currency = holding.currency
else:
cost_number = holding.cost_number
cost_currency = holding.cost_currency
exported.append(
ExportEntry(symbol, cost_currency, holding.number, cost_number,
is_mutual_fund(symbol),
holding.account if promiscuous else '', [holding]))
# Convert all the cash entries to their book and market value by currency.
cash_holdings_map = collections.defaultdict(list)
for symbol, holding in action_holdings:
if symbol != "CASH":
continue
if holding.cost_currency:
# Accumulate market and book values.
cash_holdings_map[holding.cost_currency].append(holding)
else:
# We cannot price this... no cost currency.
holdings_ignored.append(holding)
# Get the money instruments.
money_instruments = get_money_instruments(commodities_map)
# Convert all the cash values to money instruments, if possible. If not
# possible, we'll just have to ignore those values.
# Go through all the holdings to convert, and for each of those which aren't
# in terms of one of the money instruments, which we can directly add to the
# exported portfolio, attempt to convert them into currencies to one of
# those in the money instruments.
money_values_book = collections.defaultdict(D)
money_values_market = collections.defaultdict(D)
money_values_holdings = collections.defaultdict(list)
for cost_currency, holdings_list in cash_holdings_map.items():
book_value = sum(holding.book_value for holding in holdings_list)
market_value = sum(holding.market_value for holding in holdings_list)
if cost_currency in money_instruments:
# The holding is already in terms of one of the money instruments.
money_values_book[cost_currency] += book_value
money_values_market[cost_currency] += market_value
money_values_holdings[cost_currency].extend(holdings_list)
else:
# The holding is not in terms of one of the money instruments.
# Find the first available price to convert it into one
for money_currency in money_instruments:
base_quote = (cost_currency, money_currency)
_, rate = prices.get_latest_price(price_map, base_quote)
if rate is not None:
money_values_book[money_currency] += book_value * rate
money_values_market[money_currency] += market_value * rate
money_values_holdings[money_currency].extend(holdings_list)
break
else:
# We could not convert into any of the money commodities. Ignore
# those holdings.
holdings_ignored.extend(holdings_list)
for money_currency in money_values_book.keys():
book_value = money_values_book[money_currency]
market_value = money_values_market[money_currency]
holdings_list = money_values_holdings[money_currency]
symbol = money_instruments[money_currency]
assert isinstance(book_value, Decimal)
assert isinstance(market_value, Decimal)
converted.append(
ExportEntry(
symbol, money_currency,
dcontext.quantize(market_value, money_currency),
dcontext.quantize(book_value / market_value, money_currency),
is_mutual_fund(symbol), '', holdings_list))
# Add all ignored holdings to a final list.
for symbol, holding in action_holdings:
if symbol == "IGNORE":
holdings_ignored.append(holding)
return exported, converted, holdings_ignored
def compute_networth_series(since_date, end_date=None):
if end_date is None:
end_date = datetime.date.today()
(entries, errors, options_map) = beancount.loader.load_file(get_ledger_file())
account_map = get_account_map(entries)
target_currency = 'CNY'
curr_date = since_date
result = []
prev_networth = None
prev_disposable_networth = None
cum_invest_nav = decimal.Decimal("1.0")
while curr_date < end_date:
entries_to_date = [entry for entry in entries if entry.date <= curr_date]
holdings_list, price_map_to_date = holdings_reports.get_assets_holdings(
entries_to_date, options_map, target_currency
)
networth_in_cny = 0
disposable_networth_in_cny = 0
for hld in holdings_list:
acc = account_map[hld.account]
# 预付但大部分情况下不能兑现的沉没资产
is_sunk = bool(int(acc.meta.get("sunk", 0)))
# 不可支配
nondisposable = bool(int(acc.meta.get("nondisposable", 0)))
if not nondisposable:
disposable_networth_in_cny += hld.market_value
networth_in_cny += hld.market_value
txs_of_date = [
entry for entry in entries
if entry.date == curr_date and
isinstance(entry, beancount.core.data.Transaction)
]
non_trade_expenses = 0
non_trade_incomes = 0
for tx in txs_of_date:
for posting in tx.postings:
acc = posting.account
is_nt_exp = acc.startswith("Expenses:") and not acc.startswith("Expenses:Trade:")
is_nt_inc = acc.startswith("Income:") and not acc.startswith("Income:Trade:")
if is_nt_exp or is_nt_inc:
if posting.units.currency != target_currency:
base_quote = (posting.units.currency, target_currency)
_, rate = prices.get_latest_price(price_map_to_date, base_quote)
else:
rate = 1
if is_nt_exp:
non_trade_expenses += (posting.units.number * rate)
else:
non_trade_incomes -= (posting.units.number * rate)
if prev_networth:
pnl = networth_in_cny - non_trade_incomes + non_trade_expenses - prev_networth
pnl_str = ("%.2f" % pnl) if not isinstance(pnl, str) else pnl
pnl_rate_str = "%.4f%%" % (100 * pnl / prev_disposable_networth)
cum_invest_nav *= (1 + pnl / prev_disposable_networth)
else:
pnl_str = 'n/a'
pnl_rate_str = 'n/a'
result.append({
"日期": curr_date,
# 净资产=资产 - 负债（信用卡）, 包含了沉没资产
"净资产": "%.2f" % networth_in_cny,
# 可投资金额=净资产 - 不可支配资产（公积金、预付房租、宽带)
"可投资金额": "%.2f" % disposable_networth_in_cny,
# Income:Trade(已了结盈亏、分红) 以外的 Income (包含公积金收入、储蓄利息)
"非投资收入": "%.2f" % non_trade_incomes,
# Expenses:Trade 以外的 Expenses (包含社保等支出)
"非投资支出": "%.2f" % non_trade_expenses,
"投资盈亏": pnl_str,
# 投资盈亏% = 当日投资盈亏/昨日可投资金额
"投资盈亏%": pnl_rate_str,
"累计净值": "%.4f" % cum_invest_nav,
})
curr_date += datetime.timedelta(days=1)
prev_networth = networth_in_cny
prev_disposable_networth = disposable_networth_in_cny
return result
def compute_networth_series(since_date, end_date=None):
if end_date is None:
end_date = datetime.date.today()
(entries, errors,
options_map) = beancount.loader.load_file(get_ledger_file())
account_map, commodity_map = get_maps(entries)
target_currency = 'CNY'
curr_date = since_date
result = []
prev_networth = None
prev_disposable_networth = None
cum_invest_nav = decimal.Decimal("1.0")
cum_invest_nav_ytd = decimal.Decimal("1.0")
cum_invest_pnl = decimal.Decimal(0)
cum_invest_pnl_ytd = decimal.Decimal(0)
while curr_date <= end_date:
entries_to_date = [
entry for entry in entries if entry.date <= curr_date
]
holdings_list, price_map_to_date = get_assets_holdings(
entries_to_date, options_map, target_currency)
raw_networth_in_cny = decimal.Decimal(0) # 包含sunk资产的理论净资产
networth_in_cny = decimal.Decimal(0) # 不包含sunk资产的净资产
disposable_networth_in_cny = decimal.Decimal(0)
dnw_by_asset_class = {}
for hld in holdings_list:
if hld.currency == "DAY":
continue
acc = account_map[hld.account]
cmdt = commodity_map[hld.currency]
if hld.market_value is None:
raise ValueError(hld)
raw_networth_in_cny += hld.market_value
# 预付但大部分情况下不能兑现的沉没资产，比如预付的未来房租
is_sunk = bool(int(acc.meta.get("sunk", 0)))
if is_sunk:
continue
# 不可支配，比如房租押金
nondisposable = bool(int(acc.meta.get("nondisposable", 0)))
if not nondisposable:
disposable_networth_in_cny += hld.market_value
asset_class = cmdt.meta["asset-class"]
if asset_class not in dnw_by_asset_class:
dnw_by_asset_class[asset_class] = decimal.Decimal(0)
dnw_by_asset_class[asset_class] += hld.market_value
networth_in_cny += hld.market_value
txs_of_date = [
entry for entry in entries if entry.date == curr_date
and isinstance(entry, beancount.core.data.Transaction)
]
non_trade_expenses = decimal.Decimal(0)
non_trade_incomes = decimal.Decimal(0)
for tx in txs_of_date:
is_time_tx = any(
(posting.units.currency == "DAY" for posting in tx.postings))
if is_time_tx:
continue
for posting in tx.postings:
acc = posting.account
is_non_trade_exp = (
acc.startswith(EXPENSES_PREFIX)
and not acc.startswith(EXPENSES_TRADE_PREFIX)) or (
acc.startswith(EXPENSES_PREPAYMENTS_PREFIX))
is_non_trade_inc = acc.startswith(
"Income:") and not acc.startswith("Income:Trade:")
if is_non_trade_exp or is_non_trade_inc:
if posting.units.currency != target_currency:
base_quote = (posting.units.currency, target_currency)
_, rate = prices.get_latest_price(
price_map_to_date, base_quote)
else:
rate = decimal.Decimal(1)
if is_non_trade_exp:
non_trade_expenses += (posting.units.number * rate)
else:
non_trade_incomes -= (posting.units.number * rate)
if prev_networth:
pnl = networth_in_cny - non_trade_incomes + non_trade_expenses - prev_networth
pnl_str = ("%.2f" % pnl) if not isinstance(pnl, str) else pnl
pnl_rate_str = "%.4f%%" % (100 * pnl / prev_disposable_networth)
cum_invest_nav *= (1 + pnl / prev_disposable_networth)
cum_invest_nav_ytd *= (1 + pnl / prev_disposable_networth)
cum_invest_pnl += pnl
cum_invest_pnl_ytd += pnl
else:
pnl = None
pnl_str = 'n/a'
pnl_rate_str = 'n/a'
daily_status = {
"日期": curr_date,
# 理论净资产=总资产 - 负债
"理论净资产": "%.2f" % raw_networth_in_cny,
# 净资产=总资产' - 负债（信用卡）- 沉没资产
"净资产": "%.2f" % networth_in_cny,
"沉没资产": "%.2f" % (raw_networth_in_cny - networth_in_cny),
# 可投资金额=净资产 - 不可支配资产（公积金、预付房租、宽带)
"可投资净资产": "%.2f" % disposable_networth_in_cny,
# Income:Trade(已了结盈亏、分红) 以外的 Income (包含公积金收入、储蓄利息)
"非投资收入": "%.2f" % non_trade_incomes,
# Expenses:Trade 以外的 Expenses (包含社保等支出)
"非投资支出": "%.2f" % non_trade_expenses,
"投资盈亏": pnl_str,
# 投资盈亏% = 当日投资盈亏/昨日可投资金额
"投资盈亏%": pnl_rate_str,
"累计净值": "%.4f" % cum_invest_nav,
"累计盈亏": "%.2f" % cum_invest_pnl,
"当年净值": "%.4f" % cum_invest_nav_ytd,
"当年盈亏": "%.2f" % cum_invest_pnl_ytd,
}
if curr_date.weekday() >= 5:
if pnl is not None:
assert pnl <= 0.01, daily_status # 预期周末不应该有投资盈亏
assert abs(
sum(dnw_by_asset_class.values()) - disposable_networth_in_cny) < 1
assert set(dnw_by_asset_class.keys()
) <= KNOWN_ASSET_CLASSES, dnw_by_asset_class
for asset_class in KNOWN_ASSET_CLASSES:
propotion = 100 * dnw_by_asset_class.get(
asset_class, 0) / disposable_networth_in_cny
daily_status[f"{asset_class}%"] = f"{propotion:.2f}%"
result.append(daily_status)
next_date = curr_date + datetime.timedelta(days=1)
if curr_date.year != next_date.year:
cum_invest_nav_ytd = decimal.Decimal("1.0")
cum_invest_pnl_ytd = decimal.Decimal(0)
curr_date = next_date
prev_networth = networth_in_cny
prev_disposable_networth = disposable_networth_in_cny
return result
