def DoRemovalOfOption(txn, balances):
entry = CreateTransaction(txn, allow_fees=True)
item = txn['transactionItem']
inst = item['instrument']
assert inst['assetType'] == 'OPTION'
symbol = GetOptionName(inst, entry.date.year)
account = config['asset_position'].format(symbol='Options')
# Note: The contract size isn't present. If we find varying contract
# size we could consider calling the API again to find out what it is.
amt = DF(item['amount'], QO)
amt *= CSIZE
# Find the current amount in the given account to figure out the side of the
# position and the appropriate side to remove. {492fa5292636}
balance = balances[account]
try:
pos = balance[(symbol, None)]
sign = -1 if pos.units.number > 0 else 1
except KeyError:
# Could not find the position. Go short.
sign = -1
cost = CostSpec(None, None, None, None, None, False)
entry.postings.extend([
Posting(account, Amount(sign * amt, symbol), cost, Amount(ZERO, USD)),
Posting(config['pnl']),
])
return entry
def _make_trade_journal_entry(self, t: TradeConfirmation):
txn = Transaction(meta=collections.OrderedDict(),
date=t.settlement_date,
flag='*',
payee=self.payee,
narration='Sell',
tags=EMPTY_SET,
links=EMPTY_SET,
postings=[])
txn.postings.append(
Posting(
account=self.get_stock_account(t),
units=-t.quantity,
cost=CostSpec(number_per=MISSING,
number_total=None,
currency=t.gross_amount.currency,
date=None,
label=None,
merge=False),
price=t.share_price,
meta={
POSTING_DATE_KEY: t.trade_date,
TRADE_REFERENCE_NUMBER_KEY: t.reference_number
},
flag=None,
))
txn.postings.append(
Posting(
account=self.capital_gains_account + ':' + t.symbol,
units=MISSING,
meta=None,
cost=None,
price=None,
flag=None,
))
txn.postings.append(
Posting(
account=self.fees_account,
units=t.fees,
cost=None,
meta={
POSTING_DATE_KEY: t.trade_date,
TRADE_REFERENCE_NUMBER_KEY: t.reference_number,
},
price=None,
flag=None,
))
txn.postings.append(
Posting(
account=self.asset_cash_account,
units=t.net_amount,
cost=None,
meta=None,
price=None,
flag=None,
))
return txn
class TestCostToStr(unittest.TestCase):
"""
Unit tests the beancounttant module function cost_to_str().
"""
cost_per: CostSpec = CostSpec(number_per='1.00',
number_total=None,
currency='USD',
date=None,
label=None,
merge=None)
cost_no_currency: CostSpec = CostSpec(number_per='0.00',
number_total=None,
currency=None,
date=None,
label=None,
merge=None)
cost_total: CostSpec = CostSpec(number_per=None,
number_total='3.14',
currency='PIE',
date=None,
label=None,
merge=None)
cost_per_total: CostSpec = CostSpec(number_per='2.00',
number_total='3.14',
currency='PIE',
date=None,
label=None,
merge=None)
def test_cost_none(self):
self.assertEqual(cost_to_str(None), '')
def test_cost_per(self):
self.assertEqual(cost_to_str(self.cost_per), ' {1.00 USD}')
def test_cost_no_currency(self):
self.assertEqual(cost_to_str(self.cost_no_currency), ' {0.00}')
def test_cost_total(self):
self.assertEqual(cost_to_str(self.cost_total), ' {{3.14 PIE}}')
def test_cost_per_total(self):
self.assertEqual(cost_to_str(self.cost_per_total),
' {2.00 # 3.14 PIE}')
def get_cost(self) -> Optional[CostSpec]:
# TODO need a real cost here, not included in transactions CSV, probably need to
# fill it in from parsing cost-basis lots CSV post-transaction.
return CostSpec(
number_per=Decimal("1"),
number_total=None,
currency="FIXME",
date=None,
label=None,
merge=None,
)
def get_postings(self) -> List[Posting]:
postings = [
Posting(
account=self.get_primary_account(),
units=-Amount(Decimal(str(self.quantity)),
currency=self.symbol),
# TODO handle cost basis by parsing cost-basis lots CSV, so we don't end
# up getting beancount errors due to ambiguity
cost=CostSpec(
number_per=MISSING,
number_total=None,
currency=MISSING,
date=None,
label=None,
merge=None,
),
price=Amount(self.price, currency="USD"),
flag=None,
meta=self.get_meta(),
),
Posting(
account=self.get_other_account(),
units=self.amount,
cost=None,
price=None,
flag=None,
meta=self.get_meta(),
),
]
if self.action != SchwabAction.SELL_TO_OPEN:
# too early to realize gains/losses when opening a short position
postings.append(
Posting(
account=self.get_cap_gains_account(),
units=MISSING,
cost=None,
price=None,
flag=None,
meta={},
))
fees = self.fees
if fees is not None:
postings.append(
Posting(
account=self.fees_account,
units=Amount(self.fees, currency="USD"),
cost=None,
price=None,
flag=None,
meta={},
))
return postings
def get_postings(self) -> List[Posting]:
postings = [
Posting(
account=self.get_primary_account(),
units=Amount(Decimal(str(self.quantity)),
currency=self.symbol),
cost=CostSpec(
number_per=self.price,
number_total=None,
currency="USD",
date=None,
label=None,
merge=None,
),
price=None,
flag=None,
meta=self.get_meta(),
),
Posting(
account=self.get_other_account(),
units=self.amount,
cost=None,
price=None,
flag=None,
meta=self.get_meta(),
),
]
if self.action == SchwabAction.BUY_TO_CLOSE:
# need to record gains when closing a short position
postings.append(
Posting(
account=self.get_cap_gains_account(),
units=MISSING,
cost=None,
price=None,
flag=None,
meta={},
))
fees = self.fees
if fees is not None:
postings.append(
Posting(
account=self.fees_account,
units=Amount(self.fees, currency="USD"),
cost=None,
price=None,
flag=None,
meta={},
))
return postings
def DoStockSplit(txn):
entry = CreateTransaction(txn, allow_fees=True)
item = txn['transactionItem']
inst = item['instrument']
assert inst['assetType'] == 'EQUITY'
symbol = inst['symbol']
amt = DF(item['amount'], QO)
account = config['asset_position'].format(symbol=symbol)
cost = CostSpec(None, None, None, None, None, False)
entry.postings.extend([
Posting(account, Amount(-amt, symbol), cost, None),
Posting(account, Amount(amt * D('2'), symbol), cost, None),
])
return entry
def cost_spec(self, cost_comp_list, is_total):
"""Process a cost_spec grammar rule.
Args:
cost_comp_list: A list of CompoundAmount, a datetime.date, or
label ID strings.
is_total: Assume only the total cost is specified; reject the <number> # <number>
syntax, that is, no compound amounts may be specified. This is used to support
the {{...}} syntax.
Returns:
A cost-info tuple of CompoundAmount, lot date and label string. Any of these
may be set to a sentinel indicating "unset".
"""
if not cost_comp_list:
return CostSpec(MISSING, None, MISSING, None, None, False)
assert isinstance(
cost_comp_list,
list), ("Internal error in parser: {}".format(cost_comp_list))
compound_cost = None
date_ = None
label = None
merge = None
for comp in cost_comp_list:
if isinstance(comp, CompoundAmount):
if compound_cost is None:
compound_cost = comp
else:
self.errors.append(
ParserError(self.get_lexer_location(),
"Duplicate cost: '{}'.".format(comp),
None))
elif isinstance(comp, date):
if date_ is None:
date_ = comp
else:
self.errors.append(
ParserError(self.get_lexer_location(),
"Duplicate date: '{}'.".format(comp),
None))
elif comp is MERGE_COST:
if merge is None:
merge = True
self.errors.append(
ParserError(self.get_lexer_location(),
"Cost merging is not supported yet", None))
else:
self.errors.append(
ParserError(self.get_lexer_location(),
"Duplicate merge-cost spec", None))
else:
assert isinstance(comp, str), (
"Currency component is not string: '{}'".format(comp))
if label is None:
label = comp
else:
self.errors.append(
ParserError(self.get_lexer_location(),
"Duplicate label: '{}'.".format(comp),
None))
# If there was a cost_comp_list, thus a "{...}" cost basis spec, you must
# indicate that by creating a CompoundAmount(), always.
if compound_cost is None:
number_per, number_total, currency = MISSING, None, MISSING
else:
number_per, number_total, currency = compound_cost
if is_total:
if number_total is not None:
self.errors.append(
ParserError(self.get_lexer_location(), (
"Per-unit cost may not be specified using total cost "
"syntax: '{}'; ignoring per-unit cost"
).format(compound_cost), None))
# Ignore per-unit number.
number_per = ZERO
else:
# There's a single number specified; interpret it as a total cost.
number_total = number_per
number_per = ZERO
if merge is None:
merge = False
return CostSpec(number_per, number_total, currency, date_, label,
merge)
def generate_trade_entry(self, ot, file, counter):
""" Involves a commodity. One of: ['buymf', 'sellmf', 'buystock', 'sellstock', 'buyother',
'sellother', 'reinvest']"""
config = self.config
ticker, ticker_long_name = self.get_ticker_info(ot.security)
is_money_market = ticker in self.money_market_funds
# Build metadata
metadata = data.new_metadata(file.name, next(counter))
# metadata['file_account'] = self.file_account(None)
if getattr(ot, 'settleDate',
None) is not None and ot.settleDate != ot.tradeDate:
metadata['settlement_date'] = str(ot.settleDate.date())
description = f'[{ticker}] {ticker_long_name}'
target_acct = self.get_target_acct(ot)
units = ot.units
total = ot.total
# special cases
if 'sell' in ot.type:
units = -1 * abs(ot.units)
if not is_money_market:
metadata[
'todo'] = 'TODO: this entry is incomplete until lots are selected (bean-doctor context <filename> <lineno>)' # noqa: E501
if ot.type in [
'reinvest'
]: # dividends are booked to commodity_leaf. Eg: Income:Dividends:HOOLI
target_acct = self.commodity_leaf(target_acct, ticker)
else:
target_acct = self.commodity_leaf(target_acct, self.currency)
# Build transaction entry
entry = data.Transaction(metadata, ot.tradeDate.date(), self.FLAG,
ot.memo, description, data.EMPTY_SET,
data.EMPTY_SET, [])
# Main posting(s):
main_acct = self.commodity_leaf(config['main_account'], ticker)
if is_money_market: # Use price conversions instead of holding these at cost
common.create_simple_posting_with_price(entry, main_acct, units,
ticker, ot.unit_price,
self.currency)
elif 'sell' in ot.type:
common.create_simple_posting_with_cost_or_price(
entry,
main_acct,
units,
ticker,
price_number=ot.unit_price,
price_currency=self.currency,
costspec=CostSpec(None, None, None, None, None, None))
data.create_simple_posting(entry, self.config['cg'], None,
None) # capital gains posting
else: # buy stock/fund
common.create_simple_posting_with_cost(entry, main_acct, units,
ticker, ot.unit_price,
self.currency)
# "Other" account posting
reverser = 1
if units > 0 and total > 0: # (ugly) hack for some brokerages with incorrect signs (TODO: remove)
reverser = -1
data.create_simple_posting(entry, target_acct, reverser * total,
self.currency)
# Rounding errors posting
rounding_error = (reverser * total) + (ot.unit_price * units)
if 0.0005 <= abs(rounding_error) <= self.max_rounding_error:
data.create_simple_posting(entry, config['rounding_error'],
-1 * rounding_error, 'USD')
# if abs(rounding_error) > self.max_rounding_error:
# print("Transactions legs do not sum up! Difference: {}. Entry: {}, ot: {}".format(
# rounding_error, entry, ot))
return entry
def make_import_result(csv_entry: RawTransaction, accounts: Dict[str, Open],
account_to_id: Dict[str, str],
id_to_account: Dict[str, str]) -> ImportResult:
account_entry = accounts[id_to_account[account_to_id[csv_entry.account]]]
extra_postings = []
other_account = FIXME_ACCOUNT
posting_meta = collections.OrderedDict() # type: Meta
posting_meta[
description_based_source.SOURCE_DESC_KEYS[0]] = csv_entry.description
posting_meta[POSTING_DATE_KEY] = csv_entry.date
if isinstance(csv_entry, CashTransaction):
posting_meta[TRANSACTION_TYPE_KEY] = csv_entry.type
total_amount = csv_entry.units
price = None
cost = None
elif isinstance(csv_entry, FundTransaction):
total_amount = csv_entry.amount
if csv_entry.units.number > ZERO:
# Buy transaction, specify cost but not price.
cost = CostSpec(number_per=csv_entry.price.number,
currency=csv_entry.price.currency,
number_total=None,
date=None,
label=None,
merge=False)
price = None
if csv_entry.description == 'Buy':
other_account = account_entry.account + ':Cash'
elif csv_entry.description == 'Dividend':
other_account = account_entry.meta[
'dividend_account'] + ':' + csv_entry.units.currency
else:
# Sell transaction, specify price but not cost.
price = csv_entry.price
cost = CostSpec(number_per=MISSING,
number_total=None,
currency=csv_entry.price.currency,
date=None,
label=None,
merge=False)
# Add capital gains entry
extra_postings.append(
Posting(
meta=None,
account=account_entry.meta['capital_gains_account'] + ':' +
csv_entry.units.currency,
units=MISSING,
cost=None,
price=None,
flag=None,
))
else:
raise ValueError('unexpected entry type %r' % (csv_entry, ))
entry = Transaction(date=csv_entry.date,
meta=None,
narration=csv_entry.description,
flag=FLAG_OKAY,
payee=None,
tags=EMPTY_SET,
links=EMPTY_SET,
postings=[])
entry.postings.append(
Posting(account=csv_entry.account,
units=csv_entry.units,
price=price,
cost=cost,
flag=None,
meta=posting_meta))
entry.postings.extend(extra_postings)
entry.postings.append(
Posting(meta=None,
account=other_account,
units=-total_amount,
price=None,
cost=None,
flag=None))
return ImportResult(date=entry.date,
entries=[entry],
info=get_info(csv_entry))
# price = Amount(6950, "USD")
# cost = CostSpec(MISSING, None, MISSING, None, None, False)
# units = 200
# currency = "BTC
# cost_spec = CostSpec(
# number_per=MISSING,
# number_total=None,
# currency=MISSING,
# date=None,
# label=None,
# merge=False)
empty_cost_spec = CostSpec(number_per=MISSING,
number_total=None,
currency=MISSING,
date=None,
label=None,
merge=False)
def main():
position_account = "Liabilities:Crypto:Bitfinex:Positions"
margin_account = "Liabilities:Crypto:Bitfinex:Positions"
income_account = "Income:Crypto:Bitfinex:Realized"
fee_account = "Expenses:Trading:Bitfinex:Fees:Trading"
positions: typing.Dict[str, Decimal] = defaultdict(Decimal)
balances: typing.Dict[str, datetime.date] = defaultdict(
lambda: datetime.date(2000, 1, 1))
def extract(self, file, existing_entries=None):
try:
payee_df = pd.read_csv(self.payee_map_file,
sep=',',
header=0,
index_col=0,
keep_default_na=False)
except IOError:
payee_df = pd.DataFrame(columns=['RAW', 'BC', 'POSTING'])
print("Writing to new cache {}".format(self.payee_map_file),
file=sys.stderr)
new_payees = {}
entries = []
index = 0
row = {}
with open(file.name) as file_open:
for index, row in enumerate(csv.DictReader(file_open)):
payee = string.capwords(row['Naam tegenpartij'])
narration = row['Omschrijving-1']
if re.match("^'.*'$", narration):
narration = narration[1:-1]
if re.match('\\s*', payee) and re.match('.*>.*', narration):
splt = narration.split('>', 1)
payee = splt[0]
narration = splt[1]
narration = re.sub("\\s+", " ", narration).strip()
payee = re.sub("\\s+", " ", payee).strip()
payee_mpd = map_payee(payee_df, new_payees, payee, row)
if payee_mpd == "\0":
index -= 1
break
txn = data.Transaction(
meta=data.new_metadata(file.name, index),
date=parse(row['Datum'], yearfirst=False,
dayfirst=False).date(),
flag=flags.FLAG_OKAY,
payee=payee_mpd if payee_mpd else None,
narration=narration,
tags=set(),
links=set(),
postings=[],
)
txn.postings.append(
data.Posting(
self.account_root,
amount.Amount(D(row['Bedrag'].replace(',', '.')),
row['Munt']),
CostSpec(None, row['Oorspr bedrag'],
row['Oorspr munt'], None, None, None), None,
None, None))
add_post(txn, payee_df, payee, row)
entries.append(txn)
if index:
entries.append(
data.Balance(
data.new_metadata(file.name, index),
parse(row['Datum'], dayfirst=True).date() +
timedelta(days=1), self.account_root,
amount.Amount(D(row['Saldo na trn']),
row['Munt']), None, None))
if new_payees:
new_payees_df = pd.DataFrame(new_payees.items(),
columns=['RAW', 'BC'])
payee_df.to_csv(self.payee_map_file + ".old")
payee_df.append(new_payees_df,
ignore_index=True).to_csv(self.payee_map_file)
return entries
def DoTrade(txn, commodities):
entry = CreateTransaction(txn, allow_fees=True)
new_entries = [entry]
# Add the common order id as metadata, to make together multi-leg options
# orders.
if 'orderId' in txn:
match = re.match(r"([A-Z0-9]+)\.\d+", txn['orderId'])
if match:
order_id = match.group(1)
else:
order_id = txn['orderId']
entry.links.add('order-{}'.format(order_id))
# Figure out if this is a sale / clkosing transaction.
item = txn['transactionItem']
inst = item['instrument']
assetType = inst.get('assetType', None)
# It's a sale if the instruction says so. Pretty straightforward.
is_sale = item.get('instruction', None) == 'SELL'
# Add commodity leg.
amount = DF(item['amount'], QO)
if assetType in ('EQUITY', None):
# Short sales will never have the 'CLOSING' flag but they will have a
# 'SHORT SALE' description, which allows us to disambiguate them from
# closing sales. Buys to close short sales will have a description of
# 'CLOSE SHORT POSITION'.
is_closing = txn['description'] in {
'CLOSE SHORT POSITION', 'SELL TRADE'
}
symbol = inst['symbol']
account = config['asset_position'].format(symbol=symbol)
# TODO(blais): Re-enable this in v3 when booking code has been reviewed.
# This triggers and error in booking, but that's how it should rendered.
# is_closing = True
elif assetType == 'OPTION':
# Unfortunately, the 'CLOSING' flag isn't set consistently on the result
# of the API (this would be very helpful if it were). It's only used
# with options, and we only use it then.
is_closing = item.get('positionEffect', None) == 'CLOSING'
symbol = GetOptionName(inst, entry.date.year)
account = config['asset_position'].format(symbol='Options')
# Note: The contract size isn't present. If we find varying contract
# size we could consider calling the API again to find out what it is.
amount *= CSIZE
# Open a new Commodity directive for that one option product.
if not is_closing and symbol not in commodities:
meta = data.new_metadata('<ameritrade>', 0)
meta['name'] = inst['description']
if ADD_EXTRA_METADATA:
meta['assetcls'] = 'Options' # Optional
meta['strategy'] = 'RiskIncome' # Optional
if 'cusip' in inst:
meta['cusip'] = inst['cusip']
commodity = data.Commodity(meta, entry.date, symbol)
commodities[symbol] = commodity
new_entries.insert(0, commodity)
else:
assert False, "Invalid asset type: {}".format(inst)
# Create number of units and price.
units = Amount(amount, symbol)
price = DF(item['price'], QP) if 'price' in item else None
if is_sale:
units = -units
if is_closing:
# If this is a closing trade, the price is the sales price and it needs
# to be booked against a position. Set the price as price annotation,
# not cost.
cost = CostSpec(None, None, None, None, None, False)
entry.postings.append(Posting(account, units, cost, Amount(price,
USD)))
else:
# This is an opening transaction, so the price is the cost basis.
cost = CostSpec(price, None, USD, entry.date, None, False)
entry.postings.append(Posting(account, units, cost))
# Add fees postings.
entry.postings.extend(CreateFeesPostings(txn))
# Cash leg.
units = GetNetAmount(txn)
entry.postings.append(Posting(config['asset_cash'], units))
# Add a P/L leg if we're closing.
if is_closing:
entry.postings.append(Posting(config['pnl']))
return new_entries
def _make_journal_entry(self, r: Release):
txn = Transaction(meta=collections.OrderedDict(),
date=r.release_date,
flag='*',
payee=self.payee,
narration='Stock Vesting',
tags=EMPTY_SET,
links=EMPTY_SET,
postings=[])
txn.postings.append(
Posting(
account=self.get_income_account(r),
units=-r.amount_released,
cost=None,
meta={POSTING_DATE_KEY: r.release_date},
price=r.vest_price,
flag=None,
))
vest_cost_spec = CostSpec(number_per=r.vest_price.number,
currency=r.vest_price.currency,
number_total=None,
date=r.vest_date,
label=r.award_id,
merge=False)
txn.postings.append(
Posting(account=self.asset_account + ':Cash',
units=r.released_market_value_minus_taxes,
cost=None,
meta={
POSTING_DATE_KEY: r.release_date,
AWARD_NOTE_KEY:
'Market value of vested shares minus taxes.',
AWARD_ID_KEY: r.award_id,
},
price=None,
flag=None))
if r.net_release_shares is not None:
# Shares were retained
txn.postings.append(
Posting(account=self.asset_account + ':' + r.symbol,
units=r.net_release_shares,
cost=vest_cost_spec,
meta={
POSTING_DATE_KEY: r.release_date,
AWARD_ID_KEY: r.award_id,
},
price=None,
flag=None))
txn.postings.append(
Posting(
account=self.asset_account + ':Cash',
units=-Amount(number=round(
r.vest_price.number * r.net_release_shares.number, 2),
currency=r.vest_price.currency),
cost=None,
meta={
POSTING_DATE_KEY: r.release_date,
AWARD_NOTE_KEY: 'Cost of shares retained',
AWARD_ID_KEY: r.award_id,
},
price=None,
flag=None,
))
else:
# Shares were sold
# Add capital gains posting.
txn.postings.append(
Posting(
meta=None,
account=self.capital_gains_account + ':' + r.symbol,
units=-r.capital_gains,
cost=None,
price=None,
flag=None,
))
capital_gains_amount = r.capital_gains
if r.fee_amount is not None:
capital_gains_amount = amount_sub(capital_gains_amount,
r.fee_amount)
# Add cash posting for capital gains.
txn.postings.append(
Posting(
account=self.asset_account + ':Cash',
units=capital_gains_amount,
cost=None,
meta={
POSTING_DATE_KEY: r.release_date,
AWARD_NOTE_KEY: 'Capital gains less transaction fees',
AWARD_ID_KEY: r.award_id,
},
price=None,
flag=None,
))
if r.fee_amount is not None:
txn.postings.append(
Posting(
account=self.fees_account,
units=r.fee_amount,
cost=None,
meta={
POSTING_DATE_KEY: r.release_date,
AWARD_NOTE_KEY: 'Supplemental transaction fee',
AWARD_ID_KEY: r.award_id,
},
price=None,
flag=None,
))
if self.tax_accounts is None:
# Just use a single unknown account to catch all of the tax costs.
# This allows the resultant entry to match a payroll entry that includes the tax costs.
txn.postings.append(
Posting(
account=FIXME_ACCOUNT,
units=r.total_tax_amount,
cost=None,
meta=dict(),
price=None,
flag=None,
))
else:
for tax_key, tax_account_pattern in self.tax_accounts.items():
if tax_key not in r.fields:
continue
amount = parse_amount(r.fields[tax_key])
account = tax_account_pattern.format(year=r.release_date.year)
txn.postings.append(
Posting(
account=account,
units=amount,
cost=None,
meta={POSTING_DATE_KEY: r.release_date},
price=None,
flag=None,
))
return txn
