def extract_trades(entries):
"""Find all the matching trades from the metadata attached to postings.
This inspects all the postings and pairs them up using the special metadata
field that was added by this plugin when booking matching lots, and returns
pairs of those postings.
Args:
entries: The list of directives to extract from.
Returns:
A list of (number, augmenting-posting, reducing-posting).
"""
trade_map = collections.defaultdict(list)
for index, entry in enumerate(entries):
if not isinstance(entry, data.Transaction):
continue
for posting in entry.postings:
links_str = posting.meta.get(META, None)
if links_str:
links = links_str.split(',')
for link in links:
trade_map[link].append((index, entry, posting))
# Sort matches according to the index of the first entry, drop the index
# used for doing this, and convert the objects to tuples..
trades = [(data.TxnPosting(augmenting[1], augmenting[2]),
data.TxnPosting(reducing[1], reducing[2]))
for augmenting, reducing in sorted(trade_map.values())]
# Sanity check.
for matches in trades:
assert len(matches) == 2
return trades
def test_get_entry(self):
entry = self.create_empty_transaction()
posting = data.create_simple_posting(entry, 'Assets:Bank:Checking',
'123.45', 'USD')
self.assertEqual(entry, data.get_entry(entry))
self.assertEqual(entry, data.get_entry(data.TxnPosting(entry,
posting)))
def expand_sales_legs(entries, account, start, end, calculate_commission):
# Expand each of the sales legs.
balances = collections.defaultdict(inventory.Inventory)
sales = []
for txn in data.filter_txns(entries):
# If we got to the end of the period, bail out.
if txn.date >= end:
break
# Accumulate the balances before the start date.
if txn.date < start:
for posting in txn.postings:
if re.match(account, posting.account):
balance = balances[posting.account]
balance.add_position(posting)
continue
# Fallthrough: we're not in the period. Process the matching postings.
# Find reducing postings (i.e., for each lot).
txn_sales = []
for posting in txn.postings:
if re.match(account, posting.account):
balance = balances[posting.account]
reduced_position, booking = balance.add_position(posting)
# Set the cost on the posting from the reduced position.
# FIXME: Eventually that'll happen automatically during the full
# booking stage.
if booking == inventory.Booking.REDUCED:
posting = posting._replace(cost=reduced_position.cost)
# If the postings don't have a reference number, ignore them.
if 'ref' not in txn.meta:
continue
if (posting.cost and posting.units.number < ZERO):
if not posting.price:
logging.error("Missing price on %s", posting)
txn_sales.append(data.TxnPosting(txn, posting))
if txn_sales and calculate_commission:
# Find total commission.
for posting in txn.postings:
if re.search('Commission', posting.account):
commission = posting.units.number
break
else:
commission = ZERO
# Compute total number of units.
tot_units = sum(sale.posting.units.number for sale, _ in txn_sales)
# Assign a proportion of the commission to each of the sales by
# inserting it into its posting metadata. This will be processed below.
for sale, _ in txn_sales:
fraction = sale.posting.units.number / tot_units
sale.posting.meta['commission'] = fraction * commission
sales.extend(txn_sales)
return sales
def get_postings(filename, account_regexp, tag=None):
if tag:
match = lambda entry, posting: (re.match(
account_regexp, posting.account) and tag in entry.tags)
else:
match = lambda _, posting: (re.match(account_regexp, posting.account))
entries, _, _ = loader.load_file(filename)
txn_postings = [
data.TxnPosting(entry, posting) for entry in data.filter_txns(entries)
for posting in entry.postings if match(entry, posting)
]
return txn_postings
def test_posting_sortkey(self):
entries = self.create_sort_data()
txn_postings = [(data.TxnPosting(entry, entry.postings[0])
if isinstance(entry, data.Transaction) else entry)
for entry in entries]
sorted_txn_postings = sorted(txn_postings, key=data.posting_sortkey)
self.assertEqual([
data.TxnPosting, data.Open, data.Balance, data.TxnPosting,
data.TxnPosting, data.Close, data.TxnPosting
], list(map(type, sorted_txn_postings)))
self.assertEqual([900, 1002, 1001, 1008, 1009, 1000, 1100], [
entry.meta["lineno"]
for entry in map(data.get_entry, sorted_txn_postings)
])
def clear_transactions(self, entries):
errors = []
self.modified_entries = {}
groups = collections.defaultdict(list)
for entry in entries:
if (not isinstance(entry, data.Transaction)
or (entry.tags and self.ignored_tag_name in entry.tags)):
continue
posting = self.get_txn_clearing_posting(entry)
if posting:
groups[posting.account].append(data.TxnPosting(entry, posting))
# NOTE: sorting is only needed to support testing
for _, txn_postings in sorted(groups.items(), key=lambda x: x[0]):
self.clear_transaction_group(txn_postings)
return self.modified_entries, errors
def split_entries(entries, dit_component, ignored_tag):
dits, unchanged_entries, errors = [], [], []
for entry in entries:
if (isinstance(entry, data.Transaction)
and not (entry.tags and ignored_tag in entry.tags)):
dit_postings = [
posting for posting in entry.postings
if has_component(posting.account, dit_component)
]
num_dit_postings = len(dit_postings)
else:
num_dit_postings = 0
if num_dit_postings == 0:
unchanged_entries.append(entry)
else:
dits.append(data.TxnPosting(entry, dit_postings[0]))
if num_dit_postings > 1:
errors.append(
DITError(
entry.meta,
"(deposit_in_transit) Found entry with multiple postings to DIT accounts; "
"only processing posting to {} account".format(
dit_postings[0].account), entry))
return dits, unchanged_entries, errors
def book_price_conversions(entries, assets_account, income_account):
"""Rewrite transactions to insert cost basis according to a booking method.
See module docstring for full details.
Args:
entries: A list of entry instances.
assets_account: An account string, the name of the account to process.
income_account: An account string, the name of the account to use for booking
realized profit/loss.
Returns:
A tuple of
entries: A list of new, modified entries.
errors: A list of errors generated by this plugin.
matches: A list of (number, augmenting-posting, reducing-postings) for all
matched lots.
"""
# Pairs of (Position, Transaction) instances used to match augmenting
# entries with reducing ones.
pending_lots = []
# A list of pairs of matching (augmenting, reducing) postings.
all_matches = []
new_entries = []
errors = []
for eindex, entry in enumerate(entries):
# Figure out if this transaction has postings in Bitcoins without a cost.
# The purpose of this plugin is to fixup those.
if isinstance(entry, data.Transaction) and any(
is_matching(posting, assets_account)
for posting in entry.postings):
# Segregate the reducing lots, augmenting lots and other lots.
augmenting, reducing, other = [], [], []
for pindex, posting in enumerate(entry.postings):
if is_matching(posting, assets_account):
out = augmenting if posting.units.number >= ZERO else reducing
else:
out = other
out.append(posting)
# We will create a replacement list of postings with costs filled
# in, possibly more than the original list, to account for the
# different lots.
new_postings = []
# Convert all the augmenting postings to cost basis.
for posting in augmenting:
new_postings.append(
augment_inventory(pending_lots, posting, entry, eindex))
# Then process reducing postings.
if reducing:
# Process all the reducing postings, booking them to matching lots.
pnl = inventory.Inventory()
for posting in reducing:
rpostings, matches, posting_pnl, new_errors = (
reduce_inventory(pending_lots, posting, eindex))
new_postings.extend(rpostings)
all_matches.extend(matches)
errors.extend(new_errors)
pnl.add_amount(
amount.Amount(posting_pnl, posting.price.currency))
# If some reducing lots were seen in this transaction, insert an
# Income leg to absorb the P/L. We need to do this for each currency
# which incurred P/L.
if not pnl.is_empty():
for pos in pnl:
meta = data.new_metadata('<book_conversions>', 0)
new_postings.append(
data.Posting(income_account, -pos.units, None,
None, None, meta))
# Third, add back all the other unrelated legs in.
for posting in other:
new_postings.append(posting)
# Create a replacement entry.
entry = entry._replace(postings=new_postings)
new_entries.append(entry)
# Add matching metadata to all matching postings.
mod_matches = link_entries_with_metadata(new_entries, all_matches)
# Resolve the indexes to their possibly modified Transaction instances.
matches = [(data.TxnPosting(new_entries[aug_index], aug_posting),
data.TxnPosting(new_entries[red_index], red_posting))
for (aug_index, aug_posting), (red_index,
red_posting) in mod_matches]
return new_entries, errors, matches
def main():
logging.basicConfig(level=logging.INFO, format='%(levelname)-8s: %(message)s')
parser = argparse.ArgumentParser(description=__doc__.strip())
parser.add_argument('report', choices=['detail', 'aggregate', 'summary'],
help='Type of report')
parser.add_argument('filename',
help='Beancount input file')
parser.add_argument('account',
help='Account name')
parser.add_argument('--start', type=date_utils.parse_date_liberally,
help="Start date")
parser.add_argument('--end', type=date_utils.parse_date_liberally,
help="End date; if not set, at the end of star'ts year")
parser.add_argument('-o', '--output', action='store',
help="Output filename for the CSV file")
args = parser.parse_args()
calculate_commission = False
# Setup date interval.
if args.start is None:
args.start = datetime.date(datetime.date.today().year, 1, 1)
if args.end is None:
args.end = datetime.date(args.start.year + 1, 1, 1)
entries, errors, options_map = loader.load_file(args.filename)
# Expand each of the sales legs.
balances = collections.defaultdict(inventory.Inventory)
sales = []
for txn in data.filter_txns(entries):
# If we got to the end of the period, bail out.
if txn.date >= args.end:
break
# Accumulate the balances before the start date.
if txn.date < args.start:
for posting in txn.postings:
if re.match(args.account, posting.account):
balance = balances[posting.account]
balance.add_position(posting)
continue
# Fallthrough: we're not in the period. Process the matching postings.
# Find reducing postings (i.e., for each lot).
txn_sales = []
for posting in txn.postings:
if re.match(args.account, posting.account):
balance = balances[posting.account]
reduced_position, booking = balance.add_position(posting)
# Set the cost on the posting from the reduced position.
# FIXME: Eventually that'll happen automatically during the full
# booking stage.
if booking == inventory.Booking.REDUCED:
posting = posting._replace(cost=reduced_position.cost)
# If the postings don't have a reference number, ignore them.
if 'ref' not in txn.meta:
continue
if (posting.cost and
posting.units.number < ZERO):
if not posting.price:
logging.error("Missing price on %s", posting)
txn_sales.append(data.TxnPosting(txn, posting))
if txn_sales and calculate_commission:
# Find total commission.
for posting in txn.postings:
if re.search('Commission', posting.account):
commission = posting.units.number
break
else:
commission = ZERO
# Compute total number of units.
tot_units = sum(sale.posting.units.number
for sale, _ in txn_sales)
# Assign a proportion of the commission to each of the sales by
# inserting it into its posting metadata. This will be processed below.
for sale, _ in txn_sales:
fraction = sale.posting.units.number / tot_units
sale.posting.meta['commission'] = fraction * commission
sales.extend(txn_sales)
# Convert into a table of data, full detail of very single log.
Q = D('0.01')
lots = []
total_loss = collections.defaultdict(D)
total_gain = collections.defaultdict(D)
total_adj = collections.defaultdict(D)
# If no mssb number has been assigned explicitly, assign a random one. I
# need to figure out how to find those numbers again.
auto_mssb_number = itertools.count(start=1000000 + 1)
for sale in sales:
try:
sale_no = sale.txn.meta['mssb']
except KeyError:
sale_no = next(auto_mssb_number)
ref = sale.txn.meta['ref']
units = sale.posting.units
totcost = (-units.number * sale.posting.cost.number).quantize(Q)
totprice = (-units.number * sale.posting.price.number).quantize(Q)
commission_meta = sale.posting.meta.get('commission', None)
if commission_meta is None:
commission = ZERO
else:
if calculate_commission:
commission = commission_meta
else:
# Fetch the commission that was inserted by the commissions plugin.
commission = commission_meta[0].units.number
commission = commission.quantize(Q)
pnl = (totprice - totcost - commission).quantize(Q)
is_wash = sale.posting.meta.get('wash', False)
if totprice > totcost:
total_gain[units.currency] += pnl
else:
total_loss[units.currency] += pnl
if is_wash:
total_adj[units.currency] += pnl
code = 'W'
adj = -pnl
else:
code = ''
adj = ''
days_held = (sale.txn.date - sale.posting.cost.date).days
term = 'LONG' if days_held >= 365 else 'SHORT'
lot = LotSale(sale_no,
ref,
sale.posting.cost.date,
sale.txn.date,
days_held,
term,
units.currency,
-units.number.quantize(Q),
sale.posting.cost.number.quantize(Q),
sale.posting.price.number.quantize(Q),
totcost,
totprice,
commission,
totprice - commission,
pnl,
code,
adj)
lots.append(lot)
tab_detail = table.create_table(lots, fieldspec)
# Aggregate by transaction in order to be able to cross-check against the
# 1099 forms.
agglots = [aggregate_sales(lots)
for _, lots in misc_utils.groupby(
lambda lot: (lot.no, lot.ref), lots).items()]
tab_agg = table.create_table(sorted(agglots, key=lambda lot: (lot.ref, lot.no)),
fieldspec)
# Write out a summary of P/L.
summary_fields = list(enumerate(['Currency', 'Gain', 'Loss', 'Net', 'Adj/Wash']))
summary = []
gain = ZERO
loss = ZERO
adj = ZERO
for currency in sorted(total_adj.keys()):
gain += total_gain[currency]
loss += total_loss[currency]
adj += total_adj[currency]
summary.append((currency,
total_gain[currency],
total_loss[currency],
total_gain[currency] + total_loss[currency],
total_adj[currency]))
summary.append(('*', gain, loss, gain + loss, adj))
tab_summary = table.create_table(summary, summary_fields)
if args.report == 'detail':
# Render to the console.
print('Detail of all lots')
print('=' * 48)
table.render_table(tab_detail, sys.stdout, 'txt')
print()
if args.output:
with open(args.output, 'w') as file:
table.render_table(tab_detail, file, 'csv')
elif args.report == 'aggregate':
print('Aggregated by trade & Reference Number (to Match 1099/Form8459)')
print('=' * 48)
table.render_table(tab_agg, sys.stdout, 'txt')
print()
if args.output:
with open(args.output, 'w') as file:
table.render_table(tab_agg, file, 'csv')
elif args.report == 'summary':
print('Summary')
print('=' * 48)
table.render_table(tab_summary, sys.stdout, 'txt')
print()
if args.output:
with open(args.output, 'w') as file:
table.render_table(tab_summary, file, 'csv')
