def test_compute_postings_balance(self, entries, _, __):
"""
2014-01-01 open Assets:Bank:Checking
2014-01-01 open Assets:Bank:Savings
2014-01-01 open Assets:Investing
2014-01-01 open Assets:Other
2014-05-26 note Assets:Investing "Buying some shares"
2014-05-30 *
Assets:Bank:Checking 111.23 USD
Assets:Bank:Savings 222.74 USD
Assets:Bank:Savings 17.23 CAD
Assets:Investing 10000 EUR
Assets:Investing 32 HOOL {45.203 USD}
Assets:Other 1000 EUR @ 1.78 GBP
Assets:Other 1000 EUR @@ 1780 GBP
"""
postings = entries[:-1] + entries[-1].postings
computed_balance = realization.compute_postings_balance(postings)
expected_balance = inventory.Inventory()
expected_balance.add_amount(A('333.97 USD'))
expected_balance.add_amount(A('17.23 CAD'))
expected_balance.add_amount(
A('32 HOOL'),
position.Cost(D('45.203'), 'USD', datetime.date(2014, 5, 30),
None))
expected_balance.add_amount(A('12000 EUR'))
self.assertEqual(expected_balance, computed_balance)
def convert_spec_to_cost(units, cost_spec):
"""Convert a posting's CostSpec instance to a Cost.
Args:
units: An instance of Amount.
cost_spec: An instance of CostSpec.
Returns:
An instance of Cost.
"""
cost = cost_spec
errors = []
if isinstance(units, amount.Amount):
currency = units.currency
if cost_spec is not None:
number_per, number_total, cost_currency, date, label, merge = cost_spec
# Compute the cost.
if number_per is not MISSING or number_total is not None:
if number_total is not None:
# Compute the per-unit cost if there is some total cost
# component involved.
units_num = units.number
cost_total = number_total
if number_per is not MISSING:
cost_total += number_per * units_num
unit_cost = cost_total / abs(units_num)
else:
unit_cost = number_per
cost = position.Cost(unit_cost, cost_currency, date, label)
else:
cost = None
return cost
def balayageJSONtable(self, jsondata, afficherCost: bool = False):
"""Une procédure qui balaye toutes les lignes du JSON"""
self.postings = []
self.total = 0
for ligne in jsondata["table"]:
# Si debogage, affichage de l'extraction
if self.debug:
print(ligne)
print(parse_datetime(ligne["date"]).date)
if ligne["valeurpart"] == "":
ligne["valeurpart"] = "1.00"
ligne["nbpart"] = ligne["montant"]
if afficherCost and re.match("-", ligne["nbpart"]) is None:
cost = position.Cost(
Decimal(
float(ligne["montant"].replace(",", ".").replace(
" ", "").replace("\xa0", "").replace(r"\u00a", ""))
/ float(ligne["nbpart"].replace(",", ".").replace(
" ", "").replace("\xa0", "").replace(
r"\u00a", ""))).quantize(Decimal(".0001")),
"EUR",
None,
None,
)
else:
cost = None
self.postings.append(
data.Posting(
account=self.accountList[jsondata["compte"]] + ":" +
ligne["isin"].replace(" ", "").upper(),
units=amount.Amount(
Decimal(ligne["nbpart"].replace(",", ".").replace(
" ", "").replace("\xa0", "").replace(r"\u00a",
"")),
ligne["isin"].replace(" ", "").upper(),
),
cost=cost,
flag=None,
meta=None,
price=amount.Amount(
Decimal(
abs(
float(ligne["montant"].replace(
",", ".").replace(" ", "").replace(
"\xa0", "").replace(r"\u00a", "")) /
float(
ligne["nbpart"].replace(",", ".").replace(
" ", "").replace("\xa0", "").replace(
r"\u00a", "")))).quantize(
Decimal(".0001")),
"EUR",
),
))
self.total = self.total + Decimal(ligne["montant"].replace(
",", ".").replace(" ", "").replace("\xa0", "").replace(
r"\u00a", ""))
def holding_to_position(holding):
"""Convert the holding to a position.
Args:
holding: An instance of Holding.
Returns:
An instance of Position.
"""
return position.Position(
amount.Amount(holding.number, holding.currency),
(position.Cost(holding.cost_number, holding.cost_currency, None, None)
if holding.cost_number else None))
def test_cost_to_str__detail(self):
cost = position.Cost(D('101.23'), 'USD', datetime.date(2015, 9, 6),
"f4412439c31b")
self.assertEqual('101.23 USD, 2015-09-06, "f4412439c31b"',
position.cost_to_str(cost, self.dformat))
cost = position.Cost(D('101.23'), 'USD', datetime.date(2015, 9, 6),
None)
self.assertEqual('101.23 USD, 2015-09-06',
position.cost_to_str(cost, self.dformat))
cost = position.Cost(D('101.23'), 'USD', None, None)
self.assertEqual('101.23 USD',
position.cost_to_str(cost, self.dformat))
cost = position.Cost(D('101.23'), 'USD', None, "f4412439c31b")
self.assertEqual('101.23 USD, "f4412439c31b"',
position.cost_to_str(cost, self.dformat))
cost = position.Cost(None, None, None, "f4412439c31b")
self.assertEqual('"f4412439c31b"',
position.cost_to_str(cost, self.dformat))
cost = position.Cost(None, 'USD', None, "f4412439c31b")
self.assertEqual('"f4412439c31b"',
position.cost_to_str(cost, self.dformat))
def test_local_booking(self):
fileloc = data.new_metadata('<local>', 0)
date = datetime.date.today()
txn = data.Transaction(
fileloc, date, '*', None, "Narration", data.EMPTY_SET,
data.EMPTY_SET, [
data.Posting(
'Assets:Something', MISSING,
position.CostSpec(MISSING, None, MISSING, MISSING, MISSING,
MISSING), MISSING, None, None)
])
expected_txn = data.Transaction(
fileloc, date, '*', None, "Narration", data.EMPTY_SET,
data.EMPTY_SET, [
data.Posting('Assets:Something', None,
position.Cost(None, None, None, None), None, None,
None)
])
actual_txn = cmptest._local_booking(txn)
self.assertEqual(actual_txn, expected_txn)
txn = data.Transaction(
fileloc, date, '*', None, "Narration", data.EMPTY_SET,
data.EMPTY_SET, [
data.Posting(
'Assets:Something', amount.Amount(MISSING, MISSING),
position.CostSpec(MISSING, None, MISSING, MISSING, MISSING,
MISSING), amount.Amount(
MISSING, MISSING), None, None)
])
expected_txn = data.Transaction(
fileloc, date, '*', None, "Narration", data.EMPTY_SET,
data.EMPTY_SET, [
data.Posting('Assets:Something', amount.Amount(None, None),
position.Cost(None, None, None, None),
amount.Amount(None, None), None, None)
])
actual_txn = cmptest._local_booking(txn)
self.assertEqual(actual_txn, expected_txn)
def augment_inventory(pending_lots, posting, entry, eindex):
"""Add the lots from the given posting to the running inventory.
Args:
pending_lots: A list of pending ([number], Posting, Transaction) to be matched.
The number is modified in-place, destructively.
posting: The posting whose position is to be added.
entry: The parent transaction.
eindex: The index of the parent transaction housing this posting.
Returns:
A new posting with cost basis inserted to be added to a transformed transaction.
"""
number = posting.units.number
new_posting = posting._replace(units=copy.copy(posting.units),
cost=position.Cost(posting.price.number,
posting.price.currency,
entry.date, None))
pending_lots.append(([number], new_posting, eindex))
return new_posting
def test_compute_entries_balance_at_cost(self, entries, _, __):
"""
2014-01-01 open Assets:Investing
2014-01-01 open Assets:Other
2014-06-05 *
Assets:Investing 30 HOOL {40 USD}
Assets:Other
2014-06-05 *
Assets:Investing -20 HOOL {40 USD}
Assets:Other
"""
computed_balance = interpolate.compute_entries_balance(entries)
expected_balance = inventory.Inventory()
expected_balance.add_amount(A('-400 USD'))
expected_balance.add_amount(
A('10 HOOL'),
position.Cost(D('40'), 'USD', datetime.date(2014, 6, 5), None))
self.assertEqual(expected_balance, computed_balance)
def aggregate_postings(postings):
"""Aggregate postings by account and currency.
Args:
postings: A list of Posting instances.
Returns:
A list of aggregated postings.
"""
balances = collections.defaultdict(inventory.Inventory)
for posting in postings:
key = (posting.account, posting.units.currency)
balances[key].add_position(posting)
agg_postings = []
for (account, currency), balance in balances.items():
units = balance.reduce(convert.get_units)
if units.is_empty():
continue
assert len(units) == 1
units = units[0].units
cost = balance.reduce(convert.get_cost)
assert len(cost) == 1
total_cost = cost[0].units
if total_cost.currency != units.currency:
average_cost = position.Cost(total_cost.number/units.number,
total_cost.currency,
None, None)
else:
average_cost = None
posting = data.Posting(account, units, average_cost, None, None, None)
agg_postings.append(posting)
return agg_postings
def test_cost_to_str__simple(self):
cost = position.Cost(D('101.23'), 'USD', datetime.date(2015, 9, 6),
"f4412439c31b")
self.assertEqual('101.23 USD',
position.cost_to_str(cost, self.dformat, False))
cost = position.Cost(D('101.23'), 'USD', datetime.date(2015, 9, 6),
None)
self.assertEqual('101.23 USD',
position.cost_to_str(cost, self.dformat, False))
cost = position.Cost(D('101.23'), 'USD', None, None)
self.assertEqual('101.23 USD',
position.cost_to_str(cost, self.dformat, False))
cost = position.Cost(D('101.23'), 'USD', None, "f4412439c31b")
self.assertEqual('101.23 USD',
position.cost_to_str(cost, self.dformat, False))
cost = position.Cost(None, None, None, "f4412439c31b")
self.assertEqual('', position.cost_to_str(cost, self.dformat, False))
cost = position.Cost(None, 'USD', None, "f4412439c31b")
self.assertEqual('', position.cost_to_str(cost, self.dformat, False))
def _local_booking(entry):
"""Transform incompolete entries as booked.
This method converts incomplete entries with positions that sport CostSpec
instances and missing portions to using Cost and replacing MISSING instances
to None. No booking is carried out on account inventories. The purpose of
this degenerate booking is simply to transform parsed transactions for the
purpose of comparison in tests.
Args:
entry: An instance of a directive.
Returns:
A transformed list of directives.
"""
if not isinstance(entry, data.Transaction):
return entry
new_postings = []
for posting in entry.postings:
orig_posting = posting
# Fixup units.
if posting.units is MISSING:
posting = posting._replace(units=None)
elif posting.units:
posting = posting._replace(
units=_transform_incomplete_amount(posting.units))
# Fixup cost.
cost = posting.cost
if isinstance(cost, position.CostSpec):
if cost.number_per is MISSING:
cost = cost._replace(number_per=None)
if cost.number_total is MISSING:
cost = cost._replace(number_total=None)
if cost.currency is MISSING:
cost = cost._replace(currency=None)
if cost.date is MISSING:
cost = cost._replace(date=None)
if cost.label is MISSING:
cost = cost._replace(label=None)
if cost.number_total not in (None, MISSING):
if not isinstance(posting.units, amount.Amount):
raise ValueError(
"Cannot convert posting without units: {}".format(
orig_posting))
number = posting.units.number
total = ((cost.number_per or ZERO) * number +
(cost.number_total or ZERO))
cost_number = (total / number) or None
else:
cost_number = None if cost.number_per is MISSING else cost.number_per
posting = posting._replace(cost=position.Cost(
cost_number, cost.currency, cost.date, cost.label))
assert cost.date is not MISSING
assert cost.label is not MISSING
# Fixup price.
if posting.price is MISSING:
posting = posting._replace(price=None)
elif posting.price:
posting = posting._replace(
price=_transform_incomplete_amount(posting.price))
new_postings.append(posting)
return entry._replace(postings=new_postings)
def main():
import argparse, logging
logging.basicConfig(level=logging.INFO,
format='%(levelname)-8s: %(message)s')
parser = argparse.ArgumentParser(description=__doc__.strip())
parser.add_argument('output', action='store', help="Ouptut directory")
parser.add_argument('-s',
'--start',
action='store',
type=parse_time,
help="Start date")
parser.add_argument('-e',
'--end',
action='store',
type=parse_time,
help="Enddate")
parser.add_argument('--seed',
action='store',
type=int,
help="The unique seed")
args = parser.parse_args()
if args.end is None:
args.end = datetime.datetime.today()
if args.start is None:
args.start = args.end - datetime.timedelta(days=24 * 30)
if args.seed:
random.seed(args.seed)
start = args.start.date()
end = args.end.date()
rows = []
PENNY = D('0.01')
initial_balance = D(random.uniform(30000, 50000)).quantize(PENNY)
date = start
oneday = datetime.timedelta(days=1)
balance = initial_balance
inv = inventory.Inventory()
while date < end:
date += oneday
r = random.random()
s = 0
txn = None
for ttype, p in PROBAB:
if s < random.random() < s + p:
break
s += p
else:
continue
row = None
xid = random.randint(10000000, 100000000 - 1)
fees = ZERO
number = ZERO
code = ttype
if ttype == 'XFER':
desc = "CLIENT REQUESTED ELECTRONIC FUNDING"
number = D(random.uniform(3000, 15000)).quantize(PENNY)
elif ttype == 'BUY':
core = 'TRD'
stock = random.choice(STOCKS)
approx_number = D(random.uniform(2000, 10000))
price = D(random.uniform(5, 100)).quantize(PENNY)
shares = D(int(approx_number / price))
number = -shares * price
desc = "BOUGHT +{} {} @{}".format(stock, shares, price)
inv.add_amount(amount.Amount(shares, stock),
position.Cost(price, 'USD', date, None))
if shares * price > balance:
continue
fees = D('7.95')
number -= fees
elif ttype == 'SELL':
if inv.is_empty():
continue
core = 'TRD'
pos = random.choice(inv)
shares = (D(random.uniform(0.3, 0.7)) *
pos.units.number).quantize(ZERO)
price = (pos.cost.number *
D(random.normalvariate(1.0, 0.1))).quantize(PENNY)
number = price * shares
desc = "SOLD +{} {} @{} (LOT {})".format(pos.units.currency,
shares, price,
pos.cost.number)
fees = D('7.95')
number -= fees
elif ttype == 'DIV':
if inv.is_empty():
continue
pos = random.choice(inv)
desc = "ORDINARY DIVIDEND~{}".format(pos.units.currency)
number = (D(random.uniform(0.01 / 12, 0.04 / 12)) *
pos.units.number * pos.cost.number).quantize(PENNY)
else:
continue
balance += number
row = Row(date, ttype, xid, desc, fees, number, balance)
rows.append(row)
header = 'DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE'.split(',')
for date_end, report_rows in split_rows(rows, datetime.timedelta(days=90)):
filename = path.join(args.output,
"UTrade{:%Y%m%d}.csv".format(date_end))
with open(filename, 'w') as file:
wr = csv.writer(file)
wr.writerow(header)
wr.writerows(report_rows)
def extract(self, file):
# Open the CSV file and create directives.
entries = []
index = 0
for index, row in enumerate(csv.DictReader(open(file.name))):
meta = data.new_metadata(file.name, index)
date = parse(row['DATE']).date()
rtype = row['TYPE']
link = "ut{0[REF #]}".format(row)
desc = "({0[TYPE]}) {0[DESCRIPTION]}".format(row)
units = amount.Amount(D(row['AMOUNT']), self.currency)
fees = amount.Amount(D(row['FEES']), self.currency)
other = amount.add(units, fees)
if rtype == 'XFER':
assert fees.number == ZERO
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET, {link},
[
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(self.account_external, -other, None, None,
None, None),
])
elif rtype == 'DIV':
assert fees.number == ZERO
# Extract the instrument name from its description.
match = re.search(r'~([A-Z]+)$', row['DESCRIPTION'])
if not match:
logging.error("Missing instrument name in '%s'",
row['DESCRIPTION'])
continue
instrument = match.group(1)
account_dividends = self.account_dividends.format(instrument)
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET, {link},
[
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(account_dividends, -other, None, None,
None, None),
])
elif rtype in ('BUY', 'SELL'):
# Extract the instrument name, number of units, and price from
# the description. That's just what we're provided with (this is
# actually realistic of some data from some institutions, you
# have to figure out a way in your parser).
match = re.search(r'\+([A-Z]+)\b +([0-9.]+)\b +@([0-9.]+)',
row['DESCRIPTION'])
if not match:
logging.error("Missing purchase infos in '%s'",
row['DESCRIPTION'])
continue
instrument = match.group(1)
account_inst = account.join(self.account_root, instrument)
units_inst = amount.Amount(D(match.group(2)), instrument)
rate = D(match.group(3))
if rtype == 'BUY':
cost = position.Cost(rate, self.currency, None, None)
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET,
{link}, [
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(self.account_fees, fees, None, None,
None, None),
data.Posting(account_inst, units_inst, cost, None,
None, None),
])
elif rtype == 'SELL':
# Extract the lot. In practice this information not be there
# and you will have to identify the lots manually by editing
# the resulting output. You can leave the cost.number slot
# set to None if you like.
match = re.search(r'\(LOT ([0-9.]+)\)', row['DESCRIPTION'])
if not match:
logging.error("Missing cost basis in '%s'",
row['DESCRIPTION'])
continue
cost_number = D(match.group(1))
cost = position.Cost(cost_number, self.currency, None,
None)
price = amount.Amount(rate, self.currency)
account_gains = self.account_gains.format(instrument)
txn = data.Transaction(
meta, date, self.FLAG, None, desc, data.EMPTY_SET,
{link}, [
data.Posting(self.account_cash, units, None, None,
None, None),
data.Posting(self.account_fees, fees, None, None,
None, None),
data.Posting(account_inst, units_inst, cost, price,
None, None),
data.Posting(account_gains, None, None, None, None,
None),
])
else:
logging.error("Unknown row type: %s; skipping", rtype)
continue
entries.append(txn)
# Insert a final balance check.
if index:
entries.append(
data.Balance(meta, date + datetime.timedelta(days=1),
self.account_cash,
amount.Amount(D(row['BALANCE']),
self.currency), None, None))
return entries
def _extract_stock_operations(self, filename, rd):
rd = csv.reader(rd, dialect="fortuneo")
entries = []
header = True
line_index = 0
for row in rd:
# Check header
if header:
if set(row) != set(STOCK_FIELDS):
raise InvalidFormatError()
header = False
line_index += 1
continue
if len(row) != len(STOCK_FIELDS):
continue
# Extract data
row_date = datetime.strptime(row[3], "%d/%m/%Y")
label = row[0].strip() + " - " + row[1]
currency = row[9]
stock_amount = data.Amount(D(row[4]), 'STK')
stock_cost = position.Cost(
number=D(row[5]),
currency=currency,
date=row_date.date(),
label=None,
)
# Prepare the transaction
meta = data.new_metadata(filename, line_index)
txn = data.Transaction(
meta=meta,
date=row_date.date(),
flag=flags.FLAG_OKAY,
payee="",
narration=label,
tags=set(),
links=set(),
postings=[],
)
# Create the postings.
txn.postings.append(
data.Posting(account="Assets:Stock:STK",
units=stock_amount,
cost=stock_cost,
price=None,
flag=None,
meta=None))
txn.postings.append(
make_posting(self.broker_fees_account, -parse_amount(row[7])))
txn.postings.append(
make_posting(self.stock_account, parse_amount(row[8])))
# Done
entries.append(txn)
line_index += 1
return entries
def extract(self, f):
entries = []
with open(f.name, 'rb') as pdf_file:
# Read in the PDF.
read_pdf = PyPDF2.PdfFileReader(pdf_file)
all_text = "".join(
read_pdf.getPage(curr_page).extractText()
for curr_page in range(read_pdf.numPages))
data_re = re.compile(TRXN_RE)
fund_re = re.compile(FUND_RE)
fund_match = [m.groupdict() for m in fund_re.finditer(all_text)]
# Go through finding fund by fund to add transactions.
for index, row in enumerate(fund_match):
fund_text = row['fund'].strip()
text = row['data'].strip()
fund = FUND_D[fund_text + " Fund"]
matches = [m.groupdict() for m in data_re.finditer(text)]
# For each transaction found, build the associated entry
# for addition to beancount.
for index, row in enumerate(matches):
trans_date = parse(row['date']).date()
trans_desc = str.title(row['desc'].strip())
trans_amt = row['total'].replace("Œ", '-')
shares = row['shares'].replace("Œ", '-')
price = row['price'].replace("Œ", '-')
total = row['total'].replace("Œ", '-')
meta = data.new_metadata(f.name, index)
# Trans_amt in TSP statement is relative to dollars in fund
# Changes sign when relative to "Cash" account in TSP
if "-" in trans_amt:
trans_amt = trans_amt.replace("-", "")
else:
trans_amt = "-" + trans_amt
# Clean up the description to standardize naming.
# After fixed above, negative on amnt means purchase.
if ("contribution" in trans_desc.lower()
and "-" in trans_amt):
trans_desc = fund.title() + " purchase"
txn = data.Transaction(meta=meta,
date=trans_date,
flag=flags.FLAG_OKAY,
payee="Thrift Savings Plan",
narration=trans_desc,
tags=set(),
links=set(),
postings=[])
txn.postings.append(
data.Posting(
self.tsp_root + ":" + fund,
amount.Amount(D(shares), ("TSP" + fund).upper()),
position.Cost(D(price), 'USD', None, None), None,
None, None))
txn.postings.append(
data.Posting(self.cash_account,
amount.Amount(D(trans_amt), 'USD'), None,
None, None, None))
entries.append(txn)
return entries
