def get_amount(self, row):
self.assert_is_row(row)
if self.invert_amounts:
return data.Amount(-D(row.amount), self.currency)
else:
return data.Amount(D(row.amount), self.currency)
def entry_from_gl(self, entries: typing.List[Entry]) -> typing.Iterable:
"""Given a single GL_ID and as a list of Entrys
"""
first = entries[0]
postings = []
all_tags = set()
all_meta = {'lineno': 0, 'filename': "", 'gl-id': first.gl_id}
all_links = set()
for entry in entries:
self.new_accounts.add(entry.account)
if first.entry_type == "Balance":
if entry.amount and entry.date.year == 2016:
yield data.Pad(date=entry.date -
datetime.timedelta(days=1),
account=entry.account,
source_account="Equity:OpeningBalances",
meta={
'lineno': 0,
'filename': '',
'note': entry.narration,
'gl-id': entry.gl_id
})
yield data.Balance(date=entry.date,
amount=data.Amount(entry.amount,
entry.currency),
account=entry.account,
tolerance=None,
diff_amount=None,
meta={
'lineno': 0,
'filename': '',
'note': entry.narration,
'gl-id': entry.gl_id
})
else:
posting = data.Posting(entry.account,
data.Amount(entry.amount,
entry.currency),
None,
None,
flag='*',
meta=entry.meta)
all_tags.update(entry.tags)
all_links.update(entry.links)
postings.append(posting)
all_meta.update(posting.meta or {})
if postings:
yield data.Transaction(meta=all_meta,
date=entry.date,
flag='*',
payee=entry.payee,
narration=entry.narration,
tags=all_tags,
links=all_links,
postings=postings)
def get_amount(self, row):
self.assert_is_row(row)
commodity = self.investments.get(row.investment)
if self.invert_amounts:
return data.Amount(-D(row.amount), commodity)
else:
return data.Amount(D(row.amount), commodity)
def convert_transaction(txn, acct_map):
meta = {}
date = txn.GetDate().strftime('%Y-%m-%d')
flag = '*'
payee = ''
narration = txn.GetDescription()
base_currency = txn.GetCurrency().get_mnemonic()
postings = []
for split in txn.GetSplitList():
split_meta = {}
memo = split.GetMemo()
if memo:
split_meta['memo'] = memo
gnc_acct = split.GetAccount()
is_currency = gnc_acct.GetCommodity().get_namespace() == 'CURRENCY'
gnc_amount = split.GetAmount()
acct = acct_map[gnc_acct.GetGUID().to_string()]
amount = Converter.normalize_numeric(gnc_amount)
units = data.Amount(amount, acct.currencies[0])
split_flag = None
if acct.currencies[0] != base_currency:
total_cost = data.Amount(
abs(Converter.normalize_numeric(split.GetValue())),
base_currency)
num_units = abs(gnc_amount.to_double())
if num_units == 0.:
cost = price = None
else:
price = data.Amount(
decimal.Decimal(
float(split.GetValue().to_double()) /
float(gnc_amount.to_double())), base_currency)
if amount > 0 and not is_currency:
cost = data.Cost(price, base_currency,
txn.GetDate().date(), None)
else:
cost = None
postings.append(
CostBasedPosting(acct.account, units, cost, price,
total_cost, split_flag, split_meta))
else:
cost = price = None
postings.append(
data.Posting(acct.account, units, cost, price, split_flag,
split_meta))
return data.Transaction(meta, date, flag, payee, narration, None, None,
postings)
def read_price_stream(stream: typing.Iterable, price_db: typing.Dict[str,
dict],
quote_currency: str) -> data.Entries:
"""Reads an iterable of tuples and compares against existing price_db
Returns:
list of beancount Entries
"""
entries = []
for row in stream:
currency, _date, _amount = row[0:3]
if currency not in price_db:
continue
dp: datetime.datetime = dateparser.parse(_date)
assert dp, f"Unable to parse date {_date}"
date = datetime.date(dp.year, dp.month, dp.day)
amount = data.Amount(decimal.Decimal(_amount), quote_currency)
history = price_db[currency]
if date in history:
continue
entry = data.Price(date=date,
currency=currency,
amount=amount,
meta=data.new_metadata('', 0))
entries.append(entry)
return entries
def Price(price):
meta = {}
date = price.date
currency = price.commodity.mnemonic
amount = data.Amount(price.value, price.currency.mnemonic)
return data.Price(meta, date, currency, amount)
def Price(price):
meta = {}
date = price.date
currency = commodity_name(price.currency)
amount = data.Amount(price.value, currency)
commodity = commodity_name(price.commodity)
return data.Price(meta, date, commodity, amount)
def add_account(self, name, date=None, currency=None, initial_amount=None):
date = date or config.DEFAULT_DATE
name = self._format_account_name(name)
self.accounts.append(name)
currencies = currency and [currency]
self.entries.append(
bc.Open(self._get_meta(), date, name, currencies, booking=None))
if initial_amount is not None and initial_amount != 0:
assert currency is not None
self.pad_balances.append(
bc.Pad(self._get_meta(),
date=config.DEFAULT_PAD_DATE,
account=name,
source_account=config.OPENING_BALANCE_ACCOUNT))
self.pad_balances.append(
bc.Balance(self._get_meta(),
date=config.DEFAULT_BALANCE_DATE,
account=name,
amount=bc.Amount(initial_amount, currency),
tolerance=None,
diff_amount=None))
return len(self.accounts) - 1 # index of the account on self.accounts
def units_for(split):
number = data.Decimal(split.quantity).quantize(
data.Decimal(1.0) / data.Decimal(split.account.commodity.fraction))
currency = commodity_name(split.account.commodity)
return data.Amount(number, currency)
def units_for(split):
# I was having balance precision problems due to how beancount deal
# with integer precision. So multiply quantity by 1.0 to force at
# least 1 decimal place.
number = split.quantity * data.Decimal('1.0')
currency = split.account.commodity.mnemonic
return data.Amount(number, currency)
def convert_price(name, currency, gnc_price):
meta = {}
v = gnc_price.get_value()
gv = gnucash.gnucash_business.GncNumeric(instance=v)
price = Converter.normalize_numeric(gv)
amount = data.Amount(price, currency)
date = gnc_price.get_time64().strftime('%Y-%m-%d')
return data.Price(meta, date, Converter.normalize_commodity(name),
amount)
def extract(self,
file: cache._FileMemo,
existing_entries=None) -> data.Entries:
"""Given a File, let's extract the records"""
name = file.name
possible, context = self._read_file(name)
root_account = self.get_root()
if not root_account:
logger.error(f"Unable to find an account {context}")
return []
new_entries = []
for entry in possible:
target_account = self.accounts['default-transfer']
new_entries.append(
data.Transaction(
date=entry.date,
narration=entry.narration,
payee=entry.payee,
meta=entry.meta,
tags=set(),
links=set(),
flag="!",
postings=[
data.Posting(account=root_account,
units=data.Amount(entry.amount,
entry.currency),
flag="*",
cost=None,
price=None,
meta=None),
data.Posting(account=target_account,
units=data.Amount(-entry.amount,
entry.currency),
flag="!",
cost=None,
price=None,
meta=None)
]))
return new_entries
def price_for(split):
acc_comm = split.account.commodity
txn_comm = split.transaction.currency
if acc_comm == txn_comm:
return None
number = abs(split.value / split.quantity)
currency = txn_comm.mnemonic
return data.Amount(number, currency)
def extract(self, file, existing_entries):
csvfile = open(file=file.name, encoding='windows_1252')
reader = csv.reader(csvfile, delimiter=';')
meta = data.new_metadata(file.name, 0)
entries = []
for row in reader:
try:
book_date, text, credit, debit, val_date, balance = tuple(row)
book_date = datetime.strptime(book_date, '%Y-%m-%d').date()
if credit:
amount = data.Amount(Decimal(credit), self.currency)
elif debit:
amount = data.Amount(Decimal(debit), self.currency)
else:
amount = None
if balance:
balance = data.Amount(Decimal(balance), self.currency)
else:
balance = None
except Exception as e:
logging.debug(e)
else:
logging.debug((book_date, text, amount, val_date, balance))
posting = data.Posting(self.account, amount, None, None, None,
None)
entry = data.Transaction(meta, book_date, '*', '', text,
data.EMPTY_SET, data.EMPTY_SET,
[posting])
entries.append(entry)
# only add balance on SOM
book_date = book_date + timedelta(days=1)
if balance and book_date.day == 1:
entry = data.Balance(meta, book_date, self.account,
balance, None, None)
entries.append(entry)
csvfile.close()
entries = data.sorted(entries)
return entries
def extract_prices(self,
statement: FlexStatement,
existing_entries: list = None):
"""
IBFlex XML Files can contain an object called 'OpenPositions',
this is very useful because it lets us create
- Balance assertions
- Price entries from the Mark
"""
result = []
for position in statement.OpenPositions:
price = position.markPrice
safe_symbol = self.clean_symbol(position.symbol)
# Dates are 12 Midnight, let's make it the next day
date = statement.toDate + timedelta(days=1)
result.append(
# TODO De-Duplicate
data.Price(currency=safe_symbol,
amount=data.Amount(price, "USD"),
date=date,
meta={
'lineno': 0,
'filename': '',
}))
account = self.account_for_symbol(statement, position.symbol)
result.append(
data.Balance(
account=account,
amount=data.Amount(position.position * position.multiplier,
safe_symbol),
date=statement.toDate + timedelta(days=1),
meta={
'lineno': 0,
'filename': ''
},
tolerance=0.5,
diff_amount=0,
))
return result
def price_for(split):
acc_comm = split.account.commodity
txn_comm = split.transaction.currency
if acc_comm == txn_comm:
return None
try:
number = abs(split.value / split.quantity) * data.Decimal('1.000000')
except InvalidOperation: # Skip "bad" transcations (split.value and quantity are 0)
return None
currency = commodity_name(txn_comm)
return data.Amount(number, currency)
def price_for(split):
acc_comm = split.account.commodity
txn_comm = split.transaction.currency
if acc_comm == txn_comm:
return None
try:
number = abs(split.value / split.quantity)
except (decimal.DivisionByZero,
decimal.InvalidOperation): # invalid operation occurs when 0/0
if split.quantity.is_zero():
number = decimal.Decimal('0')
currency = txn_comm.mnemonic
return data.Amount(number, currency)
def get_member(self, id, name, date, balance):
#name = member_account(name)
name = self.accounts_by_id[id]
if id in self.accounts_by_id and self.accounts_by_id[id] != name:
# Generate rename
txn = bcdata.Transaction(meta={},
date=date,
flag="txn",
payee="",
narration="Rename %(from)s to %(to)s" % {
"from": self.accounts_by_id[id],
"to": name,
},
tags=set("rename"),
links=set(),
postings=[])
bcdata.create_simple_posting(txn, self.accounts_by_id[id], balance,
"EUR")
bcdata.create_simple_posting(txn, name, -balance, "EUR")
self.entries.append(txn)
if self.accounts_by_id[id] not in DONT_CLOSE:
self.entries.append(
bcdata.Close(
meta={},
date=date + datetime.timedelta(days=1),
account=self.accounts_by_id[id],
))
# Disable the balance assertion for the next day
self.last_assertion[name] = date
self.initial_balances[name] = None
self.accounts_by_id[id] = name
elif name not in self.initial_balances:
self.initial_balances[name] = balance
self.last_assertion[name] = date
self.accounts_by_id[id] = name
elif self.last_assertion.get(
name, None) != date and name != "Assets:Cash:Bar":
self.entries.append(
bcdata.Balance({"iline": str(self.line)}, date, name,
bcdata.Amount(-balance, "EUR"), None, None))
self.last_assertion[name] = date
return name
def add_transaction(self, date, payee, narration, main_account,
other_account, amount, currency, tags):
posting_common_args = dict(cost=None, price=None, flag=None, meta=None)
if config.PREFER_POSITIVE_AMOUNTS and amount < 0:
main_account, other_account = other_account, main_account
amount = -amount
postings = [
bc.Posting(main_account, bc.Amount(amount, currency),
**posting_common_args),
bc.Posting(other_account, None, **posting_common_args),
]
self.transactions.append(
bc.Transaction(self._get_meta(),
date,
flag=config.DEFAULT_FLAG,
payee=payee,
narration=narration,
tags=tags,
links=None,
postings=postings))
def get_transaction(
self,
row: Row,
file_name,
row_number,
existing_entries=None):
flag = flags.FLAG_WARNING
row = row._replace(**{
'visit_date': parse_date_liberally(row.visit_date),
'amount_billed': data.Amount(
D(row.amount_billed.replace("$", "")), self.currency),
'amount_deductible': data.Amount(
D(row.amount_deductible.replace("$", "")), self.currency),
'amount_plan_paid': data.Amount(
D(row.amount_plan_paid.replace("$", "")), self.currency),
'amount_plan_discount': data.Amount(
D(row.amount_plan_discount.replace("$", "")), self.currency),
'amount_responsible': data.Amount(
D(row.amount_responsible.replace("$", "")), self.currency),
'amount_paid_at_visit': data.Amount(
D(row.amount_paid_at_visit.replace("$", "")), self.currency),
'amount_owed': data.Amount(
D(row.amount_owed.replace("$", "")), self.currency),
})
metadata = {
'claim-number': row.claim_number,
'claim-type': row.claim_type,
'patient': row.patient,
'provider': row.provider,
'visit-date': row.visit_date,
}
postings = []
if row.amount_plan_paid:
postings.append(data.Posting(
account=self.reimbursement_account,
units=row.amount_plan_paid,
cost=None,
price=None,
flag=None,
meta={}))
if row.amount_plan_discount:
postings.append(data.Posting(
account=self.discount_account,
units=row.amount_plan_discount,
cost=None,
price=None,
flag=None,
meta={}))
if not postings:
return None
return data.Transaction( # pylint: disable=not-callable
data.new_metadata(file_name,
row_number,
metadata),
parse_date_liberally(row.date),
flag,
"United Healthcare",
self.narration_format.format(**row._asdict()),
self.tags,
set([self.link_format.format(**row._asdict())]),
postings)
def get_amount(self, row: Row):
total = row.item_total.strip(''.join(self.currency_symbols))
return data.Amount(number.D(total), row.currency)
def extract_trades(self,
statement: FlexStatement,
existing_entries: list = None):
"""
Version one does not attempt to group by order ID. This allows for perfect lot pricing and simpler
implementation. The downside is it's very verbose.
txn = data.Transaction(meta, date, self.FLAG, payee, narration,
tags, data.EMPTY_SET, [])
* https://www.interactivebrokers.com/en/software/reportguide/reportguide/tradesfq.htm
TODO: Handle Duplicates
"""
fees_account = self.accounts['fees']
match_key = 'id'
existing_by_key = self.find_existing(existing_entries, match_key)
by_order: typing.Dict[str, CombinedTrades] = {}
for trade in statement.Trades:
key = CombinedTrades.trade_key(trade)
assert key.strip(), f"Invalid Key {len(key)}"
if not trade.openCloseIndicator:
continue
if key in by_order:
combined = by_order[key]
combined.add_trade(trade.quantity, trade.tradePrice,
trade.ibCommission)
else:
combined = CombinedTrades(
ibOrderID=trade.ibOrderID,
tradeDate=trade.tradeDate,
tradePrice=trade.tradePrice,
exchange=trade.exchange,
multiplier=trade.multiplier,
ibCommission=trade.ibCommission,
symbol=trade.symbol,
currency=trade.currency,
safe_symbol=self.clean_symbol(trade.symbol),
buySell=trade.buySell,
quantity=trade.quantity,
securityID=trade.securityID,
ibCommissionCurrency=trade.ibCommissionCurrency,
openCloseIndicator=trade.openCloseIndicator.name)
by_order[key] = combined
result = []
for trade in by_order.values():
key = CombinedTrades.trade_key(trade)
if key in existing_by_key:
continue
meta = {
'lineno': 0,
'filename': '',
match_key: key,
'order_id': trade.ibOrderID,
'exchange': trade.exchange,
# 'multiplier': str(trade.multiplier),
# 'commission': trade.ibCommission,
}
# TODO Make this a parameter
payee = "IB"
if trade.securityID is None and "." in trade.symbol:
# FOREX Trade, not really a valid Symbol at all
# TODO: Better check than blank securityID
# Usually [currency].[commodity]. For example GBP.JPY
# In that case trade.currency is JPY, so we just need to parse out the GBP part
safe_symbol, _ = trade.symbol.split('.')
else:
safe_symbol = self.clean_symbol(trade.symbol)
narration = (
f"{trade.buySell.name} {trade.quantity} {safe_symbol} @ "
f"{trade.tradePrice} {trade.currency} on {trade.exchange}")
tags = data.EMPTY_SET
# cost = data.Amount(trade.cost, trade.currency)
cost_account = self.account_for_symbol(statement, trade.currency)
fees_cost_account = self.account_for_symbol(
statement, trade.ibCommissionCurrency)
comm_account = self.account_for_symbol(statement, safe_symbol)
# This is how much USD it cost us
# cost_amount = data.Cost(number=trade.netCash, currency=trade.currency, date=trade.tradeDate, label=f"xxx")
cash_amount = amount.Amount(
-trade.quantity * trade.multiplier * trade.tradePrice,
trade.currency)
unit_amount = amount.Amount(trade.quantity * trade.multiplier,
safe_symbol)
# post_price = data.Price(currency=trade.currency, amount=trade.tradePrice, meta={}, date=trade.tradeDate)
post_price = data.Amount(trade.tradePrice, trade.currency)
txn = data.Transaction(
meta,
trade.tradeDate,
self.FLAG,
"", # payee,
narration,
tags,
data.EMPTY_SET, # {link} -- what is this?
[
data.Posting(
account=comm_account,
units=unit_amount,
cost=
EMPTY_COST_SPEC, #cost_amount, # cost=(Cost, CostSpec, None),
price=post_price,
flag=self.FLAG,
meta={}),
# How does this affect our Cash?
data.Posting(
account=cost_account,
units=cash_amount,
cost=None, # cost=(Cost, CostSpec, None),
price=None,
flag=self.FLAG,
meta={}),
# Total Fees
data.Posting(account=fees_cost_account,
units=amount.Amount(
trade.ibCommission,
trade.ibCommissionCurrency),
cost=None,
price=None,
flag=self.FLAG,
meta={}),
data.Posting(account=fees_account,
units=amount.Amount(
-trade.ibCommission,
trade.ibCommissionCurrency),
cost=None,
price=None,
flag=self.FLAG,
meta={})
])
if trade.openCloseIndicator == "CLOSE":
# This is a reduction in position, add the Income Account:
data.create_simple_posting(entry=txn,
account=self.accounts.get(
'income', 'Income:FIXME'),
number=None,
currency=None)
result.append(txn)
return result
def get_price(self, row):
self.assert_is_row(row)
price_per_share = row.price_per_share.strip(
''.join(self.currency_symbols) + "()")
return data.Amount(D(price_per_share), self.currency)
def load_remote_account(connection: gspread.Client, errors: list, account: str,
options: typing.Dict[str, str]):
"""Try to Load Entries from URL into Account.
options include:
- document_name -- the Actual Google Doc name
- document_tab -- the Tab name on the Doc
- default_currency - the entry currency if None is provided
- reverse_amount - if true, assume positive entries are credits
"""
entries = []
document_name = options['document_name']
document_tab = options.get('document_tab', 0) or 0
default_currency = options['default_currency']
reverse_amount = options.get('reverse_amount', False)
if not document_name:
return
m = -1 if reverse_amount else 1
logger.info(
f"Attempting to download entries for {account} from {document_name}.{document_tab}"
)
workbook = connection.open(document_name)
sheet = None
try:
document_tab = int(document_tab)
sheet = workbook.get_worksheet(document_tab)
except ValueError:
pass
if sheet is None:
sheet = workbook.worksheet(document_tab)
records = sheet.get_all_records()
import re
row = 0
# logger.info(f"Found {len(records)} entries.")
for record in records:
row += 1
record = clean_record(record)
if 'date' not in record or not record['date']:
continue
if 'amount' not in record or not record['amount']:
continue
#if 'account' not in record or not record['account'].strip():
# continue
narration = record.pop('narration', None)
payee = record.pop('payee', None)
tagstr = record.pop('tags', '')
tags = set(re.split(r'\W+', tagstr)) if tagstr else set()
date = dateparser.parse(record.pop('date'))
if date:
date = datetime.date(year=date.year,
month=date.month,
day=date.day)
linkstr = record.pop('links', '')
links = set(re.split(r'\W+', linkstr)) if linkstr else set()
meta = {
'filename': str(options['entry_file']),
'lineno': 0,
'document-sheet-row': f"{document_name}/{document_tab}/{row+1}"
}
amount = decimal.Decimal(record.pop('amount')) * m
currency = record.pop('currency', default_currency)
entry_account = record.pop('account')
for k, v in record.items():
if v:
meta[k] = v
try:
if not entry_account:
errors.append(
f"Skipping Record with Blank Account: {meta['document-sheet-row']}"
)
logger.warning(
f"Skipping Record with Blank Account: {meta['document-sheet-row']}"
)
continue
entry = data.Transaction(
date=date,
narration=narration,
payee=payee,
tags=tags,
meta=meta,
links=links,
flag='*',
postings=[
data.Posting(account=account,
units=data.Amount(amount, currency),
cost=None,
price=None,
flag='*',
meta={}),
data.Posting(account=entry_account,
units=data.Amount(-amount, currency),
cost=None,
price=None,
flag='*',
meta={})
])
entries.append(entry)
except Exception as exc:
logger.error(f"Error while parsing {record}", exc_info=exc)
errors.append(str(exc))
logger.info(
f"Loaded {len(entries)} entries for {account} from {document_name}.{document_tab}"
)
return entries
def parse_amount(d: dict) -> bean.Amount:
if d is None:
return d
return bean.Amount(number=d["number"], currency=d["currency"])
def main():
position_account = "Liabilities:Crypto:Bitfinex:Positions"
margin_account = "Liabilities:Crypto:Bitfinex:Positions"
income_account = "Income:Crypto:Bitfinex:Realized"
fee_account = "Expenses:Trading:Bitfinex:Fees:Trading"
positions: typing.Dict[str, Decimal] = defaultdict(Decimal)
balances: typing.Dict[str, datetime.date] = defaultdict(
lambda: datetime.date(2000, 1, 1))
records = list(records_from_string(csvdata))
records.sort(key=lambda r: (r.date, r.commodity, -r.quantity))
entries = []
for record in records:
# Now, we need to Handle the Incoming Account, Only if our Absolute
# Position is decreasing:
current_position = positions[record.commodity]
new_position = current_position + record.quantity
positions[record.commodity] = new_position
# This is the account who's position we need to track for our lots
commodity_account = account.join(position_account, record.commodity)
if balances[commodity_account] != record.date:
entries.append(
data.Balance(date=record.date,
account=commodity_account,
amount=Amount(current_position, record.commodity),
tolerance=Q,
diff_amount=None,
meta=data.new_metadata("", 0)))
balances[commodity_account] = record.date
entry = data.Transaction(date=record.date,
narration=str(record),
payee="",
tags={"margin"},
links=set(),
flag='*',
meta=data.new_metadata(
"",
0,
kvlist=dict(position=str(new_position),
order_id=record.order_id)),
postings=[])
# The Commodity Adjustment
# Assets:Crypto:Bitfinex:Positions:BTC 200 BTC {} @ 6950 USD
posting = data.Posting(account=commodity_account,
units=data.Amount(record.quantity,
record.commodity),
cost=empty_cost_spec,
price=Amount(record.price, record.currency),
flag=None,
meta=data.new_metadata("", 0))
entry.postings.append(posting)
# The Currency Account Adjustment (based on Cost?)
# ; Liabilities:Crypto:Bitfinex:Borrowed:USD -1,390,102.19 USD
data.create_simple_posting(entry=entry,
account=account.join(
margin_account, record.currency),
number=-record.amount,
currency=record.currency)
if record.fee:
# We add Two fee Records
# ; Liabilities:Crypto:Bitfinex:Borrowed:USD -839.747894 USD
data.create_simple_posting(entry=entry,
account=account.join(
margin_account, record.currency),
number=record.fee,
currency=record.currency)
data.create_simple_posting(entry=entry,
account=fee_account,
number=-record.fee,
currency=record.currency)
if abs(new_position) < abs(current_position):
# Add an Income Account Entry
data.create_simple_posting(entry=entry,
account=income_account,
number=None,
currency=None)
entries.append(entry)
printer.print_entries(entries)
def parse_amount(s):
s = s.replace(',', '.')
return data.Amount(D(s), 'EUR')
def extract(self, filepath, existing):
"""Implements :py:meth:`beangulp.Importer.extract()`.
This methods costructs a transaction for each data row using
the date, narration, and amount required fields and the flag,
payee, account, currency, tag, link, balance optional fields.
Transaction metadata is constructed with :py:meth:`metadata()`
and :py:meth:`finalize()` is called on each transaction. These
can be redefine in subclasses. For customization that cannot
be implemented with these two extension points, consider
basing the importer on :py:class:`CSVReader` and implementing
the :py:class:`~beangulp.Importer` interface with tailored
processing of the data rows.
"""
entries = []
balances = defaultdict(list)
default_account = self.account(filepath)
# Compute the line number of the first data line.
offset = int(self.skiplines) + bool(self.names) + 1
for lineno, row in enumerate(self.read(filepath), offset):
# Skip empty lines.
if not row:
continue
tag = getattr(row, 'tag', None)
tags = {tag} if tag else EMPTY
link = getattr(row, 'link', None)
links = {link} if link else EMPTY
# This looks like an exercise in defensive programming
# gone too far, but we do not want to depend on any field
# being defined other than the essential ones.
flag = getattr(row, 'flag', self.flag)
payee = getattr(row, 'payee', None)
account = getattr(row, 'account', default_account)
currency = getattr(row, 'currency', self.currency)
units = data.Amount(row.amount, currency)
# Create a transaction.
txn = data.Transaction(self.metadata(filepath, lineno, row),
row.date, flag, payee, row.narration, tags, links, [
data.Posting(account, units, None, None, None, None),
])
# Apply user processing to the transaction.
txn = self.finalize(txn, row)
# Add the transaction to the output list.
entries.append(txn)
# Add balance to balances list.
balance = getattr(row, 'balance', None)
if balance is not None:
date = row.date + datetime.timedelta(days=1)
units = data.Amount(balance, currency)
meta = data.new_metadata(filepath, lineno)
balances[currency].append(data.Balance(meta, date, account, units, None, None))
if not entries:
return []
# Reverse the list if the file is in descending order.
if not entries[0].date <= entries[-1].date:
entries.reverse()
# Append balances.
for currency, balances in balances.items():
entries.append(max(balances, key=lambda x: x.date))
return entries
def extract(self,
file: cache._FileMemo,
existing_entries=None) -> data.Entries:
"""
"""
entries = []
errors = []
content = self._read_file(file.name)
records = content.pop('records')
currencies = content['currencies']
account = content['account']
if currencies:
default_currency = currencies[0]
else:
default_currency = DEFAULT_CURRENCY
row = 0
for record in records:
row += 1
record = clean_record(record)
if 'date' not in record or not record['date']:
continue
if 'amount' not in record or not record['amount']:
continue
narration = record.pop('narration', '')
payee = record.pop('payee', '')
tagstr = record.pop('tags', '')
tags = set(re.split(r'\W+', tagstr)) if tagstr else set()
# Date handling through dateparser
date = dateparser.parse(record.pop('date'))
if date:
date = datetime.date(year=date.year,
month=date.month,
day=date.day)
# Links
linkstr = record.pop('links', '')
links = set(re.split(r'\W+', linkstr)) if linkstr else set()
meta = {
'filename':
'',
'lineno':
0,
'document-sheet-row':
f"{content['document']}/{content['tab']}/{row+1}"
}
# Need more protections
amount = decimal.Decimal(record.pop('amount'))
currency = record.pop('currency', default_currency)
entry_account = record.pop('account')
meta_target = {}
meta_source = {}
for k, v in record.items():
if not v or not k:
continue
clean_key = k.lower().replace('-', '').replace('_', '')
if clean_key in ('transferaccount', 'targetaccount'):
meta_target['account'] = record[k]
else:
meta_source[k] = v
try:
if not entry_account:
errors.append(
f"Skipping Record with Blank Account: {meta['document-sheet-row']}"
)
logger.warning(
f"Skipping Record with Blank Account: {meta['document-sheet-row']}"
)
continue
entry = data.Transaction(
date=date,
narration=narration,
payee=payee,
tags=tags,
meta=meta,
links=links,
flag='*',
postings=[
data.Posting(account=account,
units=data.Amount(amount, currency),
cost=None,
price=None,
flag=None,
meta=meta_source),
data.Posting(account=entry_account,
units=data.Amount(-amount, currency),
cost=None,
price=None,
flag=None,
meta=meta_target)
])
entries.append(entry)
except Exception as exc:
logger.error(f"Error while parsing {record}", exc_info=exc)
errors.append(str(exc))
return entries
def _extract_stock_operations(self, filename, rd):
rd = csv.reader(rd, dialect="fortuneo")
entries = []
header = True
line_index = 0
for row in rd:
# Check header
if header:
if set(row) != set(STOCK_FIELDS):
raise InvalidFormatError()
header = False
line_index += 1
continue
if len(row) != len(STOCK_FIELDS):
continue
# Extract data
row_date = datetime.strptime(row[3], "%d/%m/%Y")
label = row[0].strip() + " - " + row[1]
currency = row[9]
stock_amount = data.Amount(D(row[4]), 'STK')
stock_cost = position.Cost(
number=D(row[5]),
currency=currency,
date=row_date.date(),
label=None,
)
# Prepare the transaction
meta = data.new_metadata(filename, line_index)
txn = data.Transaction(
meta=meta,
date=row_date.date(),
flag=flags.FLAG_OKAY,
payee="",
narration=label,
tags=set(),
links=set(),
postings=[],
)
# Create the postings.
txn.postings.append(
data.Posting(account="Assets:Stock:STK",
units=stock_amount,
cost=stock_cost,
price=None,
flag=None,
meta=None))
txn.postings.append(
make_posting(self.broker_fees_account, -parse_amount(row[7])))
txn.postings.append(
make_posting(self.stock_account, parse_amount(row[8])))
# Done
entries.append(txn)
line_index += 1
return entries
