def test_get_entry_accounts(self):
entries = loader.load_string(TEST_INPUT)[0]
accounts = getters.get_entry_accounts(
next(entry for entry in entries
if isinstance(entry, data.Transaction)))
self.assertEqual(
{'Assets:US:Cash', 'Expenses:Grocery', 'Expenses:Restaurant'},
accounts)
def validate_active_accounts(entries, unused_options_map):
"""Check that all references to accounts occurs on active accounts.
We basically check that references to accounts from all directives other
than Open and Close occur at dates the open-close interval of that account.
This should be good for all of the directive types where we can extract an
account name.
Note that this is more strict a check than comparing the dates: we actually
check that no references to account are made on the same day before the open
directive appears for that account. This is a nice property to have, and is
supported by our custom sorting routine that will sort open entries before
transaction entries, given the same date.
Args:
entries: A list of directives.
unused_options_map: An options map.
Returns:
A list of new errors, if any were found.
"""
error_pairs = []
active_set = set()
opened_accounts = set()
for entry in entries:
if isinstance(entry, data.Open):
active_set.add(entry.account)
opened_accounts.add(entry.account)
elif isinstance(entry, data.Close):
active_set.discard(entry.account)
else:
for account in getters.get_entry_accounts(entry):
if account not in active_set:
# Allow document and note directives that occur after an
# account is closed.
if (isinstance(entry, ALLOW_AFTER_CLOSE)
and account in opened_accounts):
continue
# Register an error to be logged later, with an appropriate
# message.
error_pairs.append((account, entry))
# Refine the error message to disambiguate between the case of an account
# that has never been seen and one that was simply not active at the time.
errors = []
for account, entry in error_pairs:
if account in opened_accounts:
message = "Invalid reference to inactive account '{}'".format(
account)
else:
message = "Invalid reference to unknown account '{}'".format(
account)
errors.append(ValidationError(entry.meta, message, entry))
return errors
def compute_entry_context(entries, context_entry, additional_accounts=None):
"""Compute the balances of all accounts referenced by entry up to entry.
This provides the inventory of the accounts to which the entry is to be
applied, before and after.
Args:
entries: A list of directives.
context_entry: The entry for which we want to obtain the before and after
context.
additional_accounts: Additional list of accounts to include in calculating
the balance. This is used when invoked for debugging, in case the booked
& interpolated transaction doesn't have all the accounts we need because
it had an error (the booking code will remove invalid postings).
Returns:
Two dicts of account-name to Inventory instance, one which represents the
context before the entry is applied, and one that represents the context
after it has been applied.
"""
assert context_entry is not None, "context_entry is missing."
# Get the set of accounts for which to compute the context.
context_accounts = getters.get_entry_accounts(context_entry)
if additional_accounts:
context_accounts.update(additional_accounts)
# Iterate over the entries until we find the target one and accumulate the
# balance.
context_before = collections.defaultdict(inventory.Inventory)
for entry in entries:
if entry is context_entry:
break
if isinstance(entry, Transaction):
for posting in entry.postings:
if not any(posting.account == account
for account in context_accounts):
continue
balance = context_before[posting.account]
balance.add_position(posting)
# Compute the after context for the entry.
context_after = copy.deepcopy(context_before)
if isinstance(context_entry, Transaction):
for posting in entry.postings:
balance = context_after[posting.account]
balance.add_position(posting)
return context_before, context_after
def compute_entry_context(entries, context_entry):
"""Compute the balances of all accounts referenced by entry up to entry.
This provides the inventory of the accounts to which the entry is to be
applied, before and after.
Args:
entries: A list of directives.
context_entry: The entry for which we want to obtain the before and after
context.
Returns:
Two dicts of account-name to Inventory instance, one which represents the
context before the entry is applied, and one that represents the context
after it has been applied.
"""
assert context_entry is not None, "context_entry is missing."
# Get the set of accounts for which to compute the context.
context_accounts = getters.get_entry_accounts(context_entry)
# Iterate over the entries until we find the target one and accumulate the
# balance.
context_before = collections.defaultdict(inventory.Inventory)
for entry in entries:
if entry is context_entry:
break
if isinstance(entry, Transaction):
for posting in entry.postings:
if not any(posting.account == account
for account in context_accounts):
continue
balance = context_before[posting.account]
balance.add_position(posting.position)
# Compute the after context for the entry.
context_after = copy.deepcopy(context_before)
if isinstance(context_entry, Transaction):
for posting in entry.postings:
balance = context_after[posting.account]
balance.add_position(posting.position)
return context_before, context_after
def validate_unused_accounts(entries, unused_options_map):
"""Check that all accounts declared open are actually used.
We check that all of the accounts that are open are at least referred to by
another directive. These are probably unused, so issue a warning (we like to
be pedantic). Note that an account that is open and then closed is
considered used--this is a valid use case that may occur in reality. If you
have a use case for an account to be open but never used, you can quiet that
warning by initializing the account with a balance asserts or a pad
directive, or even use a note will be sufficient.
(This is probably a good candidate for optional inclusion as a "pedantic"
plugin.)
Args:
entries: A list of directives.
unused_options_map: An options map.
Returns:
A list of new errors, if any were found.
"""
# Find all the accounts referenced by entries which are not Open, and the
# open directives for error reporting below.
open_map = {}
referenced_accounts = set()
for entry in entries:
if isinstance(entry, data.Open):
open_map[entry.account] = entry
continue
referenced_accounts.update(getters.get_entry_accounts(entry))
# Create a list of suitable errors, with the location of the Open directives
# corresponding to the unused accounts.
errors = [
UnusedAccountError(open_entry.meta,
"Unused account '{}'".format(account), open_entry)
for account, open_entry in open_map.items()
if account not in referenced_accounts
]
return entries, errors
def render_entry_context(entries, options_map, entry):
"""Render the context before and after a particular transaction is applied.
Args:
entries: A list of directives.
options_map: A dict of options, as produced by the parser.
entry: The entry instance which should be rendered. (Note that this object is
expected to be in the set of entries, not just structurally equal.)
Returns:
A multiline string of text, which consists of the context before the
transaction is applied, the transaction itself, and the context after it
is applied. You can just print that, it is in form that is intended to be
consumed by the user.
"""
oss = io.StringIO()
meta = entry.meta
print("Hash:{}".format(compare.hash_entry(entry)), file=oss)
print("Location: {}:{}".format(meta["filename"], meta["lineno"]), file=oss)
# Get the list of accounts sorted by the order in which they appear in the
# closest entry.
order = {}
if isinstance(entry, data.Transaction):
order = {
posting.account: index
for index, posting in enumerate(entry.postings)
}
accounts = sorted(getters.get_entry_accounts(entry),
key=lambda account: order.get(account, 10000))
# Accumulate the balances of these accounts up to the entry.
balance_before, balance_after = interpolate.compute_entry_context(
entries, entry)
# Create a format line for printing the contents of account balances.
max_account_width = max(map(len, accounts)) if accounts else 1
position_line = '{{:1}} {{:{width}}} {{:>49}}'.format(
width=max_account_width)
# Print the context before.
print(file=oss)
print("------------ Balances before transaction", file=oss)
print(file=oss)
before_hashes = set()
for account in accounts:
positions = balance_before[account].get_positions()
for position in positions:
before_hashes.add((account, hash(position)))
print(position_line.format('', account, str(position)), file=oss)
if not positions:
print(position_line.format('', account, ''), file=oss)
print(file=oss)
# Print the entry itself.
print(file=oss)
print("------------ Transaction", file=oss)
print(file=oss)
dcontext = options_map['dcontext']
printer.print_entry(entry, dcontext, render_weights=True, file=oss)
if isinstance(entry, data.Transaction):
print(file=oss)
# Print residuals.
residual = interpolate.compute_residual(entry.postings)
if not residual.is_empty():
# Note: We render the residual at maximum precision, for debugging.
print('Residual: {}'.format(residual), file=oss)
# Dump the tolerances used.
tolerances = interpolate.infer_tolerances(entry.postings, options_map)
if tolerances:
print('Tolerances: {}'.format(', '.join(
'{}={}'.format(key, value)
for key, value in sorted(tolerances.items()))),
file=oss)
# Compute the total cost basis.
cost_basis = inventory.Inventory(pos for pos in entry.postings
if pos.cost is not None).reduce(
convert.get_cost)
if not cost_basis.is_empty():
print('Basis: {}'.format(cost_basis), file=oss)
# Print the context after.
print(file=oss)
print("------------ Balances after transaction", file=oss)
print(file=oss)
for account in accounts:
positions = balance_after[account].get_positions()
for position in positions:
changed = (account, hash(position)) not in before_hashes
print(position_line.format('*' if changed else '', account,
str(position)),
file=oss)
if not positions:
print(position_line.format('', account, ''), file=oss)
print(file=oss)
return oss.getvalue()
def split_income(entries, options_map, config_str):
"""Split income transactions."""
# pylint: disable=not-callable,too-many-locals
errors = []
new_entries = []
new_accounts = set()
config = {
'income': options_map['name_income'],
'net_income': options_map['name_income'] + ':Net',
'tag': 'pretax',
'taxes': options_map['name_expenses'] + ':Taxes',
}
if config_str.strip():
try:
expr = ast.literal_eval(config_str)
config.update(expr)
except (SyntaxError, ValueError):
errors.append(
SplitIncomeError(
data.new_metadata(options_map['filename'], 0),
"Syntax error in config: {}".format(config_str), None))
return entries, errors
for entry in entries:
if not isinstance(entry, data.Transaction) or not any(
account.startswith(config['income'])
for account in getters.get_entry_accounts(entry)):
continue
# The new entry containing the raw income and taxes.
new_entry = copy.deepcopy(entry)
new_entry = new_entry._replace(
postings=[], tags=frozenset(set([config['tag']]) | entry.tags))
new_entries.append(new_entry)
income = collections.defaultdict(Inventory)
taxes = collections.defaultdict(Decimal)
for posting in list(entry.postings):
if posting.account.startswith(config['income']):
new_entry.postings.append(posting)
entry.postings.remove(posting)
income[posting.account].add_amount(posting.units)
elif re.match(config['taxes'], posting.account):
new_entry.postings.append(posting)
entry.postings.remove(posting)
taxes[posting.units.currency] += posting.units.number
for account, inv in income.items():
net_account = account.replace(config['income'],
config['net_income'])
if net_account not in new_accounts:
new_accounts.add(net_account)
new_entries.append(
data.Open(data.new_metadata('<split_income>', 0),
entry.date, net_account, None, None))
for pos in inv:
amount = pos.units
number = amount.number + taxes.pop(amount.currency, ZERO)
data.create_simple_posting(entry, net_account, number,
amount.currency)
data.create_simple_posting(new_entry, net_account, -number,
amount.currency)
return entries + new_entries, errors
def segment_periods(entries,
accounts_value,
accounts_intflows,
date_begin=None,
date_end=None):
"""Segment entries in terms of piecewise periods of internal flow.
This function iterated through the given entries and computes balances at
the beginning and end of periods without external flow entries. You should be
able to then compute the returns from these informations.
Args:
entries: A list of directives. The list may contain directives other than
than transactions as well as directives with no relation to the assets or
internal flow accounts (the function simply ignores that which is not
relevant).
accounts_value: A set of the asset accounts in the related group.
accounts_intflows: A set of the internal flow accounts in the related group.
date_begin: A datetime.date instance, the beginning date of the period to compute
returns over.
date_end: A datetime.date instance, the end date of the period to compute returns
over.
Returns:
A pair of
periods: A list of period tuples, each of which contains:
period_begin: A datetime.date instance, the first day of the period.
period_end: A datetime.date instance, the last day of the period.
balance_begin: An Inventory instance, the balance at the beginning of the period.
balance_end: An Inventory instance, the balance at the end of the period.
portfolio_entries: A list of the entries that we used in computing the portfolio.
Raises:
ValueError: If the dates create an impossible situation, the beginning
must come before the requested end, if specified.
"""
logging.info("Segmenting periods.")
logging.info("Date begin: %s", date_begin)
logging.info("Date end: %s", date_end)
if date_begin and date_end and date_begin >= date_end:
raise ValueError("Dates are not ordered correctly: {} >= {}".format(
date_begin, date_end))
accounts_related = accounts_value | accounts_intflows
is_external_flow_entry = lambda entry: (isinstance(
entry, data.Transaction) and any(posting.account not in
accounts_related
for posting in entry.postings))
# Create an iterator over the entries we care about.
portfolio_entries = [
entry for entry in entries
if getters.get_entry_accounts(entry) & accounts_value
]
iter_entries = iter(portfolio_entries)
entry = next(iter_entries)
# If a beginning cut-off has been specified, skip the entries before then
# (and make sure to accumulate the initial balance correctly).
balance = inventory.Inventory()
if date_begin is not None:
period_begin = date_begin
try:
while True:
if entry.date >= date_begin:
break
if date_end and entry.date >= date_end:
break
balance = sum_balances_for_accounts(balance, entry,
accounts_value)
entry = next(iter_entries)
except StopIteration:
# No periods found! Just return an empty list.
return [(date_begin, date_end or date_begin, balance, balance)], []
else:
period_begin = entry.date
# Main loop over the entries.
periods = []
entry_logger = misc_utils.LineFileProxy(logging.debug, ' ')
done = False
while True:
balance_begin = copy.copy(balance)
logging.debug(
",-----------------------------------------------------------")
logging.debug(" Begin: %s", period_begin)
logging.debug(" Balance: %s", balance_begin.units())
logging.debug("")
# Consume all internal flow entries, simply accumulating the total balance.
while True:
period_end = entry.date
if is_external_flow_entry(entry):
break
if date_end and entry.date >= date_end:
period_end = date_end
done = True
break
if entry:
printer.print_entry(entry, file=entry_logger)
balance = sum_balances_for_accounts(balance, entry, accounts_value)
try:
entry = next(iter_entries)
except StopIteration:
done = True
if date_end:
period_end = date_end
break
else:
done = True
balance_end = copy.copy(balance)
## FIXME: Bring this back in, this fails for now. Something about the
## initialization fails it. assert period_begin <= period_end,
## (period_begin, period_end)
periods.append((period_begin, period_end, balance_begin, balance_end))
logging.debug(" Balance: %s", balance_end.units())
logging.debug(" End: %s", period_end)
logging.debug(
"`-----------------------------------------------------------")
logging.debug("")
if done:
break
# Absorb the balance of the external flow entry.
assert is_external_flow_entry(entry), entry
if entry:
printer.print_entry(entry, file=entry_logger)
balance = sum_balances_for_accounts(balance, entry, accounts_value)
try:
entry = next(iter_entries)
except StopIteration:
# If there is an end date, insert that final period to cover the end
# date, with no changes.
if date_end:
periods.append((period_end, date_end, balance, balance))
break
period_begin = period_end
## FIXME: Bring this back in, this fails for now.
# assert all(period_begin <= period_end
# for period_begin, period_end, _, _ in periods), periods
return periods, portfolio_entries
def segment_periods(entries, accounts_value, accounts_internal):
"""Segment entries in terms of piecewise periods of internal flow.
This function iterates through the given entries and computes balances at
the beginning and end of periods without external flow entries. You should be
able to then compute the returns from this information.
Args:
entries: A list of directives. The list may contain directives other than
than transactions as well as directives with no relation to the assets or
internal flow accounts (the function simply ignores that which is not
relevant).
accounts_value: A set of the asset accounts in the related group.
accounts_internal: A set of the internal flow accounts in the related group.
Returns:
A timeline, which is a list of Segment instances.
Raises:
ValueError: If the dates create an impossible situation, the beginning
must come before the requested end, if specified.
"""
accounts_related = accounts_value | accounts_internal
is_external_flow_entry = lambda entry: (isinstance(entry, data.Transaction) and
any(posting.account not in accounts_related
for posting in entry.postings))
# Create an iterator over the entries we care about.
portfolio_entries = [entry
for entry in entries
if (isinstance(entry, data.Transaction) and
getters.get_entry_accounts(entry) & accounts_value)]
iter_entries = iter(portfolio_entries)
entry = next(iter_entries)
# If a beginning cut-off has been specified, skip the entries before then
# (and make sure to accumulate the initial balance correctly).
balance = inventory.Inventory()
period_begin = entry.date
# Main loop over the entries.
timeline = []
done = False
while True:
balance_begin = copy.copy(balance)
# Consume all internal flow entries, simply accumulating the total balance.
segment_entries = []
while True:
period_end = entry.date
if is_external_flow_entry(entry):
break
if entry:
segment_entries.append(entry)
sum_balances_for_accounts(balance, entry, accounts_value)
try:
entry = next(iter_entries)
except StopIteration:
done = True
entry = None
break
balance_end = copy.copy(balance)
## FIXME: Bring this back in, this fails for now. Something about the
## initialization fails it.
## assert period_begin <= period_end, (period_begin, period_end)
external_entries = []
segment = Segment(Snapshot(period_begin, balance_begin),
Snapshot(period_end, balance_end),
segment_entries, external_entries)
timeline.append(segment)
# Absorb the balance of the external flow entries as long as they're on
# the same date (a date change would imply a possible market value
# change and we would want to create a segment in such cases, even if
# the portfolio contents do not change).
if entry is not None:
date = entry.date
while is_external_flow_entry(entry) and entry.date == date:
external_entries.append(entry)
sum_balances_for_accounts(balance, entry, accounts_value)
try:
entry = next(iter_entries)
except StopIteration:
done = True
break
if done:
break
period_begin = period_end
## FIXME: Bring this back in, this fails for now.
# assert all(period_begin <= period_end
# for period_begin, period_end, _, _ in periods), periods
return timeline
def __call__(self, context):
pattern = self.operands[0](context)
search = re.compile(pattern, re.IGNORECASE).search
return any(
search(account)
for account in getters.get_entry_accounts(context.entry))
def __call__(self, entry):
pattern = self.eval_args(entry)[0]
search = re.compile(pattern, re.IGNORECASE).search
return any(search(account) for account in getters.get_entry_accounts(entry))
def render_entry_context(entries, options_map, entry, parsed_entry=None):
"""Render the context before and after a particular transaction is applied.
Args:
entries: A list of directives.
options_map: A dict of options, as produced by the parser.
entry: The entry instance which should be rendered. (Note that this object is
expected to be in the set of entries, not just structurally equal.)
parsed_entry: An optional incomplete, parsed but not booked nor interpolated
entry. If this is provided, this is used for inspecting the list of prior
accounts and it is also rendered.
Returns:
A multiline string of text, which consists of the context before the
transaction is applied, the transaction itself, and the context after it
is applied. You can just print that, it is in form that is intended to be
consumed by the user.
"""
oss = io.StringIO()
pr = functools.partial(print, file=oss)
header = "** {} --------------------------------"
meta = entry.meta
pr(header.format("Transaction Id"))
pr()
pr("Hash:{}".format(compare.hash_entry(entry)))
pr("Location: {}:{}".format(meta["filename"], meta["lineno"]))
pr()
pr()
# Get the list of accounts sorted by the order in which they appear in the
# closest entry.
order = {}
if parsed_entry is None:
parsed_entry = entry
if isinstance(parsed_entry, data.Transaction):
order = {
posting.account: index
for index, posting in enumerate(parsed_entry.postings)
}
accounts = sorted(getters.get_entry_accounts(parsed_entry),
key=lambda account: order.get(account, 10000))
# Accumulate the balances of these accounts up to the entry.
balance_before, balance_after = interpolate.compute_entry_context(
entries, entry, additional_accounts=accounts)
# Create a format line for printing the contents of account balances.
max_account_width = max(map(len, accounts)) if accounts else 1
position_line = '{{:1}} {{:{width}}} {{:>49}}'.format(
width=max_account_width)
# Print the context before.
pr(header.format("Balances before transaction"))
pr()
before_hashes = set()
average_costs = {}
for account in accounts:
balance = balance_before[account]
pc_balances = balance.split()
for currency, pc_balance in pc_balances.items():
if len(pc_balance) > 1:
average_costs[account] = pc_balance.average()
positions = balance.get_positions()
for position in positions:
before_hashes.add((account, hash(position)))
pr(position_line.format('', account, str(position)))
if not positions:
pr(position_line.format('', account, ''))
pr()
pr()
# Print average cost per account, if relevant.
if average_costs:
pr(header.format("Average Costs"))
pr()
for account, average_cost in sorted(average_costs.items()):
for position in average_cost:
pr(position_line.format('', account, str(position)))
pr()
pr()
# Print the entry itself.
dcontext = options_map['dcontext']
pr(header.format("Unbooked Transaction"))
pr()
if parsed_entry:
printer.print_entry(parsed_entry,
dcontext,
render_weights=True,
file=oss)
pr()
pr(header.format("Transaction"))
pr()
printer.print_entry(entry, dcontext, render_weights=True, file=oss)
pr()
if isinstance(entry, data.Transaction):
pr(header.format("Residual and Tolerances"))
pr()
# Print residuals.
residual = interpolate.compute_residual(entry.postings)
if not residual.is_empty():
# Note: We render the residual at maximum precision, for debugging.
pr('Residual: {}'.format(residual))
# Dump the tolerances used.
tolerances = interpolate.infer_tolerances(entry.postings, options_map)
if tolerances:
pr('Tolerances: {}'.format(', '.join(
'{}={}'.format(key, value)
for key, value in sorted(tolerances.items()))))
# Compute the total cost basis.
cost_basis = inventory.Inventory(pos for pos in entry.postings
if pos.cost is not None).reduce(
convert.get_cost)
if not cost_basis.is_empty():
pr('Basis: {}'.format(cost_basis))
pr()
pr()
# Print the context after.
pr(header.format("Balances after transaction"))
pr()
for account in accounts:
positions = balance_after[account].get_positions()
for position in positions:
changed = (account, hash(position)) not in before_hashes
print(position_line.format('*' if changed else '', account,
str(position)),
file=oss)
if not positions:
pr(position_line.format('', account, ''))
pr()
return oss.getvalue()
