def hierarchy(self, account_name, begin=None, end=None): """An account tree.""" if begin: tree = Tree(iter_entry_dates(self.ledger.entries, begin, end)) else: tree = self.ledger.root_tree return tree.get(account_name).serialise(end)
def find_similar_entries(entries, source_entries, comparator=None, window_days=2, filter_type=Transaction): '''Same as beancount's similar.find_similar_entries function, but with the ability to filter any type of entries''' window_head = datetime.timedelta(days=window_days) window_tail = datetime.timedelta(days=window_days + 1) if comparator is None: comparator = SimilarityComparator() # For each of the new entries, look at entries at a nearby date. duplicates = [] for entry in filter_ledger(entries, filter_type): filtered_entries = list( filter_ledger( data.iter_entry_dates(source_entries, entry.date - window_head, entry.date + window_tail), filter_type)) for source_entry in filtered_entries: if comparator(entry, source_entry): duplicates.append((entry, source_entry)) break return duplicates
def interval_balances(self, interval, account_name, accumulate=False): """Balances by interval. Arguments: interval: An interval. account_name: An account name. accumulate: A boolean, ``True`` if the balances for an interval should include all entries up to the end of the interval. Returns: A list of RealAccount instances for all the intervals. """ min_accounts = [ account for account in self.accounts.keys() if account.startswith(account_name)] interval_tuples = list( reversed(list(pairwise(self.interval_ends(interval)))) ) interval_balances = [ realization.realize(list(iter_entry_dates( self.entries, datetime.date.min if accumulate else begin_date, end_date)), min_accounts) for begin_date, end_date in interval_tuples] return interval_balances, interval_tuples
def hierarchy(self, account_name, begin=None, end=None): """An account tree.""" if begin: tree = Tree(iter_entry_dates(self.ledger.entries, begin, end)) else: tree = self.ledger.root_tree return _serialize_account_node(tree.get(account_name), end)
def interval_totals( self, interval: Interval, accounts: Union[str, Tuple[str]], conversion: str, ): """Renders totals for account (or accounts) in the intervals. Args: interval: An interval. accounts: A single account (str) or a tuple of accounts. conversion: The conversion to use. """ price_map = self.ledger.price_map for begin, end in pairwise(self.ledger.interval_ends(interval)): inventory = Inventory() entries = iter_entry_dates(self.ledger.entries, begin, end) for entry in (e for e in entries if isinstance(e, Transaction)): for posting in entry.postings: if posting.account.startswith(accounts): inventory.add_position(posting) yield { "date": begin, "balance": cost_or_value(inventory, conversion, price_map, end - ONE_DAY), "budgets": self.ledger.budgets.calculate_children(accounts, begin, end), }
def interval_balances(self, interval, account_name, accumulate=False): """Balances by interval. Arguments: interval: An interval. account_name: An account name. accumulate: A boolean, ``True`` if the balances for an interval should include all entries up to the end of the interval. Returns: A list of RealAccount instances for all the intervals. """ min_accounts = [ account for account in self.accounts.keys() if account.startswith(account_name) ] interval_tuples = list( reversed(list(pairwise(self.interval_ends(interval))))) interval_balances = [ realization.realize( list( iter_entry_dates( self.entries, datetime.date.min if accumulate else begin_date, end_date, )), min_accounts, ) for begin_date, end_date in interval_tuples ] return interval_balances, interval_tuples
def hierarchy(self, account_name, begin=None, end=None): """An account tree.""" if begin: entries = iter_entry_dates(self.ledger.entries, begin, end) tree = Tree(entries) else: tree = self.ledger.root_tree return _serialize_account_node(tree.get(account_name), end)
def interval_totals( self, filtered: FilteredLedger, interval: Interval, accounts: str | tuple[str], conversion: str, invert: bool = False, ) -> Generator[DateAndBalanceWithBudget, None, None]: """Renders totals for account (or accounts) in the intervals. Args: interval: An interval. accounts: A single account (str) or a tuple of accounts. conversion: The conversion to use. invert: invert all numbers. """ # pylint: disable=too-many-locals price_map = self.ledger.price_map for begin, end in pairwise(filtered.interval_ends(interval)): inventory = Inventory() entries = iter_entry_dates(filtered.entries, begin, end) account_inventories = {} for entry in (e for e in entries if isinstance(e, Transaction)): for posting in entry.postings: if posting.account.startswith(accounts): if posting.account not in account_inventories: account_inventories[posting.account] = Inventory() account_inventories[posting.account].add_position( posting) inventory.add_position(posting) balance = cost_or_value(inventory, conversion, price_map, end - ONE_DAY) account_balances = {} for account, acct_value in account_inventories.items(): account_balances[account] = cost_or_value( acct_value, conversion, price_map, end - ONE_DAY, ) budgets = {} if isinstance(accounts, str): budgets = self.ledger.budgets.calculate_children( accounts, begin, end) if invert: # pylint: disable=invalid-unary-operand-type balance = -balance budgets = {k: -v for k, v in budgets.items()} account_balances = {k: -v for k, v in account_balances.items()} yield DateAndBalanceWithBudget( begin, balance, account_balances, budgets, )
def hierarchy(self, account_name, begin=None, end=None): """An account tree.""" if begin: entries = iter_entry_dates(self.ledger.entries, begin, end) root_account = realization.realize(entries) else: root_account = self.ledger.root_account return _serialize_real_account( realization.get_or_create(root_account, account_name), end)
def hierarchy( self, account_name: str, begin: Optional[datetime.date] = None, end: Optional[datetime.date] = None, ): """An account tree.""" if begin is not None: tree = Tree(iter_entry_dates(self.ledger.entries, begin, end)) else: tree = self.ledger.root_tree return tree.get(account_name).serialise(end)
def hierarchy( self, account_name: str, conversion: str, begin: Optional[date] = None, end: Optional[date] = None, ) -> SerialisedTreeNode: """An account tree.""" if begin is not None and end is not None: tree = Tree(iter_entry_dates(self.ledger.entries, begin, end)) else: tree = self.ledger.root_tree return tree.get(account_name).serialise(conversion, self.ledger, end - ONE_DAY if end else None)
def interval_balances(self, interval, account_name, accumulate=False): """accumulate is False for /changes and True for /balances""" min_accounts = [account for account in self.all_accounts if account.startswith(account_name)] interval_tuples = list(reversed(self._interval_tuples(interval))) interval_balances = [ realization.realize( list(iter_entry_dates(self.entries, self.date_first if accumulate else begin_date, end_date)), min_accounts, ) for begin_date, end_date in interval_tuples ] return interval_balances, interval_tuples
def interval_balances(self, interval, account_name, accumulate=False): """accumulate is False for /changes and True for /balances""" min_accounts = [account for account in self.all_accounts if account.startswith(account_name)] interval_tuples = list(reversed(self._interval_tuples(interval))) interval_balances = [ realization.realize(list(iter_entry_dates( self.entries, self.date_first if accumulate else begin_date, end_date)), min_accounts) for begin_date, end_date in interval_tuples] return interval_balances, interval_tuples
def hierarchy( self, filtered: FilteredLedger, account_name: str, conversion: str, begin: date | None = None, end: date | None = None, ) -> SerialisedTreeNode: """An account tree.""" if begin is not None and end is not None: tree = Tree(iter_entry_dates(filtered.entries, begin, end)) else: tree = filtered.root_tree return tree.get(account_name).serialise(conversion, self.ledger.price_map, end - ONE_DAY if end else None)
def _real_account(account_name, entries, begin_date=None, end_date=None, min_accounts=None): """ Returns the realization.RealAccount instances for account_name, and their entries clamped by the optional begin_date and end_date. Warning: For efficiency, the returned result does not include any added postings to account for balances at 'begin_date'. :return: realization.RealAccount instances """ if begin_date: entries = list(iter_entry_dates(entries, begin_date, end_date)) if not min_accounts: min_accounts = [account_name] return realization.get(realization.realize(entries, min_accounts), account_name)
def _real_account(self, account_name, entries, begin_date=None, end_date=None, min_accounts=None): """ Returns the realization.RealAccount instances for account_name, and their entries clamped by the optional begin_date and end_date. Warning: For efficiency, the returned result does not include any added postings to account for balances at 'begin_date'. :return: realization.RealAccount instances """ if begin_date: entries = list(iter_entry_dates(entries, begin_date, end_date)) if not min_accounts: min_accounts = [account_name] return realization.get(realization.realize(entries, min_accounts), account_name)
def interval_totals( self, interval: Interval, accounts: Union[str, Tuple[str]], conversion: str, invert: bool = False, ) -> Generator[DateAndBalanceWithBudget, None, None]: """Renders totals for account (or accounts) in the intervals. Args: interval: An interval. accounts: A single account (str) or a tuple of accounts. conversion: The conversion to use. """ price_map = self.ledger.price_map for begin, end in pairwise(self.ledger.interval_ends(interval)): inventory = Inventory() entries = iter_entry_dates(self.ledger.entries, begin, end) for entry in (e for e in entries if isinstance(e, Transaction)): for posting in entry.postings: if posting.account.startswith(accounts): inventory.add_position(posting) balance = cost_or_value( inventory, conversion, price_map, end - ONE_DAY ) budgets = {} if isinstance(accounts, str): budgets = self.ledger.budgets.calculate_children( accounts, begin, end ) if invert: # pylint: disable=invalid-unary-operand-type balance = -balance budgets = {k: -v for k, v in budgets.items()} yield { "date": begin, "balance": balance, "budgets": budgets, }
def find_similar_entries(entries, source_entries, comparator=None, window_days=2): """Find which entries from a list are potential duplicates of a set. Note: If there are multiple entries from 'source_entries' matching an entry in 'entries', only the first match is returned. Note that this function could in theory decide to merge some of the imported entries with each other. Args: entries: The list of entries to classify as duplicate or note. source_entries: The list of entries against which to match. This is the previous, or existing set of entries to compare against. This may be null or empty. comparator: A functor used to establish the similarity of two entries. window_days: The number of days (inclusive) before or after to scan the entries to classify against. Returns: A list of pairs of entries (entry, source_entry) where entry is from 'entries' and is deemed to be a duplicate of source_entry, from 'source_entries'. """ window_head = datetime.timedelta(days=window_days) window_tail = datetime.timedelta(days=window_days + 1) if comparator is None: comparator = SimilarityComparator() # For each of the new entries, look at existing entries at a nearby date. duplicates = [] if source_entries is not None: for entry in data.filter_txns(entries): for source_entry in data.filter_txns( data.iter_entry_dates(source_entries, entry.date - window_head, entry.date + window_tail)): if comparator(entry, source_entry): duplicates.append((entry, source_entry)) break return duplicates
def interval_balances( self, filtered: FilteredLedger, interval: date.Interval, account_name: str, accumulate: bool = False, ) -> tuple[list[realization.RealAccount], list[tuple[datetime.date, datetime.date]], ]: """Balances by interval. Arguments: filtered: The currently filtered ledger. interval: An interval. account_name: An account name. accumulate: A boolean, ``True`` if the balances for an interval should include all entries up to the end of the interval. Returns: A list of RealAccount instances for all the intervals. """ min_accounts = [ account for account in self.accounts.keys() if account.startswith(account_name) ] interval_tuples = list( reversed(list(pairwise(filtered.interval_ends(interval))))) interval_balances = [ realization.realize( list( iter_entry_dates( filtered.entries, datetime.date.min if accumulate else begin_date, end_date, )), min_accounts, ) for begin_date, end_date in interval_tuples ] return interval_balances, interval_tuples
def portfolio_accounts(self, begin=None, end=None): """An account tree based on matching regex patterns.""" if begin: tree = Tree(iter_entry_dates(self.ledger.entries, begin, end)) else: tree = self.ledger.root_tree portfolios = [] for option in self.config: opt_key = option[0] if opt_key == "account_name_pattern": portfolio = self._account_name_pattern(tree, end, option[1]) elif opt_key == "account_open_metadata_pattern": portfolio = self._account_metadata_pattern( tree, end, option[1][0], option[1][1]) else: raise FavaAPIException("Portfolio List: Invalid option.") portfolios.append(portfolio) return portfolios
def interval_totals(self, interval, accounts): """Renders totals for account (or accounts) in the intervals. Args: interval: A string for the interval. accounts: A single account (str) or a tuple of accounts. """ for begin, end in pairwise(self.ledger.interval_ends(interval)): inventory = Inventory() entries = iter_entry_dates(self.ledger.entries, begin, end) for entry in filter_type(entries, Transaction): for posting in entry.postings: if posting.account.startswith(accounts): inventory.add_position(posting) yield { 'begin_date': begin, 'totals': _inventory_cost_or_value(inventory, end), 'budgets': self.ledger.budgets.calculate(accounts[0], begin, end), }
def interval_totals(self, interval, accounts): """Renders totals for account (or accounts) in the intervals. Args: interval: An interval. accounts: A single account (str) or a tuple of accounts. """ for begin, end in pairwise(self.ledger.interval_ends(interval)): inventory = CounterInventory() entries = iter_entry_dates(self.ledger.entries, begin, end) for entry in filter_type(entries, Transaction): for posting in entry.postings: if posting.account.startswith(accounts): inventory.add_position(posting) yield { 'date': begin, 'balance': cost_or_value(inventory, end), 'budgets': self.ledger.budgets.calculate_children(accounts, begin, end), }
def interval_totals(self, interval, accounts): """Renders totals for account (or accounts) in the intervals. Args: interval: An interval. accounts: A single account (str) or a tuple of accounts. """ for begin, end in pairwise(self.ledger.interval_ends(interval)): inventory = CounterInventory() entries = iter_entry_dates(self.ledger.entries, begin, end) for entry in filter_type(entries, Transaction): for posting in entry.postings: if posting.account.startswith(accounts): inventory.add_position(posting) yield { "date": begin, "balance": cost_or_value(inventory, end), "budgets": self.ledger.budgets.calculate_children( accounts, begin, end ), }
def portfolio_accounts(self, begin=None, end=None): """An account tree based on matching regex patterns.""" portfolios = [] try: self.load_report() if begin: tree = Tree(iter_entry_dates(self.ledger.entries, begin, end)) else: tree = self.ledger.root_tree for option in self.config: opt_key = option[0] if opt_key == "account_name_pattern": portfolio = self._account_name_pattern(tree, end, option[1]) elif opt_key == "account_open_metadata_pattern": portfolio = self._account_metadata_pattern( tree, end, option[1][0], option[1][1] ) else: exception = FavaAPIException("Classy Portfolio: Invalid option.") raise (exception) portfolio = ( portfolio[0], # title portfolio[1], # subtitle ( insert_rowspans(portfolio[2][0], portfolio[2][1], True), portfolio[2][1], ), # portfolio data ) portfolios.append(portfolio) except Exception as exc: traceback.print_exc(file=sys.stdout) return portfolios
def _real_account(account_name, entries, begin_date, end_date): if begin_date: entries = list(iter_entry_dates(entries, begin_date, end_date)) return realization.get_or_create(realization.realize(entries), account_name)
def test_iter_entry_dates(self): prototype = data.Transaction(data.new_metadata("misc", 200), None, '*', None, "", None, None, []) dates = [ datetime.date(2016, 1, 10), datetime.date(2016, 1, 11), datetime.date(2016, 1, 14), datetime.date(2016, 1, 15), datetime.date(2016, 1, 16), datetime.date(2016, 1, 20), datetime.date(2016, 1, 22) ] entries = [prototype._replace(date=date) for date in dates] # Same date, present. self.assertEqual([], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 14), datetime.date( 2016, 1, 14)) ]) # Same date, absent. self.assertEqual([], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 12), datetime.date( 2016, 1, 12)) ]) # Both dates exist. self.assertEqual( [datetime.date(2016, 1, 11), datetime.date(2016, 1, 14)], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 11), datetime.date(2016, 1, 15)) ]) # First doesn't exist. self.assertEqual([datetime.date(2016, 1, 14)], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 12), datetime.date( 2016, 1, 15)) ]) # Second doesn't exist. self.assertEqual([datetime.date(2016, 1, 11)], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 11), datetime.date( 2016, 1, 13)) ]) # Neither exist. self.assertEqual([ datetime.date(2016, 1, 14), datetime.date(2016, 1, 15), datetime.date(2016, 1, 16) ], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 13), datetime.date( 2016, 1, 17)) ]) # Before. self.assertEqual([datetime.date(2016, 1, 10)], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 5), datetime.date(2016, 1, 11)) ]) # After. self.assertEqual([datetime.date(2016, 1, 22)], [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 21), datetime.date( 2016, 1, 30)) ]) # Around. self.assertEqual(dates, [ entry.date for entry in data.iter_entry_dates( entries, datetime.date(2016, 1, 2), datetime.date(2016, 1, 30)) ])
def deduplicate(new_entries: List[Directive], existing_entries: List[Directive], window_days: int = 10) -> List[Directive]: """De-duplicate entries. A new non-transaction entry is considered connected to an existing entry iff they are identical. A new transaction is considered connected to an existing transaction iff: * the new transaction is no earlier than the existing transaction. * the new transaction is at most {window_days} days later than the existing transaction. * guess_transaction_duplicated returns True. If a new entry doesn't have any connection, it's considered non-duplicated. For each strongly connected subgraph, if all new entries are matched, all of them are considered duplicated. Otherwise, all of them are considered possibly duplicated. Returns new entries where: * Non-duplicated entries are preserved. * Duplicated entries are removed. * Possibly-duplicated entries are marked with DUPLICATE_META. """ window_head = datetime.timedelta(days=window_days) window_tail = datetime.timedelta(days=1) matcher = _Matcher() for new_entry in new_entries: if isinstance(new_entry, Transaction): for existing_entry in filter_txns( iter_entry_dates(existing_entries, new_entry.date - window_head, new_entry.date + window_tail)): if guess_transaction_duplicated(new_entry, existing_entry): matcher.add_edge(id(new_entry), id(existing_entry)) else: for existing_entry in iter_entry_dates( existing_entries, new_entry.date, new_entry.date + window_tail): if new_entry == existing_entry: matcher.add_edge(id(new_entry), id(existing_entry)) duplicates = set() possibly_duplicates = set() matches = matcher.matches() for subgraph in matcher.subgraphs(): n = len([True for node in subgraph if node in matches]) if n == len(subgraph): # duplicated duplicates.update(node[1] for node in subgraph if node[0]) elif n: # possibly duplicated possibly_duplicates.update(node[1] for node in subgraph if node[0]) ret = [] for new_entry in new_entries: if id(new_entry) in duplicates: continue elif id(new_entry) in possibly_duplicates and hasattr( new_entry, 'meta'): meta = copy.deepcopy(new_entry.meta) meta[DUPLICATE_META] = True ret.append(new_entry._replace(meta=meta)) else: ret.append(new_entry) return ret
def _build_graph( entries_by_file: Dict[str, List[Directive]], links: Iterable[Link], logger: ErrorLogger) -> Dict[int, List[Tuple[Transaction, str]]]: day = timedelta(days=1) edges = defaultdict(list) # id(txn) -> [(complement txn, account)] for link in links: if not link.valid(): continue entries = entries_by_file.get(link.filename, []) complement_entries = entries_by_file.get(link.complement_filename, []) for entry in filter_txns(entries): expected_complement_feature = _transaction_feature( entry, link.account, True) if not expected_complement_feature[1]: continue # find complement txn and check duplicated complement complement_txn = None complement_duplicated = False for complement_entry in filter_txns(iter_entry_dates( complement_entries, entry.date, entry.date + day)): complement_feature = _transaction_feature( complement_entry, link.complement_account, False) if complement_feature == expected_complement_feature: if not complement_txn: complement_txn = complement_entry else: complement_duplicated = True break # check duplicated duplicated = False found = False feature = _transaction_feature(entry, link.account, False) for entry2 in filter_txns(iter_entry_dates( entries, entry.date, entry.date + day)): feature2 = _transaction_feature(entry2, link.account, False) if feature2 == feature: if found: duplicated = True break found = True if not complement_txn: logger.log_error(UnresolvedLinkError( entry.meta, f'No complement transaction found for link {link}', entry, )) elif complement_duplicated: logger.log_error(UnresolvedLinkError( entry.meta, f'Multiple complement transactions found for link {link}', entry, )) elif duplicated: logger.log_error(UnresolvedLinkError( complement_txn.meta, f'Multiple complement transactions found for link {link}', complement_txn, )) else: edges[id(entry)].append((complement_txn, link.account)) edges[id(complement_txn)].append((entry, link.complement_account)) for complement_txn in filter_txns(complement_entries): complement_feature = _transaction_feature( complement_txn, link.complement_account, False) if complement_feature[1] and id(complement_txn) not in edges: logger.log_error(UnresolvedLinkError( complement_txn.meta, f'No complement transaction found for link {link}', complement_txn, )) return edges