def _parse(self, infile):
content = infile.read()
lines = stripfalse(content.split('\n'))
blocks = []
autoswitch_blocks = [] # blocks in the middle of an AutoSwitch option
block = Block()
current_block_type = BlockType.Entry
autoswitch_mode = False
for line in lines:
header, data = line[0], line[1:].strip()
if header == '!':
if data == 'Account':
current_block_type = BlockType.Account
elif data in ENTRY_HEADERS:
current_block_type = BlockType.Entry
if autoswitch_mode:
# We have a buggy qif that doesn't clear its autoswitch flag. The last block
# we added to autoswitch actually belonged to normal blocks. move it.
if autoswitch_blocks:
blocks.append(autoswitch_blocks.pop())
autoswitch_mode = False
elif data.startswith('Type:'): # if it doesn't, just ignore it
current_block_type = BlockType.Other
elif data == 'Option:AutoSwitch':
autoswitch_mode = True
elif data == 'Clear:AutoSwitch':
autoswitch_mode = False
elif header == '^':
if current_block_type != BlockType.Other:
block.type = current_block_type
if block.type == BlockType.Entry:
# Make sure we have a valid entry block (which has a valid date) and change
# the type if it's not the case.
date_line = block.get_line('D')
if date_line is None or self.clean_date(
date_line.data) is None:
block.type = BlockType.Other
if autoswitch_mode:
autoswitch_blocks.append(block)
else:
blocks.append(block)
block = Block()
if current_block_type == BlockType.Account and not autoswitch_mode:
current_block_type = BlockType.Entry
if header != '^':
block.lines.append(Line(header, data))
del block
if not blocks:
raise FileFormatError()
logging.debug('This is a QIF file. {0} blocks'.format(len(blocks)))
entry_blocks = [
block for block in blocks if block.type == BlockType.Entry
]
date_lines = (block.get_line('D') for block in entry_blocks)
str_dates = [line.data for line in date_lines if line]
self.parsing_date_format = self.guess_date_format(str_dates)
if self.parsing_date_format is None:
raise FileFormatError()
self.blocks = blocks
self.autoswitch_blocks = autoswitch_blocks
def _parse(self, infile):
content = infile.read()
lines = stripfalse(content.split('\n'))
blocks = []
autoswitch_blocks = [] # blocks in the middle of an AutoSwitch option
block = Block()
current_block_type = BlockType.Entry
autoswitch_mode = False
for line in lines:
header, data = line[0], line[1:].strip()
if header == '!':
if data == 'Account':
current_block_type = BlockType.Account
elif data in ENTRY_HEADERS:
current_block_type = BlockType.Entry
if autoswitch_mode:
# We have a buggy qif that doesn't clear its autoswitch flag. The last block
# we added to autoswitch actually belonged to normal blocks. move it.
if autoswitch_blocks:
blocks.append(autoswitch_blocks.pop())
autoswitch_mode = False
elif data.startswith('Type:'): # if it doesn't, just ignore it
current_block_type = BlockType.Other
elif data == 'Option:AutoSwitch':
autoswitch_mode = True
elif data == 'Clear:AutoSwitch':
autoswitch_mode = False
elif header == '^':
if current_block_type != BlockType.Other:
block.type = current_block_type
if block.type == BlockType.Entry:
# Make sure we have a valid entry block (which has a valid date) and change
# the type if it's not the case.
date_line = block.get_line('D')
if date_line is None or self.clean_date(date_line.data) is None:
block.type = BlockType.Other
if autoswitch_mode:
autoswitch_blocks.append(block)
else:
blocks.append(block)
block = Block()
if current_block_type == BlockType.Account and not autoswitch_mode:
current_block_type = BlockType.Entry
if header != '^':
block.lines.append(Line(header, data))
del block
if not blocks:
raise FileFormatError()
logging.debug('This is a QIF file. {0} blocks'.format(len(blocks)))
entry_blocks = [block for block in blocks if block.type == BlockType.Entry]
date_lines = (block.get_line('D') for block in entry_blocks)
str_dates = [line.data for line in date_lines if line]
self.parsing_date_format = self.guess_date_format(str_dates)
if self.parsing_date_format is None:
raise FileFormatError()
self.blocks = blocks
self.autoswitch_blocks = autoswitch_blocks
def _scan_lines(self, encoding=None):
if not encoding:
encoding = 'latin-1'
content = self.readcontent.decode(encoding, 'ignore').replace('\0', '')
rawlines = content.splitlines()
try:
reader = csv.reader(iter(rawlines), self.dialect)
except TypeError:
logging.warning("Invalid Dialect (strangely...). Delimiter: %r", self.dialect.delimiter)
lines = stripfalse(reader)
# complete smaller lines and strip whitespaces
maxlen = max(len(line) for line in lines)
for line in (l for l in lines if len(l) < maxlen):
line += [''] * (maxlen - len(line))
self.lines = lines
def _scan_lines(self, encoding=None):
if not encoding:
encoding = 'latin-1'
content = self.readcontent.decode(encoding, 'ignore').replace('\0', '')
rawlines = content.splitlines()
try:
reader = csv.reader(iter(rawlines), self.dialect)
except TypeError:
logging.warning("Invalid Dialect (strangely...). Delimiter: %r",
self.dialect.delimiter)
lines = stripfalse(reader)
# complete smaller lines and strip whitespaces
maxlen = max(len(line) for line in lines)
for line in (l for l in lines if len(l) < maxlen):
line += [''] * (maxlen - len(line))
self.lines = lines
def _scan_lines(self, encoding=None):
rawlines = self.rawlines
if encoding and encoding != self.FILE_ENCODING:
# rawlines is a list of ustrings decoded using latin-1, so if we want to re-decode them
# using another encoding, we have to re-encode them and the decode them using our encoding
redecode = lambda s: s.encode(self.FILE_ENCODING).decode(encoding, 'ignore')
rawlines = (redecode(line) for line in rawlines)
try:
reader = csv.reader(iter(rawlines), self.dialect)
except TypeError:
logging.warning("Invalid Dialect (strangely...). Delimiter: %r", self.dialect.delimiter)
lines = stripfalse(reader)
# complete smaller lines and strip whitespaces
maxlen = max(len(line) for line in lines)
for line in (l for l in lines if len(l) < maxlen):
line += [''] * (maxlen - len(line))
self.lines = lines
def balance_currencies(self, strong_split=None):
"""Balances a :ref:`multi-currency transaction <multi-currency-txn>`.
Balancing out multi-currencies transasctions can be real easy because we consider that
currencies can never mix (and we would never make the gross mistake of using market exchange
rates to do our balancing), so, if we have at least one split on each side of different
currencies, we consider ourselves balanced and do nothing.
However, we might be in a situation of "logical imbalance", which means that the transaction
doesn't logically makes sense. For example, if all our splits are on the same side, we can't
possibly balance out. If we have EUR and CAD splits, that CAD splits themselves balance out
but that EUR splits are all on the same side, we have a logical imbalance.
This method finds those imbalance and fix them by creating unsassigned splits balancing out
every currency being in that situation.
:param strong_split: The split that was last edited. See :meth:`balance`.
:type strong_split: :class:`Split`
"""
splits_with_amount = [s for s in self.splits if s.amount != 0]
if not splits_with_amount:
return
currency2balance = defaultdict(int)
for split in splits_with_amount:
currency2balance[split.amount.currency] += split.amount
imbalanced = stripfalse(currency2balance.values()
) # filters out zeros (balances currencies)
# For a logical imbalance to be possible, all imbalanced amounts must be on the same side
if imbalanced and allsame(amount > 0 for amount in imbalanced):
unassigned = [
s for s in self.splits
if s.account is None and s is not strong_split
]
for amount in imbalanced:
split = first(
s for s in unassigned
if s.amount == 0 or s.amount.currency == amount.currency)
if split is not None:
if split.amount == amount: # we end up with a null split, remove it
self.splits.remove(split)
else:
split.amount -= amount # adjust
else:
self.splits.append(Split(self, None, -amount))
def balance_currencies(self, strong_split=None):
splits_with_amount = [s for s in self.splits if s.amount != 0]
if not splits_with_amount:
return
currency2balance = defaultdict(int)
for split in splits_with_amount:
currency2balance[split.amount.currency] += split.amount
imbalanced = stripfalse(currency2balance.values()) # filters out zeros (balances currencies)
# For a logical imbalance to be possible, all imbalanced amounts must be on the same side
if imbalanced and allsame(amount > 0 for amount in imbalanced):
unassigned = [s for s in self.splits if s.account is None and s is not strong_split]
for amount in imbalanced:
split = first(s for s in unassigned if s.amount == 0 or s.amount.currency == amount.currency)
if split is not None:
if split.amount == amount: # we end up with a null split, remove it
self.splits.remove(split)
else:
split.amount -= amount # adjust
else:
self.splits.append(Split(self, None, -amount))
def balance_currencies(self, strong_split=None):
"""Balances a :ref:`multi-currency transaction <multi-currency-txn>`.
Balancing out multi-currencies transasctions can be real easy because we consider that
currencies can never mix (and we would never make the gross mistake of using market exchange
rates to do our balancing), so, if we have at least one split on each side of different
currencies, we consider ourselves balanced and do nothing.
However, we might be in a situation of "logical imbalance", which means that the transaction
doesn't logically makes sense. For example, if all our splits are on the same side, we can't
possibly balance out. If we have EUR and CAD splits, that CAD splits themselves balance out
but that EUR splits are all on the same side, we have a logical imbalance.
This method finds those imbalance and fix them by creating unsassigned splits balancing out
every currency being in that situation.
:param strong_split: The split that was last edited. See :meth:`balance`.
:type strong_split: :class:`Split`
"""
splits_with_amount = [s for s in self.splits if s.amount != 0]
if not splits_with_amount:
return
currency2balance = defaultdict(int)
for split in splits_with_amount:
currency2balance[split.amount.currency] += split.amount
imbalanced = stripfalse(currency2balance.values()) # filters out zeros (balances currencies)
# For a logical imbalance to be possible, all imbalanced amounts must be on the same side
if imbalanced and allsame(amount > 0 for amount in imbalanced):
unassigned = [s for s in self.splits if s.account is None and s is not strong_split]
for amount in imbalanced:
split = first(s for s in unassigned if s.amount == 0 or s.amount.currency == amount.currency)
if split is not None:
if split.amount == amount: # we end up with a null split, remove it
self.splits.remove(split)
else:
split.amount -= amount # adjust
else:
self.splits.append(Split(self, None, -amount))
def load(self):
"""Loads the parsed info into self.accounts and self.transactions.
You must have called parse() before calling this.
"""
def load_transaction_info(info):
description = info.description
payee = info.payee
checkno = info.checkno
date = info.date
transaction = Transaction(date, description, payee, checkno)
transaction.notes = nonone(info.notes, '')
for split_info in info.splits:
account = split_info.account
amount = split_info.amount
if split_info.amount_reversed:
amount = -amount
memo = nonone(split_info.memo, '')
split = Split(transaction, account, amount)
split.memo = memo
if split_info.reconciliation_date is not None:
split.reconciliation_date = split_info.reconciliation_date
elif split_info.reconciled: # legacy
split.reconciliation_date = transaction.date
split.reference = split_info.reference
transaction.splits.append(split)
while len(transaction.splits) < 2:
transaction.splits.append(Split(transaction, None, 0))
transaction.balance()
transaction.mtime = info.mtime
if info.reference is not None:
for split in transaction.splits:
if split.reference is None:
split.reference = info.reference
return transaction
self._load()
self.flush_account() # Implicit
# Now, we take the info we have and transform it into model instances
currencies = set()
start_date = datetime.date.max
for info in self.group_infos:
group = Group(info.name, info.type)
self.groups.append(group)
for info in self.account_infos:
account_type = info.type
if account_type not in AccountType.All:
account_type = AccountType.Asset
account_currency = self.default_currency
try:
if info.currency:
account_currency = Currency(info.currency)
except ValueError:
pass # keep account_currency as self.default_currency
account = Account(info.name, account_currency, account_type)
if info.group:
account.group = self.groups.find(info.group, account_type)
if info.budget:
self.budget_infos.append(BudgetInfo(info.name, info.budget_target, info.budget))
account.reference = info.reference
account.account_number = info.account_number
account.notes = info.notes
currencies.add(account.currency)
self.accounts.add(account)
# Pre-parse transaction info. We bring all relevant info recorded at the txn level into the split level
all_txn = self.transaction_infos + [r.transaction_info for r in self.recurrence_infos] +\
flatten([stripfalse(r.date2exception.values()) for r in self.recurrence_infos]) +\
flatten([r.date2globalchange.values() for r in self.recurrence_infos])
for info in all_txn:
split_accounts = [s.account for s in info.splits]
if info.account and info.account not in split_accounts:
info.splits.insert(0, SplitInfo(info.account, info.amount, info.currency, False))
if info.transfer and info.transfer not in split_accounts:
info.splits.append(SplitInfo(info.transfer, info.amount, info.currency, True))
for split_info in info.splits:
# this amount is just to determine the auto_create_type
str_amount = split_info.amount
if split_info.currency:
str_amount += split_info.currency
amount = self.parse_amount(str_amount, self.default_currency)
auto_create_type = AccountType.Income if amount >= 0 else AccountType.Expense
split_info.account = (
self.accounts.find(split_info.account, auto_create_type)
if split_info.account
else None
)
currency = split_info.account.currency if split_info.account is not None else self.default_currency
split_info.amount = self.parse_amount(str_amount, currency)
if split_info.amount:
currencies.add(split_info.amount.currency)
self.transaction_infos.sort(key=attrgetter('date'))
for date, transaction_infos in groupby(self.transaction_infos, attrgetter('date')):
start_date = min(start_date, date)
for position, info in enumerate(transaction_infos, start=1):
transaction = load_transaction_info(info)
self.transactions.add(transaction, position=position)
# Scheduled
for info in self.recurrence_infos:
ref = load_transaction_info(info.transaction_info)
recurrence = Recurrence(ref, info.repeat_type, info.repeat_every)
recurrence.stop_date = info.stop_date
for date, transaction_info in info.date2exception.items():
if transaction_info is not None:
exception = load_transaction_info(transaction_info)
spawn = Spawn(recurrence, exception, date, exception.date)
recurrence.date2exception[date] = spawn
else:
recurrence.delete_at(date)
for date, transaction_info in info.date2globalchange.items():
change = load_transaction_info(transaction_info)
spawn = Spawn(recurrence, change, date, change.date)
recurrence.date2globalchange[date] = spawn
self.schedules.append(recurrence)
# Budgets
TODAY = datetime.date.today()
fallback_start_date = datetime.date(TODAY.year, TODAY.month, 1)
for info in self.budget_infos:
account = self.accounts.find(info.account)
if account is None:
continue
target = self.accounts.find(info.target) if info.target else None
amount = self.parse_amount(info.amount, account.currency)
start_date = nonone(info.start_date, fallback_start_date)
budget = Budget(account, target, amount, start_date, repeat_type=info.repeat_type)
budget.notes = nonone(info.notes, '')
budget.stop_date = info.stop_date
if info.repeat_every:
budget.repeat_every = info.repeat_every
self.budgets.append(budget)
self._post_load()
self.oven.cook(datetime.date.min, until_date=None)
Currency.get_rates_db().ensure_rates(start_date, [x.code for x in currencies])
def load(self):
"""Loads the parsed info into self.accounts and self.transactions.
You must have called parse() before calling this.
"""
def load_transaction_info(info):
description = info.description
payee = info.payee
checkno = info.checkno
date = info.date
transaction = Transaction(date, description, payee, checkno)
transaction.notes = nonone(info.notes, '')
for split_info in info.splits:
account = split_info.account
amount = split_info.amount
if split_info.amount_reversed:
amount = -amount
memo = nonone(split_info.memo, '')
split = Split(transaction, account, amount)
split.memo = memo
if account is None or not of_currency(amount,
account.currency):
# fix #442: off-currency transactions shouldn't be reconciled
split.reconciliation_date = None
elif split_info.reconciliation_date is not None:
split.reconciliation_date = split_info.reconciliation_date
elif split_info.reconciled: # legacy
split.reconciliation_date = transaction.date
split.reference = split_info.reference
transaction.splits.append(split)
while len(transaction.splits) < 2:
transaction.splits.append(Split(transaction, None, 0))
transaction.balance()
transaction.mtime = info.mtime
if info.reference is not None:
for split in transaction.splits:
if split.reference is None:
split.reference = info.reference
return transaction
self._load()
self.flush_account() # Implicit
# Now, we take the info we have and transform it into model instances
currencies = set()
start_date = datetime.date.max
for info in self.group_infos:
group = Group(info.name, info.type)
self.groups.append(group)
for info in self.account_infos:
account_type = info.type
if account_type not in AccountType.All:
account_type = AccountType.Asset
account_currency = self.default_currency
try:
if info.currency:
account_currency = Currency(info.currency)
except ValueError:
pass # keep account_currency as self.default_currency
account = Account(info.name, account_currency, account_type)
if info.group:
account.group = self.groups.find(info.group, account_type)
if info.budget:
self.budget_infos.append(
BudgetInfo(info.name, info.budget_target, info.budget))
account.reference = info.reference
account.account_number = info.account_number
account.inactive = info.inactive
account.notes = info.notes
currencies.add(account.currency)
self.accounts.add(account)
# Pre-parse transaction info. We bring all relevant info recorded at the txn level into the split level
all_txn = self.transaction_infos + [r.transaction_info for r in self.recurrence_infos] +\
flatten([stripfalse(r.date2exception.values()) for r in self.recurrence_infos]) +\
flatten([r.date2globalchange.values() for r in self.recurrence_infos])
for info in all_txn:
split_accounts = [s.account for s in info.splits]
if info.account and info.account not in split_accounts:
info.splits.insert(
0,
SplitInfo(info.account, info.amount, info.currency, False))
if info.transfer and info.transfer not in split_accounts:
info.splits.append(
SplitInfo(info.transfer, info.amount, info.currency, True))
for split_info in info.splits:
# this amount is just to determine the auto_create_type
str_amount = split_info.amount
if split_info.currency:
str_amount += split_info.currency
amount = self.parse_amount(str_amount, self.default_currency)
auto_create_type = AccountType.Income if amount >= 0 else AccountType.Expense
split_info.account = (self.accounts.find(
split_info.account, auto_create_type)
if split_info.account else None)
currency = split_info.account.currency if split_info.account is not None else self.default_currency
split_info.amount = self.parse_amount(str_amount, currency)
if split_info.amount:
currencies.add(split_info.amount.currency)
self.transaction_infos.sort(key=attrgetter('date'))
for date, transaction_infos in groupby(self.transaction_infos,
attrgetter('date')):
start_date = min(start_date, date)
for position, info in enumerate(transaction_infos, start=1):
transaction = load_transaction_info(info)
self.transactions.add(transaction, position=position)
# Scheduled
for info in self.recurrence_infos:
ref = load_transaction_info(info.transaction_info)
recurrence = Recurrence(ref, info.repeat_type, info.repeat_every)
recurrence.stop_date = info.stop_date
for date, transaction_info in info.date2exception.items():
if transaction_info is not None:
exception = load_transaction_info(transaction_info)
spawn = Spawn(recurrence, exception, date, exception.date)
recurrence.date2exception[date] = spawn
else:
recurrence.delete_at(date)
for date, transaction_info in info.date2globalchange.items():
change = load_transaction_info(transaction_info)
spawn = Spawn(recurrence, change, date, change.date)
recurrence.date2globalchange[date] = spawn
self.schedules.append(recurrence)
# Budgets
TODAY = datetime.date.today()
fallback_start_date = datetime.date(TODAY.year, TODAY.month, 1)
for info in self.budget_infos:
account = self.accounts.find(info.account)
if account is None:
continue
target = self.accounts.find(info.target) if info.target else None
amount = self.parse_amount(info.amount, account.currency)
start_date = nonone(info.start_date, fallback_start_date)
budget = Budget(account,
target,
amount,
start_date,
repeat_type=info.repeat_type)
budget.notes = nonone(info.notes, '')
budget.stop_date = info.stop_date
if info.repeat_every:
budget.repeat_every = info.repeat_every
self.budgets.append(budget)
self._post_load()
self.oven.cook(datetime.date.min, until_date=None)
Currency.get_rates_db().ensure_rates(start_date,
[x.code for x in currencies])
