def do(self, *args, **options): # pylint: disable=too-many-branches
files = list_dir_files(options["path"])
# pprint(files)
for filename in files:
plain_filename = os.path.basename(filename)
if parse_filename_suffix(
plain_filename).upper() not in SVM_STATEMENT_SUFFIXES:
print("Ignoring non-SVM file {}".format(filename))
continue
if options["resolve_original_filenames"]:
found = ReferencePaymentBatchFile.objects.filter(
referencepaymentbatch__name=plain_filename).first()
if found and not found.original_filename:
assert isinstance(found, ReferencePaymentBatchFile)
found.original_filename = filename
found.save(update_fields=["original_filename"])
logger.info(
"Original SVM reference payment batch filename of %s resolved to %s",
found, filename)
if options["delete_old"]:
ReferencePaymentBatch.objects.filter(
name=plain_filename).delete()
if options["test"]:
batches = parse_svm_batches_from_file(filename)
pprint(batches)
continue
if not ReferencePaymentBatch.objects.filter(
name=plain_filename).first():
print("Importing statement file {}".format(filename))
batches = parse_svm_batches_from_file(filename)
if options["verbose"]:
pprint(batches)
with transaction.atomic():
file = ReferencePaymentBatchFile(
original_filename=filename, tag=options["tag"])
file.save()
save_or_store_media(file.file, filename)
file.save()
for data in batches:
if options["auto_create_accounts"]:
for rec_data in data["records"]:
account_number = rec_data.get("account_number")
if account_number:
get_or_create_bank_account(account_number)
create_reference_payment_batch(data,
name=plain_filename,
file=file) # pytype: disable=not-callable
file.get_total_amount(force=True)
else:
print("Skipping reference payment file {}".format(filename))
def parse_tiliote_statements_from_file(filename: str) -> list:
if parse_filename_suffix(filename).upper() not in TO_STATEMENT_SUFFIXES:
raise ValidationError(
_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"'
).format(filename=filename,
suffixes=", ".join(TO_STATEMENT_SUFFIXES),
file_type="tiliote"))
with open(filename, "rt", encoding="ISO-8859-1") as fp:
return parse_tiliote_statements(
fp.read(), filename=basename(filename)) # type: ignore
def parse_aeb43_statements_from_file(filename: str) -> list:
if parse_filename_suffix(filename).upper() not in AEB43_STATEMENT_SUFFIXES:
raise ValidationError(
_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"'
).format(filename=filename,
suffixes=", ".join(AEB43_STATEMENT_SUFFIXES),
file_type="AEB43"))
with open(filename, "rt", encoding="UTF-8") as fp:
return parse_aeb43_statements(
fp.read(), filename=os.path.basename(filename)) # type: ignore
def do(self, *args, **kwargs):
files = list_dir_files(kwargs["path"])
for filename in files:
plain_filename = os.path.basename(filename)
if parse_filename_suffix(plain_filename).upper() not in AEB43_STATEMENT_SUFFIXES:
print("Ignoring non-AEB43 file {}".format(filename))
continue
batches = parse_aeb43_statements_from_file(filename)
pprint(batches)
def camt053_parse_statement_from_file(filename: str) -> dict:
if parse_filename_suffix(
filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
raise ValidationError(
_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"'
).format(filename=filename,
suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES),
file_type="camt.053"))
with open(filename, "rb") as fp:
data = xml_to_dict(fp.read(),
array_tags=CAMT053_ARRAY_TAGS,
int_tags=CAMT053_INT_TAGS)
return data
def do(self, *args, **options): # pylint: disable=too-many-branches
if options["parse_creditor_account_data"]:
self.parse_creditor_account_data()
return
files = list_dir_files(options["path"], options["suffix"])
for filename in files:
plain_filename = os.path.basename(filename)
if parse_filename_suffix(plain_filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
print("Ignoring non-CAMT53 file {}".format(filename))
continue
if options["resolve_original_filenames"]:
found = StatementFile.objects.filter(statement__name=plain_filename).first()
if found and not found.original_filename:
assert isinstance(found, StatementFile)
found.original_filename = filename
found.save(update_fields=["original_filename"])
logger.info("Original XML statement filename of %s resolved to %s", found, filename)
if options["test"]:
statement = camt053_parse_statement_from_file(filename)
pprint(statement)
continue
if options["delete_old"]:
Statement.objects.filter(name=plain_filename).delete()
if not Statement.objects.filter(name=plain_filename).first():
print("Importing statement file {}".format(plain_filename))
statement = camt053_parse_statement_from_file(filename)
if options["verbose"]:
pprint(statement)
with transaction.atomic():
file = StatementFile(original_filename=filename, tag=options["tag"])
file.save()
save_or_store_media(file.file, filename)
file.save()
for data in [statement]:
if options["auto_create_accounts"]:
account_number = camt053_get_iban(data)
if account_number:
get_or_create_bank_account(account_number)
camt053_create_statement(data, name=plain_filename, file=file) # pytype: disable=not-callable
else:
print("Skipping statement file {}".format(filename))
def parse_creditor_account_data(self): # pylint: disable=too-many-locals,too-many-branches
for sf in StatementFile.objects.all(): # pylint: disable=too-many-nested-blocks
assert isinstance(sf, StatementFile)
full_path = sf.full_path
if os.path.isfile(full_path) and parse_filename_suffix(full_path).upper() in CAMT053_STATEMENT_SUFFIXES:
logger.info("Parsing creditor account data of %s", full_path)
statement_data = camt053_parse_statement_from_file(full_path)
d_stmt = statement_data.get("BkToCstmrStmt", {}).get("Stmt", {})
d_ntry = d_stmt.get("Ntry", [])
recs = list(StatementRecord.objects.all().filter(statement__file=sf).order_by("id"))
if len(recs) != len(d_ntry):
raise ValidationError(f"Statement record counts do not match in id={sf.id} ({sf})")
for ix, ntry in enumerate(d_ntry):
rec = recs[ix]
assert isinstance(rec, StatementRecord)
for dtl_batch in ntry.get("NtryDtls", []):
rec_detail_list = list(StatementRecordDetail.objects.all().filter(record=rec))
if len(rec_detail_list) != len(dtl_batch.get("TxDtls", [])):
raise ValidationError(f"Statement record detail counts do not match in id={sf.id} ({sf})")
for dtl_ix, dtl in enumerate(dtl_batch.get("TxDtls", [])):
d = rec_detail_list[dtl_ix]
assert isinstance(d, StatementRecordDetail)
d_parties = dtl.get("RltdPties", {})
d_dbt = d_parties.get("Dbtr", {})
d.debtor_name = d_dbt.get("Nm", "")
d_udbt = d_parties.get("UltmtDbtr", {})
d.ultimate_debtor_name = d_udbt.get("Nm", "")
d_cdtr = d_parties.get("Cdtr", {})
d.creditor_name = d_cdtr.get("Nm", "")
d_cdtr_acct = d_parties.get("CdtrAcct", {})
d_cdtr_acct_id = d_cdtr_acct.get("Id", {})
d.creditor_account = d_cdtr_acct_id.get("IBAN", "")
if d.creditor_account:
d.creditor_account_scheme = "IBAN"
else:
d_cdtr_acct_id_othr = d_cdtr_acct_id.get("Othr") or {}
d.creditor_account_scheme = d_cdtr_acct_id_othr.get("SchmeNm", {}).get("Cd", "")
d.creditor_account = d_cdtr_acct_id_othr.get("Id") or ""
logger.info("%s creditor_account %s (%s)", rec, d.creditor_account, d.creditor_account_scheme)
d.save()
if not rec.recipient_account_number:
rec.recipient_account_number = camt053_get_unified_str(rec.detail_set.all(), "creditor_account")
if rec.recipient_account_number:
rec.save(update_fields=["recipient_account_number"])
logger.info("%s recipient_account_number %s", rec, rec.recipient_account_number)