def test_marks_incomplete_sender_information(self):
with open("tests/data/testfile_sender_information") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 4)
# check that account number is 0s instead of None for matching building society
self.assertEqual(transactions[0]["category"], "credit")
self.assertEqual(transactions[0]["sender_account_number"], "00000000")
self.assertFalse(transactions[0]["incomplete_sender_info"])
# no sort code
self.assertEqual(transactions[1]["sender_sort_code"], None)
self.assertTrue(transactions[1]["incomplete_sender_info"])
self.assertTrue(transactions[1]["blocked"])
# no account number
self.assertEqual(transactions[2]["sender_account_number"], None)
self.assertTrue(transactions[2]["incomplete_sender_info"])
self.assertTrue(transactions[2]["blocked"])
# no roll number
self.assertEqual(transactions[3]["sender_roll_number"], None)
self.assertTrue(transactions[3]["incomplete_sender_info"])
self.assertFalse(transactions[3]["blocked"])
def test_populates_roll_numbers_when_relevant_sort_codes_found(self):
with open("tests/data/testfile_roll_number") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 7)
self.assertEqual(transactions[0]["category"], "credit")
self.assertEqual(transactions[0]["sender_roll_number"], "123A 123456A")
self.assertEqual(transactions[1]["category"], "credit")
self.assertEqual(transactions[1]["sender_roll_number"], "A12345678SMI")
self.assertEqual(transactions[2]["category"], "credit")
self.assertEqual(transactions[2]["sender_roll_number"], "1234/12345678")
self.assertEqual(transactions[3]["category"], "credit")
self.assertEqual(transactions[3]["sender_roll_number"], "12-123456-12345")
self.assertEqual(transactions[4]["category"], "credit")
self.assertEqual(transactions[4]["sender_roll_number"], "1234567890")
self.assertEqual(transactions[5]["category"], "debit")
self.assertEqual(transactions[5]["sender_roll_number"], "1234567890")
self.assertEqual(transactions[6]["category"], "credit")
self.assertEqual(transactions[6]["sender_roll_number"], "A12345678")
def testfile_settlement_credits(self, mock_get_conn):
with open('tests/data/testfile_settlement_credits') as f:
# records are from 36th date of 2004, i.e. 2004-02-05 (see last 5 digits in each record)
data_services_file = parse(f)
conn = mock_get_conn()
conn.batches.get.return_value = {'count': 1, 'results': [{'id': 10}]}
transactions = upload.get_transactions_from_file(data_services_file)
# test file has 3 settlement transactions which are all "administrative" credits
self.assertEqual(len(transactions), 3)
self.assertTrue(
all(transaction['category'] == 'credit'
and transaction['source'] == 'administrative'
for transaction in transactions))
# one settlement does not have a date that can be parsed so is not matched to a batch
self.assertNotIn('batch', transactions[0])
# two settlements have a date that can be parsed and matched to a batch
self.assertEqual(len(conn.batches.get.call_args_list), 2)
# first settlement is for ?-09-22 (assumed to be nearest date in the past)
self.assertEqual(transactions[1]['batch'], 10)
self.assertEqual(conn.batches.get.call_args_list[0],
((), {
'date': '2003-09-22'
}))
# second settlement is for ?-?-21 (assumed to be nearest date in the past)
self.assertEqual(transactions[2]['batch'], 10)
self.assertEqual(conn.batches.get.call_args_list[1],
((), {
'date': '2004-01-21'
}))
def test_marking_all_credit_transactions_as_unidentified(
self, mock_settings):
setup_settings(mock_settings, mark_transactions_as_unidentified=True)
with open('tests/data/testfile_1') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 3)
# transaction 0 - debit
self.assertEqual(transactions[0]['category'], 'debit')
self.assertEqual(transactions[0]['source'], 'administrative')
# transaction 1 - credit
self.assertEqual(transactions[1]['category'], 'credit')
self.assertEqual(transactions[1]['source'], 'bank_transfer')
self.assertEqual(transactions[1]['sender_account_number'], '29696666')
self.assertEqual(transactions[1]['sender_sort_code'], '608006')
self.assertEqual(transactions[1]['prisoner_number'], 'A1234BY')
self.assertEqual(transactions[1]['prisoner_dob'], '1986-12-09')
self.assertEqual(transactions[1]['blocked'], True)
self.assertEqual(transactions[1]['incomplete_sender_info'], True)
# transaction 2 - credit
self.assertEqual(transactions[2]['category'], 'credit')
self.assertEqual(transactions[2]['source'], 'bank_transfer')
self.assertEqual(transactions[2]['sender_account_number'], '78990056')
self.assertEqual(transactions[2]['sender_sort_code'], '245432')
self.assertEqual(transactions[2]['prisoner_number'], 'B4321XZ')
self.assertEqual(transactions[2]['prisoner_dob'], '1992-11-08')
self.assertEqual(transactions[2]['blocked'], True)
self.assertEqual(transactions[2]['incomplete_sender_info'], True)
def upload_transactions_from_files(files):
conn = get_authenticated_connection()
successful_transaction_count = 0
for filename in files:
logger.info('Processing %s...' % filename)
with open(filename) as f:
data_services_file = parse(f)
transactions = get_transactions_from_file(data_services_file)
if transactions:
transaction_count = len(transactions)
try:
for i in range(math.ceil(transaction_count / settings.UPLOAD_REQUEST_SIZE)):
conn.transactions.post(
clean_request_data(transactions[
i * settings.UPLOAD_REQUEST_SIZE:
(i + 1) * settings.UPLOAD_REQUEST_SIZE
])
)
stmt_date = parse_filename(filename, settings.ACCOUNT_CODE)
update_new_balance(transactions, stmt_date)
logger.info('Uploaded %d transactions from %s' % (transaction_count, filename))
successful_transaction_count += transaction_count
except SlumberHttpBaseException as e:
logger.error('Failed to upload %d transactions from %s.\n%s' % (
transaction_count,
filename,
getattr(e, 'content', e)
))
return successful_transaction_count
def upload_transactions_from_files(files):
conn = get_authenticated_connection()
successful_transaction_count = 0
for filename in files:
logger.info("Processing %s..." % filename)
with open(filename) as f:
data_services_file = parse(f)
transactions = get_transactions_from_file(data_services_file)
if transactions:
transaction_count = len(transactions)
try:
for i in range(math.ceil(transaction_count / settings.UPLOAD_REQUEST_SIZE)):
conn.transactions.post(
clean_request_data(
transactions[i * settings.UPLOAD_REQUEST_SIZE : (i + 1) * settings.UPLOAD_REQUEST_SIZE]
)
)
stmt_date = parse_filename(filename, settings.ACCOUNT_CODE).date()
update_new_balance(transactions, stmt_date)
logger.info("Uploaded %d transactions from %s" % (transaction_count, filename))
successful_transaction_count += transaction_count
except SlumberHttpBaseException as e:
logger.error(
"Failed to upload %d transactions from %s.\n%s"
% (transaction_count, filename, getattr(e, "content", e))
)
return successful_transaction_count
def test_marks_incomplete_sender_information(self, mock_settings):
setup_settings(mock_settings)
with open('tests/data/testfile_sender_information') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 4)
# check that account number is 0s instead of None for matching building society
self.assertEqual(transactions[0]['category'], 'credit')
self.assertEqual(transactions[0]['sender_account_number'], '00000000')
self.assertFalse(transactions[0]['incomplete_sender_info'])
# no sort code
self.assertEqual(transactions[1]['sender_sort_code'], None)
self.assertTrue(transactions[1]['incomplete_sender_info'])
self.assertTrue(transactions[1]['blocked'])
# no account number
self.assertEqual(transactions[2]['sender_account_number'], None)
self.assertTrue(transactions[2]['incomplete_sender_info'])
self.assertTrue(transactions[2]['blocked'])
# no roll number
self.assertEqual(transactions[3]['sender_roll_number'], None)
self.assertTrue(transactions[3]['incomplete_sender_info'])
self.assertFalse(transactions[3]['blocked'])
def test_does_not_populate_roll_number_if_not_matching_format(self):
with open('tests/data/testfile_bs_sort_code_invalid_roll_number') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 1)
self.assertEqual(transactions[0]['sender_roll_number'], None)
def test_populates_roll_numbers_when_relevant_sort_codes_found(self):
with open('tests/data/testfile_roll_number') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 7)
self.assertEqual(transactions[0]['category'], 'credit')
self.assertEqual(transactions[0]['sender_roll_number'], '123A 123456A')
self.assertEqual(transactions[1]['category'], 'credit')
self.assertEqual(transactions[1]['sender_roll_number'], 'A12345678SMI')
self.assertEqual(transactions[2]['category'], 'credit')
self.assertEqual(transactions[2]['sender_roll_number'],
'1234/12345678')
self.assertEqual(transactions[3]['category'], 'credit')
self.assertEqual(transactions[3]['sender_roll_number'],
'12-123456-12345')
self.assertEqual(transactions[4]['category'], 'credit')
self.assertEqual(transactions[4]['sender_roll_number'], '1234567890')
self.assertEqual(transactions[5]['category'], 'debit')
self.assertEqual(transactions[5]['sender_roll_number'], '1234567890')
self.assertEqual(transactions[6]['category'], 'credit')
self.assertEqual(transactions[6]['sender_roll_number'], 'A12345678')
def test_does_not_populate_roll_number_if_not_matching_format(self):
with open("tests/data/testfile_bs_sort_code_invalid_roll_number") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 1)
self.assertEqual(transactions[0]["sender_roll_number"], None)
def test_get_transactions_no_records(self, mock_logger):
with open('tests/data/testfile_no_records') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
mock_logger.info.assert_called_with('No records found.')
self.assertEqual(transactions, None)
def test_get_transactions_no_records(self, mock_logger):
with open("tests/data/testfile_no_records") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
mock_logger.info.assert_called_with("No records found.")
self.assertEqual(transactions, None)
def test_does_not_populate_roll_number_typically(self):
with open("tests/data/testfile_1") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 3)
self.assertEqual(transactions[0]["sender_roll_number"], None)
self.assertEqual(transactions[1]["sender_roll_number"], None)
self.assertEqual(transactions[2]["sender_roll_number"], None)
def test_marks_incomplete_sender_information_for_metro_bank(self):
with open("tests/data/testfile_metro_bank") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 4)
self.assertEqual(transactions[3]["sender_roll_number"], None)
self.assertTrue(transactions[3]["incomplete_sender_info"])
self.assertFalse(transactions[3]["blocked"])
def test_does_not_populate_roll_number_typically(self):
with open('tests/data/testfile_1') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 3)
self.assertEqual(transactions[0]['sender_roll_number'], None)
self.assertEqual(transactions[1]['sender_roll_number'], None)
self.assertEqual(transactions[2]['sender_roll_number'], None)
def test_get_transactions_incorrect_totals(self, mock_logger):
with open('tests/data/testfile_incorrect_totals') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
mock_logger.error.assert_called_with(
"Errors: {'account 0': ['Monetary total of debit items does not "
"match expected: counted 288615, expected 288610', "
"'Monetary total of credit items does not match expected: "
"counted 18741, expected 18732']}")
self.assertEqual(transactions, None)
def test_get_transactions(self, mock_settings):
setup_settings(mock_settings)
with open('tests/data/testfile_1') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 3)
# transaction 0 - debit
self.assertEqual(transactions[0]['category'], 'debit')
self.assertEqual(transactions[0]['source'], 'administrative')
self.assertEqual(transactions[0]['amount'], 288615)
self.assertEqual(transactions[0]['received_at'],
'2004-02-05T12:00:00+00:00')
self.assertEqual(transactions[0]['processor_type_code'], '03')
self.assertEqual(transactions[0]['reference'], 'Payment refund ')
self.assertEqual(transactions[0].get('prisoner_number'), None)
self.assertEqual(transactions[0].get('prisoner_dob'), None)
# transaction 1 - credit
self.assertEqual(transactions[1]['category'], 'credit')
self.assertEqual(transactions[1]['source'], 'bank_transfer')
self.assertEqual(transactions[1]['amount'], 8939)
self.assertEqual(transactions[1]['received_at'],
'2004-02-05T12:00:00+00:00')
self.assertEqual(transactions[1]['sender_account_number'], '29696666')
self.assertEqual(transactions[1]['sender_sort_code'], '608006')
self.assertEqual(transactions[1]['processor_type_code'], '99')
self.assertEqual(transactions[1]['prisoner_number'], 'A1234BY')
self.assertEqual(transactions[1]['prisoner_dob'], '1986-12-09')
self.assertEqual(transactions[1]['blocked'], False)
self.assertEqual(transactions[1]['incomplete_sender_info'], False)
# transaction 2 - credit
self.assertEqual(transactions[2]['category'], 'credit')
self.assertEqual(transactions[2]['source'], 'bank_transfer')
self.assertEqual(transactions[2]['amount'], 9802)
self.assertEqual(transactions[2]['received_at'],
'2004-02-05T12:00:00+00:00')
self.assertEqual(transactions[2]['sender_account_number'], '78990056')
self.assertEqual(transactions[2]['sender_sort_code'], '245432')
self.assertEqual(transactions[2]['processor_type_code'], '93')
self.assertEqual(transactions[2]['prisoner_number'], 'B4321XZ')
self.assertEqual(transactions[2]['prisoner_dob'], '1992-11-08')
self.assertEqual(transactions[2]['blocked'], False)
self.assertEqual(transactions[2]['incomplete_sender_info'], False)
def test_rbs_testfile_successful_parse(self):
with open('tests/data/testfile_rbs') as f:
output = parse(f)
self.assertTrue(output.is_valid())
self.assertEqual(len(output.accounts), 1)
self.assertEqual(len(output.account.records), 25)
self.assertEqual(
output.account.user_trailer_label.monetary_total_debit_items,
554186444)
self.assertEqual(
output.account.user_trailer_label.monetary_total_credit_items,
666903844)
def test_nwb_testfile_successful_parse(self):
with open('tests/data/testfile_nwb') as f:
output = parse(f)
self.assertTrue(output.is_valid())
self.assertEqual(len(output.accounts), 1)
self.assertEqual(len(output.account.records), 29)
self.assertEqual(
output.account.user_trailer_label.monetary_total_debit_items,
278225388)
self.assertEqual(
output.account.user_trailer_label.monetary_total_credit_items,
105598692)
def test_get_transactions_incorrect_totals(self, mock_logger):
with open("tests/data/testfile_incorrect_totals") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
mock_logger.error.assert_called_with(
"Errors: {'account 0': ['Monetary total of debit items does not "
"match expected: counted 288615, expected 288610', "
"'Monetary total of credit items does not match expected: "
"counted 18741, expected 18732']}"
)
self.assertEqual(transactions, None)
def test_marks_incomplete_sender_information_for_metro_bank(
self, mock_settings):
setup_settings(mock_settings)
with open('tests/data/testfile_metro_bank') as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 4)
self.assertEqual(transactions[3]['sender_roll_number'], None)
self.assertTrue(transactions[3]['incomplete_sender_info'])
self.assertFalse(transactions[3]['blocked'])
def test_rbs_testfile_successful_parse(self):
with open('tests/data/testfile_rbs') as f:
output = parse(f)
self.assertTrue(output.is_valid())
self.assertEqual(len(output.accounts), 1)
self.assertEqual(len(output.account.records), 25)
self.assertEqual(
output.account.user_trailer_label.monetary_total_debit_items,
554186444
)
self.assertEqual(
output.account.user_trailer_label.monetary_total_credit_items,
666903844
)
def test_nwb_testfile_successful_parse(self):
with open('tests/data/testfile_nwb') as f:
output = parse(f)
self.assertTrue(output.is_valid())
self.assertEqual(len(output.accounts), 1)
self.assertEqual(len(output.account.records), 29)
self.assertEqual(
output.account.user_trailer_label.monetary_total_debit_items,
278225388
)
self.assertEqual(
output.account.user_trailer_label.monetary_total_credit_items,
105598692
)
def test_marks_administrative_transactions(self, mock_get_conn):
with open("tests/data/testfile_administrative_credits") as f:
data_services_file = parse(f)
conn = mock_get_conn()
conn.batches.get.return_value = {"count": 1, "results": [{"id": 10}]}
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(transactions[0]["category"], "debit")
self.assertEqual(transactions[0]["source"], "administrative")
self.assertTrue("batch" not in transactions[0])
self.assertEqual(transactions[1]["category"], "credit")
self.assertEqual(transactions[1]["source"], "administrative")
self.assertTrue("batch" not in transactions[1])
self.assertEqual(transactions[2]["category"], "credit")
self.assertEqual(transactions[2]["source"], "administrative")
self.assertEqual(transactions[2]["batch"], 10)
def test_get_transactions(self):
with open("tests/data/testfile_1") as f:
data_services_file = parse(f)
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 3)
# transaction 0 - debit
self.assertEqual(transactions[0]["category"], "debit")
self.assertEqual(transactions[0]["source"], "administrative")
self.assertEqual(transactions[0]["amount"], 288615)
self.assertEqual(transactions[0]["received_at"], "2004-02-05T12:00:00+00:00")
self.assertEqual(transactions[0]["processor_type_code"], "03")
self.assertEqual(transactions[0]["reference"], "Payment refund ")
self.assertEqual(transactions[0].get("prisoner_number"), None)
self.assertEqual(transactions[0].get("prisoner_dob"), None)
# transaction 1 - credit
self.assertEqual(transactions[1]["category"], "credit")
self.assertEqual(transactions[1]["source"], "bank_transfer")
self.assertEqual(transactions[1]["amount"], 8939)
self.assertEqual(transactions[1]["received_at"], "2004-02-05T12:00:00+00:00")
self.assertEqual(transactions[1]["sender_account_number"], "29696666")
self.assertEqual(transactions[1]["sender_sort_code"], "608006")
self.assertEqual(transactions[1]["processor_type_code"], "99")
self.assertEqual(transactions[1]["prisoner_number"], "A1234BY")
self.assertEqual(transactions[1]["prisoner_dob"], "1986-12-09")
# transaction 2 - credit
self.assertEqual(transactions[2]["category"], "credit")
self.assertEqual(transactions[2]["source"], "bank_transfer")
self.assertEqual(transactions[2]["amount"], 9802)
self.assertEqual(transactions[2]["received_at"], "2004-02-05T12:00:00+00:00")
self.assertEqual(transactions[2]["sender_account_number"], "78990056")
self.assertEqual(transactions[2]["sender_sort_code"], "245432")
self.assertEqual(transactions[2]["processor_type_code"], "93")
self.assertEqual(transactions[2]["prisoner_number"], "B4321XZ")
self.assertEqual(transactions[2]["prisoner_dob"], "1992-11-08")
def test_not_marking_administrative_credits_as_unidentified(
self, mock_settings, mock_get_conn):
setup_settings(mock_settings, mark_transactions_as_unidentified=True)
with open('tests/data/testfile_administrative_credits') as f:
data_services_file = parse(f)
conn = mock_get_conn()
conn.batches.get.return_value = {'count': 1, 'results': [{'id': 10}]}
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 3)
self.assertEqual(transactions[0]['category'], 'debit')
self.assertEqual(transactions[0]['source'], 'administrative')
self.assertEqual(transactions[1]['category'], 'credit')
self.assertEqual(transactions[1]['source'], 'administrative')
self.assertEqual(transactions[1]['blocked'], False)
self.assertEqual(transactions[1]['incomplete_sender_info'], False)
self.assertEqual(transactions[2]['category'], 'credit')
self.assertEqual(transactions[2]['source'], 'administrative')
self.assertEqual(transactions[2]['blocked'], False)
self.assertEqual(transactions[2]['incomplete_sender_info'], False)
def test_marks_administrative_transactions(self, mock_get_conn):
with open('tests/data/testfile_administrative_credits') as f:
# records are from 36th date of 2004, i.e. 2004-02-05 (see last 5 digits in each record)
data_services_file = parse(f)
conn = mock_get_conn()
conn.batches.get.return_value = {'count': 1, 'results': [{'id': 10}]}
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(transactions[0]['category'], 'debit')
self.assertEqual(transactions[0]['source'], 'administrative')
self.assertTrue('batch' not in transactions[0])
self.assertEqual(transactions[1]['category'], 'credit')
self.assertEqual(transactions[1]['source'], 'administrative')
self.assertTrue('batch' not in transactions[1])
self.assertEqual(transactions[2]['category'], 'credit')
self.assertEqual(transactions[2]['source'], 'administrative')
self.assertEqual(transactions[2]['batch'], 10)
# settlement is for ?-09-22 (assumed to be nearest date in the past)
conn.batches.get.assert_called_with(date='2003-09-22')
def test_excludes_records_from_other_accounts(self, mock_get_conn):
with open('tests/data/testfile_multiple_accounts') as f:
data_services_file = parse(f)
conn = mock_get_conn()
conn.batches.get.return_value = {'count': 1, 'results': [{'id': 10}]}
transactions = upload.get_transactions_from_file(data_services_file)
self.assertEqual(len(transactions), 2)
# transaction 0 - debit
self.assertEqual(transactions[0]['category'], 'debit')
self.assertEqual(transactions[0]['source'], 'administrative')
self.assertEqual(transactions[0]['amount'], 288615)
self.assertEqual(transactions[0]['received_at'],
'2004-02-05T12:00:00+00:00')
# transaction 1 - credit (actually credit record 2 but credit 1 is filtered out)
self.assertEqual(transactions[1]['category'], 'credit')
self.assertEqual(transactions[1]['source'], 'bank_transfer')
self.assertEqual(transactions[1]['amount'], 9802)
self.assertEqual(transactions[1]['received_at'],
'2004-02-07T12:00:00+00:00')
def test_no_accounts_fails(self):
try:
parse([vhl_row])
self.fail()
except ParseError:
pass
def test_empty_file_fails(self):
try:
parse([])
self.fail()
except ParseError:
pass
def test_no_accounts_fails(self):
try:
parse([vhl_row])
self.fail()
except ParseError:
pass
def test_empty_file_fails(self):
try:
parse([])
self.fail()
except ParseError:
pass
