class LoadDailyTestCase(TestCase):
def create_test_files(self, data):
""" Create files in test_folder for tests."""
for key in data:
f = file(data[key]['file'], 'w+')
f.write('So the size is not 0')
f.close()
def create_product(self, row):
""" Helper function to save a product."""
p = Product()
p.plan_id = int(row[0])
p.institution = row[1]
p.loan_purpose = row[2]
p.pmt_type = row[3]
p.loan_type = row[4]
p.loan_term = int(row[5])
p.int_adj_term = self.c.nullable_int(row[6])
p.adj_period = self.c.nullable_int(row[7])
p.io = self.c.string_to_boolean(row[8])
p.arm_index = self.c.nullable_string(row[9])
p.int_adj_cap = self.c.nullable_int(row[10])
p.annual_cap = self.c.nullable_int(row[11])
p.loan_cap = self.c.nullable_int(row[12])
p.arm_margin = self.c.nullable_decimal(row[13])
p.ai_value = self.c.nullable_decimal(row[14])
p.min_ltv = float(row[15])
p.max_ltv = float(row[16])
p.min_fico = int(row[17])
p.max_fico = int(row[18])
p.min_loan_amt = Decimal(row[19])
p.max_loan_amt = Decimal(row[20])
p.data_timestamp = datetime.strptime('20140101', '%Y%m%d')
p.save()
return p
def setUp(self):
self.c = Command()
self.test_dir = 'ratechecker/tests/test_folder'
self.dummyargs = {
'product': {'date': '20140101', 'file': '20140101_product.txt'},
'adjustment': {'date': '20140101', 'file': '20140101_adjustment.txt'},
'rate': {'date': '20140101', 'file': '20140101_rate.txt'},
'region': {'date': '20140101', 'file': '20140101_region.txt'},
}
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
def tearDown(self):
""" Delete the test_folder dir."""
os.chdir('../../..')
path = os.path.join(os.getcwd(), self.test_dir)
shutil.rmtree(path)
def test_string_to_boolean(self):
b = self.c.string_to_boolean('abc')
self.assertEqual(b, None)
b = self.c.string_to_boolean('False')
self.assertFalse(b)
b = self.c.string_to_boolean('True')
self.assertTrue(True)
def test_nullable_int(self):
self.assertEqual(self.c.nullable_int('10'), 10)
self.assertEqual(self.c.nullable_int('10.0'), 10)
self.assertEqual(self.c.nullable_int(''), None)
self.assertEqual(self.c.nullable_int(' '), None)
def test_nullable_string(self):
self.assertEqual(self.c.nullable_string('BANK'), 'BANK')
self.assertEqual(self.c.nullable_string(' '), None)
self.assertEqual(self.c.nullable_string(''), None)
def test_nullable_decimal(self):
self.assertEqual(self.c.nullable_decimal(''), None)
self.assertEqual(self.c.nullable_decimal(' '), None)
self.assertEqual(self.c.nullable_decimal('12.5'), Decimal('12.5'))
def test_nullable_float(self):
self.assertEqual(self.c.nullable_float(''), None)
self.assertEqual(self.c.nullable_float(' '), None)
self.assertEqual(self.c.nullable_float('12.5'), 12.5)
def test_create_rate(self):
#rate_id, product_id, region_id, lock, base_rate, total_points
row = ['1', '12', '200', '40', '3.125', '3']
now = datetime.now()
r = self.c.create_rate(row, now)
self.assertEqual(r.rate_id, 1)
self.assertEqual(r.product_id, 12)
self.assertEqual(r.lock, 40)
self.assertEqual(r.base_rate, Decimal('3.125'))
self.assertEqual(r.data_timestamp, now)
self.assertEqual(r.region_id, 200)
def test_delete_temp_tables(self):
""" ... some exist, others - not."""
# don't know how to easily test table (in)existence
cursor = connection.cursor()
cursor.execute('CREATE TABLE temporary_product(a TEXT)')
self.assertRaises(OperationalError, cursor.execute, 'SELECT * FROM temporary_region')
empty = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(empty is not None)
self.c.delete_temp_tables(cursor)
self.assertRaises(OperationalError, cursor.execute, 'SELECT * FROM temporary_product')
def test_reload_old_data(self):
""" .. archive_data_to_temp_tables, delete_temp_tables, delete_data_from_base_tables """
cursor = connection.cursor()
row = [
'5999', 'SMPL', 'PURCH', 'ARM', 'JUMBO', '30', '7.0', '1',
'False', 'LIBOR', '5.0000', '2.0000', '5.0000', '2.5000',
'.5532', '1', '90', '620', '850', '417001', '2000000', 1, 1, 0
]
# Original product object
op = self.create_product(row)
self.c.archive_data_to_temp_tables(cursor)
result = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(len(result.fetchall()) == 1)
result = Product.objects.all()
self.assertTrue(result)
self.c.delete_data_from_base_tables()
result = Product.objects.all()
self.assertFalse(result)
self.c.reload_old_data(cursor)
result = Product.objects.all()
self.assertEqual(len(result), 1)
self.assertTrue(op == result[0])
#TODO: add checks for other situations
def test_handle__no_folder(self):
""" .. check that some issues are caught."""
self.c.handle()
self.assertEqual(1, self.c.status)
self.assertTrue('Error: tuple index out of range. Has a source directory been provided?'
in self.c.messages)
self.assertFalse('Warning: reloading "yesterday" data.' in self.c.messages)
def test_handle__bad_folder(self):
""" .. check that some issues are caught."""
# inexistent folder, inaccessible folder or a file all are caught in the same place
with open('not_a_folder', 'w') as fake_folder:
fake_folder.write('I am not a folder')
self.c.handle('not_a_folder')
self.assertEqual(1, self.c.status)
def test_handle__bad_zipfile(self):
""" .. check that some issues are caught."""
with open('20140101.zip', 'w') as fake_zip_archive:
fake_zip_archive.write('Some text')
self.c.handle('.')
self.assertEqual(self.c.status, 1)
self.assertTrue(' Warning: File is not a zip file.' in self.c.messages)
self.assertTrue('Warning: reloading "yesterday" data' in self.c.messages)
def test_arch_list(self):
""" .. only the correct filenames are returned and are sorted."""
self.create_test_files(self.dummyargs)
arch_name = '%s.zip' % self.dummyargs['rate']['date']
zfile = zipfile.ZipFile(arch_name, 'w')
for key in self.dummyargs:
zfile.write(self.dummyargs[key]['file'])
zfile.close()
shutil.copy(arch_name, '20130202.zip')
shutil.copy(arch_name, '20150202.zip')
result = self.c.arch_list('.')
self.assertEqual(len(result), 3)
self.assertTrue('20150202' in result[0])
self.assertTrue(arch_name in result[1])
self.assertTrue('20130202' in result[2])
def test_load_arch_data(self):
""" .. check all load_xxx_data functions here."""
zfile = self.prepare_sample_data()
result = self.c.load_arch_data(zfile)
products = Product.objects.all()
self.assertEqual(len(products), 3)
self.assertTrue(products[1].institution, 'SMPL1')
self.assertTrue(products[0].institution, 'SMPL')
self.assertTrue(products[2].institution, 'SMPL2')
self.assertTrue(products[1].loan_purpose, 'REFI')
self.assertTrue(products[1].pmt_type, 'FIXED')
#TODO: add other checks
adjustments = Adjustment.objects.all()
self.assertEqual(len(adjustments), 4)
rates = Rate.objects.all()
self.assertEqual(len(rates), 7)
regions = Region.objects.all()
self.assertEqual(len(regions), 4)
def prepare_sample_data(self):
""" solely for test_load_arch_data."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
filename = '%s_product.txt' % date
with open(filename, 'w') as prdata:
prdata.write("Is skipped anyway\n")
prdata.write("7487\tSMPL\tPURCH\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n")
prdata.write("7488\tSMPL1\tREFI\tFIXED\tCONF\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n")
prdata.write("7489\tSMPL2\tREFI\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n")
filename = filename.replace('_product', '_adjustment')
with open(filename, 'w') as adjdata:
adjdata.write("Is skipped anyway\n")
adjdata.write("7487\t67600\tP\t-0.25\t\t\t\t720\t739\t1\t60\t\n")
adjdata.write("7488\t73779\tP\t0\t850001.000\t1000000.000\t\t\t\t\t\t\n")
adjdata.write("7489\t68040\tP\t3.25\t\t\t\t620\t639\t85.010000000000005\t95\t\n")
adjdata.write("7489\t67996\tP\t1.5\t\t\t\t620\t639\t60.009999999999998\t70\t\n")
filename = filename.replace('_adjustment', '_rate')
with open(filename, 'w') as rtdata:
rtdata.write("Is skipped\n")
rtdata.write("592005597\t275387\t332\t60\t4.000\t-.375\n")
rtdata.write("592005635\t278474\t332\t30\t2.250\t1.250\n")
rtdata.write("592005636\t278474\t332\t30\t2.375\t1.000\n")
rtdata.write("592005637\t278474\t332\t30\t2.500\t.750\n")
rtdata.write("592005638\t278474\t332\t30\t2.625\t.500\n")
rtdata.write("592005639\t278474\t332\t30\t2.750\t.250\n")
rtdata.write("592005640\t278474\t332\t30\t2.875\t.000\n")
filename = filename.replace('_rate', '_region')
with open(filename, 'w') as regdata:
regdata.write("Is skipped\n")
regdata.write("12\tAK\tTrue\n")
regdata.write("12\tAL\tFalse\n")
regdata.write("12\tAR\tFalse\n")
regdata.write("12\tAZ\tFalse\n")
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region']:
zfile.write('%s_%s.txt' % (date, key))
zfile.close()
return zipfile.ZipFile(arch_name, 'r')
class LoadDailyTestCase(TestCase):
def create_test_files(self, data):
""" Create files in test_folder for tests."""
for key in data:
f = file(data[key]['file'], 'w+')
f.write('So the size is not 0')
f.close()
def create_product(self, row):
""" Helper function to save a product."""
p = Product()
p.plan_id = int(row[0])
p.institution = row[1]
p.loan_purpose = row[2]
p.pmt_type = row[3]
p.loan_type = row[4]
p.loan_term = int(row[5])
p.int_adj_term = self.c.nullable_int(row[6])
p.adj_period = self.c.nullable_int(row[7])
p.io = self.c.string_to_boolean(row[8])
p.arm_index = self.c.nullable_string(row[9])
p.int_adj_cap = self.c.nullable_int(row[10])
p.annual_cap = self.c.nullable_int(row[11])
p.loan_cap = self.c.nullable_int(row[12])
p.arm_margin = self.c.nullable_decimal(row[13])
p.ai_value = self.c.nullable_decimal(row[14])
p.min_ltv = Decimal(row[15]).quantize(Decimal('.001'))
p.max_ltv = Decimal(row[16]).quantize(Decimal('.001'))
p.min_fico = int(row[17])
p.max_fico = int(row[18])
p.min_loan_amt = Decimal(row[19])
p.max_loan_amt = Decimal(row[20])
p.data_timestamp = datetime.strptime('20140101', '%Y%m%d')
p.save()
return p
def setUp(self):
self.c = Command()
self.test_dir = '%s/test_folder' % os.path.dirname(os.path.realpath(__file__))
self.dummyargs = {
'product': {'date': '20140101', 'file': '20140101_product.txt'},
'adjustment': {'date': '20140101', 'file': '20140101_adjustment.txt'},
'rate': {'date': '20140101', 'file': '20140101_rate.txt'},
'region': {'date': '20140101', 'file': '20140101_region.txt'},
'fee': {'date': '20140101', 'file': '20140101_fee.txt'},
}
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
def tearDown(self):
""" Delete the test_folder dir."""
shutil.rmtree(self.test_dir)
def test_string_to_boolean(self):
b = self.c.string_to_boolean('abc')
self.assertEqual(b, None)
b = self.c.string_to_boolean('False')
self.assertFalse(b)
b = self.c.string_to_boolean('True')
self.assertTrue(b)
b = self.c.string_to_boolean('1')
self.assertTrue(b)
b = self.c.string_to_boolean('0')
self.assertFalse(b)
def test_nullable_int(self):
self.assertEqual(self.c.nullable_int('10'), 10)
self.assertEqual(self.c.nullable_int('10.0'), 10)
self.assertEqual(self.c.nullable_int(''), None)
self.assertEqual(self.c.nullable_int(' '), None)
def test_nullable_string(self):
self.assertEqual(self.c.nullable_string('BANK'), 'BANK')
self.assertEqual(self.c.nullable_string(' '), None)
self.assertEqual(self.c.nullable_string(''), None)
def test_nullable_decimal(self):
self.assertEqual(self.c.nullable_decimal(''), None)
self.assertEqual(self.c.nullable_decimal(' '), None)
self.assertEqual(self.c.nullable_decimal('12.5'), Decimal('12.500'))
def test_create_rate(self):
#rate_id, product_id, region_id, lock, base_rate, total_points
row = ['1', '12', '200', '40', '3.125', '3']
now = datetime.now()
r = self.c.create_rate(row, now)
self.assertEqual(r.rate_id, 1)
self.assertEqual(r.product_id, 12)
self.assertEqual(r.lock, 40)
self.assertEqual(r.base_rate, Decimal('3.125'))
self.assertEqual(r.data_timestamp, now)
self.assertEqual(r.region_id, 200)
def test_get_precalculated_results(self):
""" .. don't know what to test here."""
zfile = self.prepare_sample_data()
result = self.c.get_precalculated_results(zfile)
self.assertEqual(len(result), 1)
self.assertTrue('1' in result)
self.assertEqual(result['1'][0], '3.750')
self.assertEqual(result['1'][1], '0.125')
# Read http://alexmarandon.com/articles/python_mock_gotchas/
@patch('ratechecker.management.commands.load_daily_data.get_rates')
def test_compare_scenarios_output(self, mock_get_rates):
""" .. the function."""
mock_get_rates.return_value = {'data': {'3.750': '0.125'}}
data = {
'1': ['3.750', '0.125'],
'2': ['11', '12'],
}
cut_down = {}
cut_down['1'] = self.c.test_scenarios['1']
cut_down['2'] = self.c.test_scenarios['2']
cut_down['16'] = self.c.test_scenarios['16']
self.c.test_scenarios = cut_down
row = ['11', 'VA']
r = Region()
r.region_id = int(row[0])
r.state_id = row[1]
r.data_timestamp = datetime.now()
r.save()
result = self.c.compare_scenarios_output(data)
self.assertTrue("The following scenarios don't match: ['2'] " in self.c.messages)
def test_delete_temp_tables(self):
""" ... some exist, others - not."""
# don't know how to easily test table (in)existence
cursor = connection.cursor()
cursor.execute('CREATE TABLE temporary_product(a TEXT)')
self.assertRaises(OperationalError, cursor.execute, 'SELECT * FROM temporary_region')
empty = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(empty is not None)
self.c.delete_temp_tables(cursor)
self.assertRaises(OperationalError, cursor.execute, 'SELECT * FROM temporary_product')
def test_reload_old_data(self):
""" .. archive_data_to_temp_tables, delete_temp_tables, delete_data_from_base_tables """
cursor = connection.cursor()
row = [
'5999', 'SMPL', 'PURCH', 'ARM', 'JUMBO', '30', '7.0', '1',
'False', 'LIBOR', '5.0000', '2.0000', '5.0000', '2.5000',
'.5532', '1', '90', '620', '850', '417001', '2000000', 1, 1, 0
]
# Original product object
op = self.create_product(row)
self.c.archive_data_to_temp_tables(cursor)
result = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(len(result.fetchall()) == 1)
result = Product.objects.all()
self.assertTrue(result)
self.c.delete_data_from_base_tables()
result = Product.objects.all()
self.assertFalse(result)
self.c.reload_old_data(cursor)
result = Product.objects.all()
self.assertEqual(len(result), 1)
self.assertTrue(op == result[0])
def test_handle__no_folder(self):
""" .. check that some issues are caught."""
self.assertRaises(SystemExit, self.c.handle)
self.assertEqual(1, self.c.status)
self.assertTrue('Error: tuple index out of range. Has a source directory been provided?'
in self.c.messages)
self.assertFalse('Warning: reloading "yesterday" data.' in self.c.messages)
def test_handle__bad_folder(self):
""" .. check that some issues are caught."""
# inexistent folder, inaccessible folder or a file all are caught in the same place
with open('not_a_folder', 'w') as fake_folder:
fake_folder.write('I am not a folder')
self.assertRaises(SystemExit, self.c.handle, 'not_a_folder')
self.assertEqual(1, self.c.status)
def test_handle__bad_zipfile(self):
""" .. check that some issues are caught."""
with open('20140101.zip', 'w') as fake_zip_archive:
fake_zip_archive.write('Some text')
self.assertRaises(SystemExit, self.c.handle, '.')
self.assertEqual(self.c.status, 1)
self.assertTrue(' Warning: File is not a zip file.' in self.c.messages)
self.assertTrue('Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.load_arch_data')
def test_handle__value_error(self, mock_lad):
""" .. check that ValueError is caught."""
mock_lad.side_effect = ValueError('Value Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(' Warning: Value Error.' in self.c.messages)
self.assertTrue('Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.load_arch_data')
def test_handle__oah_exception(self, mock_lad):
""" .. check that OaHException is caught."""
mock_lad.side_effect = OaHException('OaH Exception')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(' Warning: OaH Exception.' in self.c.messages)
self.assertTrue('Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.load_arch_data')
def test_handle__integrity_error(self, mock_lad):
""" .. check that IntegrityError is caught."""
mock_lad.side_effect = IntegrityError('Integrity Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(' Warning: Integrity Error.' in self.c.messages)
self.assertTrue('Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.load_arch_data')
def test_handle__key_error(self, mock_lad):
""" .. check that KeyError is caught."""
mock_lad.side_effect = KeyError('Key Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(" Warning: 'Key Error'." in self.c.messages)
self.assertTrue('Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.arch_list')
def test_handle__operational_error(self, mock_al):
""" .. check that KeyError is caught."""
self.c.messages = [] # weird how this is not emptied for each test run
mock_al.side_effect = OperationalError('Operational Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue("Error: Operational Error." in self.c.messages)
self.assertFalse('Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.compare_scenarios_output')
@patch('ratechecker.management.commands.load_daily_data.Command.get_precalculated_results')
@patch('ratechecker.management.commands.load_daily_data.Command.load_arch_data')
def test_handle__successful_run(self, mock_lad, mock_gpr, mock_cso):
""" .. check that status and success message are set."""
mock_lad.return_value = 1
mock_gpr.return_value = 1
mock_cso.return_value = 1
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertEqual(self.c.status, 0)
success_message = 'Successfully loaded data from <%s/20140101.zip>' % self.test_dir
self.assertTrue(success_message in self.c.messages)
def test_arch_list(self):
""" .. only the correct filenames are returned and are sorted."""
self.create_test_files(self.dummyargs)
arch_name = '%s.zip' % self.dummyargs['rate']['date']
zfile = zipfile.ZipFile(arch_name, 'w')
for key in self.dummyargs:
zfile.write(self.dummyargs[key]['file'])
zfile.close()
shutil.copy(arch_name, '20130202.zip')
shutil.copy(arch_name, '20150202.zip')
result = self.c.arch_list('.')
self.assertEqual(len(result), 3)
self.assertTrue('20150202' in result[0])
self.assertTrue(arch_name in result[1])
self.assertTrue('20130202' in result[2])
def test_load_arch_data(self):
""" .. check all load_xxx_data functions here."""
zfile = self.prepare_sample_data()
result = self.c.load_arch_data(zfile)
products = Product.objects.all()
self.assertEqual(len(products), 3)
self.assertTrue(products[1].institution, 'SMPL1')
self.assertTrue(products[0].institution, 'SMPL')
self.assertTrue(products[2].institution, 'SMPL2')
self.assertTrue(products[1].loan_purpose, 'REFI')
self.assertTrue(products[1].pmt_type, 'FIXED')
#TODO: add other checks
adjustments = Adjustment.objects.all()
self.assertEqual(len(adjustments), 4)
rates = Rate.objects.all()
self.assertEqual(len(rates), 7)
regions = Region.objects.all()
self.assertEqual(len(regions), 4)
fees = Fee.objects.all()
self.assertEqual(len(fees), 4)
def test_load_xxx_data__exception(self):
""" .. check that an OaHException is being raised when number of inserted items is
different from that in the data file .. or (as in this case) is 0."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region', 'fee']:
zfile.write('%s_%s.txt' % (date, key))
self.assertRaises(OaHException, self.c.load_product_data, date, zfile)
self.assertRaises(OaHException, self.c.load_rate_data, date, zfile)
self.assertRaises(OaHException, self.c.load_region_data, date, zfile)
self.assertRaises(OaHException, self.c.load_adjustment_data, date, zfile)
self.assertRaises(OaHException, self.c.load_fee_data, date, zfile)
zfile.close()
@patch('ratechecker.management.commands.load_daily_data.reader')
def test_load_xxx_data__many_items(self, mock_reader):
""" .. check that those defs actually use bulk_create. Doesn't actually test anything,
but coveralls.io will count this as checked."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region', 'fee']:
zfile.write('%s_%s.txt' % (date, key))
mock_reader.return_value = ([ndx, 'Institution X', 'PURCH', 'FIXED', 'CONF', 30, '3.0',
'1', '0', '', '', '', '', '', '', '1.0000', '95.0000', '620',
'850', '1.0000', '417000.0000', '1', '1', '0']
for ndx in range(1003))
self.c.load_product_data(date, zfile)
mock_reader.return_value = ([ndx, ndx, 'P', '1.25', '', '', '', '700', '720', '85.01',
'95.0', ''] for ndx in range(1003))
self.c.load_adjustment_data(date, zfile)
mock_reader.return_value = ([ndx, ndx, ndx, 35, '4.000', '-0.125'] for ndx in range(1003))
self.c.load_rate_data(date, zfile)
mock_reader.return_value = ([ndx, 'DC', 1] for ndx in range(1003))
self.c.load_region_data(date, zfile)
mock_reader.return_value = ([ndx, ndx + 1, 'DC', 'Lender Name', 1, 0, 1, 100, 10, 100]
for ndx in range(1003))
self.c.load_fee_data(date, zfile)
self.assertEqual(Product.objects.all().count(), 1002)
self.assertEqual(Adjustment.objects.all().count(), 1002)
self.assertEqual(Rate.objects.all().count(), 1002)
self.assertEqual(Region.objects.all().count(), 1002)
self.assertEqual(Fee.objects.all().count(), 1002)
zfile.close()
def prepare_sample_data(self, extra_data={}):
""" solely for test_load_arch_data."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
filename = '%s_product.txt' % date
with open(filename, 'w') as prdata:
prdata.write("Is skipped anyway\n")
prdata.write("7487\tSMPL\tPURCH\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n")
prdata.write("7488\tSMPL1\tREFI\tFIXED\tCONF\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n")
prdata.write("7489\tSMPL2\tREFI\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n")
if 'product' in extra_data:
prdata.write(extra_data['product'])
filename = filename.replace('_product', '_adjustment')
with open(filename, 'w') as adjdata:
adjdata.write("Is skipped anyway\n")
adjdata.write("7487\t67600\tP\t-0.25\t\t\t\t720\t739\t1\t60\t\n")
adjdata.write("7488\t73779\tP\t0\t850001.000\t1000000.000\t\t\t\t\t\t\n")
adjdata.write("7489\t68040\tP\t3.25\t\t\t\t620\t639\t85.010000000000005\t95\t\n")
adjdata.write("7489\t67996\tP\t1.5\t\t\t\t620\t639\t60.009999999999998\t70\t\n")
if 'adjustment' in extra_data:
adjdata.write(extra_data['adjustment'])
filename = filename.replace('_adjustment', '_rate')
with open(filename, 'w') as rtdata:
rtdata.write("Is skipped\n")
rtdata.write("592005597\t275387\t332\t60\t4.000\t-.375\n")
rtdata.write("592005635\t278474\t332\t30\t2.250\t1.250\n")
rtdata.write("592005636\t278474\t332\t30\t2.375\t1.000\n")
rtdata.write("592005637\t278474\t332\t30\t2.500\t.750\n")
rtdata.write("592005638\t278474\t332\t30\t2.625\t.500\n")
rtdata.write("592005639\t278474\t332\t30\t2.750\t.250\n")
rtdata.write("592005640\t278474\t332\t30\t2.875\t.000\n")
if 'rate' in extra_data:
rtdata.write(extra_data['rate'])
filename = filename.replace('_rate', '_region')
with open(filename, 'w') as regdata:
regdata.write("Is skipped\n")
regdata.write("12\tAK\tTrue\n")
regdata.write("12\tAL\tFalse\n")
regdata.write("12\tAR\tFalse\n")
regdata.write("12\tAZ\tFalse\n")
if 'region' in extra_data:
regdata.write(extra_data['region'])
filename = filename.replace('_region', '_fee')
with open(filename, 'w') as feedata:
feedata.write("Is skipped\n")
feedata.write("11\t11111\tDC\tSMPL\t1\t1\t1\t1608.0000\t.000\t587.2700\n")
feedata.write("11\t11111\tDC\tSMPL1\t1\t0\t1\t1610.0000\t.000\t589.2700\n")
feedata.write("10\t11001\tDC\tSMPL1\t0\t1\t0\t1610.0000\t.000\t589.2700\n")
feedata.write("11\t11111\tVA\tSMPL2\t1\t1\t1\t1610.0000\t.000\t589.2700\n")
if 'fee' in extra_data:
feedata.write(extra_data['fee'])
filename = 'CoverSheet.xml'
with open(filename, 'w') as coversheet:
coversheet.write('<?xml version="1.0" encoding="utf-8"?>\n')
coversheet.write('<root>\n')
coversheet.write('<Scenarios><Scenario><ScenarioNo>1</ScenarioNo>')
coversheet.write('<AdjustedRates>3.750</AdjustedRates><AdjustedPoints>0.125')
coversheet.write('</AdjustedPoints></Scenario></Scenarios>\n')
coversheet.write('</root>')
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region', 'fee']:
zfile.write('%s_%s.txt' % (date, key))
zfile.write('CoverSheet.xml')
zfile.close()
return zipfile.ZipFile(arch_name, 'r')
class LoadDailyTestCase(TestCase):
def create_test_files(self, data):
""" Create files in test_folder for tests."""
for key in data:
f = file(data[key]['file'], 'w+')
f.write('So the size is not 0')
f.close()
def create_product(self, row):
""" Helper function to save a product."""
p = Product()
p.plan_id = int(row[0])
p.institution = row[1]
p.loan_purpose = row[2]
p.pmt_type = row[3]
p.loan_type = row[4]
p.loan_term = int(row[5])
p.int_adj_term = self.c.nullable_int(row[6])
p.adj_period = self.c.nullable_int(row[7])
p.io = self.c.string_to_boolean(row[8])
p.arm_index = self.c.nullable_string(row[9])
p.int_adj_cap = self.c.nullable_int(row[10])
p.annual_cap = self.c.nullable_int(row[11])
p.loan_cap = self.c.nullable_int(row[12])
p.arm_margin = self.c.nullable_decimal(row[13])
p.ai_value = self.c.nullable_decimal(row[14])
p.min_ltv = Decimal(row[15]).quantize(Decimal('.001'))
p.max_ltv = Decimal(row[16]).quantize(Decimal('.001'))
p.min_fico = int(row[17])
p.max_fico = int(row[18])
p.min_loan_amt = Decimal(row[19])
p.max_loan_amt = Decimal(row[20])
p.data_timestamp = datetime.strptime('20140101', '%Y%m%d')
p.save()
return p
def setUp(self):
self.c = Command()
self.test_dir = '%s/test_folder' % os.path.dirname(
os.path.realpath(__file__))
self.dummyargs = {
'product': {
'date': '20140101',
'file': '20140101_product.txt'
},
'adjustment': {
'date': '20140101',
'file': '20140101_adjustment.txt'
},
'rate': {
'date': '20140101',
'file': '20140101_rate.txt'
},
'region': {
'date': '20140101',
'file': '20140101_region.txt'
},
}
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
def tearDown(self):
""" Delete the test_folder dir."""
shutil.rmtree(self.test_dir)
def test_string_to_boolean(self):
b = self.c.string_to_boolean('abc')
self.assertEqual(b, None)
b = self.c.string_to_boolean('False')
self.assertFalse(b)
b = self.c.string_to_boolean('True')
self.assertTrue(b)
b = self.c.string_to_boolean('1')
self.assertTrue(b)
b = self.c.string_to_boolean('0')
self.assertFalse(b)
def test_nullable_int(self):
self.assertEqual(self.c.nullable_int('10'), 10)
self.assertEqual(self.c.nullable_int('10.0'), 10)
self.assertEqual(self.c.nullable_int(''), None)
self.assertEqual(self.c.nullable_int(' '), None)
def test_nullable_string(self):
self.assertEqual(self.c.nullable_string('BANK'), 'BANK')
self.assertEqual(self.c.nullable_string(' '), None)
self.assertEqual(self.c.nullable_string(''), None)
def test_nullable_decimal(self):
self.assertEqual(self.c.nullable_decimal(''), None)
self.assertEqual(self.c.nullable_decimal(' '), None)
self.assertEqual(self.c.nullable_decimal('12.5'), Decimal('12.500'))
def test_create_rate(self):
#rate_id, product_id, region_id, lock, base_rate, total_points
row = ['1', '12', '200', '40', '3.125', '3']
now = datetime.now()
r = self.c.create_rate(row, now)
self.assertEqual(r.rate_id, 1)
self.assertEqual(r.product_id, 12)
self.assertEqual(r.lock, 40)
self.assertEqual(r.base_rate, Decimal('3.125'))
self.assertEqual(r.data_timestamp, now)
self.assertEqual(r.region_id, 200)
def test_get_precalculated_results(self):
pass
def test_compare_scenarios_output(self):
pass
def test_delete_temp_tables(self):
""" ... some exist, others - not."""
# don't know how to easily test table (in)existence
cursor = connection.cursor()
cursor.execute('CREATE TABLE temporary_product(a TEXT)')
self.assertRaises(OperationalError, cursor.execute,
'SELECT * FROM temporary_region')
empty = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(empty is not None)
self.c.delete_temp_tables(cursor)
self.assertRaises(OperationalError, cursor.execute,
'SELECT * FROM temporary_product')
def test_reload_old_data(self):
""" .. archive_data_to_temp_tables, delete_temp_tables, delete_data_from_base_tables """
cursor = connection.cursor()
row = [
'5999', 'SMPL', 'PURCH', 'ARM', 'JUMBO', '30', '7.0', '1', 'False',
'LIBOR', '5.0000', '2.0000', '5.0000', '2.5000', '.5532', '1',
'90', '620', '850', '417001', '2000000', 1, 1, 0
]
# Original product object
op = self.create_product(row)
self.c.archive_data_to_temp_tables(cursor)
result = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(len(result.fetchall()) == 1)
result = Product.objects.all()
self.assertTrue(result)
self.c.delete_data_from_base_tables()
result = Product.objects.all()
self.assertFalse(result)
self.c.reload_old_data(cursor)
result = Product.objects.all()
self.assertEqual(len(result), 1)
self.assertTrue(op == result[0])
#TODO: add checks for other situations
def test_handle__no_folder(self):
""" .. check that some issues are caught."""
self.assertRaises(SystemExit, self.c.handle)
self.assertEqual(1, self.c.status)
self.assertTrue(
'Error: tuple index out of range. Has a source directory been provided?'
in self.c.messages)
self.assertFalse(
'Warning: reloading "yesterday" data.' in self.c.messages)
def test_handle__bad_folder(self):
""" .. check that some issues are caught."""
# inexistent folder, inaccessible folder or a file all are caught in the same place
with open('not_a_folder', 'w') as fake_folder:
fake_folder.write('I am not a folder')
self.assertRaises(SystemExit, self.c.handle, 'not_a_folder')
self.assertEqual(1, self.c.status)
def test_handle__bad_zipfile(self):
""" .. check that some issues are caught."""
with open('20140101.zip', 'w') as fake_zip_archive:
fake_zip_archive.write('Some text')
self.assertRaises(SystemExit, self.c.handle, '.')
self.assertEqual(self.c.status, 1)
self.assertTrue(' Warning: File is not a zip file.' in self.c.messages)
self.assertTrue(
'Warning: reloading "yesterday" data' in self.c.messages)
def test_arch_list(self):
""" .. only the correct filenames are returned and are sorted."""
self.create_test_files(self.dummyargs)
arch_name = '%s.zip' % self.dummyargs['rate']['date']
zfile = zipfile.ZipFile(arch_name, 'w')
for key in self.dummyargs:
zfile.write(self.dummyargs[key]['file'])
zfile.close()
shutil.copy(arch_name, '20130202.zip')
shutil.copy(arch_name, '20150202.zip')
result = self.c.arch_list('.')
self.assertEqual(len(result), 3)
self.assertTrue('20150202' in result[0])
self.assertTrue(arch_name in result[1])
self.assertTrue('20130202' in result[2])
def test_load_arch_data(self):
""" .. check all load_xxx_data functions here."""
zfile = self.prepare_sample_data()
result = self.c.load_arch_data(zfile)
products = Product.objects.all()
self.assertEqual(len(products), 3)
self.assertTrue(products[1].institution, 'SMPL1')
self.assertTrue(products[0].institution, 'SMPL')
self.assertTrue(products[2].institution, 'SMPL2')
self.assertTrue(products[1].loan_purpose, 'REFI')
self.assertTrue(products[1].pmt_type, 'FIXED')
#TODO: add other checks
adjustments = Adjustment.objects.all()
self.assertEqual(len(adjustments), 4)
rates = Rate.objects.all()
self.assertEqual(len(rates), 7)
regions = Region.objects.all()
self.assertEqual(len(regions), 4)
def prepare_sample_data(self):
""" solely for test_load_arch_data."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
filename = '%s_product.txt' % date
with open(filename, 'w') as prdata:
prdata.write("Is skipped anyway\n")
prdata.write(
"7487\tSMPL\tPURCH\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n"
)
prdata.write(
"7488\tSMPL1\tREFI\tFIXED\tCONF\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n"
)
prdata.write(
"7489\tSMPL2\tREFI\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n"
)
filename = filename.replace('_product', '_adjustment')
with open(filename, 'w') as adjdata:
adjdata.write("Is skipped anyway\n")
adjdata.write("7487\t67600\tP\t-0.25\t\t\t\t720\t739\t1\t60\t\n")
adjdata.write(
"7488\t73779\tP\t0\t850001.000\t1000000.000\t\t\t\t\t\t\n")
adjdata.write(
"7489\t68040\tP\t3.25\t\t\t\t620\t639\t85.010000000000005\t95\t\n"
)
adjdata.write(
"7489\t67996\tP\t1.5\t\t\t\t620\t639\t60.009999999999998\t70\t\n"
)
filename = filename.replace('_adjustment', '_rate')
with open(filename, 'w') as rtdata:
rtdata.write("Is skipped\n")
rtdata.write("592005597\t275387\t332\t60\t4.000\t-.375\n")
rtdata.write("592005635\t278474\t332\t30\t2.250\t1.250\n")
rtdata.write("592005636\t278474\t332\t30\t2.375\t1.000\n")
rtdata.write("592005637\t278474\t332\t30\t2.500\t.750\n")
rtdata.write("592005638\t278474\t332\t30\t2.625\t.500\n")
rtdata.write("592005639\t278474\t332\t30\t2.750\t.250\n")
rtdata.write("592005640\t278474\t332\t30\t2.875\t.000\n")
filename = filename.replace('_rate', '_region')
with open(filename, 'w') as regdata:
regdata.write("Is skipped\n")
regdata.write("12\tAK\tTrue\n")
regdata.write("12\tAL\tFalse\n")
regdata.write("12\tAR\tFalse\n")
regdata.write("12\tAZ\tFalse\n")
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region']:
zfile.write('%s_%s.txt' % (date, key))
zfile.close()
return zipfile.ZipFile(arch_name, 'r')
class LoadDailyTestCase(TestCase):
def create_test_files(self, data):
""" Create files in test_folder for tests."""
for key in data:
f = file(data[key]['file'], 'w+')
f.write('So the size is not 0')
f.close()
def create_product(self, row):
""" Helper function to save a product."""
p = Product()
p.plan_id = int(row[0])
p.institution = row[1]
p.loan_purpose = row[2]
p.pmt_type = row[3]
p.loan_type = row[4]
p.loan_term = int(row[5])
p.int_adj_term = self.c.nullable_int(row[6])
p.adj_period = self.c.nullable_int(row[7])
p.io = self.c.string_to_boolean(row[8])
p.arm_index = self.c.nullable_string(row[9])
p.int_adj_cap = self.c.nullable_int(row[10])
p.annual_cap = self.c.nullable_int(row[11])
p.loan_cap = self.c.nullable_int(row[12])
p.arm_margin = self.c.nullable_decimal(row[13])
p.ai_value = self.c.nullable_decimal(row[14])
p.min_ltv = Decimal(row[15]).quantize(Decimal('.001'))
p.max_ltv = Decimal(row[16]).quantize(Decimal('.001'))
p.min_fico = int(row[17])
p.max_fico = int(row[18])
p.min_loan_amt = Decimal(row[19])
p.max_loan_amt = Decimal(row[20])
p.data_timestamp = datetime.strptime('20140101', '%Y%m%d')
p.save()
return p
def setUp(self):
self.c = Command()
self.test_dir = '%s/test_folder' % os.path.dirname(
os.path.realpath(__file__))
self.dummyargs = {
'product': {
'date': '20140101',
'file': '20140101_product.txt'
},
'adjustment': {
'date': '20140101',
'file': '20140101_adjustment.txt'
},
'rate': {
'date': '20140101',
'file': '20140101_rate.txt'
},
'region': {
'date': '20140101',
'file': '20140101_region.txt'
},
'fee': {
'date': '20140101',
'file': '20140101_fee.txt'
},
}
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
def tearDown(self):
""" Delete the test_folder dir."""
shutil.rmtree(self.test_dir)
def test_string_to_boolean(self):
b = self.c.string_to_boolean('abc')
self.assertEqual(b, None)
b = self.c.string_to_boolean('False')
self.assertFalse(b)
b = self.c.string_to_boolean('True')
self.assertTrue(b)
b = self.c.string_to_boolean('1')
self.assertTrue(b)
b = self.c.string_to_boolean('0')
self.assertFalse(b)
def test_nullable_int(self):
self.assertEqual(self.c.nullable_int('10'), 10)
self.assertEqual(self.c.nullable_int('10.0'), 10)
self.assertEqual(self.c.nullable_int(''), None)
self.assertEqual(self.c.nullable_int(' '), None)
def test_nullable_string(self):
self.assertEqual(self.c.nullable_string('BANK'), 'BANK')
self.assertEqual(self.c.nullable_string(' '), None)
self.assertEqual(self.c.nullable_string(''), None)
def test_nullable_decimal(self):
self.assertEqual(self.c.nullable_decimal(''), None)
self.assertEqual(self.c.nullable_decimal(' '), None)
self.assertEqual(self.c.nullable_decimal('12.5'), Decimal('12.500'))
def test_create_rate(self):
#rate_id, product_id, region_id, lock, base_rate, total_points
row = ['1', '12', '200', '40', '3.125', '3']
now = datetime.now()
r = self.c.create_rate(row, now)
self.assertEqual(r.rate_id, 1)
self.assertEqual(r.product_id, 12)
self.assertEqual(r.lock, 40)
self.assertEqual(r.base_rate, Decimal('3.125'))
self.assertEqual(r.data_timestamp, now)
self.assertEqual(r.region_id, 200)
def test_get_precalculated_results(self):
""" .. don't know what to test here."""
zfile = self.prepare_sample_data()
result = self.c.get_precalculated_results(zfile)
self.assertEqual(len(result), 1)
self.assertTrue('1' in result)
self.assertEqual(result['1'][0], '3.750')
self.assertEqual(result['1'][1], '0.125')
# Read http://alexmarandon.com/articles/python_mock_gotchas/
@patch('ratechecker.management.commands.load_daily_data.get_rates')
def test_compare_scenarios_output(self, mock_get_rates):
""" .. the function."""
mock_get_rates.return_value = {'data': {'3.750': '0.125'}}
data = {
'1': ['3.750', '0.125'],
'2': ['11', '12'],
}
cut_down = {}
cut_down['1'] = self.c.test_scenarios['1']
cut_down['2'] = self.c.test_scenarios['2']
cut_down['16'] = self.c.test_scenarios['16']
self.c.test_scenarios = cut_down
row = ['11', 'VA']
r = Region()
r.region_id = int(row[0])
r.state_id = row[1]
r.data_timestamp = datetime.now()
r.save()
result = self.c.compare_scenarios_output(data)
self.assertTrue(
"The following scenarios don't match: ['2'] " in self.c.messages)
def test_delete_temp_tables(self):
""" ... some exist, others - not."""
# don't know how to easily test table (in)existence
cursor = connection.cursor()
cursor.execute('CREATE TABLE temporary_product(a TEXT)')
self.assertRaises(OperationalError, cursor.execute,
'SELECT * FROM temporary_region')
empty = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(empty is not None)
self.c.delete_temp_tables(cursor)
self.assertRaises(OperationalError, cursor.execute,
'SELECT * FROM temporary_product')
def test_reload_old_data(self):
""" .. archive_data_to_temp_tables, delete_temp_tables, delete_data_from_base_tables """
cursor = connection.cursor()
row = [
'5999', 'SMPL', 'PURCH', 'ARM', 'JUMBO', '30', '7.0', '1', 'False',
'LIBOR', '5.0000', '2.0000', '5.0000', '2.5000', '.5532', '1',
'90', '620', '850', '417001', '2000000', 1, 1, 0
]
# Original product object
op = self.create_product(row)
self.c.archive_data_to_temp_tables(cursor)
result = cursor.execute('SELECT * FROM temporary_product')
self.assertTrue(len(result.fetchall()) == 1)
result = Product.objects.all()
self.assertTrue(result)
self.c.delete_data_from_base_tables()
result = Product.objects.all()
self.assertFalse(result)
self.c.reload_old_data(cursor)
result = Product.objects.all()
self.assertEqual(len(result), 1)
self.assertTrue(op == result[0])
def test_handle__no_folder(self):
""" .. check that some issues are caught."""
self.assertRaises(SystemExit, self.c.handle)
self.assertEqual(1, self.c.status)
self.assertTrue(
'Error: tuple index out of range. Has a source directory been provided?'
in self.c.messages)
self.assertFalse(
'Warning: reloading "yesterday" data.' in self.c.messages)
def test_handle__bad_folder(self):
""" .. check that some issues are caught."""
# inexistent folder, inaccessible folder or a file all are caught in the same place
with open('not_a_folder', 'w') as fake_folder:
fake_folder.write('I am not a folder')
self.assertRaises(SystemExit, self.c.handle, 'not_a_folder')
self.assertEqual(1, self.c.status)
def test_handle__bad_zipfile(self):
""" .. check that some issues are caught."""
with open('20140101.zip', 'w') as fake_zip_archive:
fake_zip_archive.write('Some text')
self.assertRaises(SystemExit, self.c.handle, '.')
self.assertEqual(self.c.status, 1)
self.assertTrue(' Warning: File is not a zip file.' in self.c.messages)
self.assertTrue(
'Warning: reloading "yesterday" data' in self.c.messages)
@patch(
'ratechecker.management.commands.load_daily_data.Command.load_arch_data'
)
def test_handle__value_error(self, mock_lad):
""" .. check that ValueError is caught."""
mock_lad.side_effect = ValueError('Value Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(' Warning: Value Error.' in self.c.messages)
self.assertTrue(
'Warning: reloading "yesterday" data' in self.c.messages)
@patch(
'ratechecker.management.commands.load_daily_data.Command.load_arch_data'
)
def test_handle__oah_exception(self, mock_lad):
""" .. check that OaHException is caught."""
mock_lad.side_effect = OaHException('OaH Exception')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(' Warning: OaH Exception.' in self.c.messages)
self.assertTrue(
'Warning: reloading "yesterday" data' in self.c.messages)
@patch(
'ratechecker.management.commands.load_daily_data.Command.load_arch_data'
)
def test_handle__integrity_error(self, mock_lad):
""" .. check that IntegrityError is caught."""
mock_lad.side_effect = IntegrityError('Integrity Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(' Warning: Integrity Error.' in self.c.messages)
self.assertTrue(
'Warning: reloading "yesterday" data' in self.c.messages)
@patch(
'ratechecker.management.commands.load_daily_data.Command.load_arch_data'
)
def test_handle__key_error(self, mock_lad):
""" .. check that KeyError is caught."""
mock_lad.side_effect = KeyError('Key Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue(" Warning: 'Key Error'." in self.c.messages)
self.assertTrue(
'Warning: reloading "yesterday" data' in self.c.messages)
@patch('ratechecker.management.commands.load_daily_data.Command.arch_list')
def test_handle__operational_error(self, mock_al):
""" .. check that KeyError is caught."""
self.c.messages = [] # weird how this is not emptied for each test run
mock_al.side_effect = OperationalError('Operational Error')
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertTrue("Error: Operational Error." in self.c.messages)
self.assertFalse(
'Warning: reloading "yesterday" data' in self.c.messages)
@patch(
'ratechecker.management.commands.load_daily_data.Command.compare_scenarios_output'
)
@patch(
'ratechecker.management.commands.load_daily_data.Command.get_precalculated_results'
)
@patch(
'ratechecker.management.commands.load_daily_data.Command.load_arch_data'
)
def test_handle__successful_run(self, mock_lad, mock_gpr, mock_cso):
""" .. check that status and success message are set."""
mock_lad.return_value = 1
mock_gpr.return_value = 1
mock_cso.return_value = 1
self.prepare_sample_data()
self.assertRaises(SystemExit, self.c.handle, self.test_dir)
self.assertEqual(self.c.status, 0)
success_message = 'Successfully loaded data from <%s/20140101.zip>' % self.test_dir
self.assertTrue(success_message in self.c.messages)
def test_arch_list(self):
""" .. only the correct filenames are returned and are sorted."""
self.create_test_files(self.dummyargs)
arch_name = '%s.zip' % self.dummyargs['rate']['date']
zfile = zipfile.ZipFile(arch_name, 'w')
for key in self.dummyargs:
zfile.write(self.dummyargs[key]['file'])
zfile.close()
shutil.copy(arch_name, '20130202.zip')
shutil.copy(arch_name, '20150202.zip')
result = self.c.arch_list('.')
self.assertEqual(len(result), 3)
self.assertTrue('20150202' in result[0])
self.assertTrue(arch_name in result[1])
self.assertTrue('20130202' in result[2])
def test_load_arch_data(self):
""" .. check all load_xxx_data functions here."""
zfile = self.prepare_sample_data()
result = self.c.load_arch_data(zfile)
products = Product.objects.all()
self.assertEqual(len(products), 3)
self.assertTrue(products[1].institution, 'SMPL1')
self.assertTrue(products[0].institution, 'SMPL')
self.assertTrue(products[2].institution, 'SMPL2')
self.assertTrue(products[1].loan_purpose, 'REFI')
self.assertTrue(products[1].pmt_type, 'FIXED')
#TODO: add other checks
adjustments = Adjustment.objects.all()
self.assertEqual(len(adjustments), 4)
rates = Rate.objects.all()
self.assertEqual(len(rates), 7)
regions = Region.objects.all()
self.assertEqual(len(regions), 4)
fees = Fee.objects.all()
self.assertEqual(len(fees), 4)
def test_load_xxx_data__exception(self):
""" .. check that an OaHException is being raised when number of inserted items is
different from that in the data file .. or (as in this case) is 0."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region', 'fee']:
zfile.write('%s_%s.txt' % (date, key))
self.assertRaises(OaHException, self.c.load_product_data, date, zfile)
self.assertRaises(OaHException, self.c.load_rate_data, date, zfile)
self.assertRaises(OaHException, self.c.load_region_data, date, zfile)
self.assertRaises(OaHException, self.c.load_adjustment_data, date,
zfile)
self.assertRaises(OaHException, self.c.load_fee_data, date, zfile)
zfile.close()
@patch('ratechecker.management.commands.load_daily_data.reader')
def test_load_xxx_data__many_items(self, mock_reader):
""" .. check that those defs actually use bulk_create. Doesn't actually test anything,
but coveralls.io will count this as checked."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region', 'fee']:
zfile.write('%s_%s.txt' % (date, key))
mock_reader.return_value = ([
ndx, 'Institution X', 'PURCH', 'FIXED', 'CONF', 30, '3.0', '1',
'0', '', '', '', '', '', '', '1.0000', '95.0000', '620', '850',
'1.0000', '417000.0000', '1', '1', '0'
] for ndx in range(1003))
self.c.load_product_data(date, zfile)
mock_reader.return_value = ([
ndx, ndx, 'P', '1.25', '', '', '', '700', '720', '85.01', '95.0',
''
] for ndx in range(1003))
self.c.load_adjustment_data(date, zfile)
mock_reader.return_value = ([ndx, ndx, ndx, 35, '4.000', '-0.125']
for ndx in range(1003))
self.c.load_rate_data(date, zfile)
mock_reader.return_value = ([ndx, 'DC', 1] for ndx in range(1003))
self.c.load_region_data(date, zfile)
mock_reader.return_value = ([
ndx, ndx + 1, 'DC', 'Lender Name', 1, 0, 1, 100, 10, 100
] for ndx in range(1003))
self.c.load_fee_data(date, zfile)
self.assertEqual(Product.objects.all().count(), 1002)
self.assertEqual(Adjustment.objects.all().count(), 1002)
self.assertEqual(Rate.objects.all().count(), 1002)
self.assertEqual(Region.objects.all().count(), 1002)
self.assertEqual(Fee.objects.all().count(), 1002)
zfile.close()
def prepare_sample_data(self, extra_data={}):
""" solely for test_load_arch_data."""
self.create_test_files(self.dummyargs)
date = self.dummyargs['product']['date']
filename = '%s_product.txt' % date
with open(filename, 'w') as prdata:
prdata.write("Is skipped anyway\n")
prdata.write(
"7487\tSMPL\tPURCH\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n"
)
prdata.write(
"7488\tSMPL1\tREFI\tFIXED\tCONF\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n"
)
prdata.write(
"7489\tSMPL2\tREFI\tARM\tJUMBO\t30\t7.0\t1\tFalse\tLIBOR\t5.0000\t2.0000\t5.0000\t2.5000\t.5532\t1\t90\t620\t850\t417001\t2000000\t1\t1\t0\n"
)
if 'product' in extra_data:
prdata.write(extra_data['product'])
filename = filename.replace('_product', '_adjustment')
with open(filename, 'w') as adjdata:
adjdata.write("Is skipped anyway\n")
adjdata.write("7487\t67600\tP\t-0.25\t\t\t\t720\t739\t1\t60\t\n")
adjdata.write(
"7488\t73779\tP\t0\t850001.000\t1000000.000\t\t\t\t\t\t\n")
adjdata.write(
"7489\t68040\tP\t3.25\t\t\t\t620\t639\t85.010000000000005\t95\t\n"
)
adjdata.write(
"7489\t67996\tP\t1.5\t\t\t\t620\t639\t60.009999999999998\t70\t\n"
)
if 'adjustment' in extra_data:
adjdata.write(extra_data['adjustment'])
filename = filename.replace('_adjustment', '_rate')
with open(filename, 'w') as rtdata:
rtdata.write("Is skipped\n")
rtdata.write("592005597\t275387\t332\t60\t4.000\t-.375\n")
rtdata.write("592005635\t278474\t332\t30\t2.250\t1.250\n")
rtdata.write("592005636\t278474\t332\t30\t2.375\t1.000\n")
rtdata.write("592005637\t278474\t332\t30\t2.500\t.750\n")
rtdata.write("592005638\t278474\t332\t30\t2.625\t.500\n")
rtdata.write("592005639\t278474\t332\t30\t2.750\t.250\n")
rtdata.write("592005640\t278474\t332\t30\t2.875\t.000\n")
if 'rate' in extra_data:
rtdata.write(extra_data['rate'])
filename = filename.replace('_rate', '_region')
with open(filename, 'w') as regdata:
regdata.write("Is skipped\n")
regdata.write("12\tAK\tTrue\n")
regdata.write("12\tAL\tFalse\n")
regdata.write("12\tAR\tFalse\n")
regdata.write("12\tAZ\tFalse\n")
if 'region' in extra_data:
regdata.write(extra_data['region'])
filename = filename.replace('_region', '_fee')
with open(filename, 'w') as feedata:
feedata.write("Is skipped\n")
feedata.write(
"11\t11111\tDC\tSMPL\t1\t1\t1\t1608.0000\t.000\t587.2700\n")
feedata.write(
"11\t11111\tDC\tSMPL1\t1\t0\t1\t1610.0000\t.000\t589.2700\n")
feedata.write(
"10\t11001\tDC\tSMPL1\t0\t1\t0\t1610.0000\t.000\t589.2700\n")
feedata.write(
"11\t11111\tVA\tSMPL2\t1\t1\t1\t1610.0000\t.000\t589.2700\n")
if 'fee' in extra_data:
feedata.write(extra_data['fee'])
filename = 'CoverSheet.xml'
with open(filename, 'w') as coversheet:
coversheet.write('<?xml version="1.0" encoding="utf-8"?>\n')
coversheet.write('<root>\n')
coversheet.write('<Scenarios><Scenario><ScenarioNo>1</ScenarioNo>')
coversheet.write(
'<AdjustedRates>3.750</AdjustedRates><AdjustedPoints>0.125')
coversheet.write('</AdjustedPoints></Scenario></Scenarios>\n')
coversheet.write('</root>')
arch_name = '%s.zip' % date
zfile = zipfile.ZipFile(arch_name, 'w')
for key in ['product', 'adjustment', 'rate', 'region', 'fee']:
zfile.write('%s_%s.txt' % (date, key))
zfile.write('CoverSheet.xml')
zfile.close()
return zipfile.ZipFile(arch_name, 'r')