def test_parse_post_data(self):
"""Test parsing of POST key/value pairs for data_model in TableIO """
# Test fully loaded table (8 values) in POST
tIO = AssessTableIO(TestData,delimiters)
records = tIO.parse_POST(POST_1_8)
values = []
for record in records.values():
values.append(record.get_value())
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(values,values_data_1_8)
# Test partly loaded table (7 values) in POST
tIO = AssessTableIO(TestData,delimiters)
records = tIO.parse_POST(POST_1_7)
values = []
for record in records.values():
values.append(record.get_value())
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(values,values_data_1_7)
# Test overloaded table (9 values) in POST
tIO = AssessTableIO(TestData,delimiters)
records = tIO.parse_POST(POST_1_9)
values = []
for record in records.values():
values.append(record.get_value())
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(values,values_data_1_9)
def test_parse_post_mapppings(self):
"""Test parsing of POST key/value pairs for data_model in TableIO """
# Test partly loaded table (3 values) in POST
tIO = AssessTableIO(TestMappings,delimiters)
records = tIO.parse_POST(mappings_POST_1_3)
values = []
for record in records.values():
values.append(str(record.get_value()))
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(values,values_mappings_1_3)
# Test fully loaded table (4 values) in POST
tIO = AssessTableIO(TestMappings,delimiters)
records = tIO.parse_POST(mappings_POST_1_4)
values = []
for record in records.values():
values.append(str(record.get_value()))
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(values,values_mappings_1_4)
# Test overloaded table (5 values) in POST
tIO = AssessTableIO(TestMappings,delimiters)
records = tIO.parse_POST(mappings_POST_1_5)
values = []
for record in records.values():
values.append(str(record.get_value()))
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(values,values_mappings_1_5)
def test_get_dataframe_failure(self):
"""Test that the database can be read and exported as dataframe."""
# Test that method returns empty dataframe on no database data
TestData.objects.all().delete()
tIO = AssessTableIO(TestData,delimiters)
tIO.keys.set_headers('value')
db = tIO.get_dataframe(self.v)
dfe = pandas.DataFrame()
assert_frame_equal(db,dfe)
def test_parse_dataframe_data_success(self):
"""Test parsing of dataframe into data_model in TableIO """
# TODO: look for ways that this function might fail and test
# Test fully loaded table (8 values) in POST
t = AssessTable(TestData,'1')
t.load(False)
tIO = AssessTableIO(TestData,delimiters)
tIO.keys.set_headers('value')
DF = []
# We must unpack key/value from record, as objects cannot expected to
# identical between dataframes and load (TODO: why not???)
for key,record in tIO.parse_dataframe(dfd).items():
DF.append({key: record.get_value()})
DB = []
for key,record in t.records.items():
DB.append({key: record.get_value()})
print_traceback(tIO.errors)
self.assertEqual(tIO.errors,[])
self.assertEqual(DB,DF)
def test_parse_csv_mappings(self):
"""Test parsing of csv strings for mappings_model in TableIO """
# Test multi-colunm input
POST = {
'column_field': 'testitemb',
'csv_string': csv_mappings_string_b
}
tIO = AssessTableIO(TestMappings, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, False)
for record in records.values():
values.append(str(record.get_value()))
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_mappings_b)
# Test multi-colunm input, alternative
POST = {
'column_field': 'testitema',
'csv_string': csv_mappings_string_a
}
tIO = AssessTableIO(TestMappings, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, False)
for record in records.values():
values.append(str(record.get_value()))
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_mappings_a)
# Test one-colunm input
POST = {
'column_field': 'testitemc',
'csv_string': csv_mappings_string_c
}
tIO = AssessTableIO(TestMappings, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, True)
for record in records.values():
values.append(str(record.get_value()))
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_mappings_c)
def test_parse_post_data_invalid(self):
# Test that an invalid key length in POST results in an error and that
# the POST entry is not added to the records
tIO = AssessTableIO(TestData,delimiters)
records = tIO.parse_POST(POST_invalid_length)
key = list(POST_invalid_length.keys()).pop()
self.assertEqual(str(tIO.errors.pop()),str(KeyInvalid(key, TestData)))
self.assertEqual(records,{})
# Test that an invalid key length in POST results in an error and that
# the POST entry is not added to the records
records = tIO.parse_POST(POST_invalid_label)
key = 'bad_item in ' + list(POST_invalid_label.keys()).pop()
self.assertEqual(str(tIO.errors.pop()),str(KeyNotFound(key, TestData)))
self.assertEqual(records,{})
# Test that an invalid value field name in POST results in an error
# and that the POST entry is not added to the records
records = tIO.parse_POST(POST_invalid_value_field)
error_msg = str(NoFieldError('bad_field', TestData))
self.assertNotEqual(len(tIO.errors),0)
if len(tIO.errors) > 0:
self.assertEqual(str(tIO.errors.pop()),error_msg)
self.assertEqual(records,{})
def test_parse_dataframe_data_failure(self):
"""Test that method fail when index items and values are invalid"""
tIO = AssessTableIO(TestData,delimiters)
tIO.keys.set_headers('value')
# Test invalid decimal number
dfe = pandas.DataFrame({'testitema': ['a1'], 'testitemb': ['b1'], 'testitemc': ['c1'], 'value': 'XX'})
tIO.parse_dataframe(dfe)
e1 = str(tIO.errors.pop())
e2 = str(NotDecimalError('XX',TestData))
self.assertEqual(e1,e2)
# Test invalid index item
dfe = pandas.DataFrame({'testitema': ['a1'], 'testitemb': ['b1'], 'testitemc': ['BADITEM'], 'value': '1'})
tIO.parse_dataframe(dfe)
e1 = str(tIO.errors.pop())
e2 = str(NoItemError('BADITEM',TestItemC))
self.assertEqual(e1,e2)
# Test invalid index item in mappings_model
tIO = AssessTableIO(TestMappings,delimiters)
tIO.keys.set_headers('testitemc')
dfe = pandas.DataFrame({'testitema': ['a1'], 'testitemb': ['b1'], 'testitemc': ['BADITEM'] })
tIO.parse_dataframe(dfe)
e1 = str(tIO.errors.pop())
e2 = str(NoItemError('BADITEM',TestMappings))
self.assertEqual(e1,e2)
def test_get_dataframe_success(self):
"""Test that the database can be read and exported as dataframe."""
# Test data_model
tIO = AssessTableIO(TestData,delimiters)
tIO.keys.set_headers('value')
db = tIO.get_dataframe(self.v)
assert_frame_equal(db,dfd)
# Test that mappings_model
tIO = AssessTableIO(TestMappings,delimiters)
tIO.keys.set_headers('testitemc')
db = tIO.get_dataframe(self.v)
assert_frame_equal(db,dfm)
def test_parse_csv_data(self):
"""Test parsing of csv strings for data_model in TableIO """
# Test multi-colunm input
POST = {'column_field': 'testitemc', 'csv_string': csv_data_string_c}
tIO = AssessTableIO(TestData, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, False)
for record in records.values():
values.append(record.get_value())
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_data_c)
# Test multi-colunm input, alternative column
POST = {'column_field': 'testitema', 'csv_string': csv_data_string_a}
tIO = AssessTableIO(TestData, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, False)
for record in records.values():
values.append(record.get_value())
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_data_a)
# Test multi-colunm input, another alternative column
POST = {'column_field': 'testitemb', 'csv_string': csv_data_string_b}
tIO = AssessTableIO(TestData, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, False)
for record in records.values():
values.append(record.get_value())
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_data_b)
# Test multi-colunm input, another alternative, but with archived index
POST = {'column_field': 'testitemb', 'csv_string': csv_data_string_bb}
tIO = AssessTableIO(TestData, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, False)
for record in records.values():
values.append(record.get_value())
# This input is supposed to raise an error and return empty
self.assertNotEqual(tIO.errors, [])
self.assertEqual(values, [])
# Test one-colunm input
POST = {'column_field': 'value', 'csv_string': csv_data_string_val}
tIO = AssessTableIO(TestData, delimiters)
values = []
records = tIO.parse_csv(POST)
self.assertEqual(tIO.keys.table_one_column, True)
for record in records.values():
values.append(record.get_value())
self.assertEqual(tIO.errors, [])
self.assertEqual(values, values_data_val)
def test_parse_csv_data_failure(self):
"""Test parsing of csv strings for mappings_model in TableIO - failures."""
e1 = AssessError()
e2 = AssessError()
# Test malformed header - bad index header item
ch = ['testitemBAD', 'testitemb', 'c1', 'c2']
ct = ['testitema', 'testitemb', 'c1', 'c2']
csv_data_string_c_bad_header = """testitemBAD\ttestitemb\tc1\tc2
a1\tb1\t1\t2
a2\tb1\t5\t6"""
POST = {
'column_field': 'testitemc',
'csv_string': csv_data_string_c_bad_header
}
tIO = AssessTableIO(TestData, delimiters)
tIO.parse_csv(POST)
msg = str(NoFieldError('testitemBAD', TestData))
e2 = CSVheaderMalformed(ch, ct, msg, TestData)
e1 = tIO.errors.pop()
self.assertEqual(str(e1), str(e2))
# Test malformed header - missing index header item
ch = ['testitemb', 'c1', 'c2']
ct = ['testitema', 'testitemb', 'c1', 'c2']
csv_data_string_c_bad_header = """testitemb\tc1\tc2
a1\tb1\t1\t2
a2\tb1\t5\t6"""
POST = {
'column_field': 'testitemc',
'csv_string': csv_data_string_c_bad_header
}
tIO = AssessTableIO(TestData, delimiters)
tIO.parse_csv(POST)
msg = str(NoFieldError('testitema', TestData))
e2 = CSVheaderMalformed(ch, ct, msg, TestData)
e1 = tIO.errors.pop()
self.assertEqual(str(e1), str(e2))
# Test malformed header - missing value header item(s)
ch = ['testitema', 'testitemb']
ct = ['testitema', 'testitemb', 'c1', 'c2']
csv_data_string_c_bad_header = """testitema\ttestitemb
a2\tb1\t5\t6"""
POST = {
'column_field': 'testitemc',
'csv_string': csv_data_string_c_bad_header
}
tIO = AssessTableIO(TestData, delimiters)
tIO.parse_csv(POST)
msg = str(NoFieldError('value', TestData))
e2 = CSVheaderMalformed(ch, ct, msg, TestData)
e1 = tIO.errors.pop()
self.assertEqual(str(e1), str(e2))
# Test wrong line count: First dataline has 1 too many, last 1 too few
csv_data_string_c_bad_count = """testitema\ttestitemb\tc1\tc2
a1\tb1\t1\t2\t99
a2\tb1\t5"""
POST = {
'column_field': 'testitemc',
'csv_string': csv_data_string_c_bad_count
}
ch = ['testitema', 'testitemb', 'c1', 'c2']
tIO = AssessTableIO(TestData, delimiters)
tIO.parse_csv(POST)
# One too many row columns
row = ['a2', 'b1', '5']
e2 = CSVwrongColumnCount(row, ch, '', TestData)
e1 = tIO.errors.pop()
self.maxDiff = None
self.assertEqual(str(e1), str(e2))
# Test wrong key
row = ['a1', 'b1', '1', '2', '99']
e2 = CSVwrongColumnCount(row, ch, '', TestData)
e1 = tIO.errors.pop()
self.maxDiff = None
self.assertEqual(str(e1), str(e2))
# Test bad index item in data row
csv_data_string_c_bad_count = """testitema\ttestitemb\tc1\tc2
a1\tb1\t1\t2
BADITEM\tb1\t5\t6"""
POST = {
'column_field': 'testitemc',
'csv_string': csv_data_string_c_bad_count
}
ch = ['testitema', 'testitemb', 'c1', 'c2']
tIO = AssessTableIO(TestData, delimiters)
tIO.parse_csv(POST)
# One too many row columns
row = ['BADITEM', 'b1', '5', '6']
msg = str(NoItemError('BADITEM', TestItemA))
e2 = CSVfieldNotFound(row, ch, msg, TestData)
e1 = tIO.errors.pop()
self.maxDiff = None
self.assertEqual(str(e1), str(e2))