def setUp(self):
customers_cid_path = dev_test.path_to_test_cid('customers.ods')
self._cutplace_app = applications.CutplaceApp()
self._cutplace_app.set_cid_from_path(customers_cid_path)
self._valid_customers_csv_path = dev_test.path_to_test_data(
'valid_customers.csv')
self._broken_customers_non_csv_path = dev_test.path_to_test_data(
'valid_customers.ods')
def test_fails_on_ods_from_excel(self):
excel_path = dev_test.path_to_test_data('valid_customers.xls')
try:
list(rowio.ods_rows(excel_path))
self.fail()
except errors.DataFormatError as anticipated_error:
dev_test.assert_fnmatches(self, str(anticipated_error), '*: cannot uncompress ODS spreadsheet: *')
def test_can_extract_all_excel_field_types(self):
field_types_path = dev_test.path_to_test_data('fieldtypes.xls')
for row_number, row in enumerate(rowio.excel_rows(field_types_path)):
self.assertEqual(3, len(row))
if row_number >= 1:
_, excel_value, cutplace_value = row
self.assertEqual(cutplace_value, excel_value)
def test_fails_on_invalid_csv_source_file_with_duplicates(self):
cid = interface.Cid(dev_test.path_to_test_cid("icd_customers.xls"))
with validio.Reader(
cid,
dev_test.path_to_test_data(
"broken_customers_with_duplicates.csv")) as reader:
self.assertRaises(errors.CheckError, reader.validate_rows)
def test_fails_on_csv_source_file_with_fewer_elements_than_expected(self):
cid = interface.Cid(dev_test.path_to_test_cid("icd_customers.xls"))
with validio.Reader(
cid,
dev_test.path_to_test_data(
"broken_customers_fewer_elements.csv")) as reader:
self.assertRaises(errors.DataError, reader.validate_rows)
def test_can_open_and_validate_excel_source_file(self):
cid = interface.Cid(
dev_test.path_to_test_cid("icd_customers_excel.xls"))
with validio.Reader(
cid,
dev_test.path_to_test_data("valid_customers.xls")) as reader:
reader.validate_rows()
def test_fails_on_excel_from_ods(self):
ods_path = dev_test.path_to_test_data('valid_customers.ods')
try:
list(rowio.excel_rows(ods_path))
self.fail()
except errors.DataFormatError as anticipated_error:
dev_test.assert_fnmatches(self, str(anticipated_error), '* (R1C1): cannot read Excel file: *')
def test_fails_on_invalid_csv_source_file_with_not_observed_count_expression(
self):
cid = interface.Cid(dev_test.path_to_test_cid("icd_customers.xls"))
data_path = dev_test.path_to_test_data(
"broken_customers_with_too_many_branches.csv")
reader = validio.Reader(cid, data_path)
reader.validate_rows()
self.assertRaises(errors.CheckError, reader.close)
def test_can_validate_proper_csv_with_plugins(self):
cid_path = dev_test.path_to_test_cid('customers_with_plugins.ods')
csv_path = dev_test.path_to_test_data('valid_customers.csv')
exit_code = applications.process([
'test_can_validate_proper_csv_with_plugins', '--plugins',
dev_test.path_to_test_plugins(), cid_path, csv_path
])
self.assertEqual(0, exit_code)
def test_fails_on_delimited_with_unterminated_quote(self):
customer_cid = interface.Cid(dev_test.CID_CUSTOMERS_ODS_PATH)
broken_delimited_path = dev_test.path_to_test_data('broken_customers_with_unterminated_quote.csv')
try:
list(rowio.delimited_rows(broken_delimited_path, customer_cid.data_format))
except errors.DataFormatError as error:
error_message = '%s' % error
self.assertTrue(
'cannot parse delimited file' in error_message, 'error_message=%r' % error_message)
def test_fails_on_ods_from_csv(self):
broken_ods_path = dev_test.path_to_test_data('customers.csv')
try:
list(rowio.ods_rows(broken_ods_path))
self.fail('expected DataFormatError')
except errors.DataFormatError as error:
error_message = '%s' % error
self.assertTrue(
'cannot uncompress ODS spreadsheet:' in error_message, 'error_message=%r' % error_message)
def test_fails_on_ods_without_content_xml(self):
broken_ods_path = dev_test.path_to_test_data('broken_without_content_xml.ods')
try:
list(rowio.ods_rows(broken_ods_path))
self.fail('expected DataFormatError')
except errors.DataFormatError as error:
error_message = '%s' % error
self.assertTrue(
'cannot extract content.xml' in error_message, 'error_message=%r' % error_message)
def test_fails_on_non_existent_ods_sheet(self):
ods_path = dev_test.path_to_test_data('valid_customers.ods')
try:
list(rowio.ods_rows(ods_path, 123))
self.fail('expected DataFormatError')
except errors.DataFormatError as error:
error_message = '%s' % error
self.assertTrue(
'ODS must contain at least' in error_message, 'error_message=%r' % error_message)
def test_can_read_delimited_rows(self):
# TODO: either get rid of the CID and move it to test_iotools or use validate.Reader and move it to test_validate.
delimited_cid = interface.Cid(
dev_test.path_to_test_cid("icd_customers.xls"))
delimited_rows = rowio.delimited_rows(
dev_test.path_to_test_data("valid_customers.csv"),
delimited_cid._data_format)
first_row = next(delimited_rows)
self.assertEqual(first_row,
['38000', '23', 'John', 'Doe', 'male', '08.03.1957'])
def test_can_read_fixed_rows(self):
cid_path = dev_test.path_to_test_cid('customers_fixed.ods')
customer_cid = interface.Cid(cid_path)
fixed_path = dev_test.path_to_test_data('valid_customers_fixed.txt')
field_names_and_lengths = interface.field_names_and_lengths(customer_cid)
rows = list(rowio.fixed_rows(fixed_path, customer_cid.data_format.encoding, field_names_and_lengths))
self.assertNotEqual(0, len(rows))
for row_index in range(len(rows) - 1):
row = rows[row_index]
next_row = rows[row_index + 1]
self.assertNotEqual(0, len(row))
self.assertEqual(len(row), len(next_row))
def _build_and_validate_many_customers():
cid_path = dev_test.CID_CUSTOMERS_ODS_PATH
# TODO: Write to 'build/many_customers.csv'
many_customers_csv_path = dev_test.path_to_test_data("lots_of_customers.csv")
_build_lots_of_customers_csv(many_customers_csv_path, 50)
# Validate the data using the API, so in case of errors we get specific information.
customers_cid = interface.Cid(cid_path)
with validio.Reader(customers_cid, many_customers_csv_path) as reader:
reader.validate_rows()
# Validate the data using the command line application in order to use
# the whole tool chain from an end user's point of view.
exit_code = applications.main(["test_performance.py", cid_path, many_customers_csv_path])
if exit_code != 0:
raise ValueError("exit code of performance test must be 0 but is %d" % exit_code)
def _build_and_validate_many_customers():
icd_ods_path = dev_test.path_to_test_cid("customers.ods")
# TODO: Write to 'build/many_customers.csv'
many_customers_csv_path = dev_test.path_to_test_data(
"lots_of_customers.csv")
_build_lots_of_customers_csv(many_customers_csv_path, 50)
# Validate the data using the API, so in case of errors we get specific information.
customers_cid = interface.Cid(icd_ods_path)
with validio.Reader(customers_cid, many_customers_csv_path) as reader:
reader.validate_rows()
# Validate the data using the command line application in order to use
# the whole tool chain from an end user's point of view.
exit_code = applications.main(
["test_performance.py", icd_ods_path, many_customers_csv_path])
if exit_code != 0:
raise ValueError("exit code of performance test must be 0 but is %d" %
exit_code)
def test_fails_on_invalid_csv_with_duplicates(self):
cid = interface.Cid(dev_test.CID_CUSTOMERS_XLS_PATH)
with validio.Reader(cid, dev_test.path_to_test_data("broken_customers_with_duplicates.csv")) as reader:
self.assertRaises(errors.CheckError, reader.validate_rows)
def test_fails_on_csv_with_fewer_elements_than_expected(self):
cid = interface.Cid(dev_test.CID_CUSTOMERS_XLS_PATH)
with validio.Reader(cid, dev_test.path_to_test_data("broken_customers_fewer_elements.csv")) as reader:
dev_test.assert_raises_and_fnmatches(
self, errors.DataError,
"*(R3C1): row must contain 5 fields but only has 4: *'19253', *'Webster Inc.', *'', *''?", reader.validate_rows)
def test_can_validate_proper_csv(self):
cid_path = dev_test.path_to_test_cid('customers.xls')
csv_path = dev_test.path_to_test_data('valid_customers.csv')
exit_code = applications.process(
['test_can_validate_proper_csv', cid_path, csv_path])
self.assertEqual(0, exit_code)
def test_can_read_ods_rows(self):
ods_path = dev_test.path_to_test_data('valid_customers.ods')
self._assert_rows_contain_data(rowio.ods_rows(ods_path))
def test_fails_on_csv_with_more_elements_than_expected(self):
cid_reader = interface.Cid(dev_test.CID_CUSTOMERS_XLS_PATH)
with validio.Reader(cid_reader, dev_test.path_to_test_data("broken_customers_more_elements.csv")) as reader:
dev_test.assert_raises_and_fnmatches(
self, errors.DataError,
"*(R3C1): row must contain 5 fields but has 6, additional values are: *'error'*", reader.validate_rows)
def test_fails_on_csv_with_more_elements_than_expected(self):
cid_reader = interface.Cid(dev_test.CID_CUSTOMERS_XLS_PATH)
with validio.Reader(cid_reader, dev_test.path_to_test_data("broken_customers_more_elements.csv")) as reader:
dev_test.assert_raises_and_fnmatches(
self, errors.DataError,
"*(R3C1): row must contain 5 fields but has 6, additional values are: *'error'*", reader.validate_rows)
def test_fails_on_csv_with_fewer_elements_than_expected(self):
cid = interface.Cid(dev_test.CID_CUSTOMERS_XLS_PATH)
with validio.Reader(cid, dev_test.path_to_test_data("broken_customers_fewer_elements.csv")) as reader:
dev_test.assert_raises_and_fnmatches(
self, errors.DataError,
"*(R3C1): row must contain 5 fields but only has 4: *'19253', *'Webster Inc.', *'', *''?", reader.validate_rows)
def setUp(self):
self._cutplace_app = applications.CutplaceApp()
self._cutplace_app.set_cid_from_path(_customers_cid_path)
self._broken_customers_non_csv_path = dev_test.path_to_test_data('valid_customers.ods')
_assert_valid_customers_has_no_barret()
def test_can_read_excel_rows(self):
excel_path = dev_test.path_to_test_data('valid_customers.xls')
self._assert_rows_contain_data(rowio.excel_rows(excel_path))
def test_can_convert_ods_to_rst(self):
source_ods_path = dev_test.path_to_test_data('valid_customers.ods')
target_path = dev_test.path_to_test_result('valid_customers_from__ods.rst')
_ods.main(['--format=rst', source_ods_path, target_path])
def test_can_convert_ods_to_csv(self):
source_ods_path = dev_test.path_to_test_data('valid_customers.ods')
target_path = dev_test.path_to_test_result('valid_customers_from__ods.csv')
_ods.main([source_ods_path, target_path])
def test_fails_on_kinky_file_name(self):
source_ods_path = dev_test.path_to_test_data('valid_customers.ods')
target_path = dev_test.path_to_test_result('kinky_file_name//\\:^$\\::/')
self.assertRaises(SystemExit, _ods.main, [source_ods_path, target_path])
def setUp(self):
self._cid_path = dev_test.path_to_test_cid("icd_customers.xls")
self._cid = interface.Cid(self._cid_path)
self._data_path = dev_test.path_to_test_data("valid_customers.csv")
def setUp(self):
self._cutplace_app = applications.CutplaceApp()
self._cutplace_app.set_cid_from_path(_customers_cid_path)
self._broken_customers_non_csv_path = dev_test.path_to_test_data(
'valid_customers.ods')
_assert_valid_customers_has_no_barret()
def test_can_validate_proper_data(self):
cid_path = dev_test.path_to_test_cid('customers.ods')
data_path = dev_test.path_to_test_data('valid_customers.csv')
self.assertEqual(0, applications.main(['test', cid_path, data_path]))
def test_can_deal_with_broken_data(self):
data_path = dev_test.path_to_test_data('broken_customers.csv')
self.assertEqual(1, applications.main(['test', _customers_cid_path, data_path]))
def test_can_deal_with_broken_data(self):
cid_path = dev_test.path_to_test_cid('customers.ods')
data_path = dev_test.path_to_test_data('broken_customers.csv')
self.assertEqual(1, applications.main(['test', cid_path, data_path]))
def test_fails_on_invalid_csv_with_duplicates(self):
cid = interface.Cid(dev_test.CID_CUSTOMERS_XLS_PATH)
with validio.Reader(cid, dev_test.path_to_test_data("broken_customers_with_duplicates.csv")) as reader:
self.assertRaises(errors.CheckError, reader.validate_rows)
def test_can_open_and_validate_fixed(self):
cid = interface.Cid(dev_test.path_to_test_cid("customers_fixed.xls"))
with validio.Reader(cid, dev_test.path_to_test_data("valid_customers_fixed.txt")) as reader:
reader.validate_rows()
def test_fails_on_broken_sheet(self):
source_ods_path = dev_test.path_to_test_data('valid_customers.ods')
target_path = dev_test.path_to_test_result('valid_customers_from__ods.csv')
self.assertRaises(SystemExit, _ods.main, ['--sheet=x', source_ods_path, target_path])
self.assertRaises(SystemExit, _ods.main, ['--sheet=0', source_ods_path, target_path])
self.assertRaises(SystemExit, _ods.main, ['--sheet=17', source_ods_path, target_path])
