def testValidCsvs(self):
VALID_PREFIX = "valid_"
testsInputFolder = dev_test.getTestFolder("input")
validCsvFileNames = _tools.listdirMatching(testsInputFolder,
VALID_PREFIX + ".*\\.csv",
".*with_head.*")
validCsvPaths = list(
os.path.join(testsInputFolder, fileName)
for fileName in validCsvFileNames)
for dataPath in validCsvPaths:
# Compute the base name of the related ICD.
baseFileName = os.path.basename(dataPath)
baseFileNameWithoutCsvSuffix = os.path.splitext(baseFileName)[0]
baseFileNameWithoutValidPrefixAndCsvSuffix = baseFileNameWithoutCsvSuffix[
len(VALID_PREFIX):]
# Compute the full path of the related ICD.
icdBaseName = baseFileNameWithoutValidPrefixAndCsvSuffix.split(
"_")[0]
icdPath = dev_test.getTestIcdPath(icdBaseName + ".csv")
if not os.path.exists(icdPath):
icdPath = dev_test.getTestIcdPath(icdBaseName + ".ods")
self.assertTrue(
os.path.exists(icdPath),
u"icd '%s' (or '*.csv') for data file '%s' must be created"
% (icdPath, dataPath))
# Now validate the data.
exitCode = _cutplace.main(["test_cutplace.py", icdPath, dataPath])
self.assertEqual(exitCode, 0)
def testCanValidateIcdWithPlugins(self):
icdPath = dev_test.getTestIcdPath("customers_with_plugins.ods")
exitCode = _cutplace.main([
"test_cutplace.py", "--plugins",
dev_test.getTestPluginsPath(), icdPath
])
self.assertEqual(exitCode, 0)
def _buildAndValidateManyCustomers():
icdOdsPath = dev_test.getTestIcdPath("customers.ods")
locCsvPath = dev_test.getTestFile("input", "lots_of_customers.csv")
dev_test.createLotsOfCustomersCsv(locCsvPath, customerCount=20000)
exitCode = _cutplace.main(["test_performance.py", icdOdsPath, locCsvPath])
if exitCode != 0:
raise ValueError(u"exit code of performance test must be 0 but is %d" % exitCode)
def _buildAndValidateManyCustomers():
icdOdsPath = dev_test.getTestIcdPath("customers.ods")
locCsvPath = dev_test.getTestFile("input", "lots_of_customers.csv")
dev_test.createLotsOfCustomersCsv(locCsvPath, customerCount=20000)
exitCode = _cutplace.main(["test_performance.py", icdOdsPath, locCsvPath])
if exitCode != 0:
raise ValueError(u"exit code of performance test must be 0 but is %d" %
exitCode)
def testValidatedRowsWithBrokenDataFormat(self):
try:
icdPath = dev_test.getTestIcdPath("native_excel_formats.ods")
icd = interface.InterfaceControlDocument()
icd.read(icdPath)
for _ in icd.validatedRows(self._validCostumersCsvPath):
pass
self.fail("XLRDError expected")
except xlrd.XLRDError:
# Ignore expected error cause by wrong data format.
pass
def testIcdCustomersXls(self):
icdCustomersIcdXlsPath = dev_test.getTestIcdPath("customers.xls")
readable = open(icdCustomersIcdXlsPath, "rb")
try:
for row in _parsers.excelReader(readable):
self.assertTrue(row is not None)
self.assertTrue(len(row))
except _parsers.CutplaceXlrdImportError:
_log.warning("ignored ImportError caused by missing xlrd")
finally:
readable.close()
def testSplitValidData(self):
icdPath = dev_test.getTestIcdPath("customers.ods")
dataPath = dev_test.getTestInputPath("valid_customers_iso-8859-1.csv")
exitCode = _cutplace.process(["test_cutplace.py", "--split", icdPath, dataPath])
self.assertEqual(exitCode, 0)
acceptedDataPath = dev_test.getTestInputPath("valid_customers_iso-8859-1_accepted.csv")
rejectedDataPath = dev_test.getTestInputPath("valid_customers_iso-8859-1_rejected.txt")
self.assertNotEqual(os.path.getsize(acceptedDataPath), 0)
self.assertEqual(os.path.getsize(rejectedDataPath), 0)
os.remove(acceptedDataPath)
os.remove(rejectedDataPath)
def testSplitBrokenData(self):
icdPath = dev_test.getTestIcdPath("customers.ods")
dataPath = dev_test.getTestInputPath("broken_customers.csv")
exitCode = _cutplace.main(["test_cutplace.py", "--split", icdPath, dataPath])
self.assertEquals(exitCode, 1)
acceptedDataPath = dev_test.getTestInputPath("broken_customers_accepted.csv")
rejectedDataPath = dev_test.getTestInputPath("broken_customers_rejected.txt")
self.assertNotEqual(os.path.getsize(acceptedDataPath), 0)
self.assertNotEqual(os.path.getsize(rejectedDataPath), 0)
os.remove(acceptedDataPath)
os.remove(rejectedDataPath)
def testIcdCustomersXls(self):
icdCustomersIcdXlsPath = dev_test.getTestIcdPath("customers.xls")
readable = open(icdCustomersIcdXlsPath, "rb")
try:
for row in _parsers.excelReader(readable):
self.assertTrue(row is not None)
self.assertTrue(len(row))
except _parsers.CutplaceXlrdImportError:
_log.warning("ignored ImportError caused by missing xlrd")
finally:
readable.close()
def testValidCsvs(self):
VALID_PREFIX = "valid_"
testsInputFolder = dev_test.getTestFolder("input")
validCsvFileNames = _tools.listdirMatching(testsInputFolder, VALID_PREFIX + ".*\\.csv", ".*with_head.*")
validCsvPaths = list(os.path.join(testsInputFolder, fileName) for fileName in validCsvFileNames)
for dataPath in validCsvPaths:
# Compute the base name of the related ICD.
baseFileName = os.path.basename(dataPath)
baseFileNameWithoutCsvSuffix = os.path.splitext(baseFileName)[0]
baseFileNameWithoutValidPrefixAndCsvSuffix = baseFileNameWithoutCsvSuffix[len(VALID_PREFIX):]
# Compute the full path of the related ICD.
icdBaseName = baseFileNameWithoutValidPrefixAndCsvSuffix.split("_")[0]
icdPath = dev_test.getTestIcdPath(icdBaseName + ".csv")
if not os.path.exists(icdPath):
icdPath = dev_test.getTestIcdPath(icdBaseName + ".ods")
self.assertTrue(os.path.exists(icdPath),
u"icd '%s' (or '*.csv') for data file '%s' must be created" % (icdPath, dataPath))
# Now validate the data.
exitCode = _cutplace.main(["test_cutplace.py", icdPath, dataPath])
self.assertEqual(exitCode, 0)
def testSplitValidData(self):
icdPath = dev_test.getTestIcdPath("customers.ods")
dataPath = dev_test.getTestInputPath("valid_customers_iso-8859-1.csv")
exitCode = _cutplace.process(
["test_cutplace.py", "--split", icdPath, dataPath])
self.assertEqual(exitCode, 0)
acceptedDataPath = dev_test.getTestInputPath(
"valid_customers_iso-8859-1_accepted.csv")
rejectedDataPath = dev_test.getTestInputPath(
"valid_customers_iso-8859-1_rejected.txt")
self.assertNotEqual(os.path.getsize(acceptedDataPath), 0)
self.assertEqual(os.path.getsize(rejectedDataPath), 0)
os.remove(acceptedDataPath)
os.remove(rejectedDataPath)
def testSplitBrokenData(self):
icdPath = dev_test.getTestIcdPath("customers.ods")
dataPath = dev_test.getTestInputPath("broken_customers.csv")
exitCode = _cutplace.main(
["test_cutplace.py", "--split", icdPath, dataPath])
self.assertEquals(exitCode, 1)
acceptedDataPath = dev_test.getTestInputPath(
"broken_customers_accepted.csv")
rejectedDataPath = dev_test.getTestInputPath(
"broken_customers_rejected.txt")
self.assertNotEqual(os.path.getsize(acceptedDataPath), 0)
self.assertNotEqual(os.path.getsize(rejectedDataPath), 0)
os.remove(acceptedDataPath)
os.remove(rejectedDataPath)
def testValidFixedTxt(self):
icdPath = dev_test.getTestIcdPath("customers_fixed.ods")
dataPath = dev_test.getTestInputPath("valid_customers_fixed.txt")
exitCode = _cutplace.main(["test_cutplace.py", icdPath, dataPath])
self.assertEqual(exitCode, 0)
def setUp(self):
self._validCostumersCsvPath = dev_test.getTestInputPath("valid_customers.csv")
self._brokenCostumersCsvPath = dev_test.getTestInputPath("broken_customers.csv")
icdPath = dev_test.getTestIcdPath("customers.csv")
self._icd = interface.InterfaceControlDocument()
self._icd.read(icdPath)
def _testValidateProperIcd(self):
self._post(dev_test.getTestIcdPath("customers.csv"))
def _testValidIcd(self, suffix):
assert suffix is not None
icdPath = dev_test.getTestIcdPath("customers." + suffix)
cutPlace = _cutplace.CutPlace()
cutPlace.setIcdFromFile(icdPath)
def testCanValidateDataWithPlugins(self):
icdPath = dev_test.getTestIcdPath("customers_with_plugins.ods")
dataPath = dev_test.getTestInputPath("valid_customers.csv")
exitCode = _cutplace.main(["test_cutplace.py", "--plugins", dev_test.getTestPluginsPath(), icdPath, dataPath])
self.assertEqual(exitCode, 0)
def testBrokenNonExistentDataPath(self):
icdPath = dev_test.getTestIcdPath("customers.ods")
self.assertRaises(EnvironmentError, _cutplace.process, ["test_cutplace.py", icdPath, "no-such-data.nix"])
def testValidFixedTxt(self):
icdPath = dev_test.getTestIcdPath("customers_fixed.ods")
dataPath = dev_test.getTestInputPath("valid_customers_fixed.txt")
exitCode = _cutplace.main(["test_cutplace.py", icdPath, dataPath])
self.assertEqual(exitCode, 0)
def testValidNativeExcelFormats(self):
icdPath = dev_test.getTestIcdPath("native_excel_formats.ods")
dataPath = dev_test.getTestInputPath("valid_native_excel_formats.xls")
exitCode = _cutplace.main(["test_cutplace.py", icdPath, dataPath])
self.assertEqual(exitCode, 0)
def testBrokenNonExistentDataPath(self):
icdPath = dev_test.getTestIcdPath("customers.ods")
self.assertRaises(EnvironmentError, _cutplace.process,
["test_cutplace.py", icdPath, "no-such-data.nix"])
def _testValidIcd(self, suffix):
assert suffix is not None
icdPath = dev_test.getTestIcdPath("customers." + suffix)
cutPlace = _cutplace.CutPlace()
cutPlace.setIcdFromFile(icdPath)
def testValidNativeExcelFormats(self):
icdPath = dev_test.getTestIcdPath("native_excel_formats.ods")
dataPath = dev_test.getTestInputPath("valid_native_excel_formats.xls")
exitCode = _cutplace.main(["test_cutplace.py", icdPath, dataPath])
self.assertEqual(exitCode, 0)
def _testValidateProperIcd(self):
self._post(dev_test.getTestIcdPath("customers.csv"))