def testCanFindOutDelimitedOptions(self):
fileNameToExpectedOptionsMap = {
"valid_customers.csv": {
sniff._ITEM_DELIMITER: ",",
sniff._ESCAPE_CHARACTER: "\"",
sniff._ENCODING: 'ascii',
sniff._QUOTE_CHARACTER: "\""
},
}
for testFileName, exptectedDelimitedOptions in fileNameToExpectedOptionsMap.items(
):
testFilePath = dev_test.getTestInputPath(testFileName)
testFile = open(testFilePath, "rb")
try:
actualDelimitedOptions = sniff.delimitedOptions(testFile)
# Add actual line delimiter as expected. We cannot provide a proper expected line
# delimiter in ``exptectedDelimitedOptions`` because the actual value depend on
# the platform the repository has been checked out to.
actualLineDelimiter = actualDelimitedOptions["lineDelimiter"]
self.assertTrue(actualLineDelimiter)
exptectedDelimitedOptions[
"lineDelimiter"] = actualLineDelimiter
self.assertEqual(actualDelimitedOptions, exptectedDelimitedOptions, \
"data format for file must be %r but is %r: %r" % (exptectedDelimitedOptions, actualDelimitedOptions, testFilePath))
finally:
testFile.close()
def testCanFindOutDelimitedOptions(self):
fileNameToExpectedOptionsMap = {
"valid_customers.csv": {
sniff._ITEM_DELIMITER: ",",
sniff._ESCAPE_CHARACTER: '"',
sniff._ENCODING: "ascii",
sniff._QUOTE_CHARACTER: '"',
}
}
for testFileName, exptectedDelimitedOptions in fileNameToExpectedOptionsMap.items():
testFilePath = dev_test.getTestInputPath(testFileName)
testFile = open(testFilePath, "rb")
try:
actualDelimitedOptions = sniff.delimitedOptions(testFile)
# Add actual line delimiter as expected. We cannot provide a proper expected line
# delimiter in ``exptectedDelimitedOptions`` because the actual value depend on
# the platform the repository has been checked out to.
actualLineDelimiter = actualDelimitedOptions["lineDelimiter"]
self.assertTrue(actualLineDelimiter)
exptectedDelimitedOptions["lineDelimiter"] = actualLineDelimiter
self.assertEqual(
actualDelimitedOptions,
exptectedDelimitedOptions,
"data format for file must be %r but is %r: %r"
% (exptectedDelimitedOptions, actualDelimitedOptions, testFilePath),
)
finally:
testFile.close()
def __init__(self, readable, targetQueue, dialect, encoding="ascii"):
assert dialect is not None
assert dialect.lineDelimiter is not None
assert dialect.itemDelimiter is not None
assert encoding is not None
super(_DelimitedRowProducerThread, self).__init__(readable, targetQueue)
self._log = logging.getLogger("cutplace.parsers")
dialectKeyowrds = {
sniff._ENCODING: encoding,
sniff._ESCAPE_CHARACTER: dialect.escapeChar,
sniff._ITEM_DELIMITER: dialect.itemDelimiter,
sniff._LINE_DELIMITER: dialect.lineDelimiter,
sniff._QUOTE_CHARACTER: dialect.quoteChar
}
delimitedOptions = sniff.delimitedOptions(readable, **dialectKeyowrds)
self.readable = readable
self.encoding = encoding
self.lineDelimiter = delimitedOptions[sniff._LINE_DELIMITER]
self.itemDelimiter = delimitedOptions[sniff._ITEM_DELIMITER]
self.quoteChar = delimitedOptions[sniff._QUOTE_CHARACTER]
self.escapeChar = delimitedOptions[sniff._ESCAPE_CHARACTER]
self.blanksAroundItemDelimiter = dialect.blanksAroundItemDelimiter
def testCanCreateSniffedReaderForEmptyData(self):
emptyReadable = StringIO.StringIO("")
self.assertTrue(sniff.delimitedOptions(emptyReadable))
emptyDataFormat = sniff.createDataFormat(emptyReadable)
self.assertTrue(emptyDataFormat)
self.assertEqual(emptyDataFormat.name, data.FORMAT_DELIMITED)
emptyReader = sniff.createReader(emptyReadable)
self.assertTrue(emptyReadable)
rowCount = 0
for _ in emptyReader:
rowCount += 1
self.assertEqual(rowCount, 0)
def testCanCreateSniffedReaderForEmptyData(self):
emptyReadable = StringIO.StringIO("")
self.assertTrue(sniff.delimitedOptions(emptyReadable))
emptyDataFormat = sniff.createDataFormat(emptyReadable)
self.assertTrue(emptyDataFormat)
self.assertEqual(emptyDataFormat.name, data.FORMAT_DELIMITED)
emptyReader = sniff.createReader(emptyReadable)
self.assertTrue(emptyReadable)
rowCount = 0
for _ in emptyReader:
rowCount += 1
self.assertEqual(rowCount, 0)
