def testCanFindOutDelimitedOptions(self): fileNameToExpectedOptionsMap = { "valid_customers.csv": { sniff._ITEM_DELIMITER: ",", sniff._ESCAPE_CHARACTER: "\"", sniff._ENCODING: 'ascii', sniff._QUOTE_CHARACTER: "\"" }, } for testFileName, exptectedDelimitedOptions in fileNameToExpectedOptionsMap.items( ): testFilePath = dev_test.getTestInputPath(testFileName) testFile = open(testFilePath, "rb") try: actualDelimitedOptions = sniff.delimitedOptions(testFile) # Add actual line delimiter as expected. We cannot provide a proper expected line # delimiter in ``exptectedDelimitedOptions`` because the actual value depend on # the platform the repository has been checked out to. actualLineDelimiter = actualDelimitedOptions["lineDelimiter"] self.assertTrue(actualLineDelimiter) exptectedDelimitedOptions[ "lineDelimiter"] = actualLineDelimiter self.assertEqual(actualDelimitedOptions, exptectedDelimitedOptions, \ "data format for file must be %r but is %r: %r" % (exptectedDelimitedOptions, actualDelimitedOptions, testFilePath)) finally: testFile.close()
def testCanFindOutDelimitedOptions(self): fileNameToExpectedOptionsMap = { "valid_customers.csv": { sniff._ITEM_DELIMITER: ",", sniff._ESCAPE_CHARACTER: '"', sniff._ENCODING: "ascii", sniff._QUOTE_CHARACTER: '"', } } for testFileName, exptectedDelimitedOptions in fileNameToExpectedOptionsMap.items(): testFilePath = dev_test.getTestInputPath(testFileName) testFile = open(testFilePath, "rb") try: actualDelimitedOptions = sniff.delimitedOptions(testFile) # Add actual line delimiter as expected. We cannot provide a proper expected line # delimiter in ``exptectedDelimitedOptions`` because the actual value depend on # the platform the repository has been checked out to. actualLineDelimiter = actualDelimitedOptions["lineDelimiter"] self.assertTrue(actualLineDelimiter) exptectedDelimitedOptions["lineDelimiter"] = actualLineDelimiter self.assertEqual( actualDelimitedOptions, exptectedDelimitedOptions, "data format for file must be %r but is %r: %r" % (exptectedDelimitedOptions, actualDelimitedOptions, testFilePath), ) finally: testFile.close()
def __init__(self, readable, targetQueue, dialect, encoding="ascii"): assert dialect is not None assert dialect.lineDelimiter is not None assert dialect.itemDelimiter is not None assert encoding is not None super(_DelimitedRowProducerThread, self).__init__(readable, targetQueue) self._log = logging.getLogger("cutplace.parsers") dialectKeyowrds = { sniff._ENCODING: encoding, sniff._ESCAPE_CHARACTER: dialect.escapeChar, sniff._ITEM_DELIMITER: dialect.itemDelimiter, sniff._LINE_DELIMITER: dialect.lineDelimiter, sniff._QUOTE_CHARACTER: dialect.quoteChar } delimitedOptions = sniff.delimitedOptions(readable, **dialectKeyowrds) self.readable = readable self.encoding = encoding self.lineDelimiter = delimitedOptions[sniff._LINE_DELIMITER] self.itemDelimiter = delimitedOptions[sniff._ITEM_DELIMITER] self.quoteChar = delimitedOptions[sniff._QUOTE_CHARACTER] self.escapeChar = delimitedOptions[sniff._ESCAPE_CHARACTER] self.blanksAroundItemDelimiter = dialect.blanksAroundItemDelimiter
def testCanCreateSniffedReaderForEmptyData(self): emptyReadable = StringIO.StringIO("") self.assertTrue(sniff.delimitedOptions(emptyReadable)) emptyDataFormat = sniff.createDataFormat(emptyReadable) self.assertTrue(emptyDataFormat) self.assertEqual(emptyDataFormat.name, data.FORMAT_DELIMITED) emptyReader = sniff.createReader(emptyReadable) self.assertTrue(emptyReadable) rowCount = 0 for _ in emptyReader: rowCount += 1 self.assertEqual(rowCount, 0)