def test_quoting_and_escapes(self): """Test POSIX lexer quoting and escapes""" # Basic quoting self.assertEqual(posix_lexer('"1 2"a \' 4\' "5\t\r\n6" " "\' \'', {}), ['1 2a', ' 4', '5\t\r\n6', ' '], "Single and double quotes must not split/munge whitespace") self.assertEqual(posix_lexer(r'''"'" '"' ''', {}), ["'", '"'], "Single and double quotes must be able to quote each other") # Escaping self.assertEqual(posix_lexer(r'"\""', {}), ['"'], "Double quotes must process escapes") self.assertEqual(posix_lexer(r'foo\" bar', {}), ['foo"', 'bar'], "Escapes must be allowed outside of quotes") # Quote stripping self.assertEqual(posix_lexer(r'"Quotes"Are"Stripped"Out', {}), ['QuotesAreStrippedOut'], "POSIX parsing should strip out unescaped double quotes") self.assertEqual(posix_lexer(r"'Quotes'Are'Stripped'Out", {}), ['QuotesAreStrippedOut'], "POSIX parsing should strip out unescaped single quotes") self.assertEqual(posix_lexer(r"foo '' bar", {}), ['foo', '', 'bar'], "In POSIX rules, quotes allow empty tokens") # In POSIX rules, can't even quote a quote inside single quotes self.assertRaises(ValueError, posix_lexer, r"'\''", {})
def test_ignore_hints(self): """Test that POSIX lexer ignores any provided hints""" for inStr, outList in self.test_pairs.items(): self.assertEqual(posix_lexer(inStr, self.bad_hints), outList)
def test_basic_functionality(self): """Test basic POSIX lexer functionality""" for inStr, outList in self.test_pairs.items(): self.assertEqual(posix_lexer(inStr, {}), outList)
def test_stdin_safety(self): """Test that POSIX lexer doesn't blindly pass None to shlex.split()""" self.assertEqual(posix_lexer(None, {}), [], "POSIX lexer must not let shlex.split() treat None input as " "an excuse to read from stdin.")