def test_parse_hex_digit(self):
p = unicode5.BasicParser(
u8(b"0123456789abcdefghijklmnopqrstuvwxyz"
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
b"\xd9\xa0\xd9\xa1\xd9\xa2\xd9\xa3\xd9\xa4\xd9\xa5"
b"\xd9\xa6\xd9\xa7\xd9\xa8\xd9\xa9"))
result = []
while p.the_char is not None:
digit = p.parse_hex_digit()
if digit is not None:
result.append(digit)
else:
p.next_char()
self.assertTrue(ul('').join(result) ==
ul('0123456789abcdefABCDEF'))
# and now binary
p = unicode5.BasicParser(
b"0123456789abcdefghijklmnopqrstuvwxyz"
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ")
result = []
while p.the_char is not None:
digit = p.parse_hex_digit()
if digit is not None:
result.append(digit)
else:
p.next_char()
self.assertTrue(join_bytes(result) ==
b'0123456789abcdefABCDEF')
def test_parse_hex_digit(self):
p = unicode5.BasicParser(
u8(b"0123456789abcdefghijklmnopqrstuvwxyz"
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
b"\xd9\xa0\xd9\xa1\xd9\xa2\xd9\xa3\xd9\xa4\xd9\xa5"
b"\xd9\xa6\xd9\xa7\xd9\xa8\xd9\xa9"))
result = []
while p.the_char is not None:
digit = p.parse_hex_digit()
if digit is not None:
result.append(digit)
else:
p.next_char()
self.assertTrue(ul('').join(result) == ul('0123456789abcdefABCDEF'))
# and now binary
p = unicode5.BasicParser(b"0123456789abcdefghijklmnopqrstuvwxyz"
b"ABCDEFGHIJKLMNOPQRSTUVWXYZ")
result = []
while p.the_char is not None:
digit = p.parse_hex_digit()
if digit is not None:
result.append(digit)
else:
p.next_char()
self.assertTrue(join_bytes(result) == b'0123456789abcdefABCDEF')
def mock_challenge():
data = [byte(0)] * 56
data[0:7] = list(b"NTLMSSP")
data[8] = byte(2)
data[16] = byte(56)
data[20] = byte(1)
data[21] = byte(0x82)
for i in range3(8):
data[24 + i] = byte(random.randint(0, 255))
return join_bytes(data)
def mock_challenge():
data = [byte(0)] * 56
data[0:7] = list(b"NTLMSSP")
data[8] = byte(2)
data[16] = byte(56)
data[20] = byte(1)
data[21] = byte(0x82)
for i in range3(8):
data[24 + i] = byte(random.randint(0, 255))
return join_bytes(data)
def test_byte(self):
# bytes are different in the two versions
if sys.version_info[0] < 3:
self.assertTrue(py2.byte(0x2A) == '\x2A')
self.assertTrue(isinstance(py2.byte(0x2A), type('*')))
self.assertFalse(isinstance(py2.byte(0x2A), type(u'*')))
else:
self.assertTrue(py2.byte(0x2A) == 0x2A)
self.assertTrue(isinstance(py2.byte(0x2A), int))
if sys.version_info[0] < 3:
self.assertTrue(py2.byte('*') == '\x2A')
self.assertTrue(py2.byte('\xe9') == '\xe9')
self.assertTrue(py2.byte(b'\xe9') == '\xe9')
try:
py2.byte(u'\xe9')
self.fail("py2.byte(unicode)")
except TypeError:
pass
self.assertTrue(py2.byte(bytearray(b'\xe9')) == '\xe9')
self.assertTrue(isinstance(py2.byte('*'), type('*')))
self.assertFalse(isinstance(py2.byte('*'), type(u'*')))
else:
self.assertTrue(py2.byte('*') == 0x2A)
self.assertTrue(py2.byte('\xe9') == 0xE9)
self.assertTrue(py2.byte(b'\xe9') == 0xE9)
self.assertTrue(py2.byte(u'\xe9') == 0xE9)
self.assertTrue(py2.byte(bytearray(b'\xe9')) == 0xE9)
try:
py2.byte('\u82f1')
self.fail("py2.byte(wide char)")
except ValueError:
pass
self.assertTrue(isinstance(py2.byte('*'), int))
data = b"hello"
self.assertTrue(py2.join_bytes(list(data)) == data)
try:
py2.byte(256)
self.fail("py2.byte(large)")
except ValueError:
pass
try:
py2.byte(-1)
self.fail("py2.byte(negative)")
except ValueError:
pass
# now test byte value...
self.assertTrue(py2.byte_value(py2.byte(0)) == 0)
self.assertTrue(py2.byte_value(py2.byte(0x30)) == 0x30)
self.assertTrue(py2.byte_value(py2.byte(0xFF)) == 0xFF)
def test_byte(self):
# bytes are different in the two versions
if sys.version_info[0] < 3:
self.assertTrue(py2.byte(0x2A) == '\x2A')
self.assertTrue(isinstance(py2.byte(0x2A), type('*')))
self.assertFalse(isinstance(py2.byte(0x2A), type(u'*')))
self.assertFalse(py2.is_byte(0x82F1))
self.assertFalse(py2.is_byte(256))
self.assertTrue(py2.is_byte('\x2A'))
self.assertTrue(py2.is_byte(b'\x2A'))
self.assertFalse(py2.is_byte(u'\x2A'))
self.assertFalse(py2.is_byte(u'**'))
self.assertTrue(py2.is_byte('**'[0]))
self.assertFalse(py2.is_byte(u'**'[0]))
else:
self.assertTrue(py2.byte(0x2A) == 0x2A)
self.assertTrue(isinstance(py2.byte(0x2A), int))
self.assertTrue(py2.is_byte(0x2A))
self.assertFalse(py2.is_byte(0x82F1))
self.assertFalse(py2.is_byte('\x2A'))
self.assertFalse(py2.is_byte(b'\x2A'))
self.assertFalse(py2.is_byte('**'[0]))
self.assertFalse(py2.is_byte('**'))
self.assertFalse(py2.is_byte(b'**'))
self.assertTrue(py2.is_byte(b'**'[0]))
if sys.version_info[0] < 3:
self.assertTrue(py2.byte('*') == '\x2A')
self.assertTrue(py2.byte('\xe9') == '\xe9')
self.assertTrue(py2.byte(b'\xe9') == '\xe9')
self.assertTrue(py2.byte(u'\xe9') == '\xe9')
self.assertTrue(py2.byte(bytearray(b'\xe9')) == '\xe9')
try:
py2.byte(u'\u82f1')
self.fail("py2.byte(wide char)")
except ValueError:
pass
try:
py2.byte('\u82f1')
self.fail("py2.byte(wide char, missing u)")
except ValueError:
pass
self.assertTrue(isinstance(py2.byte('*'), type('*')))
self.assertFalse(isinstance(py2.byte('*'), type(u'*')))
else:
self.assertTrue(py2.byte('*') == 0x2A)
self.assertTrue(py2.byte('\xe9') == 0xE9)
self.assertTrue(py2.byte(b'\xe9') == 0xE9)
self.assertTrue(py2.byte(u'\xe9') == 0xE9)
self.assertTrue(py2.byte(bytearray(b'\xe9')) == 0xE9)
try:
py2.byte('\u82f1')
self.fail("py2.byte(wide char)")
except ValueError:
pass
self.assertTrue(isinstance(py2.byte('*'), int))
# test joining iterables of byte
data = b"hello"
self.assertTrue(py2.join_bytes(list(data)) == data)
# test byte_to_bstr
data = py2.byte(0x40)
self.assertTrue(py2.byte_to_bstr(data) == b'@')
self.assertTrue(isinstance(py2.byte_to_bstr(data), bytes))
for i in py2.range3(256):
b = py2.byte(i)
self.assertTrue(py2.byte_to_bstr(b)[0] == b)
# Now move on to exception handling
try:
py2.byte(256)
self.fail("py2.byte(large)")
except ValueError:
pass
try:
py2.byte(-1)
self.fail("py2.byte(negative)")
except ValueError:
pass
# now test byte value...
self.assertTrue(py2.byte_value(py2.byte(0)) == 0)
self.assertTrue(py2.byte_value(py2.byte(0x30)) == 0x30)
self.assertTrue(py2.byte_value(py2.byte(0xFF)) == 0xFF)
# force bytes
self.assertTrue(py2.force_bytes("Hello") == b"Hello")
self.assertTrue(isinstance(py2.force_bytes("Hello"), bytes))
self.assertTrue(py2.force_bytes(b"Hello") == b"Hello")
self.assertTrue(isinstance(py2.force_bytes(b"Hello"), bytes))
self.assertTrue(py2.force_bytes(u"Hello") == b"Hello")
self.assertTrue(isinstance(py2.force_bytes(u"Hello"), bytes))
try:
py2.force_bytes(py2.ul('Caf\xe9'))
self.fail("force_bytes with high-bit character")
except UnicodeEncodeError:
pass
def test_byte(self):
# bytes are different in the two versions
if sys.version_info[0] < 3:
self.assertTrue(py2.byte(0x2A) == '\x2A')
self.assertTrue(isinstance(py2.byte(0x2A), type('*')))
self.assertFalse(isinstance(py2.byte(0x2A), type(u'*')))
self.assertFalse(py2.is_byte(0x82F1))
self.assertFalse(py2.is_byte(256))
self.assertTrue(py2.is_byte('\x2A'))
self.assertTrue(py2.is_byte(b'\x2A'))
self.assertFalse(py2.is_byte(u'\x2A'))
self.assertFalse(py2.is_byte(u'**'))
self.assertTrue(py2.is_byte('**'[0]))
self.assertFalse(py2.is_byte(u'**'[0]))
else:
self.assertTrue(py2.byte(0x2A) == 0x2A)
self.assertTrue(isinstance(py2.byte(0x2A), int))
self.assertTrue(py2.is_byte(0x2A))
self.assertFalse(py2.is_byte(0x82F1))
self.assertFalse(py2.is_byte('\x2A'))
self.assertFalse(py2.is_byte(b'\x2A'))
self.assertFalse(py2.is_byte('**'[0]))
self.assertFalse(py2.is_byte('**'))
self.assertFalse(py2.is_byte(b'**'))
self.assertTrue(py2.is_byte(b'**'[0]))
if sys.version_info[0] < 3:
self.assertTrue(py2.byte('*') == '\x2A')
self.assertTrue(py2.byte('\xe9') == '\xe9')
self.assertTrue(py2.byte(b'\xe9') == '\xe9')
self.assertTrue(py2.byte(u'\xe9') == '\xe9')
self.assertTrue(py2.byte(bytearray(b'\xe9')) == '\xe9')
try:
py2.byte(u'\u82f1')
self.fail("py2.byte(wide char)")
except ValueError:
pass
try:
py2.byte('\u82f1')
self.fail("py2.byte(wide char, missing u)")
except ValueError:
pass
self.assertTrue(isinstance(py2.byte('*'), type('*')))
self.assertFalse(isinstance(py2.byte('*'), type(u'*')))
else:
self.assertTrue(py2.byte('*') == 0x2A)
self.assertTrue(py2.byte('\xe9') == 0xE9)
self.assertTrue(py2.byte(b'\xe9') == 0xE9)
self.assertTrue(py2.byte(u'\xe9') == 0xE9)
self.assertTrue(py2.byte(bytearray(b'\xe9')) == 0xE9)
try:
py2.byte('\u82f1')
self.fail("py2.byte(wide char)")
except ValueError:
pass
self.assertTrue(isinstance(py2.byte('*'), int))
# test joining iterables of byte
data = b"hello"
self.assertTrue(py2.join_bytes(list(data)) == data)
# test byte_to_bstr
data = py2.byte(0x40)
self.assertTrue(py2.byte_to_bstr(data) == b'@')
self.assertTrue(isinstance(py2.byte_to_bstr(data), bytes))
for i in py2.range3(256):
b = py2.byte(i)
self.assertTrue(py2.byte_to_bstr(b)[0] == b)
# Now move on to exception handling
try:
py2.byte(256)
self.fail("py2.byte(large)")
except ValueError:
pass
try:
py2.byte(-1)
self.fail("py2.byte(negative)")
except ValueError:
pass
# now test byte value...
self.assertTrue(py2.byte_value(py2.byte(0)) == 0)
self.assertTrue(py2.byte_value(py2.byte(0x30)) == 0x30)
self.assertTrue(py2.byte_value(py2.byte(0xFF)) == 0xFF)
# force bytes
self.assertTrue(py2.force_bytes("Hello") == b"Hello")
self.assertTrue(isinstance(py2.force_bytes("Hello"), bytes))
self.assertTrue(py2.force_bytes(b"Hello") == b"Hello")
self.assertTrue(isinstance(py2.force_bytes(b"Hello"), bytes))
self.assertTrue(py2.force_bytes(u"Hello") == b"Hello")
self.assertTrue(isinstance(py2.force_bytes(u"Hello"), bytes))
try:
py2.force_bytes(py2.ul('Caf\xe9'))
self.fail("force_bytes with high-bit character")
except UnicodeEncodeError:
pass
def test_basic(self):
# OCTET = <any 8-bit sequence of data>
for c in range3(0, 256):
self.assertTrue(is_octet(byte(c)), "is_octet(byte(%i))" % c)
# CHAR = <any US-ASCII character (octets 0 - 127)>
for c in range3(0, 128):
self.assertTrue(is_char(byte(c)), "is_char(byte(%i))" % c)
for c in range3(128, 256):
self.assertFalse(is_char(byte(c)), "is_char(byte(%i))" % c)
# upalpha = <any US-ASCII uppercase letter "A".."Z">
upalpha = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_upalpha(c) == (c in upalpha), "is_upalpha(byte(%i))" % i)
# loalpha = <any US-ASCII lowercase letter "a".."z">
loalpha = b"abcdefghijklmnopqrstuvwxyz"
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_loalpha(c) == (c in loalpha), "is_loalpha(byte(%i))" % i)
# alpha = upalpha | loalpha
alpha = upalpha + loalpha
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_alpha(c) == (c in alpha), "is_alpha(byte(%i))" % i)
# digit = <any US-ASCII digit "0".."9">
digit = b"0123456789"
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_digit(c) == (c in digit), "is_digit(byte(%i))" % i)
# ctl = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
ctl = join_bytes([byte(ic) for ic in range3(0, 32)] + [byte(127)])
for i in range3(0, 256):
c = byte(i)
self.assertTrue(is_ctl(c) == (c in ctl), "is_ctl(byte(%i))" % i)
# CR = <US-ASCII CR, carriage return (13)>
self.assertTrue(CR == byte(13), "CR")
# LF = <US-ASCII LF, linefeed (10)>
self.assertTrue(LF == byte(10), "LF")
# SP = <US-ASCII SP, space (32)>
self.assertTrue(SP == byte(32), "SP")
# HT = <US-ASCII HT, horizontal-tab (9)>
self.assertTrue(HT == byte(9), "HT")
# DQUOTE = <US-ASCII double-quote mark (34)>
self.assertTrue(DQUOTE == byte(34), "DQUOTE")
# CRLF
self.assertTrue(CRLF == join_bytes([CR, LF]), "CRLF")
# LWS = [CRLF] 1*( SP | HT )
lws_test = "; \t ;\r\n ;\r\n \r\n\t \r "
p = OctetParser(lws_test)
self.assertTrue(p.parse_lws() is None, "No LWS")
p.parse(b";")
result = p.parse_lws()
self.assertTrue(result == b" \t ", "LWS no CRLF: %s" % repr(result))
p.parse(b";")
result = p.parse_lws()
self.assertTrue(result == b"\r\n ", "LWS with CRLF: %s" % repr(result))
p.parse(b";")
self.assertTrue(p.parse_lws() == b"\r\n ", "LWS ending at CRLF")
self.assertTrue(p.parse_lws() == b"\r\n\t ", "LWS ending at CRLF")
# TEXT = <any OCTET except CTLs, but including LWS>
p = OctetParser(lws_test)
self.assertTrue(len(p.parse_text()) == 16, "TEXT ending at CR")
p = OctetParser(lws_test)
self.assertTrue(p.parse_text(True) == b"; \t ; ; ", "Unfolded TEXT")
# hexdigit = "A" | "B" | "C" | "D" | "E" | "F" | "a" | "b" | "c"
# | "d" | "e" | "f" | digit
hexdigit = b"ABCDEFabcdef" + digit
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_hex(c) == (c in hexdigit), "is_hex(byte(%i))" % i)
# words, including comment, quoted string and qdpair
word_test = b'Hi(Hi\r\n Hi)Hi<Hi>Hi@Hi,Hi;Hi:Hi\\Hi"\\"Hi\r\n Hi\\""'\
b'/Hi[Hi]Hi?Hi=Hi{Hi}Hi Hi\tHi\r\n Hi'
word_testresult = [
b"Hi", b"(Hi Hi)", b"Hi",
byte("<"), b"Hi",
byte(">"), b"Hi",
byte("@"), b"Hi",
byte(","), b"Hi",
byte(";"), b"Hi",
byte(":"), b"Hi",
byte("\\"), b"Hi", b'"\\"Hi Hi\\""',
byte("/"), b"Hi",
byte("["), b"Hi",
byte("]"), b"Hi",
byte("?"), b"Hi",
byte("="), b"Hi",
byte("{"), b"Hi",
byte("}"), b"Hi", b"Hi", b"Hi", b"Hi"
]
p = OctetParser(word_test)
p = WordParser(p.parse_text(True))
self.assertTrue(p.words == word_testresult,
"basic word parser: %s" % repr(p.words))
# token
try:
self.assertTrue(check_token(b"Hi") == "Hi")
except ValueError:
self.fail("check_token(b'Hi')")
for t in word_testresult:
if t == b"Hi":
continue
try:
check_token(t)
self.fail("Non token checked OK: %s" % t)
except ValueError:
pass
def test_basic(self):
# OCTET = <any 8-bit sequence of data>
for c in range3(0, 256):
self.assertTrue(is_octet(byte(c)), "is_octet(byte(%i))" % c)
# CHAR = <any US-ASCII character (octets 0 - 127)>
for c in range3(0, 128):
self.assertTrue(is_char(byte(c)), "is_char(byte(%i))" % c)
for c in range3(128, 256):
self.assertFalse(is_char(byte(c)), "is_char(byte(%i))" % c)
# upalpha = <any US-ASCII uppercase letter "A".."Z">
upalpha = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_upalpha(c) == (c in upalpha), "is_upalpha(byte(%i))" % i)
# loalpha = <any US-ASCII lowercase letter "a".."z">
loalpha = b"abcdefghijklmnopqrstuvwxyz"
for i in range3(0, 256):
c = byte(i)
self.assertTrue(
is_loalpha(c) == (c in loalpha), "is_loalpha(byte(%i))" % i)
# alpha = upalpha | loalpha
alpha = upalpha + loalpha
for i in range3(0, 256):
c = byte(i)
self.assertTrue(is_alpha(c) == (c in alpha),
"is_alpha(byte(%i))" % i)
# digit = <any US-ASCII digit "0".."9">
digit = b"0123456789"
for i in range3(0, 256):
c = byte(i)
self.assertTrue(is_digit(c) == (c in digit),
"is_digit(byte(%i))" % i)
# ctl = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
ctl = join_bytes([byte(ic) for ic in range3(0, 32)] + [byte(127)])
for i in range3(0, 256):
c = byte(i)
self.assertTrue(is_ctl(c) == (c in ctl), "is_ctl(byte(%i))" % i)
# CR = <US-ASCII CR, carriage return (13)>
self.assertTrue(CR == byte(13), "CR")
# LF = <US-ASCII LF, linefeed (10)>
self.assertTrue(LF == byte(10), "LF")
# SP = <US-ASCII SP, space (32)>
self.assertTrue(SP == byte(32), "SP")
# HT = <US-ASCII HT, horizontal-tab (9)>
self.assertTrue(HT == byte(9), "HT")
# DQUOTE = <US-ASCII double-quote mark (34)>
self.assertTrue(DQUOTE == byte(34), "DQUOTE")
# CRLF
self.assertTrue(CRLF == join_bytes([CR, LF]), "CRLF")
# LWS = [CRLF] 1*( SP | HT )
lws_test = "; \t ;\r\n ;\r\n \r\n\t \r "
p = OctetParser(lws_test)
self.assertTrue(p.parse_lws() is None, "No LWS")
p.parse(b";")
result = p.parse_lws()
self.assertTrue(result == b" \t ", "LWS no CRLF: %s" % repr(result))
p.parse(b";")
result = p.parse_lws()
self.assertTrue(result == b"\r\n ", "LWS with CRLF: %s" % repr(result))
p.parse(b";")
self.assertTrue(p.parse_lws() == b"\r\n ", "LWS ending at CRLF")
self.assertTrue(p.parse_lws() == b"\r\n\t ", "LWS ending at CRLF")
# TEXT = <any OCTET except CTLs, but including LWS>
p = OctetParser(lws_test)
self.assertTrue(len(p.parse_text()) == 16, "TEXT ending at CR")
p = OctetParser(lws_test)
self.assertTrue(p.parse_text(True) == b"; \t ; ; ", "Unfolded TEXT")
# hexdigit = "A" | "B" | "C" | "D" | "E" | "F" | "a" | "b" | "c"
# | "d" | "e" | "f" | digit
hexdigit = b"ABCDEFabcdef" + digit
for i in range3(0, 256):
c = byte(i)
self.assertTrue(is_hex(c) == (c in hexdigit),
"is_hex(byte(%i))" % i)
# words, including comment, quoted string and qdpair
word_test = b'Hi(Hi\r\n Hi)Hi<Hi>Hi@Hi,Hi;Hi:Hi\\Hi"\\"Hi\r\n Hi\\""'\
b'/Hi[Hi]Hi?Hi=Hi{Hi}Hi Hi\tHi\r\n Hi'
word_testresult = [
b"Hi", b"(Hi Hi)", b"Hi", byte("<"), b"Hi", byte(">"), b"Hi",
byte("@"), b"Hi", byte(","), b"Hi", byte(";"), b"Hi", byte(":"),
b"Hi", byte("\\"), b"Hi", b'"\\"Hi Hi\\""', byte("/"), b"Hi",
byte("["), b"Hi", byte("]"), b"Hi", byte("?"), b"Hi", byte("="),
b"Hi", byte("{"), b"Hi", byte("}"), b"Hi", b"Hi", b"Hi", b"Hi"]
p = OctetParser(word_test)
p = WordParser(p.parse_text(True))
self.assertTrue(p.words == word_testresult, "basic word parser: %s" %
repr(p.words))
# token
try:
self.assertTrue(check_token(b"Hi") == "Hi")
except ValueError:
self.fail("check_token(b'Hi')")
for t in word_testresult:
if t == b"Hi":
continue
try:
check_token(t)
self.fail("Non token checked OK: %s" % t)
except ValueError:
pass
