def consider_interval(self, Begin, End):
if Begin > End:
raise RegularExpressionException("Character range: '-' requires character with 'lower code' to preceed\n" + \
"found range '%s-%s' which corresponds to %i-%i as unicode code points." % \
(utf8.map_unicode_to_utf8(Begin), utf8.map_unicode_to_utf8(End), Begin, End))
self.match_set.add_interval(Interval(Begin, End))
def consider_interval(self, Begin, End):
if Begin > End:
raise RegularExpressionException("Character range: '-' requires character with 'lower code' to preceed\n" + \
"found range '%s-%s' which corresponds to %i-%i as unicode code points." % \
(utf8.map_unicode_to_utf8(Begin), utf8.map_unicode_to_utf8(End), Begin, End))
self.match_set.add_interval(Interval(Begin, End))
def pump(LetterList):
txt = ""
LetterList.sort()
for x in LetterList:
if type(x) != list:
txt += map_unicode_to_utf8(x)
txt += "(%04X)" % x
else:
for xe in x:
txt += map_unicode_to_utf8(xe)
txt += "(%04X)" % xe
txt += ", "
if len(txt) != 0: txt = txt[:-2]
return txt
def pump(LetterList):
txt = ""
LetterList.sort()
for x in LetterList:
if type(x) != list:
txt += map_unicode_to_utf8(x)
txt += "(%04X)" % x
else:
for xe in x:
txt += map_unicode_to_utf8(xe)
txt += "(%04X)" % xe
txt += ", "
if len(txt) != 0: txt = txt[:-2]
return txt
def test(the_state_machine, string_to_match):
"""Uses the given state machine to match against the 'string_to_match'."""
print
print "string = ", string_to_match
letter_code_list = utf8.map_n_utf8_to_unicode(string_to_match)
norm_db, x, x = sm.get_state_index_normalization()
state_index = the_state_machine.init_state_index
letter_n = -1
for letter_code in letter_code_list:
letter_n += 1
if letter_n % 5 == 0: sys.stdout.write("\n")
state_index = sm.states[state_index].target_map.get_resulting_target_state_index(letter_code)
sys.stdout.write("'%s' --> (%s), " % (utf8.map_unicode_to_utf8(letter_code),
repr(norm_db[state_index]).replace("L","")))
if state_index == -1: break
print
def __print_set_single_characters(CharSet, Display, ScreenWidth):
assert Display in ["hex", "utf8"]
if Display == "hex":
CharactersPerLine = 8
ColumnWidth = 6
else:
CharactersPerLine = 32
ColumnWidth = 2
# just to make sure ...
character_list = CharacterList(CharSet)
if character_list.is_empty():
sys.stdout.write("<Result = Empty Character Set>\n")
return
# Avoid memory overflow for very large sets: get character by character
last_start_character_of_line = -1
last_horizontal_offset = 0
while 1 + 1 == 2:
character_code = character_list.next()
if character_code is None: break
start_character_of_line = character_code - character_code % CharactersPerLine
horizontal_offset = character_code - start_character_of_line
if start_character_of_line > last_start_character_of_line + CharactersPerLine:
sys.stdout.write("\n...")
if start_character_of_line != last_start_character_of_line:
sys.stdout.write("\n%05X: " % start_character_of_line)
last_horizontal_offset = 0
sys.stdout.write(" " * ColumnWidth *
(horizontal_offset - last_horizontal_offset - 1))
if Display == "hex":
sys.stdout.write("%05X " % character_code)
else:
if character_code >= 0x20:
sys.stdout.write("%s " % map_unicode_to_utf8(character_code))
else:
sys.stdout.write("? ")
last_start_character_of_line = start_character_of_line
last_horizontal_offset = horizontal_offset
def __print_set_single_characters(CharSet, Display, ScreenWidth):
assert Display in ["hex", "utf8"]
if Display == "hex":
CharactersPerLine = 8
ColumnWidth = 6
else:
CharactersPerLine = 32
ColumnWidth = 2
# just to make sure ...
character_list = CharacterList(CharSet)
if character_list.is_empty():
sys.stdout.write("<Result = Empty Character Set>\n")
return
# Avoid memory overflow for very large sets: get character by character
last_start_character_of_line = -1
last_horizontal_offset = 0
while 1 + 1 == 2:
character_code = character_list.next()
if character_code is None: break
start_character_of_line = character_code - character_code % CharactersPerLine
horizontal_offset = character_code - start_character_of_line
if start_character_of_line > last_start_character_of_line + CharactersPerLine:
sys.stdout.write("\n...")
if start_character_of_line != last_start_character_of_line:
sys.stdout.write("\n%05X: " % start_character_of_line)
last_horizontal_offset = 0
sys.stdout.write(" " * ColumnWidth * (horizontal_offset - last_horizontal_offset - 1))
if Display == "hex":
sys.stdout.write("%05X " % character_code)
else:
if character_code >= 0x20:
sys.stdout.write("%s " % map_unicode_to_utf8(character_code))
else:
sys.stdout.write("? ")
last_start_character_of_line = start_character_of_line
last_horizontal_offset = horizontal_offset
def test(the_state_machine, string_to_match):
"""Uses the given state machine to match against the 'string_to_match'."""
print
print "string = ", string_to_match
letter_code_list = utf8.map_n_utf8_to_unicode(string_to_match)
norm_db, x, x = sm.get_state_index_normalization()
state_index = the_state_machine.init_state_index
letter_n = -1
for letter_code in letter_code_list:
letter_n += 1
if letter_n % 5 == 0: sys.stdout.write("\n")
state_index = sm.states[
state_index].target_map.get_resulting_target_state_index(
letter_code)
sys.stdout.write("'%s' --> (%s), " %
(utf8.map_unicode_to_utf8(letter_code),
repr(norm_db[state_index]).replace("L", "")))
if state_index == -1: break
print
def do(sh):
"""Transforms an expression of the form [a-z0-9A-Z] into a NumberSet of
code points that corresponds to the characters and character ranges mentioned.
"""
assert sh.__class__.__name__ == "StringIO" \
or sh.__class__.__name__ == "file"
def __check_letter(stream, letter):
position = stream.tell()
if stream.read(1) == letter: return True
else:
stream.seek(position)
return False
# check, if the set is thought to be inverse (preceeded by '^')
tracker = Tracker()
if __check_letter(sh, "^"): tracker.negation_f = True
char_code = None
quote_checker = DoubleQuoteChecker(
) # Checks for " appearing twice. Some users did use
# # constructs such as "-" and ended up in confusing behavior.
while 1 + 1 == 2:
char_code = utf8.__read_one_utf8_code_from_stream(sh)
quote_checker.do(char_code)
if char_code == ord("-"):
raise RegularExpressionException(
"Character range operator '-' requires a preceding character as in 'a-z'."
)
elif char_code is None:
raise RegularExpressionException(
"Missing closing ']' in character range expression.")
elif char_code == ord("]"):
break
elif char_code == ord("\\"):
char_code = snap_backslashed_character.do(sh)
if not __check_letter(sh, "-"):
# (*) Normal character
tracker.consider_letter(char_code)
else:
# (*) Character range: 'character0' '-' 'character1'
char_code_2 = utf8.__read_one_utf8_code_from_stream(sh)
quote_checker.do(char_code_2)
if char_code_2 in [None, ord(']')]:
raise RegularExpressionException(
"Character range: '-' requires a character following '-'.")
elif char_code == ord("-"):
raise RegularExpressionException(
"Character range operator '-' followed by '-'.")
elif char_code_2 == ord("\\"):
char_code_2 = snap_backslashed_character.do(sh)
# value denotes 'end', i.e first character outside the interval => add 1
if char_code == char_code_2:
utf8_string = utf8.map_unicode_to_utf8(char_code)
raise RegularExpressionException("Character range '%s-%s' has only one element.\n" \
% (utf8_string, utf8_string) + \
"In this case avoid range expression for clarity.")
tracker.consider_interval(char_code, char_code_2 + 1)
if char_code is None: break
if tracker.negation_f:
return tracker.match_set.get_complement(
Setup.buffer_encoding.source_set)
else:
return tracker.match_set
def do(sh):
"""Transforms an expression of the form [a-z0-9A-Z] into a NumberSet of
code points that corresponds to the characters and character ranges mentioned.
"""
assert sh.__class__.__name__ == "StringIO" \
or sh.__class__.__name__ == "file"
def __check_letter(stream, letter):
position = stream.tell()
if stream.read(1) == letter: return True
else: stream.seek(position); return False
# check, if the set is thought to be inverse (preceeded by '^')
tracker = Tracker()
if __check_letter(sh, "^"): tracker.negation_f = True
char_code = None
quote_checker = DoubleQuoteChecker() # Checks for " appearing twice. Some users did use
# # constructs such as "-" and ended up in confusing behavior.
while 1 + 1 == 2:
char_code = utf8.__read_one_utf8_code_from_stream(sh)
quote_checker.do(char_code)
if char_code == ord("-"):
raise RegularExpressionException("Character range operator '-' requires a preceding character as in 'a-z'.")
elif char_code is None:
raise RegularExpressionException("Missing closing ']' in character range expression.")
elif char_code == ord("]"):
break
elif char_code == ord("\\"):
char_code = snap_backslashed_character.do(sh)
if not __check_letter(sh, "-"):
# (*) Normal character
tracker.consider_letter(char_code)
else:
# (*) Character range: 'character0' '-' 'character1'
char_code_2 = utf8.__read_one_utf8_code_from_stream(sh)
quote_checker.do(char_code_2)
if char_code_2 in [None, ord(']')]:
raise RegularExpressionException("Character range: '-' requires a character following '-'.")
elif char_code == ord("-"):
raise RegularExpressionException("Character range operator '-' followed by '-'.")
elif char_code_2 == ord("\\"):
char_code_2 = snap_backslashed_character.do(sh)
# value denotes 'end', i.e first character outside the interval => add 1
if char_code == char_code_2:
utf8_string = utf8.map_unicode_to_utf8(char_code)
raise RegularExpressionException("Character range '%s-%s' has only one element.\n" \
% (utf8_string, utf8_string) + \
"In this case avoid range expression for clarity.")
tracker.consider_interval(char_code, char_code_2 + 1)
if char_code is None: break
if tracker.negation_f:
return tracker.match_set.get_complement(Setup.buffer_codec.source_set)
else:
return tracker.match_set
#! /usr/bin/env python
import sys
import os
import StringIO
sys.path.insert(0, os.environ["QUEX_PATH"])
import quex.engine.misc.utf8 as utf8
if "--hwut-info" in sys.argv:
print "UTF8: Map UTF8 String To Unicode Values"
sys.exit(0)
# the unequal sign from the utf-8 manpage
txt = chr(0xE2) + chr(0x89) + chr(0xA0)
# unicode character (C) from the utf-8 man-page
txt += chr(0xC2) + chr(0xA9)
# unicode character 'Lam' in the arabic code page
txt += chr(0xd9) + chr(0x84)
cstr = StringIO.StringIO(txt)
print "unequal-sign: E2.89.A0 -> %x" % utf8.map_utf8_to_unicode(cstr)
print "copyright-sign: C2.A9 -> %x" % utf8.map_utf8_to_unicode(cstr)
print "arabic lam: D9.84 -> %x" % utf8.map_utf8_to_unicode(cstr)
# print map(lambda x: "%x" % x, utf8.read(cstr,10000))
print "%x -> %s" % (0x2260, utf8.map_unicode_to_utf8(0x2260))
print "%x -> %s" % (0xA9, utf8.map_unicode_to_utf8(0xA9))
print "%x -> %s" % (0x644, utf8.map_unicode_to_utf8(0x644))
#! /usr/bin/env python
import sys
import os
import StringIO
sys.path.insert(0, os.environ["QUEX_PATH"])
import quex.engine.misc.utf8 as utf8
if "--hwut-info" in sys.argv:
print "UTF8: Map UTF8 String To Unicode Values"
sys.exit(0)
# the unequal sign from the utf-8 manpage
txt = chr(0xE2) + chr(0x89) + chr(0xA0)
# unicode character (C) from the utf-8 man-page
txt += chr(0xC2) + chr(0xA9)
# unicode character 'Lam' in the arabic code page
txt += chr(0xd9) + chr(0x84)
cstr = StringIO.StringIO(txt)
print "unequal-sign: E2.89.A0 -> %x" % utf8.map_utf8_to_unicode(cstr)
print "copyright-sign: C2.A9 -> %x" % utf8.map_utf8_to_unicode(cstr)
print "arabic lam: D9.84 -> %x" % utf8.map_utf8_to_unicode(cstr)
# print map(lambda x: "%x" % x, utf8.read(cstr,10000))
print "%x -> %s" % (0x2260, utf8.map_unicode_to_utf8(0x2260))
print "%x -> %s" % (0xA9, utf8.map_unicode_to_utf8(0xA9))
print "%x -> %s" % (0x644, utf8.map_unicode_to_utf8(0x644))
byte1 = value >> 8
assert value & 0xFFFF0000 == 0, "Value = %08X" % value
result += chr(byte0) + chr(byte1)
return result
# (1) Critical Borders for UTF8:
# 0x80, 0x800, 0x10000, 0x10ffff (last code element)
# (2) Critical Borders for UTF16:
# 0xD800, 0xE000
unicode_character_list = range(0x01, 0x7F) \
+ range(0x800, 0x800 +10) \
+ range(0x10000, 0x10000 + 10)
utf8_list = ""
utf16_list = ""
unicode_list = ""
for char in unicode_character_list:
utf8_list += utf8.map_unicode_to_utf8(char)
utf16_list += word_4_split(utf16.unicode_to_utf16(char))
unicode_list += byte_4_split(char)
fh = open("example/utf8.txt", "wb")
fh.write(utf8_list)
fh.close()
fh = open("example/utf16le.txt", "wb")
fh.write("\xff\xfe" + utf16_list) # BOM: Little Endian
fh.close()
fh = open("example/ucs4le.txt", "wb")
fh.write(unicode_list)
fh.close()