def __sm_newline_default(self): """Default newline: '(\n)|(\r\n)' """ global cc_type_name_db newline_set = NumberSet(ord('\n')) retour_set = NumberSet(ord('\r')) before = self.specifier_count_op_map.find_occupier(newline_set, set()) if before is not None: error.warning("Trying to implement default newline: '\\n' or '\\r\\n'.\n" "The '\\n' option is not possible, since it has been occupied by '%s'.\n" \ "No newline can be defined by default." % cc_type_name_db[before.cc_type], before.sr, SuppressCode=NotificationDB.warning_default_newline_0A_impossible) # In this case, no newline can be defined! return sm = StateMachine.from_character_set(newline_set) if Setup.dos_carriage_return_newline_f: before = self.specifier_count_op_map.find_occupier(retour_set, set()) if before is not None: error.warning("Trying to implement default newline: '\\n' or '\\r\\n'.\n" "The '\\r\\n' option is not possible, since '\\r' has been occupied by '%s'." \ % cc_type_name_db[before.cc_type], before.sr, SuppressCode=NotificationDB.warning_default_newline_0D_impossible) else: sm.add_transition_sequence(sm.init_state_index, [retour_set, newline_set]) return sm
def __init__(self): error_range_0 = NumberSet([ Interval(0b00000000, 0b01111111 + 1), Interval(0b11000000, 0b11011111 + 1), Interval(0b11100000, 0b11101111 + 1), Interval(0b11110000, 0b11110111 + 1), Interval(0b11111000, 0b11111011 + 1), Interval(0b11111100, 0b11111101 + 1), ]).get_complement(NumberSet_All()) # Adapted later error_range_N = NumberSet(Interval(0b10000000, 0b10111111+1)) \ .get_complement(NumberSet_All()) # Adapted later error_range_by_code_unit_db = { 0: error_range_0, 1: error_range_N, 2: error_range_N, 3: error_range_N, 4: error_range_N, 5: error_range_N, 6: error_range_N, 7: error_range_N, 8: error_range_N } EncodingTrafoBySplit.__init__(self, "utf8", error_range_by_code_unit_db) self.UnchangedRange = 0x7F
def __indentation_add(Info): # (0) If all involved counts are single spaces, the 'counting' can be done # easily by subtracting 'end - begin', no adaption. indent_txt = " " * 16 if Info.homogeneous_spaces(): return "" def __do(txt, CharSet, Content): txt.append(indent_txt + "if( ") __condition(txt, CharSet) txt.append(" ) { ") txt.append(Content) txt.append(" }\\\n") txt = [] spaces_db = {} # Sort same space counts together grid_db = {} # Sort same grid counts together for name, count_parameter in Info.count_db.items(): count = count_parameter.get() character_set = Info.character_set_db[name].get() if count == "bad": continue # grid counts are indicated by negative integer for count. if count >= 0: spaces_db.setdefault(count, NumberSet()).unite_with(character_set) else: grid_db.setdefault(count, NumberSet()).unite_with(character_set) for count, character_set in spaces_db.items(): __do(txt, character_set, "(I) += %i;" % count) for count, character_set in grid_db.items(): __do(txt, character_set, "(I) += (%i - ((I) %% %i));" % (abs(count), abs(count))) return "".join(txt)
def add_transition(self, Trigger, TargetStateIdx): """Adds a transition according to trigger and target index. RETURNS: The target state index (may be created newly). """ assert type(TargetStateIdx) == long \ or TargetStateIdx is None \ or TargetStateIdx in E_StateIndices, "%s" % TargetStateIdx.__class__.__name__ assert Trigger.__class__ in (int, long, list, Interval, NumberSet) or Trigger is None if Trigger is None: # This is a shorthand to trigger via the remaining triggers Trigger = self.get_trigger_set_union().get_complement(Setup.buffer_encoding.source_set) elif type(Trigger) == long: Trigger = Interval(int(Trigger), int(Trigger+1)) elif type(Trigger) == int: Trigger = Interval(Trigger, Trigger+1) elif type(Trigger) == list: Trigger = NumberSet(Trigger, ArgumentIsYoursF=True) if Trigger.__class__ == Interval: if self.__db.has_key(TargetStateIdx): self.__db[TargetStateIdx].add_interval(Trigger) else: self.__db[TargetStateIdx] = NumberSet(Trigger, ArgumentIsYoursF=True) else: if self.__db.has_key(TargetStateIdx): self.__db[TargetStateIdx].unite_with(Trigger) else: self.__db[TargetStateIdx] = Trigger return TargetStateIdx
def do(BufferCodecName, BufferCodecFileName=""): from quex.engine.state_machine.transformation.base import EncodingTrafoUnicode from quex.engine.state_machine.transformation.table import EncodingTrafoByTable from quex.engine.state_machine.transformation.utf8_state_split import EncodingTrafoUTF8 from quex.engine.state_machine.transformation.utf16_state_split import EncodingTrafoUTF16 if BufferCodecName == "utf8": return EncodingTrafoUTF8() elif BufferCodecName == "utf16": return EncodingTrafoUTF16() elif BufferCodecFileName: os.path.splitext(os.path.basename(BufferCodecFileName)) try: os.path.splitext(os.path.basename(BufferCodecFileName)) except: error.log("cannot interpret string following '--codec-file'") return EncodingTrafoByTable(FileName=BufferCodecFileName) elif BufferCodecName == "unicode": # (Still, 'icu' or 'iconv' may provide converted content, but ...) # If the internal buffer is 'unicode', then the pattern's state # machines are not converted. The requirement for the pattern's # range is the same as for the 'buffer element chunks'. return EncodingTrafoUnicode(NumberSet(Interval(0, 0x110000)), NumberSet(Interval(0, 0x110000))) elif BufferCodecName == "unit-test": return EncodingTrafoUnicode(NumberSet_All(), NumberSet_All()) else: return EncodingTrafoByTable(BufferCodecName)
def test_on_UCS_sample_sets(Trafo, unicode_to_transformed_sequence): script_list = [ "Arabic", "Armenian", "Balinese", "Bengali", "Bopomofo", "Braille", "Buginese", "Buhid", "Canadian_Aboriginal", "Cherokee", "Common", "Cuneiform", "Cypriot", "Deseret", "Gothic", "Greek", "Hanunoo", "Hebrew", "Hiragana", "Inherited", "Kannada", "Han", "Katakana", "Kharoshthi", "Khmer", "Lao", "Latin", "Limbu", "Linear_B", "Malayalam", "Mongolian", "Myanmar", "New_Tai_Lue", "Nko", "Osmanya", "Ogham", "Old_Italic", "Old_Persian", "Phoenician", "Shavian", "Syloti_Nagri", "Syriac", "Tagalog", "Tagbanwa", "Tai_Le", "Tamil", "Telugu", "Thaana", "Thai", "Tibetan", "Tifinagh", "Ugaritic", "Yi" ] sets = [X(name) for name in script_list] orig = combination.do(map(lambda x: x.sm, sets)) state_n_before, result = transform(Trafo, orig) # print result.get_graphviz_string(Option="hex") for set in sets: set.check(result, unicode_to_transformed_sequence) print "Translated %i groups without abortion on error (OK)" % len(sets) union = NumberSet() for nset in map(lambda set: set.charset, sets): union.unite_with(nset) inverse_union = NumberSet(Interval(0, 0x110000)) inverse_union.subtract(union) # print inverse_union.get_string(Option="hex") check_negative(result, inverse_union.get_intervals(PromiseToTreatWellF=True), unicode_to_transformed_sequence)
def load_UnicodeData(self): fh = open_data_base_file("UnicodeData.txt") # some rows contain aliases, so they need to get converted into values property_general_category = self.db["gc"] property_bidi_class = self.db["bc"] def convert(Property, ValueAlias): """Convert specified ValueAlias to Value of the given property.""" if Property.alias_to_name_map.has_key(ValueAlias): return Property.alias_to_name_map[ValueAlias] return ValueAlias names_db = {} general_category_db = {} bidi_class_db = {} numeric_value_db = {} names_uc1_db = {} iso_comment_db = {} for line in fh.readlines(): if line.find("#") != -1: line = line[:line.find("#")] if line == "" or line.isspace(): continue x = line.split(";") code_point = int("0x" + x[0].strip(), 16) # CodePointIdx = 0 name = x[1].strip().replace(" ", "_") # NameIdx = 1 general_category = x[2].strip().replace( " ", "_") # GeneralCategoryIdx = 2 general_category = convert(property_general_category, general_category) bidi_class = x[4].strip().replace(" ", "_") # BidiClassIdx = 4 bidi_class = convert(property_bidi_class, bidi_class) numeric_value = x[6].strip() # NumericValueIdx = 6 uc1_name = x[10].strip().replace(" ", "_") # NameUC1Idx = 10 iso_comment = x[11].strip().replace(" ", "_") # ISO_CommentIdx = 11 names_db[name] = code_point general_category_db.setdefault( general_category, NumberSet()).quick_append_value(code_point) bidi_class_db.setdefault( bidi_class, NumberSet()).quick_append_value(code_point) numeric_value_db.setdefault( numeric_value, NumberSet()).quick_append_value(code_point) names_uc1_db[uc1_name] = code_point iso_comment_db[iso_comment] = str(code_point) self.db["na"].code_point_db = names_db # Name self.db["gc"].code_point_db = general_category_db # General Category self.db["bc"].code_point_db = bidi_class_db # BidiClass self.db["nv"].code_point_db = numeric_value_db # Numeric Value self.db["na1"].code_point_db = names_uc1_db # Name Unicode 1 self.db["isc"].code_point_db = iso_comment_db # ISO_Comment
def __init__(self): EncodingTrafoBySplit.__init__(self, "utf16", CodeUnitRange=NumberSet.from_range(0, 0x10000)) self.error_range_code_unit0 = NumberSet([ Interval(0x0000, 0xDC00), Interval(0xE000, 0x10000) ]).get_complement(NumberSet_All()) self.error_range_code_unit1 = NumberSet([ Interval(0xDC00, 0xE000) ]).get_complement(NumberSet_All())
def prepare(A_list, B_list): A = NumberSet() B = NumberSet() for begin, end in A_list: A.add_interval(Interval(begin, end)) for begin, end in B_list: B.add_interval(Interval(begin, end)) A.assert_consistency() B.assert_consistency() return A, B
def __wildcard_value_match(self, WildCardValue): result = NumberSet() value_list = self.get_wildcard_value_matches(WildCardValue) if len(value_list) == 0: return None for value in value_list: result.unite_with(NumberSet(self.code_point_db[value])) # No decoupling, since result is computed each fresh and new return result
def __init__(self): drain_set = NumberSet.from_range(0, 0x100) EncodingTrafoBySplit.__init__(self, "utf8", CodeUnitRange=drain_set) self.UnchangedRange = 0x7F self.error_range_byte0 = NumberSet([ Interval(0b00000000, 0b01111111+1), Interval(0b11000000, 0b11011111+1), Interval(0b11100000, 0b11101111+1), Interval(0b11110000, 0b11110111+1), Interval(0b11111000, 0b11111011+1), Interval(0b11111100, 0b11111101+1), ]).get_complement(NumberSet_All()) self.error_range_byteN = NumberSet( Interval(0b10000000, 0b10111111+1) ).get_complement(NumberSet_All())
def __init__(self): # A character in UTF16 is at maximum represented by two code units. # => Two error ranges. error_range_0 = NumberSet([ Interval(0x0000, 0xDC00), Interval(0xE000, 0x10000) ]).get_complement(NumberSet_All()) # Adapted later error_range_1 = NumberSet([Interval(0xDC00, 0xE000)]).get_complement( NumberSet_All()) # Adapted later error_range_by_code_unit_db = {0: error_range_0, 1: error_range_1} EncodingTrafoBySplit.__init__(self, "utf16", error_range_by_code_unit_db)
def test(UC): global trafo_cp037 x = NumberSet(UC) y = x.clone() x.transform_by_table(trafo_cp037) x.assert_consistency() print "0x%02X --> 0x%s" % (UC, x.get_string(Option="hex"))
def general_checks(loop_map, appendix_sm_list): print "#_[ Checks ]__________________________________________________" print print "character sets do not intersect", all_set = NumberSet() for lei in loop_map: assert lei.character_set is not None assert not lei.character_set.has_intersection(all_set) all_set.unite_with(lei.character_set) print "[ok]" print "count actions do not appear more than once", count_action_couple_set = set() count_action_plain_set = set() appendix_sm_id_set = set() print "[ok]" ## if "Split" in sys.argv or "Plain" in sys.argv: ## list_id_set = set(sm.get_id() for sm in appendix_sm_list) ## assert appendix_sm_id_set == list_id_set ## print "appendix sm-ids are the same in loop map and sm list: [ok]" print "exit character set exits: [%s]" % any(lei.aux_count_action is None for lei in loop_map) print
def general_checks(loop_map, appendix_sm_list): print "#_[ Checks ]__________________________________________________" print print "character sets do not intersect", all_set = NumberSet() for lei in loop_map: assert lei.character_set is not None assert not lei.character_set.has_intersection(all_set) all_set.unite_with(lei.character_set) print "[ok]" print "count actions do not appear more than once", count_action_couple_set = set() count_action_plain_set = set() exit_exists_f = False appendix_sm_id_set = set() for lei in loop_map: if lei.count_action is None: assert lei.appendix_sm_id is None exit_exists_f = True elif lei.appendix_sm_id is None: assert lei.incidence_id not in count_action_plain_set count_action_plain_set.add(lei.incidence_id) else: assert lei.incidence_id not in count_action_couple_set count_action_couple_set.add(lei.incidence_id) appendix_sm_id_set.add(lei.appendix_sm_id) print "[ok]" list_id_set = set(sm.get_id() for sm in appendix_sm_list) assert appendix_sm_id_set == list_id_set print "appendix sm-ids are the same in loop map and sm list: [ok]" print "exit character set exits: [%s]" % exit_exists_f print
def get(self): # Transform 'cursor' into a number set result = NumberSet() K = len(self.__cursor) if K == 0: return None k = 0 end = 0 while k < K - 1: begin = end + self.__cursor[k] end = begin + self.__cursor[k + 1] if end > self.N: self.__cursor.pop() K -= 1 break if begin != end: result.quick_append_interval(Interval(begin, end)) k += 2 # Increment cursor k = 0 while k < K: if k == 0: self.__cursor[k] += 2 if self.__cursor[k] < 8: break else: self.__cursor[k] += 1 if self.__cursor[k] < 3: break self.__cursor[k] = 1 k += 1 return result
def __whitespace_default(self): """Try to define default whitespace ' ' or '\t' if their positions are not yet occupied in the count_command_map. """ cs0 = NumberSet(ord(" ")) cs1 = NumberSet(ord("\t")) result = NumberSet() if not self.specifier_count_op_map.find_occupier(cs0, set()): result.unite_with(cs0) if not self.specifier_count_op_map.find_occupier(cs1, set()): result.unite_with(cs1) if result.is_empty(): error.log("Trying to implement default whitespace ' ' or '\\t' failed.\n" "Characters are occupied by other elements.", self.sr) return result
def get_trigger_set_to_target(self, TargetIdx): """Returns all triggers that lead to target 'TargetIdx'. If a trigger 'None' is returned it means that the epsilon transition triggers to target state. If the TargetIndex is omitted the set of all triggers, except the epsilon triggers, are returned. """ if self.__db.has_key(TargetIdx): return self.__db[TargetIdx] else: return NumberSet()
def LineColumnCount_Default(): global _LineColumnCount_Default if _LineColumnCount_Default is None: specifier = SpecifierCountActionMap() specifier.add(NumberSet(ord('\n')), "newline", 1, SourceRef_DEFAULT) specifier.add(NumberSet(ord('\t')), "grid", 4, SourceRef_DEFAULT) specifier.define_else("space", 1, SourceRef_DEFAULT) # Define: "\else" count_command_map = specifier.finalize( Setup.buffer_codec.source_set.minimum(), Setup.buffer_codec.source_set.supremum(), # Apply: "\else" SourceRef_DEFAULT) _LineColumnCount_Default = LineColumnCount(SourceRef_DEFAULT, count_command_map) return _LineColumnCount_Default
def do(section_list, fh): """Parses a codec information file. The described codec can only be a 'static character length' encoding. That is every character in the code occupies the same number of bytes. RETURNS: [0] Set of characters in unicode which are covered by the described codec. [1] Range of values in the codec elements. """ source_set = NumberSet() drain_set = NumberSet() error_str = None try: while error_str is None: skip_whitespace(fh) source_begin = read_integer(fh) if source_begin is None: error_str = "Missing integer (source interval begin) in codec file." continue skip_whitespace(fh) source_size = read_integer(fh) if source_size is None: error_str = "Missing integer (source interval size) in codec file." continue skip_whitespace(fh) target_begin = read_integer(fh) if target_begin is None: error_str = "Missing integer (target interval begin) in codec file." continue source_end = source_begin + source_size list.append(section_list, [source_begin, source_end, target_begin]) source_set.add_interval(Interval(source_begin, source_end)) drain_set.add_interval( Interval(target_begin, target_begin + source_size)) except EndOfStreamException: pass return source_set, drain_set, error_str
def __get_remaining_set(self): ignored = (E_CharacterCountType.BAD, E_CharacterCountType.BEGIN_NEWLINE_SUPPRESSOR, E_CharacterCountType.BEGIN_NEWLINE, E_CharacterCountType.END_NEWLINE) result = NumberSet() for character_set, info in self.__map: if info.cc_type in ignored: continue result.unite_with(character_set) return result.get_complement(Setup.buffer_codec.source_set)
def __init__(self, SourceSet, Name="unicode"): # Plain 'Unicode' associates a character with a single code unit, i.e. # its 'code point'. # => Only code unit '0' is specified and everything is allowed. # ('everything allowed' is disputable, since certain ranges are # disallowed.) assert Name in ("unicode", "utf32") error_range_by_code_unit_db = {0: NumberSet()} EncodingTrafo.__init__(self, Name, SourceSet, error_range_by_code_unit_db)
def do_Number(self, number): """RETURNS: A interval sequence that implements the given number. """ result = self.do_NumberSet(NumberSet(number)) # result: list of interval sequences that implement number set. # number set contains only one element. # => one interval sequence and # the interval size must be one. if result is None: return None else: return result[0]
def _assert_consistency(self): assert not any(lme is None for lme in self) assert not any(lme.character_set is None for lme in self) assert not any((lme.iid_couple_terminal is None) and (lme.code is None) for lme in self) # Assert: Transition triggers do not intersect! total = NumberSet() for lme in self: assert not lme.character_set.has_intersection(total) total.unite_with(lme.character_set)
def verify(A, TrafoInfo): result = NumberSet() for interval in A.get_intervals(): for x in range(interval.begin, interval.end): for source_begin, source_end, target_begin in TrafoInfo: if x >= source_begin and x < source_end: offset = x - source_begin y = target_begin + offset result.add_interval(Interval(y)) result.assert_consistency() return result
def test(X): print "#_______________________________________________" nset = NumberSet([ Interval(x, y) for x, y in X]) clone = nset.clone() print "#NumberSet: %s" % nset result = nset.clone() result.complement(all) print "#NumberSet.inverse: %s" % result assert result.is_equal(nset.get_complement(all)) assert result.intersection(nset).is_empty() assert result.union(nset).is_all()
def load_Composition_Exclusion(self): # Column 0 contains what is interesting ... table = parse_table("CompositionExclusions.txt", NumberColumnList=[0]) number_set = NumberSet() for row in table: begin = row[0] number_set.quick_append_interval(Interval(begin, begin + 1)) number_set.clean() self.db["CE"].code_point_db = number_set
def get_number_set(Cursor): if len(Cursor) == 2: return S_None cursor = copy(Cursor) cursor.pop(0) # element 0 and '-1' are just helping values, no interval borders. result = [] while len(cursor) != 1: begin = cursor.pop(0) end = cursor.pop(0) result.append(Interval(begin, end)) return NumberSet(result)
def CounterSetupLineColumn_Default(): global _CounterSetupLineColumn_Default if _CounterSetupLineColumn_Default is None: count_command_map = CountOpMap() count_command_map.add(NumberSet(ord('\n')), "newline", 1, SourceRef_DEFAULT) count_command_map.add(NumberSet(ord('\t')), "grid", 4, SourceRef_DEFAULT) count_command_map.define_else("space", 1, SourceRef_DEFAULT) # Define: "\else" count_command_map.assign_else_count_command( Setup.buffer_codec.source_set.minimum(), Setup.buffer_codec.source_set.supremum(), # Apply: "\else" SourceRef_DEFAULT) _CounterSetupLineColumn_Default = ParserDataLineColumn( SourceRef_DEFAULT, count_command_map) return _CounterSetupLineColumn_Default
def get_all(): """RETURNS: A state machine that 'eats' absolutely everything, i.e. .--- \Any ---. | | (0)--- \Any --->(( 0 ))<--------' """ result = StateMachine() i = index.get() state = State(AcceptanceF=True) state.add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), i) result.states[i] = state result.get_init_state().add_transition( NumberSet(Interval(-sys.maxint, sys.maxint)), i) return result