class EncodingTrafoUTF16(EncodingTrafoBySplit): UnchangedRange = 0x10000 def __init__(self): EncodingTrafoBySplit.__init__(self, "utf16", CodeUnitRange=NumberSet.from_range(0, 0x10000)) self.error_range_code_unit0 = NumberSet([ Interval(0x0000, 0xDC00), Interval(0xE000, 0x10000) ]).get_complement(NumberSet_All()) self.error_range_code_unit1 = NumberSet([ Interval(0xDC00, 0xE000) ]).get_complement(NumberSet_All()) def prune(self, number_set): global ForbiddenRange number_set.subtract(ForbiddenRange) number_set.mask(0, 0x110000) def get_interval_sequences(self, Orig): interval_1word, intervals_2word = _get_contigous_intervals(Orig) result = [] if interval_1word is not None: result.append([interval_1word]) if intervals_2word is not None: result.extend( _get_trigger_sequence_for_interval(interval) for interval in intervals_2word ) return result def lexatom_n_per_character(self, CharacterSet): """If all characters in a unicode character set state machine require the same number of bytes to be represented this number is returned. Otherwise, 'None' is returned. RETURNS: N > 0 number of bytes required to represent any character in the given state machine. None characters in the state machine require different numbers of bytes. """ assert isinstance(CharacterSet, NumberSet) interval_list = CharacterSet.get_intervals(PromiseToTreatWellF=True) front = interval_list[0].begin # First element of number set back = interval_list[-1].end - 1 # Last element of number set # Determine number of bytes required to represent the first and the # last character of the number set. The number of bytes per character # increases monotonously, so only borders have to be considered. front_chunk_n = len(unicode_to_utf16(front)) back_chunk_n = len(unicode_to_utf16(back)) if front_chunk_n != back_chunk_n: return None else: return front_chunk_n def _plug_encoding_error_detectors(self, sm): """Adorn states with transitions to the 'on_encoding_error' handler if the input value lies beyond the limits. The state machine is an implementation of linear sequences of intervals. Thus, the 'code unit position' can be be determined by the number of transitions from the init state. sm = mini state machine that implements the transition sequences. Bad ranges for code units (a 2 byte): 1st: 0xDC00 - 0xCFFF 2nd: 0x0000 - 0xDBFF, 0xE000 - 0x11000 """ # 'CodeUnit[0]' appears at the init state # (Adapt trigger map before entering the 'on bad lexatom state' init_tm = sm.get_init_state().target_map.get_map() workset = set(init_tm.iterkeys()) for si, trigger_set in init_tm.iteritems(): assert not trigger_set.has_intersection(self.error_range_code_unit0) bad_lexatom_state_index = self._plug_encoding_error_detector_single_state(sm, init_tm) # 'CodeUnit[>0]' appear all at later states done = set([bad_lexatom_state_index]) while workset: si = workset.pop() tm = sm.states[si].target_map.get_map() done.add(si) # Only add bad lexatom detection to state that transit on lexatoms # (Bad lexatom states, btw. do not have transitions) if not tm: continue for trigger_set in tm.itervalues(): assert not trigger_set.has_intersection(self.error_range_code_unit1) workset.update(new_si for new_si in tm.iterkeys() if new_si not in done) tm[bad_lexatom_state_index] = self.error_range_code_unit1 def _plug_encoding_error_detector_single_state(self, sm, target_map): bad_lexatom_state_index = sm.access_bad_lexatom_state() if target_map: target_map[bad_lexatom_state_index] = self.error_range_code_unit0 return bad_lexatom_state_index def adapt_source_and_drain_range(self, LexatomByteN): EncodingTrafoBySplit.adapt_source_and_drain_range(self, LexatomByteN) self.error_range_code_unit0.mask_interval(self.lexatom_range) self.error_range_code_unit1.mask_interval(self.lexatom_range) if LexatomByteN == -1: return elif LexatomByteN >= 2: return else: # if there are less than 2 byte for the lexatoms, then only the # unicode range from 0x00 to 0xFF can be treated. self.source_set.mask(0x00, 0x100)
class EncodingTrafoUTF8(EncodingTrafoBySplit): def __init__(self): drain_set = NumberSet.from_range(0, 0x100) EncodingTrafoBySplit.__init__(self, "utf8", CodeUnitRange=drain_set) self.UnchangedRange = 0x7F self.error_range_byte0 = NumberSet([ Interval(0b00000000, 0b01111111+1), Interval(0b11000000, 0b11011111+1), Interval(0b11100000, 0b11101111+1), Interval(0b11110000, 0b11110111+1), Interval(0b11111000, 0b11111011+1), Interval(0b11111100, 0b11111101+1), ]).get_complement(NumberSet_All()) self.error_range_byteN = NumberSet( Interval(0b10000000, 0b10111111+1) ).get_complement(NumberSet_All()) def adapt_source_and_drain_range(self, LexatomByteN): EncodingTrafoBySplit.adapt_source_and_drain_range(self, LexatomByteN) self.error_range_byte0.mask_interval(self.lexatom_range) self.error_range_byteN.mask_interval(self.lexatom_range) def prune(self, X): pass def get_interval_sequences(self, Orig): """Orig = Unicode Trigger Set. It is transformed into a sequence of intervals that cover all elements of Orig in a representation as UTF8 code units. A transition from state '1' to state '2' on 'Orig' is then equivalent to the transitions along the code unit sequence. """ db = _split_by_transformed_sequence_length(Orig) if db is None: return [] result = [] for seq_length, interval in db.items(): interval_list = _get_contiguous_interval_sequences(interval, seq_length) result.extend( _get_trigger_sequence_for_contigous_byte_range_interval(interval, seq_length) for interval in interval_list) return result def lexatom_n_per_character(self, CharacterSet): """If all characters in a unicode character set state machine require the same number of bytes to be represented this number is returned. Otherwise, 'None' is returned. RETURNS: N > 0 number of bytes required to represent any character in the given state machine. None characters in the state machine require different numbers of bytes. """ assert isinstance(CharacterSet, NumberSet) interval_list = CharacterSet.get_intervals(PromiseToTreatWellF=True) front = interval_list[0].begin # First element of number set back = interval_list[-1].end - 1 # Last element of number set # Determine number of bytes required to represent the first and the # last character of the number set. The number of bytes per character # increases monotonously, so only borders have to be considered. front_chunk_n = len(unicode_to_utf8(front)) back_chunk_n = len(unicode_to_utf8(back)) if front_chunk_n != back_chunk_n: return None else: return front_chunk_n def _plug_encoding_error_detectors(self, sm): """Adorn states with transitions to the 'on_encoding_error' handler if the input value lies beyond the limits. The state machine is an implementation of linear sequences of intervals. Thus, the 'byte position' can be be determined by the number of transitions from the init state. sm = mini state machine that implements the transition sequences. UTF8 Encodings in binary look like the following (see 'man utf8'). 1 byte: 0xxxxxxx 2 byte: 110xxxxx 10xxxxxx 3 byte: 1110xxxx 10xxxxxx 10xxxxxx 4 byte: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 5 byte: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 6 byte: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxxx The resulting byte ranges can be observed in 'error_range_byte0' for Byte[0] and 'error_range_byteN' for Byte[>0]. """ # 'Byte[0]' appears at the init state # (Adapt trigger map before entering the 'on bad lexatom state' init_tm = sm.get_init_state().target_map.get_map() workset = set(init_tm.iterkeys()) for si, trigger_set in init_tm.iteritems(): assert not trigger_set.has_intersection(self.error_range_byte0) bad_lexatom_state_index = self._plug_encoding_error_detector_single_state(sm, init_tm) # 'Byte[>0]' appear all at later states done = set([bad_lexatom_state_index]) while workset: si = workset.pop() tm = sm.states[si].target_map.get_map() done.add(si) # Only add bad lexatom detection to state that transit on lexatoms # (Bad lexatom states, btw. do not have transitions) if not tm: continue for trigger_set in tm.itervalues(): assert not trigger_set.has_intersection(self.error_range_byteN) workset.update(new_si for new_si in tm.iterkeys() if new_si not in done) tm[bad_lexatom_state_index] = self.error_range_byteN def _plug_encoding_error_detector_single_state(self, sm, target_map): bad_lexatom_state_index = sm.access_bad_lexatom_state() if target_map: target_map[bad_lexatom_state_index] = self.error_range_byte0 return bad_lexatom_state_index