def get_tokens(self,
message: str,
characters: Characters = None) -> List[Token]:
"""Convert the passed message into tokens.
:param characters: the Control Characters to use for tokenizing. If omitted, use a default set.
:param message: The EDI message
:return: Token[]
"""
self.characters = characters or Characters()
self._char = None
self._message = iter(message)
self._message_index = 0
self.read_next_char()
self.token_selector = {
self.characters.component_separator:
Token.Type.COMPONENT_SEPARATOR,
self.characters.data_separator: Token.Type.DATA_SEPARATOR,
self.characters.segment_terminator: Token.Type.TERMINATOR,
}
tokens = []
while not self.end_of_message():
token = self.get_next_token()
tokens.append(token)
return tokens
def __init__(self, characters: Characters = None):
super().__init__()
if characters is None:
characters = Characters()
self.characters = characters
self.replace_map = characters.escaped_syntax_dic
# Thanks to "Bor González Usach" for this wonderful piece of code:
# https://gist.github.com/bgusach/a967e0587d6e01e889fd1d776c5f3729
substrs = sorted(self.replace_map, key=len, reverse=True)
self.regexp = re.compile("|".join(map(re.escape, substrs)))
def __init__(self, characters: Characters = None):
super().__init__()
if characters is None:
characters = Characters()
self.characters = characters
def __init__(self, characters=None):
super(Serializer, self).__init__()
if characters is None:
characters = Characters()
self.characters = characters