def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() EscapedStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() SuffixedStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() PrefixedIntegerTokenBuilder.__escape_z__() SuffixedIntegerTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() RealExponentTokenBuilder.__escape_z__() SuffixedRealTokenBuilder.__escape_z__() IdentifierTokenBuilder.__escape_z__() PrefixedIdentifierTokenBuilder.__escape_z__() CaseInsensitiveListTokenBuilder.__escape_z__() CaseSensitiveListTokenBuilder.__escape_z__() SingleCharacterTokenBuilder.__escape_z__() SlashSlashCommentTokenBuilder.__escape_z__() SlashStarCommentTokenBuilder.__escape_z__() ClassTypeTokenBuilder.__escape_z__() HexRealExponentTokenBuilder.__escape_z__() NestedCommentTokenBuilder.__escape_z__() return 'Escape ?Z'
def __escape_z__():
InvalidTokenBuilder.__escape_z__()
WhitespaceTokenBuilder.__escape_z__()
NewlineTokenBuilder.__escape_z__()
IdentifierTokenBuilder.__escape_z__()
EscapedStringTokenBuilder.__escape_z__()
CaseInsensitiveListTokenBuilder.__escape_z__()
CaseSensitiveListTokenBuilder.__escape_z__()
TripleQuoteStringTokenBuilder.__escape_z__()
LeadToEndOfLineTokenBuilder.__escape_z__()
BlockTokenBuilder.__escape_z__()
BraceCommentTokenBuilder.__escape_z__()
SlashSlashCommentTokenBuilder.__escape_z__()
SlashStarCommentTokenBuilder.__escape_z__()
GenericNumberTokenBuilder.__escape_z__()
return 'Escape ?Z'
def __escape_z__():
InvalidTokenBuilder.__escape_z__()
WhitespaceTokenBuilder.__escape_z__()
NewlineTokenBuilder.__escape_z__()
EscapedStringTokenBuilder.__escape_z__()
IntegerTokenBuilder.__escape_z__()
IntegerExponentTokenBuilder.__escape_z__()
RealTokenBuilder.__escape_z__()
RealExponentTokenBuilder.__escape_z__()
IdentifierTokenBuilder.__escape_z__()
CaseInsensitiveListTokenBuilder.__escape_z__()
CaseSensitiveListTokenBuilder.__escape_z__()
PrefixedIntegerTokenBuilder.__escape_z__()
RegexTokenBuilder.__escape_z__()
SlashSlashCommentTokenBuilder.__escape_z__()
SlashStarCommentTokenBuilder.__escape_z__()
return 'Escape ?Z'
def __escape_z__():
InvalidTokenBuilder.__escape_z__()
WhitespaceTokenBuilder.__escape_z__()
NewlineTokenBuilder.__escape_z__()
EscapedStringTokenBuilder.__escape_z__()
IntegerTokenBuilder.__escape_z__()
IntegerExponentTokenBuilder.__escape_z__()
RealTokenBuilder.__escape_z__()
RealExponentTokenBuilder.__escape_z__()
PrefixedIntegerTokenBuilder.__escape_z__()
SuffixedIdentifierTokenBuilder.__escape_z__()
CaseInsensitiveListTokenBuilder.__escape_z__()
CaseSensitiveListTokenBuilder.__escape_z__()
SingleCharacterTokenBuilder.__escape_z__()
PrefixedIdentifierTokenBuilder.__escape_z__()
TripleQuoteStringTokenBuilder.__escape_z__()
SlashSlashCommentTokenBuilder.__escape_z__()
SlashStarCommentTokenBuilder.__escape_z__()
SwiftArgumentTokenBuilder.__escape_z__()
SwiftSymbolTokenBuilder.__escape_z__()
return 'Escape ?Z'
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder("'")
integer_exponent_tb = IntegerExponentTokenBuilder("'")
hex_integer_tb = PrefixedIntegerTokenBuilder('0x', False,
'0123456789abcdefABCDEF')
binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01')
real_tb = RealTokenBuilder(False, False, "'")
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "'")
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
annotation_tb = PrefixedIdentifierTokenBuilder('@', 'annotation',
False)
operand_types.append('annotation')
symbol_tb = PrefixedIdentifierTokenBuilder('#', 'symbol', True)
operand_types.append('symbol')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
raw_string_tb = PrefixedRawStringTokenBuilder('r', True, quotes)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '~/', '%', '^', '=', '==', '!=', '>', '>=',
'<', '<=', '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=',
'>>=', '!', '&', '|', '~', '<<', '>>', '~/=', '||', '&&', '.',
'..', ':', '?', '??', '??=', 'as', 'is', 'is!', '++', '--', 'new'
]
self.unary_operators = [
'+', '-', '*', '!', '~', '.', '..'
'?.', '++', '--', 'new'
]
self.postfix_operators = ['++', '--']
groupers = ['(', ')', ',', '[', ']', '{', '}']
group_starts = ['(', '[', ',', '{']
group_mids = [',']
group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'abstract', 'assert', 'async', 'await', 'break', 'case', 'catch',
'class', 'const', 'continue', 'covariant', 'default', 'deferred',
'do', 'dynamic', 'else', 'enum', 'export', 'extends', 'external',
'factory', 'final', 'finally', 'for', 'Function', 'get', 'hide',
'if', 'implements', 'import', 'in', 'interface', 'library',
'mixin', 'on', 'operator', 'part', 'rethrow', 'return', 'set',
'show', 'static', 'switch', 'sync', 'throw', 'try', 'typedef',
'var', 'void', 'while', 'with', 'yield'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = ['int', 'double', 'String', 'List', 'bool', 'void']
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['false', 'true', 'null', 'this', 'super']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, hex_integer_tb, binary_integer_tb,
real_tb, real_exponent_tb, keyword_tb, types_tb, values_tb,
groupers_tb, known_operator_tb, identifier_tb, annotation_tb,
symbol_tb, class_type_tb, string_tb, raw_string_tb,
slash_slash_comment_tb, slash_star_comment_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(
tokens, ['statement terminator', 'newline'], ['{'],
['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
stmt_separator_tb = SingleCharacterTokenBuilder(
';', 'statement separator', False)
integer_tb = IntegerTokenBuilder('_')
integer_exponent_tb = IntegerExponentTokenBuilder('_')
real_tb = RealTokenBuilder(True, True, '_')
real_exponent_tb = RealExponentTokenBuilder(True, True, 'E', '_')
operand_types.append('number')
argument_tb = SwiftArgumentTokenBuilder()
leads = '_'
extras = '_'
suffixes = '?'
identifier_tb = SuffixedIdentifierTokenBuilder(leads, extras, suffixes)
operand_types.append('identifier')
attribute_tb = PrefixedIdentifierTokenBuilder('@', 'attribute', False)
symbol_tb = SwiftSymbolTokenBuilder('.', 'symbol', True)
operand_types.append('symbol')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 10)
triple_quote_comment_tb = TripleQuoteStringTokenBuilder(quotes)
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
operand_types.append('string')
known_operators = [
'+', '-', '*', '/', '%', '==', '!=', '>', '<', '>=', '<=', '&&',
'||', '!', '&', '|', '^', '~', '<<', '>>', '===', '=', '+=', '-=',
'*=', '/=', '%=', '<<=', '>>=', '&=', '^=', '|=', '...', '..<',
'?', ':', '.', '++', '--', '->', '??', '\\.', '&+', '&-', '&*'
]
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
self.unary_operators = ['+', '-', '!', '~', '&', '++', '--', ':', '?']
self.postfix_operators = ['++', '--', ':', '!', '?']
groupers = ['(', ')', ',', '[', ']', '{', '}']
group_starts = ['(', '[', ',', '{']
group_mids = [',']
group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
keywords = [
'associatedtype', 'class', 'deinit', 'enum', 'extension',
'fileprivate', 'func', 'import', 'init', 'inout', 'internal',
'let', 'open', 'operator', 'private', 'protocol', 'public',
'static', 'struct', 'subscript', 'typealias', 'var', 'break',
'case', 'continue', 'default', 'defer', 'do', 'else',
'fallthrough', 'for', 'guard', 'if', 'in', 'repeat', 'return',
'switch', 'where', 'while', 'as', 'Any', 'catch', 'is', 'rethrows',
'super', 'throw', 'throws', 'try', 'try?', 'try!', '#available',
'#colorLiteral', '#column', '#else', '#elseif', '#endif', '#file',
'#fileLiteral', '#function', '#if', '#imageLiteral', '#line',
'#selector', '#sourceLocation', 'associativity', 'convenience',
'dynamic', 'didSet', 'final', 'get', 'infix', 'indirect', 'lazy',
'left', 'mutating', 'none', 'nonmutating', 'optional', 'override',
'postfix', 'precedence', 'prefix', 'Protocol', 'required', 'right',
'set', 'Type', 'unowned', 'weak', 'willSet'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'char',
'double',
'float',
'int',
'long',
'short',
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['nil', 'Self', 'false', 'true']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, stmt_separator_tb, integer_tb,
integer_exponent_tb, real_tb, real_exponent_tb, argument_tb,
keyword_tb, types_tb, values_tb, known_operator_tb, groupers_tb,
identifier_tb, attribute_tb, symbol_tb, string_tb,
slash_slash_comment_tb, slash_star_comment_tb,
triple_quote_comment_tb, self.unknown_operator_tb,
invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
self.tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid')
self.convert_keywords_to_identifiers(['.'])
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types = ['number', 'string', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder(None)
integer_exponent_tb = IntegerExponentTokenBuilder(None)
real_tb = RealTokenBuilder(False, False, None)
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', None)
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
directive_tb = DirectiveTokenBuilder()
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 10)
prefixed_string_tb = PrefixedStringTokenBuilder('@', False, quotes)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
directives = [
'#define', '#undef', '#ifdef', '#ifndef', '#if', '#endif', '#else',
'#elif', '#import', '#line', '#include'
]
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
c_preprocessor_tb = CaseSensitiveListTokenBuilder(
directives, 'preprocessor', False)
c_warning_tb = LeadToEndOfLineTokenBuilder('#warning', True,
'preprocessor')
c_error_tb = LeadToEndOfLineTokenBuilder('#error', True,
'preprocessor')
c_pragma_tb = LeadToEndOfLineTokenBuilder('#pragma', True,
'preprocessor')
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '=', '==', '!=', '>', '>=', '<', '<=',
'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '!',
'&', '|', '<<', '>>', '~', '.', '->', '++', '--', '&&', '||', '^',
'?', '##'
]
self.unary_operators = [
'+', '-', '*', '!', '&', '^', '~', '++', '--', '##'
]
self.postfix_operators = ['++', '--', '&', '->', '*', '^']
groupers = ['(', ')', ',', '[', ']', '{', '}', ':']
group_starts = ['(', '[', ',', '{']
group_ends = [')', ']', '}']
group_mids = [',', ':']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'atomic', 'break', 'bycopy', 'byref', 'case', 'continue',
'default', 'do', 'else', 'for', 'goto', 'if', 'IMP', 'in',
'inline', 'inout', 'nonatomic', 'oneway', 'out', 'Protocol',
'restrict', 'retain', 'return', 'SEL', 'sizeof', 'switch',
'typedef', 'while', '@interface', '@end', '@implementation',
'@protocol', '@class', '@public', '@protected', '@private',
'@property', '@try', '@throw', '@catch()', '@finally',
'@synthesize', '@dynamic', '@selector'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'auto', 'char', 'const', 'double', 'enum', 'extern', 'float', 'id',
'int', 'long', 'register', 'short', 'signed', 'static', 'struct',
'union', 'unsigned', 'void', 'volatile', '_Bool', '_Complex',
'_Imaginary', 'BOOL', 'Class'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['self', 'super', 'nil', 'YES', 'NO', 'NULL', '...']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, real_tb, real_exponent_tb,
keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb,
directive_tb, identifier_tb, class_type_tb, string_tb,
prefixed_string_tb, slash_slash_comment_tb, slash_star_comment_tb,
c_preprocessor_tb, c_warning_tb, c_error_tb, c_pragma_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
# tokens = Examiner.combine_identifier_colon(tokens, ['statement terminator', 'newline'], ['{'], ['whitespace', 'comment', 'line description'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.convert_values_to_operators()
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence(['*', ';'])
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'string', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder('_')
integer_exponent_tb = IntegerExponentTokenBuilder('_')
real_tb = RealTokenBuilder(False, False, '_')
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', '_')
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
decorator_tb = PrefixedIdentifierTokenBuilder('@', 'decorator', False)
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '=', '==', '!=', '>', '>=', '<', '<=',
'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '!',
'&', '|', '~', '<<', '>>', '>>>', '>>>=', '^', '.', '::', '++',
'--', '&&', '||', '?', '->', 'new'
]
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
self.unary_operators = ['+', '-', '!', '~', '++', '--', 'new']
self.postfix_operators = ['++', '--']
groupers = ['(', ')', ',', '[', ']', '{', '}', ':']
group_starts = ['(', '[', ',', '{']
group_ends = [')', ']', '}']
group_mids = [',', ':']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
keywords = [
'abstract', 'assert', 'break', 'case', 'catch', 'class', 'const',
'continue', 'default', 'do', 'else', 'enum', 'extends', 'final',
'finally', 'for', 'goto', 'if', 'implements', 'import',
'instanceof', 'interface', 'native', 'package', 'private',
'protected', 'public', 'return', 'static', 'strictfp', 'super',
'switch', 'synchronized', 'throw', 'throws', 'transient', 'try',
'volatile', 'while'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'boolean', 'byte', 'char', 'double', 'float', 'int', 'long',
'short', 'string', 'void', 'Integer', 'String', 'StringBuilder',
'File', 'Exception', 'IOException'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['false', 'null', 'this', 'true']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, terminators_tb, integer_tb,
integer_exponent_tb, real_tb, real_exponent_tb, keyword_tb,
types_tb, values_tb, known_operator_tb, groupers_tb, identifier_tb,
class_type_tb, decorator_tb, string_tb, slash_slash_comment_tb,
slash_star_comment_tb, self.unknown_operator_tb,
invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(
tokens, ['statement terminator', 'newline'], ['{'],
['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.convert_keywords_to_identifiers(['::', '.'])
self.convert_operators_to_identifiers(['::', '.'])
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'string', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, comment):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
number_tb = GenericNumberTokenBuilder()
operand_types.append('number')
leads = ''
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
quotes = ['"', "'", "’", '`']
string_tb = EscapedStringTokenBuilder(quotes, 0)
triple_string_tb = TripleQuoteStringTokenBuilder(quotes)
operand_types.append('string')
comment_tbs = []
if comment == 'ada':
comment_tbs = [LeadToEndOfLineTokenBuilder('--', True, 'comment')]
if comment == 'hash':
comment_tbs = [LeadToEndOfLineTokenBuilder('#', True, 'comment')]
if comment == 'bang':
comment_tbs = [LeadToEndOfLineTokenBuilder('!', True, 'comment')]
if comment == 'cobol-inline':
comment_tbs = [LeadToEndOfLineTokenBuilder('*>', True, 'comment')]
if comment == 'percent':
comment_tbs = [LeadToEndOfLineTokenBuilder('%', True, 'comment')]
if comment == 'cobol':
pass
if comment == 'fortran':
pass
if comment == 'basic':
comment_tbs = [
LeadToEndOfLineTokenBuilder("REM", False, 'comment'),
LeadToEndOfLineTokenBuilder("'", True, 'comment')
]
if comment == 'c':
comment_tbs = [SlashStarCommentTokenBuilder()]
if comment == 'cpp':
comment_tbs = [
SlashSlashCommentTokenBuilder(),
SlashStarCommentTokenBuilder()
]
if comment == 'pascal':
comment_tbs = [
BraceCommentTokenBuilder(),
BlockTokenBuilder('(*', '*)', 'comment')
]
known_operators = [
'+', '-', '*', '/', '%', '=', '==', '!=', '>', '>=', '<', '<=',
'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '!',
'&', '|', '~', '<<', '>>', '^', '.', '..', '...', ':', '++', '--',
'->', '&&', '||', '?', '##', '\\', '_', '@', '#', '$', '`', '```'
]
groupers = ['(', ')', ',', '[', ']', '{', '}', ';']
# group_starts = ['(', '[', ',', '{']
group_mids = [',', ';']
# group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders1 = [
whitespace_tb, newline_tb, number_tb, groupers_tb,
known_operator_tb, identifier_tb, string_tb, triple_string_tb
]
tokenbuilders2 = [self.unknown_operator_tb, invalid_token_builder]
tokenbuilders = tokenbuilders1 + comment_tbs + tokenbuilders2
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(
tokens, ['newline', 'statement separator'], ['{'],
['whitespace', 'comment', 'line description'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.calc_statistics()
# tokens = self.source_tokens()
# tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
# allow_pairs = []
# self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
# self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs)
# self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
# self.calc_operand_n_confidence(tokens, operand_types, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
# self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder('_')
integer_exponent_tb = IntegerExponentTokenBuilder('_')
hex_integer_tb = PrefixedIntegerTokenBuilder(
'0x', False, '0123456789ABCDEFabcdef_')
real_tb = RealTokenBuilder(True, True, '_')
real_exponent_tb = RealExponentTokenBuilder(True, True, 'E', '_')
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
decorator_tb = PrefixedIdentifierTokenBuilder('@', 'decorator', False)
operand_types.append('decorator')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
triple_quote_string_tb = TripleQuoteStringTokenBuilder(quotes)
operand_types.append('string')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
class_tb = ClassTypeTokenBuilder()
operand_types.append('class')
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '=', '+=', '-=', '*=', '/=', '%=', '++',
'--', '&&', '||', '!', '==', '!=', '===', '!==', '<', '>', '<=',
'>=', '!!', '?.', '?:', '::', '..', ':', '?', '.'
]
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
self.unary_operators = ['+', '-', '!', '*', '++', '--']
self.postfix_operators = ['++', '--', ':']
groupers = ['->', '(', ')', ',', '[', ']', '{', '}']
group_starts = ['(', '[', ',', '{']
group_mids = ['->', ',']
group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
keywords = [
'as', 'as?', 'break', 'class', 'continue', 'do', 'else', 'for',
'fun', 'if', 'in', '!in', 'is', '!is', 'object', 'package',
'return', 'super', 'throw', 'try', 'typealias', 'typeof', 'val',
'var', 'when', 'while', 'by', 'catch', 'constructor', 'delegate',
'dynamic', 'field', 'file', 'finally', 'get', 'import', 'init',
'param', 'property', 'receiver', 'set', 'setparam', 'where',
'actual', 'abstract', 'annotation', 'companion', 'const',
'crossinline', 'data', 'enum', 'expect', 'external', 'final',
'infix', 'inline', 'inner', 'internal', 'lateinit', 'noinline',
'open', 'operator', 'out', 'override', 'private', 'protected',
'public', 'reified', 'sealed', 'suspend', 'tailrec', 'vararg'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'Byte', 'Short', 'Int', 'Long', 'Float', 'Double', 'Char', 'u',
'f', 'ul', 'UInt', 'ULong', 'UByte', 'UShort'
]
type_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['false', 'null', 'this', 'true']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, terminators_tb, integer_tb,
integer_exponent_tb, hex_integer_tb, real_tb, real_exponent_tb,
keyword_tb, type_tb, values_tb, known_operator_tb, groupers_tb,
identifier_tb, class_tb, decorator_tb, string_tb,
triple_quote_string_tb, slash_slash_comment_tb,
slash_star_comment_tb, self.unknown_operator_tb,
invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
self.tokens = self.combine_numbers_and_adjacent_types(tokens)
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'string', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder(None)
integer_exponent_tb = IntegerExponentTokenBuilder(None)
real_tb = RealTokenBuilder(False, False, None)
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', None)
hex_constant_tb = PrefixedIntegerTokenBuilder(
'0H', False, '0123456789ABCDEFabcdef')
octal_constant_tb = PrefixedIntegerTokenBuilder(
'0O', False, '01234567')
binary_constant_tb = PrefixedIntegerTokenBuilder('0B', False, '01')
operand_types.append('number')
leads = '_$'
extras = '_$'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
operand_types.append('string')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
terminators_tb = CaseInsensitiveListTokenBuilder(
[';'], 'statement terminator', False)
known_operators = [
'+', '-', '*', '/', '%', '=', '==', '!=', '===', '!==', '>', '>=',
'<', '<=', '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=',
'>>=', '!', '&', '|', '~', '<<', '>>', '=>', '^', '.', ':', '++',
'--', '&&', '||', '?', '$', '?.', 'new', 'delete'
]
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
self.unary_operators = [
'+', '-', '!', '~', '++', '--', ':', '$', 'new', 'delete'
]
self.postfix_operators = ['++', '--', ':']
groupers = ['(', ')', ',', '[', ']', '{', '}']
group_starts = ['(', '[', ',', '{']
group_mids = [',']
group_ends = [')', ']', '}']
groupers_tb = CaseSensitiveListTokenBuilder(groupers, 'group', False)
regex_tb = RegexTokenBuilder()
keywords = [
'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger',
'default', 'do', 'else', 'enum', 'export', 'extends', 'finally',
'for', 'function', 'if', 'import', 'in', 'instanceof', 'return',
'switch', 'throw', 'try', 'typeof', 'while', 'with', 'as',
'implements', 'interface', 'let', 'package', 'private',
'protected', 'public', 'static', 'yield', 'constructor', 'declare',
'get', 'module', 'require', 'set', 'type', 'from', 'of'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'any', 'boolean', 'byte', 'char', 'number', 'string', 'symbol',
'void', 'never', 'object'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['this', 'super', 'null', 'true', 'false', 'undefined']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, terminators_tb, integer_tb,
integer_exponent_tb, real_tb, real_exponent_tb, hex_constant_tb,
octal_constant_tb, binary_constant_tb, keyword_tb, types_tb,
values_tb, known_operator_tb, groupers_tb, regex_tb, identifier_tb,
string_tb, slash_slash_comment_tb, slash_star_comment_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
self.tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid')
self.convert_keywords_to_identifiers(['.'])
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'string', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder("'")
integer_exponent_tb = IntegerExponentTokenBuilder("'")
hex_integer_tb = PrefixedIntegerTokenBuilder('0x', False,
'0123456789abcdefABCDEF')
binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01')
suffixed_integer_tb = SuffixedIntegerTokenBuilder(
['U', 'L', 'LL', 'ULL', 'LLU'], False, None)
real_tb = RealTokenBuilder(False, False, "'")
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "'")
suffixed_real_tb = SuffixedRealTokenBuilder(False, False, ['f', 'l'],
False, None)
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
directives = [
'#define', '#undef', '#ifdef', '#ifndef', '#if', '#endif', '#else',
'#elif', '#line', '#include', '#pragma'
]
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
c_preprocessor_tb = CaseSensitiveListTokenBuilder(
directives, 'preprocessor', False)
c_warning_tb = LeadToEndOfLineTokenBuilder('#warning', True,
'preprocessor')
c_error_tb = LeadToEndOfLineTokenBuilder('#error', True,
'preprocessor')
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '=', '==', '!=', '>', '>=', '<', '<=',
'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '!',
'&', '|', '~', '<<', '>>', '^', '.', '++', '--', '->', '&&', '||',
'?', '##', '::', '<=>', '.*', '->*', 'new', 'delete', 'and',
'and_eq', 'bitand', 'bitor', 'compl', 'not', 'not_eq', 'or',
'or_eq', 'xor', 'xor_eq'
]
self.unary_operators = [
'+', '-', '*', '!', '&', '~', '++', '--', 'new', 'delete', 'compl',
'not'
]
self.postfix_operators = ['++', '--', '&', '*']
groupers = ['(', ')', ',', '[', ']', '{', '}', ':']
group_starts = ['(', '[', ',', '{']
group_ends = [')', ']', '}']
group_mids = [',', ':']
group_mids = [',', ':']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'alignas',
'alignof',
'asm',
'atomic_cancel',
'atomic_commit',
'atomic_noexcept',
'audit',
'auto',
'axiom',
'break',
'case',
'catch',
'class',
'concept',
'const',
'consteval',
'constexpr',
'const_cast',
'continue',
'co_await',
'co_return',
'co_yield',
'decltype',
'default',
'do',
'dynamic_cast',
'else',
'enum',
'explicit',
'export',
'extern',
'final',
'for',
'friend',
'goto',
'if',
'import',
'inline',
'module',
'mutable',
'namespace',
'noexcept',
'nullptr',
'operator',
'override',
'private',
'protected',
'public',
'private:',
'protected:',
'public:',
'reflexpr',
'register',
'reinterpret_cast',
'requires',
'return',
'signed',
'sizeof',
'static',
'static_assert',
'static_cast',
'struct',
'switch',
'synchronized',
'template',
'thread_local',
'throw',
'transaction_safe',
'transaction_safe_dynamic'
'try',
'typedef',
'typeid',
'typename',
'union',
'unsigned',
'using',
'virtual',
'volatile',
'while',
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'bool', 'char', 'char8_t', 'char16_t', 'char32_t', 'double',
'float', 'int', 'long', 'short', 'void', 'wchar_t'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['false', 'this', 'true', 'cout', 'cin']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, hex_integer_tb, binary_integer_tb,
suffixed_integer_tb, real_tb, real_exponent_tb, suffixed_real_tb,
keyword_tb, types_tb, values_tb, known_operator_tb, groupers_tb,
identifier_tb, class_type_tb, string_tb, slash_slash_comment_tb,
slash_star_comment_tb, c_preprocessor_tb, c_error_tb, c_warning_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(
tokens, ['statement terminator', 'newline'], ['{'],
['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence(['*', ';'])
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'string']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, tab_size, wide):
super().__init__()
self.operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder(None)
integer_exponent_tb = IntegerExponentTokenBuilder(None)
binary_integer_tb = SuffixedIntegerTokenBuilder(['B'], False, None)
hex_integer_tb = SuffixedIntegerTokenBuilder(['H'], False, 'ABCDEF')
octal_integer_tb = SuffixedIntegerTokenBuilder(['O'], False, None)
decimal_integer_tb = SuffixedIntegerTokenBuilder(['D'], False, None)
real_tb = RealTokenBuilder(True, False, None)
real_exponent_tb = RealExponentTokenBuilder(True, False, 'E', None)
binary_real_tb = SuffixedRealTokenBuilder(True, True, ['B'], False,
None)
self.operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
self.operand_types.append('identifier')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
self.operand_types.append('string')
label_tb = PL1LabelTokenBuilder()
self.operand_types.append('label')
slash_star_comment_tb = SlashStarCommentTokenBuilder()
directives = [
'%ACTIVATE', '%DEACTIVATE', '%DECLARE', '%DCL', '%DICTIONARY',
'%DO', '%ELSE', '%END', '%FATAL', '%GOTO', '%IF', '%INCLUDE',
'%LIST', '%NOLIST', '%PAGE', '%PROCEDURE', '%PROC', '%REPLACE',
'%RETURN', '%THEN'
]
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
preprocessor_tb = CaseInsensitiveListTokenBuilder(
directives, 'preprocessor', False)
title_tb = LeadToEndOfLineTokenBuilder('%TITLE', True, 'preprocessor')
subtitle_tb = LeadToEndOfLineTokenBuilder('%SBTTL', True,
'preprocessor')
error_tb = LeadToEndOfLineTokenBuilder('%ERROR', True, 'preprocessor')
warn_tb = LeadToEndOfLineTokenBuilder('%WARN', True, 'preprocessor')
inform_tb = LeadToEndOfLineTokenBuilder('%INFORM', True,
'preprocessor')
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '**', '>', '<', '=', '>=', '<=', '<>', '^>',
'^<', '^=', '^', '~>', '~<', '~=', '~', '&', '&:', ':=', '|', '|:',
'||', '!', '!:', '!!', ':', '@', 'NOT', 'AND', 'OR', 'XOR',
'MINUS', 'PLUS', 'MOD'
]
self.unary_operators = ['+', '-', '^', '~', '@', 'NOT']
self.postfix_operators = []
groupers = ['(', ')', ',', '[', ']', '{', '}']
self.group_starts = ['(', '[', ',', '{']
self.group_mids = [',']
self.group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'ADDRESS', 'AT', 'BASED', 'BY', 'CALL', 'CASE', 'CLOSE', 'DATA',
'DECLARE', 'DISABLE', 'DO', 'ELSE', 'ENABLE', 'END', 'EOF',
'EXTERNAL', 'GO', 'GOTO', 'HALT', 'IF', 'INITIAL', 'INTERRUPT',
'LABEL', 'LITERALLY', 'OFFSET', 'ON', 'OPEN', 'OTHERWISE', 'OTHER',
'PROCEDURE', 'PROC', 'PUBLIC', 'READ', 'REENTRANT', 'RETURN',
'SELECTOR', 'STRUCTURE', 'THEN', 'TO', 'WHILE', 'WRITE'
]
keyword_tb = CaseInsensitiveListTokenBuilder(keywords, 'keyword',
False)
attributes = [
'ALIGNED', 'ANY', 'AREA', 'BASED', 'BUILTIN', 'CONDITION', 'COND',
'CONTROLLED', 'CTL', 'DEFINED', 'DEF', 'DIRECT', 'ENTRY',
'ENVIRONMENT', 'ENV', 'EXTERNAL', 'EXT', 'FILE', 'GLOBALDEF',
'GLOBALREF', 'INITIAL', 'INIT', 'INPUT', 'INTERNAL', 'INT'
'KEYED', 'LABEL', 'LIKE', 'LIST', 'MEMBER', 'NONVARYING', 'NONVAR',
'OPTIONAL', 'OPTIONS', 'OUTPUT', 'PARAMETER', 'PARM', 'PICTURE',
'PIC', 'POSITION', 'POS', 'PRECISION', 'PREC', 'PRINT', 'READONLY',
'RECORD', 'REFER', 'RETURNS', 'SEQUENTIAL', 'SEQL', 'STATIC',
'STREAM', 'STRUCTURE', 'TRUNCATE', 'UNALIGNED', 'UNAL', 'UNION',
'UPDATE', 'VARIABLE', 'VARYING', 'VAR'
]
attributes_tb = CaseInsensitiveListTokenBuilder(
attributes, 'attribute', False)
functions = [
'ABS', 'ADJUSTRPL', 'BLOCKINPUT', 'BLOCKINWORD', 'BLOCKINDWORD',
'BLOCKOUTPUT', 'BLOCKOUTWORD', 'BLOCKOUTDWORD', 'BUILDPTR',
'BYTESWAP', 'CMPD', 'CARRY', 'CAUSEINTERRUPT',
'CLEARTASKSWITCHEDFLAG', 'CMPB', 'CMPW', 'CONTROLREGISTER', 'DEC',
'DOUBLE', 'DEBUGREGISTER', 'FINDB', 'FINDD', 'FINDRD', 'FINDHW',
'FINDRB', 'FINDRHW', 'FINDRW', 'FINDW', 'FIX', 'FLAGS', 'FLOAT',
'GETACCESSRIGHTS', 'GETREALERROR', 'GETSEGMENTLIMIT', 'HIGH',
'IABS', 'INHWORD', 'INITREALMATHUNITSKIPRB', 'INPUT', 'INT SIZE',
'INWORD SIZE', 'INVALIDATEDATACACHE', 'INVALIDATETLBENTRY'
'INDWORD', 'LAST', 'LENGTH', 'LOCALTABLE', 'LOCKSET', 'LOW',
'MACHINESTATUS', 'MOVB', 'MOVBIT', 'MOVD', 'MOVE', 'MOVHW',
'MOVRB', 'MOVRBIT'
'MOVRD', 'MOVRHW', 'MOVRW', 'MOVW', 'NIL', 'OFFSETOF', 'OUTDWORD',
'OUTHWORD', 'OUTPUT', 'OUTWORD', 'PARITY', 'RESTOREGLOBALTABLE',
'RESTOREINTERRUPTABLE', 'RESTOREREALSTATUS', 'ROL', 'ROR', 'SAL',
'SAR', 'SAVEGLOBALTABLE', 'SAVEINTERRUPTTABLE', 'SAVEREALSTATUS',
'SCANBIT', 'SCANRBIT', 'SCL', 'SCR', 'SEGMENTREADABLE',
'SEGMENTWRITABLE', 'SELECTOROF', 'SETB', 'SETHW', 'SETREALMODE',
'SETW', 'SHL', 'SHLD', 'SHR', 'SHRD', 'SETD', 'SIGN', 'SIGNED',
'SKIPB', 'SKIPD', 'SKIPRD'
'SKIPHW', 'SKIPRHW', 'SKIPRW', 'SKIPW', 'STACKBASE', 'STACKPTR',
'TASKREGISTER', 'TESTREGISTER', 'TIME', 'UNSIGN',
'WAITFORINTERRUPT', 'WBINVALIDATEDATACACHE', 'XLAT', 'ZERO'
]
function_tb = CaseInsensitiveListTokenBuilder(functions, 'function',
True)
format_items = [
'A', 'B', 'B1', 'B2', 'B3', 'B4', 'COLUMN', 'COL', 'E', 'F', 'P',
'R', 'TAB', 'X'
]
format_item_tb = CaseSensitiveListTokenBuilder(format_items, 'format',
True)
self.operand_types.append('format')
options = [
'APPEND', 'BACKUP_DATE', 'BATCH', 'BLOCK_BOUNDARY_FORMAT',
'BLOCK_IO', 'BLOCK_SIZE', 'BUCKET_SIZE', 'BY', 'CANCEL_CONTROL_O',
'CARRIAGE_RETURN_FORMAT', 'CONTIGUOUS', 'CONTIGUOUS_BEST_TRY',
'CREATION_DATE', 'CURRENT_POSITION', 'DEFAULT_FILE_NAME',
'DEFERRED_WRITE', 'DELETE', 'EDIT', 'EXPIRATION_DATE',
'EXTENSION_SIZE', 'FAST_DELETE', 'FILE_ID', 'FILE_ID_TO',
'FILE_SIZE', 'FIXED_CONTROL_FROM', 'FIXED_CONTROL_SIZE',
'FIXED_CONTROL_SIZE_TO', 'FIXED_CONTROL_TO',
'FIXED_LENGTH_RECORDS', 'FROM', 'GROUP_PROTECTION', 'IDENT',
'IGNORE_LINE_MARKS', 'IN', 'INDEXED', 'INDEX_NUMBER',
'INITIAL_FILL', 'INTO', 'KEY', 'KEYFROM', 'KEYTO', 'LINESIZE',
'LOCK_ON_READ', 'LOCK_ON_WRITE', 'MAIN PROCEDURE',
'MANUAL_UNLOCKING', 'MATCH_GREATER', 'MATCH_GREATER_EQUAL',
'MATCH_NEXT', 'MATCH_NEXT_EQUAL', 'MAXIMUM_RECORD_NUMBER',
'MAXIMUM_RECORD_SIZE', 'MULTIBLOCK_COUNT', 'MULTIBUFFER_COUNT',
'NOLOCK', 'NONEXISTENT_RECORD', 'NONRECURSIVE', 'NORESCAN',
'NO_ECHO', 'NO_FILTER', 'NO_SHARE', 'OWNER_GROUP', 'OWNER_ID',
'OWNER_MEMBER', 'OWNER_PROTECTION', 'PAGE', 'PAGESIZE',
'PRINTER_FORMAT', 'PROMPT', 'PURGE_TYPE_AHEAD', 'READ_AHEAD',
'READ_CHECK', 'READ_REGARDLESS', 'RECORD_ID', 'RECORD_ID_ACCESS',
'RECORD_ID_TO', 'RECURSIVE', 'REPEAT', 'RESCAN',
'RETRIEVAL_POINTERS', 'REVISION_DATE', 'REWIND_ON_CLOSE',
'REWIND_ON_OPEN', 'SCALARVARYING', 'SET READ', 'SHARED_READ',
'SHARED_WRITE', 'SKIP', 'SNAP', 'SPOOL', 'STATEMENT', 'SUPERSEDE',
'SYSTEM', 'SYSTEM_PROTECTION', 'TEMPORARY', 'TIMEOUT_PERIOD',
'TITLE', 'TO', 'UNDERFLOW', 'UFL', 'UNTIL', 'USER_OPEN',
'WAIT_FOR_RECORD', 'WHILE', 'WORLD_PROTECTION', 'WRITE_BEHIND',
'WRITE_CHECK'
]
options_tb = CaseInsensitiveListTokenBuilder(options, 'option', False)
conditions = [
'ANYCONDITION', 'CONVERSION', 'CONV', 'ENDFILE', 'ENDPAGE',
'FINISH', 'FIXEDOVERFLOW', 'FOFL', 'OVERFLOW', 'OFL', 'STORAGE',
'STRINGRANGE', 'STRG', 'SUBSCRIPTRANGE', 'SUBRG', 'UNDEFINEDFILE',
'UNDF', 'VAXCONDITION', 'ZERODIVIDE', 'ZDIV'
]
conditions_tb = CaseInsensitiveListTokenBuilder(
conditions, 'condition', False)
subroutines = [
'DISPLAY', 'EXTEND', 'FLUSH', 'NEXT_VOLUME', 'RELEASE', 'RESIGNAL',
'REWIND', 'SPACEBLOCK'
]
subroutines_tb = CaseInsensitiveListTokenBuilder(
subroutines, 'subroutine', False)
types = [
'ADDRESS', 'BYTE', 'CHARINT', 'DWORD', 'HWORD', 'INTEGER',
'LONGINT', 'OFFSET', 'POINTER', 'REAL', 'SHORTINT', 'STRUCTURE',
'QWORD', 'WORD'
]
types_tb = CaseInsensitiveListTokenBuilder(types, 'type', True)
self.operand_types.append('type')
values = ['SYSIN', 'SYSPRINT', 'TRUE', 'FALSE']
values_tb = CaseInsensitiveListTokenBuilder(values, 'value', True)
self.operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
# tokenize as free-format
tokenbuilders_free = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, binary_integer_tb, hex_integer_tb,
octal_integer_tb, decimal_integer_tb, real_tb, real_exponent_tb,
binary_real_tb, keyword_tb, format_item_tb, function_tb,
attributes_tb, options_tb, conditions_tb, subroutines_tb, types_tb,
values_tb, groupers_tb, known_operator_tb, identifier_tb,
string_tb, label_tb, slash_star_comment_tb, preprocessor_tb,
title_tb, subtitle_tb, error_tb, warn_tb, inform_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer_free = Tokenizer(tokenbuilders_free)
tokens_free = tokenizer_free.tokenize(code)
tokens_free = Examiner.combine_adjacent_identical_tokens(
tokens_free, 'invalid operator')
tokens_free = Examiner.combine_adjacent_identical_tokens(
tokens_free, 'invalid')
self.tokens = tokens_free
self.calc_statistics()
statistics_free = self.statistics
self.statistics = {}
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators,
self.group_ends, allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
self.group_starts, allow_pairs)
self.calc_group_confidence(tokens, self.group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, self.operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
confidences_free = self.confidences
self.confidences = {}
errors_free = self.errors
self.errors = []
# tokenize as fixed-format
tokenbuilders_fixed = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, binary_integer_tb, hex_integer_tb,
octal_integer_tb, decimal_integer_tb, real_tb, real_exponent_tb,
binary_real_tb, keyword_tb, function_tb, attributes_tb, options_tb,
conditions_tb, subroutines_tb, types_tb, values_tb, groupers_tb,
known_operator_tb, identifier_tb, string_tb, label_tb,
slash_star_comment_tb, preprocessor_tb, title_tb, subtitle_tb,
error_tb, warn_tb, inform_tb, self.unknown_operator_tb,
invalid_token_builder
]
comment_start_tb = PL1CommentStartTokenBuilder()
comment_middle_tb = PL1CommentMiddleTokenBuilder()
comment_end_tb = PL1CommentEndTokenBuilder()
type1_tokenbuilders = [comment_start_tb]
tokenbuilders_fixed_1 = tokenbuilders_fixed + type1_tokenbuilders + [
invalid_token_builder
]
tokenizer_fixed_1 = Tokenizer(tokenbuilders_fixed_1)
type2_tokenbuilders = [
comment_start_tb, comment_middle_tb, comment_end_tb
]
tokenbuilders_fixed_2 = tokenbuilders_fixed + type2_tokenbuilders + [
invalid_token_builder
]
tokenizer_fixed_2 = Tokenizer(tokenbuilders_fixed_2)
tokens_fixed = self.tokenize_code(code, tab_size, tokenizer_fixed_1,
tokenizer_fixed_2, wide)
tokens_fixed = Examiner.combine_adjacent_identical_tokens(
tokens_fixed, 'invalid operator')
tokens_fixed = Examiner.combine_adjacent_identical_tokens(
tokens_fixed, 'invalid')
tokens_fixed = Examiner.combine_adjacent_identical_tokens(
tokens_fixed, 'whitespace')
tokens_fixed = self.convert_broken_comments_to_comments(tokens_fixed)
self.tokens = tokens_fixed
self.calc_statistics()
statistics_fixed = self.statistics
self.statistics = {}
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators,
self.group_ends, allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
self.group_starts, allow_pairs)
self.calc_group_confidence(tokens, self.group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, self.operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
confidences_fixed = self.confidences
self.confidences = {}
errors_fixed = self.errors
self.errors = []
# compute confidence for free-format and fixed-format
confidence_free = 1.0
if len(confidences_free) == 0:
confidence_free = 0.0
else:
for key in confidences_free:
factor = confidences_free[key]
confidence_free *= factor
confidence_fixed = 1.0
if len(confidences_fixed) == 0:
confidence_fixed = 0.0
else:
for key in confidences_fixed:
factor = confidences_fixed[key]
confidence_fixed *= factor
# select the better of free-format and spaced-format
if confidence_fixed > confidence_free:
self.tokens = tokens_fixed
self.statistics = statistics_fixed
self.confidences = confidences_fixed
self.errors = errors_fixed
else:
self.tokens = tokens_free
self.statistics = statistics_free
self.confidences = confidences_free
self.errors = errors_free
def __init__(self, code, block_comment_limit):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder("'")
integer_exponent_tb = IntegerExponentTokenBuilder("'")
hex_integer_tb = PrefixedIntegerTokenBuilder('0x', False, '0123456789abcdefABCDEF_')
binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01_')
suffixed_integer_tb = SuffixedIntegerTokenBuilder(['U', 'L', 'LU', 'UL'], False, None)
real_tb = RealTokenBuilder(False, False, "'")
suffixed_real_tb = SuffixedRealTokenBuilder(False, False, ['f', 'l', 'i'], False, None)
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "'")
hex_real_tb = HexRealExponentTokenBuilder()
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
attribute_tb = PrefixedIdentifierTokenBuilder('@', 'attribute', False)
operand_types.append('attribute')
# string suffix: c,w,d
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
r_string_tb = PrefixedStringTokenBuilder('r', True, quotes)
backtick_string_tb = EscapedStringTokenBuilder(['`'], 0)
x_string_tb = PrefixedStringTokenBuilder('x', True, quotes)
q_string_tb = PrefixedStringTokenBuilder('q', True, quotes)
# q{} string
cwd_string_tb = SuffixedStringTokenBuilder(quotes, 'cwd', False)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
slash_plus_comment_tb = NestedCommentTokenBuilder('/+', '+/', block_comment_limit)
line_continuation_tb = SingleCharacterTokenBuilder('\\', 'line continuation', False)
terminators_tb = SingleCharacterTokenBuilder(';', 'statement terminator', False)
known_operators = [
'/', '/=', '.', '..', '...', '&', '&=', '&&', '|', '|=', '||',
'-', '-=', '--', '+', '+=', '++', '<', '<=', '<<', '<<=', '>', '>=',
'>>=', '>>>=', '>>', '>>>', '!', '!=', '?', ',', ':', '$',
'=', '==', '*', '*=', '%', '%=', '^', '^=', '^^', '^^=', '~', '~=',
'@', '=>', '#',
'new', 'delete',
'typeof', 'is'
]
self.unary_operators = [
'+', '-', '*',
'!', '&', '~',
'++', '--', ':',
'new', 'delete',
'typeof', 'is'
]
self.postfix_operators = [
'++', '--', '&', ':'
]
groupers = ['(', ')', ',', '[', ']', '{', '}']
group_starts = ['(', '[', ',', '{']
group_mids = [',']
group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(known_operators, 'operator', False)
keywords = [
'abstract', 'alias', 'align', 'asm', 'assert', 'auto',
'body', 'break', 'case', 'cast', 'catch', 'class', 'const', 'continue',
'debug', 'default', 'delegate', 'deprecated', 'do',
'else', 'enum', 'export', 'extern',
'final', 'finally', 'for', 'foreach', 'foreach_reverse', 'function',
'goto',
'if', 'immutable', 'import', 'in', 'inout', 'interface', 'invariant',
'lazy',
'macro', 'mixin', 'module',
'nothrow',
'out', 'override',
'package', 'pragma', 'private', 'protected', 'public', 'pure',
'ref', 'return',
'scope', 'shared', 'static', 'struct', 'switch', 'synchronized',
'template', 'throw', 'try', 'typeid',
'union', 'unittest', 'version', 'while', 'with',
'__gshared', '__traits', '__vector', '__parameters'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'bool', 'byte', 'cdouble', 'cent', 'cfloat', 'char', 'creal',
'dchar', 'double', 'float', 'idouble', 'ifloat', 'int', 'ireal',
'long', 'real', 'short', 'ubyte', 'ucent', 'uint', 'ulong', 'ushort',
'void', 'wchar'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = [
'false', 'null', 'super', 'this', 'true',
'__FILE__', '__FILE_FULL_PATH__', '__MODULE__', '__LINE__',
'__FUNCTION__', '__PRETTY_FUNCTION__',
'__DATE__', '__EOF__', '__TIME__','__TIMESTAMP__',
'__VENDOR__', '__VERSION__'
]
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb,
whitespace_tb,
line_continuation_tb,
terminators_tb,
integer_tb,
integer_exponent_tb,
hex_integer_tb,
binary_integer_tb,
suffixed_integer_tb,
real_tb,
real_exponent_tb,
suffixed_real_tb,
hex_real_tb,
keyword_tb,
types_tb,
values_tb,
groupers_tb,
known_operator_tb,
identifier_tb,
attribute_tb,
class_type_tb,
string_tb,
r_string_tb,
x_string_tb,
backtick_string_tb,
q_string_tb,
cwd_string_tb,
slash_slash_comment_tb,
slash_star_comment_tb,
slash_plus_comment_tb,
self.unknown_operator_tb,
invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(tokens, ['statement terminator', 'newline'], ['{'], ['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
number_suffixes = ['f', 'F', 'i', 'I', 'u', 'U', 'l', 'L', 'ul', 'uL', 'Ul', 'UL', 'lu', 'lU', 'Lu', 'LU']
tokens = self.combine_tokens_and_adjacent_types(tokens, 'number', 'identifier', number_suffixes)
string_suffixes = ['c', 'w', 'd']
self.tokens = self.combine_tokens_and_adjacent_types(tokens, 'string', 'identifier', string_suffixes)
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder("'")
integer_exponent_tb = IntegerExponentTokenBuilder("_")
hex_integer_tb = PrefixedIntegerTokenBuilder(
'0x', False, '_0123456789abcdefABCDEF')
binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '_01')
suffixed_integer_tb = SuffixedIntegerTokenBuilder([
'G',
'L',
'I',
], False, '_')
real_tb = RealTokenBuilder(False, False, "_")
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "_")
suffixed_real_tb = SuffixedRealTokenBuilder(False, False,
['G', 'D', 'F'], False,
'_')
operand_types.append('number')
leads = '@_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
triple_quote_string_tb = TripleQuoteStringTokenBuilder(quotes)
regex_tb = RegexTokenBuilder()
# dollar-slash slash-dollar strings (allow newline)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
shebang_tb = SheBangTokenBuilder()
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+',
'-',
'*',
'/',
'%',
'**',
'=',
'==',
'!=',
'===',
'!==',
'>',
'>=',
'<',
'<=',
'+=',
'-=',
'*=',
'/=',
'%=',
'**=',
'&=',
'|=',
'^=',
'<<=',
'>>=',
'!',
'&',
'|',
'~',
'<<',
'>>',
'>>>',
'^',
'?.',
'?:',
'<>',
'>>>=',
'.',
'.&',
'.@',
'::',
'=~',
'==~',
'*.',
'*:',
'..',
'..<',
'<=>',
'++',
'--',
'->',
'&&',
'||',
'?',
'##',
'as',
'in',
'!in',
'instanceof',
'!instanceof',
'new',
]
self.unary_operators = ['+', '-', '*', '!', '&', '~', '++', '--', '?']
self.postfix_operators = ['++', '--', '&', '*']
groupers = ['(', ')', ',', '[', ']', '{', '}', ':']
# group_starts = ['(', '[', ',', '{']
group_ends = [')', ']', '}']
group_mids = [',', ':']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'assert', 'break', 'case', 'catch', 'class', 'const', 'continue',
'def', 'default', 'do', 'else', 'enum', 'extends', 'finally',
'for', 'goto', 'if', 'implements', 'import', 'interface', 'new',
'package', 'return', 'super', 'switch', 'throw', 'throws', 'trait',
'try', 'var', 'while'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'byte', 'char', 'double', 'float', 'int', 'long', 'short',
'Java.lang.BigInteger'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['null', 'true', 'false', 'this']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, terminators_tb, integer_tb,
integer_exponent_tb, hex_integer_tb, binary_integer_tb,
suffixed_integer_tb, real_tb, real_exponent_tb, suffixed_real_tb,
keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb,
identifier_tb, class_type_tb, string_tb, triple_quote_string_tb,
regex_tb, slash_slash_comment_tb, slash_star_comment_tb,
shebang_tb, self.unknown_operator_tb, invalid_token_builder
]
# shebang line at start
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(
tokens, ['statement terminator', 'newline'], ['{'],
['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence(['*', ';'])
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
# self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder("'")
integer_exponent_tb = IntegerExponentTokenBuilder("'")
hex_integer_tb = PrefixedIntegerTokenBuilder(
'0x', False, '0123456789abcdefABCDEF_')
long_integer_tb = SuffixedIntegerTokenBuilder('L', False, None)
real_tb = RealTokenBuilder(False, False, "'")
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "'")
float_real_tb = SuffixedRealTokenBuilder(False, False, ['f'], False,
None)
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
symbol_tb = PrefixedIdentifierTokenBuilder("'", 'symbol', True)
operand_types.append('symbol')
quotes = ['"']
string_tb = EscapedStringTokenBuilder(quotes, 0)
triple_string_tb = TripleQuoteStringTokenBuilder(quotes)
operand_types.append('string')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '&', '|', '^', '<<', '>>', '&&', '||',
'=', '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=',
'>:', '⇒', '=>', '=', '<%', '<:', '←', '<-', '#', '@', '==', '!=',
'>', '<', '>=', '<=', '!', '~', '<<<', '>>>', '.', '++', '--',
'new'
]
self.unary_operators = ['+', '-', '*', '!', '~', '++', '--', 'new']
self.postfix_operators = ['++', '--']
groupers = ['(', ')', ',', '[', ']', '{', '}', ':']
group_starts = ['(', '[', ',', '{']
group_mids = [',', ':']
group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'abstract', 'case', 'catch', 'class', 'def', 'do', 'else',
'extends', 'final', 'finally', 'for', 'forSome', 'if', 'implicit',
'import', 'lazy', 'match', 'object', 'override', 'package',
'private', 'protected', 'return', 'sealed', 'then', 'throw',
'trait', 'try', 'type', 'val', 'var', 'while', 'with', 'yield'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
values = ['false', 'true', 'null', 'this', 'super']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, hex_integer_tb, long_integer_tb,
real_tb, real_exponent_tb, float_real_tb, keyword_tb, values_tb,
groupers_tb, known_operator_tb, identifier_tb, symbol_tb,
string_tb, triple_string_tb, slash_slash_comment_tb,
slash_star_comment_tb, self.unknown_operator_tb,
invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
self.tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid')
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
integer_tb = IntegerTokenBuilder(None)
integer_exponent_tb = IntegerExponentTokenBuilder(None)
real_tb = RealTokenBuilder(False, False, None)
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', None)
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
quotes = ['"', "'", '`', "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '&', '!', '^', '<<', '>>', '&^', '=',
'+=', '-=', '*=', '<<=', '>>=', '&^=', '&&', '||', '<-', '++',
'--', '==', '!=', '<=', '>=', ':=', '...', '.', ':', '<', '>'
]
self.unary_operators = ['+', '-', '*', '!', '&', '<-', ':']
self.postfix_operators = ['++', '--', ':']
groupers = ['(', ')', ',', '[', ']', '{', '}']
# group_starts = ['(', '[', ',', '{']
group_mids = [',']
group_ends = [')', ']', '}']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'break', 'case', 'chan', 'const', 'continue', 'default', 'defer',
'else', 'fallthrough', 'for', 'func', 'go', 'goto', 'if', 'import',
'interface', 'map', 'package', 'range', 'return'
'select', 'struct', 'switch', 'type', 'var'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = [
'uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32',
'int64', 'float32', 'float64', 'complex64', 'complex128', 'byte',
'rune', 'string', 'uint', 'int', 'uintptr'
]
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['nil', 'true', 'false']
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb, whitespace_tb, line_continuation_tb, terminators_tb,
integer_tb, integer_exponent_tb, real_tb, real_exponent_tb,
keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb,
identifier_tb, class_type_tb, string_tb, slash_slash_comment_tb,
slash_star_comment_tb, self.unknown_operator_tb,
invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
# tokens = Examiner.combine_identifier_colon(tokens, ['newline'], ['{'], ['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
# self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
# self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_format_confidence()
self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, tab_size, wide):
super().__init__()
self.operand_types = []
self.whitespace_tb = WhitespaceTokenBuilder()
self.newline_tb = NewlineTokenBuilder()
self.integer_tb = IntegerTokenBuilder(None)
self.integer_exponent_tb = IntegerExponentTokenBuilder(None)
self.binary_integer_tb = SuffixedIntegerTokenBuilder(['B'], False,
None)
self.real_tb = RealTokenBuilder(False, False, None)
self.real_exponent_tb = RealExponentTokenBuilder(
False, False, 'E', None)
self.binary_real_tb = SuffixedRealTokenBuilder(True, True, ['B'],
False, None)
self.operand_types.append('number')
leads = '_'
extras = '_'
self.identifier_tb = IdentifierTokenBuilder(leads, extras)
self.operand_types.append('identifier')
quotes = ['"', "'", "’"]
self.string_tb = EscapedStringTokenBuilder(quotes, 0)
self.operand_types.append('string')
self.label_tb = PL1LabelTokenBuilder()
self.operand_types.append('label')
self.slash_star_comment_tb = SlashStarCommentTokenBuilder()
self.jcl_tb = JCLTokenBuilder()
directives = [
'%ACTIVATE', '%DEACTIVATE', '%DECLARE', '%DCL', '%DICTIONARY',
'%DO', '%ELSE', '%END', '%FATAL', '%GOTO', '%IF', '%INCLUDE',
'%LIST', '%NOLIST', '%PAGE', '%PROCEDURE', '%PROC', '%REPLACE',
'%RETURN', '%THEN'
]
self.line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
self.preprocessor_tb = CaseInsensitiveListTokenBuilder(
directives, 'preprocessor', False)
self.title_tb = LeadToEndOfLineTokenBuilder('%TITLE', True,
'preprocessor')
self.subtitle_tb = LeadToEndOfLineTokenBuilder('%SBTTL', True,
'preprocessor')
self.error_tb = LeadToEndOfLineTokenBuilder('%ERROR', True,
'preprocessor')
self.warn_tb = LeadToEndOfLineTokenBuilder('%WARN', True,
'preprocessor')
self.inform_tb = LeadToEndOfLineTokenBuilder('%INFORM', True,
'preprocessor')
self.terminators_tb = SingleCharacterTokenBuilder(
';', 'statement terminator', False)
known_operators = [
'+', '-', '*', '/', '**', '>', '<', '=', '>=', '<=', '¬>', '¬<',
'¬=', '^>', '^<', '^=', '^', '~>', '~<', '~=', '~', '¬', '&', '&:',
'|', '|:', '||', '!', '!:', '!!', ':'
]
self.unary_operators = ['+', '-', '^', '~', '¬']
self.postfix_operators = []
groupers = ['(', ')', ',', '[', ']', '{', '}']
self.group_starts = ['(', '[', ',', '{']
self.group_mids = [',']
self.group_ends = [')', ']', '}']
self.groupers_tb = CaseInsensitiveListTokenBuilder(
groupers, 'group', False)
self.known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'ALLOCATE', 'ALLOC', 'BEGIN', 'CALL', 'CLOSE', 'DECLARE', 'DCL',
'DO', 'ELSE', 'END', 'FORMAT', 'FREE', 'GET', 'GOTO', 'GO TO',
'IF', 'LEAVE', 'ON', 'OPEN', 'OTHERWISE', 'OTHER', 'PROCEDURE',
'PROC', 'PUT', 'READ', 'RETURN', 'REVERT', 'REWRITE', 'SELECT',
'SIGNAL', 'STOP', 'THEN', 'WHEN', 'WRITE'
]
self.keyword_tb = CaseInsensitiveListTokenBuilder(
keywords, 'keyword', False)
attributes = [
'ALIGNED', 'ANY', 'AREA', 'BASED', 'BUILTIN', 'CONDITION', 'COND',
'CONTROLLED', 'CTL', 'DEFINED', 'DEF', 'DIRECT', 'ENTRY',
'ENVIRONMENT', 'ENV', 'EXTERNAL', 'EXT', 'FILE', 'GLOBALDEF',
'GLOBALREF', 'INITIAL', 'INIT', 'INPUT', 'INTERNAL', 'INT'
'KEYED', 'LABEL', 'LIKE', 'LIST', 'MEMBER', 'NONVARYING', 'NONVAR',
'OPTIONAL', 'OPTIONS', 'OUTPUT', 'PARAMETER', 'PARM', 'PICTURE',
'PIC', 'POSITION', 'POS', 'PRECISION', 'PREC', 'PRINT', 'READONLY',
'RECORD', 'REFER', 'RETURNS', 'SEQUENTIAL', 'SEQL', 'STATIC',
'STREAM', 'STRUCTURE', 'TRUNCATE', 'UNALIGNED', 'UNAL', 'UNION',
'UPDATE', 'VARIABLE', 'VARYING', 'VAR'
]
self.attributes_tb = CaseInsensitiveListTokenBuilder(
attributes, 'attribute', False)
functions = [
'ABS', 'ACOS', 'ACTUALCOUNT', 'ADD', 'ADDR', 'ADDREL',
'ALLOCATION', 'ALLOCN', 'ASIN', 'ATAN', 'ATAND', 'ATANH',
'AUTOMATIC', 'AUTO', 'BINARY', 'BIN', 'BIT', 'BOOL', 'BYTE',
'BYTESIZE', 'CEIL', 'CHARACTER', 'CHAR', 'COLLATE', 'COPY', 'COS',
'COSD', 'COSH', 'DATE', 'DATETIME', 'DECIMAL', 'DEC', 'DECODE',
'DESCRIPTOR', 'DESC', 'DIMENSION', 'DIM', 'DIVIDE', 'EMPTY',
'ENCODE', 'ERROR', 'EVERY', 'EXP', 'FIXED', 'FLOAT', 'FLOOR',
'HBOUND', 'HIGH', 'INDEX', 'INFORM', 'INT', 'LBOUND', 'LENGTH',
'LINE', 'LINENO', 'LOG', 'LOG10', 'LOG2', 'LOW', 'LTRIM', 'MAX',
'MAXLENGTH', 'MIN', 'MOD', 'MULTIPLY', 'NULL', 'OFFSET',
'ONARGSLIST', 'ONCHAR', 'ONCODE', 'ONFILE', 'ONKEY', 'ONSOURCE',
'PAGENO', 'POINTER', 'PTR', 'POSINT', 'PRESENT', 'PROD', 'RANK',
'REFERENCE', 'REVERSE', 'ROUND', 'RTRIM', 'SEARCH', 'SIGN', 'SIN',
'SIND', 'SINH', 'SIZE', 'SOME', 'SQRT', 'STRING', 'SUBSTR',
'SUBTRACT', 'SUM', 'TAN', 'TAND', 'TANH', 'TIME', 'TRANSLATE',
'TRIM', 'TRUNC', 'UNSPEC', 'VALID', 'VALUE', 'VAL', 'VARIANT',
'VERIFY', 'WARN'
]
self.function_tb = CaseInsensitiveListTokenBuilder(
functions, 'function', True)
format_items = [
'A', 'B', 'B1', 'B2', 'B3', 'B4', 'COLUMN', 'COL', 'E', 'F', 'P',
'R', 'TAB', 'X'
]
self.format_item_tb = CaseSensitiveListTokenBuilder(
format_items, 'format', True)
self.operand_types.append('format')
options = [
'APPEND', 'BACKUP_DATE', 'BATCH', 'BLOCK_BOUNDARY_FORMAT',
'BLOCK_IO', 'BLOCK_SIZE', 'BUCKET_SIZE', 'BY', 'CANCEL_CONTROL_O',
'CARRIAGE_RETURN_FORMAT', 'CONTIGUOUS', 'CONTIGUOUS_BEST_TRY',
'CREATION_DATE', 'CURRENT_POSITION', 'DEFAULT_FILE_NAME',
'DEFERRED_WRITE', 'DELETE', 'EDIT', 'EXPIRATION_DATE',
'EXTENSION_SIZE', 'FAST_DELETE', 'FILE_ID', 'FILE_ID_TO',
'FILE_SIZE', 'FIXED_CONTROL_FROM', 'FIXED_CONTROL_SIZE',
'FIXED_CONTROL_SIZE_TO', 'FIXED_CONTROL_TO',
'FIXED_LENGTH_RECORDS', 'FROM', 'GROUP_PROTECTION', 'IDENT',
'IGNORE_LINE_MARKS', 'IN', 'INDEXED', 'INDEX_NUMBER',
'INITIAL_FILL', 'INTO', 'KEY', 'KEYFROM', 'KEYTO', 'LINESIZE',
'LOCK_ON_READ', 'LOCK_ON_WRITE', 'MAIN PROCEDURE',
'MANUAL_UNLOCKING', 'MATCH_GREATER', 'MATCH_GREATER_EQUAL',
'MATCH_NEXT', 'MATCH_NEXT_EQUAL', 'MAXIMUM_RECORD_NUMBER',
'MAXIMUM_RECORD_SIZE', 'MULTIBLOCK_COUNT', 'MULTIBUFFER_COUNT',
'NOLOCK', 'NONEXISTENT_RECORD', 'NONRECURSIVE', 'NORESCAN',
'NO_ECHO', 'NO_FILTER', 'NO_SHARE', 'OWNER_GROUP', 'OWNER_ID',
'OWNER_MEMBER', 'OWNER_PROTECTION', 'PAGE', 'PAGESIZE',
'PRINTER_FORMAT', 'PROMPT', 'PURGE_TYPE_AHEAD', 'READ_AHEAD',
'READ_CHECK', 'READ_REGARDLESS', 'RECORD_ID', 'RECORD_ID_ACCESS',
'RECORD_ID_TO', 'RECURSIVE', 'REPEAT', 'RESCAN',
'RETRIEVAL_POINTERS', 'REVISION_DATE', 'REWIND_ON_CLOSE',
'REWIND_ON_OPEN', 'SCALARVARYING', 'SET READ', 'SHARED_READ',
'SHARED_WRITE', 'SKIP', 'SNAP', 'SPOOL', 'STATEMENT', 'SUPERSEDE',
'SYSTEM', 'SYSTEM_PROTECTION', 'TEMPORARY', 'TIMEOUT_PERIOD',
'TITLE', 'TO', 'UNDERFLOW', 'UFL', 'UNTIL', 'USER_OPEN',
'WAIT_FOR_RECORD', 'WHILE', 'WORLD_PROTECTION', 'WRITE_BEHIND',
'WRITE_CHECK'
]
self.options_tb = CaseInsensitiveListTokenBuilder(
options, 'option', False)
conditions = [
'ANYCONDITION', 'CONVERSION', 'CONV', 'ENDFILE', 'ENDPAGE',
'FINISH', 'FIXEDOVERFLOW', 'FOFL', 'OVERFLOW', 'OFL', 'STORAGE',
'STRINGRANGE', 'STRG', 'SUBSCRIPTRANGE', 'SUBRG', 'UNDEFINEDFILE',
'UNDF', 'VAXCONDITION', 'ZERODIVIDE', 'ZDIV'
]
self.conditions_tb = CaseInsensitiveListTokenBuilder(
conditions, 'condition', False)
subroutines = [
'DISPLAY', 'EXTEND', 'FLUSH', 'NEXT_VOLUME', 'RELEASE', 'RESIGNAL',
'REWIND', 'SPACEBLOCK'
]
self.subroutines_tb = CaseInsensitiveListTokenBuilder(
subroutines, 'subroutine', False)
types = [
'FIXED', 'BINARY', 'FLOAT', 'DECIMAL', 'BIT', 'CHARACTER',
'PICTURE'
]
self.types_tb = CaseInsensitiveListTokenBuilder(types, 'type', True)
self.operand_types.append('type')
values = ['SYSIN', 'SYSPRINT']
self.values_tb = CaseInsensitiveListTokenBuilder(values, 'value', True)
self.operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
# tokenize as free-format
tokenbuilders_free = [
self.newline_tb, self.whitespace_tb, self.line_continuation_tb,
self.terminators_tb, self.integer_tb, self.integer_exponent_tb,
self.binary_integer_tb, self.real_tb, self.real_exponent_tb,
self.binary_real_tb, self.keyword_tb, self.function_tb,
self.attributes_tb, self.options_tb, self.conditions_tb,
self.subroutines_tb, self.types_tb, self.values_tb,
self.groupers_tb, self.known_operator_tb, self.identifier_tb,
self.string_tb, self.label_tb, self.slash_star_comment_tb,
self.preprocessor_tb, self.title_tb, self.subtitle_tb,
self.error_tb, self.warn_tb, self.inform_tb, self.jcl_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer_free = Tokenizer(tokenbuilders_free)
tokens_free = tokenizer_free.tokenize(code)
tokens_free = Examiner.combine_adjacent_identical_tokens(
tokens_free, 'invalid operator')
tokens_free = Examiner.combine_adjacent_identical_tokens(
tokens_free, 'invalid')
self.tokens = tokens_free
self.calc_statistics()
statistics_free = self.statistics
self.statistics = {}
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators,
self.group_ends, allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
self.group_starts, allow_pairs)
self.calc_group_confidence(tokens, self.group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, self.operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
confidences_free = self.confidences
self.confidences = {}
errors_free = self.errors
self.errors = []
# tokenize as fixed-format
tokenbuilders_fixed = [
self.newline_tb, self.whitespace_tb, self.line_continuation_tb,
self.terminators_tb, self.integer_tb, self.integer_exponent_tb,
self.binary_integer_tb, self.real_tb, self.real_exponent_tb,
self.binary_real_tb, self.keyword_tb, self.function_tb,
self.attributes_tb, self.options_tb, self.conditions_tb,
self.subroutines_tb, self.types_tb, self.values_tb,
self.groupers_tb, self.known_operator_tb, self.identifier_tb,
self.string_tb, self.label_tb, self.slash_star_comment_tb,
self.preprocessor_tb, self.title_tb, self.subtitle_tb,
self.error_tb, self.warn_tb, self.inform_tb, self.jcl_tb,
self.unknown_operator_tb, invalid_token_builder
]
comment_start_tb = PL1CommentStartTokenBuilder()
comment_middle_tb = PL1CommentMiddleTokenBuilder()
comment_end_tb = PL1CommentEndTokenBuilder()
type1_tokenbuilders = [comment_start_tb]
tokenbuilders_fixed_1 = tokenbuilders_fixed + type1_tokenbuilders + [
invalid_token_builder
]
tokenizer_fixed_1 = Tokenizer(tokenbuilders_fixed_1)
type2_tokenbuilders = [
comment_start_tb, comment_middle_tb, comment_end_tb
]
tokenbuilders_fixed_2 = tokenbuilders_fixed + type2_tokenbuilders + [
invalid_token_builder
]
tokenizer_fixed_2 = Tokenizer(tokenbuilders_fixed_2)
tokens_fixed = self.tokenize_code(code, tab_size, tokenizer_fixed_1,
tokenizer_fixed_2, wide)
tokens_fixed = Examiner.combine_adjacent_identical_tokens(
tokens_fixed, 'invalid operator')
tokens_fixed = Examiner.combine_adjacent_identical_tokens(
tokens_fixed, 'invalid')
tokens_fixed = Examiner.combine_adjacent_identical_tokens(
tokens_fixed, 'whitespace')
tokens_fixed = self.convert_broken_comments_to_comments(tokens_fixed)
self.tokens = tokens_fixed
self.calc_statistics()
statistics_fixed = self.statistics
self.statistics = {}
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence()
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators,
self.group_ends, allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
self.group_starts, allow_pairs)
self.calc_group_confidence(tokens, self.group_mids)
operand_types_2 = ['number', 'symbol']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, self.operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
confidences_fixed = self.confidences
self.confidences = {}
errors_fixed = self.errors
self.errors = []
# compute confidence for free-format and fixed-format
confidence_free = 1.0
if len(confidences_free) == 0:
confidence_free = 0.0
else:
for key in confidences_free:
factor = confidences_free[key]
confidence_free *= factor
confidence_fixed = 1.0
if len(confidences_fixed) == 0:
confidence_fixed = 0.0
else:
for key in confidences_fixed:
factor = confidences_fixed[key]
confidence_fixed *= factor
# select the better of free-format and spaced-format
if confidence_fixed > confidence_free:
self.tokens = tokens_fixed
self.statistics = statistics_fixed
self.confidences = confidences_fixed
self.errors = errors_fixed
else:
self.tokens = tokens_free
self.statistics = statistics_free
self.confidences = confidences_free
self.errors = errors_free
def __init__(self, code, year):
super().__init__()
operand_types = []
whitespace_tb = WhitespaceTokenBuilder()
newline_tb = NewlineTokenBuilder()
integer_tb = IntegerTokenBuilder("'")
integer_exponent_tb = IntegerExponentTokenBuilder("'")
hex_integer_tb = PrefixedIntegerTokenBuilder('0x', False,
'0123456789abcdefABCDEF')
binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01')
suffixed_integer_tb = SuffixedIntegerTokenBuilder(
['U', 'L', 'LL', 'ULL', 'LLU'], False, None)
real_tb = RealTokenBuilder(False, False, "'")
real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "'")
suffixed_real_tb = SuffixedRealTokenBuilder(False, False, ['f', 'l'],
False, None)
operand_types.append('number')
leads = '_'
extras = '_'
identifier_tb = IdentifierTokenBuilder(leads, extras)
operand_types.append('identifier')
quotes = ['"', "'", "’"]
string_tb = EscapedStringTokenBuilder(quotes, 0)
operand_types.append('string')
class_type_tb = ClassTypeTokenBuilder()
operand_types.append('class')
slash_slash_comment_tb = SlashSlashCommentTokenBuilder()
slash_star_comment_tb = SlashStarCommentTokenBuilder()
directives = [
'#define', '#undef', '#ifdef', '#ifndef', '#if', '#endif', '#else',
'#elif', '#line', '#include', '#pragma'
]
line_continuation_tb = SingleCharacterTokenBuilder(
'\\', 'line continuation', False)
c_preprocessor_tb = CaseSensitiveListTokenBuilder(
directives, 'preprocessor', False)
c_warning_tb = LeadToEndOfLineTokenBuilder('#warning', True,
'preprocessor')
c_error_tb = LeadToEndOfLineTokenBuilder('#error', True,
'preprocessor')
terminators_tb = SingleCharacterTokenBuilder(';',
'statement terminator',
False)
known_operators = [
'+', '-', '*', '/', '%', '=', '==', '!=', '>', '>=', '<', '<=',
'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '!',
'&', '|', '~', '<<', '>>', '^', '.', '++', '--', '->', '&&', '||',
'?', '##'
]
self.unary_operators = ['+', '-', '*', '!', '&', '~', '++', '--']
self.postfix_operators = ['++', '--', '&', '*']
groupers = ['(', ')', ',', '[', ']', '{', '}', ':']
group_starts = ['(', '[', ',', '{']
group_ends = [')', ']', '}']
group_mids = [',', ':']
groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False)
known_operator_tb = CaseSensitiveListTokenBuilder(
known_operators, 'operator', False)
keywords = [
'auto', 'break', 'case', 'const', 'continue', 'default', 'do',
'else', 'enum', 'extern', 'for', 'goto', 'if', 'inline',
'register', 'return', 'signed', 'sizeof', 'static', 'struct',
'switch', 'typedef', 'union', 'unsigned', 'volatile', 'while'
]
keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False)
types = ['char', 'double', 'float', 'int', 'long', 'short']
types_89 = ['void']
types_99 = ['bool', 'complex']
if year in ['89', '99']:
types += types_89
if year in ['99']:
types += types_99
types_tb = CaseSensitiveListTokenBuilder(types, 'type', True)
operand_types.append('type')
values = ['NULL']
values_89 = []
values_99 = ['...', 'true', 'false']
if year in ['89', '99']:
values += values_89
if year in ['99']:
values += values_99
values_tb = CaseSensitiveListTokenBuilder(values, 'value', True)
operand_types.append('value')
invalid_token_builder = InvalidTokenBuilder()
tokenbuilders = [
newline_tb,
whitespace_tb,
line_continuation_tb,
terminators_tb,
integer_tb,
integer_exponent_tb,
hex_integer_tb,
binary_integer_tb,
suffixed_integer_tb,
real_tb,
real_exponent_tb,
suffixed_real_tb,
keyword_tb,
types_tb,
values_tb,
groupers_tb,
known_operator_tb,
identifier_tb,
class_type_tb,
string_tb,
]
if year in ['99']:
tokenbuilders += [
slash_slash_comment_tb,
]
tokenbuilders += [
slash_star_comment_tb, c_preprocessor_tb, c_error_tb, c_warning_tb,
self.unknown_operator_tb, invalid_token_builder
]
tokenizer = Tokenizer(tokenbuilders)
tokens = tokenizer.tokenize(code)
tokens = Examiner.combine_adjacent_identical_tokens(
tokens, 'invalid operator')
tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid')
tokens = Examiner.combine_identifier_colon(
tokens, ['statement terminator', 'newline'], ['{'],
['whitespace', 'comment'])
self.tokens = tokens
self.convert_identifiers_to_labels()
self.calc_statistics()
tokens = self.source_tokens()
tokens = Examiner.join_all_lines(tokens)
self.calc_token_confidence()
self.calc_token_2_confidence(['*', ';'])
num_operators = self.count_my_tokens(['operator', 'invalid operator'])
if num_operators > 0:
self.calc_operator_confidence(num_operators)
allow_pairs = []
self.calc_operator_2_confidence(tokens, num_operators, allow_pairs)
self.calc_operator_3_confidence(tokens, num_operators, group_ends,
allow_pairs)
self.calc_operator_4_confidence(tokens, num_operators,
group_starts, allow_pairs)
self.calc_group_confidence(tokens, group_mids)
operand_types_2 = ['number']
self.calc_operand_n_confidence(tokens, operand_types_2, 2)
self.calc_operand_n_confidence(tokens, operand_types, 4)
self.calc_keyword_confidence()
self.calc_paired_blockers_confidence(['{'], ['}'])
self.calc_line_length_confidence(code, self.max_expected_line)
