def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() EscapedStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() SuffixedStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() PrefixedIntegerTokenBuilder.__escape_z__() SuffixedIntegerTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() RealExponentTokenBuilder.__escape_z__() SuffixedRealTokenBuilder.__escape_z__() IdentifierTokenBuilder.__escape_z__() PrefixedIdentifierTokenBuilder.__escape_z__() CaseInsensitiveListTokenBuilder.__escape_z__() CaseSensitiveListTokenBuilder.__escape_z__() SingleCharacterTokenBuilder.__escape_z__() SlashSlashCommentTokenBuilder.__escape_z__() SlashStarCommentTokenBuilder.__escape_z__() ClassTypeTokenBuilder.__escape_z__() HexRealExponentTokenBuilder.__escape_z__() NestedCommentTokenBuilder.__escape_z__() return 'Escape ?Z'
def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() EscapedStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() PrefixedIntegerTokenBuilder.__escape_z__() SuffixedIntegerTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() AssemblyCommentTokenBuilder.__escape_z__() return 'Escape ?Z'
def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() EscapedStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() PrefixedRawStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() RealExponentTokenBuilder.__escape_z__() IdentifierTokenBuilder.__escape_z__() PrefixedIdentifierTokenBuilder.__escape_z__() CaseInsensitiveListTokenBuilder.__escape_z__() CaseSensitiveListTokenBuilder.__escape_z__() SingleCharacterTokenBuilder.__escape_z__() LeadToEndOfLineTokenBuilder.__escape_z__() TripleQuoteStringTokenBuilder.__escape_z__() RawTripleQuoteCommentTokenBuilder.__escape_z__() return 'Escape ?Z'
def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() EscapedStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() RealExponentTokenBuilder.__escape_z__() IdentifierTokenBuilder.__escape_z__() CaseInsensitiveListTokenBuilder.__escape_z__() CaseSensitiveListTokenBuilder.__escape_z__() BlockTokenBuilder.__escape_z__() TripleQuoteStringTokenBuilder.__escape_z__() SlashSlashCommentTokenBuilder.__escape_z__() TripleSlashCommentTokenBuilder.__escape_z__() LeadToEndOfLineTokenBuilder.__escape_z__() NullTokenBuilder.__escape_z__() ClassTypeTokenBuilder.__escape_z__() return 'Escape ?Z'
def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() StuffedQuoteStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() RealExponentTokenBuilder.__escape_z__() CaseInsensitiveListTokenBuilder.__escape_z__() CaseSensitiveListTokenBuilder.__escape_z__() SingleCharacterTokenBuilder.__escape_z__() LeadToEndOfLineTokenBuilder.__escape_z__() BlockTokenBuilder.__escape_z__() CobolIdentifierTokenBuilder.__escape_z__() PictureTokenBuilder.__escape_z__() CRPictureTokenBuilder.__escape_z__() CobolPreprocessorTokenBuilder.__escape_z__() AsteriskCommentTokenBuilder.__escape_z__() return 'Escape ?Z'
def __escape_z__(): InvalidTokenBuilder.__escape_z__() WhitespaceTokenBuilder.__escape_z__() NewlineTokenBuilder.__escape_z__() EscapedStringTokenBuilder.__escape_z__() PrefixedStringTokenBuilder.__escape_z__() IntegerTokenBuilder.__escape_z__() IntegerExponentTokenBuilder.__escape_z__() PrefixedIntegerTokenBuilder.__escape_z__() SuffixedIntegerTokenBuilder.__escape_z__() RealTokenBuilder.__escape_z__() IdentifierTokenBuilder.__escape_z__() CaseInsensitiveListTokenBuilder.__escape_z__() CaseSensitiveListTokenBuilder.__escape_z__() LeadToEndOfLineTokenBuilder.__escape_z__() SingleCharacterTokenBuilder.__escape_z__() LabelTokenBuilder.__escape_z__() AssemblyCommentTokenBuilder.__escape_z__() MultilineCommentTokenBuilder.__escape_z__() HashQuoteCharTokenBuilder.__escape_z__() return 'Escape ?Z'
def __init__(self, code): super().__init__() operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder(None) integer_exponent_tb = IntegerExponentTokenBuilder(None) real_tb = RealTokenBuilder(False, False, None) real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', None) operand_types.append('number') leads = '_' extras = '_' identifier_tb = IdentifierTokenBuilder(leads, extras) operand_types.append('identifier') directive_tb = DirectiveTokenBuilder() quotes = ['"', "'", "’"] string_tb = EscapedStringTokenBuilder(quotes, 10) prefixed_string_tb = PrefixedStringTokenBuilder('@', False, quotes) operand_types.append('string') class_type_tb = ClassTypeTokenBuilder() operand_types.append('class') slash_slash_comment_tb = SlashSlashCommentTokenBuilder() slash_star_comment_tb = SlashStarCommentTokenBuilder() directives = [ '#define', '#undef', '#ifdef', '#ifndef', '#if', '#endif', '#else', '#elif', '#import', '#line', '#include' ] line_continuation_tb = SingleCharacterTokenBuilder( '\\', 'line continuation', False) c_preprocessor_tb = CaseSensitiveListTokenBuilder( directives, 'preprocessor', False) c_warning_tb = LeadToEndOfLineTokenBuilder('#warning', True, 'preprocessor') c_error_tb = LeadToEndOfLineTokenBuilder('#error', True, 'preprocessor') c_pragma_tb = LeadToEndOfLineTokenBuilder('#pragma', True, 'preprocessor') terminators_tb = SingleCharacterTokenBuilder(';', 'statement terminator', False) known_operators = [ '+', '-', '*', '/', '%', '=', '==', '!=', '>', '>=', '<', '<=', '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '!', '&', '|', '<<', '>>', '~', '.', '->', '++', '--', '&&', '||', '^', '?', '##' ] self.unary_operators = [ '+', '-', '*', '!', '&', '^', '~', '++', '--', '##' ] self.postfix_operators = ['++', '--', '&', '->', '*', '^'] groupers = ['(', ')', ',', '[', ']', '{', '}', ':'] group_starts = ['(', '[', ',', '{'] group_ends = [')', ']', '}'] group_mids = [',', ':'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) keywords = [ 'atomic', 'break', 'bycopy', 'byref', 'case', 'continue', 'default', 'do', 'else', 'for', 'goto', 'if', 'IMP', 'in', 'inline', 'inout', 'nonatomic', 'oneway', 'out', 'Protocol', 'restrict', 'retain', 'return', 'SEL', 'sizeof', 'switch', 'typedef', 'while', '@interface', '@end', '@implementation', '@protocol', '@class', '@public', '@protected', '@private', '@property', '@try', '@throw', '@catch()', '@finally', '@synthesize', '@dynamic', '@selector' ] keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) types = [ 'auto', 'char', 'const', 'double', 'enum', 'extern', 'float', 'id', 'int', 'long', 'register', 'short', 'signed', 'static', 'struct', 'union', 'unsigned', 'void', 'volatile', '_Bool', '_Complex', '_Imaginary', 'BOOL', 'Class' ] types_tb = CaseSensitiveListTokenBuilder(types, 'type', True) operand_types.append('type') values = ['self', 'super', 'nil', 'YES', 'NO', 'NULL', '...'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, line_continuation_tb, terminators_tb, integer_tb, integer_exponent_tb, real_tb, real_exponent_tb, keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb, directive_tb, identifier_tb, class_type_tb, string_tb, prefixed_string_tb, slash_slash_comment_tb, slash_star_comment_tb, c_preprocessor_tb, c_warning_tb, c_error_tb, c_pragma_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid operator') tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid') # tokens = Examiner.combine_identifier_colon(tokens, ['statement terminator', 'newline'], ['{'], ['whitespace', 'comment', 'line description']) self.tokens = tokens self.convert_identifiers_to_labels() self.convert_values_to_operators() self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_all_lines(tokens) self.calc_token_confidence() self.calc_token_2_confidence(['*', ';']) num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = ['number', 'string', 'symbol'] self.calc_operand_n_confidence(tokens, operand_types_2, 2) self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_paired_blockers_confidence(['{'], ['}']) self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, block_comment_limit): super().__init__() operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() line_continuation_tb = SingleCharacterTokenBuilder( '\\', 'line continuation', False) integer_tb = IntegerTokenBuilder('_') integer_exponent_tb = IntegerExponentTokenBuilder('_') real_tb = RealTokenBuilder(False, True, '_') real_exponent_tb = RealExponentTokenBuilder(False, True, 'E', '_') octal_integer_tb = PrefixedIntegerTokenBuilder('0o', True, '01234567_') hex_integer_tb = PrefixedIntegerTokenBuilder( '0x', True, '0123456789ABCDEFabcdef_') binary_integer_tb = PrefixedIntegerTokenBuilder('0b', True, '01_') operand_types.append('number') leads = '_' extras = '_' identifier_tb = IdentifierTokenBuilder(leads, extras) operand_types.append('identifier') lifetime_tb = IdentifierTokenBuilder("'", extras) attribute_tb = RustAttributeTokenBuilder() quotes = ['"'] string_tb = EscapedStringTokenBuilder(quotes, 10) bstring_tb = PrefixedStringTokenBuilder('b', True, quotes) rstring_tb = RustRawStringTokenBuilder() operand_types.append('string') char_tb = SingleCharStringTokenBuilder() class_type_tb = ClassTypeTokenBuilder() operand_types.append('class') slash_slash_comment_tb = SlashSlashCommentTokenBuilder() slash_star_comment_tb = NestedCommentTokenBuilder( '/*', '*/', block_comment_limit) terminators_tb = SingleCharacterTokenBuilder(';', 'statement terminator', False) known_operators = [ '+', '-', '*', '/', '%', '^', '!', '&', '|', '&&', '||', '<<', '>>', '+=', '-=', '*=', '/=', '%=', '^=', '&=', '|-', '<<=', '>>=', '=', '==', '!=', '>', '<', '>=', '<=', '@', '.', '..', '...', '->', '#', '$', '?', 'in', '&mut' ] self.unary_operators = ['+', '-', '*', '!', '&', '&mut'] self.postfix_operators = [] groupers = ['(', ')', ',', '[', ']', '{', '}', ':', '::', '=>'] group_starts = ['(', '[', ',', '{'] group_mids = [',', ':', '::', '=>'] group_ends = [')', ']', '}', ')|'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) keywords = [ 'as', 'break', 'const', 'continue', 'crate' 'else', 'enum', 'extern', 'fn', 'for', 'if', 'impl', 'let', 'loop', 'match', 'mod', 'move', 'mut', 'pub', 'ref', 'return', 'static', 'struct', 'trait', 'type', 'unsafe', 'use', 'where', 'while' ] keywords_2018 = ['dyn', 'union', 'static'] keywords_future = [ 'abstract', 'become', 'box', 'do', 'final', 'macro', 'override', 'priv', 'typeof', 'unsized', 'virtual', 'yield', 'async', 'await', 'try' ] keywords += keywords_2018 keywords += keywords_future keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) types = [ 'Self', 'u8', 'i8', 'u16', 'i16', 'u32', 'i32', 'u64', 'i64', 'u128', 'i128', 'usize', 'isize', 'f32', 'f64' ] types_tb = CaseSensitiveListTokenBuilder(types, 'type', True) operand_types.append('type') values = ['self', 'true', 'false', 'super', '_'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, line_continuation_tb, terminators_tb, integer_tb, integer_exponent_tb, octal_integer_tb, hex_integer_tb, binary_integer_tb, real_tb, real_exponent_tb, keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb, identifier_tb, char_tb, lifetime_tb, class_type_tb, attribute_tb, string_tb, bstring_tb, rstring_tb, slash_slash_comment_tb, slash_star_comment_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid operator') tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid') self.tokens = self.combine_numbers_and_adjacent_types(tokens) self.convert_operators_to_identifiers() self.convert_bars_to_groups() self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_all_lines(tokens) self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = ['number', 'symbol'] self.calc_operand_n_confidence(tokens, operand_types_2, 2) self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_paired_blockers_confidence(['{'], ['}']) self.calc_line_format_confidence() self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, tab_size): super().__init__() operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder("'") integer_exponent_tb = IntegerExponentTokenBuilder("'") real_tb = RealTokenBuilder(True, True, None) hex_integer_1_tb = PrefixedIntegerTokenBuilder( '$', False, '0123456789abcdefABCDEF') hex_integer_2_tb = PrefixedIntegerTokenBuilder( '#$', False, '0123456789abcdefABCDEF') hex_integer_3_tb = PrefixedIntegerTokenBuilder( '&', False, '0123456789abcdefABCDEF') hex_integer_h_tb = SuffixedIntegerTokenBuilder(['h'], False, 'abcdefABCDEF') binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01') suffixed_integer_tb = SuffixedIntegerTokenBuilder( ['Q', 'A', 'O', 'D', 'B'], False, None) operand_types.append('number') leads = '_$#.' extras = '_$#.' identifier_tb = IbmAsmIdentifierTokenBuilder(leads, extras) operand_types.append('identifier') quotes = ['"', "'", "’"] string_tb = EscapedStringTokenBuilder(quotes, 0) hex_string_tb = PrefixedStringTokenBuilder('X', False, quotes) char_string_tb = PrefixedStringTokenBuilder('C', False, quotes) operand_types.append('string') known_operators = ['+', '-', '*', '/', '=', '&', '#', '?'] self.unary_operators = ['+', '-', '=', '&', '#', '?'] self.postfix_operators = [] groupers = ['(', ')', ',', '[', ']', '{', '}', ':', '<', '>'] group_starts = ['(', '[', ',', '{', '<'] group_ends = [')', ']', '}', '>'] group_mids = [','] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) # keywords = [] # keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) # types = [] # types_tb = CaseSensitiveListTokenBuilder(types, 'type', True) values = ['*'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') comment_tb = AssemblyCommentTokenBuilder(';*') title_directive_tb = LeadToEndOfLineTokenBuilder( 'TITLE', False, 'directive') subtitle_directive_tb = LeadToEndOfLineTokenBuilder( 'SUBTTL', False, 'directive') include_directive_tb = LeadToEndOfLineTokenBuilder( 'INCLUDE', False, 'directive') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, integer_tb, integer_exponent_tb, hex_integer_1_tb, hex_integer_2_tb, hex_integer_3_tb, hex_integer_h_tb, binary_integer_tb, suffixed_integer_tb, real_tb, values_tb, groupers_tb, known_operator_tb, title_directive_tb, subtitle_directive_tb, include_directive_tb, identifier_tb, string_tb, hex_string_tb, char_string_tb, comment_tb, self.unknown_operator_tb, invalid_token_builder ] opcode_tokenbuilders = [identifier_tb, invalid_token_builder] args_tokenbuilders = [ integer_tb, integer_exponent_tb, hex_integer_1_tb, hex_integer_2_tb, hex_integer_3_tb, hex_integer_h_tb, binary_integer_tb, suffixed_integer_tb, real_tb, values_tb, groupers_tb, known_operator_tb, identifier_tb, string_tb, hex_string_tb, char_string_tb, comment_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) opcode_tokenizer = Tokenizer(opcode_tokenbuilders) args_tokenizer = Tokenizer(args_tokenbuilders) # tokenize as free-format tokens_free = tokenizer.tokenize(code) tokens_free = Examiner.combine_adjacent_identical_tokens( tokens_free, 'invalid operator') tokens_free = Examiner.combine_adjacent_identical_tokens( tokens_free, 'invalid') tokens_free = Examiner.convert_values_to_operators( tokens_free, known_operators) self.tokens = tokens_free self.convert_asm_identifiers_to_labels() self.calc_statistics() statistics_free = self.statistics self.statistics = {} self.calc_confidences(operand_types, group_starts, group_mids, group_ends, None) self.calc_line_length_confidence(code, self.max_expected_line) confidences_free = self.confidences self.confidences = {} errors_free = self.errors self.errors = [] # tokenize as space-format opcode_extras = '.&=,()+-*/' label_leads = '.&$@' label_mids = '.&$#@' label_ends = ':,' comment_leads = '*;!' line_comment_leads = '' use_line_id = False tokens_space, indents = Tokenizer.tokenize_asm_code( code, tab_size, opcode_tokenizer, opcode_extras, args_tokenizer, label_leads, label_mids, label_ends, comment_leads, line_comment_leads, use_line_id) tokens_space = Examiner.combine_adjacent_identical_tokens( tokens_space, 'invalid operator') tokens_space = Examiner.combine_adjacent_identical_tokens( tokens_space, 'invalid') tokens_space = Examiner.combine_identifier_colon( tokens_space, ['newline'], [], []) tokens_space = Tokenizer.combine_number_and_adjacent_identifier( tokens_space) tokens_space = Examiner.convert_values_to_operators( tokens_space, known_operators) self.tokens = tokens_space self.convert_asm_identifiers_to_labels() self.calc_statistics() statistics_space = self.statistics self.statistics = {} self.calc_confidences(operand_types, group_starts, group_mids, group_ends, indents) self.calc_line_length_confidence(code, self.max_expected_line) confidences_space = self.confidences self.confidences = {} errors_space = self.errors self.errors = [] # select the better of free-format and spaced-format confidence_free = 1.0 for key in confidences_free: factor = confidences_free[key] confidence_free *= factor confidence_space = 1.0 for key in confidences_space: factor = confidences_space[key] confidence_space *= factor if confidence_space > confidence_free: self.tokens = tokens_space self.statistics = statistics_space self.confidences = confidences_space self.errors = errors_space else: self.tokens = tokens_free self.statistics = statistics_free self.confidences = confidences_free self.errors = errors_free
def __init__(self, code, tab_size, processor): super().__init__() self.newlines_important = 'always' operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder("'") integer_exponent_tb = IntegerExponentTokenBuilder("'") real_tb = RealTokenBuilder(True, True, None) hex_integer_1_tb = PrefixedIntegerTokenBuilder('$', False, '0123456789abcdefABCDEF') hex_integer_2_tb = PrefixedIntegerTokenBuilder('#$', False, '0123456789abcdefABCDEF') hex_integer_3_tb = PrefixedIntegerTokenBuilder('&', False, '0123456789abcdefABCDEF') hex_integer_h_tb = SuffixedIntegerTokenBuilder(['h'], False, 'abcdefABCDEF') binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01') suffixed_integer_tb = SuffixedIntegerTokenBuilder(['Q', 'A', 'O', 'D', 'B'], False, None) operand_types.append('number') leads = '$#.@&' extras = '$#.@&' identifier_tb = IbmAsmIdentifierTokenBuilder(leads, extras) operand_types.append('identifier') quotes = ['"', "'", "’"] string_tb = EscapedStringTokenBuilder(quotes, 0) hex_string_tb = PrefixedStringTokenBuilder('X', False, quotes) char_string_tb = PrefixedStringTokenBuilder('C', False, quotes) operand_types.append('string') known_operators = [ '+', '-', '*', '/', '=', '&', '#', '?', "'" ] self.unary_operators = [ '+', '-', '=', '&', '#', '?', "'" ] self.postfix_operators = [] groupers = ['(', ')', ',', '[', ']', '{', '}', ':', '<', '>'] group_starts = ['(', '[', ',', '{', '<'] group_ends = [')', ']', '}', '>'] group_mids = [','] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) known_operator_tb = CaseSensitiveListTokenBuilder(known_operators, 'operator', False) preprocessors = [ 'MACRO', 'MEND' ] preprocessor_tb = CaseInsensitiveListTokenBuilder(preprocessors, 'preprocesssor', False) directives = [ 'CSECT', 'DC', 'DROP', 'DS', 'EJECT', 'END', 'ENTRY', 'EQU', 'EXTRN', 'FREEMAIN', 'GETMAIN', 'GLOBAL', 'NAM', 'NAME', 'ORG', 'PAGE', 'PARAM', 'PROC', 'PUBLIC', 'RETURN', 'STIMER', 'TITLE', 'SUBTTL', 'USING' ] directive_tb = CaseInsensitiveListTokenBuilder(directives, 'directive', False) keywords = [] keywords_360 = [ 'A', 'ABEND', 'AD', 'ADR', 'AE', 'AER', 'AH', 'AL', 'ALR', 'AP', 'AR', 'AU', 'AUR', 'AW', 'AWR', 'AXR', 'B', 'BAL', 'BALR', 'BAS', 'BASR', 'BC', 'BCR', 'BCT', 'BCTR', 'BE', 'BH', 'BL', 'BM', 'BNE', 'BNH', 'BNL', 'BNM', 'BNP', 'BNO', 'BNZ', 'BO', 'BP', 'BR', 'BXH', 'BXLE', 'BZ', 'C', 'CD', 'CDR', 'CE', 'CER', 'CH', 'CL', 'CLC', 'CLI', 'CLR', 'CP', 'CR', 'CVB', 'CVD', 'D', 'DD', 'DDR', 'DE', 'DER', 'DIAGNOSE', 'DP', 'DR', 'ED', 'EDMK', 'EX', 'HDR', 'HER', 'HIO', 'IC', 'ISK', 'L', 'LA', 'LCR', 'LCDR', 'LCER', 'LD', 'LDR', 'LE', 'LER', 'LH', 'LM', 'LNDR', 'LNER', 'LNR', 'LPDR', 'LPER', 'LPR', 'LPSW', 'LR', 'LRDR', 'LRER', 'LTDR', 'LTER', 'LTR', 'M', 'MD', 'MDR', 'ME', 'MER', 'MH', 'MP', 'MR', 'MVC', 'MVI', 'MVN', 'MVO', 'MVZ', 'MXD', 'MXDR', 'MXR', 'N', 'NC', 'NI', 'NOP', 'NOPR', 'NR', 'O', 'OC', 'OI', 'OR', 'PACK', 'RDD', 'S', 'SD', 'SDR', 'SE', 'SER', 'SH', 'SIO', 'SL', 'SLA', 'SLDA', 'SLDL', 'SLL', 'SLR', 'SP', 'SPM', 'SR', 'SRA', 'SRDL', 'SRP', 'SSK', 'SSM', 'SRDA', 'SRL', 'ST', 'STC', 'STD', 'STE', 'STH', 'STM', 'SU', 'SUR', 'SVC', 'SW', 'SWR', 'SXR', 'TCH', 'TIO', 'TM', 'TR', 'TRT', 'TS', 'UNPK', 'UNPKU', 'WRD', 'X', 'XC', 'XI', 'XR', 'ZAP' ] keywords_370 = [ 'BRXH', 'BRXLE', 'CLCL', 'HDV', 'LAM', 'LEDR', 'MS', 'MVCL', 'RIO', 'SIOF', 'STAM', 'VA', 'VACD', 'VACDR', 'VACE', 'VACER', 'VAD', 'VADQ', 'VADR', 'VADS', 'VAE', 'VAEQ', 'VAER', 'VAES', 'VAQ', 'VAR', 'VAS', 'VC', 'VCD', 'VCDQ', 'VCDR', 'VCDS', 'VCE', 'VCEQ', 'VCER', 'VCES', 'VCQ', 'VCR', 'VCS', 'VDD', 'VDDQ', 'VDDR', 'VDDS', 'VDE', 'VDEQ', 'VDER', 'VDES', 'VL', 'VLCDR', 'VLCER', 'VLCR', 'VLD', 'VLDQ', 'VLDR', 'VLEQ', 'VLH', 'VLINT', 'VLM', 'VLMD', 'VLMDQ', 'VLMDR', 'VLMEQ', 'VLMQ', 'VLMR', 'VLNDR', 'VLNER', 'VLNR', 'VLPDR', 'VLPER', 'VLPR', 'VLQ', 'VLR', 'VLY', 'VLYD', 'VLZDR', 'VLZR', 'VM', 'VMAD', 'VMADQ', 'VMADS', 'VMAE', 'VMAEQ', 'VMAES', 'VMCD', 'VMCE', 'VMCER', 'VMD', 'VMDQ', 'VMDR', 'VMDS', 'VME', 'VMEQ', 'VMER', 'VMES', 'VMQ', 'VMR', 'VMS', 'VMSD', 'VMSDQ', 'VMSDS', 'VMSE', 'VMSEQ', 'VMSES', 'VN', 'VNQ', 'VNR', 'VNS', 'VO', 'VOQ', 'VOR', 'VOS', 'VS', 'VSD', 'VSDQ', 'VSDR', 'VSDS', 'VSE', 'VSEQ', 'VSER', 'VSES', 'VSQD', 'VSQDR', 'VSQE', 'VSQER', 'VSQ', 'VSR', 'VSS', 'VST', 'VSTD', 'VSTE', 'VSTH', 'VSTKD', 'VSTMD', 'VTAD', 'VTAE', 'VTSD', 'VTSE', 'VX', 'VXQ', 'VXR', 'VXS', 'VMXSE', 'VMNSE', 'VMXAE', 'VLELE', 'VSELE', 'VMXDS', 'VMNSD', 'VMXAD', 'VLELD', 'VXELD', 'VSPSD', 'VAPSD', 'VTVM', 'VCVM', 'VCZVM', 'VCOVM', 'VXVC', 'VXVMM', 'VRRS', 'VRSVC', 'VRSV', 'VLVM', 'VLCVM', 'VSTVM', 'VNVM', 'VOVM', 'VXVM', ' VSRSV', 'VMRSV', 'VSRRS', 'VLVCA', 'VRCL', 'VSVMM', 'VLVXA', 'VSVTP', 'VACSV', 'VACRS', 'STNSM', 'SOTSM', 'SIOP', 'MC', 'LRA', 'CONCS', 'DISCS', 'STIDP', 'SCK', 'SPT', 'STPT', 'SPKA', 'IPK', 'PTLB', 'SPX', 'STPX', 'STAP', 'RRB', 'PC', 'SAC', 'IPTE', 'IVSK', 'IAC', 'SSAR', 'EPAR', 'ESAR', 'PT', 'ISKE', 'RRBE', 'SSKE', 'TB', 'STCTL', 'LCTL', 'CS', 'CDS', 'CLM', 'STCM', 'ICM', 'MVCK', 'MVCP', 'MVCS', 'VLI', 'VSTI', 'VLID', 'VSTID', 'VSRL', 'VSLL', 'VLBIX', 'LASP', 'TPROT', 'STRAG', 'MVCSK', 'MVCDK', 'DPFET', 'MVHHI', 'MVGHI', 'MVHI', 'CHHSI', 'CLHHSI', 'CGHSI', 'CLGHSI', 'CHSI', 'CLFHSI', 'TBEGIN', 'TBEGINC', 'MVCIN', 'UNPKA' ] keywords_390 = [ 'BASSM', 'BSG', 'BSM', 'CLRCH', 'CMPS', 'CLRIO', 'CMSG', 'LAE', 'LXDR', 'MDE', 'PFPO', 'PR', 'PTFF', 'SAM24', 'SAM31', 'SCKPF', 'TAM', 'TMPS', 'TMSG', 'TRACE', 'TRAP2', 'TMH',' TMLH', 'TML', 'TMLL', 'TMHH', 'TMHL', 'BRC', 'BRAS', 'BRCT', 'BRCTG', 'LHI', 'LGHI', 'AHI', 'AGHI', 'MHI', 'MGHI', 'CHI', 'CGHI', 'MVCLE', 'CLCLE', 'UPT', 'SIE', 'PCF', 'CFC', 'DEP', 'DCTP', 'MAD', 'MUN', 'STCAP', 'SERVC', 'IPM', 'DXR', 'PGIN', 'PGOUT', 'CSCH', 'HSCH', 'MSCH', 'SSCH', 'STSCH', 'TSCH', 'TPI', 'SAL', 'RSCH', 'STCRW', 'STCPS', 'RCHP', 'SCHM', 'STZP', 'SZP', 'TPZI', 'BAKR', 'CKSM', 'MADS', 'SQDR', 'STURA', 'MSTA', 'PALB', 'EREG', 'ESTA', 'LURA', 'TAR', 'SQDR', 'SAR', 'EAR', 'CSP', 'MSR', 'MVPG', 'MVST', 'CUSE', 'BSG', 'CLST', 'SRST', 'XSCH', 'RP', 'STCKE', 'SACF', 'STSI', 'SRNM', 'STFPC', 'LFPC', 'TRE', 'CUUTF', 'CUTFU', 'STFL', 'LPSWE', 'TRAP4', 'LPEBR', 'LNEBR', 'LTEBR', 'LCEBR', 'LDEBR', 'LXDBR', 'LDEBR', 'MXDBR', 'KEBR', 'CEBR', 'AEBR', 'SEBR', 'MDEBR', 'DEBR', 'MAEBR', 'MSEBR', 'LPDBR', 'LCDBR', 'SQEBR', 'MEEBR', 'KDBR', 'CDBR', 'ADBR', 'MDBR', 'DDBR', 'SDBR', 'LDER', 'LXDR', 'MAER', 'MSER', 'SQXR', 'MEER', 'MADR', 'MSDR', 'LPXBR', 'LNXBR', 'LTXBR', 'LCXBR', 'LCXBR', 'LEDBR', 'LDXBR', 'LEXBR', 'FIXBR', 'KXBR', 'CXBR', 'AXBR', 'SXBR', 'MXBR', 'DXBR', 'TBEDR', 'TBDR', 'DIEBR', 'FIEBR', 'THDER', 'DIDBR', 'FIDBR', 'LPXR', 'LNXR', 'LTXR', 'LCXR', 'LXR', 'LEXR', 'FIXR', 'CXR', 'LZER', 'LZDR', 'LZXR', 'FIER', 'FIDR', 'SFPC', 'EFPC', 'CEFBR', 'CDFBR', 'CXFBR', 'CEGBR', 'CEFR', 'CDFR', 'CXFR', 'CFDR', 'CFXR', 'CEGR', 'CDGR', 'CXGR', 'CGER', 'CGDR', 'CGXR', 'CDGBR', 'CXGBR', 'CGDBR', 'CGEBR', 'CGXBR', 'LMC', 'LPGR', 'LNGR', 'LTGR', 'LCGR', 'LGC', 'LURAG', 'AGR', 'SGR', 'ALGR', 'SLGR', 'MSGR', 'DSGR', 'EREGG', 'LRVGR', 'LPGFR', 'LNGFR', 'LTGFR', 'LCGFR', 'LGFR', 'LLGFR', 'LLGTR', 'AGFR', 'SGFR', 'ALGFR', 'SLGFR', 'MSGFR', 'DSGFR', 'LRVR', 'CGR', 'CLGR', 'STURG', 'CGFR', 'CLGFR', 'BCTGR', 'NGR', 'OGR', 'XGR', 'MLGR', 'DLGR', 'ALCGR', 'SLBGR', 'EPSW', 'TRTT', 'TRTO', 'TROT', 'TROO', 'MLR', 'DLR', 'ALCR', 'SLBR', 'ESEA', 'LARL', 'LGFI', 'BRCL', 'BRASL', 'XIHF', 'XILF', 'IIHF', 'IILF', 'NIHF', 'NILF', 'OIHF', 'OILF', 'LLIHF', 'LLILF', 'LLHRL', 'LGHRL', 'LHRL', 'AGFI', 'AFI', 'ALGFI', 'ALFI', 'CGFI', 'CFI', 'LLGFRL', 'STRL', 'EXRL', 'PFDRL', 'CGHRL','CHRL', 'CLGHRL', 'CLHRL', 'CGRL', 'CLGRL', 'CRL', 'CLGFRL', 'CLRL', 'MVCOS', 'ECTG', 'CSST', 'PKU', 'LRAG', 'LG', 'AG', 'SG', 'ALG', 'SLG', 'MSG', 'DSG', 'CVBG', 'LRVG', 'LGF', 'LGH', 'LLGF', 'LLGT', 'AGF', 'SGF', 'ALGF', 'SLGF', 'MSGF', 'DSGF', 'LRV', 'LRVH', 'CG', 'CLG', 'STG', 'CVDG', 'STRVG', 'CGF', 'CLGF', 'STRV', 'STRVH', 'BCTG', 'NG', 'OG', 'XG', 'MLG', 'DLG', 'ALCG', 'SLBG', 'STPQ', 'LPQ', 'LLGC', 'LLGH', 'ML', 'DL', 'ALC', 'SLB', 'PKA', 'DIL', 'BDIL', 'ANUM', 'COMP', 'MCPU', 'MIO', 'BIFLAG', 'MULDIV', 'LMG', 'SRAG', 'SLAG', 'SRLG', 'SLLG', 'TRACG', 'RLLG', 'RLL', 'CLMH', 'CLMY', 'CLT', 'CLTH', 'CLTL', 'CLTNE', 'CLTE', 'CLTNL', 'CLTNH', 'STMG', 'STCTG', 'STMH', 'STCMH', 'LCTLG', 'CSG', 'CDSG', 'BXHG', 'BXLEG', 'ICMH', 'MVCLU', 'CLCLU', 'LMH', 'LMY', 'TP', 'SRAK', 'SLAK', 'SRLK', 'SRLK', 'LOCG', 'BRXHG', 'BRXLG', 'LDEB', 'LXDB', 'LXEB', 'MXDB', 'KEB', 'CEB', 'AEB', 'SEB', 'MDEB', 'DEB', 'MAEB', 'MSEB', 'TCEB', 'TCDB', 'TCXB', 'SQEB', 'SQDB', 'MEEB', 'KDB', 'CDB', 'ADB', 'SDB', 'MDB', 'DDB', 'MADB', 'MSDB', 'LDE', 'LXD', 'LXE', 'SQE', 'SQD', 'MEE', 'PLO', 'LMD' ] keywords_z = [ 'IIHH', 'IIHL', 'IILH', 'IILL', 'LLIHH', 'LLIHL', 'LLILH', 'LLILL', 'NIHH', 'NIHL', 'NILH', 'NILL', 'OIHH', 'OIHL', 'OILH', 'OILL', 'SAM64' ] if processor in ['360', '370', '390', 'system-z']: keywords += keywords_360 if processor in ['370', '390', 'system-z']: keywords += keywords_370 if processor in ['390', 'system-z']: keywords += keywords_390 if processor in ['system-z']: keywords += keywords_z opcode_tb = CaseInsensitiveListTokenBuilder(keywords, 'keyword', False) registers = [ 'R0', 'R1', 'R2', 'R3', 'R4', 'R5', 'R6', 'R7', 'R8', 'R9', 'R10', 'R11', 'R12', 'R13', 'R14', 'R15', 'FP0', 'FP2', 'FP4', 'FP6' ] register_tb = CaseInsensitiveListTokenBuilder(registers, 'register', True) values = ['*'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') comment_tb = LeadToEndOfLineTokenBuilder('!', False, 'comment') line_comment_tb = AssemblyCommentTokenBuilder('*') include_directive_tb = LeadToEndOfLineTokenBuilder('INCLUDE', False, 'directive') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, integer_tb, integer_exponent_tb, hex_integer_1_tb, hex_integer_2_tb, hex_integer_3_tb, hex_integer_h_tb, binary_integer_tb, suffixed_integer_tb, real_tb, values_tb, groupers_tb, known_operator_tb, register_tb, opcode_tb, directive_tb, include_directive_tb, preprocessor_tb, identifier_tb, string_tb, hex_string_tb, char_string_tb, comment_tb, line_comment_tb, self.unknown_operator_tb, invalid_token_builder ] opcode_tokenbuilders = [ whitespace_tb, opcode_tb, directive_tb, include_directive_tb, preprocessor_tb, identifier_tb, self.unknown_operator_tb, invalid_token_builder ] args_tokenbuilders = [ whitespace_tb, integer_tb, integer_exponent_tb, hex_integer_1_tb, hex_integer_2_tb, hex_integer_3_tb, hex_integer_h_tb, binary_integer_tb, suffixed_integer_tb, real_tb, values_tb, groupers_tb, known_operator_tb, register_tb, identifier_tb, string_tb, hex_string_tb, char_string_tb, comment_tb, line_comment_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) opcode_tokenizer = Tokenizer(opcode_tokenbuilders) args_tokenizer = Tokenizer(args_tokenbuilders) # tokenize as free-format tokens_free = tokenizer.tokenize(code) tokens_free = Examiner.combine_adjacent_identical_tokens(tokens_free, 'invalid operator') tokens_free = Examiner.combine_adjacent_identical_tokens(tokens_free, 'invalid') tokens_free = AssemblyIBMExaminer.convert_keywords_to_identifiers(tokens_free) tokens_free = Examiner.convert_values_to_operators(tokens_free, known_operators) self.tokens = tokens_free self.convert_asm_identifiers_to_labels() self.convert_asm_keywords_to_identifiers() self.calc_statistics() statistics1 = self.statistics self.statistics = {} self.calc_confidences(operand_types, group_starts, group_mids, group_ends, None) self.calc_line_length_confidence(code, self.max_expected_line) confidences_free = self.confidences self.confidences = {} errors_free = self.errors self.errors = [] # tokenize as space-format opcode_extras = '.&=,()+-*/' label_leads = '.&$@' label_mids = '.&$#@' label_ends = ':,' comment_leads = '!' line_comment_leads = '*' use_line_id = True tokens_space, indents = Tokenizer.tokenize_asm_code(code, tab_size, opcode_tokenizer, opcode_extras, args_tokenizer, label_leads, label_mids, label_ends, comment_leads, line_comment_leads, use_line_id) tokens_space = Examiner.combine_adjacent_identical_tokens(tokens_space, 'invalid operator') tokens_space = Examiner.combine_adjacent_identical_tokens(tokens_space, 'invalid') tokens_space = Examiner.combine_identifier_colon(tokens_space, ['newline'], [], []) tokens_space = Tokenizer.combine_number_and_adjacent_identifier(tokens_space) tokens_space = AssemblyIBMExaminer.convert_opcodes_to_keywords(tokens_space, keywords) tokens_space = AssemblyIBMExaminer.convert_keywords_to_identifiers(tokens_space) tokens_space = Examiner.convert_values_to_operators(tokens_space, known_operators) self.tokens = tokens_space self.convert_asm_identifiers_to_labels() self.convert_asm_keywords_to_identifiers() self.calc_statistics() statistics2 = self.statistics self.statistics = {} self.calc_confidences(operand_types, group_starts, group_mids, group_ends, indents) self.calc_line_length_confidence(code, self.max_expected_line) confidences_space = self.confidences self.confidences = {} errors_space = self.errors self.errors = [] # select the better of free-format and spaced-format confidence_free = 1.0 for key in confidences_free: factor = confidences_free[key] confidence_free *= factor confidence_space = 1.0 for key in confidences_space: factor = confidences_space[key] confidence_space *= factor if confidence_space > confidence_free: self.tokens = tokens_space self.statistics = statistics2 self.confidences = confidences_space self.errors = errors_space else: self.tokens = tokens_free self.statistics = statistics1 self.confidences = confidences_free self.errors = errors_free
def __init__(self, code): super().__init__() self.newlines_important = 'always' operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() stmt_separator_tb = SingleCharacterTokenBuilder( ';', 'statement separator', False) integer_tb = IntegerTokenBuilder('_') integer_exponent_tb = IntegerExponentTokenBuilder('_') real_tb = RealTokenBuilder(False, False, '_') real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', '_') operand_types.append('number') leads = '_' extras = '_' identifier_tb = IdentifierTokenBuilder(leads, extras) operand_types.append('identifier') decorator_tb = PrefixedIdentifierTokenBuilder('@', 'decorator', False) quotes = ['"', "'", "’"] string_tb = EscapedStringTokenBuilder(quotes, 0) raw_string_tb = PrefixedRawStringTokenBuilder('r', True, quotes) byte_string_tb = PrefixedStringTokenBuilder('b', True, quotes) unicode_string_tb = PrefixedStringTokenBuilder('u', True, quotes) fast_string_tb = PrefixedStringTokenBuilder('f', True, quotes) operand_types.append('string') triple_quote_comment_tb = TripleQuoteStringTokenBuilder(quotes) raw_triple_quote_comment_tb = RawTripleQuoteCommentTokenBuilder() hash_comment_tb = LeadToEndOfLineTokenBuilder('#', True, 'comment') known_operators = [ '+', '-', '*', '/', '%', '@', '=', ':=', '==', '>', '>=', '<', '<=', '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '&', '|', '~', '<<', '>>', '**', '.', ':', '++', '--', 'and', 'or', 'in', 'is', 'not' ] known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) self.unary_operators = ['+', '-', 'not', '~', '++', '--', '.'] self.postfix_operators = ['++', '--', ':'] self.adjective_operators = ['not'] self.keyword_postfix = [':'] groupers = ['(', ')', ',', '[', ']', '{', '}'] group_starts = ['(', '[', ',', '{'] group_mids = [','] group_ends = [')', ']', '}'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) continuation_chars = ['\\'] line_continuation_tb = CaseInsensitiveListTokenBuilder( continuation_chars, 'line continuation', False) keywords = [ 'as', 'assert', 'break', 'case', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'lambda', 'match', 'nonlocal', 'pass', 'print', 'raise', 'return', 'try', 'while', 'while', 'with', 'yield' ] keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) values = ['False', 'None', 'True'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, line_continuation_tb, stmt_separator_tb, integer_tb, integer_exponent_tb, real_tb, real_exponent_tb, keyword_tb, values_tb, known_operator_tb, groupers_tb, identifier_tb, decorator_tb, string_tb, raw_string_tb, byte_string_tb, unicode_string_tb, fast_string_tb, hash_comment_tb, triple_quote_comment_tb, raw_triple_quote_comment_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid operator') self.tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid') self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_parens_continued_lines(tokens) tokens = Examiner.join_operator_continued_lines( tokens, self.postfix_operators) self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [['not', 'in']] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = [ 'number', 'string', 'identifier', 'variable', 'symbol' ] self.calc_operand_n_confidence(tokens, operand_types_2, 2) self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_line_format_confidence() self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, variant): super().__init__() operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder(None) integer_exponent_tb = IntegerExponentTokenBuilder(None) real_tb = RealTokenBuilder(True, True, None) real_exponent_tb = RealExponentTokenBuilder(True, True, 'E', None) operand_types.append('number') leads = '_' extras = '_' identifier_tb = IdentifierTokenBuilder(leads, extras) operand_types.append('identifier') class_type_tb = ClassTypeTokenBuilder() operand_types.append('class') quotes = ['"'] string_tb = EscapedStringTokenBuilder(quotes, 0) triple_quote_string_tb = TripleQuoteStringTokenBuilder(quotes) prefixed_string_tb = PrefixedStringTokenBuilder('@', False, quotes) char_tb = FsharpCharTokenBuilder(["'", "’"]) operand_types.append('string') slash_slash_comment_tb = NullTokenBuilder() parens_star_comment_tb = BlockTokenBuilder('(*', '*)', 'comment') triple_slash_comment_tb = NullTokenBuilder() if variant in ['fsharp']: slash_slash_comment_tb = SlashSlashCommentTokenBuilder() triple_slash_comment_tb = TripleSlashCommentTokenBuilder() directives = [ '#if', '#else', '#elif', '#endif', '#define', '#undef', '#line', '#region', '#endregion', '#pragma' ] preprocessor_tb = CaseSensitiveListTokenBuilder( directives, 'preprocessor', False) c_warning_tb = LeadToEndOfLineTokenBuilder('#warning', True, 'preprocessor') c_error_tb = LeadToEndOfLineTokenBuilder('#error', True, 'preprocessor') known_operators = [ 'and', 'as', 'in', 'mod', 'not', 'of', 'or', 'when', '::', '+', '-', '*', '/', '+.', '-.', '*.', '/.', '=', "'", '->', '>', '<', '>=', '<=', '==', '^', '||', '.', '#' ] known_operators_fsharp = [ 'new', '!', '!=', '%', '%%', '%?', '&', '&&', '&&&', '(|', '|)', '*?', '**', '+?', '-?', '->', '..', '.. ..', '/?', ':', ':=', ':/', '<<', '<<<', '<-', '<>', '<>?', '<=?', '<|', '<||', '<|||', '<@', '@>', '<@@', '@@>', '=?', '==', '>?', '>>', '>>>', '>=?', '?', '|||', '^^^', '?>=', '?>', '?<=', '?<', '?=', '?<>', '?+', '?-', '?*', '?/', '>=?', '>?', '<=?', '<?', '=?', '<>?', '+?', '-?', '*?', '/?', '?>=?', '?>?', '?<=?', '?<?', '?=?', '?<>?', '?+?', '?-?', '?*?', '?/?', '@', '|>', '||>', '|||>', '~~', '~~~', '~-', '~+', ':>', ':?>', "'" ] if variant in ['fsharp']: known_operators += known_operators_fsharp self.unary_operators = ['new', 'not', "'", '-'] self.postfix_operators = ["'"] known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) groupers = [ '(', ')', ',', '[', ']', '{', '}', 'begin', 'end', ';', '|' ] groupers_fsharp = ['[|', '|]', '[<', '>]', '^'] if variant in ['fsharp']: groupers += groupers_fsharp # group_starts = ['(', '[', ',', '{', '[|', '[<'] group_mids = [',', ';', '^', '|'] group_ends = [')', ']', '}', '|]', '>]'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) keywords = [ 'assert', 'class', 'def', 'do', 'done', 'downto', 'else', 'exception', 'failwith', 'for', 'fun', 'function', 'if', 'inherit', 'lazy', 'let', 'match', 'method', 'module', 'object', 'open', 'raise', 'rec', 'sig', 'then', 'to', 'try', 'type', 'val', 'virtual', 'while', 'with' ] keywords_fsharp = [ 'abstract', 'break', 'default', 'delegate', 'downcast', 'elif', 'extern', 'finally', 'fixed', 'global', 'inline', 'interface', 'internal', 'let!', 'match!', 'member', 'mutable', 'namespace', 'override', 'private', 'public', 'return', 'return!', 'upcast', 'use', 'use!', 'yield', 'yield!' ] if variant in ['fsharp']: keywords += keywords_fsharp keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) types = [ 'bool', 'byte', 'char', 'double', 'float', 'int', 'list', 'long', 'number', 'object', 'range', 'string', 'struct', 'union', 'unit', 'void' ] types_fsharp = [ 'decimal', 'sbyte', 'short', 'uint', 'ulong', 'ushort', 'void' ] if variant in ['fsharp']: types += types_fsharp types_tb = CaseSensitiveListTokenBuilder(types, 'type', True) operand_types.append('type') values = ['base', 'false', 'null', 'true', '_'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, integer_tb, integer_exponent_tb, real_tb, real_exponent_tb, keyword_tb, types_tb, values_tb, known_operator_tb, groupers_tb, identifier_tb, class_type_tb, string_tb, triple_quote_string_tb, prefixed_string_tb, char_tb, triple_slash_comment_tb, slash_slash_comment_tb, parens_star_comment_tb, preprocessor_tb, c_error_tb, c_warning_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid operator') self.tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid') self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_all_lines(tokens) self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) # self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = ['number', 'string', 'symbol'] self.calc_operand_n_confidence(tokens, operand_types_2, 2) # self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_paired_blockers_confidence(['{'], ['}']) self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, year): super().__init__() if year is not None and year not in ['90', '1990', '95', '1995', '2003', '2008']: raise CodeStatException('Unknown year for language') operand_types = [] kind_integer_tb = KindIntegerTokenBuilder() kind_real_tb = KindRealTokenBuilder() operand_types.append('number') leads = '_' extras = '_' identifier_tb = IdentifierTokenBuilder(leads, extras) operand_types.append('identifier') bang_comment_tb = LeadToEndOfLineTokenBuilder('!', False, 'comment') quotes = ["'", '"', "’"] string_tb = StuffedQuoteStringTokenBuilder(quotes, False) binary_string_tb = PrefixedStringTokenBuilder('B', False, quotes) octal_string_tb = PrefixedStringTokenBuilder('O', False, quotes) hex_string_tb = PrefixedStringTokenBuilder('Z', False, quotes) operand_types.append('string') known_operators = [ '=', '+', '-', '*', '/', '**', '==', '>', '>=', '<', '<=', '/=', '.EQ.', '.NE.', '.LT.', '.LE.', '.GT.', '.GE.', '.AND.', '.OR.', '.NOT.', '.EQV.', '.NEQV.', ':', '::', '=>', '%' ] known_operator_tb = CaseInsensitiveListTokenBuilder(known_operators, 'operator', False) self.unary_operators = [ '+', '-' ] user_operator_tb = UserDefinedOperatorTokenBuilder() continuation_tb = SingleCharacterTokenBuilder('&', 'line continuation', False) stmt_separator_tb = SingleCharacterTokenBuilder(';', 'statement separator', False) groupers = ['(', ')', ',', '[', ']'] # group_starts = ['(', '[', ',', '{'] group_mids = [','] # group_ends = [')', ']', '}'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) keywords = [ 'IF', 'THEN', 'ELSE', 'ENDIF', 'END IF', 'GO', 'TO', 'GOTO', 'GO TO', 'READ', 'WRITE', 'BACKSPACE', 'REWIND', 'ENDFILE', 'FORMAT', 'PRINT', 'DO', 'CONTINUE', 'PROGRAM', 'SUBROUTINE', 'FUNCTION', 'BLOCK DATA', 'IMPLICIT', 'SAVE', 'COMMON', 'DIMENSION', 'EQUIVALENCE', 'DATA', 'EXTERNAL', 'CALL', 'RETURN', 'STOP', 'END', 'INQUIRE', 'INTRINSIC', 'PARAMETER', 'allocate', 'case', 'contains', 'cycle', 'deallocate', 'elsewhere', 'exit', 'include', 'interface', 'intent', 'kind', 'module', 'namelist', 'nullify', 'only', 'operator', 'optional', 'pointer', 'private', 'procedure', 'public', 'recursive', 'result', 'select', 'sequence', 'target', 'type', 'use', 'while', 'where', 'enddo', 'end do', 'none' ] keywords_95 = [ 'FORALL', 'PURE', 'ELEMENTAL' ] keywords_2003 = [ 'abstract', 'allocatable', 'associate', 'bind', 'class', 'enum', 'end enum', 'import', 'protected', 'select type', 'type guard', 'value', 'wait' ] keywords_2008 = [ 'block', 'contiguous' ] if year in ['95', '2003', '2008']: keywords += keywords_95 if year in ['2003', '2008']: keywords += keywords_2003 if year in ['2008']: keywords += keywords_2008 keyword_tb = CaseInsensitiveListTokenBuilder(keywords, 'keyword', False) types = [ 'INTEGER', 'REAL', 'COMPLEX', 'DOUBLE PRECISION', 'DOUBLEPRECISION', 'DOUBLE', 'PRECISION', 'LOGICAL', 'CHARACTER' ] types_tb = CaseInsensitiveListTokenBuilder(types, 'type', True) operand_types.append('type') tokenbuilders = [ self.newline_tb, self.whitespace_tb, continuation_tb, stmt_separator_tb, self.integer_tb, self.integer_exponent_tb, kind_integer_tb, self.real_tb, self.real_exponent_tb, self.double_exponent_tb, kind_real_tb, keyword_tb, types_tb, known_operator_tb, user_operator_tb, groupers_tb, identifier_tb, string_tb, binary_string_tb, octal_string_tb, hex_string_tb, bang_comment_tb, self.jcl_tb, self.unknown_operator_tb, self.invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid operator') tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid') tokens = Examiner.combine_identifier_colon(tokens, ['newline'], [], ['whitespace', 'comment', 'line description']) self.tokens = tokens self.convert_identifiers_to_labels() self.convert_numbers_to_lineNumbers() self.convert_stars_to_io_channels() self.calc_statistics() tokens = self.source_tokens() self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) # self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) # self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = ['number', 'string', 'identifier', 'variable', 'symbol'] self.calc_operand_n_confidence(tokens, operand_types_2, 2) self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, block_comment_limit): super().__init__() operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder("'") integer_exponent_tb = IntegerExponentTokenBuilder("'") hex_integer_tb = PrefixedIntegerTokenBuilder('0x', False, '0123456789abcdefABCDEF_') binary_integer_tb = PrefixedIntegerTokenBuilder('0b', False, '01_') suffixed_integer_tb = SuffixedIntegerTokenBuilder(['U', 'L', 'LU', 'UL'], False, None) real_tb = RealTokenBuilder(False, False, "'") suffixed_real_tb = SuffixedRealTokenBuilder(False, False, ['f', 'l', 'i'], False, None) real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', "'") hex_real_tb = HexRealExponentTokenBuilder() operand_types.append('number') leads = '_' extras = '_' identifier_tb = IdentifierTokenBuilder(leads, extras) operand_types.append('identifier') attribute_tb = PrefixedIdentifierTokenBuilder('@', 'attribute', False) operand_types.append('attribute') # string suffix: c,w,d quotes = ['"', "'", "’"] string_tb = EscapedStringTokenBuilder(quotes, 0) r_string_tb = PrefixedStringTokenBuilder('r', True, quotes) backtick_string_tb = EscapedStringTokenBuilder(['`'], 0) x_string_tb = PrefixedStringTokenBuilder('x', True, quotes) q_string_tb = PrefixedStringTokenBuilder('q', True, quotes) # q{} string cwd_string_tb = SuffixedStringTokenBuilder(quotes, 'cwd', False) operand_types.append('string') class_type_tb = ClassTypeTokenBuilder() operand_types.append('class') slash_slash_comment_tb = SlashSlashCommentTokenBuilder() slash_star_comment_tb = SlashStarCommentTokenBuilder() slash_plus_comment_tb = NestedCommentTokenBuilder('/+', '+/', block_comment_limit) line_continuation_tb = SingleCharacterTokenBuilder('\\', 'line continuation', False) terminators_tb = SingleCharacterTokenBuilder(';', 'statement terminator', False) known_operators = [ '/', '/=', '.', '..', '...', '&', '&=', '&&', '|', '|=', '||', '-', '-=', '--', '+', '+=', '++', '<', '<=', '<<', '<<=', '>', '>=', '>>=', '>>>=', '>>', '>>>', '!', '!=', '?', ',', ':', '$', '=', '==', '*', '*=', '%', '%=', '^', '^=', '^^', '^^=', '~', '~=', '@', '=>', '#', 'new', 'delete', 'typeof', 'is' ] self.unary_operators = [ '+', '-', '*', '!', '&', '~', '++', '--', ':', 'new', 'delete', 'typeof', 'is' ] self.postfix_operators = [ '++', '--', '&', ':' ] groupers = ['(', ')', ',', '[', ']', '{', '}'] group_starts = ['(', '[', ',', '{'] group_mids = [','] group_ends = [')', ']', '}'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) known_operator_tb = CaseSensitiveListTokenBuilder(known_operators, 'operator', False) keywords = [ 'abstract', 'alias', 'align', 'asm', 'assert', 'auto', 'body', 'break', 'case', 'cast', 'catch', 'class', 'const', 'continue', 'debug', 'default', 'delegate', 'deprecated', 'do', 'else', 'enum', 'export', 'extern', 'final', 'finally', 'for', 'foreach', 'foreach_reverse', 'function', 'goto', 'if', 'immutable', 'import', 'in', 'inout', 'interface', 'invariant', 'lazy', 'macro', 'mixin', 'module', 'nothrow', 'out', 'override', 'package', 'pragma', 'private', 'protected', 'public', 'pure', 'ref', 'return', 'scope', 'shared', 'static', 'struct', 'switch', 'synchronized', 'template', 'throw', 'try', 'typeid', 'union', 'unittest', 'version', 'while', 'with', '__gshared', '__traits', '__vector', '__parameters' ] keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) types = [ 'bool', 'byte', 'cdouble', 'cent', 'cfloat', 'char', 'creal', 'dchar', 'double', 'float', 'idouble', 'ifloat', 'int', 'ireal', 'long', 'real', 'short', 'ubyte', 'ucent', 'uint', 'ulong', 'ushort', 'void', 'wchar' ] types_tb = CaseSensitiveListTokenBuilder(types, 'type', True) operand_types.append('type') values = [ 'false', 'null', 'super', 'this', 'true', '__FILE__', '__FILE_FULL_PATH__', '__MODULE__', '__LINE__', '__FUNCTION__', '__PRETTY_FUNCTION__', '__DATE__', '__EOF__', '__TIME__','__TIMESTAMP__', '__VENDOR__', '__VERSION__' ] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, line_continuation_tb, terminators_tb, integer_tb, integer_exponent_tb, hex_integer_tb, binary_integer_tb, suffixed_integer_tb, real_tb, real_exponent_tb, suffixed_real_tb, hex_real_tb, keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb, identifier_tb, attribute_tb, class_type_tb, string_tb, r_string_tb, x_string_tb, backtick_string_tb, q_string_tb, cwd_string_tb, slash_slash_comment_tb, slash_star_comment_tb, slash_plus_comment_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid operator') tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid') tokens = Examiner.combine_identifier_colon(tokens, ['statement terminator', 'newline'], ['{'], ['whitespace', 'comment']) self.tokens = tokens self.convert_identifiers_to_labels() number_suffixes = ['f', 'F', 'i', 'I', 'u', 'U', 'l', 'L', 'ul', 'uL', 'Ul', 'UL', 'lu', 'lU', 'Lu', 'LU'] tokens = self.combine_tokens_and_adjacent_types(tokens, 'number', 'identifier', number_suffixes) string_suffixes = ['c', 'w', 'd'] self.tokens = self.combine_tokens_and_adjacent_types(tokens, 'string', 'identifier', string_suffixes) self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_all_lines(tokens) self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = ['number', 'symbol'] self.calc_operand_n_confidence(tokens, operand_types_2, 2) self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_paired_blockers_confidence(['{'], ['}']) self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, block_comment_limit): super().__init__() self.newlines_important = 'parens' operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder(None) integer_exponent_tb = IntegerExponentTokenBuilder(None) hex_integer_tb = PrefixedIntegerTokenBuilder('0x', False, '0123456789abcdefABCDEF') real_tb = RealTokenBuilder(False, False, None) real_exponent_tb = RealExponentTokenBuilder(False, False, 'E', None) imaginary_tb = SuffixedRealTokenBuilder(False, False, ['im', 'cx'], True, None) operand_types.append('number') leads = '_' extras = '_' suffixes = '!' identifier_tb = SuffixedIdentifierTokenBuilder(leads, extras, suffixes) operand_types.append('identifier') symbol_tb = PrefixedIdentifierTokenBuilder(':', 'symbol', True) operand_types.append('symbol') attribute_tb = PrefixedIdentifierTokenBuilder('@', 'attribute', False) operand_types.append('attribute') dollar_sign_tb = SingleCharacterTokenBuilder('$', 'identifier', True) quotes = ['"', "'", "’"] string_tb = EscapedStringTokenBuilder(quotes, 0) raw_string_tb = PrefixedRawStringTokenBuilder('raw', True, quotes) b_string_tb = PrefixedStringTokenBuilder('b', True, quotes) triple_quote_string_tb = TripleQuoteStringTokenBuilder(quotes) operand_types.append('string') comment_tb = LeadToEndOfLineTokenBuilder('#', True, 'comment') nested_comment_tb = NestedCommentTokenBuilder('#=', '=#', block_comment_limit) line_continuation_tb = SingleCharacterTokenBuilder( '\\', 'line continuation', False) terminators_tb = SingleCharacterTokenBuilder(';', 'statement terminator', False) known_operators = [ 'where', 'in', 'isa', '′', "'", '+', '-', '*', '/', '\\', '^', '%', '//', '<<', '>>', '<<<', '>>>', ':', '=', '==', '!=', '===', '!==', '+=', '-=', '*=', '/=', '^=', '%=', '<', '>', '<=', '>=', '~', '&', '|', '!', '&&', '||', '?', '.', '<:', '>:', '::', '->', '...', '..', '∀', '≤', '≥', '⊻', '⊽', '⊼' ] # 0x391 through 0x3a9 (capital) # 0x3b1 through 0x3c9 (small) greek_letters = [ 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η', 'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'ο', 'π', 'ρ', 'ς', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω' ] greek_letter_tb = CaseSensitiveListTokenBuilder( greek_letters, 'identifier', True) self.unary_operators = [ 'isa', '+', '-', '~', '!', '.', ':', '::', "'", '<:', '>:', 'in', '..' ] self.postfix_operators = ['...', '′'] groupers = ['(', ')', ',', '[', ']', '{', '}'] # group_starts = ['(', '[', ',', '{'] group_mids = [','] group_ends = [')', ']', '}'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) keywords = [ 'baremodule', 'begin', 'break', 'catch', 'const', 'continue', 'do', 'else', 'elseif', 'end', 'export', 'finally', 'for', 'function', 'global', 'if', 'import', 'let', 'local', 'macro', 'module', 'quote', 'return', 'struct', 'try', 'using', 'while', 'abstract', 'mutable', 'primitive', 'type' ] keyword_tb = CaseSensitiveListTokenBuilder(keywords, 'keyword', False) types = [ 'Int8', 'UInt8', 'Int16', 'UInt16', 'Int32', 'UInt32', 'Int64', 'UInt64', 'Int128', 'UInt128', 'Float16', 'Float32', 'Float64', 'Bool', 'Char' ] types_tb = CaseSensitiveListTokenBuilder(types, 'type', True) operand_types.append('type') values = ['false', 'true'] values_tb = CaseSensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, line_continuation_tb, terminators_tb, integer_tb, integer_exponent_tb, hex_integer_tb, real_tb, real_exponent_tb, imaginary_tb, keyword_tb, types_tb, values_tb, groupers_tb, known_operator_tb, identifier_tb, symbol_tb, attribute_tb, dollar_sign_tb, greek_letter_tb, string_tb, raw_string_tb, b_string_tb, triple_quote_string_tb, comment_tb, nested_comment_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid operator') tokens = Examiner.combine_adjacent_identical_tokens(tokens, 'invalid') tokens = JuliaExaminer.split_symbols_to_operators_identifiers( tokens, group_ends) self.tokens = tokens self.convert_keywords_to_identifiers() self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_parens_continued_lines(tokens) tokens = Examiner.join_operator_continued_lines( tokens, self.postfix_operators) self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) # self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) operand_types_2 = ['number', 'identifier', 'symbol'] self.calc_operand_confidence(tokens, operand_types_2) self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_paired_blockers_confidence(['{'], ['}']) self.calc_line_length_confidence(code, self.max_expected_line)
def __init__(self, code, year, extension): super().__init__() if year is not None and year not in ['2002', '2014']: raise CodeStatException('Unknown year for language') operand_types = [] whitespace_tb = WhitespaceTokenBuilder() newline_tb = NewlineTokenBuilder() integer_tb = IntegerTokenBuilder(None) integer_exponent_tb = IntegerExponentTokenBuilder(None) real_tb = RealTokenBuilder(False, True, None) real_exponent_tb = RealExponentTokenBuilder(False, True, 'E', None) identifier_tb = CobolIdentifierTokenBuilder() quotes = ['"', "'", "’"] string_tb = StuffedQuoteStringTokenBuilder(quotes, False) n_string_tb = PrefixedStringTokenBuilder('N', False, quotes) nx_string_tb = PrefixedStringTokenBuilder('NX', False, quotes) picture_tb = PictureTokenBuilder() cr_picture_tb = CRPictureTokenBuilder() inline_comment_tb = LeadToEndOfLineTokenBuilder('*>', True, 'comment') star_comment_tb = AsteriskCommentTokenBuilder() terminators_tb = SingleCharacterTokenBuilder('.', 'statement terminator', False) known_operators = [ 'ADD', 'SUBTRACT', 'MULTIPLY', 'DIVIDE', '+', '-', '*', '/', '**', '=', '<>', '>', '>=', '<', '<=', 'AND', 'OR', 'NOT', 'B-AND', 'B-NOT', 'B-OR', 'B-XOR', ':' ] known_operator_tb = CaseSensitiveListTokenBuilder( known_operators, 'operator', False) self.unary_operators = ['+', '-', 'NOT'] groupers = ['(', ')', ','] group_starts = ['('] group_mids = [','] # group_ends = [')'] groupers_tb = CaseInsensitiveListTokenBuilder(groupers, 'group', False) keywords = [ 'ACCEPT', 'ACCESS', 'ADD', 'ADDRESS', 'ADVANCING', 'AFTER', 'ALL', 'ALPHABET', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER', 'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER', 'ALTERNATE', 'AND', 'ANY', 'APPLY', 'ARE', 'AREA', 'AREAS', 'ASCENDING', 'ASSIGN', 'AT', 'AUTHOR', 'BEFORE', 'BEGINNING', 'BELL', 'BINARY', 'BLOCK', 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CALL', 'CANCEL', 'CBL', 'CD', 'CF', 'CH', 'CHARACTER', 'CHARACTERS', 'CLOCK-UNITS', 'CLOSE', 'COBOL', 'CODE', 'CODE-SET', 'COL', 'COLLATING', 'COLS', 'COLUMN', 'COMMA', 'COMMON', 'COMMUNICATION', 'COMP', 'COMPUTATIONAL', 'COMPUTE', 'CONFIGURATION', 'CONTAINS', 'CONTENT', 'CONTINUE', 'CONTROL', 'CONTROLS', 'CONVERTING', 'COPY', 'CORR', 'CORRESPONDING', 'COUNT', 'CURRENCY', 'DATA', 'DATE', 'DATE-COMPILED', 'DATE-WRITTEN', 'DAY', 'DAY-OF-WEEK', 'DE', 'DEBUG-CONTENTS', 'DEBUG-ITEM', 'DEBUG-LINE', 'DEBUG-NAME', 'DEBUG-SUB-1', 'DEBUG-SUB-2', 'DEBUG-SUB-3', 'DECIMAL-POINT', 'DECLARATIVES', 'DELETE', 'DELIMITED', 'DELIMITER', 'DEPENDING', 'DESCENDING', 'DESTINATION', 'DISABLE', 'DIVIDE', 'DIVISION', 'DOWN', 'DUPLICATES', 'DYNAMIC', 'EGI', 'ELSE', 'EMI', 'ENABLE', 'END', 'END-ACCEPT', 'END-ADD', 'END-CALL', 'END-COMPUTE', 'END-DELETE', 'END-DISPLAY', 'END-DIVIDE', 'END-EVALUATE', 'END-EXEC', 'END-IF', 'END-MULTIPLY', 'END-OF-PAGE', 'END-PERFORM', 'END-READ', 'END-RECEIVE', 'END-RETURN', 'END-REWRITE', 'END-SEARCH', 'END-START', 'END-STRING', 'END-SUBTRACT', 'END-UNSTRING', 'END-WRITE', 'ENTER', 'ENVIRONMENT', 'EOL', 'EOP', 'EQUAL', 'ERROR', 'ESI', 'EVALUATE', 'EVERY', 'EXCEPTION', 'EXEC', 'EXIT', 'EXTEND', 'EXTERNAL', 'FD', 'FILE', 'FILE-CONTROL', 'FILLER', 'FINAL', 'FIRST', 'FOOTING', 'FOR', 'FROM', 'FULL', 'GENERATE', 'GIVING', 'GLOBAL', 'GO', 'GOBACK', 'GREATER', 'GROUP', 'HEADING', 'HIGH-VALUE', 'HIGH-VALUES', 'I-O', 'I-O-CONTROL', 'IDENTIFICATION', 'IF', 'IN', 'INDEX', 'INDEXED', 'INDICATE', 'INITIAL', 'INITIALIZE', 'INITIATE', 'INPUT', 'INPUT-OUTPUT', 'INSPECT', 'INSTALLATION', 'INTO', 'INVALID', 'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL', 'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LESS', 'LIMIT', 'LIMITS', 'LINAGE', 'LINAGE-COUNTER', 'LINE', 'LINE-COUNTER', 'LINES', 'LINKAGE', 'LOCK', 'LOW-VALUE', 'LOW-VALUES', 'MEMORY', 'MERGE', 'MESSAGE', 'MODE', 'MODULES', 'MOVE', 'MULTIPLE', 'MULTIPLY', 'NATIVE', 'NEGATIVE', 'NEXT', 'NOT', 'NUMBER', 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED', 'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OMITTED', 'OPEN', 'OPTIONAL', 'OR', 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PAGE-COUNTER', 'PARAGRAPH', 'PERFORM', 'PF', 'PH', 'PIC', 'PICTURE', 'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRINTING', 'PROCEDURE', 'PROCEDURES', 'PROCEED', 'PROGRAM', 'PROGRAM-ID', 'PURGE', 'QUEUE', 'QUOTE', 'QUOTES', 'RANDOM', 'RD', 'READ', 'RECEIVE', 'RECORD', 'RECORDS', 'REDEFINES', 'REEL', 'REFERENCE', 'RELATIVE', 'RELEASE', 'REMAINDER', 'REMOVAL', 'RENAMES', 'REPLACE', 'REPLACING', 'REPORT', 'REPORTING', 'REPORTS', 'RERUN', 'RESERVE', 'RESET', 'RESUME', 'RETRY', 'RETURN', 'REVERSED', 'REWIND', 'REWRITE', 'RF', 'RH', 'RIGHT', 'ROUNDED', 'RUN', 'SAME', 'SD', 'SEARCH', 'SECTION', 'SECURE', 'SECURITY', 'SEGMENT', 'SEGMENT-LIMIT', 'SELECT', 'SEND', 'SENTENCE', 'SEPARATE', 'SEQUENCE', 'SEQUENTIAL', 'SET', 'SIGN', 'SIZE', 'SORT', 'SORT-MERGE', 'SOURCE', 'SOURCE-COMPUTER', 'SPECIAL-NAMES', 'STANDARD', 'STANDARD-1', 'STANDARD-2', 'START', 'STATUS', 'STOP', 'STRING', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3', 'SUBTRACT', 'SUM', 'SUPPRESS', 'SYMBOLIC', 'SYNC', 'SYNCHRONIZED', 'TABLE', 'TALLY', 'TALLYING', 'TAPE', 'TERMINAL', 'TERMINATE', 'TEST', 'TEXT', 'THAN', 'THEN', 'THROUGH', 'THRU', 'TIME', 'TIMES', 'TITLE', 'TO', 'TOP', 'TRAILING', 'TYPE', 'UNIT', 'UNSTRING', 'UNTIL', 'UP', 'UPON', 'USAGE', 'USE', 'USING', 'VALUE', 'VALUES', 'VARYING', 'WHEN', 'WITH', 'WORDS', 'WORKING-STORAGE', 'WRITE' ] keywords_2002 = [ 'ACTIVE-CLASS', 'ALIGNED', 'ALLOCATE', 'ANYCASE', 'ARITHMETIC', 'AUTO', 'AUTOMATIC', 'BACKGROUND-COLOR', 'BASED', 'BASIS', 'BINARY-CHAR', 'BINARY-DOUBLE', 'BINARY-LONG', 'BINARY-SHORT', 'BIT', 'BLINK', 'BOOLEAN', 'CENTER', 'CLASS', 'CLASS-ID', 'CLASSIFICATION', 'COLUMNS', 'COM-REG', 'CONDITION', 'CONSTANT', 'CRT', 'CURSOR', 'CYCLE', 'DATA-POINTER', 'DBCS', 'DEBUGGING', 'DETAIL', 'DISPLAY', 'DISPLAY-1', 'DISPLAY-OF', 'EC', 'EGCS', 'EJECT', 'END-INVOKE', 'ENDING', 'ENTRY-CONVENTION', 'ENTRY-FIELD', 'EO', 'EOS', 'ERASE', 'EXCEPTION-OBJECT', 'EXCLUSIVE', 'EXPANDS', 'EXTERN', 'FACTORY', 'FLOAT-EXTENDED', 'FLOAT-LONG', 'FLOAT-SHORT', 'FOREGROUND-COLOR', 'FOREVER', 'FORMAT', 'FREE', 'FUNCTION', 'FUNCTION-ID', 'GET', 'GROUP-USAGE', 'HIGHLIGHT', 'IGNORING', 'IMPLEMENTS', 'INHERITS', 'INITIALIZED', 'INSERT', 'INTERFACE', 'INTERFACE-ID', 'INTRINSIC', 'INVOKE', 'KANJI', 'LC_ALL', 'LC_COLLATE', 'LC_CTYPE', 'LC_MESSAGES', 'LC_MONEY', 'LC_NUMERIC', 'LC_TIME', 'LOCAL-STORAGE', 'LOCALE', 'LOWLIGHT', 'MANUAL', 'METACLASS', 'METHOD', 'METHOD-ID', 'MINUS', 'MORE-LABELS', 'NATIONAL', 'NATIONAL-EDITED', 'NATIONAL-OF', 'NATIVE_BINARY', 'NESTED', 'NEW', 'NONE', 'NORMAL', 'OBJECT', 'OBJECT-REFERENCE', 'ONLY', 'OPTIONS', 'OVERRIDE', 'PHYSICAL', 'PRESENT', 'PREVIOUS', 'PROCEDURE-POINTER', 'PROCESSING', 'PROGRAM-POINTER', 'PROPERTY', 'PROTOTYPE', 'RAISE', 'RAISING', 'READY', 'RECURSIVE', 'REFERENCES', 'RELATION', 'RELOAD', 'REPOSITORY', 'REQUIRED', 'RETURN-CODE', 'RETURNING', 'ROUNDING', 'SCREEN', 'SECONDS', 'SERVICE', 'SHARING', 'SHIFT-IN', 'SHIFT-OUT', 'SIGNED', 'SKIP1', 'SKIP2', 'SKIP3', 'SORT-CONTROL', 'SORT-CORE-SIZE', 'SORT-FILE-SIZE', 'SORT-MESSAGE', 'SORT-MODE-SIZE', 'SORT-RETURN', 'SOURCES', 'STATEMENT', 'STEP', 'STRONG', 'SYMBOL', 'SYSTEM-DEFAULT', 'TRACE', 'TYPEDEF', 'UCS-4', 'UNDERLINE', 'UNIVERSAL', 'UNLOCK', 'UNSIGNED', 'USER-DEFAULT', 'UTF-16', 'UTF-8', 'VAL-STATUS', 'VALID', 'VALIDATE', 'VALIDATE-STATUS', 'WHEN-COMPILED', 'WRITE-ONLY', 'YYYYDDD', 'YYYYMMDD', ] keywords_2014 = [ 'AWAY-FROM-ZERO', 'NEAREST-AWAY-FROM-ZERO', 'NEAREST-EVEN', 'NEAREST-TOWARD-ZERO', 'TOWARD-GREATER', 'TOWARD-LESSER', 'CAPACITY', 'FLOAT-BINARY-128', 'FLOAT-BINARY-32', 'FLOAT-BINARY-64', 'FLOAT-DECIMAL-16', 'FLOAT-DECIMAL-34', 'FLOAT-INFINITY', 'FLOAT-NOT-A-NUMBER', 'FUNCTION-POINTER', 'INTERMEDIATE', 'PHYSICAL', 'PREFIXED', 'PROHIBITED', 'SHORT', 'STANDARD-BINARY', 'STANDARD-DECIMAL', 'TRUNCATION' ] keywords_ibm = ['ABSENT', 'ID', 'PASSWORD', 'UNBOUNDED'] keywords_gnu = [ 'ARGUMENT-NUMBER', 'ARGUMENT-VALUE', 'ASCII', 'BINARY-C-LONG', 'BINARY-SEQUENTIAL', 'CARD-PUNCH', 'CARD-READER', 'CASSETTE', 'CHAIN', 'CHAINING', 'COLOR', 'COMMAND-LINE', 'COMMIT', 'COMP-1', 'COMP-2', 'COMP-3', 'COMP-4', 'COMP-5', 'COMP-6', 'COMP-X', 'COMPUTATIONAL-1', 'COMPUTATIONAL-2', 'COMPUTATIONAL-3', 'COMPUTATIONAL-4', 'COMPUTATIONAL-5', 'COMPUTATIONAL-6', 'COMPUTATIONAL-X', 'CONVERSION', 'CRT-UNDER', 'DISC', 'DISK', 'EBCDIC', 'ECHO', 'END-CHAIN', 'ENTRY', 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'ESCAPE', 'F', 'FILE-ID', 'FIXED', 'FLOAT-DECIMAL-7', 'ID', 'IGNORE', 'KEPT', 'KEYBOARD', 'LEFT-JUSTIFY', 'LEFTLINE', 'LINE-SEQUENTIAL', 'LOWER', 'MAGNETIC-TAPE', 'NAME', 'NO-ECHO', 'NOTHING', 'OVERLINE', 'PRINT', 'PRINTER', 'PRINTER-1', 'PROCEDURE-POINTER', 'PROCEDURES', 'PROMPT', 'PROTECTED', 'RECORDING', 'REVERSE', 'RIGHT-JUSTIFY', 'ROLLBACK', 'S', 'SCROLL', 'SIGNED-INT', 'SIGNED-LONG', 'SIGNED-SHORT', 'SPACE-FILL', 'STATIC', 'STDCALL', 'SYSTEM-OFFSET', 'TAB', 'TIME-OUT', 'TRAILING-SIGN', 'U', 'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT', 'UPDATE', 'UPPER', 'USER', 'V', 'VARIABLE', 'WAIT', 'WRAP', 'ZERO-FILL' ] keywords_acu = [ '3-D', 'ACTION', 'ACTIVE-X', 'ADJUSTABLE-COLUMNS', 'ALIGNMENT', 'AUTO-DECIMAL', 'AUTO-SPIN', 'BACKGROUND-HIGH', 'BACKGROUND-LOW', 'BACKGROUND-STANDARD', 'BAR', 'BITMAP', 'BITMAP-END', 'BITMAP-HANDLE', 'BITMAP-NUMBER', 'BITMAP-START', 'BITMAP-TRAILING', 'BITMAP-TRANSPARENT-COLOR', 'BITMAP-WIDTH', 'BOX', 'BOXED', 'BUSY', 'BUTTONS', 'CALENDAR-FONT', 'CANCEL-BUTTON', 'CELL', 'CELL-COLOR', 'CELL-DATA', 'CELL-FONT', 'CELL-PROTECTION', 'CENTERED-HEADING', 'CENTURY-DATE', 'CHECK-BOX', 'CLEAR-SELECTION', 'CLINE', 'CLINES', 'COLORS', 'COLUMN-COLOR', 'COLUMN-DIVIDERS', 'COLUMN-FONT', 'COLUMN-HEADINGS', 'COLUMN-PROTECTION', 'COMBO-BOX', 'COPY-SELECTION', 'CSIZE', 'CURSOR-COL', 'CURSOR-COLOR', 'CURSOR-FRAME-WIDTH', 'CURSOR-ROW', 'CURSOR-X', 'CURSOR-Y', 'CUSTOM-PRINT-TEMPLATE', 'DASHED', 'DATA-COLUMNS', 'DATA-TYPES', 'DATE-ENTRY', 'DEFAULT-BUTTON', 'DEFAULT-FONT', 'DESTROY', 'DISPLAY-COLUMNS', 'DISPLAY-FORMAT', 'DOTDASH', 'DOTTED', 'DOUBLE', 'DRAG-COLOR', 'DROP-DOWN', 'DROP-LIST', 'END-COLOR', 'END-MODIFY', 'ENGRAVED', 'ENSURE-VISIBLE', 'ENTRY-FIELD', 'ENTRY-REASON', 'ESCAPE-BUTTON', 'EVENT', 'EVENT-LIST', 'EXCEPTION-VALUE', 'EXPAND', 'EXTERNAL-FORM', 'FILE-NAME', 'FILE-POS', 'FILL-COLOR', 'FILL-COLOR-2', 'FILL-PERCENT', 'FINISH-REASON', 'FIXED-FONT', 'FIXED-WIDTH', 'FLAT', 'FLAT-BUTTONS', 'FLOAT', 'FLOATING', 'FONT', 'FRAME', 'FRAMED', 'FULL-HEIGHT', 'GRID', 'GO-BACK', 'GO-FORWARD', 'GO-HOME', 'GO-SEARCH', 'GRAPHICAL', 'GRID', 'GROUP-VALUE', 'HANDLE', 'HAS-CHILDREN', 'HEADING-COLOR', 'HEADING-DIVIDER-COLOR', 'HEADING-FONT', 'HEAVY', 'HEIGHT-IN-CELLS', 'HIDDEN-DATA', 'HIGH-COLOR', 'HOT-TRACK', 'HSCROLL', 'HSCROLL-POS', 'ICON', 'IDENTIFIED', 'INDEPENDENT', 'INQUIRE', 'INSERTION-INDEX', 'INSERTION-ROWS', 'ITEM', 'ITEM-TEXT', 'ITEM-TO-ADD', 'ITEM-TO-DELETE', 'ITEM-TO-EMPTY', 'ITEM-VALUE', 'LABEL', 'LABEL-OFFSET', 'LARGE-FONT', 'LARGE-OFFSET', 'LAST-ROW', 'LAYOUT-DATA', 'LAYOUT-MANAGER', 'LEADING-SHIFT', 'LEFT-TEXT', 'LINES-AT-ROOT', 'LIST-BOX', 'LM-RESIZE', 'LONG-DATE', 'LOW-COLOR', 'LOWERED', 'MASS-UPDATE', 'MAX-LINES', 'MAX-PROGRESS', 'MAX-TEXT', 'MAX-VAL', 'MEDIUM-FONT', 'MENU', 'MIN-VAL', 'MODIFY', 'MULTILINE', 'NAVIGATE-URL', 'NEXT-ITEM', 'NO-AUTOSEL', 'NO-AUTO-DEFAULT', 'NO-BOX', 'NO-DIVIDERS', 'NO-F4', 'NO-FOCUS', 'NO-GROUP-TAB', 'NO-KEY-LETTER', 'NO-SEARCH', 'NO-UPDOWN', 'NOTAB', 'NOTIFY', 'NOTIFY-CHANGE', 'NOTIFY-DBLCLICK', 'NOTIFY-SELCHANGE', 'NUM-COL-HEADINGS', 'NUM-ROWS', 'OK-BUTTON', 'OVERLAP-LEFT', 'OVERLAP-TOP', 'PAGE-SETUP', 'PAGED', 'PARENT', 'PERMANENT', 'PIXEL', 'PLACEMENT', 'POP-UP', 'POSITION-SHIFT', 'PRINT-NO-PROMPT', 'PRINT-PREVIEW', 'PRIORITY', 'PROGRESS', 'PROPERTIES', 'PROPERTY', 'PUSH-BUTTON', 'QUERY-INDEX', 'RADIO-BUTTON', 'RAISED', 'READ-ONLY', 'RECORD-DATA', 'RECORD-TO-ADD', 'RECORD-TO-DELETE', 'REFRESH', 'REGION-COLOR', 'RESET-GRID', 'RESET-LIST', 'RESET-TABS', 'RIGHT-ALIGN', 'RIMMED', 'ROW-COLOR', 'ROW-COLOR-PATTERN', 'ROW-DIVIDERS', 'ROW-FONT', 'ROW-HEADINGS', 'ROW-PROTECTION', 'SAVE-AS', 'SAVE-AS-NO-PROMPT', 'SCROLL-BAR', 'SEARCH-OPTIONS', 'SEARCH-TEXT', 'SELECT-ALL', 'SELECTION-INDEX', 'SELECTION-TEXT', 'SELF-ACT', 'SEPARATION', 'SHADING', 'SHADOW', 'SHORT-DATE', 'SHOW-LINES', 'SHOW-NONE', 'SHOW-SEL-ALWAYS', 'SMALL-FONT', 'SORT-ORDER', 'SPINNER', 'SQUARE', 'START-X', 'START-Y', 'STATIC-LIST', 'STATUS-BAR', 'STATUS-TEXT', 'STYLE', 'SUBWINDOW', 'TAB-TO-ADD', 'TAB-TO-DELETE', 'TEMPORARY', 'TERMINATION-VALUE', 'THREAD', 'THREADS', 'THUMB-POSITION', 'TILED-HEADINGS', 'TITLE', 'TITLE-POSITION', 'TRADITIONAL-FONT', 'TRAILING-SHIFT', 'TRANSPARENT', 'TREE-VIEW', 'UNFRAMED', 'UNSORTED', 'USE-ALT', 'USE-RETURN', 'USE TAB', 'VALUE-FORMAT', 'VARIANT', 'VERTICAL', 'VERY-HEAVY', 'VIRTUAL-WIDTH', 'VPADDING', 'VSCROLL', 'VSCROLL-BAR', 'VSCROLL-POS', 'VTOP', 'WEB-BROWSER', 'WIDTH', 'WIDTH-IN-CELLS', 'WINDOW', 'X', 'Y' ] if year in ['2002', '2014']: keywords += keywords_2002 if year == '2014': keywords += keywords_2014 if extension.lower() == 'acu': keywords += keywords_acu if extension.lower() == 'ibm': keywords += keywords_ibm if extension.lower() == 'gnu': keywords += keywords_gnu keyword_tb = CaseInsensitiveListTokenBuilder(keywords, 'keyword', False) values = [ 'BLANK', 'SPACE', 'SPACES', 'ZERO', 'ZEROES', 'ZEROS', 'FALSE', 'NO', 'OFF', 'ON', 'TRUE' ] values_2002 = ['NULL', 'NULLS', 'SELF', 'SUPER'] if year in ['2002', '2014']: values += values_2002 value_tb = CaseInsensitiveListTokenBuilder(values, 'value', True) operand_types.append('value') cobol_preprocessor_tb = CobolPreprocessorTokenBuilder() exec_tb = BlockTokenBuilder('EXEC', 'END-EXEC', 'exec block') invalid_token_builder = InvalidTokenBuilder() tokenbuilders = [ newline_tb, whitespace_tb, terminators_tb, integer_tb, integer_exponent_tb, real_tb, real_exponent_tb, picture_tb, cr_picture_tb, keyword_tb, star_comment_tb, # before operator, to catch single star as comment known_operator_tb, groupers_tb, value_tb, identifier_tb, string_tb, n_string_tb, nx_string_tb, inline_comment_tb, cobol_preprocessor_tb, exec_tb, self.unknown_operator_tb, invalid_token_builder ] tokenizer = Tokenizer(tokenbuilders) tokens = tokenizer.tokenize(code) tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid operator') self.tokens = Examiner.combine_adjacent_identical_tokens( tokens, 'invalid') self.convert_numbers_to_pictures() self.convert_numbers_to_levels() self.calc_statistics() tokens = self.source_tokens() tokens = Examiner.join_all_lines(tokens) self.calc_token_confidence() self.calc_token_2_confidence() num_operators = self.count_my_tokens(['operator', 'invalid operator']) if num_operators > 0: self.calc_operator_confidence(num_operators) allow_pairs = [] self.calc_operator_2_confidence(tokens, num_operators, allow_pairs) # self.calc_operator_3_confidence(tokens, num_operators, group_ends, allow_pairs) self.calc_operator_4_confidence(tokens, num_operators, group_starts, allow_pairs) self.calc_group_confidence(tokens, group_mids) # self.calc_operand_n_confidence(tokens, operand_types, 2) # self.calc_operand_n_confidence(tokens, operand_types, 4) self.calc_keyword_confidence() self.calc_line_length_confidence(code, self.max_expected_line) self.calc_picture_confidence() expected_keyword_confidence = self.check_expected_keywords() self.confidences['expected_keywords'] = expected_keyword_confidence