def parse(self, tokens): # while the next token is not begin (start of compound statement), scan the block for declarations while (True): # variable declaration part if (tokens.match_lookahead(TokenKind.Identifier, 'var')): # read variable declaration part # and append to current active variables # local variables can overwrite global variables current_part = PascalVarDeclaration(self) current_part.parse(tokens) self._variables.update(current_part.variables) elif (tokens.match_lookahead(TokenKind.Identifier, 'procedure') or tokens.match_lookahead(TokenKind.Identifier, 'function')): current_part = PascalFunction(self, tokens) self._functions[current_part.name] = current_part current_part.parse(tokens) elif (tokens.match_lookahead(TokenKind.Identifier, 'type')): current_part = PascalTypeDeclaration(self) current_part.parse(tokens) self._types.update(current_part.types) elif tokens.match_lookahead(TokenKind.Identifier, 'begin'): break else: raise_error( ('Unknown block token...' + str(tokens.next_token())), '', is_critical=False) self._contents.append(current_part) # at this point we must be at a begin self._compound_statement = PascalCompoundStatement(self, True) self._compound_statement.parse(tokens) tokens.match_token(TokenKind.Symbol)
def resolve_function_call(self, function): """ Assumes the file has checked itself for the function before calling this """ from pas_file_cache import get_unit_named logger.debug("File: Resolving Function call: %s", function.name) for unit_reference in self._contents.uses: unit = unit_reference.points_to.contents for unit_function in unit.functions: if unit_function.name.lower() == function.name.lower(): return unit_function for unit_declared_function in unit.function_declarations: if unit_declared_function.name.lower() == function.name.lower( ): return unit_declared_function # check the System unit... system = get_unit_named('System') if (system != None): for unit_function in system.contents.functions: if unit_function.name.lower() == function.name.lower(): return unit_function raise_error(("File: Unable to resolve function: %s in %s" % (function.name, self._filename)), '', is_critical=False)
def parse(self, tokens): # while the next token is not begin (start of compound statement), scan the block for declarations while (True): # variable declaration part if (tokens.match_lookahead(TokenKind.Identifier, 'var')): # read variable declaration part # and append to current active variables # local variables can overwrite global variables current_part = PascalVarDeclaration(self) current_part.parse(tokens) self._variables.update(current_part.variables) elif (tokens.match_lookahead(TokenKind.Identifier, 'procedure') or tokens.match_lookahead(TokenKind.Identifier, 'function')): current_part = PascalFunction(self, tokens) self._functions[current_part.name] = current_part current_part.parse(tokens) elif (tokens.match_lookahead(TokenKind.Identifier, 'type')): current_part = PascalTypeDeclaration(self) current_part.parse(tokens) self._types.update(current_part.types) elif tokens.match_lookahead(TokenKind.Identifier, 'begin'): break else: raise_error(('Unknown block token...' + str(tokens.next_token())), '', is_critical=False) self._contents.append(current_part) # at this point we must be at a begin self._compound_statement = PascalCompoundStatement(self, True) self._compound_statement.parse(tokens) tokens.match_token(TokenKind.Symbol)
def __init__(self, path=None, is_virtual=False): import os from pascal_parser.tokeniser.pas_meta_comment import PascalMetaComment self._filename = '' self._name = '' self._contains_kind = None self._contents = None # PascalProgram or PascalLibrary self._code = dict() self._is_parsed = False self._meta_comment = None self._is_virtual = is_virtual self._meta_comment = None self._stream = None if not self._is_virtual: self._stream = SGTokenStream(path) self._meta_comment = PascalMetaComment(self._stream) self._meta_comment.process_meta_comments() self._filename = os.path.basename(path).split('.')[0] # program? if self._stream.match_lookahead(TokenKind.Identifier, 'program'): self._contains_kind = 'program'; self._contents = PascalProgram(self) # unit? elif self._stream.match_lookahead(TokenKind.Identifier, 'unit'): self._contains_kind = 'unit'; self._contents = PascalUnit(self) else: #logger raise_error(("Error in program syntax: %s in %s", self._stream.next_token(), self._filename), '', is_critical=False) self._name = self._stream.lookahead(2)[1].value # file's 'name' is after the kind of file is stated else: self._is_parsed = True
def resolve_variable(self, var_name): from pas_file_cache import get_unit_named logger.debug("File: Resolving Variable: %s", var_name) for unit_reference in self._contents.uses: unit = unit_reference.points_to.contents # could be a variable... for unit_name, unit_var in unit.variables.items(): if unit_name.lower() == var_name.lower(): return unit_var # or it could be an enumeration value... for (name, type) in unit.types.items(): if type.kind is 'enumeration': for val in type.values: if val.name.lower() == var_name.lower(): return val # check the System unit... system = get_unit_named('System') if (system != None): for name, unit_var in system.contents.variables.items(): if name.lower() == var_name.lower(): return unit_var raise_error(("File: Unable to resolve variable: %s in %s" % (var_name, self._filename)), '', is_critical=False)
def get_template(name): if name is None: return None if (name in _templates): return _templates[name] else: raise_error(("Error getting template: " + name), '', is_critical=True)
def add_file(file): ''' ''' if file != None: _files[file.name.lower()] = file if file.contains_kind == 'unit': _loaded_units[file.name.lower()] = file logger.info("File Cache: Added unit: %s", file.name) else: logger.info("File Cache: Added program: %s", file.name) else: raise_error("File Cache: Unable to add None unit", 'FileIO', is_critical=False)
def get_type(name): ''' gets a type from the type cache and returns it if no type is found then it returns None ''' if name is None: raise_error(("Type Cache: name was None"), '', is_critical=False) if name in _loaded_types: return _loaded_types[name] else: return None
def get_type(name): ''' gets a type from the type cache and returns it if no type is found then it returns None ''' if name is None: raise_error(("Type Cache: name was None"), '', is_critical=False) if name in _loaded_types: return _loaded_types[name] else: return None
def match_one_token(self, token_lst): """ Finds and returns the token match within a list of tokenIdentifier - value pairs. Will throw an error if no match could be found """ matched = self.match_one_lookahead(token_lst, consume=False) tok = self.next_token() if not matched: raise_error(('TokenStream %s: unexpected %s(%s) expected %s', self._tokeniser.line_details(), tok._kind, tok._value, map(lambda n: '%s(%s)' % (n[0],n[1]),token_lst)), exception_msg, is_critical=False) return tok
def add_file(file): ''' ''' if file != None: _files[file.name.lower()] = file if file.contains_kind == 'unit': _loaded_units[file.name.lower()] = file logger.info("File Cache: Added unit: %s", file.name) else: logger.info("File Cache: Added program: %s", file.name) else: raise_error("File Cache: Unable to add None unit", 'FileIO', is_critical=False)
def match_one_token(self, token_lst): """ Finds and returns the token match within a list of tokenIdentifier - value pairs. Will throw an error if no match could be found """ matched = self.match_one_lookahead(token_lst, consume=False) tok = self.next_token() if not matched: raise_error(('TokenStream %s: unexpected %s(%s) expected %s', self._tokeniser.line_details(), tok._kind, tok._value, map(lambda n: '%s(%s)' % (n[0], n[1]), token_lst)), exception_msg, is_critical=False) return tok
def resolve_type(self, type): from pas_file_cache import get_unit_named logger.debug("File: Resolving Type: %s", type) for unit_reference in self._contents.uses: if unit_reference.points_to != None: unit = unit_reference.points_to.contents for unit_name, unit_type in unit.types.items(): if unit_name.lower() == type.lower(): return unit_type # check the System unit... system = get_unit_named('System') if (system != None): for name, unit_type in system.contents.types.items(): if name.lower() == type.lower(): return unit_type raise_error(("File: Unable to resolve type: %s in %s" % (type, self._filename)), '', is_critical=False)
def match_token(self, token_kind, token_value = None): """ Looks at the next token, and if it is the same kind as 'token_kind' and has the same value as 'token_value' then it is returned. Otherwise an error occurs and the program stops. If 'token_value' is None then only the 'token_kind' is checked. """ tok = self.next_token() if tok._kind != token_kind or (token_value != None and token_value != tok._value.lower()): raise_error(('TokenStream %s: found a %s (%s) expected %s (%s)' % (self._tokeniser.line_details(), tok._kind, tok._value, token_kind, token_value)), '', is_critical=False) logger.debug('TokenStream : Matched token %s (%s)', tok._kind, tok._value) return tok
def load_templates(lib, extension): import glob import os path = os.path.join(os.path.dirname(os.path.realpath(__file__)), lib) # print 'here ', path file_paths = glob.glob(path + '*' + extension) if len(file_paths) > 0: for path in file_paths: file = open(path, "r") template = '' for line in file.readlines(): template += line _templates[os.path.basename(path)] = template file.close() else: raise_error(("Template path is empty: %s" % path), '', is_critical=True)
def resolve_type(self, type): from pas_file_cache import get_unit_named logger.debug("File: Resolving Type: %s", type) for unit_reference in self._contents.uses: if unit_reference.points_to != None: unit = unit_reference.points_to.contents for unit_name, unit_type in unit.types.items(): if unit_name.lower() == type.lower(): return unit_type # check the System unit... system = get_unit_named('System') if (system != None): for name, unit_type in system.contents.types.items(): if name.lower() == type.lower(): return unit_type raise_error(("File: Unable to resolve type: %s in %s" % (type, self._filename)), '', is_critical=False)
def parse(self, tokens, do_resolve=True): logger.debug("Parsing uses clause") tokens.match_token(TokenKind.Identifier, 'uses') while (True): if (tokens.match_lookahead(TokenKind.Symbol, ';')): tokens.match_token(TokenKind.Symbol, ';') break elif (tokens.match_lookahead(TokenKind.Symbol, ',')): tokens.match_token(TokenKind.Symbol, ',') elif (tokens.match_lookahead(TokenKind.Identifier)): new_reference = PascalUnitReference() new_reference.parse(tokens, self._file_owner, do_resolve) self._units.append(new_reference) else: raise_error( ('Error reading uses clause: ' + str(tokens.next_token())), '', is_critical=False) logger.debug("Finished parsing uses clause")
def resolve_function_call(self, function): """ Assumes the file has checked itself for the function before calling this """ from pas_file_cache import get_unit_named logger.debug("File: Resolving Function call: %s", function.name) for unit_reference in self._contents.uses: unit = unit_reference.points_to.contents for unit_function in unit.functions: if unit_function.name.lower() == function.name.lower(): return unit_function for unit_declared_function in unit.function_declarations: if unit_declared_function.name.lower() == function.name.lower(): return unit_declared_function # check the System unit... system = get_unit_named('System') if (system != None): for unit_function in system.contents.functions: if unit_function.name.lower() == function.name.lower(): return unit_function raise_error(("File: Unable to resolve function: %s in %s" % (function.name, self._filename)), '', is_critical=False)
def resolve_variable(self, var_name): from pas_file_cache import get_unit_named logger.debug("File: Resolving Variable: %s", var_name) for unit_reference in self._contents.uses: unit = unit_reference.points_to.contents # could be a variable... for unit_name, unit_var in unit.variables.items(): if unit_name.lower() == var_name.lower(): return unit_var # or it could be an enumeration value... for (name, type) in unit.types.items(): if type.kind is 'enumeration': for val in type.values: if val.name.lower() == var_name.lower(): return val # check the System unit... system = get_unit_named('System') if (system != None): for name, unit_var in system.contents.variables.items(): if name.lower() == var_name.lower(): return unit_var raise_error(("File: Unable to resolve variable: %s in %s" %( var_name, self._filename)), '', is_critical=False)
def match_token(self, token_kind, token_value=None): """ Looks at the next token, and if it is the same kind as 'token_kind' and has the same value as 'token_value' then it is returned. Otherwise an error occurs and the program stops. If 'token_value' is None then only the 'token_kind' is checked. """ tok = self.next_token() if tok._kind != token_kind or (token_value != None and token_value != tok._value.lower()): raise_error( ('TokenStream %s: found a %s (%s) expected %s (%s)' % (self._tokeniser.line_details(), tok._kind, tok._value, token_kind, token_value)), '', is_critical=False) logger.debug('TokenStream : Matched token %s (%s)', tok._kind, tok._value) return tok
def __init__(self, path=None, is_virtual=False): import os from pascal_parser.tokeniser.pas_meta_comment import PascalMetaComment self._filename = '' self._name = '' self._contains_kind = None self._contents = None # PascalProgram or PascalLibrary self._code = dict() self._is_parsed = False self._meta_comment = None self._is_virtual = is_virtual self._meta_comment = None self._stream = None if not self._is_virtual: self._stream = SGTokenStream(path) self._meta_comment = PascalMetaComment(self._stream) self._meta_comment.process_meta_comments() self._filename = os.path.basename(path).split('.')[0] # program? if self._stream.match_lookahead(TokenKind.Identifier, 'program'): self._contains_kind = 'program' self._contents = PascalProgram(self) # unit? elif self._stream.match_lookahead(TokenKind.Identifier, 'unit'): self._contains_kind = 'unit' self._contents = PascalUnit(self) else: #logger raise_error(("Error in program syntax: %s in %s", self._stream.next_token(), self._filename), '', is_critical=False) self._name = self._stream.lookahead(2)[ 1].value # file's 'name' is after the kind of file is stated else: self._is_parsed = True
def main(): logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', stream=sys.stdout) import c_lib import pas_lib import converter_helper converter_helper.converters["c_lib"] = c_lib converter_helper.converters["pas_lib"] = pas_lib script_path = os.path.realpath(__file__) + '/' swingame_path = os.path.realpath(script_path + '../../../../../') + '/' destination = os.path.join(swingame_path, 'Dist', 'HowTo', 'Source_Code' ) source = os.path.join(destination, 'HowTos') lib_source = os.path.join(source, 'lib') # destination = os.path.normpath("../../Dist/HowTo/Source_Code") print "Current Directory: %s" %os.getcwd() print "Source Directory: %s" %source print "Library Source Directory: %s" %lib_source print "Destination Directory: %s" %destination print '*' * 70 if not os.path.exists(source): raise_error("Source directory does not exist %s" %source, '', is_critical=False) if not os.path.exists(lib_source): raise_error("Library directory does not exist %s" %lib_source, '', is_critical=False) if not os.path.exists(destination): raise_error("Destination directory does not exist %s" %destination, '', is_critical=False) print " Adding Units:" print '*' * 70 # add units in the lib_source directory to the file list add_file(PascalFile.create_unit_from('System', None, ['LongInt', 'Byte', 'String', 'Single', 'Pointer', 'LongWord', 'int64', 'Word', 'Integer', 'Boolean'], None)) add_file(PascalFile.create_unit_from('SysUtils', None, None, None)) dir_contents = glob.glob(os.path.join(lib_source, "*.pas")) if len(dir_contents) > 0: for fname in dir_contents: try: add_file(PascalFile(fname)) except Exception: print " Error adding unit: %s", fname else: print "Library directory was empty: %s" %lib_source print '*' * 70 print " Adding Programs:" print '*' * 70 # adds files in the source directory to the file list dir_contents = glob.glob(os.path.join(source, "*.pas")) if len(dir_contents) > 0: for fname in dir_contents: add_file(PascalFile(fname)) else: print raise_error("Source directory was empty: %s" %source, '', is_critical=True) print '*' * 70 print " Parsing files:" print '*' * 70 for (name, file) in files().items(): if (not file.is_parsed) and (file.contains_kind == 'program'): file.parse() print '*' * 70 print ' Converting files:' print '*' * 70 for (name, file) in files().items(): if file.is_parsed and file.contains_kind == 'program': run_convert(file) print '*' * 70 print ' Writing files: ' print '*' * 70 for (name, file) in files().items(): if file.is_parsed and file.contains_kind == 'program': write_file(file, destination) print ".", print ' Done.'
def main(): logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', stream=sys.stdout) import c_lib import pas_lib import converter_helper converter_helper.converters["c_lib"] = c_lib converter_helper.converters["pas_lib"] = pas_lib script_path = os.path.realpath(__file__) + '/' swingame_path = os.path.realpath(script_path + '../../../../../') + '/' destination = os.path.join(swingame_path, 'Dist', 'HowTo', 'Source_Code') source = os.path.join(destination, 'HowTos') lib_source = os.path.join(source, 'lib') # destination = os.path.normpath("../../Dist/HowTo/Source_Code") print "Current Directory: %s" % os.getcwd() print "Source Directory: %s" % source print "Library Source Directory: %s" % lib_source print "Destination Directory: %s" % destination print '*' * 70 if not os.path.exists(source): raise_error("Source directory does not exist %s" % source, '', is_critical=False) if not os.path.exists(lib_source): raise_error("Library directory does not exist %s" % lib_source, '', is_critical=False) if not os.path.exists(destination): raise_error("Destination directory does not exist %s" % destination, '', is_critical=False) print " Adding Units:" print '*' * 70 # add units in the lib_source directory to the file list add_file( PascalFile.create_unit_from('System', None, [ 'LongInt', 'Byte', 'String', 'Single', 'Pointer', 'LongWord', 'int64', 'Word', 'Integer', 'Boolean' ], None)) add_file(PascalFile.create_unit_from('SysUtils', None, None, None)) dir_contents = glob.glob(os.path.join(lib_source, "*.pas")) if len(dir_contents) > 0: for fname in dir_contents: try: add_file(PascalFile(fname)) except Exception: print " Error adding unit: %s", fname else: print "Library directory was empty: %s" % lib_source print '*' * 70 print " Adding Programs:" print '*' * 70 # adds files in the source directory to the file list dir_contents = glob.glob(os.path.join(source, "*.pas")) if len(dir_contents) > 0: for fname in dir_contents: add_file(PascalFile(fname)) else: print raise_error("Source directory was empty: %s" % source, '', is_critical=True) print '*' * 70 print " Parsing files:" print '*' * 70 for (name, file) in files().items(): if (not file.is_parsed) and (file.contains_kind == 'program'): file.parse() print '*' * 70 print ' Converting files:' print '*' * 70 for (name, file) in files().items(): if file.is_parsed and file.contains_kind == 'program': run_convert(file) print '*' * 70 print ' Writing files: ' print '*' * 70 for (name, file) in files().items(): if file.is_parsed and file.contains_kind == 'program': write_file(file, destination) print ".", print ' Done.'