def GenerateTags(buff): """Create a DocStruct object that represents a Scheme document @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() rtags.SetElementDescription('function', "Function Definitions") for lnum, line in enumerate(buff): line = line.strip() # Skip comment and empty lines if line.startswith(u";") or not line: continue # Check Function Definitions if line.startswith('(') and u'define' in line: llen = len(line) idx = 1 idx += (len(line[idx:]) - len(line[idx:].lstrip())) if llen > idx and line[idx:].startswith(u'define') and \ (llen > (idx + 6)) and line[idx+6].isspace(): idx = parselib.SkipWhitespace(line, idx + 6) if llen > idx and line[idx] == u'(': # function with parameters idx = parselib.SkipWhitespace(line, idx + 1) name = GetIdentifierName(line[idx:]) if name is not None: rtags.AddFunction(taglib.Function(name, lnum)) return rtags
def CaptureClassElements(scope, line, idx): """Get recursively capture all the elements defined on the line from the index. @param scope: Scope object to append element to @param line: string of text to parse @param idx: current index in line @return: new index """ idx = parselib.SkipWhitespace(line, idx) dend = line[idx:].find(u"{") + idx if idx >= len(line) or idx == dend: return idx segments = line[idx:dend].strip().split() if len(segments): token = segments[0] idx += len(token) if token.startswith(u'.'): # Descendant class nextscope = taglib.Class(token, scope.GetLine()) scope.AddElement('class', nextscope) # Recurse to look for more children idx = CaptureClassElements(nextscope, line, idx) elif token.startswith(u'#'): # An ID obj = taglib.Variable(token, scope.GetLine()) scope.AddElement('variable', obj) else: # Element p, div, etc.. obj = CSSTag(token, scope.GetLine()) scope.AddElement('tag_red', obj) return idx
def GenerateTags(buff): """Create a DocStruct object that represents an Editra Style Sheet @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() rtags.SetElementDescription('styletag', "Style Tags") c_element = None # Currently found document element incomment = False # Inside a comment indef = False # Inside a style definition {} for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Check if valid item to add to document if c_element is not None and line[idx] == u'{': rtags.AddElement('styletag', c_element) c_element = None # Check for coments if line[idx] == u'/' and llen > idx and line[idx+1] == u'*': idx += 2 incomment = True elif line[idx] == u'*' and llen > idx and line[idx+1] == u'/': idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif line[idx] == u'{': idx += 1 indef = True elif indef and line[idx] == u'}': idx += 1 indef = False elif not indef and line[idx].isalpha(): # Found start of tag name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: # Make a tag but don't add it to the DocStruct till we # find if a { is the next non space character to follow c_element = StyleTag(name, lnum) idx += len(name) else: # This should never happen but if it does there must # be something wrong with the document or the parse has # gone afowl. idx += 1 else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a vala dacument @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() # Setup document structure rtags.SetElementDescription('function', "Function Definitions") inclass = False # Inside a class defintion incomment = False # Inside a comment infundef = False # Inside a function definition lastclass = None lastfun = None openb = 0 # Keep track of open brackets for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Check for coments if line[idx:].startswith(u'/*'): idx += 2 incomment = True elif line[idx:].startswith(u'//') or line[idx:].startswith(u'#'): break # go to next line elif line[idx:].startswith(u'*/'): idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 continue elif line[idx] == u'{': idx += 1 openb += 1 # Class name must be followed by a { if not inclass and lastclass is not None: inclass = True rtags.AddClass(lastclass) elif lastfun is not None: infundef = True lastfun = None else: pass continue elif line[idx] == u'}': idx += 1 openb -= 1 if inclass and openb == 0: inclass = False lastclass = None elif infundef and inclass and openb == 1: infundef = False elif infundef and openb == 0: infundef = False lastfun = None else: pass continue elif not infundef and parselib.IsToken(line, idx, u'class'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 5) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: idx += len(name) # Move past the class name lastclass = taglib.Class(name, lnum) continue match = RE_METH.match(line[idx:]) if match is not None: # Check that not a method call sline = line.strip() if sline.endswith(u';'): idx += match.end(2) continue # Most likely a definition so store it in the DocStruct name = match.group(2) # Secondary check for other end cases regex will find if name in KEYWORDS: idx += match.end(2) continue lastfun = name if inclass and lastclass is not None: lastclass.AddMethod( taglib.Method(name, lnum, lastclass.GetName())) else: rtags.AddFunction(taglib.Function(name, lnum)) idx += match.end(2) else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a Php Script @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() # Setup document structure rtags.SetElementDescription('function', "Function Definitions") inphp = False # Are we in a php section or not inclass = False # Inside a class defintion incomment = False # Inside a comment infundef = False # Inside a function definition lastclass = None lastfun = None instring = False openb = 0 # Keep track of open brackets for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Walk through strings ignoring contents if instring or line[idx] in (u"'", u'"'): idx, instring = parselib.FindStringEnd(line[idx:], idx) # For multiline strings if instring: continue # Check if in a <?php ?> block or not if line[idx:].startswith(u'<?'): idx += 2 if line[idx:].startswith(u'php'): idx += 5 inphp = True elif line[idx:].startswith(u'?>'): idx += 2 inphp = False # Skip anything not in side of a php section if not inphp: idx += 1 continue # Check for coments if line[idx:].startswith(u'/*'): idx += 2 incomment = True elif line[idx:].startswith(u'//') or line[idx:].startswith(u'#'): break # go to next line elif line[idx:].startswith(u'*/'): idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif line[idx] == u'{': idx += 1 openb += 1 # Class name must be followed by a { if not inclass and lastclass is not None: inclass = True rtags.AddClass(lastclass) elif lastfun is not None: infundef = True lastfun = None else: pass elif line[idx] == u'}': idx += 1 openb -= 1 if inclass and openb == 0: inclass = False lastclass = None elif infundef and inclass and openb == 1: infundef = False elif infundef and openb == 0: infundef = False lastfun = None else: pass elif not infundef and parselib.IsToken(line, idx, u'class'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 5) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: idx += len(name) # Move past the class name lastclass = taglib.Class(name, lnum) elif parselib.IsToken(line, idx, u'function'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 8) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: lastfun = name # Skip whitespace idx = parselib.SkipWhitespace(line, idx + len(name)) if line[idx] != u'(': continue if inclass and lastclass is not None: lastclass.AddMethod(taglib.Method(name, lnum, lastclass.GetName())) else: rtags.AddFunction(taglib.Function(name, lnum)) elif inclass and parselib.IsToken(line, idx, u'var'): # Look for class variables idx += 3 parts = line[idx:].split() if len(parts) and parts[0].startswith(u'$'): name = parselib.GetFirstIdentifier(parts[0][1:]) if name is not None and lastclass is not None: name = u'$' + name lastclass.AddVariable(taglib.Variable(name, lnum, lastclass.GetName())) idx += len(name) else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a SQL document @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() # Setup document structure rtags.SetElementDescription('function', "Function Definitions") rtags.SetElementDescription('procedure', "Procedure Definitions") rtags.SetElementDescription('package', "Packages") rtags.SetElementPriority('package', 3) rtags.SetElementPriority('function', 2) rtags.SetElementPriority('procedure', 1) # State Variables inpackage = False # Inside a package incomment = False # Inside a comment infunpro = False # Inside a function or proceedure definition lastname = None # Name of last found element lastpkg = None # Last found package object lastpkgname = None # Name of last package for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Check for coments if line[idx:].startswith(u'/*'): idx += 2 incomment = True elif line[idx:].startswith(u'--') or line[idx:].startswith(u'#'): break # go to next line elif line[idx:].startswith(u'*/'): idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif (inpackage or infunpro) and \ parselib.IsToken(line, idx, u'end', True): idx = parselib.SkipWhitespace(line, idx + 3) name = parselib.GetFirstIdentifier(line[idx:]) if inpackage and name == lastpkgname: inpackage = False lastpkgname = None elif infunpro and name == lastname: infunpro = False lastname = None elif not infunpro and parselib.IsToken(line, idx, u'package', True): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 7) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None and name.lower() == u'body': idx = parselib.SkipWhitespace(line, idx + 3) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: inpackage = True lastpkgname = name lastpkg = taglib.Package(name, lnum) rtags.AddElement('package', lastpkg) elif parselib.IsToken(line, idx, u'function', True): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 8) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: infunpro = True lastname = name if lastpkg is not None: lastpkg.AddElement('function', taglib.Function(name, lnum)) else: rtags.AddFunction(taglib.Function(name, lnum)) elif parselib.IsToken(line, idx, u'procedure', True): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 9) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: infunpro = True lastname = name if lastpkg is not None: lastpkg.AddElement('procedure', taglib.Procedure(name, lnum)) else: rtags.AddElement('procedure', taglib.Procedure(name, lnum)) else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a haXe Script @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() # Setup document structure rtags.SetElementDescription('class', "Class Definitions") rtags.SetElementDescription('function', "Function Definitions") inclass = False # Inside a class defintion incomment = False # Inside a comment infundef = False # Inside a function definition lastclass = None lastfun = None openb = 0 # Keep track of open brackets for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Check for coments if line[idx:].startswith(u'/*'): idx += 2 incomment = True elif line[idx:].startswith(u'//'): break # go to next line elif line[idx:].startswith(u'*/'): idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif line[idx] == u'{': idx += 1 openb += 1 # Class name must be followed by a { if not inclass and lastclass is not None: inclass = True rtags.AddClass(lastclass) elif lastfun is not None: infundef = True lastfun = None else: pass elif line[idx] == u'}': idx += 1 openb -= 1 if inclass and openb == 0: inclass = False lastclass = None elif infundef and inclass and openb == 1: infundef = False elif infundef and openb == 0: infundef = False lastfun = None else: pass elif not infundef and parselib.IsToken(line, idx, u'class'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 5) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: idx += len(name) # Move past the class name lastclass = taglib.Class(name, lnum) elif parselib.IsToken(line, idx, u'function'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 8) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: lastfun = name # Skip whitespace idx = parselib.SkipWhitespace(line, idx + len(name)) if line[idx] != u'(': continue if inclass and lastclass is not None: lastclass.AddMethod( taglib.Method(name, lnum, lastclass.GetName())) else: rtags.AddFunction(taglib.Function(name, lnum)) elif inclass and not infundef and parselib.IsToken( line, idx, u'var'): # Look for class variables idx = parselib.SkipWhitespace(line, idx + 3) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None and lastclass is not None: lastclass.AddVariable( taglib.Variable(name, lnum, lastclass.GetName())) idx += len(name) else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a Verilog document @param buff: a file like buffer object (StringIO) @todo: add support for parsing module definitions / class variables """ rtags = taglib.DocStruct() rtags.SetElementDescription('class', "Class Definitions") rtags.SetElementDescription('task', "Task Definitions") rtags.SetElementDescription('function', "Function Definitions") rtags.SetElementPriority('class', 3) rtags.SetElementPriority('task', 2) rtags.SetElementPriority('function', 1) # Variables to track parse state inclass = False # Inside a class defintion incomment = False # Inside a comment intask = False # Inside a task definition infunction = False # Inside a function definition # Parse the text for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip any leading Whitespace idx = parselib.SkipWhitespace(line, idx) # Check for coments if line[idx:].startswith(u'/*'): idx += 2 incomment = True elif line[idx:].startswith(u'//'): break # go to next line elif line[idx:].startswith(u'*/'): idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif parselib.IsToken(line, idx, u'class'): idx = parselib.SkipWhitespace(line, idx + 5) cname = parselib.GetFirstIdentifier(line[idx:]) if cname is not None: inclass = True rtags.AddClass(taglib.Class(cname, lnum)) break # go to next line elif inclass and parselib.IsToken(line, idx, u'endclass'): inclass = False break # go to next line elif parselib.IsToken(line, idx, u'task'): idx += 4 tname = parselib.GetTokenParenLeft(line[idx:]) if tname is not None: intask = True if inclass: lclass = rtags.GetLastClass() task = taglib.Function(tname, lnum, 'task', lclass.GetName()) lclass.AddElement('task', task) else: task = taglib.Function(tname, lnum, 'task') rtags.AddElement('task', task) break # goto next line elif parselib.IsToken(line, idx, u'function'): idx += 8 fname = parselib.GetTokenParenLeft(line[idx:]) if fname is not None: infunction = True if inclass: lclass = rtags.GetLastClass() lclass.AddMethod(taglib.Method(fname, lnum)) else: rtags.AddFunction(taglib.Function(fname, lnum)) break elif intask and parselib.IsToken(line, idx, u'endtask'): intask = False break # go to next line elif infunction and parselib.IsToken(line, idx, 'endfunction'): infunction = False break else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a Cascading Style Sheets @param buff: a file like buffer object (StringIO) @todo: add support for parsing selectors and grouping classes and identities of each selector in a subscope. """ rtags = taglib.DocStruct() # Setup document structure # Use variables node for global identities rtags.SetElementDescription('variable', "Identities") rtags.SetElementDescription('tag_red', "Elements") # Use classes for global classes # Uses DocStruct builtin c_tag = None # Currently found tag incomment = False # Inside a comment indef = False # Inside a style definition {} for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): idx = parselib.SkipWhitespace(line, idx) # Check for comments if llen > idx + 1 and line[idx] == u'/' and line[idx + 1] == u'*': idx += 2 incomment = True elif llen > idx + 1 and line[idx] == u'*' and line[idx + 1] == u'/': idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif line[idx] == u'{': idx += 1 indef = True elif indef and line[idx] == u'}': idx += 1 indef = False elif not indef and line[idx] in (u'.', u'#'): # Classes and ID's if idx == 0 or line[idx - 1].isspace(): names = line[idx:].split() if len(names): name = names[0] else: name = None if name is not None: if line[idx] == u'.': # See if we already have found previous # defs using this class identifier cobj = rtags.GetElement('class', name) if cobj is None: cobj = taglib.Class(name, lnum) rtags.AddClass(cobj) # Update the index idx += len(name) # Grab all other defs that may be children of # the current one. idx = CaptureClassElements(cobj, line, idx) else: # Stand alone ID rtags.AddVariable(taglib.Variable(name, lnum)) idx += len(name) continue # TODO: smarter skip ahead to speed up parse idx += 1 elif not indef and not line[idx].isspace(): # Possible element nparen = line[idx:].find(u'{') + idx token = line[idx:nparen] if token: idx += len(token) if not token.startswith(u"@"): obj = CSSTag(token.strip(), lnum) rtags.AddElement('tag_red', obj) else: idx += 1 else: # TODO: smarter skip ahead to speed up parse idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a Ferite document @param buff: a file like buffer object (StringIO) """ rtags = taglib.DocStruct() # Setup document structure rtags.SetElementDescription('namespace', "Namespaces") rtags.SetElementDescription('class', "Class Definitions") rtags.SetElementDescription('protocol', "Protocols") rtags.SetElementDescription('function', "Function Definitions") rtags.SetElementPriority('namespace', 4) rtags.SetElementPriority('class', 3) rtags.SetElementPriority('protocol', 2) rtags.SetElementPriority('function', 1) # Variables for tracking parse state incomment = False # Inside a comment innamespace = False # Inside a namespace inclass = False # Inside a class defintion inprotocol = False # Inside a protocol infundef = False # Inside a function definition lastnspace = None # Last Namespace lastclass = None # Last Class lastprotocol = None # Last Protocol lastfun = None # last Function openb = 0 # Keep track of open brackets for scope resolution def InSubScope(): return innamespace or inclass or inprotocol or infundef # Parse the contents of the buffer for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Check for comments if line[idx:].startswith(u'/*'): idx += 2 incomment = True elif line[idx:].startswith(u'//'): break # go to next line elif line[idx:].startswith(u'*/'): idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif line[idx] == u'{': idx += 1 openb += 1 # Namespace/Class/Protocol names must be followed by a { if not InSubScope() and lastnspace is not None: innamespace = True rtags.AddElement('namespace', lastnspace) elif not inclass and lastclass is not None: inclass = True if lastnspace is not None: # Class is in a namespace lastnspace.AddElement('class', lastclass) else: # Class is at the global scope rtags.AddClass(lastclass) elif not InSubScope() and lastprotocol is not None: inprotocol = True rtags.AddElement('protocol', lastprotocol) elif lastfun is not None: infundef = True lastfun = None else: pass elif line[idx] == u'}': idx += 1 openb -= 1 # Check if the scope needs to change if innamespace and openb == 0: innamespace = False lastnspace = None elif innamespace and inclass and openb == 1: inclass = False lastclass = None elif (innamespace and inclass and infundef and openb == 2) or \ (innamespace and infundef and openb == 1): infundef = False lastfun = None elif inclass and openb == 0: inclass = False lastclass = None elif inclass and infundef and openb == 1: infundef = False lastfun = None elif inprotocol and openb == 0: inprotocol = False lastprotocol = None elif inprotocol and infundef and openb == 1: infundef = False lastfun = None elif infundef and openb == 0: infundef = False lastfun = None else: pass elif not infundef and parselib.IsToken(line, idx, u'class'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 5) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: idx += len(name) # Move past the class name lastclass = taglib.Class(name, lnum) elif not infundef and not inclass and \ parselib.IsToken(line, idx, 'namespace'): idx = parselib.SkipWhitespace(line, idx + 9) name = GetElementName(line[idx:]) if name is not None: idx += len(name) lastnspace = taglib.Namespace(name, lnum) elif parselib.IsToken(line, idx, u'protocol'): idx = parselib.SkipWhitespace(line, idx + 8) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: idx += len(name) lastprotocol = Protocol(name, lnum) elif parselib.IsToken(line, idx, u'function'): # Skip whitespace idx = parselib.SkipWhitespace(line, idx + 8) name = parselib.GetFirstIdentifier(line[idx:]) if name is not None: lastfun = name # Skip whitespace idx = parselib.SkipWhitespace(line, idx + len(name)) if line[idx] != u'(': continue tfun = taglib.Function(name, lnum) if innamespace and not inclass and lastnspace: lastnspace.AddElement('function', tfun) elif inclass and lastclass is not None: lastclass.AddMethod( taglib.Method(name, lnum, lastclass.GetName())) elif inprotocol and lastprotocol is not None: lastprotocol.AddElement('function', tfun) else: rtags.AddFunction(tfun) else: idx += 1 return rtags
def GenerateTags(buff): """Create a DocStruct object that represents a Cascading Style Sheets @param buff: a file like buffer object (StringIO) @todo: add support for parsing selectors and grouping classes and identities of each selector in a subscope. """ rtags = taglib.DocStruct() # Setup document structure # rtags.SetElementDescription('tag', "Selectors") # Use variables node for global identities rtags.SetElementDescription('variable', "Identities") # Use classes for global classes # Uses DocStruct builtin c_tag = None # Currenly found tag incomment = False # Inside a comment indef = False # Inside a style definition {} for lnum, line in enumerate(buff): line = line.strip() llen = len(line) idx = 0 while idx < len(line): # Skip Whitespace idx = parselib.SkipWhitespace(line, idx) # Check if valid item to add to document # if c_tag is not None and line[idx] == u'{': # rtags.AddElement('tag', c_tag) # c_tag = None # Check for coments if llen > idx + 1 and line[idx] == u'/' and line[idx + 1] == u'*': idx += 2 incomment = True elif llen > idx + 1 and line[idx] == u'*' and line[idx + 1] == u'/': idx += 2 incomment = False # At end of line if idx >= llen: break # Look for tags if incomment: idx += 1 elif line[idx] == u'{': idx += 1 indef = True elif indef and line[idx] == u'}': idx += 1 indef = False elif not indef and line[idx] in (u'.', u'#'): if idx == 0 or line[idx - 1].isspace(): name = parselib.GetFirstIdentifier(line[idx + 1:]) if name is not None: name = line[idx] + name if line[idx] == u'.': rtags.AddClass(taglib.Class(name, lnum)) else: rtags.AddVariable(taglib.Variable(name, lnum)) idx += len(name) idx += 1 else: idx += 1 return rtags