def _open_index(self): indexPath = self._index_path if indexPath == "neo4j": from storage.neo4j import Neo4jStorage self._storage = Neo4jStorage() elif indexPath == "dummy": from storage.dummy import DummyStorage self._storage = DummyStorage() elif indexPath.find(".sqlite3") > 0: from storage.sqlite3 import Sqlite3Storage self._storage = Sqlite3Storage(indexPath) elif indexPath.find("/") > 0: from storage.shelve import ShelveStorage self._storage = ShelveStorage(indexPath) else: raise Exception("Cannot open index '" + indexPath + "'!")
class PhpIndex: def __init__( self, index_path, update_callback=None, error_callback=None, finished_callback=None, indexPathManager=None ): if index_path == None: raise Exception("Cannot open index from empty file-path!") self._parsers = {} self._index_path_manager = indexPathManager self._index_path = index_path self._update_callback = update_callback self._error_callback = error_callback self._finished_callback = finished_callback self._open_index() ### BUILD API def build(self, work_dir): try: while work_dir[-1:] == "/": work_dir = work_dir[:-1] self._open_index() self._storage.empty() self._all_files_count = 0 self._done_files_count = 0 self._storage.begin() self._index_internals() self._collect_directory(work_dir) self._index_directory(work_dir) self._storage.sync() if self._finished_callback != None: self._finished_callback() except sqlite3.OperationalError as exception: print(dir(exception)) self._error_callback("Database error: " + str(exception)) raise exception def update(self, work_dir): try: while work_dir[-1:] == "/": work_dir = work_dir[:-1] self._open_index() self._all_files_count = 0 self._done_files_count = 0 self._storage.begin() self._remove_deleted(work_dir) self._collect_directory(work_dir, True) self._update_directory(work_dir) self._storage.sync() if self._finished_callback != None: self._finished_callback() except sqlite3.Error as exception: self._error_callback("Database error: " + exception.args[0]) except sqlite3.OperationalError as exception: self._error_callback("Database error: " + exception.strerror) def set_build_callback(self, callback): self._callback = callback ### INDEX ACCESS API ### HELPERS def _open_index(self): indexPath = self._index_path if indexPath == "neo4j": from storage.neo4j import Neo4jStorage self._storage = Neo4jStorage() elif indexPath == "dummy": from storage.dummy import DummyStorage self._storage = DummyStorage() elif indexPath.find(".sqlite3") > 0: from storage.sqlite3 import Sqlite3Storage self._storage = Sqlite3Storage(indexPath) elif indexPath.find("/") > 0: from storage.shelve import ShelveStorage self._storage = ShelveStorage(indexPath) else: raise Exception("Cannot open index '" + indexPath + "'!") def get_index(self): return self._storage ### BUILD HELPERS def _index_internals(self): storage = self._storage csvFilePath = os.path.dirname(__file__) + "/php.internals.csv" reader = csv.reader(open(csvFilePath, "r"), delimiter=",") for row in reader: typeName = row[0] docComment = "" namespace = "\\" if typeName == "function": typeName, name = row storage.add_function("INTERNAL", namespace, name, docComment, 0, 0, []) elif typeName in ["class", "interface", "trait"]: typeName, className, classType, parentName, interfaces, isFinal, isAbstract, docComment = row interfaces = interfaces.split(",") isFinal = isFinal == "true" isAbstract = isAbstract == "true" storage.add_class( "INTERNAL", namespace, className, classType, parentName, interfaces, isFinal, isAbstract, docComment, 0, 0, ) elif typeName == "member": typeName, memberName, className, isStatic, visibility, docComment = row storage.add_member("INTERNAL", namespace, className, memberName, 0, 0, isStatic, visibility, docComment) pass elif typeName == "method": typeName, methodName, className, isStatic, visibility, parameters, docComment = row parameters = parameters.split(",") isStatic = isStatic == "true" storage.add_method( "INTERNAL", namespace, className, methodName, isStatic, visibility, docComment, 0, 0, parameters ) pass elif typeName == "variable": pass elif typeName == "constant": typeName, name, value = row storage.add_constant("INTERNAL", name, 0, 0) def _remove_deleted(self, directory): allFilesToCheck = self._storage.get_all_files() currentFileCount = 0 for filePath in allFilesToCheck: self._update_callback(currentFileCount, len(allFilesToCheck), filePath) currentFileCount += 1 if not self.__is_file_indexable(filePath): self._unindex_phpfile(filePath) def _collect_directory(self, directory, only_updates=False): for entry in os.listdir(directory): entryPath = directory + "/" + entry if os.path.isdir(entryPath): self._collect_directory(entryPath, only_updates) elif os.path.isfile(entryPath): if self.__is_file_indexable(entryPath): currentMTime = int(os.path.getmtime(entryPath)) indexedMTime = int(self._storage.getmtime(entryPath)) if currentMTime > indexedMTime or not only_updates: indexedHash = self._storage.gethash(entryPath) currentHash = hashlib.sha256(open(entryPath, "rb").read()).digest() if currentHash != indexedHash: self._all_files_count += 1 def _index_directory(self, directory): for entry in os.listdir(directory): entryPath = directory + "/" + entry if os.path.isdir(entryPath): self._index_directory(entryPath) elif os.path.isfile(entryPath): if self.__is_file_indexable(entryPath): self._done_files_count += 1 if self._update_callback != None: self._update_callback(self._done_files_count, self._all_files_count, entryPath) self._index_phpfile(entryPath) def _update_directory(self, directory): for entry in os.listdir(directory): entryPath = directory + "/" + entry if os.path.isdir(entryPath): self._update_directory(entryPath) elif os.path.isfile(entryPath): if self.__is_file_indexable(entryPath): currentMTime = int(os.path.getmtime(entryPath)) indexedMTime = int(self._storage.getmtime(entryPath)) if currentMTime > indexedMTime: indexedHash = self._storage.gethash(entryPath) currentHash = hashlib.sha256(open(entryPath, "rb").read()).digest() if currentHash != indexedHash: self._done_files_count += 1 if self._update_callback != None: self._update_callback(self._done_files_count, self._all_files_count, entryPath) self._unindex_phpfile(entryPath) self._index_phpfile(entryPath) def __is_file_indexable(self, filePath): shouldInclude = True if self._index_path_manager != None: shouldInclude = self._index_path_manager.shouldIncludePathInIndex(filePath) if filePath[-4:] == ".php" and shouldInclude: return True else: return False def reindex_phpfile(self, filePath): self._storage.begin() self._unindex_phpfile(filePath) self._index_phpfile(filePath) self._storage.sync() def _index_phpfile(self, filePath): with open(filePath, "r", encoding="ISO-8859-1") as f: code = f.read() tokens, comments = token_get_all(code, filePath) blocks, namespace, use_statements, use_statement_index, constants = parse_php_tokens(tokens) # add extracted data to index hashValue = hashlib.sha256(open(filePath, "rb").read()).digest() self._storage.add_file(filePath, namespace, int(os.path.getmtime(filePath)), hashValue) for block in blocks: if len(block) > 2: if block[2] == "class": tokenIndex = block[3] className = block[4] parentName = block[5] interfaces = block[6] isFinal = block[7] isAbstract = block[8] classType = block[9] members = block[10] classconstants = block[11] docComment = block[12] uses = block[13] line = tokens[tokenIndex][2] column = tokens[tokenIndex][3] if parentName in use_statements: parentName = use_statements[parentName] if parentName != None and parentName[0] != "\\": if len(namespace) > 1: parentName = "\\" + namespace + "\\" + parentName else: parentName = "\\" + parentName cleanedInterfaces = [] for interface in interfaces: if interface in use_statements: interface = use_statements[interface] if interface[0] != "\\": interface = "\\" + namespace + "\\" + interface cleanedInterfaces.append(interface) interfaces = cleanedInterfaces self._storage.add_class( filePath, namespace, className, classType, parentName, interfaces, isFinal, isAbstract, docComment, line, column, ) for constTokenIndex in classconstants: constantName = tokens[constTokenIndex + 1][1] constantValue = tokens[constTokenIndex + 3][1] constantLine = tokens[constTokenIndex][2] constantColumn = tokens[constTokenIndex][3] constantDocComment = "" if tokens[constTokenIndex - 1][0] == T_DOC_COMMENT: constantDocComment = tokens[constTokenIndex - 1][1] self._storage.add_class_constant( filePath, namespace, className, constantName, constantValue, constantDocComment, constantLine, constantColumn, ) for memberTokenIndex in members: memberName = tokens[memberTokenIndex][1] memberLine = tokens[memberTokenIndex][2] memberColumn = tokens[memberTokenIndex][3] isStatic = False visibility = "public" memberKeywordIndex = memberTokenIndex - 1 while tokens[memberKeywordIndex][1] in ["public", "protected", "private", "static"]: if tokens[memberKeywordIndex][1] == "static": isStatic = True if tokens[memberKeywordIndex][1] in ["public", "protected", "private"]: visibility = tokens[memberKeywordIndex][1] memberKeywordIndex -= 1 memberDocComment = "" if tokens[memberKeywordIndex][0] in [T_DOC_COMMENT, T_COMMENT]: memberDocComment = tokens[memberKeywordIndex][1] # try to find declaration in comments ( // @var $foo \Bar) typeHint = None annotations = get_annotations_by_doccomment(memberDocComment) if "var" in annotations: for annotation in annotations["var"]: if len(annotation) == 1: typeHint = annotation[0] if len(annotation) == 2: variable, varTypeId = annotation if varTypeId[0] == "$": tempVar = variable variable = varTypeId varTypeId = tempVar if variable == "$" + memberName: typeHint = varTypeId if typeHint != None and len(typeHint) > 0: if typeHint in use_statements: typeHint = use_statements[typeHint] if typeHint[0] != "\\": typeHint = "\\" + namespace + "\\" + typeHint self._storage.add_member( filePath, namespace, className, memberName, memberLine, memberColumn, isStatic, visibility, memberDocComment, typeHint, ) self.__index_uses(filePath, className, "", uses, namespace, use_statements) if block[2] == "method": tokenIndex = block[3] className = block[4] methodName = block[5] keywords = block[6] doccomment = block[7] arguments = block[8] uses = block[9] line = tokens[tokenIndex][2] column = tokens[tokenIndex][3] isStatic = "static" in keywords visibility = list(set(keywords) & set(["public", "protected", "private"])) if len(visibility) == 1: visibility = visibility[0] else: visibility = "public" self._storage.add_method( filePath, namespace, className, methodName, isStatic, visibility, doccomment, line, column, arguments, ) self.__index_uses(filePath, className, methodName, uses, namespace, use_statements) if block[2] == "function" and block[4] != None: tokenIndex = block[3] functionName = block[4] doccomment = block[5] arguments = block[6] uses = block[7] line = tokens[tokenIndex][2] column = tokens[tokenIndex][3] self._storage.add_function(filePath, namespace, functionName, doccomment, line, column, arguments) self.__index_uses(filePath, "", functionName, uses, namespace, use_statements) for constantIndex in constants: constantName = tokens[constantIndex + 2][1] constantLine = tokens[constantIndex][2] constantColumn = tokens[constantIndex][3] self._storage.add_constant(filePath, constantName, constantLine, constantColumn) def _unindex_phpfile(self, filePath): self._storage.removeFile(filePath) def __index_uses(self, filePath, className, containingName, uses, namespace, use_statements={}): if className in use_statements: className = use_statements[className] if len(className) > 0 and className[0] != "\\": className = namespace + "\\" + className for line, column, usedName, useType in uses: if useType == "method": self._storage.add_method_use(usedName, filePath, line, column, className, containingName) elif useType == "member": self._storage.add_member_use(usedName, filePath, line, column, className, containingName) elif useType == "function": self._storage.add_function_use(usedName, filePath, line, column, className, containingName) elif useType == "class": if usedName in use_statements: usedName = use_statements[usedName] if len(usedName) > 0 and usedName[0] != "\\": usedName = namespace + "\\" + usedName self._storage.add_class_use(usedName, filePath, line, column, className, containingName) elif useType == "constant": self._storage.add_constant_use(usedName, filePath, line, column, className, containingName)