class CrossRefXMLEncoder: def __init__(self): self.crossRef = CrossReference() self.root = Element("CrossReference", version="1.0") def __init__(self, crossRef): self.crossRef = crossRef self.root = Element("CrossReference", version="1.0") def getCrossReference(self): return self.crossRef def outputRoutineVariables(self, parent, variables, tag): variablesElement = SubElement(parent, tag) for var in variables: varElement = SubElement(variablesElement, "Var", name=var.getName(), EK=str(var.getNotKilledExp()), N=str(var.getNewed()), C=str(var.getChanged()), K=str(var.getKilled())) def outputRoutineCalledRoutines(self, parent, calledRoutines): calledRoutinesElement = SubElement(parent, "CalledRoutines") for callInfo in calledRoutines: varElement = SubElement(calledRoutinesElement, "Routine", name=callInfo.getCalledRoutine().getName()) def outputRoutinesAsXML(self, parent): allRoutineElement = SubElement(parent, "Routines") allRoutines = self.crossRef.getAllRoutines() for routine in allRoutines.values(): packageName = "" package = routine.getPackage() if package: packageName = package.getName() routineElement = SubElement(allRoutineElement, "Routine", name=routine.getName(), package=packageName) self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "LocalVariables") self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "GlobalVariables") self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "NakedGlobals") self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "MarkedItems") self.outputRoutineCalledRoutines(routineElement, routine.getCalledRoutines()) def outputOrphanRoutinesAsXML(self, parent): routinesElement = SubElement(parent, "OrphanRoutines") for routineName in self.crossRef.getOrphanRoutines(): routineElement = SubElement(routinesElement, "Routine", name=routineName) def outputPackagesAsXML(self, parent): allPackageElement = SubElement(parent, "Packages") allPackages = self.crossRef.getAllPackages() for packageName in allPackages.iterkeys(): packageElement = SubElement( allPackageElement, "Package", name=packageName, total="%d" % len(allPackages[packageName].getAllRoutines())) def outputRoutinesAsPlainLog(self, outputFile): outputFile.write("Total Routines, %s" % len(allRoutines)) def outputPackagesAsPlainLog(self, outputFile): allPackages = self.crossRef.getAllPackages() outputFile.write("Total Packages, %d" % len(allPackages)) for packageName, package in allPackages.iteritems(): outputFile.write("Package,%s, Total,%d" % (packageName, len(package.getAllRoutines()))) def outputAsXML(self, outputFile): self.outputPackagesAsXML(self.root) self.outputRoutinesAsXML(self.root) self.outputOrphanRoutinesAsXML(self.root) fileHandle = open(outputFile, 'w') ElementTree(self.root).write(fileHandle, "utf-8") def outputAsPlainLog(self, outputFile): fileHandle = open(outputFile, 'w') self.outputPackagesAsPlainLog(fileHandle) self.outputRoutinesAsPlainLog(fileHandle) self.outputOrphanRoutinesAsPlainLog(fileHandle) def loadFromXML(self, inputFile): pass
class CrossRefXMLEncoder: def __init__(self): self.crossRef = CrossReference() self.root = Element("CrossReference", version="1.0") def __init__(self, crossRef): self.crossRef = crossRef self.root = Element("CrossReference", version="1.0") def getCrossReference(self): return self.crossRef def outputRoutineVariables(self, parent, variables, tag): variablesElement = SubElement(parent, tag) for var in variables: varElement = SubElement(variablesElement, "Var", name=var.getName(), EK=str(var.getNotKilledExp()), N=str(var.getNewed()), C=str(var.getChanged()), K=str(var.getKilled())) def outputRoutineCalledRoutines(self, parent, calledRoutines): calledRoutinesElement = SubElement(parent, "CalledRoutines") for callInfo in calledRoutines: varElement = SubElement(calledRoutinesElement, "Routine", name=callInfo.getCalledRoutine().getName()) def outputRoutinesAsXML(self, parent): allRoutineElement = SubElement(parent, "Routines") allRoutines = self.crossRef.getAllRoutines() for routine in allRoutines.values(): packageName = "" package = routine.getPackage() if package: packageName = package.getName() routineElement = SubElement(allRoutineElement, "Routine", name=routine.getName(), package=packageName) self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "LocalVariables") self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "GlobalVariables") self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "NakedGlobals") self.outputRoutineVariables(routineElement, routine.getLocalVariables(), "MarkedItems") self.outputRoutineCalledRoutines(routineElement, routine.getCalledRoutines()) def outputOrphanRoutinesAsXML(self, parent): routinesElement = SubElement(parent, "OrphanRoutines") for routineName in self.crossRef.getOrphanRoutines(): routineElement = SubElement(routinesElement, "Routine", name=routineName) def outputPackagesAsXML(self, parent): allPackageElement = SubElement(parent, "Packages") allPackages = self.crossRef.getAllPackages() for packageName in allPackages.iterkeys(): packageElement = SubElement(allPackageElement, "Package", name=packageName, total="%d" % len(allPackages[packageName].getAllRoutines())) def outputRoutinesAsPlainLog(self, outputFile): outputFile.write("Total Routines, %s" % len(allRoutines)) def outputPackagesAsPlainLog(self, outputFile): allPackages = self.crossRef.getAllPackages() outputFile.write("Total Packages, %d" % len(allPackages)) for packageName, package in allPackages.iteritems(): outputFile.write("Package,%s, Total,%d" % (packageName, len(package.getAllRoutines()))) def outputAsXML(self, outputFile): self.outputPackagesAsXML(self.root) self.outputRoutinesAsXML(self.root) self.outputOrphanRoutinesAsXML(self.root) fileHandle = open(outputFile, 'w') ElementTree(self.root).write(fileHandle, "utf-8") def outputAsPlainLog(self, outputFile): fileHandle = open(outputFile, 'w') self.outputPackagesAsPlainLog(fileHandle) self.outputRoutinesAsPlainLog(fileHandle) self.outputOrphanRoutinesAsPlainLog(fileHandle) def loadFromXML(self, inputFile): pass
class InitCrossReferenceGenerator(object): def __init__(self): logger.progress("Create cross reference generator...") self.crossRef = CrossReference() @property def crossReference(self): return self.crossRef def parsePercentRoutineMappingFile(self, mappingFile): result = csv.DictReader(open(mappingFile, 'r')) for row in result: self.crossRef.addPercentRoutineMapping(row['Name'], row['Source'], row['Package']) def parsePackagesFile(self, packageFilename): result = csv.DictReader(open(packageFilename, 'r')) crossRef = self.crossRef currentPackage = None index = 0 for row in result: packageName = row['Directory Name'] if packageName: currentPackage = crossRef.getPackageByName(packageName) if not currentPackage: crossRef.addPackageByName(packageName) currentPackage = crossRef.getPackageByName(packageName) currentPackage.setOriginalName(row['Package Name']) vdlId = row['VDL ID'] if vdlId and len(vdlId): currentPackage.setDocLink(getVDLHttpLinkByID(vdlId)) else: if not currentPackage: logger.warn("row is not under any package: %s" % row) continue if len(row['Prefixes']): currentPackage.addNamespace(row['Prefixes']) if len(row['Globals']): currentPackage.addGlobalNamespace(row['Globals']) logger.info("Total # of Packages is %d" % (len(crossRef.getAllPackages()))) def parsePlatformDependentRoutineFile(self, routineCSVFile): routineFile = open(routineCSVFile, 'r') sniffer = csv.Sniffer() dialect = sniffer.sniff(routineFile.read(1024)) routineFile.seek(0) hasHeader = sniffer.has_header(routineFile.read(1024)) routineFile.seek(0) result = csv.reader(routineFile, dialect) currentName = "" routineDict = dict() crossRef = self.crossRef index = 0 for line in result: if hasHeader and index == 0: index += 1 continue if line[0]: currentName = line[0] if line[0] not in routineDict: routineDict[currentName] = [] routineDict[currentName].append(line[-1]) routineDict[currentName].append([line[1], line[2]]) for (routineName, mappingList) in iteritems(routineDict): crossRef.addPlatformDependentRoutineMapping( routineName, mappingList[0], mappingList[1:]) #=============================================================================== # Find all globals by source zwr and package.csv files version v2 #=============================================================================== def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = codecs.open(file, 'r', encoding='utf-8', errors='ignore') lineNo = 0 fileName = os.path.basename(file) result = ZWR_FILENO_REGEX.search(fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = ZWR_NAMESPACE_REGEX.search(fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ", '') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 if not fileNo: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError as es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error( "Duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close() logger.info( "Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d" % (len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet))) sortedKeyList = sorted( list(allGlobals.keys()), key=lambda item: float(allGlobals[item].getFileNo())) for key in sortedKeyList: globalVar = allGlobals[key] # fix the uncategoried item if globalVar.getFileNo() in fileNoPackageMappingDict: globalVar.setPackage(allPackages[fileNoPackageMappingDict[ globalVar.getFileNo()]]) crossReference.addGlobalToPackage(globalVar, globalVar.getPackage().getName()) #=========================================================================== # find all the package name and routines by reading the repository directory #=========================================================================== def findPackagesAndRoutinesBySource(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) allRoutines = self.crossRef.getAllRoutines() allPackages = self.crossRef.getAllPackages() crossReference = self.crossRef for file in searchFiles: routineName = os.path.basename(file).split(".")[0] needRename = crossReference.routineNeedRename(routineName) if needRename: origName = routineName routineName = crossReference.getRenamedRoutineName(routineName) if crossReference.isPlatformDependentRoutineByName(routineName): continue packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Routines") - 1] crossReference.addRoutineToPackageByName(routineName, packageName) if needRename: routine = crossReference.getRoutineByName(routineName) assert (routine) routine.setOriginalName(origName) if A_ROUTINE_EX.search(routineName): pass logger.info("Total package is %d and Total Routines are %d" % (len(allPackages), len(allRoutines)))