class InitCrossReferenceGenerator(object): def __init__(self): self.crossRef = CrossReference() @property def crossReference(self): return self.crossRef def parsePercentRoutineMappingFile(self, mappingFile): result = csv.DictReader(open(mappingFile, "rb")) for row in result: self.crossRef.addPercentRoutineMapping(row['Name'], row['Source'], row['Package']) def parsePackagesFile(self, packageFilename): result = csv.DictReader(open(packageFilename, 'rb')) crossRef = self.crossRef currentPackage = None index = 0 for row in result: packageName = row['Directory Name'] if len(packageName) > 0: currentPackage = crossRef.getPackageByName(packageName) if not currentPackage: logger.debug("Package [%s] not found" % packageName) crossRef.addPackageByName(packageName) currentPackage = crossRef.getPackageByName(packageName) currentPackage.setOriginalName(row['Package Name']) vdlId = row['VDL ID'] if vdlId and len(vdlId): currentPackage.setDocLink(getVDLHttpLinkByID(vdlId)) else: if not currentPackage: logger.warn("row is not under any package: %s" % row) continue if len(row['Prefixes']): currentPackage.addNamespace(row['Prefixes']) if len(row['Globals']): currentPackage.addGlobalNamespace(row['Globals']) logger.info("Total # of Packages is %d" % (len(crossRef.getAllPackages()))) def parsePlatformDependentRoutineFile(self, routineCSVFile): routineFile = open(routineCSVFile, "rb") sniffer = csv.Sniffer() dialect = sniffer.sniff(routineFile.read(1024)) routineFile.seek(0) hasHeader = sniffer.has_header(routineFile.read(1024)) routineFile.seek(0) result = csv.reader(routineFile, dialect) currentName = "" routineDict = dict() crossRef = self.crossRef index = 0 for line in result: if hasHeader and index == 0: index += 1 continue if len(line[0]) > 0: currentName = line[0] if line[0] not in routineDict: routineDict[currentName] = [] routineDict[currentName].append(line[-1]) routineDict[currentName].append([line[1], line[2]]) for (routineName, mappingList) in routineDict.iteritems(): crossRef.addPlatformDependentRoutineMapping( routineName, mappingList[0], mappingList[1:]) #=============================================================================== # Find all globals by source zwr and package.csv files version v2 #=============================================================================== def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): logger.info("Package: %s is new" % packageName) crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = open(file, 'r') lineNo = 0 fileName = os.path.basename(file) result = re.search("(?P<fileNo>^[0-9.]+)(-1)?\+(?P<des>.*)\.zwr$", fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = re.search("(?P<namespace>^[^.]+)\.zwr$", fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ", '') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo == 1: assert re.search('ZWR', line.strip()) if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 logger.debug("globalName: %s, Des: %s, fileNo: %s, package: %s" % (globalName, globalDes, fileNo, packageName)) if len(fileNo) == 0: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError, es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error( "Error, duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close() logger.info( "Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d" % (len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet))) sortedKeyList = sorted( allGlobals.keys(), key=lambda item: float(allGlobals[item].getFileNo())) for key in sortedKeyList: globalVar = allGlobals[key] # fix the uncategoried item if globalVar.getFileNo() in fileNoPackageMappingDict: globalVar.setPackage(allPackages[fileNoPackageMappingDict[ globalVar.getFileNo()]]) crossReference.addGlobalToPackage(globalVar, globalVar.getPackage().getName())
class InitCrossReferenceGenerator(object): def __init__(self): self.crossRef = CrossReference() @property def crossReference(self): return self.crossRef def parsePercentRoutineMappingFile(self, mappingFile): result = csv.DictReader(open(mappingFile, "rb")) for row in result: self.crossRef.addPercentRoutineMapping(row['Name'], row['Source'], row['Package']) def parsePackagesFile(self, packageFilename): result = csv.DictReader(open(packageFilename, 'rb')) crossRef = self.crossRef currentPackage = None index = 0 for row in result: packageName = row['Directory Name'] if packageName: currentPackage = crossRef.getPackageByName(packageName) if not currentPackage: crossRef.addPackageByName(packageName) currentPackage = crossRef.getPackageByName(packageName) currentPackage.setOriginalName(row['Package Name']) vdlId = row['VDL ID'] if vdlId and len(vdlId): currentPackage.setDocLink(getVDLHttpLinkByID(vdlId)) else: if not currentPackage: logger.warn("row is not under any package: %s" % row) continue if len(row['Prefixes']): currentPackage.addNamespace(row['Prefixes']) if len(row['Globals']): currentPackage.addGlobalNamespace(row['Globals']) logger.info("Total # of Packages is %d" % (len(crossRef.getAllPackages()))) def parsePlatformDependentRoutineFile(self, routineCSVFile): routineFile = open(routineCSVFile, "rb") sniffer = csv.Sniffer() dialect = sniffer.sniff(routineFile.read(1024)) routineFile.seek(0) hasHeader = sniffer.has_header(routineFile.read(1024)) routineFile.seek(0) result = csv.reader(routineFile, dialect) currentName = "" routineDict = dict() crossRef = self.crossRef index = 0 for line in result: if hasHeader and index == 0: index += 1 continue if line[0]: currentName = line[0] if line[0] not in routineDict: routineDict[currentName] = [] routineDict[currentName].append(line[-1]) routineDict[currentName].append([line[1], line[2]]) for (routineName, mappingList) in routineDict.iteritems(): crossRef.addPlatformDependentRoutineMapping(routineName, mappingList[0], mappingList[1:]) #=============================================================================== # Find all globals by source zwr and package.csv files version v2 #=============================================================================== def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = open(file, 'r') lineNo = 0 fileName = os.path.basename(file) result = ZWR_FILENO_REGEX.search(fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = ZWR_NAMESPACE_REGEX.search(fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ",'') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 if not fileNo: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError, es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error("Duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close() logger.info("Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d" % (len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet))) sortedKeyList = sorted(allGlobals.keys(), key=lambda item: float(allGlobals[item].getFileNo())) for key in sortedKeyList: globalVar = allGlobals[key] # fix the uncategoried item if globalVar.getFileNo() in fileNoPackageMappingDict: globalVar.setPackage(allPackages[fileNoPackageMappingDict[globalVar.getFileNo()]]) crossReference.addGlobalToPackage(globalVar, globalVar.getPackage().getName())