def __postParsing__(self, Routine, CrossReference):
routineDetail = routineTag.search(self._varName.strip())
if routineDetail:
routineName = routineDetail.group('name')
if not validRoutineName.search(routineName):
logger.warning("invalid Routine Name: %s in routine:%s, package: %s" %
(routineName, Routine, Routine.getPackage()))
return
if (routineName.startswith("%")):
CrossReference.addPercentRoutine(routineName)
# ignore mumps routine for now
if CrossReference.isMumpsRoutine(routineName):
return
# routineName=routineName[1:]
if CrossReference.routineNeedRename(routineName):
routineName = CrossReference.getRenamedRoutineName(routineName)
tag = ""
if routineDetail.group('external'):
tag += routineDetail.group('external')
if routineDetail.group('tag'):
tag += routineDetail.group('tag')
if not CrossReference.hasRoutine(routineName):
# automatically categorize the routine by the namespace
# if could not find one, assign to Uncategorized
defaultPackageName = "Uncategorized"
(namespace, package) = CrossReference.categorizeRoutineByNamespace(routineName)
if namespace and package:
defaultPackageName = package.getName()
CrossReference.addRoutineToPackageByName(routineName, defaultPackageName, False)
routine = CrossReference.getRoutineByName(routineName)
Routine.addCalledRoutines(routine, tag, self._varValue)
def checkCSVDeps(self, CrossReference, optionText, keyVal):
if CrossReference._inputTemplateDeps:
if (keyVal == "Input_Template") and (
optionText in CrossReference._inputTemplateDeps.keys()):
for entry in CrossReference._inputTemplateDeps[optionText]:
foundGlobal = CrossReference.getGlobalByFileNo(entry[3])
if foundGlobal:
self._curRoutine.addGlobalVariables(
GlobalVariable(foundGlobal.getName(), "", "RJ"))
if CrossReference._sortTemplateDeps:
if (keyVal == "Sort_Template") and (
optionText in CrossReference._sortTemplateDeps.keys()):
for entry in CrossReference._sortTemplateDeps[optionText]:
foundGlobal = CrossReference.getGlobalByFileNo(entry[3])
if foundGlobal:
self._curRoutine.addGlobalVariables(
GlobalVariable(foundGlobal.getName(), "", "RJ"))
if CrossReference._printTemplateDeps:
if (keyVal == "Print_Template") and (
optionText in CrossReference._printTemplateDeps.keys()):
for entry in CrossReference._printTemplateDeps[optionText]:
foundGlobal = CrossReference.getGlobalByFileNo(entry[3])
if foundGlobal:
self._curRoutine.addGlobalVariables(
GlobalVariable(foundGlobal.getName(), "", "RJ"))
def __postParsing__(self, Routine, CrossReference):
routineDetail = routineTag.search(self._varName.strip())
if routineDetail:
routineName = routineDetail.group('name')
if not validRoutineName.search(routineName):
logger.warning(
"invalid Routine Name: %s in routine:%s, package: %s" %
(routineName, Routine, Routine.getPackage()))
return
if (routineName.startswith("%")):
CrossReference.addPercentRoutine(routineName)
# ignore mumps routine for now
if CrossReference.isMumpsRoutine(routineName):
return
# routineName=routineName[1:]
if CrossReference.routineNeedRename(routineName):
routineName = CrossReference.getRenamedRoutineName(routineName)
tag = ""
if routineDetail.group('external'):
tag += routineDetail.group('external')
if routineDetail.group('tag'):
tag += routineDetail.group('tag')
if not CrossReference.hasRoutine(routineName):
# automatically categorize the routine by the namespace
# if could not find one, assign to Uncategorized
defaultPackageName = "Uncategorized"
(namespace, package
) = CrossReference.categorizeRoutineByNamespace(routineName)
if namespace and package:
defaultPackageName = package.getName()
CrossReference.addRoutineToPackageByName(
routineName, defaultPackageName, False)
routine = CrossReference.getRoutineByName(routineName)
Routine.addCalledRoutines(routine, tag, self._varValue)
def __addVarToRoutine__(self, Routine, CrossReference):
globalVar = CrossReference.getGlobalByName(self._varName)
if not globalVar:
# this is to fix a problem with the name convention of a top level global
# like ICD9 can be referred as eith ICD9 or ICD9(
altName = getAlternateGlobalName(self._varName)
globalVar = CrossReference.getGlobalByName(altName)
if globalVar:
self._varName = altName
Routine.addGlobalVariables(
GlobalVariable(self._varName, self._varPrefix, self._varValue))
def __addVarToRoutine__(self, Routine, CrossReference):
globalVar = CrossReference.getGlobalByName(self._varName)
if not globalVar:
# this is to fix a problem with the name convention of a top level global
# like ICD9 can be referred as eith ICD9 or ICD9(
altName = getAlternateGlobalName(self._varName)
globalVar = CrossReference.getGlobalByName(altName)
if globalVar:
logger.debug("Changing global name from %s to %s" % (self._varName, altName))
self._varName = altName
Routine.addGlobalVariables(GlobalVariable(self._varName,
self._varPrefix,
self._varValue))
def __postParsing__(self, Routine, CrossReference):
globalVar = CrossReference.getGlobalByName(self._varName)
if not globalVar:
globalVar = CrossReference.addNonFileManGlobalByName(self._varName)
routineName = Routine.getName()
# case to handle the platform dependent routines
if CrossReference.isPlatformDependentRoutineByName(routineName):
genericRoutine = CrossReference.getGenericPlatformDepRoutineByName(routineName)
assert genericRoutine
globalVar.addReferencedRoutine(genericRoutine)
genericRoutine.addReferredGlobal(globalVar)
else:
globalVar.addReferencedRoutine(Routine)
Routine.addReferredGlobal(globalVar)
def __postParsing__(self, Routine, CrossReference):
globalVar = CrossReference.getGlobalByName(self._varName)
if not globalVar:
globalVar = CrossReference.addNonFileManGlobalByName(self._varName)
routineName = Routine.getName()
# case to handle the platform dependent routines
if CrossReference.isPlatformDependentRoutineByName(routineName):
genericRoutine = CrossReference.getGenericPlatformDepRoutineByName(routineName)
assert genericRoutine
globalVar.addReferencedRoutine(genericRoutine)
genericRoutine.addReferredGlobal(globalVar)
else:
globalVar.addReferencedRoutine(Routine)
Routine.addReferredGlobal(globalVar)
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference):
logger.debug("Current Type is [%s]" % fType)
result = self.UNDEFINED_POINTER.search(fType)
if result:
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
return
result = self.POINTER_TO_REGEX.search(fType)
if result:
fileNo = result.group('File')
filePointedTo = CrossReference.getGlobalByFileNo(fileNo)
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
if not filePointedTo:
logger.error("Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line))
else:
self._field.setPointedToFile(filePointedTo)
return
# deal with file pointer to subFiles
result = self.SUBFILE_REGEX.search(fType)
if result:
# create a field for sub file type
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation)
fileNo = result.group('File')
logger.debug("Pointer to subFile %s" % fileNo)
subFile = Global.getSubFileByFileNo(fileNo)
if not subFile: # this is a new subfile
subFile = FileManFile(fileNo, fName, self._curFile)
self._curFile.addFileManSubFile(subFile)
logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo()))
if self._isSubFile:
Global.addFileManSubFile(subFile)
self._field.setPointedToSubFile(subFile)
CrossReference.addFileManSubFile(subFile)
return
for (key, value) in self.StringTypeMappingDict.iteritems():
if fType.startswith(key):
self._field = FileManFieldFactory.createField(fieldNo, fName, value, fLocation)
break
if not self._field:
# double check the loc and type
if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX:
fType = line[self.MAXIMIUM_TYPE_START_INDEX:]
if fLocation:
fLocation = line[line.find(fLocation):self.MAXIMIUM_TYPE_START_INDEX]
logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation))
self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference)
assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference):
result = UNDEFINED_POINTER.search(fType)
if result:
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
return
result = POINTER_TO_REGEX.search(fType)
if result:
fileNo = result.group('File')
filePointedTo = CrossReference.getGlobalByFileNo(fileNo)
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER,
fLocation)
if not filePointedTo:
logger.warning("Could not find file pointed to [%s], [%s], line:[%s]" %
(fileNo, self._curFile, line))
else:
self._field.setPointedToFile(filePointedTo)
return
# deal with file pointer to subFiles
result = SUBFILE_REGEX.search(fType)
if result:
# create a field for sub file type
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_SUBFILE_POINTER,
fLocation)
fileNo = result.group('File')
subFile = Global.getSubFileByFileNo(fileNo)
if not subFile: # this is a new subfile
subFile = FileManFile(fileNo, fName, self._curFile)
self._curFile.addFileManSubFile(subFile)
if self._isSubFile:
Global.addFileManSubFile(subFile)
self._field.setPointedToSubFile(subFile)
CrossReference.addFileManSubFile(subFile)
return
for (key, value) in iteritems(self.StringTypeMappingDict):
if fType.startswith(key):
self._field = FileManFieldFactory.createField(fieldNo, fName, value, fLocation)
break
if not self._field:
# double check the loc and type
if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX:
fType = line[self.MAXIMIUM_TYPE_START_INDEX:]
if fLocation:
fLocation = line[line.find(fLocation):self.MAXIMIUM_TYPE_START_INDEX]
self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference)
assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def loadCrossRefFromMongoDB():
conn = pymongo.Connection()
vistadb = conn.vista
packageTable = vistadb.packages
packages = packageTable.find()
crossRef = CrossReference()
packageDecoder = PackageJSONDecoder(crossRef)
for packageJson in packages:
logger.info("decoding Package: %s" % packageJson['name'])
packageDecoder.decodePackage(packageJson)
routinesTable = vistadb.routines
routines = routinesTable.find()
routineDecoder = RoutineJSONDecode(crossRef)
for routineJson in routines:
logger.info("decoding Routine: %s" % routineJson['name'])
routineDecoder.decodeRoutine(routineJson)
fileManFileDecoder = FileManFileDecode(crossRef)
globalsTable = vistadb.globals
globalFiles = globalsTable.find()
for globalFileJson in globalFiles:
logger.info("decoding global: %s" % globalFileJson['name'])
fileManFileDecoder.decodeGlobal(globalFileJson)
subFilesTable = vistadb.subfiles
subFiles = subFilesTable.find()
for subFileJson in subFiles:
logger.info("decoding subfile: %s" % subFileJson['file_no'])
fileManFileDecoder.decodeSubFile(subFileJson)
return crossRef
def parseLine(self, line, Global, CrossReference):
assert self._global
strippedLine = line.rstrip(" ")
if len(strippedLine) == 0:
return
value = strippedLine[self.POINTED_TO_BY_VALUE_INDEX:]
logger.debug("Parsing line [%s]" % value)
result = self.POINTED_TO_BY_VALUE.search(value)
if result:
fileManNo = result.group("FileNo")
fieldNo = result.group('fieldNo')
subFileNo = result.group('subFieldNo')
logger.debug("File # %s, field # %s, sub-field # %s" %
(fileManNo, fieldNo, subFileNo))
pointedByGlobal = CrossReference.getGlobalByFileNo(fileManNo)
if pointedByGlobal:
self._global.addPointedToByFile(pointedByGlobal, fieldNo,
subFileNo)
logger.debug("added global to pointed list: %s, %s, %s" %
(fileManNo, fieldNo, subFileNo))
else:
logger.warning("Could not find global based on %s, %s" %
(fileManNo, result.group("Name")))
else:
logger.error(
"Could not parse pointer reference [%s] in file [%s]" %
(line, self._global.getFileNo()))
def __parsingVariablePointer__(self, Global, CrossReference):
index, fileList, found = 0, None, False
indentValue = self.__getDefaultIndentLevel__(self._curFile,
self.DEFAULT_NAME_INDENT)
for index in range(len(self._lines)):
if not found:
if re.search(
"^ {%d,%d}FILE ORDER PREFIX LAYGO MESSAGE$" %
(self.DEFAULT_NAME_INDENT, indentValue),
self._lines[index]):
found = True
continue
else:
if re.search("^ {%d,}$" % indentValue, self._lines[index]):
break
else:
result = re.search("^ +(?P<File>[0-9\.]+) +",
self._lines[index])
if result:
filePointedTo = CrossReference.getGlobalByFileNo(
result.group('File'))
if not filePointedTo:
# log an error for now, will handle this case later
logger.error(
"INVALID File! File is %s, Global is %s" %
(result.group('File'), Global))
continue
if not fileList: fileList = []
fileList.append(filePointedTo)
self._field.setPointedToFiles(fileList)
def parseLine(self, line, Global, CrossReference):
assert self._global
strippedLine = line.rstrip(" ")
if not strippedLine:
return
value = strippedLine[self.POINTED_TO_BY_VALUE_INDEX:]
result = POINTED_TO_BY_VALUE_REGEX.search(value)
if result:
fileManNo = result.group("FileNo")
fieldNo = result.group('fieldNo')
subFileNo = result.group('subFieldNo')
pointedByGlobal = CrossReference.getGlobalByFileNo(fileManNo)
if pointedByGlobal:
self._global.addPointedToByFile(pointedByGlobal, fieldNo, subFileNo)
else:
logger.warning("Could not find global based on %s, %s" %
(fileManNo, result.group("Name")))
else:
logger.error("Could not parse pointer reference [%s] in file [%s]" % (line, self._global.getFileNo()))
def parseLine(self, line, Global, CrossReference):
assert self._global
strippedLine = line.rstrip(" ")
if len(strippedLine) == 0:
return
value = strippedLine[self.POINTED_TO_BY_VALUE_INDEX:]
logger.debug("Parsing line [%s]" % value)
result = self.POINTED_TO_BY_VALUE.search(value)
if result:
fileManNo = result.group("FileNo")
fieldNo = result.group('fieldNo')
subFileNo = result.group('subFieldNo')
logger.debug("File # %s, field # %s, sub-field # %s" % (fileManNo, fieldNo, subFileNo))
pointedByGlobal = CrossReference.getGlobalByFileNo(fileManNo)
if pointedByGlobal:
self._global.addPointedToByFile(pointedByGlobal, fieldNo, subFileNo)
logger.debug("added global to pointed list: %s, %s, %s" %
(fileManNo, fieldNo, subFileNo))
else:
logger.warning("Could not find global based on %s, %s" %
(fileManNo, result.group("Name")))
else:
logger.error("Could not parse pointer reference [%s] in file [%s]" % (line, self._global.getFileNo()))
def __parsingVariablePointer__(self, Global, CrossReference):
index, fileList, found = 0, None, False
indentValue = self.__getDefaultIndentLevel__(self._curFile,
self.DEFAULT_NAME_INDENT)
for index in range(len(self._lines)):
if not found:
if re.search("^ {%d,%d}FILE ORDER PREFIX LAYGO MESSAGE$" % (self.DEFAULT_NAME_INDENT, indentValue),
self._lines[index]):
found = True
continue
else:
if re.search("^ {%d,}$" % indentValue, self._lines[index]):
break
else:
result = FILE_REGEX.search(self._lines[index])
if result:
filePointedTo = CrossReference.getGlobalByFileNo(result.group('File'))
if not filePointedTo:
# log an error for now, will handle this case later
logger.error("INVALID File! File is %s, Global is %s" % (result.group('File'), Global))
continue
if not fileList: fileList = []
fileList.append(filePointedTo)
self._field.setPointedToFiles(fileList)
def __init__(self): self.crossRef = CrossReference()
def __init__(self):
self.crossRef = CrossReference()
class InitCrossReferenceGenerator(object):
def __init__(self):
self.crossRef = CrossReference()
@property
def crossReference(self):
return self.crossRef
def parsePercentRoutineMappingFile(self, mappingFile):
result = csv.DictReader(open(mappingFile, "rb"))
for row in result:
self.crossRef.addPercentRoutineMapping(row['Name'], row['Source'],
row['Package'])
def parsePackagesFile(self, packageFilename):
result = csv.DictReader(open(packageFilename, 'rb'))
crossRef = self.crossRef
currentPackage = None
index = 0
for row in result:
packageName = row['Directory Name']
if len(packageName) > 0:
currentPackage = crossRef.getPackageByName(packageName)
if not currentPackage:
logger.debug("Package [%s] not found" % packageName)
crossRef.addPackageByName(packageName)
currentPackage = crossRef.getPackageByName(packageName)
currentPackage.setOriginalName(row['Package Name'])
vdlId = row['VDL ID']
if vdlId and len(vdlId):
currentPackage.setDocLink(getVDLHttpLinkByID(vdlId))
else:
if not currentPackage:
logger.warn("row is not under any package: %s" % row)
continue
if len(row['Prefixes']):
currentPackage.addNamespace(row['Prefixes'])
if len(row['Globals']):
currentPackage.addGlobalNamespace(row['Globals'])
logger.info("Total # of Packages is %d" %
(len(crossRef.getAllPackages())))
def parsePlatformDependentRoutineFile(self, routineCSVFile):
routineFile = open(routineCSVFile, "rb")
sniffer = csv.Sniffer()
dialect = sniffer.sniff(routineFile.read(1024))
routineFile.seek(0)
hasHeader = sniffer.has_header(routineFile.read(1024))
routineFile.seek(0)
result = csv.reader(routineFile, dialect)
currentName = ""
routineDict = dict()
crossRef = self.crossRef
index = 0
for line in result:
if hasHeader and index == 0:
index += 1
continue
if len(line[0]) > 0:
currentName = line[0]
if line[0] not in routineDict:
routineDict[currentName] = []
routineDict[currentName].append(line[-1])
routineDict[currentName].append([line[1], line[2]])
for (routineName, mappingList) in routineDict.iteritems():
crossRef.addPlatformDependentRoutineMapping(
routineName, mappingList[0], mappingList[1:])
#===============================================================================
# Find all globals by source zwr and package.csv files version v2
#===============================================================================
def findGlobalsBySourceV2(self, dirName, pattern):
searchFiles = glob.glob(os.path.join(dirName, pattern))
logger.info("Total Search Files are %d " % len(searchFiles))
crossReference = self.crossRef
allGlobals = crossReference.getAllGlobals()
allPackages = crossReference.getAllPackages()
skipFile = []
fileNoSet = set()
for file in searchFiles:
packageName = os.path.dirname(file)
packageName = packageName[packageName.index("Packages") +
9:packageName.index("Globals") - 1]
if not crossReference.hasPackage(packageName):
logger.info("Package: %s is new" % packageName)
crossReference.addPackageByName(packageName)
package = allPackages.get(packageName)
zwrFile = open(file, 'r')
lineNo = 0
fileName = os.path.basename(file)
result = re.search("(?P<fileNo>^[0-9.]+)(-1)?\+(?P<des>.*)\.zwr$",
fileName)
if result:
fileNo = result.group('fileNo')
if fileNo.startswith('0'): fileNo = fileNo[1:]
globalDes = result.group('des')
else:
result = re.search("(?P<namespace>^[^.]+)\.zwr$", fileName)
if result:
namespace = result.group('namespace')
# package.addGlobalNamespace(namespace)
continue
else:
continue
globalName = "" # find out the global name by parsing the global file
logger.debug("Parsing file: %s" % file)
for line in zwrFile:
if lineNo == 0:
globalDes = line.strip()
# Removing the extra text in the header of the ZWR file
# to tell if it needs to be added or skipped
globalDes = globalDes.replace("OSEHRA ZGO Export: ", '')
if globalDes.startswith("^"):
logger.info("No Description: Skip this file: %s" %
file)
skipFile.append(file)
namespace = globalDes[1:]
package.addGlobalNamespace(namespace)
break
if lineNo == 1:
assert re.search('ZWR', line.strip())
if lineNo >= 2:
info = line.strip().split('=')
globalName = info[0]
detail = info[1].strip("\"")
if globalName.find(',') > 0:
result = globalName.split(',')
if len(result) == 2 and result[1] == "0)":
globalName = result[0]
break
elif globalName.endswith("(0)"):
globalName = globalName.split('(')[0]
break
else:
continue
lineNo = lineNo + 1
logger.debug("globalName: %s, Des: %s, fileNo: %s, package: %s" %
(globalName, globalDes, fileNo, packageName))
if len(fileNo) == 0:
if file not in skipFile:
logger.warn("Warning: No FileNo found for file %s" % file)
continue
globalVar = Global(globalName, fileNo, globalDes,
allPackages.get(packageName))
try:
fileNum = float(globalVar.getFileNo())
except ValueError, es:
logger.error("error: %s, globalVar:%s file %s" %
(es, globalVar, file))
continue
# crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
if globalVar.getName() not in allGlobals:
allGlobals[globalVar.getName()] = globalVar
if fileNo not in fileNoSet:
fileNoSet.add(fileNo)
else:
logger.error(
"Error, duplicated file No [%s,%s,%s,%s] file:%s " %
(fileNo, globalName, globalDes, packageName, file))
zwrFile.close()
logger.info(
"Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d"
%
(len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet)))
sortedKeyList = sorted(
allGlobals.keys(),
key=lambda item: float(allGlobals[item].getFileNo()))
for key in sortedKeyList:
globalVar = allGlobals[key]
# fix the uncategoried item
if globalVar.getFileNo() in fileNoPackageMappingDict:
globalVar.setPackage(allPackages[fileNoPackageMappingDict[
globalVar.getFileNo()]])
crossReference.addGlobalToPackage(globalVar,
globalVar.getPackage().getName())
def __init__(self):
self.crossRef = CrossReference()
self.root = Element("CrossReference", version="1.0")
class CrossRefXMLEncoder:
def __init__(self):
self.crossRef = CrossReference()
self.root = Element("CrossReference", version="1.0")
def __init__(self, crossRef):
self.crossRef = crossRef
self.root = Element("CrossReference", version="1.0")
def getCrossReference(self):
return self.crossRef
def outputRoutineVariables(self, parent, variables, tag):
variablesElement = SubElement(parent, tag)
for var in variables:
varElement = SubElement(variablesElement, "Var",
name=var.getName(),
EK=str(var.getNotKilledExp()),
N=str(var.getNewed()),
C=str(var.getChanged()),
K=str(var.getKilled()))
def outputRoutineCalledRoutines(self, parent, calledRoutines):
calledRoutinesElement = SubElement(parent, "CalledRoutines")
for callInfo in calledRoutines:
varElement = SubElement(calledRoutinesElement, "Routine",
name=callInfo.getCalledRoutine().getName())
def outputRoutinesAsXML(self, parent):
allRoutineElement = SubElement(parent, "Routines")
allRoutines = self.crossRef.getAllRoutines()
for routine in allRoutines.values():
packageName = ""
package = routine.getPackage()
if package:
packageName = package.getName()
routineElement = SubElement(allRoutineElement, "Routine",
name=routine.getName(),
package=packageName)
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"LocalVariables")
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"GlobalVariables")
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"NakedGlobals")
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"MarkedItems")
self.outputRoutineCalledRoutines(routineElement,
routine.getCalledRoutines())
def outputOrphanRoutinesAsXML(self, parent):
routinesElement = SubElement(parent, "OrphanRoutines")
for routineName in self.crossRef.getOrphanRoutines():
routineElement = SubElement(routinesElement, "Routine", name=routineName)
def outputPackagesAsXML(self, parent):
allPackageElement = SubElement(parent, "Packages")
allPackages = self.crossRef.getAllPackages()
for packageName in allPackages.iterkeys():
packageElement = SubElement(allPackageElement, "Package",
name=packageName,
total="%d" % len(allPackages[packageName].getAllRoutines()))
def outputRoutinesAsPlainLog(self, outputFile):
outputFile.write("Total Routines, %s" % len(allRoutines))
def outputPackagesAsPlainLog(self, outputFile):
allPackages = self.crossRef.getAllPackages()
outputFile.write("Total Packages, %d" % len(allPackages))
for packageName, package in allPackages.iteritems():
outputFile.write("Package,%s, Total,%d" % (packageName, len(package.getAllRoutines())))
def outputAsXML(self, outputFile):
self.outputPackagesAsXML(self.root)
self.outputRoutinesAsXML(self.root)
self.outputOrphanRoutinesAsXML(self.root)
fileHandle = open(outputFile, 'w')
ElementTree(self.root).write(fileHandle, "utf-8")
def outputAsPlainLog(self, outputFile):
fileHandle = open(outputFile, 'w')
self.outputPackagesAsPlainLog(fileHandle)
self.outputRoutinesAsPlainLog(fileHandle)
self.outputOrphanRoutinesAsPlainLog(fileHandle)
def loadFromXML(self, inputFile):
pass
def __init__(self):
logger.progress("Create cross reference generator...")
self.crossRef = CrossReference()
class CrossRefXMLEncoder:
def __init__(self):
self.crossRef = CrossReference()
self.root = Element("CrossReference", version="1.0")
def __init__(self, crossRef):
self.crossRef = crossRef
self.root = Element("CrossReference", version="1.0")
def getCrossReference(self):
return self.crossRef
def outputRoutineVariables(self, parent, variables, tag):
variablesElement = SubElement(parent, tag)
for var in variables:
varElement = SubElement(variablesElement,
"Var",
name=var.getName(),
EK=str(var.getNotKilledExp()),
N=str(var.getNewed()),
C=str(var.getChanged()),
K=str(var.getKilled()))
def outputRoutineCalledRoutines(self, parent, calledRoutines):
calledRoutinesElement = SubElement(parent, "CalledRoutines")
for callInfo in calledRoutines:
varElement = SubElement(calledRoutinesElement,
"Routine",
name=callInfo.getCalledRoutine().getName())
def outputRoutinesAsXML(self, parent):
allRoutineElement = SubElement(parent, "Routines")
allRoutines = self.crossRef.getAllRoutines()
for routine in allRoutines.values():
packageName = ""
package = routine.getPackage()
if package:
packageName = package.getName()
routineElement = SubElement(allRoutineElement,
"Routine",
name=routine.getName(),
package=packageName)
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"LocalVariables")
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"GlobalVariables")
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"NakedGlobals")
self.outputRoutineVariables(routineElement,
routine.getLocalVariables(),
"MarkedItems")
self.outputRoutineCalledRoutines(routineElement,
routine.getCalledRoutines())
def outputOrphanRoutinesAsXML(self, parent):
routinesElement = SubElement(parent, "OrphanRoutines")
for routineName in self.crossRef.getOrphanRoutines():
routineElement = SubElement(routinesElement,
"Routine",
name=routineName)
def outputPackagesAsXML(self, parent):
allPackageElement = SubElement(parent, "Packages")
allPackages = self.crossRef.getAllPackages()
for packageName in allPackages.iterkeys():
packageElement = SubElement(
allPackageElement,
"Package",
name=packageName,
total="%d" % len(allPackages[packageName].getAllRoutines()))
def outputRoutinesAsPlainLog(self, outputFile):
outputFile.write("Total Routines, %s" % len(allRoutines))
def outputPackagesAsPlainLog(self, outputFile):
allPackages = self.crossRef.getAllPackages()
outputFile.write("Total Packages, %d" % len(allPackages))
for packageName, package in allPackages.iteritems():
outputFile.write("Package,%s, Total,%d" %
(packageName, len(package.getAllRoutines())))
def outputAsXML(self, outputFile):
self.outputPackagesAsXML(self.root)
self.outputRoutinesAsXML(self.root)
self.outputOrphanRoutinesAsXML(self.root)
fileHandle = open(outputFile, 'w')
ElementTree(self.root).write(fileHandle, "utf-8")
def outputAsPlainLog(self, outputFile):
fileHandle = open(outputFile, 'w')
self.outputPackagesAsPlainLog(fileHandle)
self.outputRoutinesAsPlainLog(fileHandle)
self.outputOrphanRoutinesAsPlainLog(fileHandle)
def loadFromXML(self, inputFile):
pass
def __init__(self):
self.crossRef = CrossReference()
self.root = Element("CrossReference", version="1.0")
class InitCrossReferenceGenerator(object):
def __init__(self):
self.crossRef = CrossReference()
@property
def crossReference(self):
return self.crossRef
def parsePercentRoutineMappingFile(self, mappingFile):
result = csv.DictReader(open(mappingFile, "rb"))
for row in result:
self.crossRef.addPercentRoutineMapping(row['Name'],
row['Source'],
row['Package'])
def parsePackagesFile(self, packageFilename):
result = csv.DictReader(open(packageFilename, 'rb'))
crossRef = self.crossRef
currentPackage = None
index = 0
for row in result:
packageName = row['Directory Name']
if packageName:
currentPackage = crossRef.getPackageByName(packageName)
if not currentPackage:
crossRef.addPackageByName(packageName)
currentPackage = crossRef.getPackageByName(packageName)
currentPackage.setOriginalName(row['Package Name'])
vdlId = row['VDL ID']
if vdlId and len(vdlId):
currentPackage.setDocLink(getVDLHttpLinkByID(vdlId))
else:
if not currentPackage:
logger.warn("row is not under any package: %s" % row)
continue
if len(row['Prefixes']):
currentPackage.addNamespace(row['Prefixes'])
if len(row['Globals']):
currentPackage.addGlobalNamespace(row['Globals'])
logger.info("Total # of Packages is %d" % (len(crossRef.getAllPackages())))
def parsePlatformDependentRoutineFile(self, routineCSVFile):
routineFile = open(routineCSVFile, "rb")
sniffer = csv.Sniffer()
dialect = sniffer.sniff(routineFile.read(1024))
routineFile.seek(0)
hasHeader = sniffer.has_header(routineFile.read(1024))
routineFile.seek(0)
result = csv.reader(routineFile, dialect)
currentName = ""
routineDict = dict()
crossRef = self.crossRef
index = 0
for line in result:
if hasHeader and index == 0:
index += 1
continue
if line[0]:
currentName = line[0]
if line[0] not in routineDict:
routineDict[currentName] = []
routineDict[currentName].append(line[-1])
routineDict[currentName].append([line[1], line[2]])
for (routineName, mappingList) in routineDict.iteritems():
crossRef.addPlatformDependentRoutineMapping(routineName,
mappingList[0],
mappingList[1:])
#===============================================================================
# Find all globals by source zwr and package.csv files version v2
#===============================================================================
def findGlobalsBySourceV2(self, dirName, pattern):
searchFiles = glob.glob(os.path.join(dirName, pattern))
logger.info("Total Search Files are %d " % len(searchFiles))
crossReference = self.crossRef
allGlobals = crossReference.getAllGlobals()
allPackages = crossReference.getAllPackages()
skipFile = []
fileNoSet = set()
for file in searchFiles:
packageName = os.path.dirname(file)
packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1]
if not crossReference.hasPackage(packageName):
crossReference.addPackageByName(packageName)
package = allPackages.get(packageName)
zwrFile = open(file, 'r')
lineNo = 0
fileName = os.path.basename(file)
result = ZWR_FILENO_REGEX.search(fileName)
if result:
fileNo = result.group('fileNo')
if fileNo.startswith('0'): fileNo = fileNo[1:]
globalDes = result.group('des')
else:
result = ZWR_NAMESPACE_REGEX.search(fileName)
if result:
namespace = result.group('namespace')
# package.addGlobalNamespace(namespace)
continue
else:
continue
globalName = "" # find out the global name by parsing the global file
logger.debug("Parsing file: %s" % file)
for line in zwrFile:
if lineNo == 0:
globalDes = line.strip()
# Removing the extra text in the header of the ZWR file
# to tell if it needs to be added or skipped
globalDes = globalDes.replace("OSEHRA ZGO Export: ",'')
if globalDes.startswith("^"):
logger.info("No Description: Skip this file: %s" % file)
skipFile.append(file)
namespace = globalDes[1:]
package.addGlobalNamespace(namespace)
break
if lineNo >= 2:
info = line.strip().split('=')
globalName = info[0]
detail = info[1].strip("\"")
if globalName.find(',') > 0:
result = globalName.split(',')
if len(result) == 2 and result[1] == "0)":
globalName = result[0]
break
elif globalName.endswith("(0)"):
globalName = globalName.split('(')[0]
break
else:
continue
lineNo = lineNo + 1
if not fileNo:
if file not in skipFile:
logger.warn("Warning: No FileNo found for file %s" % file)
continue
globalVar = Global(globalName, fileNo, globalDes,
allPackages.get(packageName))
try:
fileNum = float(globalVar.getFileNo())
except ValueError, es:
logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file))
continue
# crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
if globalVar.getName() not in allGlobals:
allGlobals[globalVar.getName()] = globalVar
if fileNo not in fileNoSet:
fileNoSet.add(fileNo)
else:
logger.error("Duplicated file No [%s,%s,%s,%s] file:%s " %
(fileNo, globalName, globalDes, packageName, file))
zwrFile.close()
logger.info("Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d" %
(len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet)))
sortedKeyList = sorted(allGlobals.keys(),
key=lambda item: float(allGlobals[item].getFileNo()))
for key in sortedKeyList:
globalVar = allGlobals[key]
# fix the uncategoried item
if globalVar.getFileNo() in fileNoPackageMappingDict:
globalVar.setPackage(allPackages[fileNoPackageMappingDict[globalVar.getFileNo()]])
crossReference.addGlobalToPackage(globalVar,
globalVar.getPackage().getName())
