def unitTestRoutineJSONEncoder(outputFile):
testPA = Package("TestPA")
testPB = Package("TestPB")
testPC = Package("TestPC")
calledRoutineA = Routine("TestRA", testPB)
calledRoutineB = Routine("TestRB", testPC)
testRoutine = Routine("TestRA", testPA)
localVar = LocalVariable("TSTLA", ">>", "YU+4,XU+1*")
testRoutine.addLocalVariables(localVar)
localVar = LocalVariable("TSTLB", ">>", "YU+1*,XU+3*")
testRoutine.addLocalVariables(localVar)
globalVar = GlobalVariable("TSTGA", ">>", "YU+4,XU+1*")
testRoutine.addGlobalVariables(globalVar)
nakedVar = NakedGlobal("^(20.2", None, "YU+4,XU+1")
testRoutine.addNakedGlobals(nakedVar)
labelRef = LabelReference("DD", None, "YU+4,XU+1")
testRoutine.addLabelReference(labelRef)
globalVar = Global("^TMP(\"TEST\"", None, None, None)
testRoutine.addReferredGlobal(globalVar)
globalVar = Global("^TMP($J", None, None, None)
testRoutine.addReferredGlobal(globalVar)
testRoutine.addCalledRoutines(calledRoutineA, "$$DT", "N+1,Y+3")
testRoutine.addCalledRoutines(calledRoutineB, "$$OUT", "Y+1,Z+3")
output = RoutineJSONEncoder().outputRoutine(testRoutine)
outputFile = open(outputFile, "wb")
outputFile.write(output)
outputFile.write("\n")
def __outputGlobal__(self, Global):
outputDict = self._outputDict
outputDict["name"] = Global.getName()
outputDict["package"] = Global.getPackage().getName()
outputDict[
"accessed_by_routines"] = self.__outputAccessByRoutineList__(
Global)
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line,
Global, CrossReference):
logger.debug("Current Type is [%s]" % fType)
result = self.UNDEFINED_POINTER.search(fType)
if result:
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER,
fLocation)
return
result = self.POINTER_TO_REGEX.search(fType)
if result:
fileNo = result.group('File')
filePointedTo = CrossReference.getGlobalByFileNo(fileNo)
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER,
fLocation)
if not filePointedTo:
logger.error(
"Could not find file pointed to [%s], [%s], line:[%s]" %
(fileNo, self._curFile, line))
else:
self._field.setPointedToFile(filePointedTo)
return
# deal with file pointer to subFiles
result = self.SUBFILE_REGEX.search(fType)
if result:
# create a field for sub file type
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_SUBFILE_POINTER,
fLocation)
fileNo = result.group('File')
logger.debug("Pointer to subFile %s" % fileNo)
subFile = Global.getSubFileByFileNo(fileNo)
if not subFile:
subFile = FileManFile(fileNo, fName, self._curFile)
self._curFile.addFileManSubFile(subFile)
logger.debug("Added subFile %s to File %s" %
(fileNo, self._curFile.getFileNo()))
if self._isSubFile:
Global.addFileManSubFile(subFile)
self._field.setPointedToSubFile(subFile)
return
for (key, value) in self.StringTypeMappingDict.iteritems():
if fType.startswith(key):
self._field = FileManFieldFactory.createField(
fieldNo, fName, value, fLocation)
break
if not self._field:
# double check the loc and type
if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX:
fType = line[self.MAXIMIUM_TYPE_START_INDEX:]
if fLocation:
fLocation = line[line.find(fLocation):self.
MAXIMIUM_TYPE_START_INDEX]
logger.warn("new Type is [%s], loc is [%s]" %
(fType, fLocation))
self.__createFieldByType__(fieldNo, fType, fName, fLocation,
line, Global, CrossReference)
assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (
fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def __outputIndividualFileManFile__(self, outDir, Global):
logger.info("Writing Global %s" % Global)
jsonResult = self._fileManFileEncoder.outputResult(Global)
outputFile = open(os.path.join(outDir,"Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),'wb')
outputFile.write(jsonResult)
outputFile.write("\n")
outputFile.close()
logger.info("Writing FileManFile %s" % Global.getFileNo())
outputFile = open(os.path.join(outDir,"FileManFile_%s.json" % (Global.getFileNo())),'wb')
outputFile.write(jsonResult)
outputFile.write("\n")
outputFile.close()
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference):
logger.debug("Current Type is [%s]" % fType)
result = self.UNDEFINED_POINTER.search(fType)
if result:
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
return
result = self.POINTER_TO_REGEX.search(fType)
if result:
fileNo = result.group('File')
filePointedTo = CrossReference.getGlobalByFileNo(fileNo)
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
if not filePointedTo:
logger.error("Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line))
else:
self._field.setPointedToFile(filePointedTo)
return
# deal with file pointer to subFiles
result = self.SUBFILE_REGEX.search(fType)
if result:
# create a field for sub file type
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation)
fileNo = result.group('File')
logger.debug("Pointer to subFile %s" % fileNo)
subFile = Global.getSubFileByFileNo(fileNo)
if not subFile: # this is a new subfile
subFile = FileManFile(fileNo, fName, self._curFile)
self._curFile.addFileManSubFile(subFile)
logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo()))
if self._isSubFile:
Global.addFileManSubFile(subFile)
self._field.setPointedToSubFile(subFile)
CrossReference.addFileManSubFile(subFile)
return
for (key, value) in self.StringTypeMappingDict.iteritems():
if fType.startswith(key):
self._field = FileManFieldFactory.createField(fieldNo, fName, value, fLocation)
break
if not self._field:
# double check the loc and type
if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX:
fType = line[self.MAXIMIUM_TYPE_START_INDEX:]
if fLocation:
fLocation = line[line.find(fLocation):self.MAXIMIUM_TYPE_START_INDEX]
logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation))
self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference)
assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def outputCrossRefAsJSON(self, outDir):
allRoutines = self._crossRef.getAllRoutines()
for routine in allRoutines.itervalues():
self.__outputIndividualRoutine__(outDir, routine)
allGlobals = self._crossRef.getAllGlobals()
for Global in allGlobals.itervalues():
if not Global.isFileManFile():
self.__outputIndividualGlobal__(outDir, Global)
else:
self.__outputIndividualFileManFile__(outDir, Global)
if not Global.getAllSubFiles(): continue
for subFile in Global.getAllSubFiles().itervalues():
self.__outputIndividualSubFile__(outDir, subFile)
for Package in self._crossRef.getAllPackages().itervalues():
self.__outputIndividualPackage__(outDir, Package)
def outputCrossRefAsJSON(self, outDir):
allRoutines = self._crossRef.getAllRoutines()
for routine in allRoutines.itervalues():
self.__outputIndividualRoutine__(outDir, routine)
allGlobals = self._crossRef.getAllGlobals()
for Global in allGlobals.itervalues():
if not Global.isFileManFile():
self.__outputIndividualGlobal__(outDir, Global)
else:
self.__outputIndividualFileManFile__(outDir, Global)
if not Global.getAllSubFiles(): continue
for subFile in Global.getAllSubFiles().itervalues():
self.__outputIndividualSubFile__(outDir, subFile)
for Package in self._crossRef.getAllPackages().itervalues():
self.__outputIndividualPackage__(outDir, Package)
def _generateSchema(self):
files = getKeys(self._ddRoot, float) # sort files by float value
logging.debug("Parsing files %s" % files)
for file in files:
if file not in self._allSchema:
self._allSchema[file] = Global("", file, "")
self._generateFileSchema(self._ddRoot[file], self._allSchema[file])
def __outputIndividualFileManFile__(self, outDir, Global):
logger.info("Writing Global %s" % Global)
jsonResult = self._fileManFileEncoder.outputResult(Global)
outputFile = open(
os.path.join(
outDir,
"Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),
'wb')
outputFile.write(jsonResult)
outputFile.write("\n")
outputFile.close()
logger.info("Writing FileManFile %s" % Global.getFileNo())
outputFile = open(
os.path.join(outDir, "FileManFile_%s.json" % (Global.getFileNo())),
'wb')
outputFile.write(jsonResult)
outputFile.write("\n")
outputFile.close()
def __outputIndividualGlobal__(self, outDir, Global):
logger.info("Writing Global %s" % Global)
outputFile = open(
os.path.join(
outDir,
"Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),
'wb')
outputFile.write(self._globalEncoder.outputResult(Global))
outputFile.write("\n")
outputFile.close()
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'):
files = getKeys(ddRoot, float) # sort files by float value
for file in files:
if file not in self._allSchema:
self._allSchema[file] = Global("", file, "")
self._generateFileSchema(ddRoot[file], self._allSchema[file])
self._updateMultiple()
self._sccSet = self._generateSCCSet()
return self._allSchema
def __handleAllGlobals__(self, Package, PackageJson):
crossRef = self._crossRef
for globalName in PackageJson['non_fileman_globals']:
crossRef.addGlobalToPackageByName(globalName, Package.getName())
packageName = PackageJson["name"]
package = crossRef.getPackageByName(packageName)
for fileManGlobalDict in PackageJson['fileman_globals']:
globalName = fileManGlobalDict['name']
fileNo = fileManGlobalDict['file_no']
fileName = fileManGlobalDict['file_name']
GlobalVar = Global(globalName, fileNo, fileName, package)
crossRef.addGlobalToPackage(GlobalVar, packageName)
def __outputGlobal__(self, Global):
outputDict = self._outputDict
outputDict["name"] = Global.getName()
outputDict["package"] = Global.getPackage().getName()
outputDict["accessed_by_routines"] = self.__outputAccessByRoutineList__(Global)
def __outputAccessByRoutineList__(self, Global):
outputList = [{
"package": x.getName(),
"routines": [z.getName() for z in y]
} for (x, y) in Global.getAllReferencedRoutines().iteritems()]
return outputList
def __outputIndividualGlobal__(self, outDir, Global):
logger.info("Writing Global %s" % Global)
outputFile = open(os.path.join(outDir,"Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),'wb')
outputFile.write(self._globalEncoder.outputResult(Global))
outputFile.write("\n")
outputFile.close()
def __outputAccessByRoutineList__(self, Global):
outputList =[{"package":x.getName(),"routines":[z.getName() for z in y]} for (x,y) in Global.getAllReferencedRoutines().iteritems()]
return outputList
def onSectionStart(self, line, section, Global, CrossReference):
logger.debug("[%s]" % line)
self._lines = []
result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(
line)
assert result
fileNo = result.group('FileNo')
fieldNo = result.group("FieldNo")
self._isSubFile = float(fileNo) != float(Global.getFileNo())
if self._isSubFile:
logger.debug("%s is a subfile" % fileNo)
self._curFile = Global.getSubFileByFileNo(fileNo)
assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (
fileNo, Global.getFileNo(), line)
else:
self._curFile = Global
restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
startIdent = self.DEFAULT_NAME_INDENT
#if len(fileNo) + 4 > startIdent:
# startIdent = self
defaultIdentLevel = self.__getDefaultIndentLevel__(
self._curFile, self.DEFAULT_NAME_INDENT)
if restOfLineStart > defaultIdentLevel:
logger.debug(
"FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d"
% (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
try:
floatValue = float(fieldNo)
except ValueError:
logger.error("invalid fieldNo %s" % fieldNo)
fieldNo = line[line.find(",") + 1:defaultIdentLevel]
floatValue = float(fieldNo)
restOfLine = line[line.find("," + fieldNo) + len(fieldNo) + 1:].strip()
logger.debug("Parsing [%s]" % restOfLine)
result = self.NAME_LOC_TYPE_REGEX.search(restOfLine)
fName, fType, fLocation = None, None, None
if result:
logger.debug("FileNo: %s, Field#: %s, Name: %s, Loc %s, Type %s" %
(fileNo, fieldNo, result.group('Name').rstrip(),
result.group('Loc'), result.group('Type')))
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
if fLocation == ";":
fLocation = None
fType = result.group('Type').strip()
else:
# handle three cases, 1. no location info 2. no type info 3. Both
if restOfLine.find(";") != -1: # missing type info
logger.warn("Missing Type information [%s]" % line)
result = self.NAME_LOC_REGEX.search(restOfLine)
if result:
logger.debug("Name: %s, Loc %s" %
(result.group('Name'), result.group('Loc')))
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
else:
logger.error("Could not parse [%s]" % restOfLine)
return
else: # missing location, assume at least two space seperate name and type
logger.warn("Missing location information [%s]" % line)
result = self.NAME_TYPE_REGEX.search(restOfLine)
if result:
fName = result.group('Name').strip()
fType = result.group('Type').strip()
logger.debug("Name: %s, Type %s" %
(result.group('Name'), result.group('Type')))
else:
logger.warn("Guessing Name: %s at line [%s]" %
(restOfLine.strip(), line))
stripedType = ""
if fType:
stripedType = self.__stripFieldAttributes__(fType)
if len(stripedType) > 0:
self.__createFieldByType__(fieldNo, stripedType, fName, fLocation,
line, Global, CrossReference)
else:
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
logger.debug("Add field %s to File %s" %
(fName, self._curFile.getFileNo()))
self._curFile.addFileManField(self._field)
if len(stripedType) > 0:
self.__parseFieldAttributes__(fType)
def _generateSchema(self):
files = getKeys(self._ddRoot, float) # sort files by float value
for file in files:
if file not in self._allSchema:
self._allSchema[file] = Global("", file, "")
self._generateFileSchema(self._ddRoot[file], self._allSchema[file])
def findGlobalsBySourceV2(self, dirName, pattern):
searchFiles = glob.glob(os.path.join(dirName, pattern))
logger.info("Total Search Files are %d " % len(searchFiles))
crossReference = self.crossRef
allGlobals = crossReference.getAllGlobals()
allPackages = crossReference.getAllPackages()
skipFile = []
fileNoSet = set()
for file in searchFiles:
packageName = os.path.dirname(file)
packageName = packageName[packageName.index("Packages") +
9:packageName.index("Globals") - 1]
if not crossReference.hasPackage(packageName):
logger.info("Package: %s is new" % packageName)
crossReference.addPackageByName(packageName)
package = allPackages.get(packageName)
zwrFile = open(file, 'r')
lineNo = 0
fileName = os.path.basename(file)
result = re.search("(?P<fileNo>^[0-9.]+)(-1)?\+(?P<des>.*)\.zwr$",
fileName)
if result:
fileNo = result.group('fileNo')
if fileNo.startswith('0'): fileNo = fileNo[1:]
globalDes = result.group('des')
else:
result = re.search("(?P<namespace>^[^.]+)\.zwr$", fileName)
if result:
namespace = result.group('namespace')
# package.addGlobalNamespace(namespace)
continue
else:
continue
globalName = "" # find out the global name by parsing the global file
logger.debug("Parsing file: %s" % file)
for line in zwrFile:
if lineNo == 0:
globalDes = line.strip()
# Removing the extra text in the header of the ZWR file
# to tell if it needs to be added or skipped
globalDes = globalDes.replace("OSEHRA ZGO Export: ", '')
if globalDes.startswith("^"):
logger.info("No Description: Skip this file: %s" %
file)
skipFile.append(file)
namespace = globalDes[1:]
package.addGlobalNamespace(namespace)
break
if lineNo == 1:
assert re.search('ZWR', line.strip())
if lineNo >= 2:
info = line.strip().split('=')
globalName = info[0]
detail = info[1].strip("\"")
if globalName.find(',') > 0:
result = globalName.split(',')
if len(result) == 2 and result[1] == "0)":
globalName = result[0]
break
elif globalName.endswith("(0)"):
globalName = globalName.split('(')[0]
break
else:
continue
lineNo = lineNo + 1
logger.debug("globalName: %s, Des: %s, fileNo: %s, package: %s" %
(globalName, globalDes, fileNo, packageName))
if len(fileNo) == 0:
if file not in skipFile:
logger.warn("Warning: No FileNo found for file %s" % file)
continue
globalVar = Global(globalName, fileNo, globalDes,
allPackages.get(packageName))
try:
fileNum = float(globalVar.getFileNo())
except ValueError, es:
logger.error("error: %s, globalVar:%s file %s" %
(es, globalVar, file))
continue
# crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
if globalVar.getName() not in allGlobals:
allGlobals[globalVar.getName()] = globalVar
if fileNo not in fileNoSet:
fileNoSet.add(fileNo)
else:
logger.error(
"Error, duplicated file No [%s,%s,%s,%s] file:%s " %
(fileNo, globalName, globalDes, packageName, file))
zwrFile.close()
def findGlobalsBySourceV2(self, dirName, pattern):
searchFiles = glob.glob(os.path.join(dirName, pattern))
logger.info("Total Search Files are %d " % len(searchFiles))
crossReference = self.crossRef
allGlobals = crossReference.getAllGlobals()
allPackages = crossReference.getAllPackages()
skipFile = []
fileNoSet = set()
for file in searchFiles:
packageName = os.path.dirname(file)
packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1]
if not crossReference.hasPackage(packageName):
crossReference.addPackageByName(packageName)
package = allPackages.get(packageName)
zwrFile = open(file, 'r')
lineNo = 0
fileName = os.path.basename(file)
result = ZWR_FILENO_REGEX.search(fileName)
if result:
fileNo = result.group('fileNo')
if fileNo.startswith('0'): fileNo = fileNo[1:]
globalDes = result.group('des')
else:
result = ZWR_NAMESPACE_REGEX.search(fileName)
if result:
namespace = result.group('namespace')
# package.addGlobalNamespace(namespace)
continue
else:
continue
globalName = "" # find out the global name by parsing the global file
logger.debug("Parsing file: %s" % file)
for line in zwrFile:
if lineNo == 0:
globalDes = line.strip()
# Removing the extra text in the header of the ZWR file
# to tell if it needs to be added or skipped
globalDes = globalDes.replace("OSEHRA ZGO Export: ",'')
if globalDes.startswith("^"):
logger.info("No Description: Skip this file: %s" % file)
skipFile.append(file)
namespace = globalDes[1:]
package.addGlobalNamespace(namespace)
break
if lineNo >= 2:
info = line.strip().split('=')
globalName = info[0]
detail = info[1].strip("\"")
if globalName.find(',') > 0:
result = globalName.split(',')
if len(result) == 2 and result[1] == "0)":
globalName = result[0]
break
elif globalName.endswith("(0)"):
globalName = globalName.split('(')[0]
break
else:
continue
lineNo = lineNo + 1
if not fileNo:
if file not in skipFile:
logger.warn("Warning: No FileNo found for file %s" % file)
continue
globalVar = Global(globalName, fileNo, globalDes,
allPackages.get(packageName))
try:
fileNum = float(globalVar.getFileNo())
except ValueError, es:
logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file))
continue
# crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
if globalVar.getName() not in allGlobals:
allGlobals[globalVar.getName()] = globalVar
if fileNo not in fileNoSet:
fileNoSet.add(fileNo)
else:
logger.error("Duplicated file No [%s,%s,%s,%s] file:%s " %
(fileNo, globalName, globalDes, packageName, file))
zwrFile.close()
def _generateFileZeroSchema(self):
while (len(self._zeroFiles) > 0):
file = self._zeroFiles.pop(0)
if file not in self._allSchema:
self._allSchema[file] = Global("", file, "")
self._generateFileSchema(self._ddRoot[file], self._allSchema[file])
def onSectionStart(self, line, section, Global, CrossReference):
self._lines = []
result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line)
assert result
fileNo = result.group('FileNo')
fieldNo = result.group("FieldNo")
self._isSubFile = float(fileNo) != float(Global.getFileNo())
if self._isSubFile:
self._curFile = Global.getSubFileByFileNo(fileNo)
assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line)
else:
self._curFile = Global
restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
startIdent = self.DEFAULT_NAME_INDENT
defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT)
if restOfLineStart > defaultIdentLevel:
logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" %
(fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
try:
floatValue = float(fieldNo)
except ValueError:
logger.error("invalid fieldNo %s" % fieldNo)
fieldNo = line[line.find(",")+1:defaultIdentLevel]
floatValue = float(fieldNo)
restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip()
result = NAME_LOC_TYPE_REGEX.search(restOfLine)
fName, fType, fLocation = None, None, None
if result:
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
if fLocation == ";":
fLocation = None
fType = result.group('Type').strip()
else:
# handle three cases, 1. no location info 2. no type info 3. Both
if restOfLine.find(";") != -1: # missing type info
logger.warn("Missing Type information [%s]" % line)
result = NAME_LOC_REGEX.search(restOfLine)
if result:
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
else:
logger.error("Could not parse [%s]" % restOfLine)
return
else: # missing location, assume at least two space seperate name and type
result = NAME_TYPE_REGEX.search(restOfLine)
if result:
fName = result.group('Name').strip()
fType = result.group('Type').strip()
else:
logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line))
stripedType = ""
if fType:
stripedType = self.__stripFieldAttributes__(fType)
if stripedType:
self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference)
else:
self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
self._curFile.addFileManField(self._field)
if stripedType:
self.__parseFieldAttributes__(fType)
def _setFieldSpecificData(self, zeroFields, fileField, rootNode,
fileSchema, filePointedTo, subFile):
if fileField.getType() == FileManField.FIELD_TYPE_FILE_POINTER:
fileGlobalRoot = ""
if len(zeroFields) >= 3:
fileGlobalRoot = zeroFields[2]
if filePointedTo:
if filePointedTo not in self._allSchema:
""" create a new fileman file """
self._allSchema[filePointedTo] = Global(
fileGlobalRoot, filePointedTo, "")
pointedToFile = self._allSchema[filePointedTo]
assert pointedToFile.isRootFile()
fileField.setPointedToFile(pointedToFile)
globalName = pointedToFile.getName()
fileNo = fileSchema.getFileNo()
if fileSchema.isSubFile():
fileNo = fileSchema.getRootFile().getFileNo()
self._addToFileDepDict(fileNo, pointedToFile.getFileNo())
if fileGlobalRoot:
if not globalName:
pointedToFile.setName(fileGlobalRoot)
elif globalName != fileGlobalRoot:
logging.error(
"%s: FileMan global root mismatch %s: %s" %
(zeroFields, globalName, fileGlobalRoot))
else:
logging.info("@TODO, find file global root for # %s" %
filePointedTo)
elif fileGlobalRoot:
self._noPointedToFiles[fileGlobalRoot] = Global(fileGlobalRoot)
logging.info("@TODO, set the file number for %s" %
fileGlobalRoot)
else:
logging.warn(
"No pointed to file set for file:%s: field:%r 0-index:%s" %
(fileSchema.getFileNo(), fileField, zeroFields))
elif fileField.getType() == FileManField.FIELD_TYPE_SUBFILE_POINTER:
if subFile:
if subFile not in self._allSchema:
self._allSchema[subFile] = FileManFile(
subFile, "", fileSchema)
subFileSchema = self._allSchema[subFile]
subFileSchema.setParentFile(fileSchema)
fileSchema.addFileManSubFile(subFileSchema)
fileField.setPointedToSubFile(subFileSchema)
else:
logging.warn(
"No subfile is set for file:%s, field:%r 0-index:%s" %
(fileSchema.getFileNo(), fileField, zeroFields))
elif fileField.getType(
) == FileManField.FIELD_TYPE_SET and not subFile:
setDict = dict(
[x.split(':') for x in zeroFields[2].rstrip(';').split(';')])
fileField.setSetMembers(setDict)
elif fileField.getType(
) == FileManField.FIELD_TYPE_VARIABLE_FILE_POINTER:
if "V" in rootNode: # parsing variable pointer
vptrs = parsingVariablePointer(rootNode['V'])
vpFileSchemas = []
if vptrs:
logging.debug("variable points: %s" % vptrs)
for x in vptrs:
if x not in self._allSchema:
self._allSchema[x] = Global("", x, "")
pointedToFile = self._allSchema[x]
if pointedToFile.isSubFile():
logging.error(
"Field: %r point to subFile: %s, parent: %s" %
(fileField, pointedToFile,
pointedToFile.getParentFile()))
else:
fileNo = fileSchema.getFileNo()
if fileSchema.isSubFile():
fileNo = fileSchema.getRootFile().getFileNo()
self._addToFileDepDict(fileNo,
pointedToFile.getFileNo())
vpFileSchemas.append(self._allSchema[x])
fileField.setPointedToFiles(vpFileSchemas)
elif fileField.getType() == FileManField.FIELD_TYPE_COMPUTED:
if len(zeroFields) >= 5:
logging.debug("Computed Mumps Code: %s for %r" %
("".join(zeroFields[4:]), fileField))
def onSectionEnd(self, line, section, Global, CrossReference):
if self._curLine:
self._lines.append(self._curLine)
Global.setDescription(self._lines)
def onSectionEnd(self, line, section, Global, CrossReference):
if len(self._curLine) > 0:
self._lines.append(self._curLine)
Global.setDescription(self._lines)
def findGlobalsBySourceV2(self, dirName, pattern):
searchFiles = glob.glob(os.path.join(dirName, pattern))
logger.info("Total Search Files are %d " % len(searchFiles))
crossReference = self.crossRef
allGlobals = crossReference.getAllGlobals()
allPackages = crossReference.getAllPackages()
skipFile = []
fileNoSet = set()
for file in searchFiles:
packageName = os.path.dirname(file)
packageName = packageName[packageName.index("Packages") +
9:packageName.index("Globals") - 1]
if not crossReference.hasPackage(packageName):
crossReference.addPackageByName(packageName)
package = allPackages.get(packageName)
zwrFile = codecs.open(file, 'r', encoding='utf-8', errors='ignore')
lineNo = 0
fileName = os.path.basename(file)
result = ZWR_FILENO_REGEX.search(fileName)
if result:
fileNo = result.group('fileNo')
if fileNo.startswith('0'): fileNo = fileNo[1:]
globalDes = result.group('des')
else:
result = ZWR_NAMESPACE_REGEX.search(fileName)
if result:
namespace = result.group('namespace')
# package.addGlobalNamespace(namespace)
continue
else:
continue
globalName = "" # find out the global name by parsing the global file
logger.debug("Parsing file: %s" % file)
for line in zwrFile:
if lineNo == 0:
globalDes = line.strip()
# Removing the extra text in the header of the ZWR file
# to tell if it needs to be added or skipped
globalDes = globalDes.replace("OSEHRA ZGO Export: ", '')
if globalDes.startswith("^"):
logger.info("No Description: Skip this file: %s" %
file)
skipFile.append(file)
namespace = globalDes[1:]
package.addGlobalNamespace(namespace)
break
if lineNo >= 2:
info = line.strip().split('=')
globalName = info[0]
detail = info[1].strip("\"")
if globalName.find(',') > 0:
result = globalName.split(',')
if len(result) == 2 and result[1] == "0)":
globalName = result[0]
break
elif globalName.endswith("(0)"):
globalName = globalName.split('(')[0]
break
else:
continue
lineNo = lineNo + 1
if not fileNo:
if file not in skipFile:
logger.warn("Warning: No FileNo found for file %s" % file)
continue
globalVar = Global(globalName, fileNo, globalDes,
allPackages.get(packageName))
try:
fileNum = float(globalVar.getFileNo())
except ValueError as es:
logger.error("error: %s, globalVar:%s file %s" %
(es, globalVar, file))
continue
# crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
if globalVar.getName() not in allGlobals:
allGlobals[globalVar.getName()] = globalVar
if fileNo not in fileNoSet:
fileNoSet.add(fileNo)
else:
logger.error(
"Duplicated file No [%s,%s,%s,%s] file:%s " %
(fileNo, globalName, globalDes, packageName, file))
zwrFile.close()
logger.info(
"Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d"
%
(len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet)))
sortedKeyList = sorted(
list(allGlobals.keys()),
key=lambda item: float(allGlobals[item].getFileNo()))
for key in sortedKeyList:
globalVar = allGlobals[key]
# fix the uncategoried item
if globalVar.getFileNo() in fileNoPackageMappingDict:
globalVar.setPackage(allPackages[fileNoPackageMappingDict[
globalVar.getFileNo()]])
crossReference.addGlobalToPackage(globalVar,
globalVar.getPackage().getName())
