def generateFileIndex(self, inputFileName, allSchemaDict, fileNumber):
self._allSchemaDict = allSchemaDict
schemaFile = allSchemaDict[fileNumber]
if not schemaFile.hasField('.01'):
logging.error("File does not have a .01 field, ignore")
return
keyField = schemaFile.getFileManFieldByFieldNo('.01')
keyLoc = keyField.getLocation()
if not keyLoc:
logging.error(".01 field does not have a location")
return
self._curFileNo = fileNumber
glbLoc = self._glbLocMap[fileNumber]
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot: continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
(ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc,
keyField)
if detail:
self._addFileKeyIndex(fileNumber, ien, detail)
elif ien:
logging.info("No name associated with ien: %s, file: %s" %
(ien, fileNumber))
else:
logging.info("No index for data with ien: %s, file: %s" %
(ien, fileNumber))
def parseZWRGlobalFileBySchemaV2(self,
inputFileName,
allSchemaDict,
fileNumber,
glbLoc=None):
self._allSchemaDict = allSchemaDict
schemaFile = allSchemaDict[fileNumber]
self._glbData[fileNumber] = FileManFileData(
fileNumber, self.getFileManFileNameByFileNo(fileNumber))
self._curFileNo = fileNumber
if not glbLoc:
glbLoc = self._glbLocMap.get(fileNumber)
logging.info("File: %s global loc: %s" % (fileNumber, glbLoc))
elif fileNumber in self._glbLocMap:
logging.info("global loc %s, %s" %
(glbLoc, self._glbLocMap[fileNumber]))
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot:
continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
self._parseDataBySchema(fileDataRoot, schemaFile,
self._glbData[fileNumber])
self._resolveSelfPointer()
if self._crossRef:
self._updateCrossReference()
def getFreeTextLink(dataEntry, value, **kargs):
if value:
if 'glbData' in kargs:
glbData = kargs['glbData']
# Acquire the field and file of the target information
file,field = kargs["targetField"].split("/");
# Check if the target file hasn't already been parsed
if file not in glbData.outFileManData:
glbData._glbData[file] = FileManFileData(file,
glbData.getFileManFileNameByFileNo(file))
for pathName in glbData.allFiles[file]["path"]:
# Taken from the FileManGlobalDataParser
glbData._createDataRootByZWRFile(pathName)
glbLoc = glbData._glbLocMap.get(file)
for dataRoot in readGlobalNodeFromZWRFileV2(pathName, glbLoc):
if not dataRoot:
continue
glbData._dataRoot = dataRoot
fileDataRoot = dataRoot
glbData._parseDataBySchema(fileDataRoot, glbData._allSchemaDict[file],
glbData._glbData[file])
# Once the information is available check for the target in any available
# information in that file
dataEntries = glbData.outFileManData[file].dataEntries
for entry in dataEntries:
if value == dataEntries[entry].fields[field].value:
return '<a href="%s/%s/%s-%s.html">%s</a>' % (VIV_URL, dataEntries[entry].fileNo.replace(".","_"),
file, entry, value)
return value
def generateFileFieldMap(self, inputFileName, fileNumber, fieldNo):
schemaFile = self._allSchemaDict[fileNumber]
if not schemaFile.hasField(fieldNo):
logging.error("File does not have a [%s] field, ignore", fieldNo)
return
keyField = schemaFile.getFileManFieldByFieldNo(fieldNo)
keyLoc = keyField.getLocation()
if not keyLoc:
logging.error("[%s] field does not have a location", fieldNo)
return
glbLoc = self._glbLocMap[fileNumber]
fieldMap = {}
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot: continue
fileDataRoot = dataRoot
(ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc,
keyField)
if detail:
fieldMap[detail] = ien
elif ien:
logging.info("No name associated with ien: %s, file: %s" %
(ien, fileNumber))
else:
logging.info("No index for data with ien: %s, file: %s" %
(ien, fileNumber))
return fieldMap
def getFreeTextLink(dataEntry, value, **kargs):
if value:
if 'glbData' in kargs:
glbData = kargs['glbData']
# Acquire the field and file of the target information
file, field = kargs["targetField"].split("/")
# Check if the target file hasn't already been parsed
if file not in glbData.outFileManData:
glbData._glbData[file] = FileManFileData(
file, glbData.getFileManFileNameByFileNo(file))
for pathName in glbData.allFiles[file]["path"]:
# Taken from the FileManGlobalDataParser
glbData._createDataRootByZWRFile(pathName)
glbLoc = glbData._glbLocMap.get(file)
for dataRoot in readGlobalNodeFromZWRFileV2(
pathName, glbLoc):
if not dataRoot:
continue
glbData._dataRoot = dataRoot
fileDataRoot = dataRoot
glbData._parseDataBySchema(
fileDataRoot, glbData._allSchemaDict[file],
glbData._glbData[file])
# Once the information is available check for the target in any available
# information in that file
dataEntries = glbData.outFileManData[file].dataEntries
for entry in dataEntries:
if value == dataEntries[entry].fields[field].value:
return '<a href="%s/%s/%s-%s.html">%s</a>' % (
FILES_URL, dataEntries[entry].fileNo.replace(
".", "_"), file, entry, value)
return value
def generateFileIndex(self, inputFileName, allSchemaDict,
fileNumber):
self._allSchemaDict = allSchemaDict
schemaFile = allSchemaDict[fileNumber]
if not schemaFile.hasField('.01'):
logging.error("File does not have a .01 field, ignore")
return
keyField = schemaFile.getFileManFieldByFieldNo('.01')
keyLoc = keyField.getLocation()
if not keyLoc:
logging.error(".01 field does not have a location")
return
self._curFileNo = fileNumber
glbLoc = self._glbLocMap[fileNumber]
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot: continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
(ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField)
if detail:
self._addFileKeyIndex(fileNumber, ien, detail)
elif ien:
logging.info("No name associated with ien: %s, file: %s" % (ien, fileNumber))
else:
logging.info("No index for data with ien: %s, file: %s" % (ien, fileNumber))
def generateFileFieldMap(self, inputFileList, fileNumber, fieldNo):
"""
Generate a map Field Value => IEN
"""
schemaFile = self._allSchemaDict[fileNumber]
if not schemaFile.hasField(fieldNo):
logger.error("File does not have a [%s] field, ignore", fieldNo)
return dict()
keyField = schemaFile.getFileManFieldByFieldNo(fieldNo)
keyLoc = keyField.getLocation()
if not keyLoc:
logger.error("[%s] field does not have a location", fieldNo)
return dict()
glbLoc = self._glbLocMap[fileNumber]
fieldMap = {}
for inputFileName in inputFileList:
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot:
continue
fileDataRoot = dataRoot
(ien,
detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc,
keyField)
if detail:
fieldMap[detail] = ien
return fieldMap
def generateFileIndex(self, inputFileName, fileNumber):
schemaFile = self._allSchemaDict[fileNumber]
if not schemaFile.hasField('.01'):
logger.error("File %s does not have a .01 field, ignore" %
fileNumber)
return
keyField = schemaFile.getFileManFieldByFieldNo('.01')
keyLoc = keyField.getLocation()
if not keyLoc:
logger.error("File %s .01 field does not have a location, ignore" %
fileNumber)
return
self._curFileNo = fileNumber
if fileNumber in self._glbLocMap:
glbLoc = self._glbLocMap[fileNumber]
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot:
continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
(ien,
detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc,
keyField)
if detail:
self._addFileKeyIndex(fileNumber, ien, detail)
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'):
files = getKeys(ddRoot, float) # sort files by float value
for file in files:
if file not in self._allSchema:
self._allSchema[file] = Global("", file, "")
self._generateFileSchema(ddRoot[file], self._allSchema[file])
self._updateMultiple()
self._sccSet = self._generateSCCSet()
return self._allSchema
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'):
files = getKeys(ddRoot, float) # sort files by float value
for file in files:
if file not in self._allSchema:
self._allSchema[file] = Global("", file, "")
self._generateFileSchema(ddRoot[file], self._allSchema[file])
self._updateMultiple()
self._sccSet = self._generateSCCSet()
return self._allSchema
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'):
self._ddRoot = ddRoot
self._generateSchema()
self._updateMultiple()
self._generateNoPointerToFileList()
outSCCLst = self._generateSCCSet()
totalFiles = 0
for idx, scc in enumerate(outSCCLst):
scc = scc - self._isolatedFile
totalFiles += len(scc)
return self._allSchema
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'):
self._ddRoot = ddRoot
self._generateSchema()
self._updateMultiple()
self._generateNoPointerToFileList()
outSCCLst = self._generateSCCSet()
totalFiles = 0
for idx, scc in enumerate(outSCCLst):
scc = scc - self._isolatedFile
totalFiles += len(scc)
return self._allSchema
def parseZWRGlobalFileBySchemaV2(self, inputFileName, fileNumber, glbLoc=None):
schemaFile = self._allSchemaDict[fileNumber]
self._glbData[fileNumber] = FileManFileData(fileNumber,
self.getFileManFileNameByFileNo(fileNumber))
self._curFileNo = fileNumber
if not glbLoc:
glbLoc = self._glbLocMap.get(fileNumber)
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot:
continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
self._parseDataBySchema(fileDataRoot, schemaFile,
self._glbData[fileNumber])
self._resolveSelfPointer()
if self._crossRef:
self._updateCrossReference()
def parseZWRGlobalFileBySchemaV2(self, inputFileName, fileNumber, glbLoc=None):
schemaFile = self._allSchemaDict[fileNumber]
self._glbData[fileNumber] = FileManFileData(fileNumber,
self.getFileManFileNameByFileNo(fileNumber))
self._curFileNo = fileNumber
if not glbLoc:
glbLoc = self._glbLocMap.get(fileNumber)
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot:
continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
self._parseDataBySchema(fileDataRoot, schemaFile,
self._glbData[fileNumber])
self._resolveSelfPointer()
if self._crossRef:
self._updateCrossReference()
def generateFileFieldMap(self, inputFileList, fileNumber, fieldNo):
schemaFile = self._allSchemaDict[fileNumber]
if not schemaFile.hasField(fieldNo):
logger.error("File does not have a [%s] field, ignore", fieldNo)
return
keyField = schemaFile.getFileManFieldByFieldNo(fieldNo)
keyLoc = keyField.getLocation()
if not keyLoc:
logger.error("[%s] field does not have a location", fieldNo)
return
glbLoc = self._glbLocMap[fileNumber]
fieldMap = {}
for inputFileName in inputFileList:
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot: continue
fileDataRoot = dataRoot
(ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField)
if detail:
fieldMap[detail] = ien
return fieldMap
def generateFileIndex(self, inputFileName, fileNumber):
schemaFile = self._allSchemaDict[fileNumber]
if not schemaFile.hasField('.01'):
logger.error("File %s does not have a .01 field, ignore" % fileNumber)
return
keyField = schemaFile.getFileManFieldByFieldNo('.01')
keyLoc = keyField.getLocation()
if not keyLoc:
logger.error("File %s .01 field does not have a location, ignore" % fileNumber)
return
self._curFileNo = fileNumber
if fileNumber in self._glbLocMap:
glbLoc = self._glbLocMap[fileNumber]
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot: continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
(ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField)
if detail:
self._addFileKeyIndex(fileNumber, ien, detail)
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, "^DD"):
self._ddRoot = ddRoot
# logging.info("Printing Global Node:")
# printGlobal(ddRoot)
self._generateSchema()
self._updateMultiple()
self._generateNoPointerToFileList()
outSCCLst = self._generateSCCSet()
if "101" in self._fileDep:
print self._fileDep["101"]
totalFiles = 0
for idx, scc in enumerate(outSCCLst):
scc = scc - self._isolatedFile
print "%s: %s" % (idx, scc)
totalFiles += len(scc)
if "101" in scc:
print "Total # files need to read at the same time is %s" % len(scc)
print "Total Files before 101 is %s" % totalFiles
# self._updateFileDepByFile('200', set())
# print len(self._fileDep['200']), self._fileDep['200']
return self._allSchema
def parseZWRGlobalFileBySchemaV2(self, inputFileName, allSchemaDict,
fileNumber, glbLoc=None):
self._allSchemaDict = allSchemaDict
schemaFile = allSchemaDict[fileNumber]
self._glbData[fileNumber] = FileManFileData(fileNumber,
self.getFileManFileNameByFileNo(fileNumber))
self._curFileNo = fileNumber
if not glbLoc:
glbLoc = self._glbLocMap.get(fileNumber)
logging.info("File: %s global loc: %s" % (fileNumber, glbLoc))
elif fileNumber in self._glbLocMap:
logging.info("global loc %s, %s" % (glbLoc, self._glbLocMap[fileNumber]))
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot:
continue
self._dataRoot = dataRoot
fileDataRoot = dataRoot
self._parseDataBySchema(fileDataRoot, schemaFile,
self._glbData[fileNumber])
self._resolveSelfPointer()
if self._crossRef:
self._updateCrossReference()
def parseSchemaDDFileV2(self, inputDDZWRFile):
for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'):
self._ddRoot = ddRoot
#logging.info("Printing Global Node:")
#printGlobal(ddRoot)
self._generateSchema()
self._updateMultiple()
self._generateNoPointerToFileList()
outSCCLst = self._generateSCCSet()
if '101' in self._fileDep:
print self._fileDep['101']
totalFiles = 0
for idx, scc in enumerate(outSCCLst):
scc = scc - self._isolatedFile
print "%s: %s" % (idx, scc)
totalFiles += len(scc)
if '101' in scc:
print "Total # files need to read at the same time is %s" % len(
scc)
print "Total Files before 101 is %s" % totalFiles
#self._updateFileDepByFile('200', set())
#print len(self._fileDep['200']), self._fileDep['200']
return self._allSchema
def generateFileFieldMap(self, inputFileName, fileNumber, fieldNo):
schemaFile = self._allSchemaDict[fileNumber]
if not schemaFile.hasField(fieldNo):
logging.error("File does not have a [%s] field, ignore", fieldNo)
return
keyField = schemaFile.getFileManFieldByFieldNo(fieldNo)
keyLoc = keyField.getLocation()
if not keyLoc:
logging.error("[%s] field does not have a location", fieldNo)
return
glbLoc = self._glbLocMap[fileNumber]
fieldMap = {}
for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc):
if not dataRoot: continue
fileDataRoot = dataRoot
(ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField)
if detail:
fieldMap[detail] = ien
elif ien:
logging.info("No name associated with ien: %s, file: %s" % (ien, fileNumber))
else:
logging.info("No index for data with ien: %s, file: %s" % (ien, fileNumber))
return fieldMap
