def generateFileIndex(self, inputFileName, allSchemaDict, fileNumber): self._allSchemaDict = allSchemaDict schemaFile = allSchemaDict[fileNumber] if not schemaFile.hasField('.01'): logging.error("File does not have a .01 field, ignore") return keyField = schemaFile.getFileManFieldByFieldNo('.01') keyLoc = keyField.getLocation() if not keyLoc: logging.error(".01 field does not have a location") return self._curFileNo = fileNumber glbLoc = self._glbLocMap[fileNumber] for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue self._dataRoot = dataRoot fileDataRoot = dataRoot (ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField) if detail: self._addFileKeyIndex(fileNumber, ien, detail) elif ien: logging.info("No name associated with ien: %s, file: %s" % (ien, fileNumber)) else: logging.info("No index for data with ien: %s, file: %s" % (ien, fileNumber))
def parseZWRGlobalFileBySchemaV2(self, inputFileName, allSchemaDict, fileNumber, glbLoc=None): self._allSchemaDict = allSchemaDict schemaFile = allSchemaDict[fileNumber] self._glbData[fileNumber] = FileManFileData( fileNumber, self.getFileManFileNameByFileNo(fileNumber)) self._curFileNo = fileNumber if not glbLoc: glbLoc = self._glbLocMap.get(fileNumber) logging.info("File: %s global loc: %s" % (fileNumber, glbLoc)) elif fileNumber in self._glbLocMap: logging.info("global loc %s, %s" % (glbLoc, self._glbLocMap[fileNumber])) for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue self._dataRoot = dataRoot fileDataRoot = dataRoot self._parseDataBySchema(fileDataRoot, schemaFile, self._glbData[fileNumber]) self._resolveSelfPointer() if self._crossRef: self._updateCrossReference()
def getFreeTextLink(dataEntry, value, **kargs): if value: if 'glbData' in kargs: glbData = kargs['glbData'] # Acquire the field and file of the target information file,field = kargs["targetField"].split("/"); # Check if the target file hasn't already been parsed if file not in glbData.outFileManData: glbData._glbData[file] = FileManFileData(file, glbData.getFileManFileNameByFileNo(file)) for pathName in glbData.allFiles[file]["path"]: # Taken from the FileManGlobalDataParser glbData._createDataRootByZWRFile(pathName) glbLoc = glbData._glbLocMap.get(file) for dataRoot in readGlobalNodeFromZWRFileV2(pathName, glbLoc): if not dataRoot: continue glbData._dataRoot = dataRoot fileDataRoot = dataRoot glbData._parseDataBySchema(fileDataRoot, glbData._allSchemaDict[file], glbData._glbData[file]) # Once the information is available check for the target in any available # information in that file dataEntries = glbData.outFileManData[file].dataEntries for entry in dataEntries: if value == dataEntries[entry].fields[field].value: return '<a href="%s/%s/%s-%s.html">%s</a>' % (VIV_URL, dataEntries[entry].fileNo.replace(".","_"), file, entry, value) return value
def generateFileFieldMap(self, inputFileName, fileNumber, fieldNo): schemaFile = self._allSchemaDict[fileNumber] if not schemaFile.hasField(fieldNo): logging.error("File does not have a [%s] field, ignore", fieldNo) return keyField = schemaFile.getFileManFieldByFieldNo(fieldNo) keyLoc = keyField.getLocation() if not keyLoc: logging.error("[%s] field does not have a location", fieldNo) return glbLoc = self._glbLocMap[fileNumber] fieldMap = {} for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue fileDataRoot = dataRoot (ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField) if detail: fieldMap[detail] = ien elif ien: logging.info("No name associated with ien: %s, file: %s" % (ien, fileNumber)) else: logging.info("No index for data with ien: %s, file: %s" % (ien, fileNumber)) return fieldMap
def getFreeTextLink(dataEntry, value, **kargs): if value: if 'glbData' in kargs: glbData = kargs['glbData'] # Acquire the field and file of the target information file, field = kargs["targetField"].split("/") # Check if the target file hasn't already been parsed if file not in glbData.outFileManData: glbData._glbData[file] = FileManFileData( file, glbData.getFileManFileNameByFileNo(file)) for pathName in glbData.allFiles[file]["path"]: # Taken from the FileManGlobalDataParser glbData._createDataRootByZWRFile(pathName) glbLoc = glbData._glbLocMap.get(file) for dataRoot in readGlobalNodeFromZWRFileV2( pathName, glbLoc): if not dataRoot: continue glbData._dataRoot = dataRoot fileDataRoot = dataRoot glbData._parseDataBySchema( fileDataRoot, glbData._allSchemaDict[file], glbData._glbData[file]) # Once the information is available check for the target in any available # information in that file dataEntries = glbData.outFileManData[file].dataEntries for entry in dataEntries: if value == dataEntries[entry].fields[field].value: return '<a href="%s/%s/%s-%s.html">%s</a>' % ( FILES_URL, dataEntries[entry].fileNo.replace( ".", "_"), file, entry, value) return value
def generateFileFieldMap(self, inputFileList, fileNumber, fieldNo): """ Generate a map Field Value => IEN """ schemaFile = self._allSchemaDict[fileNumber] if not schemaFile.hasField(fieldNo): logger.error("File does not have a [%s] field, ignore", fieldNo) return dict() keyField = schemaFile.getFileManFieldByFieldNo(fieldNo) keyLoc = keyField.getLocation() if not keyLoc: logger.error("[%s] field does not have a location", fieldNo) return dict() glbLoc = self._glbLocMap[fileNumber] fieldMap = {} for inputFileName in inputFileList: for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue fileDataRoot = dataRoot (ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField) if detail: fieldMap[detail] = ien return fieldMap
def generateFileIndex(self, inputFileName, fileNumber): schemaFile = self._allSchemaDict[fileNumber] if not schemaFile.hasField('.01'): logger.error("File %s does not have a .01 field, ignore" % fileNumber) return keyField = schemaFile.getFileManFieldByFieldNo('.01') keyLoc = keyField.getLocation() if not keyLoc: logger.error("File %s .01 field does not have a location, ignore" % fileNumber) return self._curFileNo = fileNumber if fileNumber in self._glbLocMap: glbLoc = self._glbLocMap[fileNumber] for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue self._dataRoot = dataRoot fileDataRoot = dataRoot (ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField) if detail: self._addFileKeyIndex(fileNumber, ien, detail)
def parseSchemaDDFileV2(self, inputDDZWRFile): for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'): files = getKeys(ddRoot, float) # sort files by float value for file in files: if file not in self._allSchema: self._allSchema[file] = Global("", file, "") self._generateFileSchema(ddRoot[file], self._allSchema[file]) self._updateMultiple() self._sccSet = self._generateSCCSet() return self._allSchema
def parseSchemaDDFileV2(self, inputDDZWRFile): for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'): self._ddRoot = ddRoot self._generateSchema() self._updateMultiple() self._generateNoPointerToFileList() outSCCLst = self._generateSCCSet() totalFiles = 0 for idx, scc in enumerate(outSCCLst): scc = scc - self._isolatedFile totalFiles += len(scc) return self._allSchema
def parseZWRGlobalFileBySchemaV2(self, inputFileName, fileNumber, glbLoc=None): schemaFile = self._allSchemaDict[fileNumber] self._glbData[fileNumber] = FileManFileData(fileNumber, self.getFileManFileNameByFileNo(fileNumber)) self._curFileNo = fileNumber if not glbLoc: glbLoc = self._glbLocMap.get(fileNumber) for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue self._dataRoot = dataRoot fileDataRoot = dataRoot self._parseDataBySchema(fileDataRoot, schemaFile, self._glbData[fileNumber]) self._resolveSelfPointer() if self._crossRef: self._updateCrossReference()
def generateFileFieldMap(self, inputFileList, fileNumber, fieldNo): schemaFile = self._allSchemaDict[fileNumber] if not schemaFile.hasField(fieldNo): logger.error("File does not have a [%s] field, ignore", fieldNo) return keyField = schemaFile.getFileManFieldByFieldNo(fieldNo) keyLoc = keyField.getLocation() if not keyLoc: logger.error("[%s] field does not have a location", fieldNo) return glbLoc = self._glbLocMap[fileNumber] fieldMap = {} for inputFileName in inputFileList: for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue fileDataRoot = dataRoot (ien, detail) = self._getKeyNameBySchema(fileDataRoot, keyLoc, keyField) if detail: fieldMap[detail] = ien return fieldMap
def parseSchemaDDFileV2(self, inputDDZWRFile): for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, "^DD"): self._ddRoot = ddRoot # logging.info("Printing Global Node:") # printGlobal(ddRoot) self._generateSchema() self._updateMultiple() self._generateNoPointerToFileList() outSCCLst = self._generateSCCSet() if "101" in self._fileDep: print self._fileDep["101"] totalFiles = 0 for idx, scc in enumerate(outSCCLst): scc = scc - self._isolatedFile print "%s: %s" % (idx, scc) totalFiles += len(scc) if "101" in scc: print "Total # files need to read at the same time is %s" % len(scc) print "Total Files before 101 is %s" % totalFiles # self._updateFileDepByFile('200', set()) # print len(self._fileDep['200']), self._fileDep['200'] return self._allSchema
def parseZWRGlobalFileBySchemaV2(self, inputFileName, allSchemaDict, fileNumber, glbLoc=None): self._allSchemaDict = allSchemaDict schemaFile = allSchemaDict[fileNumber] self._glbData[fileNumber] = FileManFileData(fileNumber, self.getFileManFileNameByFileNo(fileNumber)) self._curFileNo = fileNumber if not glbLoc: glbLoc = self._glbLocMap.get(fileNumber) logging.info("File: %s global loc: %s" % (fileNumber, glbLoc)) elif fileNumber in self._glbLocMap: logging.info("global loc %s, %s" % (glbLoc, self._glbLocMap[fileNumber])) for dataRoot in readGlobalNodeFromZWRFileV2(inputFileName, glbLoc): if not dataRoot: continue self._dataRoot = dataRoot fileDataRoot = dataRoot self._parseDataBySchema(fileDataRoot, schemaFile, self._glbData[fileNumber]) self._resolveSelfPointer() if self._crossRef: self._updateCrossReference()
def parseSchemaDDFileV2(self, inputDDZWRFile): for ddRoot in readGlobalNodeFromZWRFileV2(inputDDZWRFile, '^DD'): self._ddRoot = ddRoot #logging.info("Printing Global Node:") #printGlobal(ddRoot) self._generateSchema() self._updateMultiple() self._generateNoPointerToFileList() outSCCLst = self._generateSCCSet() if '101' in self._fileDep: print self._fileDep['101'] totalFiles = 0 for idx, scc in enumerate(outSCCLst): scc = scc - self._isolatedFile print "%s: %s" % (idx, scc) totalFiles += len(scc) if '101' in scc: print "Total # files need to read at the same time is %s" % len( scc) print "Total Files before 101 is %s" % totalFiles #self._updateFileDepByFile('200', set()) #print len(self._fileDep['200']), self._fileDep['200'] return self._allSchema