def parsePackagesFile(self, packageFilename): result = csv.DictReader(open(packageFilename, 'rb')) crossRef = self.crossRef currentPackage = None index = 0 for row in result: packageName = row['Directory Name'] if packageName: currentPackage = crossRef.getPackageByName(packageName) if not currentPackage: crossRef.addPackageByName(packageName) currentPackage = crossRef.getPackageByName(packageName) currentPackage.setOriginalName(row['Package Name']) vdlId = row['VDL ID'] if vdlId and len(vdlId): currentPackage.setDocLink(getVDLHttpLinkByID(vdlId)) else: if not currentPackage: logger.warn("row is not under any package: %s" % row) continue if len(row['Prefixes']): currentPackage.addNamespace(row['Prefixes']) if len(row['Globals']): currentPackage.addGlobalNamespace(row['Globals']) logger.info("Total # of Packages is %d" % (len(crossRef.getAllPackages())))
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference): logger.debug("Current Type is [%s]" % fType) result = self.UNDEFINED_POINTER.search(fType) if result: self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) return result = self.POINTER_TO_REGEX.search(fType) if result: fileNo = result.group('File') filePointedTo = CrossReference.getGlobalByFileNo(fileNo) self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) if not filePointedTo: logger.error( "Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line)) else: self._field.setPointedToFile(filePointedTo) return # deal with file pointer to subFiles result = self.SUBFILE_REGEX.search(fType) if result: # create a field for sub file type self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation) fileNo = result.group('File') logger.debug("Pointer to subFile %s" % fileNo) subFile = Global.getSubFileByFileNo(fileNo) if not subFile: subFile = FileManFile(fileNo, fName, self._curFile) self._curFile.addFileManSubFile(subFile) logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo())) if self._isSubFile: Global.addFileManSubFile(subFile) self._field.setPointedToSubFile(subFile) return for (key, value) in self.StringTypeMappingDict.iteritems(): if fType.startswith(key): self._field = FileManFieldFactory.createField( fieldNo, fName, value, fLocation) break if not self._field: # double check the loc and type if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX: fType = line[self.MAXIMIUM_TYPE_START_INDEX:] if fLocation: fLocation = line[line.find(fLocation):self. MAXIMIUM_TYPE_START_INDEX] logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation)) self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference) assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % ( fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def parsePackagesFile(self, packageFilename): result = csv.DictReader(open(packageFilename, 'rb')) crossRef = self.crossRef currentPackage = None index = 0 for row in result: packageName = row['Directory Name'] if len(packageName) > 0: currentPackage = crossRef.getPackageByName(packageName) if not currentPackage: logger.debug("Package [%s] not found" % packageName) crossRef.addPackageByName(packageName) currentPackage = crossRef.getPackageByName(packageName) currentPackage.setOriginalName(row['Package Name']) vdlId = row['VDL ID'] if vdlId and len(vdlId): currentPackage.setDocLink(getVDLHttpLinkByID(vdlId)) else: if not currentPackage: logger.warn("row is not under any package: %s" % row) continue if len(row['Prefixes']): currentPackage.addNamespace(row['Prefixes']) if len(row['Globals']): currentPackage.addGlobalNamespace(row['Globals']) logger.info("Total # of Packages is %d" % (len(crossRef.getAllPackages())))
def getPackageHRefLink(pkgName, icrEntry, **kargs): global pgkUpperCaseNameDict if pkgName in pkgMap: pkgLink = getPackageHtmlFileName(pkgMap[pkgName]) return '<a href=\"%s%s\">%s</a>' % (DOX_URL, pkgLink, pkgName) crossRef = None if 'crossRef' in kargs: crossRef = kargs['crossRef'] if crossRef: if len(pgkUpperCaseNameDict) == 0 : for name in crossRef.getAllPackages().iterkeys(): pgkUpperCaseNameDict[name.upper()] = name upperName = normalizeName(pkgName).upper() if upperName in pgkUpperCaseNameDict: addToPackageMap(icrEntry, pgkUpperCaseNameDict[upperName]) return '<a href=\"%s%s\">%s</a>' % (DOX_URL, getPackageHtmlFileName(pgkUpperCaseNameDict[upperName]), pkgName) pkg = crossRef.getPackageByName(pkgName) if not pkg: pkgRename = normalizeName(pkgName).title() # logger.warn('[%s] renamed as [%s]', pkgName, pkgRename) pkg = crossRef.getPackageByName(pkgRename) if not pkg: pkgRename = normalizeName(pkgName) pkg = crossRef.getPackageByName(pkgRename) if pkg: addToPackageMap(icrEntry, pkg.getName()) pkgLink = getPackageHtmlFileName(pkg.getName()) return '<a href=\"%s%s\">%s</a>' % (DOX_URL, pkgLink, pkgName) else: logger.warn('Can not find mapping for package: [%s]', pkgName) return pkgName
def _updateHL7Reference(self): protocol = self._glbData['101'] for ien in sorted(list(protocol.dataEntries), key=lambda x: float(x)): protocolEntry = protocol.dataEntries[ien] if '4' in protocolEntry.fields: type = protocolEntry.fields['4'].value if (type != 'event driver' and type != 'subscriber'): entryName = protocolEntry.name namespace, package = \ self._crossRef.__categorizeVariableNameByNamespace__(entryName) if package: package.protocol.append(protocolEntry) # only care about the event drive and subscriber type elif (type == 'event driver' or type == 'subscriber'): entryName = protocolEntry.name namespace, package = \ self._crossRef.__categorizeVariableNameByNamespace__(entryName) if package: package.hl7.append(protocolEntry) elif '12' in protocolEntry.fields: # check the packge it belongs pass else: logger.warn("Cannot find a package for HL7: %s" % entryName) for field in ('771', '772'): if field not in protocolEntry.fields: continue hl7Rtn = protocolEntry.fields[field].value if not hl7Rtn: continue for rtn, tag, pos in getMumpsRoutine(hl7Rtn): hl7Info = {"name": entryName, "ien": ien} if tag: hl7Info['tag'] = tag self._rtnRefDict.setdefault(rtn, {}).setdefault('101', []).append(hl7Info)
def getPackageHRefLink(pkgName, icrEntry, **kargs): if pkgName in pkgMap: pkgLink = getPackageHtmlFileName(pkgMap[pkgName]) return '<a href=\"%s%s\">%s</a>' % (dox_url, pkgLink , pkgName) crossRef = None if 'crossRef' in kargs: crossRef = kargs['crossRef'] if crossRef: if len(pgkUpperCaseNameDict) == 0 : for name in crossRef.getAllPackages().iterkeys(): pgkUpperCaseNameDict[name.upper()] = name upperName = normalizeName(pkgName).upper() if upperName in pgkUpperCaseNameDict: addToPackageMap(icrEntry, pgkUpperCaseNameDict[upperName]) return '<a href=\"%s%s\">%s</a>' % (dox_url, getPackageHtmlFileName(pgkUpperCaseNameDict[upperName]) , pkgName) pkg = crossRef.getPackageByName(pkgName) if not pkg: pkgRename = normalizeName(pkgName).title() # logger.warn('[%s] renamed as [%s]', pkgName, pkgRename) pkg = crossRef.getPackageByName(pkgRename) if not pkg: pkgRename = normalizeName(pkgName) pkg = crossRef.getPackageByName(pkgRename) if pkg: addToPackageMap(icrEntry, pkg.getName()) pkgLink = getPackageHtmlFileName(pkg.getName()) return '<a href=\"%s%s\">%s</a>' % (dox_url, pkgLink , pkgName) else: logger.warn('Can not find mapping for package: [%s]', pkgName) return pkgName
def filterResult(self, line): """ return None to stop reading more information return False to keep reading more information return GlobalNode to generate the result """ retNode = None subscripts, value, rootSub = findSubscriptValue(line) if not subscripts: # must have some subscripts return None if not self.rootSub: self.rootSub = rootSub if rootSub != self.rootSub: # not under the same root, ignore retNode = self.curRoot if self.glbLoc: logger.warn("Different root, expected: %s, real: %s, ignore for now" % (self.rootSub, rootSub)) self.curRoot = None return True else: self.rootSub = rootSub self.curCommonSub = subscripts[0:self.index+1] self.curRoot = createGlobalNode(subscripts, value, rootSub) if retNode: retNode = retNode.getRootNode() for sub in self.curCommonSub: retNode = retNode[sub] return retNode else: return True if self.commonSubscript and subscripts[0:self.index] != self.commonSubscript: logger.warn("Different subsript, expected: %s, real: %s, ignore for now" % (self.commonSubscript, subscripts[0:self.index])) retNode = self.curRoot self.curRoot = None if retNode: retNode = retNode.getRootNode() for sub in self.commonSubscript: retNode = retNode[sub] return retNode else: return True if self.curCommonSub is None: self.curCommonSub = subscripts[0:self.index+1] self.curRoot = createGlobalNode(subscripts, value, rootSub, self.curRoot) return True curCommonScript = os.path.commonprefix([subscripts, self.curCommonSub]) if self.curCommonSub == curCommonScript: self.curRoot = createGlobalNode(subscripts, value, rootSub, self.curRoot) return True else: retNode = self.curRoot if retNode: retNode = retNode.getRootNode() for subscript in curCommonScript: retNode = retNode[subscript] self.curRoot = createGlobalNode(subscripts, value, rootSub) self.curCommonSub = curCommonScript + subscripts[len(curCommonScript):self.index+1] return retNode
def convertJson(inputJsonFile, date, MRepositDir, patchRepositDir, generateHTML, generatePDF, outDir=None, pdfOutDir=None, local=False): if not generateHTML and not generatePDF: raise Exception("Nothing to generate!") global DOX_URL global VIVIAN_URL DOX_URL = getDOXURL(local) VIVIAN_URL = getViViaNURL(local) if generateHTML: if not outDir: raise Exception("Must specify Output directory") if not os.path.exists(outDir): # Will also create intermediate directories if needed os.makedirs(outDir) if generatePDF: if not pdfOutDir: raise Exception("Must specify PDF Output directory") # Will also create intermediate directories if needed if not os.path.exists(pdfOutDir): os.makedirs(pdfOutDir) crossRef = parseCrossReferenceGeneratorArgs(MRepositDir, patchRepositDir) global RPC_NAME_TO_IEN_MAPPING RPC_NAME_TO_IEN_MAPPING = generateSingleFileFieldToIenMappingBySchema(MRepositDir, crossRef, RPC_FILE_NO, RPC_NAME_FIELD_NO) with open(inputJsonFile, 'r') as inputFile: pkgJson = {} # group by package allpkgJson = [] inputJson = json.load(inputFile) for icrEntry in inputJson: if 'NUMBER' not in icrEntry: logger.error("Could not parse entry: " + str(icrEntry)) continue if generatePDF: _generateICRIndividualPagePDF(icrEntry, date, pdfOutDir) if generateHTML: _generateICRIndividualPage(icrEntry, date, outDir, crossRef) summaryInfo = _convertICREntryToSummaryInfo(icrEntry, crossRef) allpkgJson.append(summaryInfo) if 'CUSTODIAL PACKAGE' in icrEntry: pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'],[]).append(summaryInfo) if generateHTML: _generateICRSummaryPageImpl(allpkgJson, 'ICR List', 'All', date, outDir, isForAll=True) for pkgName, outJson in pkgJson.iteritems(): _generateICRSummaryPageImpl(outJson, 'ICR List', pkgName, date, outDir) logger.warn('Total # entry in PACKAGE_MAP is [%s]', len(PACKAGE_MAP)) logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson)) _generatePkgDepSummaryPage(inputJson, date, outDir, crossRef)
def parse(self, inputFilename, outputFilename): with open(inputFilename, 'r') as ICRFile: for line in ICRFile: line = line.rstrip("\r\n") self._curLineNo += 1 """ get rid of lines that are ignored """ if self.isIgnoredLine(line): continue match = START_OF_RECORD.match(line) if match: self._startOfNewItem(match, line) continue match = GENERIC_START_OF_RECORD.search(line) if not match: match = DBA_COMMENTS.match(line) if match and match.group('name') in ICR_FILE_KEYWORDS: fieldName = match.group('name') if isSubFile(fieldName): self._curField = fieldName self._startOfSubFile(match, line) else: logger.debug('field name is: %s', fieldName) logger.debug('cur field is: %s', self._curField) """ Check to see if fieldName is already in the out list """ if isWordProcessingField(self._curField): if self._ignoreKeywordInWordProcessingFields( fieldName): self._appendWordsFieldLine(line) continue # figure out where to store the record self._curField = fieldName self._rewindStack() self._findKeyValueInLine(match, line, self._curRecord) elif self._curField and self._curField in self._curRecord: if len(line.strip()) == 0 and not isWordProcessingField( self._curField): logger.warn( 'Ignore blank line for current field: [%s]', self._curField) continue self._appendWordsFieldLine(line) else: if self._curRecord: if len(line.strip()) == 0: continue print 'No field associated with line %s: %s ' % ( self._curLineNo, line) logger.info('End of file now') if len(self._curStack) > 0: self._curField = None self._rewindStack() if self._curRecord: logger.info('Add last record: %s', self._curRecord) self._outObject.append(self._curRecord) # pprint.pprint(self._outObject); with open(outputFilename, 'w') as out_file: json.dump(self._outObject, out_file, indent=4)
def convertJson(inputJsonFile, date, MRepositDir, patchRepositDir, generateHTML, generatePDF, outDir=None, pdfOutDir=None, local=False): if not generateHTML and not generatePDF: raise Exception("Nothing to generate!") global DOX_URL global VIVIAN_URL DOX_URL = getDOXURL(local) VIVIAN_URL = getViViaNURL(local) if generateHTML: if not outDir: raise Exception("Must specify Output directory") if not os.path.exists(outDir): # Will also create intermediate directories if needed os.makedirs(outDir) if generatePDF: if not pdfOutDir: raise Exception("Must specify PDF Output directory") # Will also create intermediate directories if needed if not os.path.exists(pdfOutDir): os.makedirs(pdfOutDir) crossRef = parseCrossReferenceGeneratorArgs(MRepositDir, patchRepositDir) global RPC_NAME_TO_IEN_MAPPING RPC_NAME_TO_IEN_MAPPING = generateSingleFileFieldToIenMappingBySchema(MRepositDir, crossRef, RPC_FILE_NO, RPC_NAME_FIELD_NO) with open(inputJsonFile, 'r') as inputFile: pkgJson = {} # group by package allpkgJson = [] inputJson = json.load(inputFile) for icrEntry in inputJson: if generatePDF: _generateICRIndividualPagePDF(icrEntry, date, pdfOutDir) if generateHTML: _generateICRIndividualPage(icrEntry, date, outDir, crossRef) summaryInfo = _convertICREntryToSummaryInfo(icrEntry, crossRef) allpkgJson.append(summaryInfo) if 'CUSTODIAL PACKAGE' in icrEntry: pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'],[]).append(summaryInfo) if generateHTML: _generateICRSummaryPageImpl(allpkgJson, 'ICR List', 'All', date, outDir, isForAll=True) for pkgName, outJson in pkgJson.iteritems(): _generateICRSummaryPageImpl(outJson, 'ICR List', pkgName, date, outDir) logger.warn('Total # entry in PACKAGE_MAP is [%s]', len(PACKAGE_MAP)) logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson)) _generatePkgDepSummaryPage(inputJson, date, outDir, crossRef)
def parse(self, inputFilename, outputFilename): with open(inputFilename, "r") as ICRFile: for line in ICRFile: line = line.rstrip("\r\n") self._curLineNo += 1 """ get rid of lines that are ignored """ if self.isIgnoredLine(line): continue match = START_OF_RECORD.match(line) if match: self._startOfNewItem(match, line) continue match = GENERIC_START_OF_RECORD.search(line) if not match: match = DBA_COMMENTS.match(line) if match and match.group("name") in ICR_FILE_KEYWORDS: fieldName = match.group("name") if isSubFile(fieldName): self._curField = fieldName self._startOfSubFile(match, line) else: logger.debug("field name is: %s", fieldName) logger.debug("cur field is: %s", self._curField) """ Check to see if fieldName is already in the out list """ if isWordProcessingField(self._curField): if self._ignoreKeywordInWordProcessingFields(fieldName): self._appendWordsFieldLine(line) continue # figure out where to store the record self._curField = fieldName self._rewindStack() self._findKeyValueInLine(match, line, self._curRecord) elif self._curField and self._curField in self._curRecord: if len(line.strip()) == 0 and not isWordProcessingField(self._curField): logger.warn("Ignore blank line for current field: [%s]", self._curField) continue self._appendWordsFieldLine(line) else: if self._curRecord: if len(line.strip()) == 0: continue print "No field associated with line %s: %s " % (self._curLineNo, line) logger.info("End of file now") if len(self._curStack) > 0: self._curField = None self._rewindStack() if self._curRecord: logger.info("Add last record: %s", self._curRecord) self._outObject.append(self._curRecord) # pprint.pprint(self._outObject); with open(outputFilename, "w") as out_file: json.dump(self._outObject, out_file, indent=4)
def _convertIndividualFieldValue(self, field, icrEntry, value): if isWordProcessingField(field): if type(value) is list: value = "\n".join(value) value = '<pre>\n' + cgi.escape(value) + '\n</pre>\n' return value if field in field_convert_map: if type(value) is list: logger.warn('field: [%s], value:[%s], icrEntry: [%s]', field, value, icrEntry) return value value = field_convert_map[field](value, icrEntry, crossRef=self._crossRef) return value return value
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference): logger.debug("Current Type is [%s]" % fType) result = self.UNDEFINED_POINTER.search(fType) if result: self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) return result = self.POINTER_TO_REGEX.search(fType) if result: fileNo = result.group('File') filePointedTo = CrossReference.getGlobalByFileNo(fileNo) self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) if not filePointedTo: logger.error("Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line)) else: self._field.setPointedToFile(filePointedTo) return # deal with file pointer to subFiles result = self.SUBFILE_REGEX.search(fType) if result: # create a field for sub file type self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation) fileNo = result.group('File') logger.debug("Pointer to subFile %s" % fileNo) subFile = Global.getSubFileByFileNo(fileNo) if not subFile: # this is a new subfile subFile = FileManFile(fileNo, fName, self._curFile) self._curFile.addFileManSubFile(subFile) logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo())) if self._isSubFile: Global.addFileManSubFile(subFile) self._field.setPointedToSubFile(subFile) CrossReference.addFileManSubFile(subFile) return for (key, value) in self.StringTypeMappingDict.iteritems(): if fType.startswith(key): self._field = FileManFieldFactory.createField(fieldNo, fName, value, fLocation) break if not self._field: # double check the loc and type if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX: fType = line[self.MAXIMIUM_TYPE_START_INDEX:] if fLocation: fLocation = line[line.find(fLocation):self.MAXIMIUM_TYPE_START_INDEX] logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation)) self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference) assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def _getTableRows(self, fileManData, fileNo): rows = [] for ien in getKeys(fileManData.dataEntries.keys(), float): dataEntry = fileManData.dataEntries[ien] if not dataEntry.name: logger.warn("No name for %s" % dataEntry) continue name = dataEntry.name if isFilePointerType(dataEntry): link, name = convertFilePointerToHtml(name) dataHtmlLink = "<a href=\"../%s/%s\">%s</a>" % (fileNo.replace( ".", "_"), getDataEntryHtmlFile( ien, fileNo), str(name).replace("\xa0", "")) rows.append([dataHtmlLink, ien]) return rows
def _generateICRSummaryPage(self, inputJson, date): pkgJson = {} # group by package allpgkJson = [] for icrEntry in inputJson: self._generateICRIndividualPage(icrEntry, date) summaryInfo = self._convertICREntryToSummaryInfo(icrEntry) allpgkJson.append(summaryInfo) if 'CUSTODIAL PACKAGE' in icrEntry: pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'],[]).append(summaryInfo) self._generateICRSummaryPageImpl(allpgkJson, 'ICR List', 'All', date, True) for pkgName, outJson in pkgJson.iteritems(): self._generateICRSummaryPageImpl(outJson, 'ICR List', pkgName, date) logger.warn('Total # entry in pkgMap is [%s]', len(pkgMap)) logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson)) self._generatePkgDepSummaryPage(inputJson, date)
def parseFileManDbJSONFile(self, dbJsonFile): logger.info("Start parsing JSON file [%s]" % dbJsonFile) with open(dbJsonFile, 'r') as jsonFile: dbCallJson = json.load(jsonFile) for pkgItem in dbCallJson: """ find all the routines under that package """ routines = pkgItem['routines'] for rtn in routines: rtnName = rtn['name'] routine = self._crossRef.getRoutineByName(rtnName) if not routine: logger.warn("Can not find routine [%s]" % rtnName) continue fileManGlobals = rtn['Globals'] self._addFileManGlobals(routine, fileManGlobals) fileManFileNos = rtn['FileMan calls'] self._addFileManDBCalls(routine, fileManFileNos)
def parseFileManDbJSONFile(self, dbJsonFile): logger.progress("Start parsing JSON file [%s]" % dbJsonFile) with open(dbJsonFile, 'r') as jsonFile: dbCallJson = json.load(jsonFile) for pkgItem in dbCallJson: # find all the routines under that package routines = pkgItem['routines'] for rtn in routines: rtnName = rtn['name'] routine = self._crossRef.getRoutineByName(rtnName) if not routine: logger.warn("Cannot find routine [%s]" % rtnName) continue fileManGlobals = rtn['Globals'] self._addFileManGlobals(routine, fileManGlobals) fileManFileNos = rtn['FileMan calls'] self._addFileManDBCalls(routine, fileManFileNos)
def visitRoutine(self, routine, outputDir): calledRoutines = routine.getCalledRoutines() if not calledRoutines or len(calledRoutines) == 0: logger.warn("No called Routines found! for package:%s" % routineName) return routineName = routine.getName() if not routine.getPackage(): logger.error("ERROR: package: %s does not belongs to a package" % routineName) return packageName = routine.getPackage().getName() try: dirName = os.path.join(outputDir, packageName) if not os.path.exists(dirName): os.makedirs(dirName) except OSError, e: logger.error("Error making dir %s : Error: %s" % (dirName, e)) return
def _generateICRSummaryPage(self, inputJson, date): self.failures = [] pkgJson = {} # group by package allpgkJson = [] for icrEntry in inputJson: self._generateICRIndividualPage(icrEntry, date) if self._generatePDF: self._generateICRIndividualPagePDF(icrEntry, date) summaryInfo = self._convertICREntryToSummaryInfo(icrEntry) allpgkJson.append(summaryInfo) if 'CUSTODIAL PACKAGE' in icrEntry: pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'],[]).append(summaryInfo) self._generateICRSummaryPageImpl(allpgkJson, 'ICR List', 'All', date, True) for pkgName, outJson in pkgJson.iteritems(): self._generateICRSummaryPageImpl(outJson, 'ICR List', pkgName, date) logger.warn('Total # entry in pkgMap is [%s]', len(pkgMap)) logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson)) self._generatePkgDepSummaryPage(inputJson, date)
def _generateICRSummaryPage(self, inputJson): pkgJson = {} # group by package allpgkJson = [] for icrEntry in inputJson: self._generateICRIndividualPage(icrEntry) summaryInfo = self._convertICREntryToSummaryInfo(icrEntry) allpgkJson.append(summaryInfo) if 'CUSTODIAL PACKAGE' in icrEntry: pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'],[]).append(summaryInfo) self._generateICRSummaryPageImpl(allpgkJson, 'ICR List', 'All', True) for pkgName, outJson in pkgJson.iteritems(): self._generateICRSummaryPageImpl(outJson, 'ICR List', pkgName) logger.warn('Total # entry in pkgMap is [%s]', len(pkgMap)) logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson)) pprint.pprint(set(pkgJson.keys()) - set(pkgMap.keys())) pprint.pprint(set(pgkUpperCaseNameDict.values()) - set(pkgMap.values())) # pprint.pprint(pkgMap) self._generatePkgDepSummaryPage(inputJson)
def _parseSchemaField(self, fieldNo, rootNode, fileSchema): if '0' not in rootNode: logger.warn('%s does not have a 0 subscript' % rootNode) return None zeroFields = rootNode["0"].value if not zeroFields: logger.warn("No value: %s for %s" % (zeroFields, rootNode['0'])) return None zeroFields = zeroFields.split('^') if len(zeroFields) < 2: return FileManFieldFactory.createField( fieldNo, zeroFields[0], FileManField.FIELD_TYPE_NONE, None) types, specifier, filePointedTo, subFile = \ self.parseFieldTypeSpecifier(zeroFields[1]) location = None if len(zeroFields) >= 4 and zeroFields[3]: location = zeroFields[3].strip(' ') if location == ';': # No location information location = None elif location.split(';')[-1] == '0': # 0 means multiple multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER if not types: types = [multipleType] if multipleType in types and types[0] != multipleType: types.remove(multipleType) types.insert(0, multipleType) if not subFile: subFile = filePointedTo if not types: logger.debug('Cannot determine the type for %s, fn: %s, file:%s' % (zeroFields, fieldNo, fileSchema.getFileNo())) types = [FileManField.FIELD_TYPE_NONE] if types and types[0] == FileManField.FIELD_TYPE_SUBFILE_POINTER: if subFile and subFile == fileSchema.getFileNo(): logger.warning("Recursive subfile pointer for %s" % subFile) types = [FileManField.FIELD_TYPE_NONE] fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0], types[0], location) if specifier: fileField.setSpecifier(specifier) self._setFieldSpecificData(zeroFields, fileField, rootNode, fileSchema, filePointedTo, subFile) return fileField
def _parseSchemaField(self, fieldNo, rootNode, fileSchema): if '0' not in rootNode: logger.warn('%s does not have a 0 subscript' % rootNode) return None zeroFields = rootNode["0"].value if not zeroFields: logger.warn("No value: %s for %s" % (zeroFields, rootNode['0'])) return None zeroFields = zeroFields.split('^') if len(zeroFields) < 2: return FileManFieldFactory.createField(fieldNo, zeroFields[0], FileManField.FIELD_TYPE_NONE, None) types, specifier, filePointedTo, subFile = \ self.parseFieldTypeSpecifier(zeroFields[1]) location = None if len(zeroFields) >= 4 and zeroFields[3]: location = zeroFields[3].strip(' ') if location == ';': # No location information location = None elif location.split(';')[-1] == '0': # 0 means multiple multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER if not types: types = [multipleType] if multipleType in types and types[0] != multipleType: types.remove(multipleType) types.insert(0, multipleType) if not subFile: subFile = filePointedTo if not types: logger.debug('Cannot determine the type for %s, fn: %s, file:%s' % (zeroFields, fieldNo, fileSchema.getFileNo())) types = [FileManField.FIELD_TYPE_NONE] if types and types[0] == FileManField.FIELD_TYPE_SUBFILE_POINTER: if subFile and subFile == fileSchema.getFileNo(): logger.warning("Recursive subfile pointer for %s" % subFile) types = [FileManField.FIELD_TYPE_NONE] fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0], types[0], location) if specifier: fileField.setSpecifier(specifier) self._setFieldSpecificData(zeroFields, fileField, rootNode, fileSchema, filePointedTo, subFile) return fileField
def printAllNamespaces(self): crossRef = self._crossRef allPackages = crossRef.getAllPackages() namespaces = set() excludeNamespace = set() for package in allPackages.itervalues(): for namespace in package.getNamespaces(): if (namespace.startswith("!")): excludeNamespace.add(namespace) else: namespaces.add(namespace) sortedSet = sorted(namespaces) sortedExclude = sorted(excludeNamespace) logger.info("Total # of namespace: %d" % len(sortedSet)) logger.info("Total # of excluded namespaces: %d" % len(sortedExclude)) logger.info(sortedSet) logger.info(sortedExclude) for item in excludeNamespace: if item[1:] not in sortedSet: logger.warn("item: %s not in the namespace set" % item[1:])
def _ignoreKeywordInWordProcessingFields(self, fieldName): """ This is a HACK to circuvent the case that there is a keyword value like pair in the sub file word processing fields the keyword is not part of the subFile, we assume it is part of word processing field if any of the parent field already has that field. """ logger.debug("current field is [%s]", self._curField) if self._curRecord and fieldName in self._curRecord: logger.warn("fieldName: [%s] is already parsed, ignore fields", fieldName) return True """ This is some special logic to ignore some of the fields in word processing field """ if fieldName == "ROUTINE": recordToCheck = self._curRecord if self._curStack and len(self._curStack) > 0: # we are in subfile mode and it is a world processing field recordToCheck = self._curStack[0][0] if "REMOTE PROCEDURE" in recordToCheck: logger.warn("Ignore ROUTINE field as it is a REMOTE PROCEDURE type") return True for stackItem in self._curStack: if fieldName in stackItem[0]: logger.warn( "fieldName: [%s] is already parsed in [%s], ignore the words fields", fieldName, stackItem[1] ) return True return False
def _ignoreKeywordInWordProcessingFields(self, fieldName): """ This is a HACK to circuvent the case that there is a keyword value like pair in the sub file word processing fields the keyword is not part of the subFile, we assume it is part of word processing field if any of the parent field already has that field. """ logger.debug('current field is [%s]', self._curField) if self._curRecord and fieldName in self._curRecord: logger.warn('fieldName: [%s] is already parsed, ignore fields', fieldName) return True """ This is some special logic to ignore some of the fields in word processing field """ if fieldName == 'ROUTINE': recordToCheck = self._curRecord if self._curStack and len( self._curStack ) > 0: # we are in subfile mode and it is a world processing field recordToCheck = self._curStack[0][0] if 'REMOTE PROCEDURE' in recordToCheck: logger.warn( 'Ignore ROUTINE field as it is a REMOTE PROCEDURE type') return True for stackItem in self._curStack: if fieldName in stackItem[0]: logger.warn( 'fieldName: [%s] is already parsed in [%s], ignore the words fields', fieldName, stackItem[1]) return True return False
def _generateICRIndividualPagePDF(icrJson, date, pdfOutDir): ien = icrJson['NUMBER'] if 'CUSTODIAL PACKAGE' in icrJson: packageName = icrJson['CUSTODIAL PACKAGE'] pdfOutDir = os.path.join(pdfOutDir, normalizePackageName(packageName)) if not os.path.exists(pdfOutDir): os.mkdir(pdfOutDir) else: # TODO: PDF will not be included in a package bundle and will not be # accessible from the Dox pages logger.warn("Could not find package for: ICR %s" % ien) pdfFile = os.path.join(pdfOutDir, 'ICR-' + ien + '.pdf') # Setup the pdf document buf = io.BytesIO() doc = SimpleDocTemplate( buf, rightMargin=old_div(inch, 2), leftMargin=old_div(inch, 2), topMargin=old_div(inch, 2), bottomMargin=old_div(inch, 2), pagesize=letter, ) pdf = [] # Title pdf.append( Paragraph("%s %s (%s)" % (icrJson['NAME'], 'ICR', ien), STYLES['Heading1'])) # Table _icrDataEntryToPDF(pdf, icrJson, doc) try: doc.build(pdf) with open(pdfFile, 'w') as fd: fd.write(buf.getvalue()) except: global FAILURES FAILURES.append(pdfFile)
def _convertFileManDataToHtml(self, fileManData): fileManDataFileNo = fileManData.fileNo pathSafeFileManDataFileNo = fileManDataFileNo.replace(".", "_") for ien in getKeys(fileManData.dataEntries.keys(), float): dataEntry = fileManData.dataEntries[ien] name = dataEntry.name if not name: logger.warn("no name for %s" % dataEntry) continue outDir = self.outDir fileNo = dataEntry.fileNo if fileNo: outDir = os.path.join(self.outDir, fileNo.replace(".", "_")) tName = "%s-%s" % (pathSafeFileManDataFileNo, ien) if isFilePointerType(dataEntry): link, name = convertFilePointerToHtml(name) outHtmlFileName = getDataEntryHtmlFile(ien, fileManDataFileNo) with open(os.path.join(outDir, outHtmlFileName), 'w') as output: output.write("<html>") outputDataRecordTableHeader(output, tName) output.write("<body id=\"dt_example\">") output.write("""<div id="container" style="width:80%">""") output.write("<h1>%s (%s) %s (%s)</h1>\n" % (name, ien, fileManData.name, fileManDataFileNo)) if fileNo in ['19', '101']: # Todo: Check if the object exists in options/menus first. output.write( "<a style='font-size: 15px;' href='%s../vista_menus.php#%s?name=%s'>View in ViViaN Menu</a>" % (VIV_URL, fileNo, urllib.quote_plus(name))) outputFileEntryTableList(output, tName) """ table body """ output.write("<tbody>\n") self._fileManDataEntryToHtml(output, dataEntry, True) output.write("</tbody>\n") output.write("</table>\n") output.write("</div>\n") output.write("</div>\n") output.write("</body></html>")
def _generateICRIndividualPagePDF(icrJson, date, pdfOutDir): ien = icrJson['NUMBER'] if 'CUSTODIAL PACKAGE' in icrJson: packageName = icrJson['CUSTODIAL PACKAGE'] pdfOutDir = os.path.join(pdfOutDir, normalizePackageName(packageName)) if not os.path.exists(pdfOutDir): os.mkdir(pdfOutDir) else: # TODO: PDF will not be included in a package bundle and will not be # accessible from the Dox pages logger.warn("Could not find package for: ICR %s" % ien) pdfFile = os.path.join(pdfOutDir, 'ICR-' + ien + '.pdf') # Setup the pdf document buf = io.BytesIO() doc = SimpleDocTemplate( buf, rightMargin=inch/2, leftMargin=inch/2, topMargin=inch/2, bottomMargin=inch/2, pagesize=letter, ) pdf = [] # Title pdf.append(Paragraph("%s %s (%s)" % (icrJson['NAME'], 'ICR', ien), STYLES['Heading1'])) # Table _icrDataEntryToPDF(pdf, icrJson, doc) try: doc.build(pdf) with open(pdfFile, 'w') as fd: fd.write(buf.getvalue()) except: global FAILURES FAILURES.append(pdfFile)
def _convertFileManDataToHtml(self, fileManData): fileManDataFileNo = fileManData.fileNo pathSafeFileManDataFileNo = fileManDataFileNo.replace(".", "_") for ien in getKeys(fileManData.dataEntries.keys(), float): dataEntry = fileManData.dataEntries[ien] name = dataEntry.name if not name: logger.warn("no name for %s" % dataEntry) continue outDir = self.outDir fileNo = dataEntry.fileNo if fileNo: outDir = os.path.join(self.outDir, fileNo.replace(".","_")) tName = "%s-%s" % (pathSafeFileManDataFileNo, ien) if isFilePointerType(dataEntry): link, name = convertFilePointerToHtml(name) outHtmlFileName = getDataEntryHtmlFileName(ien, fileManDataFileNo) with open(os.path.join(outDir, outHtmlFileName), 'w') as output: output.write ("<html>") outputDataRecordTableHeader(output, tName) output.write("<body id=\"dt_example\">") output.write("""<div id="container" style="width:80%">""") output.write ("<h1>%s (%s) %s (%s)</h1>\n" % (name, ien, fileManData.name, fileManDataFileNo)) if fileNo in ['19','101']: # Todo: Check if the object exists in options/menus first. output.write("<a style='font-size: 15px;' href='%s../vista_menus.php#%s?name=%s'>View in ViViaN Menu</a>" % (VIV_URL, fileNo, urllib.quote_plus(name))) outputFileEntryTableList(output, tName) """ table body """ output.write("<tbody>\n") self._fileManDataEntryToHtml(output, dataEntry, True) output.write("</tbody>\n") output.write("</table>\n") output.write("</div>\n") output.write("</div>\n") output.write ("</body></html>")
def _updateHL7Reference(self): protocol = self._glbData['101'] outJSON = {} for ien in sorted(protocol.dataEntries.keys(), key=lambda x: float(x)): protocolEntry = protocol.dataEntries[ien] if '4' in protocolEntry.fields: type = protocolEntry.fields['4'].value if (type != 'event driver' and type != 'subscriber'): entryName = protocolEntry.name namespace, package = \ self._crossRef.__categorizeVariableNameByNamespace__(entryName) if package: package.protocol.append(protocolEntry) # only care about the event drive and subscriber type elif (type == 'event driver' or type == 'subscriber'): entryName = protocolEntry.name namespace, package = \ self._crossRef.__categorizeVariableNameByNamespace__(entryName) if package: package.hl7.append(protocolEntry) elif '12' in protocolEntry.fields: # check the packge it belongs pass else: logger.warn("Cannot find a package for HL7: %s" % entryName) for field in ('771', '772'): if field not in protocolEntry.fields: continue hl7Rtn = protocolEntry.fields[field].value if not hl7Rtn: continue for rtn, tag, pos in getMumpsRoutine(hl7Rtn): hl7Info = {"name": entryName, "ien": ien} if tag: hl7Info['tag'] = tag self._rtnRefDict.setdefault(rtn,{}).setdefault('101',[]).append(hl7Info)
def parse(self, inputFilename, outputFilename): global date with open(inputFilename,'r') as ICRFile: for line in ICRFile: line = line.rstrip("\r\n") self._curLineNo +=1 """ get rid of lines that are ignored """ if self.isIgnoredLine(line): continue match = INTEGRATION_REFERENCES_LIST.match(line) if match: date = match.group(1).strip() continue match = START_OF_RECORD.match(line) if match and not self._DBAComments and not self._generalDescription: self._startOfNewItem(match, line) continue match = GENERIC_START_OF_RECORD.search(line) if not match: match = DBA_COMMENTS.match(line) if match: self._DBAComments = True if match and match.group('name') in ICR_FILE_KEYWORDS: fieldName = match.group('name') if fieldName == 'DBA Comments': self._DBAComments = True elif fieldName == 'GENERAL DESCRIPTION': self._generalDescription = True if self._DBAComments: if fieldName in ICR_FILE_KEYWORDS: self._DBAComments = False elif self._generalDescription: if line.startswith(" STATUS:"): # Starts with exactly 2 spaces self._generalDescription = False if self._DBAComments: fieldName = 'DBA Comments' if self._curField == fieldName: self._appendWordsFieldLine(line) else: self._curField = fieldName name = match.group('name') # this is the name part restOfLine = line[match.end():] self._curRecord[name] = restOfLine.strip() elif self._generalDescription: fieldName = 'GENERAL DESCRIPTION' if self._curField == fieldName: self._appendWordsFieldLine(line) else: self._curField = fieldName name = match.group('name') # this is the name part restOfLine = line[match.end():] self._curRecord[name] = restOfLine.strip() elif isSubFile(fieldName): self._curField = fieldName self._startOfSubFile(match, line) else: logger.debug('field name is: %s', fieldName) logger.debug('cur field is: %s', self._curField) """ Check to see if fieldName is already in the out list """ if isWordProcessingField(self._curField): if self._ignoreKeywordInWordProcessingFields(fieldName): self._appendWordsFieldLine(line) continue # figure out where to store the record self._curField = fieldName self._rewindStack(); self._findKeyValueInLine(match, line, self._curRecord) elif self._curField and self._curField in self._curRecord: if len(line.strip()) == 0 and not isWordProcessingField(self._curField): logger.warn('Ignore blank line for current field: [%s]', self._curField) continue self._appendWordsFieldLine(line) else: if self._curRecord: if len(line.strip()) == 0: continue print 'No field associated with line %s: %s ' % (self._curLineNo, line) logger.info('End of file now') if len(self._curStack) > 0: self._curField = None self._rewindStack() if self._curRecord: logger.info('Add last record: %s', self._curRecord) self._outObject.append(self._curRecord) # pprint.pprint(self._outObject); with open(outputFilename, 'w') as out_file: json.dump(self._outObject,out_file, indent=4)
def onSectionStart(self, line, section, Global, CrossReference): logger.debug("[%s]" % line) self._lines = [] result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search( line) assert result fileNo = result.group('FileNo') fieldNo = result.group("FieldNo") self._isSubFile = float(fileNo) != float(Global.getFileNo()) if self._isSubFile: logger.debug("%s is a subfile" % fileNo) self._curFile = Global.getSubFileByFileNo(fileNo) assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % ( fileNo, Global.getFileNo(), line) else: self._curFile = Global restOfLineStart = line.find("," + fieldNo) + len(fieldNo) startIdent = self.DEFAULT_NAME_INDENT #if len(fileNo) + 4 > startIdent: # startIdent = self defaultIdentLevel = self.__getDefaultIndentLevel__( self._curFile, self.DEFAULT_NAME_INDENT) if restOfLineStart > defaultIdentLevel: logger.debug( "FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel)) try: floatValue = float(fieldNo) except ValueError: logger.error("invalid fieldNo %s" % fieldNo) fieldNo = line[line.find(",") + 1:defaultIdentLevel] floatValue = float(fieldNo) restOfLine = line[line.find("," + fieldNo) + len(fieldNo) + 1:].strip() logger.debug("Parsing [%s]" % restOfLine) result = self.NAME_LOC_TYPE_REGEX.search(restOfLine) fName, fType, fLocation = None, None, None if result: logger.debug("FileNo: %s, Field#: %s, Name: %s, Loc %s, Type %s" % (fileNo, fieldNo, result.group('Name').rstrip(), result.group('Loc'), result.group('Type'))) fName = result.group('Name').strip() fLocation = result.group('Loc').strip() if fLocation == ";": fLocation = None fType = result.group('Type').strip() else: # handle three cases, 1. no location info 2. no type info 3. Both if restOfLine.find(";") != -1: # missing type info logger.warn("Missing Type information [%s]" % line) result = self.NAME_LOC_REGEX.search(restOfLine) if result: logger.debug("Name: %s, Loc %s" % (result.group('Name'), result.group('Loc'))) fName = result.group('Name').strip() fLocation = result.group('Loc').strip() else: logger.error("Could not parse [%s]" % restOfLine) return else: # missing location, assume at least two space seperate name and type logger.warn("Missing location information [%s]" % line) result = self.NAME_TYPE_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fType = result.group('Type').strip() logger.debug("Name: %s, Type %s" % (result.group('Name'), result.group('Type'))) else: logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line)) stripedType = "" if fType: stripedType = self.__stripFieldAttributes__(fType) if len(stripedType) > 0: self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference) else: self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation) logger.debug("Add field %s to File %s" % (fName, self._curFile.getFileNo())) self._curFile.addFileManField(self._field) if len(stripedType) > 0: self.__parseFieldAttributes__(fType)
def _parseIndividualFieldDetail(self, value, fieldAttr, outDataEntry): value = value.strip(' ') if not value: return fieldDetail = value pointerFileNo = None if fieldAttr.isSetType(): setDict = fieldAttr.getSetMembers() if setDict and value in setDict: fieldDetail = setDict[value] elif fieldAttr.isFilePointerType() or fieldAttr.isVariablePointerType(): fileNo = None ien = None if fieldAttr.isFilePointerType(): filePointedTo = fieldAttr.getPointedToFile() if filePointedTo: fileNo = filePointedTo.getFileNo() ien = value else: fieldDetail = 'No Pointed to File' else: # for variable pointer type vpInfo = value.split(';') if len(vpInfo) != 2: logger.error("Unknown variable pointer format: %s" % value) fieldDetail = "Unknow Variable Pointer" else: fileNo = self.getFileNoByGlobalLocation(vpInfo[1]) ien = vpInfo[0] if not fileNo: logger.warn("Could not find File for %s" % value) fieldDetail = 'Global Root: %s, IEN: %s' % (vpInfo[1], ien) if fileNo and ien: fieldDetail = '^'.join((fileNo, ien)) idxName = self._getFileKeyIndex(fileNo, ien) if idxName: idxes = str(idxName).split('^') if len(idxes) == 1: fieldDetail = '^'.join((fieldDetail, str(idxName))) elif len(idxes) == 3: fieldDetail = '^'.join((fieldDetail, str(idxes[-1]))) elif fileNo == self._curFileNo: pointerFileNo = fileNo elif fieldAttr.getType() == FileManField.FIELD_TYPE_DATE_TIME: # datetime if value.find(',') >=0: fieldDetail = horologToDateTime(value) else: outDt = fmDtToPyDt(value) if outDt: fieldDetail = outDt else: logger.warn("Could not parse Date/Time: %s" % value) elif fieldAttr.getName().upper().startswith("TIMESTAMP"): # timestamp field if value.find(',') >=0: fieldDetail = horologToDateTime(value) if outDataEntry: dataField = FileManDataField(fieldAttr.getFieldNo(), fieldAttr.getType(), fieldAttr.getName(), fieldDetail) if pointerFileNo: self._addDataFieldToPointerRef(pointerFileNo, value, dataField) outDataEntry.addField(dataField) if fieldAttr.getFieldNo() == '.01': outDataEntry.name = fieldDetail outDataEntry.type = fieldAttr.getType() return fieldDetail
def _generateDataTableHtml(self, fileManData, fileNo): outDir = self.outDir isLargeFile = len(fileManData.dataEntries) > 4500 tName = normalizePackageName(fileManData.name) outDir = os.path.join(self.outDir, fileNo.replace(".", "_")) if not os.path.exists(outDir): os.mkdir(outDir) with open("%s/%s.html" % (outDir, fileNo), 'w') as output: output.write("<html>\n") if isLargeFile: ajexSrc = "%s_array.txt" % fileNo outputLargeDataListTableHeader(output, ajexSrc, tName) else: outputDataListTableHeader(output, tName) output.write("<body id=\"dt_example\">") output.write("""<div id="container" style="width:80%">""") output.write("<h1>File %s(%s) Data List</h1>" % (tName, fileNo)) writeTableListInfo(output, tName) if not isLargeFile: output.write("<tbody>\n") for ien in getKeys(fileManData.dataEntries.keys(), float): dataEntry = fileManData.dataEntries[ien] if not dataEntry.name: logger.warn("no name for %s" % dataEntry) continue name = dataEntry.name if isFilePointerType(dataEntry): link, name = convertFilePointerToHtml(dataEntry.name) dataHtmlLink = "<a href=\"../%s/%s\">%s</a>" % ( fileNo.replace( ".", "_"), getDataEntryHtmlFile(ien, fileNo), name) tableRow = [dataHtmlLink, dataEntry.ien] output.write("<tr>\n") """ table body """ for item in tableRow: output.write("<td>%s</td>\n" % item) output.write("</tr>\n") output.write("</tbody>\n") output.write("</table>\n") output.write("</div>\n") output.write("</div>\n") output.write("</body></html>\n") if isLargeFile: logger.info("Writing Ajax file: %s" % ajexSrc) """ Write out the data file in JSON format """ outJson = {"aaData": []} with open(os.path.join(outDir, ajexSrc), 'w') as output: outArray = outJson["aaData"] for ien in getKeys(fileManData.dataEntries.keys(), float): dataEntry = fileManData.dataEntries[ien] if not dataEntry.name: logger.warn("No name for %s" % dataEntry) continue name = dataEntry.name if isFilePointerType(dataEntry): link, name = convertFilePointerToHtml(dataEntry.name) dataHtmlLink = "<a href=\"../%s/%s\">%s</a>" % ( fileNo.replace( ".", "_"), getDataEntryHtmlFile( ien, fileNo), str(name).replace("\xa0", "")) outArray.append([dataHtmlLink, ien]) json.dump(outJson, output)
def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = codecs.open(file, 'r', encoding='utf-8', errors='ignore') lineNo = 0 fileName = os.path.basename(file) result = ZWR_FILENO_REGEX.search(fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = ZWR_NAMESPACE_REGEX.search(fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ", '') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 if not fileNo: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError as es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error( "Duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close() logger.info( "Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d" % (len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet))) sortedKeyList = sorted( list(allGlobals.keys()), key=lambda item: float(allGlobals[item].getFileNo())) for key in sortedKeyList: globalVar = allGlobals[key] # fix the uncategoried item if globalVar.getFileNo() in fileNoPackageMappingDict: globalVar.setPackage(allPackages[fileNoPackageMappingDict[ globalVar.getFileNo()]]) crossReference.addGlobalToPackage(globalVar, globalVar.getPackage().getName())
def isIgnoredLine(self, line): for regEx in LINES_TO_IGNORE: if regEx.match(line): logger.warn('Ignore line %s', line) return True return False
def _setFieldSpecificData(self, zeroFields, fileField, rootNode, fileSchema, filePointedTo, subFile): if fileField.getType() == FileManField.FIELD_TYPE_FILE_POINTER: fileGlobalRoot = "" if len(zeroFields) >= 3: fileGlobalRoot = zeroFields[2] if filePointedTo: if filePointedTo not in self._allSchema: """ create a new fileman file """ self._allSchema[filePointedTo] = Global( fileGlobalRoot, filePointedTo, "") pointedToFile = self._allSchema[filePointedTo] assert pointedToFile.isRootFile() fileField.setPointedToFile(pointedToFile) globalName = pointedToFile.getName() fileNo = fileSchema.getFileNo() if fileSchema.isSubFile(): fileNo = fileSchema.getRootFile().getFileNo() self._addToFileDepDict(fileNo, pointedToFile.getFileNo()) if fileGlobalRoot: if not globalName: pointedToFile.setName(fileGlobalRoot) elif globalName != fileGlobalRoot: logger.error( "%s: FileMan global root mismatch '%s' : '%s'" % (zeroFields, globalName, fileGlobalRoot)) else: logger.info("@TODO, find file global root for # %s" % filePointedTo) elif fileGlobalRoot: self._noPointedToFiles[fileGlobalRoot] = Global(fileGlobalRoot) logger.info("@TODO, set the file number for %s" % fileGlobalRoot) else: logger.warn( "No pointed to file set for file:%s: field:%r 0-index:%s" % (fileSchema.getFileNo(), fileField, zeroFields)) elif fileField.getType() == FileManField.FIELD_TYPE_SUBFILE_POINTER: if subFile: if subFile not in self._allSchema: self._allSchema[subFile] = FileManFile( subFile, "", fileSchema) subFileSchema = self._allSchema[subFile] subFileSchema.setParentFile(fileSchema) fileSchema.addFileManSubFile(subFileSchema) fileField.setPointedToSubFile(subFileSchema) else: logger.warn( "No subfile is set for file:%s, field:%r 0-index:%s" % (fileSchema.getFileNo(), fileField, zeroFields)) elif fileField.getType( ) == FileManField.FIELD_TYPE_SET and not subFile: setDict = dict( [x.split(':') for x in zeroFields[2].rstrip(';').split(';')]) fileField.setSetMembers(setDict) elif fileField.getType( ) == FileManField.FIELD_TYPE_VARIABLE_FILE_POINTER: if "V" in rootNode: # parsing variable pointer vptrs = parsingVariablePointer(rootNode['V']) vpFileSchemas = [] if vptrs: for x in vptrs: if x not in self._allSchema: self._allSchema[x] = Global("", x, "") pointedToFile = self._allSchema[x] if pointedToFile.isSubFile(): logger.error( "Field: %r point to subFile: %s, parent: %s" % (fileField, pointedToFile, pointedToFile.getParentFile())) else: fileNo = fileSchema.getFileNo() if fileSchema.isSubFile(): fileNo = fileSchema.getRootFile().getFileNo() self._addToFileDepDict(fileNo, pointedToFile.getFileNo()) vpFileSchemas.append(self._allSchema[x]) fileField.setPointedToFiles(vpFileSchemas)
def isIgnoredLine(self, line): for regEx in LINES_TO_IGNORE: if regEx.match(line): logger.warn("Ignore line %s", line) return True return False
def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): logger.info("Package: %s is new" % packageName) crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = open(file, 'r') lineNo = 0 fileName = os.path.basename(file) result = re.search("(?P<fileNo>^[0-9.]+)(-1)?\+(?P<des>.*)\.zwr$", fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = re.search("(?P<namespace>^[^.]+)\.zwr$", fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ", '') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo == 1: assert re.search('ZWR', line.strip()) if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 logger.debug("globalName: %s, Des: %s, fileNo: %s, package: %s" % (globalName, globalDes, fileNo, packageName)) if len(fileNo) == 0: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError, es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error( "Error, duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close()
def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = open(file, 'r') lineNo = 0 fileName = os.path.basename(file) result = ZWR_FILENO_REGEX.search(fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = ZWR_NAMESPACE_REGEX.search(fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ",'') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 if not fileNo: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError, es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error("Duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close()
def _setFieldSpecificData(self, zeroFields, fileField, rootNode, fileSchema, filePointedTo, subFile): if fileField.getType() == FileManField.FIELD_TYPE_FILE_POINTER: fileGlobalRoot = "" if len(zeroFields) >= 3: fileGlobalRoot = zeroFields[2] if filePointedTo: if filePointedTo not in self._allSchema: """ create a new fileman file """ self._allSchema[filePointedTo] = Global(fileGlobalRoot, filePointedTo, "") pointedToFile = self._allSchema[filePointedTo] assert pointedToFile.isRootFile() fileField.setPointedToFile(pointedToFile) globalName = pointedToFile.getName() fileNo = fileSchema.getFileNo() if fileSchema.isSubFile(): fileNo = fileSchema.getRootFile().getFileNo() self._addToFileDepDict(fileNo, pointedToFile.getFileNo()) if fileGlobalRoot: if not globalName: pointedToFile.setName(fileGlobalRoot) elif globalName != fileGlobalRoot: logger.warning("%s: FileMan global root mismatch '%s' : '%s'" % (zeroFields, globalName, fileGlobalRoot)) else: logger.info("@TODO, find file global root for # %s" % filePointedTo) elif fileGlobalRoot: self._noPointedToFiles[fileGlobalRoot] = Global(fileGlobalRoot) logger.info("@TODO, set the file number for %s" % fileGlobalRoot) else: logger.warn("No pointed to file set for file:%s: field:%r 0-index:%s" % (fileSchema.getFileNo(), fileField, zeroFields)) elif fileField.getType() == FileManField.FIELD_TYPE_SUBFILE_POINTER: if subFile: if subFile not in self._allSchema: self._allSchema[subFile] = FileManFile(subFile, "", fileSchema) subFileSchema = self._allSchema[subFile] subFileSchema.setParentFile(fileSchema) fileSchema.addFileManSubFile(subFileSchema) fileField.setPointedToSubFile(subFileSchema) else: logger.warn("No subfile is set for file:%s, field:%r 0-index:%s" % (fileSchema.getFileNo(), fileField, zeroFields)) elif fileField.getType() == FileManField.FIELD_TYPE_SET and not subFile: setDict = dict([x.split(':') for x in zeroFields[2].rstrip(';').split(';')]) fileField.setSetMembers(setDict) elif fileField.getType() == FileManField.FIELD_TYPE_VARIABLE_FILE_POINTER: if "V" in rootNode: # parsing variable pointer vptrs = parsingVariablePointer(rootNode['V']) vpFileSchemas = [] if vptrs: for x in vptrs: if x not in self._allSchema: self._allSchema[x] = Global("", x, "") pointedToFile = self._allSchema[x] if pointedToFile.isSubFile(): logger.error("Field: %r point to subFile: %s, parent: %s" % (fileField, pointedToFile, pointedToFile.getParentFile())) else: fileNo = fileSchema.getFileNo() if fileSchema.isSubFile(): fileNo = fileSchema.getRootFile().getFileNo() self._addToFileDepDict(fileNo, pointedToFile.getFileNo()) vpFileSchemas.append(self._allSchema[x]) fileField.setPointedToFiles(vpFileSchemas)
def parse(self, inputFilename, outputFilename): global date with open(inputFilename,'r') as ICRFile: for line in ICRFile: line = line.rstrip("\r\n") self._curLineNo +=1 # get rid of lines that are ignored if self.isIgnoredLine(line): continue match = INTEGRATION_REFERENCES_LIST.match(line) if match: date = match.group(1).strip() continue match = START_OF_RECORD.match(line) if match and not self._DBAComments and not self._generalDescription: self._startOfNewItem(match, line) continue match = GENERIC_START_OF_RECORD.search(line) if not match: match = DBA_COMMENTS.match(line) if match: self._DBAComments = True if match and match.group('name') in ICR_FILE_KEYWORDS: fieldName = match.group('name') if fieldName == 'DBA Comments': self._DBAComments = True elif fieldName == 'GENERAL DESCRIPTION': self._generalDescription = True if self._DBAComments: if fieldName in ICR_FILE_KEYWORDS: self._DBAComments = False elif self._generalDescription: if line.startswith(" STATUS:"): # Starts with exactly 2 spaces self._generalDescription = False if self._DBAComments: fieldName = 'DBA Comments' if self._curField == fieldName: self._appendWordsFieldLine(line) else: self._curField = fieldName name = match.group('name') # this is the name part restOfLine = line[match.end():] self._curRecord[name] = restOfLine.strip() elif self._generalDescription: fieldName = 'GENERAL DESCRIPTION' if self._curField == fieldName: self._appendWordsFieldLine(line) else: # Starting to process general description self._curField = fieldName self._rewindStack(); self._findKeyValueInLine(match, line, self._curRecord) elif isSubFile(fieldName): self._curField = fieldName self._startOfSubFile(match, line) else: logger.debug('field name is: %s', fieldName) logger.debug('cur field is: %s', self._curField) """ Check to see if fieldName is already in the out list """ if isWordProcessingField(self._curField): if self._ignoreKeywordInWordProcessingFields(fieldName): self._appendWordsFieldLine(line) continue # figure out where to store the record self._curField = fieldName self._rewindStack(); self._findKeyValueInLine(match, line, self._curRecord) elif self._curField and self._curField in self._curRecord: if len(line.strip()) == 0 and not isWordProcessingField(self._curField): logger.warn('Ignore blank line for current field: [%s]', self._curField) continue self._appendWordsFieldLine(line) else: if self._curRecord: if len(line.strip()) == 0: continue print 'No field associated with line %s: %s ' % (self._curLineNo, line) logger.info('End of file now') if len(self._curStack) > 0: self._curField = None self._rewindStack() if self._curRecord: logger.info('Add last record: %s', self._curRecord) self._outObject.append(self._curRecord) # pprint.pprint(self._outObject); with open(outputFilename, 'w') as out_file: json.dump(self._outObject,out_file, indent=4)
def onSectionStart(self, line, section, Global, CrossReference): self._lines = [] result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line) assert result fileNo = result.group('FileNo') fieldNo = result.group("FieldNo") self._isSubFile = float(fileNo) != float(Global.getFileNo()) if self._isSubFile: self._curFile = Global.getSubFileByFileNo(fileNo) assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line) else: self._curFile = Global restOfLineStart = line.find("," + fieldNo) + len(fieldNo) startIdent = self.DEFAULT_NAME_INDENT defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT) if restOfLineStart > defaultIdentLevel: logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel)) try: floatValue = float(fieldNo) except ValueError: logger.error("invalid fieldNo %s" % fieldNo) fieldNo = line[line.find(",")+1:defaultIdentLevel] floatValue = float(fieldNo) restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip() result = NAME_LOC_TYPE_REGEX.search(restOfLine) fName, fType, fLocation = None, None, None if result: fName = result.group('Name').strip() fLocation = result.group('Loc').strip() if fLocation == ";": fLocation = None fType = result.group('Type').strip() else: # handle three cases, 1. no location info 2. no type info 3. Both if restOfLine.find(";") != -1: # missing type info logger.warn("Missing Type information [%s]" % line) result = NAME_LOC_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fLocation = result.group('Loc').strip() else: logger.error("Could not parse [%s]" % restOfLine) return else: # missing location, assume at least two space seperate name and type result = NAME_TYPE_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fType = result.group('Type').strip() else: logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line)) stripedType = "" if fType: stripedType = self.__stripFieldAttributes__(fType) if stripedType: self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference) else: self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation) self._curFile.addFileManField(self._field) if stripedType: self.__parseFieldAttributes__(fType)
def _parseIndividualFieldDetail(self, value, fieldAttr, outDataEntry): value = value.strip(' ') if not value: return fieldDetail = value pointerFileNo = None if fieldAttr.isSetType(): setDict = fieldAttr.getSetMembers() if setDict and value in setDict: fieldDetail = setDict[value] elif fieldAttr.isFilePointerType() or fieldAttr.isVariablePointerType( ): fileNo = None ien = None if fieldAttr.isFilePointerType(): filePointedTo = fieldAttr.getPointedToFile() if filePointedTo: fileNo = filePointedTo.getFileNo() ien = value else: fieldDetail = 'No Pointed to File' else: # for variable pointer type vpInfo = value.split(';') if len(vpInfo) != 2: logger.error("Unknown variable pointer format: %s" % value) fieldDetail = "Unknow Variable Pointer" else: fileNo = self.getFileNoByGlobalLocation(vpInfo[1]) ien = vpInfo[0] if not fileNo: logger.warn("Could not find File for %s" % value) fieldDetail = 'Global Root: %s, IEN: %s' % (vpInfo[1], ien) if fileNo and ien: fieldDetail = '^'.join((fileNo, ien)) idxName = self._getFileKeyIndex(fileNo, ien) if idxName: idxes = str(idxName).split('^') if len(idxes) == 1: fieldDetail = '^'.join((fieldDetail, str(idxName))) elif len(idxes) == 3: fieldDetail = '^'.join((fieldDetail, str(idxes[-1]))) elif fileNo == self._curFileNo: pointerFileNo = fileNo elif fieldAttr.getType( ) == FileManField.FIELD_TYPE_DATE_TIME: # datetime if value.find(',') >= 0: fieldDetail = horologToDateTime(value) else: outDt = fmDtToPyDt(value) if outDt: fieldDetail = outDt else: logger.warn("Could not parse Date/Time: %s" % value) elif fieldAttr.getName().upper().startswith( "TIMESTAMP"): # timestamp field if value.find(',') >= 0: fieldDetail = horologToDateTime(value) if outDataEntry: dataField = FileManDataField(fieldAttr.getFieldNo(), fieldAttr.getType(), fieldAttr.getName(), fieldDetail) if pointerFileNo: self._addDataFieldToPointerRef(pointerFileNo, value, dataField) outDataEntry.addField(dataField) if fieldAttr.getFieldNo() == '.01': outDataEntry.name = fieldDetail outDataEntry.type = fieldAttr.getType() return fieldDetail
def convertJson(inputJsonFile, date, MRepositDir, patchRepositDir, generateHTML, generatePDF, outDir=None, pdfOutDir=None, local=False): if not generateHTML and not generatePDF: raise Exception("Nothing to generate!") global DOX_URL global VIVIAN_URL DOX_URL = getDOXURL(local) VIVIAN_URL = getViViaNURL(local) if generateHTML: if not outDir: raise Exception("Must specify Output directory") if not os.path.exists(outDir): # Will also create intermediate directories if needed os.makedirs(outDir) if generatePDF: if not pdfOutDir: raise Exception("Must specify PDF Output directory") # Will also create intermediate directories if needed if not os.path.exists(pdfOutDir): os.makedirs(pdfOutDir) from InitCrossReferenceGenerator import parseCrossReferenceGeneratorArgs crossRef = parseCrossReferenceGeneratorArgs(MRepositDir, patchRepositDir) global RPC_NAME_TO_IEN_MAPPING RPC_NAME_TO_IEN_MAPPING = generateSingleFileFieldToIenMappingBySchema(MRepositDir, crossRef) with open(inputJsonFile, 'r') as inputFile: pkgJson = {} # group by package allpkgJson = [] inputJson = json.load(inputFile) for icrEntry in inputJson: if 'NUMBER' not in icrEntry: logger.error("Could not parse entry: " + str(icrEntry)) continue if 'CUSTODIAL PACKAGE' in icrEntry: pkgName = icrEntry['CUSTODIAL PACKAGE'] if crossRef.getMappedPackageName(pkgName) is None: crossRef.addMappedPackage(pkgName, crossRef.normalizePackageName(pkgName).title()) logger.warning("Adding package " + pkgName + " to package name map.") if generatePDF: _generateICRIndividualPagePDF(icrEntry, date, pdfOutDir) if generateHTML: _generateICRIndividualPage(icrEntry, date, outDir, crossRef) summaryInfo = _convertICREntryToSummaryInfo(icrEntry, crossRef) allpkgJson.append(summaryInfo) if 'CUSTODIAL PACKAGE' in icrEntry: pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'], []).append(summaryInfo) if generateHTML: _generateICRSummaryPageImpl(allpkgJson, 'ICR List', 'All', date, outDir, crossRef, isForAll=True) for pkgName, outJson in iteritems(pkgJson): _generateICRSummaryPageImpl(outJson, 'ICR List', pkgName, date, outDir, crossRef) logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson)) _generatePkgDepSummaryPage(inputJson, date, outDir, crossRef)