Ejemplo n.º 1
0
def _addToPackageMap(icrEntry, pkgName):
    if 'CUSTODIAL PACKAGE' in icrEntry:
        icrPkg = icrEntry['CUSTODIAL PACKAGE']
        if icrPkg not in PACKAGE_MAP:
            PACKAGE_MAP[icrPkg] = pkgName
        elif PACKAGE_MAP[icrPkg] != pkgName:
            logger.warning('[%s] mapped to [%s] and [%s]', icrPkg, PACKAGE_MAP[icrPkg], pkgName)
Ejemplo n.º 2
0
    def __postParsing__(self, Routine, CrossReference):
        routineDetail = routineTag.search(self._varName.strip())
        if routineDetail:
            routineName = routineDetail.group('name')
            if not validRoutineName.search(routineName):
                logger.warning("invalid Routine Name: %s in routine:%s, package: %s" %
                             (routineName, Routine, Routine.getPackage()))
                return
            if (routineName.startswith("%")):
               CrossReference.addPercentRoutine(routineName)
               # ignore mumps routine for now
               if CrossReference.isMumpsRoutine(routineName):
                   return
#                   routineName=routineName[1:]
            if CrossReference.routineNeedRename(routineName):
                routineName = CrossReference.getRenamedRoutineName(routineName)
            tag = ""
            if routineDetail.group('external'):
                tag += routineDetail.group('external')
            if routineDetail.group('tag'):
                tag += routineDetail.group('tag')
            if not CrossReference.hasRoutine(routineName):
                # automatically categorize the routine by the namespace
                # if could not find one, assign to Uncategorized
                defaultPackageName = "Uncategorized"
                (namespace, package) = CrossReference.categorizeRoutineByNamespace(routineName)
                if namespace and package:
                    defaultPackageName = package.getName()
                CrossReference.addRoutineToPackageByName(routineName, defaultPackageName, False)
            routine = CrossReference.getRoutineByName(routineName)
            Routine.addCalledRoutines(routine, tag, self._varValue)
Ejemplo n.º 3
0
def _getPackageHRefLink(pkgName, icrEntry, **kargs):
    global pgkUpperCaseNameDict
    if pkgName in PACKAGE_MAP:
        pkgLink = getPackageHtmlFileName(PACKAGE_MAP[pkgName])
        return '<a href=\"%s%s\">%s</a>' % (DOX_URL, pkgLink, pkgName)
    crossRef = None
    if 'crossRef' in kargs:
        crossRef = kargs['crossRef']
    if crossRef:
        if not pgkUpperCaseNameDict:
            for name in crossRef.getAllPackages().iterkeys():
                pgkUpperCaseNameDict[name.upper()] = name
        upperName = _normalizeName(pkgName).upper()
        if upperName in pgkUpperCaseNameDict:
            _addToPackageMap(icrEntry, pgkUpperCaseNameDict[upperName])
            return '<a href=\"%s%s\">%s</a>' % (DOX_URL,
                                                getPackageHtmlFileName(pgkUpperCaseNameDict[upperName]),
                                                pkgName)
        pkg = crossRef.getPackageByName(pkgName)
        if not pkg:
            pkgRename = _normalizeName(pkgName).title()
            pkg = crossRef.getPackageByName(pkgRename)
        if not pkg:
            pkgRename = _normalizeName(pkgName)
            pkg = crossRef.getPackageByName(pkgRename)
        if pkg:
            _addToPackageMap(icrEntry, pkg.getName())
            pkgLink = getPackageHtmlFileName(pkg.getName())
            return '<a href=\"%s%s\">%s</a>' % (DOX_URL, pkgLink, pkgName)
        else:
            logger.warning('Cannot find mapping for package: [%s]', pkgName)
    return pkgName
Ejemplo n.º 4
0
def _getFileManFileHRefLink(fileNo, icrEntry, **kargs):
    crossRef = None
    if 'crossRef' in kargs:
        crossRef = kargs['crossRef']
    if crossRef:
        fileInfo = crossRef.getGlobalByFileNo(fileNo)
        if fileInfo:
            linkName = getGlobalHtmlFileNameByName(fileInfo.getName())
            # _addToPackageMap(icrEntry, fileInfo.getPackage().getName())
            return '<a href=\"%s%s\">%s</a>' % (DOX_URL, linkName, fileNo)
        else:
            logger.warning('Cannot find file: [%s]', fileNo)
    return fileNo
Ejemplo n.º 5
0
 def _parseSubFileField(self, dataRoot, fieldAttr, outDataEntry):
   subFile = fieldAttr.getPointedToSubFile()
   if fieldAttr.hasSubType(FileManField.FIELD_TYPE_WORD_PROCESSING):
     outLst = self._parsingWordProcessingNode(dataRoot)
     outDataEntry.addField(FileManDataField(fieldAttr.getFieldNo(),
                                      FileManField.FIELD_TYPE_WORD_PROCESSING,
                                      fieldAttr.getName(),
                                      outLst))
   elif subFile:
     subFileData = FileManFileData(subFile.getFileNo(),
                                   subFile.getFileManName())
     self._parseDataBySchema(dataRoot, subFile, subFileData)
     outDataEntry.addField(FileManDataField(fieldAttr.getFieldNo(),
                                       FileManField.FIELD_TYPE_SUBFILE_POINTER,
                                       fieldAttr.getName(),
                                       subFileData))
   else:
     logger.warning("Do not know how to intepret the schema %s" % fieldAttr)
Ejemplo n.º 6
0
 def parseLine(self, line, Global, CrossReference):
     assert self._global
     strippedLine = line.rstrip(" ")
     if not strippedLine:
         return
     value = strippedLine[self.POINTED_TO_BY_VALUE_INDEX:]
     result = POINTED_TO_BY_VALUE_REGEX.search(value)
     if result:
         fileManNo = result.group("FileNo")
         fieldNo = result.group('fieldNo')
         subFileNo = result.group('subFieldNo')
         pointedByGlobal = CrossReference.getGlobalByFileNo(fileManNo)
         if pointedByGlobal:
             self._global.addPointedToByFile(pointedByGlobal, fieldNo, subFileNo)
         else:
             logger.warning("Could not find global based on %s, %s" %
                            (fileManNo, result.group("Name")))
     else:
         logger.error("Could not parse pointer reference [%s] in file [%s]" % (line, self._global.getFileNo()))
Ejemplo n.º 7
0
 def _parseSchemaField(self, fieldNo, rootNode, fileSchema):
   if '0' not in rootNode:
     logger.warn('%s does not have a 0 subscript' % rootNode)
     return None
   zeroFields = rootNode["0"].value
   if not zeroFields:
     logger.warn("No value: %s for %s" % (zeroFields, rootNode['0']))
     return None
   zeroFields = zeroFields.split('^')
   if len(zeroFields) < 2:
     return FileManFieldFactory.createField(fieldNo, zeroFields[0],
                                            FileManField.FIELD_TYPE_NONE, None)
   types, specifier, filePointedTo, subFile = \
       self.parseFieldTypeSpecifier(zeroFields[1])
   location = None
   if len(zeroFields) >= 4 and zeroFields[3]:
     location = zeroFields[3].strip(' ')
     if location == ';': # No location information
       location = None
     elif location.split(';')[-1] == '0': # 0 means multiple
       multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER
       if not types:
         types = [multipleType]
       if multipleType in types and types[0] != multipleType:
         types.remove(multipleType)
         types.insert(0, multipleType)
         if not subFile: subFile = filePointedTo
   if not types:
     logger.debug('Cannot determine the type for %s, fn: %s, file:%s' %
                  (zeroFields, fieldNo, fileSchema.getFileNo()))
     types = [FileManField.FIELD_TYPE_NONE]
   if types and types[0]  == FileManField.FIELD_TYPE_SUBFILE_POINTER:
     if subFile and subFile == fileSchema.getFileNo():
       logger.warning("Recursive subfile pointer for %s" % subFile)
       types = [FileManField.FIELD_TYPE_NONE]
   fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0],
                                               types[0], location)
   if specifier:
     fileField.setSpecifier(specifier)
   self._setFieldSpecificData(zeroFields, fileField, rootNode,
                             fileSchema, filePointedTo, subFile)
   return fileField
 def parseLine(self, line, Global, CrossReference):
     assert self._global
     strippedLine = line.rstrip(" ")
     if len(strippedLine) == 0:
         return
     value = strippedLine[self.POINTED_TO_BY_VALUE_INDEX:]
     logger.debug("Parsing line [%s]" % value)
     result = self.POINTED_TO_BY_VALUE.search(value)
     if result:
         fileManNo = result.group("FileNo")
         fieldNo = result.group('fieldNo')
         subFileNo = result.group('subFieldNo')
         logger.debug("File # %s, field # %s, sub-field # %s" % (fileManNo, fieldNo, subFileNo))
         pointedByGlobal = CrossReference.getGlobalByFileNo(fileManNo)
         if pointedByGlobal:
             self._global.addPointedToByFile(pointedByGlobal, fieldNo, subFileNo)
             logger.debug("added global to pointed list: %s, %s, %s" %
                         (fileManNo, fieldNo, subFileNo))
         else:
             logger.warning("Could not find global based on %s, %s" %
                            (fileManNo, result.group("Name")))
     else:
         logger.error("Could not parse pointer reference [%s] in file [%s]" % (line, self._global.getFileNo()))
Ejemplo n.º 9
0
 def onSectionStart(self, line, section, crossRef):
     if section != IXindexLogFileParser.ROUTINE:
         logger.error("Invalid section Header --> %s", line)
         return False
     routineName = ROUTINE_START.search(line).group('name')
     if VALID_ROUTINE_NAME.search(
             routineName
     ) != None:  #, "Invalid RoutineName: [%s] Line: [%s]" % (routineName, line)
         if self._crossRef.isPlatformDependentRoutineByName(routineName):
             self._curRoutine = self._crossRef.getPlatformDependentRoutineByName(
                 routineName)
             return True
         renamedRoutineName = routineName
         if self._crossRef.routineNeedRename(routineName):
             renamedRoutineName = self._crossRef.getRenamedRoutineName(
                 routineName)
         if not self._crossRef.hasRoutine(renamedRoutineName):
             logger.warning("Invalid Routine: '%s': Rename Routine '%s'" %
                            (routineName, renamedRoutineName))
             return False
         self._curRoutine = self._crossRef.getRoutineByName(
             renamedRoutineName)
         self._curRoutine._structuredCode = structuredSource
         self._curPackage = self._curRoutine.getPackage()
         return True
     match = VALID_OBJECT.search(routineName).group("name")
     if match[:2] == "dd":
         fileNo = match[2:]
         if '.' not in fileNo:
             fileNo += ".0"
         self._curRoutine = self._crossRef.getGlobalByFileNo(fileNo)
         if not self._curRoutine:
             self._curRoutine = self._crossRef.getFileManSubFileByFileNo(
                 fileNo)
             if self._curRoutine:
                 self._curRoutine.setPackage(self._curPackage)
Ejemplo n.º 10
0
 def _setFieldSpecificData(self, zeroFields, fileField, rootNode,
                          fileSchema, filePointedTo, subFile):
   if fileField.getType() == FileManField.FIELD_TYPE_FILE_POINTER:
     fileGlobalRoot = ""
     if len(zeroFields) >= 3:
       fileGlobalRoot = zeroFields[2]
     if filePointedTo:
       if filePointedTo not in self._allSchema:
         """ create a new fileman file """
         self._allSchema[filePointedTo] = Global(fileGlobalRoot,
                                                 filePointedTo,
                                                 "")
       pointedToFile = self._allSchema[filePointedTo]
       assert pointedToFile.isRootFile()
       fileField.setPointedToFile(pointedToFile)
       globalName = pointedToFile.getName()
       fileNo = fileSchema.getFileNo()
       if fileSchema.isSubFile():
         fileNo = fileSchema.getRootFile().getFileNo()
       self._addToFileDepDict(fileNo,
                              pointedToFile.getFileNo())
       if fileGlobalRoot:
         if not globalName:
           pointedToFile.setName(fileGlobalRoot)
         elif globalName != fileGlobalRoot:
           logger.warning("%s: FileMan global root mismatch '%s' : '%s'" %
                         (zeroFields, globalName, fileGlobalRoot))
       else:
         logger.info("@TODO, find file global root for # %s" % filePointedTo)
     elif fileGlobalRoot:
       self._noPointedToFiles[fileGlobalRoot] = Global(fileGlobalRoot)
       logger.info("@TODO, set the file number for %s" % fileGlobalRoot)
     else:
       logger.warn("No pointed to file set for file:%s: field:%r 0-index:%s" %
                    (fileSchema.getFileNo(), fileField, zeroFields))
   elif fileField.getType() == FileManField.FIELD_TYPE_SUBFILE_POINTER:
     if subFile:
       if subFile not in self._allSchema:
         self._allSchema[subFile] = FileManFile(subFile, "", fileSchema)
       subFileSchema = self._allSchema[subFile]
       subFileSchema.setParentFile(fileSchema)
       fileSchema.addFileManSubFile(subFileSchema)
       fileField.setPointedToSubFile(subFileSchema)
     else:
       logger.warn("No subfile is set for file:%s, field:%r 0-index:%s" %
                    (fileSchema.getFileNo(), fileField, zeroFields))
   elif fileField.getType() == FileManField.FIELD_TYPE_SET and not subFile:
     setDict = dict([x.split(':') for x in zeroFields[2].rstrip(';').split(';')])
     fileField.setSetMembers(setDict)
   elif fileField.getType() == FileManField.FIELD_TYPE_VARIABLE_FILE_POINTER:
     if "V" in rootNode: # parsing variable pointer
       vptrs = parsingVariablePointer(rootNode['V'])
       vpFileSchemas = []
       if vptrs:
         for x in vptrs:
           if x not in self._allSchema:
             self._allSchema[x] = Global("", x, "")
           pointedToFile = self._allSchema[x]
           if pointedToFile.isSubFile():
             logger.error("Field: %r point to subFile: %s, parent: %s" %
                          (fileField, pointedToFile,
                           pointedToFile.getParentFile()))
           else:
             fileNo = fileSchema.getFileNo()
             if fileSchema.isSubFile():
               fileNo = fileSchema.getRootFile().getFileNo()
             self._addToFileDepDict(fileNo,
                                    pointedToFile.getFileNo())
           vpFileSchemas.append(self._allSchema[x])
         fileField.setPointedToFiles(vpFileSchemas)
Ejemplo n.º 11
0
 def onSectionStart(self, line, section, Global, CrossReference):
     self._lines = []
     result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line)
     assert result
     fileNo = result.group('FileNo')
     fieldNo = result.group("FieldNo")
     self._isSubFile = float(fileNo) != float(Global.getFileNo())
     if self._isSubFile:
         self._curFile = Global.getSubFileByFileNo(fileNo)
         assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line)
     else:
         self._curFile = Global
     restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
     startIdent = self.DEFAULT_NAME_INDENT
     defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT)
     if restOfLineStart > defaultIdentLevel:
         logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" %
                         (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
         try:
             floatValue = float(fieldNo)
         except ValueError:
             logger.error("invalid fieldNo %s" % fieldNo)
             fieldNo = line[line.find(",")+1:defaultIdentLevel]
             floatValue = float(fieldNo)
     restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip()
     result = NAME_LOC_TYPE_REGEX.search(restOfLine)
     fName, fType, fLocation = None, None, None
     if result:
         fName = result.group('Name').strip()
         fLocation = result.group('Loc').strip()
         if fLocation == ";":
             fLocation = None
         fType = result.group('Type').strip()
     else:
         # handle three cases, 1. no location info 2. no type info 3. Both
         if restOfLine.find(";") != -1: # missing type info
             logger.warn("Missing Type information [%s]" % line)
             result = NAME_LOC_REGEX.search(restOfLine)
             if result:
                 fName = result.group('Name').strip()
                 fLocation = result.group('Loc').strip()
             else:
                 logger.error("Could not parse [%s]" % restOfLine)
                 return
         else: # missing location, assume at least two space seperate name and type
             result = NAME_TYPE_REGEX.search(restOfLine)
             if result:
                 fName = result.group('Name').strip()
                 fType = result.group('Type').strip()
             else:
                 logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line))
     stripedType = ""
     if fType:
         stripedType = self.__stripFieldAttributes__(fType)
     if stripedType:
         self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference)
     else:
         self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
     self._curFile.addFileManField(self._field)
     if stripedType:
         self.__parseFieldAttributes__(fType)
Ejemplo n.º 12
0
 def _setFieldSpecificData(self, zeroFields, fileField, rootNode,
                           fileSchema, filePointedTo, subFile):
     if fileField.getType() == FileManField.FIELD_TYPE_FILE_POINTER:
         fileGlobalRoot = ""
         if len(zeroFields) >= 3:
             fileGlobalRoot = zeroFields[2]
         if filePointedTo:
             if filePointedTo not in self._allSchema:
                 """ create a new fileman file """
                 self._allSchema[filePointedTo] = Global(
                     fileGlobalRoot, filePointedTo, "")
             pointedToFile = self._allSchema[filePointedTo]
             assert pointedToFile.isRootFile()
             fileField.setPointedToFile(pointedToFile)
             globalName = pointedToFile.getName()
             fileNo = fileSchema.getFileNo()
             if fileSchema.isSubFile():
                 fileNo = fileSchema.getRootFile().getFileNo()
             self._addToFileDepDict(fileNo, pointedToFile.getFileNo())
             if fileGlobalRoot:
                 if not globalName:
                     pointedToFile.setName(fileGlobalRoot)
                 elif globalName != fileGlobalRoot:
                     logger.warning(
                         "%s: FileMan global root mismatch '%s' : '%s'" %
                         (zeroFields, globalName, fileGlobalRoot))
             else:
                 logger.info("@TODO, find file global root for # %s" %
                             filePointedTo)
         elif fileGlobalRoot:
             self._noPointedToFiles[fileGlobalRoot] = Global(fileGlobalRoot)
             logger.info("@TODO, set the file number for %s" %
                         fileGlobalRoot)
         else:
             logger.warn(
                 "No pointed to file set for file:%s: field:%r 0-index:%s" %
                 (fileSchema.getFileNo(), fileField, zeroFields))
     elif fileField.getType() == FileManField.FIELD_TYPE_SUBFILE_POINTER:
         if subFile:
             if subFile not in self._allSchema:
                 self._allSchema[subFile] = FileManFile(
                     subFile, "", fileSchema)
             subFileSchema = self._allSchema[subFile]
             subFileSchema.setParentFile(fileSchema)
             fileSchema.addFileManSubFile(subFileSchema)
             fileField.setPointedToSubFile(subFileSchema)
         else:
             logger.warn(
                 "No subfile is set for file:%s, field:%r 0-index:%s" %
                 (fileSchema.getFileNo(), fileField, zeroFields))
     elif fileField.getType(
     ) == FileManField.FIELD_TYPE_SET and not subFile and zeroFields[2]:
         setDict = dict(
             [x.split(':') for x in zeroFields[2].rstrip(';').split(';')])
         fileField.setSetMembers(setDict)
     elif fileField.getType(
     ) == FileManField.FIELD_TYPE_VARIABLE_FILE_POINTER:
         if "V" in rootNode:  # parsing variable pointer
             vptrs = parsingVariablePointer(rootNode['V'])
             vpFileSchemas = []
             if vptrs:
                 for x in vptrs:
                     if x not in self._allSchema:
                         self._allSchema[x] = Global("", x, "")
                     pointedToFile = self._allSchema[x]
                     if pointedToFile.isSubFile():
                         logger.error(
                             "Field: %r point to subFile: %s, parent: %s" %
                             (fileField, pointedToFile,
                              pointedToFile.getParentFile()))
                     else:
                         fileNo = fileSchema.getFileNo()
                         if fileSchema.isSubFile():
                             fileNo = fileSchema.getRootFile().getFileNo()
                         self._addToFileDepDict(fileNo,
                                                pointedToFile.getFileNo())
                     vpFileSchemas.append(self._allSchema[x])
                 fileField.setPointedToFiles(vpFileSchemas)
Ejemplo n.º 13
0
def convertJson(inputJsonFile, date, MRepositDir, patchRepositDir,
                generateHTML, generatePDF, outDir=None, pdfOutDir=None,
                local=False):
    if not generateHTML and not generatePDF:
        raise Exception("Nothing to generate!")

    global DOX_URL
    global VIVIAN_URL
    DOX_URL = getDOXURL(local)
    VIVIAN_URL = getViViaNURL(local)

    if generateHTML:
        if not outDir:
            raise Exception("Must specify Output directory")
        if not os.path.exists(outDir):
            # Will also create intermediate directories if needed
            os.makedirs(outDir)

    if generatePDF:
        if not pdfOutDir:
            raise Exception("Must specify PDF Output directory")
        # Will also create intermediate directories if needed
        if not os.path.exists(pdfOutDir):
            os.makedirs(pdfOutDir)

    from InitCrossReferenceGenerator import parseCrossReferenceGeneratorArgs
    crossRef = parseCrossReferenceGeneratorArgs(MRepositDir,
                                                patchRepositDir)
    global RPC_NAME_TO_IEN_MAPPING
    RPC_NAME_TO_IEN_MAPPING = generateSingleFileFieldToIenMappingBySchema(MRepositDir,
                                                                          crossRef)


    with open(inputJsonFile, 'r') as inputFile:
        pkgJson = {} # group by package
        allpkgJson = []
        inputJson = json.load(inputFile)

        for icrEntry in inputJson:
            if 'NUMBER' not in icrEntry:
                logger.error("Could not parse entry: " + str(icrEntry))
                continue

            if 'CUSTODIAL PACKAGE' in icrEntry:
                pkgName = icrEntry['CUSTODIAL PACKAGE']
                if crossRef.getMappedPackageName(pkgName) is None:
                    crossRef.addMappedPackage(pkgName,
                                              crossRef.normalizePackageName(pkgName).title())
                    logger.warning("Adding package " + pkgName + " to package name map.")
            if generatePDF:
                _generateICRIndividualPagePDF(icrEntry, date, pdfOutDir)
            if generateHTML:
                _generateICRIndividualPage(icrEntry, date, outDir, crossRef)
                summaryInfo = _convertICREntryToSummaryInfo(icrEntry, crossRef)
                allpkgJson.append(summaryInfo)
                if 'CUSTODIAL PACKAGE' in icrEntry:
                    pkgJson.setdefault(icrEntry['CUSTODIAL PACKAGE'], []).append(summaryInfo)
        if generateHTML:
            _generateICRSummaryPageImpl(allpkgJson, 'ICR List', 'All', date,
                                        outDir, crossRef, isForAll=True)
            for pkgName, outJson in iteritems(pkgJson):
                _generateICRSummaryPageImpl(outJson, 'ICR List', pkgName, date,
                                            outDir, crossRef)
            logger.warn('Total # entry in pkgJson is [%s]', len(pkgJson))
            _generatePkgDepSummaryPage(inputJson, date, outDir, crossRef)