Exemplo n.º 1
0
 def __outputIndividualFileManFile__(self, outDir, Global):
     logger.info("Writing Global %s" % Global)
     jsonResult = self._fileManFileEncoder.outputResult(Global)
     outputFile = open(os.path.join(outDir,"Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),'wb')
     outputFile.write(jsonResult)
     outputFile.write("\n")
     outputFile.close()
     logger.info("Writing FileManFile %s" % Global.getFileNo())
     outputFile = open(os.path.join(outDir,"FileManFile_%s.json" % (Global.getFileNo())),'wb')
     outputFile.write(jsonResult)
     outputFile.write("\n")
     outputFile.close()
Exemplo n.º 2
0
 def __outputIndividualFileManFile__(self, outDir, Global):
     logger.info("Writing Global %s" % Global)
     jsonResult = self._fileManFileEncoder.outputResult(Global)
     outputFile = open(
         os.path.join(
             outDir,
             "Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),
         'wb')
     outputFile.write(jsonResult)
     outputFile.write("\n")
     outputFile.close()
     logger.info("Writing FileManFile %s" % Global.getFileNo())
     outputFile = open(
         os.path.join(outDir, "FileManFile_%s.json" % (Global.getFileNo())),
         'wb')
     outputFile.write(jsonResult)
     outputFile.write("\n")
     outputFile.close()
Exemplo n.º 3
0
 def onSectionStart(self, line, section, Global, CrossReference):
     logger.debug("[%s]" % line)
     self._lines = []
     result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(
         line)
     assert result
     fileNo = result.group('FileNo')
     fieldNo = result.group("FieldNo")
     self._isSubFile = float(fileNo) != float(Global.getFileNo())
     if self._isSubFile:
         logger.debug("%s is a subfile" % fileNo)
         self._curFile = Global.getSubFileByFileNo(fileNo)
         assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (
             fileNo, Global.getFileNo(), line)
     else:
         self._curFile = Global
     restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
     startIdent = self.DEFAULT_NAME_INDENT
     #if len(fileNo) + 4 > startIdent:
     #    startIdent = self
     defaultIdentLevel = self.__getDefaultIndentLevel__(
         self._curFile, self.DEFAULT_NAME_INDENT)
     if restOfLineStart > defaultIdentLevel:
         logger.debug(
             "FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d"
             % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
         try:
             floatValue = float(fieldNo)
         except ValueError:
             logger.error("invalid fieldNo %s" % fieldNo)
             fieldNo = line[line.find(",") + 1:defaultIdentLevel]
             floatValue = float(fieldNo)
     restOfLine = line[line.find("," + fieldNo) + len(fieldNo) + 1:].strip()
     logger.debug("Parsing [%s]" % restOfLine)
     result = self.NAME_LOC_TYPE_REGEX.search(restOfLine)
     fName, fType, fLocation = None, None, None
     if result:
         logger.debug("FileNo: %s, Field#: %s, Name: %s, Loc %s, Type %s" %
                      (fileNo, fieldNo, result.group('Name').rstrip(),
                       result.group('Loc'), result.group('Type')))
         fName = result.group('Name').strip()
         fLocation = result.group('Loc').strip()
         if fLocation == ";":
             fLocation = None
         fType = result.group('Type').strip()
     else:
         # handle three cases, 1. no location info 2. no type info 3. Both
         if restOfLine.find(";") != -1:  # missing type info
             logger.warn("Missing Type information [%s]" % line)
             result = self.NAME_LOC_REGEX.search(restOfLine)
             if result:
                 logger.debug("Name: %s, Loc %s" %
                              (result.group('Name'), result.group('Loc')))
                 fName = result.group('Name').strip()
                 fLocation = result.group('Loc').strip()
             else:
                 logger.error("Could not parse [%s]" % restOfLine)
                 return
         else:  # missing location, assume at least two space seperate name and type
             logger.warn("Missing location information [%s]" % line)
             result = self.NAME_TYPE_REGEX.search(restOfLine)
             if result:
                 fName = result.group('Name').strip()
                 fType = result.group('Type').strip()
                 logger.debug("Name: %s, Type %s" %
                              (result.group('Name'), result.group('Type')))
             else:
                 logger.warn("Guessing Name: %s at line [%s]" %
                             (restOfLine.strip(), line))
     stripedType = ""
     if fType:
         stripedType = self.__stripFieldAttributes__(fType)
     if len(stripedType) > 0:
         self.__createFieldByType__(fieldNo, stripedType, fName, fLocation,
                                    line, Global, CrossReference)
     else:
         self._field = FileManFieldFactory.createField(
             fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
     logger.debug("Add field %s to File %s" %
                  (fName, self._curFile.getFileNo()))
     self._curFile.addFileManField(self._field)
     if len(stripedType) > 0:
         self.__parseFieldAttributes__(fType)
Exemplo n.º 4
0
    def findGlobalsBySourceV2(self, dirName, pattern):
        searchFiles = glob.glob(os.path.join(dirName, pattern))
        logger.info("Total Search Files are %d " % len(searchFiles))
        crossReference = self.crossRef
        allGlobals = crossReference.getAllGlobals()
        allPackages = crossReference.getAllPackages()
        skipFile = []
        fileNoSet = set()
        for file in searchFiles:
            packageName = os.path.dirname(file)
            packageName = packageName[packageName.index("Packages") +
                                      9:packageName.index("Globals") - 1]
            if not crossReference.hasPackage(packageName):
                logger.info("Package: %s is new" % packageName)
                crossReference.addPackageByName(packageName)
            package = allPackages.get(packageName)
            zwrFile = open(file, 'r')
            lineNo = 0
            fileName = os.path.basename(file)
            result = re.search("(?P<fileNo>^[0-9.]+)(-1)?\+(?P<des>.*)\.zwr$",
                               fileName)
            if result:
                fileNo = result.group('fileNo')
                if fileNo.startswith('0'): fileNo = fileNo[1:]
                globalDes = result.group('des')
            else:
                result = re.search("(?P<namespace>^[^.]+)\.zwr$", fileName)
                if result:
                    namespace = result.group('namespace')
                    #                    package.addGlobalNamespace(namespace)
                    continue
                else:
                    continue
            globalName = ""  # find out the global name by parsing the global file
            logger.debug("Parsing file: %s" % file)
            for line in zwrFile:
                if lineNo == 0:
                    globalDes = line.strip()
                    # Removing the extra text in the header of the ZWR file
                    # to tell if it needs to be added or skipped
                    globalDes = globalDes.replace("OSEHRA ZGO Export: ", '')
                    if globalDes.startswith("^"):
                        logger.info("No Description: Skip this file: %s" %
                                    file)
                        skipFile.append(file)
                        namespace = globalDes[1:]
                        package.addGlobalNamespace(namespace)
                        break
                if lineNo == 1:
                    assert re.search('ZWR', line.strip())
                if lineNo >= 2:
                    info = line.strip().split('=')
                    globalName = info[0]
                    detail = info[1].strip("\"")
                    if globalName.find(',') > 0:
                        result = globalName.split(',')
                        if len(result) == 2 and result[1] == "0)":
                            globalName = result[0]
                            break
                    elif globalName.endswith("(0)"):
                        globalName = globalName.split('(')[0]
                        break
                    else:
                        continue
                lineNo = lineNo + 1
            logger.debug("globalName: %s, Des: %s, fileNo: %s, package: %s" %
                         (globalName, globalDes, fileNo, packageName))
            if len(fileNo) == 0:
                if file not in skipFile:
                    logger.warn("Warning: No FileNo found for file %s" % file)
                continue
            globalVar = Global(globalName, fileNo, globalDes,
                               allPackages.get(packageName))
            try:
                fileNum = float(globalVar.getFileNo())
            except ValueError, es:
                logger.error("error: %s, globalVar:%s file %s" %
                             (es, globalVar, file))
                continue


#            crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
            if globalVar.getName() not in allGlobals:
                allGlobals[globalVar.getName()] = globalVar
            if fileNo not in fileNoSet:
                fileNoSet.add(fileNo)
            else:
                logger.error(
                    "Error, duplicated file No [%s,%s,%s,%s] file:%s " %
                    (fileNo, globalName, globalDes, packageName, file))
            zwrFile.close()
Exemplo n.º 5
0
 def onSectionStart(self, line, section, Global, CrossReference):
     self._lines = []
     result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line)
     assert result
     fileNo = result.group('FileNo')
     fieldNo = result.group("FieldNo")
     self._isSubFile = float(fileNo) != float(Global.getFileNo())
     if self._isSubFile:
         self._curFile = Global.getSubFileByFileNo(fileNo)
         assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line)
     else:
         self._curFile = Global
     restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
     startIdent = self.DEFAULT_NAME_INDENT
     defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT)
     if restOfLineStart > defaultIdentLevel:
         logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" %
                         (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
         try:
             floatValue = float(fieldNo)
         except ValueError:
             logger.error("invalid fieldNo %s" % fieldNo)
             fieldNo = line[line.find(",")+1:defaultIdentLevel]
             floatValue = float(fieldNo)
     restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip()
     result = NAME_LOC_TYPE_REGEX.search(restOfLine)
     fName, fType, fLocation = None, None, None
     if result:
         fName = result.group('Name').strip()
         fLocation = result.group('Loc').strip()
         if fLocation == ";":
             fLocation = None
         fType = result.group('Type').strip()
     else:
         # handle three cases, 1. no location info 2. no type info 3. Both
         if restOfLine.find(";") != -1: # missing type info
             logger.warn("Missing Type information [%s]" % line)
             result = NAME_LOC_REGEX.search(restOfLine)
             if result:
                 fName = result.group('Name').strip()
                 fLocation = result.group('Loc').strip()
             else:
                 logger.error("Could not parse [%s]" % restOfLine)
                 return
         else: # missing location, assume at least two space seperate name and type
             result = NAME_TYPE_REGEX.search(restOfLine)
             if result:
                 fName = result.group('Name').strip()
                 fType = result.group('Type').strip()
             else:
                 logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line))
     stripedType = ""
     if fType:
         stripedType = self.__stripFieldAttributes__(fType)
     if stripedType:
         self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference)
     else:
         self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
     self._curFile.addFileManField(self._field)
     if stripedType:
         self.__parseFieldAttributes__(fType)
Exemplo n.º 6
0
  def findGlobalsBySourceV2(self, dirName, pattern):
    searchFiles = glob.glob(os.path.join(dirName, pattern))
    logger.info("Total Search Files are %d " % len(searchFiles))
    crossReference = self.crossRef
    allGlobals = crossReference.getAllGlobals()
    allPackages = crossReference.getAllPackages()
    skipFile = []
    fileNoSet = set()
    for file in searchFiles:
      packageName = os.path.dirname(file)
      packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1]
      if not crossReference.hasPackage(packageName):
        crossReference.addPackageByName(packageName)
      package = allPackages.get(packageName)
      zwrFile = open(file, 'r')
      lineNo = 0
      fileName = os.path.basename(file)
      result = ZWR_FILENO_REGEX.search(fileName)
      if result:
        fileNo = result.group('fileNo')
        if fileNo.startswith('0'): fileNo = fileNo[1:]
        globalDes = result.group('des')
      else:
        result = ZWR_NAMESPACE_REGEX.search(fileName)
        if result:
            namespace = result.group('namespace')
#                    package.addGlobalNamespace(namespace)
            continue
        else:
            continue
      globalName = "" # find out the global name by parsing the global file
      logger.debug("Parsing file: %s" % file)
      for line in zwrFile:
        if lineNo == 0:
          globalDes = line.strip()
          # Removing the extra text in the header of the ZWR file
          # to tell if it needs to be added or skipped
          globalDes = globalDes.replace("OSEHRA ZGO Export: ",'')
          if globalDes.startswith("^"):
            logger.info("No Description: Skip this file: %s" % file)
            skipFile.append(file)
            namespace = globalDes[1:]
            package.addGlobalNamespace(namespace)
            break
        if lineNo >= 2:
          info = line.strip().split('=')
          globalName = info[0]
          detail = info[1].strip("\"")
          if globalName.find(',') > 0:
              result = globalName.split(',')
              if len(result) == 2 and result[1] == "0)":
                  globalName = result[0]
                  break
          elif globalName.endswith("(0)"):
              globalName = globalName.split('(')[0]
              break
          else:
              continue
        lineNo = lineNo + 1
      if not fileNo:
        if file not in skipFile:
          logger.warn("Warning: No FileNo found for file %s" % file)
        continue
      globalVar = Global(globalName, fileNo, globalDes,
                         allPackages.get(packageName))
      try:
        fileNum = float(globalVar.getFileNo())
      except ValueError, es:
        logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file))
        continue
#            crossReference.addGlobalToPackage(globalVar, packageName)
      # only add to allGlobals dict as we have to change the package later on
      if globalVar.getName() not in allGlobals:
        allGlobals[globalVar.getName()] = globalVar
      if fileNo not in fileNoSet:
        fileNoSet.add(fileNo)
      else:
        logger.error("Duplicated file No [%s,%s,%s,%s] file:%s " %
                      (fileNo, globalName, globalDes, packageName, file))
      zwrFile.close()
Exemplo n.º 7
0
    def findGlobalsBySourceV2(self, dirName, pattern):
        searchFiles = glob.glob(os.path.join(dirName, pattern))
        logger.info("Total Search Files are %d " % len(searchFiles))
        crossReference = self.crossRef
        allGlobals = crossReference.getAllGlobals()
        allPackages = crossReference.getAllPackages()
        skipFile = []
        fileNoSet = set()
        for file in searchFiles:
            packageName = os.path.dirname(file)
            packageName = packageName[packageName.index("Packages") +
                                      9:packageName.index("Globals") - 1]
            if not crossReference.hasPackage(packageName):
                crossReference.addPackageByName(packageName)
            package = allPackages.get(packageName)
            zwrFile = codecs.open(file, 'r', encoding='utf-8', errors='ignore')
            lineNo = 0
            fileName = os.path.basename(file)
            result = ZWR_FILENO_REGEX.search(fileName)
            if result:
                fileNo = result.group('fileNo')
                if fileNo.startswith('0'): fileNo = fileNo[1:]
                globalDes = result.group('des')
            else:
                result = ZWR_NAMESPACE_REGEX.search(fileName)
                if result:
                    namespace = result.group('namespace')
                    #                    package.addGlobalNamespace(namespace)
                    continue
                else:
                    continue
            globalName = ""  # find out the global name by parsing the global file
            logger.debug("Parsing file: %s" % file)
            for line in zwrFile:
                if lineNo == 0:
                    globalDes = line.strip()
                    # Removing the extra text in the header of the ZWR file
                    # to tell if it needs to be added or skipped
                    globalDes = globalDes.replace("OSEHRA ZGO Export: ", '')
                    if globalDes.startswith("^"):
                        logger.info("No Description: Skip this file: %s" %
                                    file)
                        skipFile.append(file)
                        namespace = globalDes[1:]
                        package.addGlobalNamespace(namespace)
                        break
                if lineNo >= 2:
                    info = line.strip().split('=')
                    globalName = info[0]
                    detail = info[1].strip("\"")
                    if globalName.find(',') > 0:
                        result = globalName.split(',')
                        if len(result) == 2 and result[1] == "0)":
                            globalName = result[0]
                            break
                    elif globalName.endswith("(0)"):
                        globalName = globalName.split('(')[0]
                        break
                    else:
                        continue
                lineNo = lineNo + 1
            if not fileNo:
                if file not in skipFile:
                    logger.warn("Warning: No FileNo found for file %s" % file)
                continue
            globalVar = Global(globalName, fileNo, globalDes,
                               allPackages.get(packageName))
            try:
                fileNum = float(globalVar.getFileNo())
            except ValueError as es:
                logger.error("error: %s, globalVar:%s file %s" %
                             (es, globalVar, file))
                continue


#            crossReference.addGlobalToPackage(globalVar, packageName)
# only add to allGlobals dict as we have to change the package later on
            if globalVar.getName() not in allGlobals:
                allGlobals[globalVar.getName()] = globalVar
            if fileNo not in fileNoSet:
                fileNoSet.add(fileNo)
            else:
                logger.error(
                    "Duplicated file No [%s,%s,%s,%s] file:%s " %
                    (fileNo, globalName, globalDes, packageName, file))
            zwrFile.close()
        logger.info(
            "Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d"
            %
            (len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet)))

        sortedKeyList = sorted(
            list(allGlobals.keys()),
            key=lambda item: float(allGlobals[item].getFileNo()))
        for key in sortedKeyList:
            globalVar = allGlobals[key]
            # fix the uncategoried item
            if globalVar.getFileNo() in fileNoPackageMappingDict:
                globalVar.setPackage(allPackages[fileNoPackageMappingDict[
                    globalVar.getFileNo()]])
            crossReference.addGlobalToPackage(globalVar,
                                              globalVar.getPackage().getName())