def __outputIndividualFileManFile__(self, outDir, Global): logger.info("Writing Global %s" % Global) jsonResult = self._fileManFileEncoder.outputResult(Global) outputFile = open(os.path.join(outDir,"Global_%s.json" % base64.urlsafe_b64encode(Global.getName())),'wb') outputFile.write(jsonResult) outputFile.write("\n") outputFile.close() logger.info("Writing FileManFile %s" % Global.getFileNo()) outputFile = open(os.path.join(outDir,"FileManFile_%s.json" % (Global.getFileNo())),'wb') outputFile.write(jsonResult) outputFile.write("\n") outputFile.close()
def __outputIndividualFileManFile__(self, outDir, Global): logger.info("Writing Global %s" % Global) jsonResult = self._fileManFileEncoder.outputResult(Global) outputFile = open( os.path.join( outDir, "Global_%s.json" % base64.urlsafe_b64encode(Global.getName())), 'wb') outputFile.write(jsonResult) outputFile.write("\n") outputFile.close() logger.info("Writing FileManFile %s" % Global.getFileNo()) outputFile = open( os.path.join(outDir, "FileManFile_%s.json" % (Global.getFileNo())), 'wb') outputFile.write(jsonResult) outputFile.write("\n") outputFile.close()
def onSectionStart(self, line, section, Global, CrossReference): logger.debug("[%s]" % line) self._lines = [] result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search( line) assert result fileNo = result.group('FileNo') fieldNo = result.group("FieldNo") self._isSubFile = float(fileNo) != float(Global.getFileNo()) if self._isSubFile: logger.debug("%s is a subfile" % fileNo) self._curFile = Global.getSubFileByFileNo(fileNo) assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % ( fileNo, Global.getFileNo(), line) else: self._curFile = Global restOfLineStart = line.find("," + fieldNo) + len(fieldNo) startIdent = self.DEFAULT_NAME_INDENT #if len(fileNo) + 4 > startIdent: # startIdent = self defaultIdentLevel = self.__getDefaultIndentLevel__( self._curFile, self.DEFAULT_NAME_INDENT) if restOfLineStart > defaultIdentLevel: logger.debug( "FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel)) try: floatValue = float(fieldNo) except ValueError: logger.error("invalid fieldNo %s" % fieldNo) fieldNo = line[line.find(",") + 1:defaultIdentLevel] floatValue = float(fieldNo) restOfLine = line[line.find("," + fieldNo) + len(fieldNo) + 1:].strip() logger.debug("Parsing [%s]" % restOfLine) result = self.NAME_LOC_TYPE_REGEX.search(restOfLine) fName, fType, fLocation = None, None, None if result: logger.debug("FileNo: %s, Field#: %s, Name: %s, Loc %s, Type %s" % (fileNo, fieldNo, result.group('Name').rstrip(), result.group('Loc'), result.group('Type'))) fName = result.group('Name').strip() fLocation = result.group('Loc').strip() if fLocation == ";": fLocation = None fType = result.group('Type').strip() else: # handle three cases, 1. no location info 2. no type info 3. Both if restOfLine.find(";") != -1: # missing type info logger.warn("Missing Type information [%s]" % line) result = self.NAME_LOC_REGEX.search(restOfLine) if result: logger.debug("Name: %s, Loc %s" % (result.group('Name'), result.group('Loc'))) fName = result.group('Name').strip() fLocation = result.group('Loc').strip() else: logger.error("Could not parse [%s]" % restOfLine) return else: # missing location, assume at least two space seperate name and type logger.warn("Missing location information [%s]" % line) result = self.NAME_TYPE_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fType = result.group('Type').strip() logger.debug("Name: %s, Type %s" % (result.group('Name'), result.group('Type'))) else: logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line)) stripedType = "" if fType: stripedType = self.__stripFieldAttributes__(fType) if len(stripedType) > 0: self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference) else: self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation) logger.debug("Add field %s to File %s" % (fName, self._curFile.getFileNo())) self._curFile.addFileManField(self._field) if len(stripedType) > 0: self.__parseFieldAttributes__(fType)
def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): logger.info("Package: %s is new" % packageName) crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = open(file, 'r') lineNo = 0 fileName = os.path.basename(file) result = re.search("(?P<fileNo>^[0-9.]+)(-1)?\+(?P<des>.*)\.zwr$", fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = re.search("(?P<namespace>^[^.]+)\.zwr$", fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ", '') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo == 1: assert re.search('ZWR', line.strip()) if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 logger.debug("globalName: %s, Des: %s, fileNo: %s, package: %s" % (globalName, globalDes, fileNo, packageName)) if len(fileNo) == 0: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError, es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error( "Error, duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close()
def onSectionStart(self, line, section, Global, CrossReference): self._lines = [] result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line) assert result fileNo = result.group('FileNo') fieldNo = result.group("FieldNo") self._isSubFile = float(fileNo) != float(Global.getFileNo()) if self._isSubFile: self._curFile = Global.getSubFileByFileNo(fileNo) assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line) else: self._curFile = Global restOfLineStart = line.find("," + fieldNo) + len(fieldNo) startIdent = self.DEFAULT_NAME_INDENT defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT) if restOfLineStart > defaultIdentLevel: logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel)) try: floatValue = float(fieldNo) except ValueError: logger.error("invalid fieldNo %s" % fieldNo) fieldNo = line[line.find(",")+1:defaultIdentLevel] floatValue = float(fieldNo) restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip() result = NAME_LOC_TYPE_REGEX.search(restOfLine) fName, fType, fLocation = None, None, None if result: fName = result.group('Name').strip() fLocation = result.group('Loc').strip() if fLocation == ";": fLocation = None fType = result.group('Type').strip() else: # handle three cases, 1. no location info 2. no type info 3. Both if restOfLine.find(";") != -1: # missing type info logger.warn("Missing Type information [%s]" % line) result = NAME_LOC_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fLocation = result.group('Loc').strip() else: logger.error("Could not parse [%s]" % restOfLine) return else: # missing location, assume at least two space seperate name and type result = NAME_TYPE_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fType = result.group('Type').strip() else: logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line)) stripedType = "" if fType: stripedType = self.__stripFieldAttributes__(fType) if stripedType: self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference) else: self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation) self._curFile.addFileManField(self._field) if stripedType: self.__parseFieldAttributes__(fType)
def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = open(file, 'r') lineNo = 0 fileName = os.path.basename(file) result = ZWR_FILENO_REGEX.search(fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = ZWR_NAMESPACE_REGEX.search(fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ",'') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 if not fileNo: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError, es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error("Duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close()
def findGlobalsBySourceV2(self, dirName, pattern): searchFiles = glob.glob(os.path.join(dirName, pattern)) logger.info("Total Search Files are %d " % len(searchFiles)) crossReference = self.crossRef allGlobals = crossReference.getAllGlobals() allPackages = crossReference.getAllPackages() skipFile = [] fileNoSet = set() for file in searchFiles: packageName = os.path.dirname(file) packageName = packageName[packageName.index("Packages") + 9:packageName.index("Globals") - 1] if not crossReference.hasPackage(packageName): crossReference.addPackageByName(packageName) package = allPackages.get(packageName) zwrFile = codecs.open(file, 'r', encoding='utf-8', errors='ignore') lineNo = 0 fileName = os.path.basename(file) result = ZWR_FILENO_REGEX.search(fileName) if result: fileNo = result.group('fileNo') if fileNo.startswith('0'): fileNo = fileNo[1:] globalDes = result.group('des') else: result = ZWR_NAMESPACE_REGEX.search(fileName) if result: namespace = result.group('namespace') # package.addGlobalNamespace(namespace) continue else: continue globalName = "" # find out the global name by parsing the global file logger.debug("Parsing file: %s" % file) for line in zwrFile: if lineNo == 0: globalDes = line.strip() # Removing the extra text in the header of the ZWR file # to tell if it needs to be added or skipped globalDes = globalDes.replace("OSEHRA ZGO Export: ", '') if globalDes.startswith("^"): logger.info("No Description: Skip this file: %s" % file) skipFile.append(file) namespace = globalDes[1:] package.addGlobalNamespace(namespace) break if lineNo >= 2: info = line.strip().split('=') globalName = info[0] detail = info[1].strip("\"") if globalName.find(',') > 0: result = globalName.split(',') if len(result) == 2 and result[1] == "0)": globalName = result[0] break elif globalName.endswith("(0)"): globalName = globalName.split('(')[0] break else: continue lineNo = lineNo + 1 if not fileNo: if file not in skipFile: logger.warn("Warning: No FileNo found for file %s" % file) continue globalVar = Global(globalName, fileNo, globalDes, allPackages.get(packageName)) try: fileNum = float(globalVar.getFileNo()) except ValueError as es: logger.error("error: %s, globalVar:%s file %s" % (es, globalVar, file)) continue # crossReference.addGlobalToPackage(globalVar, packageName) # only add to allGlobals dict as we have to change the package later on if globalVar.getName() not in allGlobals: allGlobals[globalVar.getName()] = globalVar if fileNo not in fileNoSet: fileNoSet.add(fileNo) else: logger.error( "Duplicated file No [%s,%s,%s,%s] file:%s " % (fileNo, globalName, globalDes, packageName, file)) zwrFile.close() logger.info( "Total # of Packages is %d and Total # of Globals is %d, Total Skip File %d, total FileNo is %d" % (len(allPackages), len(allGlobals), len(skipFile), len(fileNoSet))) sortedKeyList = sorted( list(allGlobals.keys()), key=lambda item: float(allGlobals[item].getFileNo())) for key in sortedKeyList: globalVar = allGlobals[key] # fix the uncategoried item if globalVar.getFileNo() in fileNoPackageMappingDict: globalVar.setPackage(allPackages[fileNoPackageMappingDict[ globalVar.getFileNo()]]) crossReference.addGlobalToPackage(globalVar, globalVar.getPackage().getName())