def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference): logger.debug("Current Type is [%s]" % fType) result = self.UNDEFINED_POINTER.search(fType) if result: self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) return result = self.POINTER_TO_REGEX.search(fType) if result: fileNo = result.group('File') filePointedTo = CrossReference.getGlobalByFileNo(fileNo) self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) if not filePointedTo: logger.error( "Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line)) else: self._field.setPointedToFile(filePointedTo) return # deal with file pointer to subFiles result = self.SUBFILE_REGEX.search(fType) if result: # create a field for sub file type self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation) fileNo = result.group('File') logger.debug("Pointer to subFile %s" % fileNo) subFile = Global.getSubFileByFileNo(fileNo) if not subFile: subFile = FileManFile(fileNo, fName, self._curFile) self._curFile.addFileManSubFile(subFile) logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo())) if self._isSubFile: Global.addFileManSubFile(subFile) self._field.setPointedToSubFile(subFile) return for (key, value) in self.StringTypeMappingDict.iteritems(): if fType.startswith(key): self._field = FileManFieldFactory.createField( fieldNo, fName, value, fLocation) break if not self._field: # double check the loc and type if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX: fType = line[self.MAXIMIUM_TYPE_START_INDEX:] if fLocation: fLocation = line[line.find(fLocation):self. MAXIMIUM_TYPE_START_INDEX] logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation)) self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference) assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % ( fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def _parseSchemaField(self, fieldNo, rootNode, fileSchema): if '0' not in rootNode: logging.warn('%s does not have a 0 subscript' % rootNode) return None zeroFields = rootNode["0"].value if not zeroFields: logging.warn("No value: %s for %s" % (zeroFields, rootNode['0'])) return None zeroFields = zeroFields.split('^') if len(zeroFields) < 2: return FileManFieldFactory.createField( fieldNo, zeroFields[0], FileManField.FIELD_TYPE_NONE, None) types, specifier, filePointedTo, subFile = \ self.parseFieldTypeSpecifier(zeroFields[1]) location = None if len(zeroFields) >= 4 and zeroFields[3]: location = zeroFields[3].strip(' ') if location == ';': # No location information location = None elif location.split(';')[-1] == '0': # 0 means multiple multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER if not types: logging.debug('Set type to be multiple for %s' % zeroFields) types = [multipleType] if multipleType in types and types[0] != multipleType: logging.debug('Change type to be multiple for %s' % zeroFields) types.remove(multipleType) types.insert(0, multipleType) if not subFile: subFile = filePointedTo if not types: logging.warn('Can not determine the type for %s, fn: %s, file:%s' % (zeroFields, fieldNo, fileSchema.getFileNo())) types = [FileManField.FIELD_TYPE_NONE] if types and types[0] == FileManField.FIELD_TYPE_SUBFILE_POINTER: if subFile and subFile == fileSchema.getFileNo(): logging.error("recursive subfile pointer for %s" % subFile) types = [FileManField.FIELD_TYPE_NONE] logging.debug( '%s is %s, %s, %s, %s' % (zeroFields[1], types, specifier, filePointedTo, subFile)) fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0], types[0], location) if specifier: fileField.setSpecifier(specifier) logging.debug("Adding specifier: %s to %r" % (specifier, fileField)) self._setFieldSpecificData(zeroFields, fileField, rootNode, fileSchema, filePointedTo, subFile) return fileField
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference): logger.debug("Current Type is [%s]" % fType) result = self.UNDEFINED_POINTER.search(fType) if result: self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) return result = self.POINTER_TO_REGEX.search(fType) if result: fileNo = result.group('File') filePointedTo = CrossReference.getGlobalByFileNo(fileNo) self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER, fLocation) if not filePointedTo: logger.error("Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line)) else: self._field.setPointedToFile(filePointedTo) return # deal with file pointer to subFiles result = self.SUBFILE_REGEX.search(fType) if result: # create a field for sub file type self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation) fileNo = result.group('File') logger.debug("Pointer to subFile %s" % fileNo) subFile = Global.getSubFileByFileNo(fileNo) if not subFile: # this is a new subfile subFile = FileManFile(fileNo, fName, self._curFile) self._curFile.addFileManSubFile(subFile) logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo())) if self._isSubFile: Global.addFileManSubFile(subFile) self._field.setPointedToSubFile(subFile) CrossReference.addFileManSubFile(subFile) return for (key, value) in self.StringTypeMappingDict.iteritems(): if fType.startswith(key): self._field = FileManFieldFactory.createField(fieldNo, fName, value, fLocation) break if not self._field: # double check the loc and type if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX: fType = line[self.MAXIMIUM_TYPE_START_INDEX:] if fLocation: fLocation = line[line.find(fLocation):self.MAXIMIUM_TYPE_START_INDEX] logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation)) self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference) assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def __decodeFileManField__(self, fileManFile, fileManFieldJson, rootFileManFile=None): fieldNo = fileManFieldJson['field_no'] fieldName = fileManFieldJson['name'] location = fileManFieldJson['location'] type = fileManFieldJson['type'] fileManField = FileManFieldFactory.createField(fieldNo, fieldName, type, location) fileManField.setTypeName(fileManFieldJson['typename']) fileManField.setRequired(fileManFieldJson['isRequired']) fileManField.setAudited(fileManFieldJson['isAudited']) fileManField.setAddNewEntryWithoutAsking(fileManFieldJson['isAddNewEntryWithoutAsking']) fileManField.setMultiplyAsked(fileManFieldJson['isMultiplyAsked']) if fileManField.isFilePointerType(): fileNo = fileManFieldJson['filePointedTo'] if fileNo: filePointedTo = self._crossRef.getGlobalByFileNo(fileNo) assert filePointedTo fileManField.setPointedToFile(filePointedTo) elif fileManField.isSetType(): fileManField.setSetMembers(fileManFieldJson['set_memembers']) elif fileManField.isVariablePointerType(): pointedToFiles = [self._crossRef.getGlobalByFileNo(x) for x in fileManFieldJson['pointedToFileList']] fileManField.setPointedToFiles (pointedToFiles) elif fileManField.isWordProcessingType(): fileManField.setNoWrap(fileManFieldJson['isNoWrap']) elif fileManField.isSubFilePointerType(): subFileNo = fileManFieldJson['subfilePointedTo'] parentFile = fileManFile if rootFileManFile: parentFile = rootFileManFile fileManField.setPointedToSubFile(parentFile.getSubFileByFileNo(subFileNo)) fileManFile.addFileManField(fileManField)
def _parseSchemaField(self, fieldNo, rootNode, fileSchema): if '0' not in rootNode: logging.warn('%s does not have a 0 subscript' % rootNode) return None zeroFields = rootNode["0"].value if not zeroFields: logging.warn("No value: %s for %s" % (zeroFields, rootNode['0'])) return None zeroFields = zeroFields.split('^') if len(zeroFields) < 2: return FileManFieldFactory.createField(fieldNo, zeroFields[0], FileManField.FIELD_TYPE_NONE, None) types, specifier, filePointedTo, subFile = \ self.parseFieldTypeSpecifier(zeroFields[1]) location = None if len(zeroFields) >= 4 and zeroFields[3]: location = zeroFields[3].strip(' ') if location == ';': # No location information location = None elif location.split(';')[-1] == '0': # 0 means multiple multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER if not types: logging.debug('Set type to be multiple for %s' % zeroFields) types = [multipleType] if multipleType in types and types[0] != multipleType: logging.debug('Change type to be multiple for %s' % zeroFields) types.remove(multipleType) types.insert(0, multipleType) if not subFile: subFile = filePointedTo if not types: logging.warn('Can not determine the type for %s, fn: %s, file:%s' % (zeroFields, fieldNo, fileSchema.getFileNo())) types = [FileManField.FIELD_TYPE_NONE] if types and types[0] == FileManField.FIELD_TYPE_SUBFILE_POINTER: if subFile and subFile == fileSchema.getFileNo(): logging.error("recursive subfile pointer for %s" % subFile) types = [FileManField.FIELD_TYPE_NONE] logging.debug('%s is %s, %s, %s, %s' % (zeroFields[1], types, specifier, filePointedTo, subFile)) fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0], types[0], location) if specifier: fileField.setSpecifier(specifier) logging.debug("Adding specifier: %s to %r" % (specifier, fileField)) self._setFieldSpecificData(zeroFields, fileField, rootNode, fileSchema, filePointedTo, subFile) return fileField
def __decodeFileManField__(self, fileManFile, fileManFieldJson, rootFileManFile=None): fieldNo = fileManFieldJson['field_no'] fieldName = fileManFieldJson['name'] location = fileManFieldJson['location'] type = fileManFieldJson['type'] fileManField = FileManFieldFactory.createField(fieldNo, fieldName, type, location) fileManField.setTypeName(fileManFieldJson['typename']) fileManField.setRequired(fileManFieldJson['isRequired']) fileManField.setAudited(fileManFieldJson['isAudited']) fileManField.setAddNewEntryWithoutAsking( fileManFieldJson['isAddNewEntryWithoutAsking']) fileManField.setMultiplyAsked(fileManFieldJson['isMultiplyAsked']) if fileManField.isFilePointerType(): fileNo = fileManFieldJson['filePointedTo'] if fileNo: filePointedTo = self._crossRef.getGlobalByFileNo(fileNo) assert filePointedTo fileManField.setPointedToFile(filePointedTo) elif fileManField.isSetType(): fileManField.setSetMembers(fileManFieldJson['set_memembers']) elif fileManField.isVariablePointerType(): pointedToFiles = [ self._crossRef.getGlobalByFileNo(x) for x in fileManFieldJson['pointedToFileList'] ] fileManField.setPointedToFiles(pointedToFiles) elif fileManField.isWordProcessingType(): fileManField.setNoWrap(fileManFieldJson['isNoWrap']) elif fileManField.isSubFilePointerType(): subFileNo = fileManFieldJson['subfilePointedTo'] parentFile = fileManFile if rootFileManFile: parentFile = rootFileManFile fileManField.setPointedToSubFile( parentFile.getSubFileByFileNo(subFileNo)) fileManFile.addFileManField(fileManField)
def onSectionStart(self, line, section, Global, CrossReference): logger.debug("[%s]" % line) self._lines = [] result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search( line) assert result fileNo = result.group('FileNo') fieldNo = result.group("FieldNo") self._isSubFile = float(fileNo) != float(Global.getFileNo()) if self._isSubFile: logger.debug("%s is a subfile" % fileNo) self._curFile = Global.getSubFileByFileNo(fileNo) assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % ( fileNo, Global.getFileNo(), line) else: self._curFile = Global restOfLineStart = line.find("," + fieldNo) + len(fieldNo) startIdent = self.DEFAULT_NAME_INDENT #if len(fileNo) + 4 > startIdent: # startIdent = self defaultIdentLevel = self.__getDefaultIndentLevel__( self._curFile, self.DEFAULT_NAME_INDENT) if restOfLineStart > defaultIdentLevel: logger.debug( "FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel)) try: floatValue = float(fieldNo) except ValueError: logger.error("invalid fieldNo %s" % fieldNo) fieldNo = line[line.find(",") + 1:defaultIdentLevel] floatValue = float(fieldNo) restOfLine = line[line.find("," + fieldNo) + len(fieldNo) + 1:].strip() logger.debug("Parsing [%s]" % restOfLine) result = self.NAME_LOC_TYPE_REGEX.search(restOfLine) fName, fType, fLocation = None, None, None if result: logger.debug("FileNo: %s, Field#: %s, Name: %s, Loc %s, Type %s" % (fileNo, fieldNo, result.group('Name').rstrip(), result.group('Loc'), result.group('Type'))) fName = result.group('Name').strip() fLocation = result.group('Loc').strip() if fLocation == ";": fLocation = None fType = result.group('Type').strip() else: # handle three cases, 1. no location info 2. no type info 3. Both if restOfLine.find(";") != -1: # missing type info logger.warn("Missing Type information [%s]" % line) result = self.NAME_LOC_REGEX.search(restOfLine) if result: logger.debug("Name: %s, Loc %s" % (result.group('Name'), result.group('Loc'))) fName = result.group('Name').strip() fLocation = result.group('Loc').strip() else: logger.error("Could not parse [%s]" % restOfLine) return else: # missing location, assume at least two space seperate name and type logger.warn("Missing location information [%s]" % line) result = self.NAME_TYPE_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fType = result.group('Type').strip() logger.debug("Name: %s, Type %s" % (result.group('Name'), result.group('Type'))) else: logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line)) stripedType = "" if fType: stripedType = self.__stripFieldAttributes__(fType) if len(stripedType) > 0: self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference) else: self._field = FileManFieldFactory.createField( fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation) logger.debug("Add field %s to File %s" % (fName, self._curFile.getFileNo())) self._curFile.addFileManField(self._field) if len(stripedType) > 0: self.__parseFieldAttributes__(fType)
def onSectionStart(self, line, section, Global, CrossReference): self._lines = [] result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line) assert result fileNo = result.group('FileNo') fieldNo = result.group("FieldNo") self._isSubFile = float(fileNo) != float(Global.getFileNo()) if self._isSubFile: self._curFile = Global.getSubFileByFileNo(fileNo) assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line) else: self._curFile = Global restOfLineStart = line.find("," + fieldNo) + len(fieldNo) startIdent = self.DEFAULT_NAME_INDENT defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT) if restOfLineStart > defaultIdentLevel: logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" % (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel)) try: floatValue = float(fieldNo) except ValueError: logger.error("invalid fieldNo %s" % fieldNo) fieldNo = line[line.find(",")+1:defaultIdentLevel] floatValue = float(fieldNo) restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip() result = NAME_LOC_TYPE_REGEX.search(restOfLine) fName, fType, fLocation = None, None, None if result: fName = result.group('Name').strip() fLocation = result.group('Loc').strip() if fLocation == ";": fLocation = None fType = result.group('Type').strip() else: # handle three cases, 1. no location info 2. no type info 3. Both if restOfLine.find(";") != -1: # missing type info logger.warn("Missing Type information [%s]" % line) result = NAME_LOC_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fLocation = result.group('Loc').strip() else: logger.error("Could not parse [%s]" % restOfLine) return else: # missing location, assume at least two space seperate name and type result = NAME_TYPE_REGEX.search(restOfLine) if result: fName = result.group('Name').strip() fType = result.group('Type').strip() else: logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line)) stripedType = "" if fType: stripedType = self.__stripFieldAttributes__(fType) if stripedType: self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference) else: self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation) self._curFile.addFileManField(self._field) if stripedType: self.__parseFieldAttributes__(fType)