def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line,
Global, CrossReference):
logger.debug("Current Type is [%s]" % fType)
result = self.UNDEFINED_POINTER.search(fType)
if result:
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER,
fLocation)
return
result = self.POINTER_TO_REGEX.search(fType)
if result:
fileNo = result.group('File')
filePointedTo = CrossReference.getGlobalByFileNo(fileNo)
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_FILE_POINTER,
fLocation)
if not filePointedTo:
logger.error(
"Could not find file pointed to [%s], [%s], line:[%s]" %
(fileNo, self._curFile, line))
else:
self._field.setPointedToFile(filePointedTo)
return
# deal with file pointer to subFiles
result = self.SUBFILE_REGEX.search(fType)
if result:
# create a field for sub file type
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_SUBFILE_POINTER,
fLocation)
fileNo = result.group('File')
logger.debug("Pointer to subFile %s" % fileNo)
subFile = Global.getSubFileByFileNo(fileNo)
if not subFile:
subFile = FileManFile(fileNo, fName, self._curFile)
self._curFile.addFileManSubFile(subFile)
logger.debug("Added subFile %s to File %s" %
(fileNo, self._curFile.getFileNo()))
if self._isSubFile:
Global.addFileManSubFile(subFile)
self._field.setPointedToSubFile(subFile)
return
for (key, value) in self.StringTypeMappingDict.iteritems():
if fType.startswith(key):
self._field = FileManFieldFactory.createField(
fieldNo, fName, value, fLocation)
break
if not self._field:
# double check the loc and type
if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX:
fType = line[self.MAXIMIUM_TYPE_START_INDEX:]
if fLocation:
fLocation = line[line.find(fLocation):self.
MAXIMIUM_TYPE_START_INDEX]
logger.warn("new Type is [%s], loc is [%s]" %
(fType, fLocation))
self.__createFieldByType__(fieldNo, fType, fName, fLocation,
line, Global, CrossReference)
assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (
fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def _parseSchemaField(self, fieldNo, rootNode, fileSchema):
if '0' not in rootNode:
logging.warn('%s does not have a 0 subscript' % rootNode)
return None
zeroFields = rootNode["0"].value
if not zeroFields:
logging.warn("No value: %s for %s" % (zeroFields, rootNode['0']))
return None
zeroFields = zeroFields.split('^')
if len(zeroFields) < 2:
return FileManFieldFactory.createField(
fieldNo, zeroFields[0], FileManField.FIELD_TYPE_NONE, None)
types, specifier, filePointedTo, subFile = \
self.parseFieldTypeSpecifier(zeroFields[1])
location = None
if len(zeroFields) >= 4 and zeroFields[3]:
location = zeroFields[3].strip(' ')
if location == ';': # No location information
location = None
elif location.split(';')[-1] == '0': # 0 means multiple
multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER
if not types:
logging.debug('Set type to be multiple for %s' %
zeroFields)
types = [multipleType]
if multipleType in types and types[0] != multipleType:
logging.debug('Change type to be multiple for %s' %
zeroFields)
types.remove(multipleType)
types.insert(0, multipleType)
if not subFile: subFile = filePointedTo
if not types:
logging.warn('Can not determine the type for %s, fn: %s, file:%s' %
(zeroFields, fieldNo, fileSchema.getFileNo()))
types = [FileManField.FIELD_TYPE_NONE]
if types and types[0] == FileManField.FIELD_TYPE_SUBFILE_POINTER:
if subFile and subFile == fileSchema.getFileNo():
logging.error("recursive subfile pointer for %s" % subFile)
types = [FileManField.FIELD_TYPE_NONE]
logging.debug(
'%s is %s, %s, %s, %s' %
(zeroFields[1], types, specifier, filePointedTo, subFile))
fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0],
types[0], location)
if specifier:
fileField.setSpecifier(specifier)
logging.debug("Adding specifier: %s to %r" %
(specifier, fileField))
self._setFieldSpecificData(zeroFields, fileField, rootNode, fileSchema,
filePointedTo, subFile)
return fileField
def __createFieldByType__(self, fieldNo, fType, fName, fLocation, line, Global, CrossReference):
logger.debug("Current Type is [%s]" % fType)
result = self.UNDEFINED_POINTER.search(fType)
if result:
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
return
result = self.POINTER_TO_REGEX.search(fType)
if result:
fileNo = result.group('File')
filePointedTo = CrossReference.getGlobalByFileNo(fileNo)
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_FILE_POINTER, fLocation)
if not filePointedTo:
logger.error("Could not find file pointed to [%s], [%s], line:[%s]" % (fileNo, self._curFile, line))
else:
self._field.setPointedToFile(filePointedTo)
return
# deal with file pointer to subFiles
result = self.SUBFILE_REGEX.search(fType)
if result:
# create a field for sub file type
self._field = FileManFieldFactory.createField(fieldNo, fName,
FileManField.FIELD_TYPE_SUBFILE_POINTER, fLocation)
fileNo = result.group('File')
logger.debug("Pointer to subFile %s" % fileNo)
subFile = Global.getSubFileByFileNo(fileNo)
if not subFile: # this is a new subfile
subFile = FileManFile(fileNo, fName, self._curFile)
self._curFile.addFileManSubFile(subFile)
logger.debug("Added subFile %s to File %s" % (fileNo, self._curFile.getFileNo()))
if self._isSubFile:
Global.addFileManSubFile(subFile)
self._field.setPointedToSubFile(subFile)
CrossReference.addFileManSubFile(subFile)
return
for (key, value) in self.StringTypeMappingDict.iteritems():
if fType.startswith(key):
self._field = FileManFieldFactory.createField(fieldNo, fName, value, fLocation)
break
if not self._field:
# double check the loc and type
if line.find(fType) > self.MAXIMIUM_TYPE_START_INDEX:
fType = line[self.MAXIMIUM_TYPE_START_INDEX:]
if fLocation:
fLocation = line[line.find(fLocation):self.MAXIMIUM_TYPE_START_INDEX]
logger.warn("new Type is [%s], loc is [%s]" % (fType, fLocation))
self.__createFieldByType__(fieldNo, fType, fName, fLocation, line, Global, CrossReference)
assert self._field, "Could not find the right type for %s, %s, %s, %s, %s" % (fType, fLocation, fieldNo, line, self._curFile.getFileNo())
def __decodeFileManField__(self, fileManFile, fileManFieldJson, rootFileManFile=None):
fieldNo = fileManFieldJson['field_no']
fieldName = fileManFieldJson['name']
location = fileManFieldJson['location']
type = fileManFieldJson['type']
fileManField = FileManFieldFactory.createField(fieldNo, fieldName, type, location)
fileManField.setTypeName(fileManFieldJson['typename'])
fileManField.setRequired(fileManFieldJson['isRequired'])
fileManField.setAudited(fileManFieldJson['isAudited'])
fileManField.setAddNewEntryWithoutAsking(fileManFieldJson['isAddNewEntryWithoutAsking'])
fileManField.setMultiplyAsked(fileManFieldJson['isMultiplyAsked'])
if fileManField.isFilePointerType():
fileNo = fileManFieldJson['filePointedTo']
if fileNo:
filePointedTo = self._crossRef.getGlobalByFileNo(fileNo)
assert filePointedTo
fileManField.setPointedToFile(filePointedTo)
elif fileManField.isSetType():
fileManField.setSetMembers(fileManFieldJson['set_memembers'])
elif fileManField.isVariablePointerType():
pointedToFiles = [self._crossRef.getGlobalByFileNo(x) for x in fileManFieldJson['pointedToFileList']]
fileManField.setPointedToFiles (pointedToFiles)
elif fileManField.isWordProcessingType():
fileManField.setNoWrap(fileManFieldJson['isNoWrap'])
elif fileManField.isSubFilePointerType():
subFileNo = fileManFieldJson['subfilePointedTo']
parentFile = fileManFile
if rootFileManFile:
parentFile = rootFileManFile
fileManField.setPointedToSubFile(parentFile.getSubFileByFileNo(subFileNo))
fileManFile.addFileManField(fileManField)
def _parseSchemaField(self, fieldNo, rootNode, fileSchema):
if '0' not in rootNode:
logging.warn('%s does not have a 0 subscript' % rootNode)
return None
zeroFields = rootNode["0"].value
if not zeroFields:
logging.warn("No value: %s for %s" % (zeroFields, rootNode['0']))
return None
zeroFields = zeroFields.split('^')
if len(zeroFields) < 2:
return FileManFieldFactory.createField(fieldNo, zeroFields[0],
FileManField.FIELD_TYPE_NONE, None)
types, specifier, filePointedTo, subFile = \
self.parseFieldTypeSpecifier(zeroFields[1])
location = None
if len(zeroFields) >= 4 and zeroFields[3]:
location = zeroFields[3].strip(' ')
if location == ';': # No location information
location = None
elif location.split(';')[-1] == '0': # 0 means multiple
multipleType = FileManField.FIELD_TYPE_SUBFILE_POINTER
if not types:
logging.debug('Set type to be multiple for %s' % zeroFields)
types = [multipleType]
if multipleType in types and types[0] != multipleType:
logging.debug('Change type to be multiple for %s' % zeroFields)
types.remove(multipleType)
types.insert(0, multipleType)
if not subFile: subFile = filePointedTo
if not types:
logging.warn('Can not determine the type for %s, fn: %s, file:%s' %
(zeroFields, fieldNo, fileSchema.getFileNo()))
types = [FileManField.FIELD_TYPE_NONE]
if types and types[0] == FileManField.FIELD_TYPE_SUBFILE_POINTER:
if subFile and subFile == fileSchema.getFileNo():
logging.error("recursive subfile pointer for %s" % subFile)
types = [FileManField.FIELD_TYPE_NONE]
logging.debug('%s is %s, %s, %s, %s' %
(zeroFields[1], types, specifier, filePointedTo, subFile))
fileField = FileManFieldFactory.createField(fieldNo, zeroFields[0],
types[0], location)
if specifier:
fileField.setSpecifier(specifier)
logging.debug("Adding specifier: %s to %r" % (specifier, fileField))
self._setFieldSpecificData(zeroFields, fileField, rootNode,
fileSchema, filePointedTo, subFile)
return fileField
def __decodeFileManField__(self,
fileManFile,
fileManFieldJson,
rootFileManFile=None):
fieldNo = fileManFieldJson['field_no']
fieldName = fileManFieldJson['name']
location = fileManFieldJson['location']
type = fileManFieldJson['type']
fileManField = FileManFieldFactory.createField(fieldNo, fieldName,
type, location)
fileManField.setTypeName(fileManFieldJson['typename'])
fileManField.setRequired(fileManFieldJson['isRequired'])
fileManField.setAudited(fileManFieldJson['isAudited'])
fileManField.setAddNewEntryWithoutAsking(
fileManFieldJson['isAddNewEntryWithoutAsking'])
fileManField.setMultiplyAsked(fileManFieldJson['isMultiplyAsked'])
if fileManField.isFilePointerType():
fileNo = fileManFieldJson['filePointedTo']
if fileNo:
filePointedTo = self._crossRef.getGlobalByFileNo(fileNo)
assert filePointedTo
fileManField.setPointedToFile(filePointedTo)
elif fileManField.isSetType():
fileManField.setSetMembers(fileManFieldJson['set_memembers'])
elif fileManField.isVariablePointerType():
pointedToFiles = [
self._crossRef.getGlobalByFileNo(x)
for x in fileManFieldJson['pointedToFileList']
]
fileManField.setPointedToFiles(pointedToFiles)
elif fileManField.isWordProcessingType():
fileManField.setNoWrap(fileManFieldJson['isNoWrap'])
elif fileManField.isSubFilePointerType():
subFileNo = fileManFieldJson['subfilePointedTo']
parentFile = fileManFile
if rootFileManFile:
parentFile = rootFileManFile
fileManField.setPointedToSubFile(
parentFile.getSubFileByFileNo(subFileNo))
fileManFile.addFileManField(fileManField)
def onSectionStart(self, line, section, Global, CrossReference):
logger.debug("[%s]" % line)
self._lines = []
result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(
line)
assert result
fileNo = result.group('FileNo')
fieldNo = result.group("FieldNo")
self._isSubFile = float(fileNo) != float(Global.getFileNo())
if self._isSubFile:
logger.debug("%s is a subfile" % fileNo)
self._curFile = Global.getSubFileByFileNo(fileNo)
assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (
fileNo, Global.getFileNo(), line)
else:
self._curFile = Global
restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
startIdent = self.DEFAULT_NAME_INDENT
#if len(fileNo) + 4 > startIdent:
# startIdent = self
defaultIdentLevel = self.__getDefaultIndentLevel__(
self._curFile, self.DEFAULT_NAME_INDENT)
if restOfLineStart > defaultIdentLevel:
logger.debug(
"FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d"
% (fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
try:
floatValue = float(fieldNo)
except ValueError:
logger.error("invalid fieldNo %s" % fieldNo)
fieldNo = line[line.find(",") + 1:defaultIdentLevel]
floatValue = float(fieldNo)
restOfLine = line[line.find("," + fieldNo) + len(fieldNo) + 1:].strip()
logger.debug("Parsing [%s]" % restOfLine)
result = self.NAME_LOC_TYPE_REGEX.search(restOfLine)
fName, fType, fLocation = None, None, None
if result:
logger.debug("FileNo: %s, Field#: %s, Name: %s, Loc %s, Type %s" %
(fileNo, fieldNo, result.group('Name').rstrip(),
result.group('Loc'), result.group('Type')))
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
if fLocation == ";":
fLocation = None
fType = result.group('Type').strip()
else:
# handle three cases, 1. no location info 2. no type info 3. Both
if restOfLine.find(";") != -1: # missing type info
logger.warn("Missing Type information [%s]" % line)
result = self.NAME_LOC_REGEX.search(restOfLine)
if result:
logger.debug("Name: %s, Loc %s" %
(result.group('Name'), result.group('Loc')))
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
else:
logger.error("Could not parse [%s]" % restOfLine)
return
else: # missing location, assume at least two space seperate name and type
logger.warn("Missing location information [%s]" % line)
result = self.NAME_TYPE_REGEX.search(restOfLine)
if result:
fName = result.group('Name').strip()
fType = result.group('Type').strip()
logger.debug("Name: %s, Type %s" %
(result.group('Name'), result.group('Type')))
else:
logger.warn("Guessing Name: %s at line [%s]" %
(restOfLine.strip(), line))
stripedType = ""
if fType:
stripedType = self.__stripFieldAttributes__(fType)
if len(stripedType) > 0:
self.__createFieldByType__(fieldNo, stripedType, fName, fLocation,
line, Global, CrossReference)
else:
self._field = FileManFieldFactory.createField(
fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
logger.debug("Add field %s to File %s" %
(fName, self._curFile.getFileNo()))
self._curFile.addFileManField(self._field)
if len(stripedType) > 0:
self.__parseFieldAttributes__(fType)
def onSectionStart(self, line, section, Global, CrossReference):
self._lines = []
result = DataDictionaryListFileLogParser.FILEMAN_FIELD_START.search(line)
assert result
fileNo = result.group('FileNo')
fieldNo = result.group("FieldNo")
self._isSubFile = float(fileNo) != float(Global.getFileNo())
if self._isSubFile:
self._curFile = Global.getSubFileByFileNo(fileNo)
assert self._curFile, "Could not find subFile [%s] in file [%s] line [%s]" % (fileNo, Global.getFileNo(), line)
else:
self._curFile = Global
restOfLineStart = line.find("," + fieldNo) + len(fieldNo)
startIdent = self.DEFAULT_NAME_INDENT
defaultIdentLevel = self.__getDefaultIndentLevel__(self._curFile, self.DEFAULT_NAME_INDENT)
if restOfLineStart > defaultIdentLevel:
logger.warning("FileNo: %s, FieldNo: %s, line: %s, may not be a valid field no, %d, %d" %
(fileNo, fieldNo, line, restOfLineStart, defaultIdentLevel))
try:
floatValue = float(fieldNo)
except ValueError:
logger.error("invalid fieldNo %s" % fieldNo)
fieldNo = line[line.find(",")+1:defaultIdentLevel]
floatValue = float(fieldNo)
restOfLine = line[line.find("," + fieldNo) + len(fieldNo)+1:].strip()
result = NAME_LOC_TYPE_REGEX.search(restOfLine)
fName, fType, fLocation = None, None, None
if result:
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
if fLocation == ";":
fLocation = None
fType = result.group('Type').strip()
else:
# handle three cases, 1. no location info 2. no type info 3. Both
if restOfLine.find(";") != -1: # missing type info
logger.warn("Missing Type information [%s]" % line)
result = NAME_LOC_REGEX.search(restOfLine)
if result:
fName = result.group('Name').strip()
fLocation = result.group('Loc').strip()
else:
logger.error("Could not parse [%s]" % restOfLine)
return
else: # missing location, assume at least two space seperate name and type
result = NAME_TYPE_REGEX.search(restOfLine)
if result:
fName = result.group('Name').strip()
fType = result.group('Type').strip()
else:
logger.warn("Guessing Name: %s at line [%s]" % (restOfLine.strip(), line))
stripedType = ""
if fType:
stripedType = self.__stripFieldAttributes__(fType)
if stripedType:
self.__createFieldByType__(fieldNo, stripedType, fName, fLocation, line, Global, CrossReference)
else:
self._field = FileManFieldFactory.createField(fieldNo, fName, FileManField.FIELD_TYPE_NONE, fLocation)
self._curFile.addFileManField(self._field)
if stripedType:
self.__parseFieldAttributes__(fType)