def initrestofgrammar(self):
try:
self.nextmessage = getattr(self.module, 'nextmessage')
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
self.nextmessage = None
try:
self.nextmessage2 = getattr(self.module, 'nextmessage2')
if self.nextmessage is None:
raise botslib.GrammarError(_(u'Grammar "$grammar": if nextmessage2: nextmessage has to be used.'),grammar=self.grammarname)
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
self.nextmessage2 = None
try:
self.nextmessageblock = getattr(self.module, 'nextmessageblock')
if self.nextmessage:
raise botslib.GrammarError(_(u'Grammar "$grammar": nextmessageblock and nextmessage not both allowed.'),grammar=self.grammarname)
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
self.nextmessageblock = None
if self._checkstructurerequired:
try:
self._dostructure()
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
raise botslib.GrammarError(_(u'Grammar "$grammar": no structure, is required.'),grammar=self.grammarname)
except:
self.structurefromgrammar[0]['error'] = True #mark the structure as having errors
raise
try:
self._dorecorddefs()
except:
self.recorddefs['BOTS_1$@#%_error'] = True #mark structure has been read with errors
raise
else:
self.recorddefs['BOTS_1$@#%_error'] = False #mark structure has been read and checked
self.structure = copy.deepcopy(self.structurefromgrammar) #(deep)copy structure for use in translation (in translation values are changed, so use a copy)
self._linkrecorddefs2structure(self.structure)
def _checkstructure(self,structure,mpath):
''' Recursive
1. Check structure.
2. Add keys: mpath, count
'''
if not isinstance(structure,list):
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": not a list.'),grammar=self.grammarname,mpath=mpath)
for i in structure:
if not isinstance(i,dict):
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record should be a dict: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if ID not in i:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record without ID: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if not isinstance(i[ID],basestring):
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": recordID of record is not a string: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if not i[ID]:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": recordID of record is empty: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if MIN not in i:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record without MIN: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if MAX not in i:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record without MAX: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if not isinstance(i[MIN],int):
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record where MIN is not whole number: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if not isinstance(i[MAX],int):
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record where MAX is not whole number: "$record".'),grammar=self.grammarname,mpath=mpath,record=i)
if i[MIN] > i[MAX]:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record where MIN > MAX: "$record".'),grammar=self.grammarname,mpath=mpath,record=str(i)[:100])
i[MPATH]=mpath+[[i[ID]]]
i[COUNT]=0
if LEVEL in i:
self._checkstructure(i[LEVEL],i[MPATH])
def _checkbackcollision(self, structure, collision=None):
''' Recursive.
Check if grammar has collision problem.
A message with collision problems is ambigious.
'''
headerissave = False
if not collision:
collision = []
for i in structure:
#~ print 'check back',i[MPATH], 'with',collision
if i[ID] in collision:
raise botslib.GrammarError(_(
u'Grammar "$grammar", in structure: back-collision detected at record "$mpath".'
),
grammar=self.grammarname,
mpath=i[MPATH])
if i[MIN]:
collision = []
headerissave = True
collision.append(i[ID])
if LEVEL in i:
returncollision, returnheaderissave = self._checkbackcollision(
i[LEVEL], [i[ID]])
collision.extend(returncollision)
if returnheaderissave: #if one of segment(groups) is required, there is always a segment after the header segment; so remove header from nowcollision:
collision.remove(i[ID])
return collision, headerissave #collision is used to update on higher level; cleared indicates the header segment can not collide anymore
def _manipulatefieldformat(self, field, recordID):
try:
field[BFORMAT] = self.formatconvert[field[FORMAT]]
except KeyError:
raise botslib.GrammarError(
u'Grammar "$grammar", record "$record", field "$field": format "$format" has to be one of "$keys".',
grammar=self.grammarname,
record=recordID,
field=field[ID],
format=field[FORMAT],
keys=self.formatconvert.keys())
def _linkrecorddefs2structure(self,structure):
''' recursive
for each record in structure: add the pointer to the right recorddefinition.
'''
for i in structure:
try:
i[FIELDS] = self.recorddefs[i[ID]]
except KeyError:
raise botslib.GrammarError(_(u'Grammar "$grammar": in recorddef no record "$record".'),grammar=self.grammarname,record=i[ID])
if LEVEL in i:
self._linkrecorddefs2structure(i[LEVEL])
def syntaxread(soortpythonfile,editype,grammarname):
''' dispatch function for class Grammar or subclass
read only grammar
'''
try:
classtocall = globals()[editype]
except KeyError:
raise botslib.GrammarError(_(u'Read grammar for type "$soort" editype "$editype" messagetype "$messagetype", but editype is unknown.'), soort=soortpythonfile,editype=editype, messagetype=grammarname)
terug = classtocall(soortpythonfile,editype,grammarname)
terug.initsyntax(includedefault=False)
return terug
def grammarread(editype,grammarname):
''' dispatch function for class Grammar or subclass
read whole grammar
'''
try:
classtocall = globals()[editype]
except KeyError:
raise botslib.GrammarError(_(u'Read grammar for editype "$editype" messagetype "$messagetype", but editype is unknown.'), editype=editype, messagetype=grammarname)
terug = classtocall('grammars',editype,grammarname)
terug.initsyntax(includedefault=True)
terug.initrestofgrammar()
return terug
def _dostructure(self):
''' 1. check the structure for validity.
2. adapt in structure: Add keys: mpath, count
3. remember that structure is checked and adapted (so when grammar is read again, no checking/adapt needed)
'''
try:
structurefromgrammar = getattr(self.module, 'structure')
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
raise botslib.GrammarError(
_(u'Grammar "$grammar": no structure, is required.'),
grammar=self.grammarname)
self.structure = copy.deepcopy(structurefromgrammar)
if len(self.structure) != 1: #every structure has only 1 root!!
raise botslib.GrammarError(_(
u'Grammar "$grammar", in structure: only one root record allowed.'
),
grammar=self.grammarname)
self._checkstructure(self.structure, [])
if self.syntax['checkcollision']:
self._checkbackcollision(self.structure)
self._checknestedcollision(self.structure)
self._checkbotscollision(self.structure)
def _checkbotscollision(self,structure,collision=None):
''' Recursive.
Within one level: no twice the same tag. Bots can not handle this.
'''
if not collision:
collision=[]
for i in structure:
if i[ID] in collision:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure: bots-collision detected at record "$mpath".'),grammar=self.grammarname,mpath=i[MPATH])
collision.append(i[ID])
if LEVEL in i:
self._checkbotscollision(i[LEVEL])
return
def initsyntax(self,includedefault):
''' Update default syntax from class with syntax read from grammar. '''
if includedefault:
self.syntax = copy.deepcopy(self.__class__.defaultsyntax) #copy syntax from class data
else:
self.syntax = {}
try:
syntaxfromgrammar = getattr(self.module, 'syntax')
except AttributeError:
pass #there is no syntax in the grammar, is OK.
else:
if not isinstance(syntaxfromgrammar,dict):
raise botslib.GrammarError(_(u'Grammar "$grammar": syntax is not a dict{}.'),grammar=self.grammarname)
self.syntax.update(syntaxfromgrammar)
def initrestofgrammar(self):
try:
self.nextmessage = getattr(self.module, 'nextmessage')
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
self.nextmessage = None
try:
self.nextmessage2 = getattr(self.module, 'nextmessage2')
if self.nextmessage is None:
raise botslib.GrammarError(_(
u'Grammar "$grammar": if nextmessage2: nextmessage has to be used.'
),
grammar=self.grammarname)
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
self.nextmessage2 = None
try:
self.nextmessageblock = getattr(self.module, 'nextmessageblock')
if self.nextmessage:
raise botslib.GrammarError(_(
u'Grammar "$grammar": nextmessageblock and nextmessage not both allowed.'
),
grammar=self.grammarname)
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
self.nextmessageblock = None
if self._checkstructurerequired:
self._dostructure()
self._dorecorddefs()
self._linkrecorddefs2structure(self.structure)
if self.syntax['noBOTSID'] and len(self.recorddefs) != 1:
raise botslib.GrammarError(_(
u'Grammar "$grammar": if syntax["noBOTSID"]: there can be only one record in recorddefs.'
),
grammar=self.grammarname)
if self.nextmessageblock is not None and len(self.recorddefs) != 1:
raise botslib.GrammarError(_(
u'Grammar "$grammar": if nextmessageblock: there can be only one record in recorddefs.'
),
grammar=self.grammarname)
def _dostructure(self):
''' 1. check the structure for validity.
2. adapt in structure: Add keys: mpath, count
3. remember that structure is checked and adapted (so when grammar is read again, no checking/adapt needed)
'''
self.structurefromgrammar = getattr(self.module, 'structure')
if len(self.structurefromgrammar) != 1: #every structure has only 1 root!!
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure: only one root record allowed.'),grammar=self.grammarname)
#check if structure is read & checked earlier in this run. If so, we can skip all checks.
if 'error' in self.structurefromgrammar[0]:
pass # grammar has been read before, but there are errors. Do nothing here, same errors will be raised again.
elif MPATH in self.structurefromgrammar[0]:
return # grammar has been red before, with no errors. Do no checks.
self._checkstructure(self.structurefromgrammar,[])
if self.syntax['checkcollision']:
self._checkbackcollision(self.structurefromgrammar)
self._checknestedcollision(self.structurefromgrammar)
self._checkbotscollision(self.structurefromgrammar)
def _checknestedcollision(self,structure,collision=None):
''' Recursive.
Check if grammar has collision problem.
A message with collision problems is ambiguous.
'''
if not collision:
levelcollision = []
else:
levelcollision = collision[:]
for i in reversed(structure):
checkthissegment = True
if LEVEL in i:
checkthissegment = self._checknestedcollision(i[LEVEL],levelcollision + [i[ID]])
#~ print 'check nested',checkthissegment, i[MPATH], 'with',levelcollision
if checkthissegment and i[ID] in levelcollision:
raise botslib.GrammarError(_(u'Grammar "$grammar", in structure: nesting collision detected at record "$mpath".'),grammar=self.grammarname,mpath=i[MPATH])
if i[MIN]:
levelcollision = [] #enecessarympty uppercollision
return bool(levelcollision)
def _checkfield(self, field, recordID):
#'normalise' field: make list equal length
if len(field) == 3: # that is: composite
field += [None, False, None, None, 'A']
#~ field[ISFIELD] =
elif len(field) == 4: # that is: field (not a composite)
field += [True, 0, 0, 'A']
#~ field[ISFIELD] = True
#~ field[MINLENGTH] = 0
elif len(
field
) == 8: # this happens when there are errors in a table and table is read again
raise botslib.GrammarError(_(
u'Grammar "$grammar": error in grammar; error is already reported in this run.'
),
grammar=self.grammarname)
else:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": list has invalid number of arguments.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID])
if not isinstance(field[ID], basestring) or not field[ID]:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": fieldID has to be a string.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID])
if not isinstance(field[MANDATORY], basestring):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": mandatory/conditional has to be a string.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID])
if not field[MANDATORY] or field[MANDATORY] not in ['M', 'C']:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": mandatory/conditional must be "M" or "C".'
),
grammar=self.grammarname,
record=recordID,
field=field[ID])
if field[ISFIELD]: # that is: field, and not a composite
#get MINLENGTH (from tuple or if fixed
if isinstance(field[LENGTH], tuple):
if not isinstance(field[LENGTH][0], int) and not isinstance(
field[LENGTH][0], float):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": min length "$min" has to be a number.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
min=field[LENGTH])
if not isinstance(field[LENGTH][1], int) and not isinstance(
field[LENGTH][1], float):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": max length "$max" has to be a number.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
max=field[LENGTH])
if field[LENGTH][0] > field[LENGTH][1]:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": min length "$min" must be > max length "$max".'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
min=field[LENGTH][0],
max=field[LENGTH][1])
field[MINLENGTH] = field[LENGTH][0]
field[LENGTH] = field[LENGTH][1]
elif isinstance(field[LENGTH], int) or isinstance(
field[LENGTH], float):
if isinstance(self, fixed):
field[MINLENGTH] = field[LENGTH]
else:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": length "$len" has to be number or (min,max).'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
len=field[LENGTH])
if field[LENGTH] < 1:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": length "$len" has to be at least 1.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
len=field[LENGTH])
if field[MINLENGTH] < 0:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": minlength "$len" has to be at least 0.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
len=field[LENGTH])
#format
if not isinstance(field[FORMAT], basestring):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": format "$format" has to be a string.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
format=field[FORMAT])
self._manipulatefieldformat(field, recordID)
#~ if field[FORMAT] in ['N','I','R']:
if field[BFORMAT] in ['N', 'I', 'R']:
if isinstance(field[LENGTH], float):
field[DECIMALS] = int(
round((field[LENGTH] - int(field[LENGTH])) *
10)) #fill DECIMALS
field[LENGTH] = int(round(field[LENGTH]))
if field[DECIMALS] >= field[LENGTH]:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": field length "$len" has to be greater that nr of decimals "$decimals".'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
len=field[LENGTH],
decimals=field[DECIMALS])
if isinstance(field[MINLENGTH], float):
field[MINLENGTH] = int(round(field[MINLENGTH]))
else: #if format 'R', A, D, T
if isinstance(field[LENGTH], float):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": if format "$format", no length "$len".'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
format=field[FORMAT],
len=field[LENGTH])
if isinstance(field[MINLENGTH], float):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": if format "$format", no minlength "$len".'
),
grammar=self.grammarname,
record=recordID,
field=field[ID],
format=field[FORMAT],
len=field[MINLENGTH])
else: #check composite
if not isinstance(field[SUBFIELDS], list):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field": is a composite field, has to have subfields.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID])
if len(field[SUBFIELDS]) < 2:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record", field "$field" has < 2 sfields.'
),
grammar=self.grammarname,
record=recordID,
field=field[ID])
def _dorecorddefs(self):
''' 1. check the recorddefinitions for validity.
2. adapt in field-records: normalise length lists, set bool ISFIELD, etc
3. remember that recorddef is checked and adapted (so when grammar is read again, no checking/adapt needed)
'''
try:
recorddefsfromgrammar = getattr(self.module, 'recorddefs')
except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere
raise botslib.GrammarError(
_(u'Grammar "$grammar": no recorddefs.'),
grammar=self.grammarname)
self.recorddefs = copy.deepcopy(recorddefsfromgrammar)
if not isinstance(self.recorddefs, dict):
raise botslib.GrammarError(
_(u'Grammar "$grammar": recorddefs is not a dict{}.'),
grammar=self.grammarname)
for recordID, fields in self.recorddefs.iteritems():
for field in fields: #check if field 'BOTSID' is present:
if field[ID] == 'BOTSID':
break
else: #so there is no field 'BOTSID' in record
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record": no field BOTSID.'),
grammar=self.grammarname,
record=recordID)
if not isinstance(recordID, basestring):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record": is not a string.'),
grammar=self.grammarname,
record=recordID)
if not recordID:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record": recordID with empty string.'
),
grammar=self.grammarname,
record=recordID)
if not isinstance(fields, list):
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record": no correct fields found.'
),
grammar=self.grammarname,
record=recordID)
if isinstance(self, xml) or isinstance(self, json):
if len(fields) < 1:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record": too few fields.'
),
grammar=self.grammarname,
record=recordID)
else:
if len(fields) < 2:
raise botslib.GrammarError(_(
u'Grammar "$grammar", record "$record": too few fields.'
),
grammar=self.grammarname,
record=recordID)
for field in fields:
self._checkfield(field, recordID)
if not field[ISFIELD]: # if composite
for sfield in field[SUBFIELDS]:
self._checkfield(sfield, recordID)
def _dorecorddefs(self):
''' 1. check the recorddefinitions for validity.
2. adapt in field-records: normalise length lists, set bool ISFIELD, etc
'''
try:
self.recorddefs = getattr(self.module, 'recorddefs')
except AttributeError:
raise botslib.GrammarError(_(u'Grammar "$grammar": no recorddefs.'),grammar=self.grammarname)
if not isinstance(self.recorddefs,dict):
raise botslib.GrammarError(_(u'Grammar "$grammar": recorddefs is not a dict{}.'),grammar=self.grammarname)
#check if grammar is read & checked earlier in this run. If so, we can skip all checks.
if 'BOTS_1$@#%_error' in self.recorddefs: #if checked before
if self.recorddefs['BOTS_1$@#%_error']: #if grammar had errors
raise botslib.GrammarError(_(u'Grammar "$grammar" has error that is already reported in this run.'),grammar=self.grammarname)
return #no error, skip checks
for recordID ,fields in self.recorddefs.iteritems():
if not isinstance(recordID,basestring):
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": is not a string.'),grammar=self.grammarname,record=recordID)
if not recordID:
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": recordID with empty string.'),grammar=self.grammarname,record=recordID)
if not isinstance(fields,list):
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": no correct fields found.'),grammar=self.grammarname,record=recordID)
if isinstance(self,(xml,json)):
if len (fields) < 1:
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": too few fields.'),grammar=self.grammarname,record=recordID)
else:
if len (fields) < 2:
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": too few fields.'),grammar=self.grammarname,record=recordID)
hasBOTSID = False #to check if BOTSID is present
fieldnamelist = [] #to check for double fieldnames
for field in fields:
self._checkfield(field,recordID)
if not field[ISFIELD]: # if composite
for sfield in field[SUBFIELDS]:
self._checkfield(sfield,recordID)
if sfield[ID] in fieldnamelist:
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": field "$field" appears twice. Field names should be unique within a record.'),grammar=self.grammarname,record=recordID,field=sfield[ID])
fieldnamelist.append(sfield[ID])
else:
if field[ID] == 'BOTSID':
hasBOTSID = True
if field[ID] in fieldnamelist:
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": field "$field" appears twice. Field names should be unique within a record.'),grammar=self.grammarname,record=recordID,field=field[ID])
fieldnamelist.append(field[ID])
if not hasBOTSID: #there is no field 'BOTSID' in record
raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": no field BOTSID.'),grammar=self.grammarname,record=recordID)
if self.syntax['noBOTSID'] and len(self.recorddefs) != 1:
raise botslib.GrammarError(_(u'Grammar "$grammar": if syntax["noBOTSID"]: there can be only one record in recorddefs.'),grammar=self.grammarname)
if self.nextmessageblock is not None and len(self.recorddefs) != 1:
raise botslib.GrammarError(_(u'Grammar "$grammar": if nextmessageblock: there can be only one record in recorddefs.'),grammar=self.grammarname)
