def initrestofgrammar(self): try: self.nextmessage = getattr(self.module, 'nextmessage') except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere self.nextmessage = None try: self.nextmessage2 = getattr(self.module, 'nextmessage2') if self.nextmessage is None: raise botslib.GrammarError(_(u'Grammar "$grammar": if nextmessage2: nextmessage has to be used.'),grammar=self.grammarname) except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere self.nextmessage2 = None try: self.nextmessageblock = getattr(self.module, 'nextmessageblock') if self.nextmessage: raise botslib.GrammarError(_(u'Grammar "$grammar": nextmessageblock and nextmessage not both allowed.'),grammar=self.grammarname) except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere self.nextmessageblock = None if self._checkstructurerequired: try: self._dostructure() except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere raise botslib.GrammarError(_(u'Grammar "$grammar": no structure, is required.'),grammar=self.grammarname) except: self.structurefromgrammar[0]['error'] = True #mark the structure as having errors raise try: self._dorecorddefs() except: self.recorddefs['BOTS_1$@#%_error'] = True #mark structure has been read with errors raise else: self.recorddefs['BOTS_1$@#%_error'] = False #mark structure has been read and checked self.structure = copy.deepcopy(self.structurefromgrammar) #(deep)copy structure for use in translation (in translation values are changed, so use a copy) self._linkrecorddefs2structure(self.structure)
def _checkstructure(self,structure,mpath): ''' Recursive 1. Check structure. 2. Add keys: mpath, count ''' if not isinstance(structure,list): raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": not a list.'),grammar=self.grammarname,mpath=mpath) for i in structure: if not isinstance(i,dict): raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record should be a dict: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if ID not in i: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record without ID: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if not isinstance(i[ID],basestring): raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": recordID of record is not a string: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if not i[ID]: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": recordID of record is empty: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if MIN not in i: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record without MIN: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if MAX not in i: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record without MAX: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if not isinstance(i[MIN],int): raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record where MIN is not whole number: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if not isinstance(i[MAX],int): raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record where MAX is not whole number: "$record".'),grammar=self.grammarname,mpath=mpath,record=i) if i[MIN] > i[MAX]: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure, at "$mpath": record where MIN > MAX: "$record".'),grammar=self.grammarname,mpath=mpath,record=str(i)[:100]) i[MPATH]=mpath+[[i[ID]]] i[COUNT]=0 if LEVEL in i: self._checkstructure(i[LEVEL],i[MPATH])
def _checkbackcollision(self, structure, collision=None): ''' Recursive. Check if grammar has collision problem. A message with collision problems is ambigious. ''' headerissave = False if not collision: collision = [] for i in structure: #~ print 'check back',i[MPATH], 'with',collision if i[ID] in collision: raise botslib.GrammarError(_( u'Grammar "$grammar", in structure: back-collision detected at record "$mpath".' ), grammar=self.grammarname, mpath=i[MPATH]) if i[MIN]: collision = [] headerissave = True collision.append(i[ID]) if LEVEL in i: returncollision, returnheaderissave = self._checkbackcollision( i[LEVEL], [i[ID]]) collision.extend(returncollision) if returnheaderissave: #if one of segment(groups) is required, there is always a segment after the header segment; so remove header from nowcollision: collision.remove(i[ID]) return collision, headerissave #collision is used to update on higher level; cleared indicates the header segment can not collide anymore
def _manipulatefieldformat(self, field, recordID): try: field[BFORMAT] = self.formatconvert[field[FORMAT]] except KeyError: raise botslib.GrammarError( u'Grammar "$grammar", record "$record", field "$field": format "$format" has to be one of "$keys".', grammar=self.grammarname, record=recordID, field=field[ID], format=field[FORMAT], keys=self.formatconvert.keys())
def _linkrecorddefs2structure(self,structure): ''' recursive for each record in structure: add the pointer to the right recorddefinition. ''' for i in structure: try: i[FIELDS] = self.recorddefs[i[ID]] except KeyError: raise botslib.GrammarError(_(u'Grammar "$grammar": in recorddef no record "$record".'),grammar=self.grammarname,record=i[ID]) if LEVEL in i: self._linkrecorddefs2structure(i[LEVEL])
def syntaxread(soortpythonfile,editype,grammarname): ''' dispatch function for class Grammar or subclass read only grammar ''' try: classtocall = globals()[editype] except KeyError: raise botslib.GrammarError(_(u'Read grammar for type "$soort" editype "$editype" messagetype "$messagetype", but editype is unknown.'), soort=soortpythonfile,editype=editype, messagetype=grammarname) terug = classtocall(soortpythonfile,editype,grammarname) terug.initsyntax(includedefault=False) return terug
def grammarread(editype,grammarname): ''' dispatch function for class Grammar or subclass read whole grammar ''' try: classtocall = globals()[editype] except KeyError: raise botslib.GrammarError(_(u'Read grammar for editype "$editype" messagetype "$messagetype", but editype is unknown.'), editype=editype, messagetype=grammarname) terug = classtocall('grammars',editype,grammarname) terug.initsyntax(includedefault=True) terug.initrestofgrammar() return terug
def _dostructure(self): ''' 1. check the structure for validity. 2. adapt in structure: Add keys: mpath, count 3. remember that structure is checked and adapted (so when grammar is read again, no checking/adapt needed) ''' try: structurefromgrammar = getattr(self.module, 'structure') except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere raise botslib.GrammarError( _(u'Grammar "$grammar": no structure, is required.'), grammar=self.grammarname) self.structure = copy.deepcopy(structurefromgrammar) if len(self.structure) != 1: #every structure has only 1 root!! raise botslib.GrammarError(_( u'Grammar "$grammar", in structure: only one root record allowed.' ), grammar=self.grammarname) self._checkstructure(self.structure, []) if self.syntax['checkcollision']: self._checkbackcollision(self.structure) self._checknestedcollision(self.structure) self._checkbotscollision(self.structure)
def _checkbotscollision(self,structure,collision=None): ''' Recursive. Within one level: no twice the same tag. Bots can not handle this. ''' if not collision: collision=[] for i in structure: if i[ID] in collision: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure: bots-collision detected at record "$mpath".'),grammar=self.grammarname,mpath=i[MPATH]) collision.append(i[ID]) if LEVEL in i: self._checkbotscollision(i[LEVEL]) return
def initsyntax(self,includedefault): ''' Update default syntax from class with syntax read from grammar. ''' if includedefault: self.syntax = copy.deepcopy(self.__class__.defaultsyntax) #copy syntax from class data else: self.syntax = {} try: syntaxfromgrammar = getattr(self.module, 'syntax') except AttributeError: pass #there is no syntax in the grammar, is OK. else: if not isinstance(syntaxfromgrammar,dict): raise botslib.GrammarError(_(u'Grammar "$grammar": syntax is not a dict{}.'),grammar=self.grammarname) self.syntax.update(syntaxfromgrammar)
def initrestofgrammar(self): try: self.nextmessage = getattr(self.module, 'nextmessage') except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere self.nextmessage = None try: self.nextmessage2 = getattr(self.module, 'nextmessage2') if self.nextmessage is None: raise botslib.GrammarError(_( u'Grammar "$grammar": if nextmessage2: nextmessage has to be used.' ), grammar=self.grammarname) except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere self.nextmessage2 = None try: self.nextmessageblock = getattr(self.module, 'nextmessageblock') if self.nextmessage: raise botslib.GrammarError(_( u'Grammar "$grammar": nextmessageblock and nextmessage not both allowed.' ), grammar=self.grammarname) except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere self.nextmessageblock = None if self._checkstructurerequired: self._dostructure() self._dorecorddefs() self._linkrecorddefs2structure(self.structure) if self.syntax['noBOTSID'] and len(self.recorddefs) != 1: raise botslib.GrammarError(_( u'Grammar "$grammar": if syntax["noBOTSID"]: there can be only one record in recorddefs.' ), grammar=self.grammarname) if self.nextmessageblock is not None and len(self.recorddefs) != 1: raise botslib.GrammarError(_( u'Grammar "$grammar": if nextmessageblock: there can be only one record in recorddefs.' ), grammar=self.grammarname)
def _dostructure(self): ''' 1. check the structure for validity. 2. adapt in structure: Add keys: mpath, count 3. remember that structure is checked and adapted (so when grammar is read again, no checking/adapt needed) ''' self.structurefromgrammar = getattr(self.module, 'structure') if len(self.structurefromgrammar) != 1: #every structure has only 1 root!! raise botslib.GrammarError(_(u'Grammar "$grammar", in structure: only one root record allowed.'),grammar=self.grammarname) #check if structure is read & checked earlier in this run. If so, we can skip all checks. if 'error' in self.structurefromgrammar[0]: pass # grammar has been read before, but there are errors. Do nothing here, same errors will be raised again. elif MPATH in self.structurefromgrammar[0]: return # grammar has been red before, with no errors. Do no checks. self._checkstructure(self.structurefromgrammar,[]) if self.syntax['checkcollision']: self._checkbackcollision(self.structurefromgrammar) self._checknestedcollision(self.structurefromgrammar) self._checkbotscollision(self.structurefromgrammar)
def _checknestedcollision(self,structure,collision=None): ''' Recursive. Check if grammar has collision problem. A message with collision problems is ambiguous. ''' if not collision: levelcollision = [] else: levelcollision = collision[:] for i in reversed(structure): checkthissegment = True if LEVEL in i: checkthissegment = self._checknestedcollision(i[LEVEL],levelcollision + [i[ID]]) #~ print 'check nested',checkthissegment, i[MPATH], 'with',levelcollision if checkthissegment and i[ID] in levelcollision: raise botslib.GrammarError(_(u'Grammar "$grammar", in structure: nesting collision detected at record "$mpath".'),grammar=self.grammarname,mpath=i[MPATH]) if i[MIN]: levelcollision = [] #enecessarympty uppercollision return bool(levelcollision)
def _checkfield(self, field, recordID): #'normalise' field: make list equal length if len(field) == 3: # that is: composite field += [None, False, None, None, 'A'] #~ field[ISFIELD] = elif len(field) == 4: # that is: field (not a composite) field += [True, 0, 0, 'A'] #~ field[ISFIELD] = True #~ field[MINLENGTH] = 0 elif len( field ) == 8: # this happens when there are errors in a table and table is read again raise botslib.GrammarError(_( u'Grammar "$grammar": error in grammar; error is already reported in this run.' ), grammar=self.grammarname) else: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": list has invalid number of arguments.' ), grammar=self.grammarname, record=recordID, field=field[ID]) if not isinstance(field[ID], basestring) or not field[ID]: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": fieldID has to be a string.' ), grammar=self.grammarname, record=recordID, field=field[ID]) if not isinstance(field[MANDATORY], basestring): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": mandatory/conditional has to be a string.' ), grammar=self.grammarname, record=recordID, field=field[ID]) if not field[MANDATORY] or field[MANDATORY] not in ['M', 'C']: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": mandatory/conditional must be "M" or "C".' ), grammar=self.grammarname, record=recordID, field=field[ID]) if field[ISFIELD]: # that is: field, and not a composite #get MINLENGTH (from tuple or if fixed if isinstance(field[LENGTH], tuple): if not isinstance(field[LENGTH][0], int) and not isinstance( field[LENGTH][0], float): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": min length "$min" has to be a number.' ), grammar=self.grammarname, record=recordID, field=field[ID], min=field[LENGTH]) if not isinstance(field[LENGTH][1], int) and not isinstance( field[LENGTH][1], float): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": max length "$max" has to be a number.' ), grammar=self.grammarname, record=recordID, field=field[ID], max=field[LENGTH]) if field[LENGTH][0] > field[LENGTH][1]: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": min length "$min" must be > max length "$max".' ), grammar=self.grammarname, record=recordID, field=field[ID], min=field[LENGTH][0], max=field[LENGTH][1]) field[MINLENGTH] = field[LENGTH][0] field[LENGTH] = field[LENGTH][1] elif isinstance(field[LENGTH], int) or isinstance( field[LENGTH], float): if isinstance(self, fixed): field[MINLENGTH] = field[LENGTH] else: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": length "$len" has to be number or (min,max).' ), grammar=self.grammarname, record=recordID, field=field[ID], len=field[LENGTH]) if field[LENGTH] < 1: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": length "$len" has to be at least 1.' ), grammar=self.grammarname, record=recordID, field=field[ID], len=field[LENGTH]) if field[MINLENGTH] < 0: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": minlength "$len" has to be at least 0.' ), grammar=self.grammarname, record=recordID, field=field[ID], len=field[LENGTH]) #format if not isinstance(field[FORMAT], basestring): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": format "$format" has to be a string.' ), grammar=self.grammarname, record=recordID, field=field[ID], format=field[FORMAT]) self._manipulatefieldformat(field, recordID) #~ if field[FORMAT] in ['N','I','R']: if field[BFORMAT] in ['N', 'I', 'R']: if isinstance(field[LENGTH], float): field[DECIMALS] = int( round((field[LENGTH] - int(field[LENGTH])) * 10)) #fill DECIMALS field[LENGTH] = int(round(field[LENGTH])) if field[DECIMALS] >= field[LENGTH]: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": field length "$len" has to be greater that nr of decimals "$decimals".' ), grammar=self.grammarname, record=recordID, field=field[ID], len=field[LENGTH], decimals=field[DECIMALS]) if isinstance(field[MINLENGTH], float): field[MINLENGTH] = int(round(field[MINLENGTH])) else: #if format 'R', A, D, T if isinstance(field[LENGTH], float): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": if format "$format", no length "$len".' ), grammar=self.grammarname, record=recordID, field=field[ID], format=field[FORMAT], len=field[LENGTH]) if isinstance(field[MINLENGTH], float): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": if format "$format", no minlength "$len".' ), grammar=self.grammarname, record=recordID, field=field[ID], format=field[FORMAT], len=field[MINLENGTH]) else: #check composite if not isinstance(field[SUBFIELDS], list): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field": is a composite field, has to have subfields.' ), grammar=self.grammarname, record=recordID, field=field[ID]) if len(field[SUBFIELDS]) < 2: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record", field "$field" has < 2 sfields.' ), grammar=self.grammarname, record=recordID, field=field[ID])
def _dorecorddefs(self): ''' 1. check the recorddefinitions for validity. 2. adapt in field-records: normalise length lists, set bool ISFIELD, etc 3. remember that recorddef is checked and adapted (so when grammar is read again, no checking/adapt needed) ''' try: recorddefsfromgrammar = getattr(self.module, 'recorddefs') except AttributeError: #if grammarpart does not exist set to None; test required grammarpart elsewhere raise botslib.GrammarError( _(u'Grammar "$grammar": no recorddefs.'), grammar=self.grammarname) self.recorddefs = copy.deepcopy(recorddefsfromgrammar) if not isinstance(self.recorddefs, dict): raise botslib.GrammarError( _(u'Grammar "$grammar": recorddefs is not a dict{}.'), grammar=self.grammarname) for recordID, fields in self.recorddefs.iteritems(): for field in fields: #check if field 'BOTSID' is present: if field[ID] == 'BOTSID': break else: #so there is no field 'BOTSID' in record raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record": no field BOTSID.'), grammar=self.grammarname, record=recordID) if not isinstance(recordID, basestring): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record": is not a string.'), grammar=self.grammarname, record=recordID) if not recordID: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record": recordID with empty string.' ), grammar=self.grammarname, record=recordID) if not isinstance(fields, list): raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record": no correct fields found.' ), grammar=self.grammarname, record=recordID) if isinstance(self, xml) or isinstance(self, json): if len(fields) < 1: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record": too few fields.' ), grammar=self.grammarname, record=recordID) else: if len(fields) < 2: raise botslib.GrammarError(_( u'Grammar "$grammar", record "$record": too few fields.' ), grammar=self.grammarname, record=recordID) for field in fields: self._checkfield(field, recordID) if not field[ISFIELD]: # if composite for sfield in field[SUBFIELDS]: self._checkfield(sfield, recordID)
def _dorecorddefs(self): ''' 1. check the recorddefinitions for validity. 2. adapt in field-records: normalise length lists, set bool ISFIELD, etc ''' try: self.recorddefs = getattr(self.module, 'recorddefs') except AttributeError: raise botslib.GrammarError(_(u'Grammar "$grammar": no recorddefs.'),grammar=self.grammarname) if not isinstance(self.recorddefs,dict): raise botslib.GrammarError(_(u'Grammar "$grammar": recorddefs is not a dict{}.'),grammar=self.grammarname) #check if grammar is read & checked earlier in this run. If so, we can skip all checks. if 'BOTS_1$@#%_error' in self.recorddefs: #if checked before if self.recorddefs['BOTS_1$@#%_error']: #if grammar had errors raise botslib.GrammarError(_(u'Grammar "$grammar" has error that is already reported in this run.'),grammar=self.grammarname) return #no error, skip checks for recordID ,fields in self.recorddefs.iteritems(): if not isinstance(recordID,basestring): raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": is not a string.'),grammar=self.grammarname,record=recordID) if not recordID: raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": recordID with empty string.'),grammar=self.grammarname,record=recordID) if not isinstance(fields,list): raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": no correct fields found.'),grammar=self.grammarname,record=recordID) if isinstance(self,(xml,json)): if len (fields) < 1: raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": too few fields.'),grammar=self.grammarname,record=recordID) else: if len (fields) < 2: raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": too few fields.'),grammar=self.grammarname,record=recordID) hasBOTSID = False #to check if BOTSID is present fieldnamelist = [] #to check for double fieldnames for field in fields: self._checkfield(field,recordID) if not field[ISFIELD]: # if composite for sfield in field[SUBFIELDS]: self._checkfield(sfield,recordID) if sfield[ID] in fieldnamelist: raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": field "$field" appears twice. Field names should be unique within a record.'),grammar=self.grammarname,record=recordID,field=sfield[ID]) fieldnamelist.append(sfield[ID]) else: if field[ID] == 'BOTSID': hasBOTSID = True if field[ID] in fieldnamelist: raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": field "$field" appears twice. Field names should be unique within a record.'),grammar=self.grammarname,record=recordID,field=field[ID]) fieldnamelist.append(field[ID]) if not hasBOTSID: #there is no field 'BOTSID' in record raise botslib.GrammarError(_(u'Grammar "$grammar", record "$record": no field BOTSID.'),grammar=self.grammarname,record=recordID) if self.syntax['noBOTSID'] and len(self.recorddefs) != 1: raise botslib.GrammarError(_(u'Grammar "$grammar": if syntax["noBOTSID"]: there can be only one record in recorddefs.'),grammar=self.grammarname) if self.nextmessageblock is not None and len(self.recorddefs) != 1: raise botslib.GrammarError(_(u'Grammar "$grammar": if nextmessageblock: there can be only one record in recorddefs.'),grammar=self.grammarname)