def evaluateretryrun(type,stuff2evaluate): resultlast={OPEN:0,ERROR:0,OK:0,DONE:0} didretry = False for row in botslib.query('''SELECT idta FROM filereport GROUP BY idta HAVING MAX(statust) != %(statust)s''', {'statust':DONE}): didretry = True for tadict in botslib.query('''SELECT ''' + tavars + ''' FROM ta WHERE idta= %(idta)s ''', {'idta':row['idta']}): break else: #there really should be a corresponding ta raise botslib.PanicError(_(u'MaintenanceRetry: could not find transaction "$txt".'),txt=row['idta']) mytrace = Trace(tadict,stuff2evaluate) resultlast[mytrace.statusttree]+=1 if mytrace.statusttree == DONE: mytrace.errortext = '' #~ mytrace.ta.update(tracestatus=mytrace.statusttree) #ts for retried filereports is tricky: is this the time the file was originally received? best would be to use ts of prepare... #that is quite difficult, so use time of this run rootta=botslib.OldTransaction(stuff2evaluate) rootta.syn('ts') #get the timestamp of this run mytrace.ts = rootta.ts insert_filereport(mytrace) del mytrace.ta del mytrace if not didretry: return 0 #no error return finish_evaluation(stuff2evaluate,resultlast,type)
def run(self): ''' reinjects files with failed communication. ''' #bots keeps track of last time automaticretrycommunication was done; reason: performance idta_lastretry = botslib.unique('bots__automaticretrycommunication',updatewith=self.minta4query) if idta_lastretry == 1: #this is the first time automaticretrycommunication is run. #do not do anything this run, in order to avoid sending older files. return False #no run for row in botslib.query('''SELECT MIN(idta) AS min_idta FROM filereport WHERE idta > %(idta_lastretry)s AND statust = %(statust)s ''', {'statust':ERROR,'idta_lastretry':idta_lastretry}): startidta = row['min_idta'] if not startidta: return False #no run do_retransmit = False for row in botslib.query('''SELECT idta,parent,numberofresends FROM ta WHERE idta > %(startidta)s AND status = %(status)s AND statust = %(statust)s ''', {'statust':ERROR,'status':EXTERNOUT,'startidta':startidta}): do_retransmit = True ta_outgoing = botslib.OldTransaction(row['idta']) ta_outgoing.update(retransmit=False,statust=RESEND) #set retransmit back to False ta_resend = botslib.OldTransaction(row['parent']) #parent ta with status RAWOUT; this is where the outgoing file is kept ta_externin = ta_resend.copyta(status=EXTERNIN,statust=DONE) #inject; status is DONE so this ta is not used further ta_externin.copyta(status=FILEOUT,statust=OK,numberofresends=row['numberofresends']) #reinjected file is ready as new input if do_retransmit: return super(automaticretrycommunication, self).run() else: return False #no run
def finish_evaluation(stuff2evaluate,resultlast,type): #count nr files send for row in botslib.query('''SELECT COUNT(*) as count FROM ta WHERE idta > %(rootidta)s AND status=%(status)s AND statust=%(statust)s ''', {'status':EXTERNOUT,'rootidta':stuff2evaluate,'statust':DONE}): send = row['count'] #count process errors for row in botslib.query('''SELECT COUNT(*) as count FROM ta WHERE idta >= %(rootidta)s AND status=%(status)s AND statust=%(statust)s''', {'status':PROCESS,'rootidta':stuff2evaluate,'statust':ERROR}): processerrors = row['count'] #generate report (in database) rootta=botslib.OldTransaction(stuff2evaluate) rootta.syn('ts') #get the timestamp of this run LastReceived=resultlast[DONE]+resultlast[OK]+resultlast[OPEN]+resultlast[ERROR] status = bool(resultlast[OK]+resultlast[OPEN]+resultlast[ERROR]+processerrors) botslib.change(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,status,type) VALUES (%(idta)s, %(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s, %(send)s,%(processerrors)s,%(ts)s,%(lastreceived)s,%(status)s,%(type)s) ''', {'idta':stuff2evaluate, 'lastopen':resultlast[OPEN],'lasterror':resultlast[ERROR],'lastok':resultlast[OK],'lastdone':resultlast[DONE], 'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':LastReceived,'status':status,'type':type[2:]}) return generate_report(stuff2evaluate) #return report status: 0 (no error) or 1 (error)
def run(self): ''' prepare the files indicated by user to be rereceived. Return: indication if files should be rereceived. ''' do_retransmit = False for row in botslib.query('''SELECT idta FROM filereport WHERE retransmit = %(retransmit)s ''', {'retransmit':1}): do_retransmit = True botslib.changeq('''UPDATE filereport SET retransmit = %(retransmit)s WHERE idta = %(idta)s ''', {'idta':row['idta'],'retransmit':0}) for row2 in botslib.query('''SELECT idta FROM ta WHERE parent = %(parent)s ''', {'parent':row['idta']}): ta_rereceive = botslib.OldTransaction(row2['idta']) ta_externin = ta_rereceive.copyta(status=EXTERNIN,statust=DONE,parent=0) #inject; status is DONE so this ta is not used further ta_externin.copyta(status=FILEIN,statust=OK) #reinjected file is ready as new input if do_retransmit: return super(rereceive, self).run() else: return False #no run
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize): #count nr files send for row in botslib.query('''SELECT COUNT(*) as count FROM ta WHERE idta > %(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s ''', {'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}): send = row['count'] #count process errors for row in botslib.query('''SELECT COUNT(*) as count FROM ta WHERE idta >= %(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s''', {'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}): processerrors = row['count'] #generate report (in database) rootta = botslib.OldTransaction(rootidtaofrun) rootta.syn('ts') #get the timestamp of this run lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR] status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors) botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors, ts,lastreceived,status,type,filesize,acceptance) VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s, %(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s) ''', {'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK], 'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived, 'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False))}) #20120830: if new run with nothing received and no process errors: delete ta's. if command == 'new' and not lastreceived and not processerrors: botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def email_error_report(rootidtaofrun): for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidtaofrun)s''', {'rootidtaofrun':rootidtaofrun}): break else: raise botslib.PanicError(_(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s')%{'time':unicode(results['ts'])[:16]} reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':unicode(results['ts'])[:19]} reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n')%(results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors')%(results['lasterror']) reporttext += _(u' %d files with errors,\n')%(results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck')%(results['lastok']) reporttext += _(u' %d files got stuck,\n')%(results['lastok']) if results['lastopen']: subject += _(u'; %d system errors')%(results['lastopen']) reporttext += _(u' %d system errors,\n')%(results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors')%(results['processerrors']) reporttext += _(u' %d errors in processes.\n')%(results['processerrors']) reporttext += _(u' %d files send in run.\n')%(results['send']) botsglobal.logger.info(reporttext) #log the report texts # only send email report if there are errors. # sendreportifprocesserror (in bots.ini): no email reports if only process errors if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)): # Include details about process errors in the email report; if debug is True: includes trace if results['processerrors']: for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext FROM ta WHERE idta>=%(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}): reporttext += '\nProcess error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) # Include details about file errors in the email report; if debug is True: includes trace if results['lasterror'] or results['lastopen'] or results['lastok']: for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename FROM filereport WHERE idta>%(rootidtaofrun)s AND statust!=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'statust':DONE}): reporttext += '\nFile error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) botslib.sendbotserrorreport(subject,reporttext) return int(results['status']) #return report status: 0 (no error) or 1 (error)
def email_error_report(rootidtaofrun): for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidtaofrun)s''', {'rootidtaofrun':rootidtaofrun}): break else: raise botslib.PanicError(_(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]} reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]} reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n')%(results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors')%(results['lasterror']) reporttext += _(u' %d files with errors,\n')%(results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck')%(results['lastok']) reporttext += _(u' %d files got stuck,\n')%(results['lastok']) if results['lastopen']: subject += _(u'; %d system errors')%(results['lastopen']) reporttext += _(u' %d system errors,\n')%(results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors')%(results['processerrors']) reporttext += _(u' %d errors in processes.\n')%(results['processerrors']) reporttext += _(u' %d files send in run.\n')%(results['send']) botsglobal.logger.info(reporttext) #log the report texts # only send email report if there are errors. # sendreportifprocesserror (in bots.ini): no email reports if only process errors if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)): # Include details about process errors in the email report; if debug is True: includes trace if results['processerrors']: for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext FROM ta WHERE idta>=%(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}): reporttext += '\nProcess error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) # Include details about file errors in the email report; if debug is True: includes trace if results['lasterror'] or results['lastopen'] or results['lastok']: for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename FROM filereport WHERE idta>%(rootidtaofrun)s AND statust!=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'statust':DONE}): reporttext += '\nFile error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) botslib.sendbotserrorreport(subject,reporttext) return int(results['status']) #return report status: 0 (no error) or 1 (error)
def preparerecommunication(): #for each out-communication process that went wrong: retransmit = False #indicate retransmit for row in botslib.query( '''SELECT idta,tochannel FROM ta WHERE statust!=%(statust)s AND status=%(status)s AND retransmit=%(retransmit)s ''', { 'status': PROCESS, 'retransmit': True, 'statust': DONE }): run_outgoing = botslib.OldTransaction(row['idta']) run_outgoing.update(retransmit=False) #set retransmit back to False #get rootidta of run where communication failed for row2 in botslib.query( '''SELECT max(idta) as rootidta FROM ta WHERE script=%(script)s AND idta<%(thisidta)s ''', { 'script': 0, 'thisidta': row['idta'] }): rootidta = row2['rootidta'] #get endidta of run where communication failed for row3 in botslib.query( '''SELECT min(idta) as endidta FROM ta WHERE script=%(script)s AND idta>%(thisidta)s ''', { 'script': 0, 'thisidta': row['idta'] }): endidta = row3['endidta'] if not endidta: endidta = sys.maxint - 1 #reinject for row4 in botslib.query( '''SELECT idta FROM ta WHERE idta<%(endidta)s AND idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND tochannel=%(tochannel)s ''', { 'statust': OK, 'status': RAWOUT, 'rootidta': rootidta, 'endidta': endidta, 'tochannel': row['tochannel'] }): retransmit = True ta_outgoing = botslib.OldTransaction(row4['idta']) ta_outgoing_copy = ta_outgoing.copyta(status=RAWOUT, statust=OK) ta_outgoing.update(statust=DONE) return retransmit
def prepareretransmit(): ''' prepare the retransmittable files. Return: indication if files should be retransmitted.''' retransmit = False #indicate retransmit #for rereceive for row in botslib.query( '''SELECT idta,reportidta FROM filereport WHERE retransmit=%(retransmit)s ''', {'retransmit': True}): retransmit = True botslib.change( '''UPDATE filereport SET retransmit=%(retransmit)s WHERE idta=%(idta)s AND reportidta=%(reportidta)s ''', { 'idta': row['idta'], 'reportidta': row['reportidta'], 'retransmit': False }) for row2 in botslib.query( '''SELECT idta FROM ta WHERE parent=%(parent)s AND status=%(status)s''', { 'parent': row['idta'], 'status': RAWIN }): ta_rereceive = botslib.OldTransaction(row2['idta']) ta_externin = ta_rereceive.copyta( status=EXTERNIN, statust=DONE, parent=0 ) #inject; status is DONE so this ta is not used further ta_raw = ta_externin.copyta( status=RAWIN, statust=OK) #reinjected file is ready as new input #for resend; this one is slow. Can be improved by having a separate list of idta to resend for row in botslib.query( '''SELECT idta,parent FROM ta WHERE retransmit=%(retransmit)s AND status=%(status)s''', { 'retransmit': True, 'status': EXTERNOUT }): retransmit = True ta_outgoing = botslib.OldTransaction(row['idta']) ta_outgoing.update( retransmit=False) #is reinjected; set retransmit back to False ta_resend = botslib.OldTransaction( row['parent'] ) #parent ta with status RAWOUT; this is where the outgoing file is kept ta_externin = ta_resend.copyta( status=EXTERNIN, statust=DONE, parent=0) #inject; status is DONE so this ta is not used further ta_raw = ta_externin.copyta( status=RAWOUT, statust=OK) #reinjected file is ready as new input return retransmit
def finish_evaluation(stuff2evaluate, resultlast, evaluate_type): #count nr files send for row in botslib.query( '''SELECT COUNT(*) as count FROM ta WHERE idta > %(rootidta)s AND status=%(status)s AND statust=%(statust)s ''', { 'status': EXTERNOUT, 'rootidta': stuff2evaluate, 'statust': DONE }): send = row['count'] #count process errors for row in botslib.query( '''SELECT COUNT(*) as count FROM ta WHERE idta >= %(rootidta)s AND status=%(status)s AND statust=%(statust)s''', { 'status': PROCESS, 'rootidta': stuff2evaluate, 'statust': ERROR }): processerrors = row['count'] #generate report (in database) rootta = botslib.OldTransaction(stuff2evaluate) rootta.syn('ts') #get the timestamp of this run lastreceived = resultlast[DONE] + resultlast[OK] + resultlast[ OPEN] + resultlast[ERROR] status = bool(resultlast[OK] + resultlast[OPEN] + resultlast[ERROR] + processerrors) botslib.change( u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,status,type) VALUES (%(idta)s, %(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s, %(send)s,%(processerrors)s,%(ts)s,%(lastreceived)s,%(status)s,%(type)s) ''', { 'idta': stuff2evaluate, 'lastopen': resultlast[OPEN], 'lasterror': resultlast[ERROR], 'lastok': resultlast[OK], 'lastdone': resultlast[DONE], 'send': send, 'processerrors': processerrors, 'ts': rootta.ts, 'lastreceived': lastreceived, 'status': status, 'type': evaluate_type[2:] }) return generate_report( stuff2evaluate) #return report status: 0 (no error) or 1 (error)
def _buildevaluationstructure(self, tacurrent): """ recursive,for each db-ta: - fill global talist with the children (and children of children, etc) """ # gather next steps/ta's for tacurrent; if tacurrent.child: # find successor by using child relation ship for row in botslib.query( """SELECT """ + TAVARS + """ FROM ta WHERE idta=%(child)s""", {"child": tacurrent.child}, ): realdict = dict([(key, row[key]) for key in row.keys()]) tacurrent.talijst = [botslib.OldTransaction(**realdict)] else: # find successor by using parent-relationship; mostly this relation except for merge operations talijst = [] for row in botslib.query( """SELECT """ + TAVARS + """ FROM ta WHERE idta > %(currentidta)s AND parent=%(currentidta)s """, # adding the idta > %(parent)s to selection speeds up a lot. {"currentidta": tacurrent.idta}, ): realdict = dict([(key, row[key]) for key in row.keys()]) talijst.append(botslib.OldTransaction(**realdict)) # filter: # one ta might have multiple children; 2 possible reasons for that: # 1. split up # 2. error is processing the file; and retried # Here case 2 (error/retry) is filtered; it is not interesting to evaluate the older errors! # So: if the same filename and different script: use newest idta # shortcut: when an error occurs in a split all is turned back. # so: split up is OK as a whole or because of retries. # so: if split, and different scripts: split is becaue of retries: use newest idta. # ~ print tacurrent.talijst if len(talijst) > 1 and talijst[0].script != talijst[1].script: # find higest idta highest_ta = talijst[0] for ta_object in talijst[1:]: if ta_object.idta > highest_ta.idta: highest_ta = ta_object tacurrent.talijst = [highest_ta] else: tacurrent.talijst = talijst # recursive build: for child in tacurrent.talijst: self._buildevaluationstructure(child)
def finish_evaluation(stuff2evaluate, resultlast, evaluate_type): # count nr files send for row in botslib.query( """SELECT COUNT(*) as count FROM ta WHERE idta > %(rootidta)s AND status=%(status)s AND statust=%(statust)s """, {"status": EXTERNOUT, "rootidta": stuff2evaluate, "statust": DONE}, ): send = row["count"] # count process errors for row in botslib.query( """SELECT COUNT(*) as count FROM ta WHERE idta >= %(rootidta)s AND status=%(status)s AND statust=%(statust)s""", {"status": PROCESS, "rootidta": stuff2evaluate, "statust": ERROR}, ): processerrors = row["count"] # generate report (in database) rootta = botslib.OldTransaction(stuff2evaluate) rootta.syn("ts") # get the timestamp of this run lastreceived = resultlast[DONE] + resultlast[OK] + resultlast[OPEN] + resultlast[ERROR] status = bool(resultlast[OK] + resultlast[OPEN] + resultlast[ERROR] + processerrors) botslib.change( u"""INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,status,type) VALUES (%(idta)s, %(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s, %(send)s,%(processerrors)s,%(ts)s,%(lastreceived)s,%(status)s,%(type)s) """, { "idta": stuff2evaluate, "lastopen": resultlast[OPEN], "lasterror": resultlast[ERROR], "lastok": resultlast[OK], "lastdone": resultlast[DONE], "send": send, "processerrors": processerrors, "ts": rootta.ts, "lastreceived": lastreceived, "status": status, "type": evaluate_type[2:], }, ) return generate_report(stuff2evaluate) # return report status: 0 (no error) or 1 (error)
def evaluaterun(evaluate_type, stuff2evaluate): ''' traces all received files. Write a filereport for each file, and writes a report for the run. ''' resultlast = { OPEN: 0, ERROR: 0, OK: 0, DONE: 0 } #gather results of all filereports for runreport #look at infiles from this run; trace them to determine their tracestatus. for tadict in botslib.query( '''SELECT ''' + TAVARS + ''' FROM ta WHERE idta > %(rootidta)s AND status=%(status)s ''', { 'status': EXTERNIN, 'rootidta': stuff2evaluate }): botsglobal.logger.debug(u'evaluate %s.', tadict['idta']) mytrace = Trace(tadict, stuff2evaluate) resultlast[mytrace.statusttree] += 1 insert_filereport(mytrace) del mytrace.ta_object del mytrace return finish_evaluation(stuff2evaluate, resultlast, evaluate_type)
def _cleanprocessnothingreceived(): ''' delete all --new runs that received no files; including all process under the run processes are organised as trees, so recursive. ''' def core(idta): #select db-ta's referring to this db-ta for row in botslib.query('''SELECT idta FROM ta WHERE idta > %(idta)s AND script=%(idta)s''', {'idta':idta}): core(row['idta']) ta=botslib.OldTransaction(idta) ta.delete() return #select root-processes older than hoursnotrefferedarekept vanaf = datetime.datetime.today() - datetime.timedelta(hours=botsglobal.ini.getint('settings','hoursrunwithoutresultiskept',1)) for row in botslib.query('''SELECT idta FROM report WHERE type = 'new' AND lastreceived=0 AND ts < %(vanaf)s''', {'vanaf':vanaf}): core(row['idta']) #delete report botslib.change('''DELETE FROM report WHERE idta=%(idta)s ''',{'idta':row['idta']})
def partnerlookup(value,field,field_where_value_is_searched='idpartner',safe=False): ''' lookup via table partner. lookup value is returned, exception if not there. when using 'field_where_value_is_searched' with other values as ='idpartner', partner tabel is only indexed on idpartner (so uniqueness is not guaranteerd). should work OK if not too many partners. parameter safe can be: - True: if not found, return value - False: if not found throw exception - None: if not found, return None ''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM partner WHERE '''+field_where_value_is_searched+ ''' = %(value)s ''',{'value':value}): if row[field]: return row[field] #nothing found in partner table if safe is None: return None elif safe: #if safe is True return value else: raise botslib.CodeConversionError(_(u'No result found for partner lookup; either partner "%(idpartner)s" does not exist or field "%(field)s" has no value.'), {'idpartner':value,'field':field})
def generate_report(stuff2evaluate): for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidta)s''', {'rootidta':stuff2evaluate}): break else: raise botslib.PanicError(_(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]} reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]} reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n')%(results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors')%(results['lasterror']) reporttext += _(u' %d files with errors,\n')%(results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck')%(results['lastok']) reporttext += _(u' %d files got stuck,\n')%(results['lastok']) if results['lastopen']: subject += _(u'; %d system errors')%(results['lastopen']) reporttext += _(u' %d system errors,\n')%(results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors')%(results['processerrors']) reporttext += _(u' %d errors in processes.\n')%(results['processerrors']) reporttext += _(u' %d files send in run.\n')%(results['send']) botsglobal.logger.info(reporttext) # sendreportifprocesserror allows blocking of email reports for process errors if (results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True))): botslib.sendbotserrorreport(subject,reporttext) return int(results['status']) #return report status: 0 (no error) or 1 (error)
def prepareautomaticrecommunication(): ''' reinjects all files for which communication failed (status = RAWOUT) ''' retransmit = False #indicate retransmit #bots keeps track of last time automaticretrycommunication was done; reason is mainly performance startidta = max( botslib.keeptrackoflastretry('bots__automaticretrycommunication', botslib.getlastrun()), botslib.get_idta_last_error()) #reinject for row4 in botslib.query( '''SELECT idta FROM ta WHERE idta>%(startidta)s AND status=%(status)s AND statust=%(statust)s ''', { 'statust': OK, 'status': RAWOUT, 'startidta': startidta }): retransmit = True ta_outgoing = botslib.OldTransaction(row4['idta']) ta_outgoing_copy = ta_outgoing.copyta(status=RAWOUT, statust=OK) ta_outgoing.update(statust=DONE) return retransmit
def postprocess(routedict,function,status=FILEOUT,rootidta=None,**argv): ''' for postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - zip files. If errors occur during processing, no ta are left with status FILEOUT ! postprocess is called right before the out-communicatiation ''' if rootidta is None: rootidta = botsglobal.currentrun.get_minta4query_routepart() nr_files = 0 for row in botslib.query(u'''SELECT idta,filename FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND tochannel=%(tochannel)s ''', {'status':status,'statust':OK,'idroute':routedict['idroute'],'tochannel':routedict['tochannel'],'rootidta':rootidta}): try: botsglobal.logger.debug(u'Start postprocessing "%(name)s" for file "%(filename)s".', {'name':function.__name__,'filename':row['filename']}) ta_from = botslib.OldTransaction(row['idta']) ta_from.filename = row['filename'] function(ta_from=ta_from,endstatus=status,routedict=routedict,**argv) except: txt = botslib.txtexc() ta_from.update(statust=ERROR,errortext=txt) ta_from.deletechildren() else: botsglobal.logger.debug(u'OK postprocessing "%(name)s" for file "%(filename)s".', {'name':function.__name__,'filename':row['filename']}) ta_from.update(statust=DONE) nr_files += 1 return nr_files
def _cleanprocessnothingreceived(): ''' delete all --new runs that recieved no files; including all process under the run processes are organised as trees, so recursive. ''' def core(idta): #select db-ta's referring to this db-ta for row in botslib.query( '''SELECT idta FROM ta WHERE idta > %(idta)s AND script=%(idta)s''', {'idta': idta}): core(row['idta']) ta = botslib.OldTransaction(idta) ta.delete() return #select root-processes older than hoursnotrefferedarekept vanaf = datetime.datetime.today() - datetime.timedelta( hours=botsglobal.ini.getint('settings', 'hoursrunwithoutresultiskept', 1)) for row in botslib.query( '''SELECT idta FROM report WHERE type = 'new' AND lastreceived=0 AND ts < %(vanaf)s''', {'vanaf': vanaf}): core(row['idta']) #delete report botslib.change('''DELETE FROM report WHERE idta=%(idta)s ''', {'idta': row['idta']})
def partnerlookup(value, field, field_where_value_is_searched='idpartner', safe=False): ''' lookup via table partner. lookup value is returned, exception if not there. when using 'field_where_value_is_searched' with other values as ='idpartner', partner tabel is only indexed on idpartner (so uniqueness is not guaranteerd). should work OK if not too many partners. parameter safe can be: - True: if not found, return value - False: if not found throw exception - None: if not found, return None ''' for row in botslib.query( u'''SELECT ''' + field + ''' FROM partner WHERE ''' + field_where_value_is_searched + ''' = %(value)s ''', {'value': value}): if row[field]: return row[field] #nothing found in partner table if safe is None: return None elif safe: #if safe is True return value else: raise botslib.CodeConversionError( _(u'No result found for partner lookup; either partner "%(idpartner)s" does not exist or field "%(field)s" has no value.' ), { 'idpartner': value, 'field': field })
def sqlite_database_is_version3(): for row in botslib.query('''PRAGMA table_info(routes)'''): if row['name'] == 'translateind': if row['type'] == 'bool': return False else: return True raise Exception('Could determine version of database')
def sqlite_database_is_version3(): for row in botslib.query('''PRAGMA table_info(routes)'''): if row['name'] == 'translateind': if row['type'] == 'bool': return False else: return True raise Exception('Could not determine version of database')
def preprocess(routedict, function, status=FILEIN, **argv): ''' for pre- and postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - unzipping zipped files. - convert excel to csv - password protected files. Select files from INFILE -> SET_FOR_PROCESSING using criteria Than the actual processing function is called. The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN If errors occur during processing, no ta are left with status FILEIN ! preprocess is called right after the in-communicatiation ''' nr_files = 0 preprocessnumber = botslib.getpreprocessnumber() if not botslib.addinfo(change={'status': preprocessnumber}, where={ 'status': status, 'idroute': routedict['idroute'], 'fromchannel': routedict['fromchannel'] }): #check if there is something to do return 0 for row in botslib.query( u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND fromchannel=%(fromchannel)s ''', { 'status': preprocessnumber, 'statust': OK, 'idroute': routedict['idroute'], 'fromchannel': routedict['fromchannel'], 'rootidta': botslib.get_minta4query() }): try: botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".', function.__name__, row['filename']) ta_set_for_processing = botslib.OldTransaction(row['idta']) ta_processing = ta_set_for_processing.copyta( status=preprocessnumber + 1) ta_processing.filename = row['filename'] function(ta_from=ta_processing, endstatus=status, routedict=routedict, **argv) except: txt = botslib.txtexc() ta_processing.failure() ta_processing.update(statust=ERROR, errortext=txt) else: botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".', function.__name__, row['filename']) ta_set_for_processing.update(statust=DONE) ta_processing.update(statust=DONE) nr_files += 1 return nr_files
def _buildevaluationstructure(self, tacurrent): ''' recursive,for each db-ta: - fill global talist with the children (and children of children, etc) ''' #gather next steps/ta's for tacurrent; if tacurrent.child: #find successor by using child relation ship for row in botslib.query( '''SELECT ''' + TAVARS + ''' FROM ta WHERE idta=%(child)s''', {'child': tacurrent.child}): realdict = dict([(key, row[key]) for key in row.keys()]) tacurrent.talijst = [botslib.OldTransaction(**realdict)] else: #find successor by using parent-relationship; mostly this relation except for merge operations talijst = [] for row in botslib.query( '''SELECT ''' + TAVARS + ''' FROM ta WHERE idta > %(currentidta)s AND parent=%(currentidta)s ''', #adding the idta > %(parent)s to selection speeds up a lot. {'currentidta': tacurrent.idta}): realdict = dict([(key, row[key]) for key in row.keys()]) talijst.append(botslib.OldTransaction(**realdict)) #filter: #one ta might have multiple children; 2 possible reasons for that: #1. split up #2. error is processing the file; and retried #Here case 2 (error/retry) is filtered; it is not interesting to evaluate the older errors! #So: if the same filename and different script: use newest idta #shortcut: when an error occurs in a split all is turned back. #so: split up is OK as a whole or because of retries. #so: if split, and different scripts: split is becaue of retries: use newest idta. #~ print tacurrent.talijst if len(talijst) > 1 and talijst[0].script != talijst[1].script: #find higest idta highest_ta = talijst[0] for ta_object in talijst[1:]: if ta_object.idta > highest_ta.idta: highest_ta = ta_object tacurrent.talijst = [highest_ta] else: tacurrent.talijst = talijst #recursive build: for child in tacurrent.talijst: self._buildevaluationstructure(child)
def routedispatcher(routestorun, type=None): ''' run all route(s). ''' if type == '--retransmit': if not prepareretransmit(): return 0 elif type == '--retrycommunication': if not preparerecommunication(): return 0 elif type == '--automaticretrycommunication': if not prepareautomaticrecommunication(): return 0 elif type == '--retry': if not prepareretry(): return 0 stuff2evaluate = botslib.getlastrun() botslib.set_minta4query() for route in routestorun: foundroute = False botslib.setpreprocessnumber(SET_FOR_PROCESSING) for routedict in botslib.query( '''SELECT idroute , fromchannel_id as fromchannel, tochannel_id as tochannel, fromeditype, frommessagetype, alt, frompartner_id as frompartner, topartner_id as topartner, toeditype, tomessagetype, seq, frompartner_tochannel_id, topartner_tochannel_id, testindicator, translateind, defer FROM routes WHERE idroute=%(idroute)s AND active=%(active)s ORDER BY seq''', { 'idroute': route, 'active': True }): botsglobal.logger.info(_(u'running route %(idroute)s %(seq)s'), { 'idroute': routedict['idroute'], 'seq': routedict['seq'] }) botslib.setrouteid(routedict['idroute']) foundroute = True router(routedict) botslib.setrouteid('') botsglobal.logger.debug(u'finished route %s %s', routedict['idroute'], routedict['seq']) if not foundroute: botsglobal.logger.warning(_(u'there is no (active) route "%s".'), route) return stuff2evaluate
def _cleanarchive(): ''' delete all archive directories older than maxdaysarchive days.''' vanaf = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d') for row in botslib.query('''SELECT archivepath FROM channel '''): if row['archivepath']: vanafdir = botslib.join(row['archivepath'],vanaf) for dir in glob.glob(botslib.join(row['archivepath'],'*')): if dir < vanafdir: shutil.rmtree(dir,ignore_errors=True)
def persist_lookup(domein,botskey): ''' lookup persistent values in db. ''' for row in botslib.query(u'''SELECT content FROM persist WHERE domein=%(domein)s AND botskey=%(botskey)s''', {'domein':domein,'botskey':botskey}): return pickle.loads(str(row['content'])) return None
def getcodeset(ccodeid,leftcode,field='rightcode'): ''' Get a code set ''' return list(botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND leftcode = %(leftcode)s''', {'ccodeid':ccodeid, 'leftcode':leftcode, }))
def reverse_ccode(ccodeid,rightcode,field='leftcode'): ''' as ccode but reversed lookup.''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND rightcode = %(rightcode)s''', {'ccodeid':ccodeid, 'rightcode':rightcode, }): return row[field] raise botslib.CodeConversionError(_(u'Value "$value" not in code-conversion, user table "$table".'),value=rightcode,table=ccodeid)
def _cleantransactions(): vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30)) lijst = list( botslib.query('''SELECT idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf})) for rootta in lijst: botslib.change('''DELETE FROM filereport WHERE reportidta = %(rootta)s''',{'rootta':rootta['idta']}) botslib.change('''DELETE FROM report WHERE idta = %(rootta)s''',{'rootta':rootta['idta']}) #~ botslib.change('''DELETE FROM filereport WHERE idta = %(rootta)s''',{'rootta':rootta['idta']}) for index in range(1,len(lijst)): botslib.change('''DELETE FROM ta WHERE idta >= %(minrootta)s AND idta < %(maxrootta)s''', {'minrootta':lijst[index-1]['idta'],'maxrootta':lijst[index]['idta']})
def rcodetconversion(ccodeid,rightcode,field='leftcode'): ''' as codetconversion but reverses the dictionary first''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND rightcode = %(rightcode)s''', {'ccodeid':ccodeid, 'rightcode':rightcode, }): return row[field] raise botslib.CodeConversionError(_(u'Value "$value" not in codetconversions, user table "$table".'),value=rightcode,tabel=ccodeid)
def core(idta): #select db-ta's referring to this db-ta for row in botslib.query('''SELECT idta FROM ta WHERE idta > %(idta)s AND script=%(idta)s''', {'idta':idta}): core(row['idta']) ta=botslib.OldTransaction(idta) ta.delete() return
def safercodetconversion(ccodeid,rightcode,field='leftcode'): ''' as codetconversion but reverses the dictionary first''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND rightcode = %(rightcode)s''', {'ccodeid':ccodeid, 'rightcode':rightcode, }): return row[field] return rightcode
def getcodeset(ccodeid,leftcode,field='rightcode'): ''' Returns a list of all 'field' values in ccode with right ccodeid and leftcode. ''' terug = [] for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND leftcode = %(leftcode)s''', {'ccodeid':ccodeid,'leftcode':leftcode}): terug.append(row[field]) return terug
def _cleanarchive(): ''' delete all archive directories older than maxdaysarchive days.''' vanaf = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d') for row in botslib.query('''SELECT archivepath FROM channel WHERE archivepath != '' '''): vanafdir = botslib.join(row['archivepath'],vanaf) for entry in glob.iglob(botslib.join(row['archivepath'],'*')): if entry < vanafdir: if entry.endswith('.zip'): os.remove(entry) else: shutil.rmtree(entry,ignore_errors=True)
def get_minta4query_route(self): ''' find out where route was started. if not started in crashed run, value for recovery run will be found. ''' for row in botslib.query('''SELECT MIN(idta) as route_idta FROM ta WHERE idta > %(rootidta_of_current_run)s AND script = %(rootidta_of_current_run)s AND idroute = %(idroute)s ''', {'rootidta_of_current_run':self.get_minta4query(),'idroute':botslib.getrouteid()}): return row['route_idta']
def core(idta): #select db-ta's referring to this db-ta for row in botslib.query( '''SELECT idta FROM ta WHERE idta > %(idta)s AND script=%(idta)s''', {'idta': idta}): core(row['idta']) ta = botslib.OldTransaction(idta) ta.delete() return
def run(self): #get rootidta of crashed run for row in botslib.query('''SELECT MAX(idta) as crashed_idta FROM ta WHERE idta < %(rootidta_of_current_run)s AND script = 0 ''', {'rootidta_of_current_run':self.minta4query}): self.minta4query_crash = row['crashed_idta'] if not self.minta4query_crash: return False #no run rootofcrashedrun = botslib.OldTransaction(self.minta4query_crash) rootofcrashedrun.update(statust=DONE) #clean up things from crash ********************************** #delete run report botslib.changeq('''DELETE FROM report WHERE idta = %(rootofcrashedrun)s''',{'rootofcrashedrun':rootofcrashedrun.idta}) #delete file reports botslib.changeq('''DELETE FROM filereport WHERE idta>%(rootofcrashedrun)s''',{'rootofcrashedrun':rootofcrashedrun.idta}) #delete ta's after ERROR and OK for crashed merges mergedidtatodelete = set() for row in botslib.query('''SELECT child FROM ta WHERE idta > %(rootofcrashedrun)s AND statust = %(statust)s AND status != %(status)s AND child != 0 ''', {'rootofcrashedrun':rootofcrashedrun.idta,'status':PROCESS,'statust':OK}): mergedidtatodelete.add(row['child']) for idta in mergedidtatodelete: ta_object = botslib.OldTransaction(idta) ta_object.delete() #delete ta's after ERROR and OK for other for row in botslib.query('''SELECT idta FROM ta WHERE idta > %(rootofcrashedrun)s AND ( statust = %(statust1)s OR statust = %(statust2)s ) AND status != %(status)s AND child = 0 ''', {'rootofcrashedrun':rootofcrashedrun.idta,'status':PROCESS,'statust1':OK,'statust2':ERROR}): ta_object = botslib.OldTransaction(row['idta']) ta_object.deletechildren() return super(crashrecovery, self).run()
def preparerecommunication(): #for each out-communication process that went wrong: retransmit = False #indicate retransmit for row in botslib.query('''SELECT idta,tochannel FROM ta WHERE statust!=%(statust)s AND status=%(status)s AND retransmit=%(retransmit)s ''', {'status':PROCESS,'retransmit':True,'statust':DONE}): run_outgoing = botslib.OldTransaction(row['idta']) run_outgoing.update(retransmit=False) #set retransmit back to False #get rootidta of run where communication failed for row2 in botslib.query('''SELECT max(idta) as rootidta FROM ta WHERE script=%(script)s AND idta<%(thisidta)s ''', {'script':0,'thisidta':row['idta']}): rootidta = row2['rootidta'] #get endidta of run where communication failed for row3 in botslib.query('''SELECT min(idta) as endidta FROM ta WHERE script=%(script)s AND idta>%(thisidta)s ''', {'script':0,'thisidta':row['idta']}): endidta = row3['endidta'] if not endidta: endidta = sys.maxint - 1 #reinject for row4 in botslib.query('''SELECT idta FROM ta WHERE idta<%(endidta)s AND idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND tochannel=%(tochannel)s ''', {'statust':OK,'status':RAWOUT,'rootidta':rootidta,'endidta':endidta,'tochannel':row['tochannel']}): retransmit = True ta_outgoing = botslib.OldTransaction(row4['idta']) ta_outgoing_copy = ta_outgoing.copyta(status=RAWOUT,statust=OK) ta_outgoing.update(statust=DONE) return retransmit
def _cleantransactions(): ''' delete records from report, filereport and ta. best indexes are on idta/reportidta; this should go fast. ''' vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30)) for row in botslib.query('''SELECT MAX(idta) as max_idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf}): maxidta = row['max_idta'] if maxidta is None: #if there is no maxidta to delete, do nothing return botslib.changeq('''DELETE FROM report WHERE idta < %(maxidta)s''',{'maxidta':maxidta}) botslib.changeq('''DELETE FROM filereport WHERE idta < %(maxidta)s''',{'maxidta':maxidta}) botslib.changeq('''DELETE FROM ta WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
def persist_lookup(domein, botskey): """ lookup persistent values in db. """ for row in botslib.query( u"""SELECT content FROM persist WHERE domein=%(domein)s AND botskey=%(botskey)s""", {"domein": domein, "botskey": botskey}, ): return pickle.loads(str(row["content"])) return None
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize): #count nr files send for row in botslib.query('''SELECT COUNT(*) as count FROM ta WHERE idta > %(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s ''', {'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}): send = row['count'] #count process errors for row in botslib.query('''SELECT COUNT(*) as count FROM ta WHERE idta >= %(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s''', {'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}): processerrors = row['count'] #generate report (in database) rootta = botslib.OldTransaction(rootidtaofrun) rootta.syn('ts') #get the timestamp of this run lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR] status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors) #give information about the used command line parameters for each run. Problem is that there is only 35pos for this (in MySQL, PostgreSQL). #~ commandline = if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': commandline = ' '.join(sys.argv) else: commandline = ' '.join([arg for arg in sys.argv[1:] if arg!='-cconfig' and not arg.startswith('--')])[:35] botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors, ts,lastreceived,status,type,filesize,acceptance,rsrv1) VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s, %(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s,%(rsrv1)s) ''', {'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK], 'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived, 'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False)), 'rsrv1':commandline}) #20120830: if new run with nothing received and no process errors: delete ta's. if command == 'new' and not lastreceived and not processerrors: botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def codetconversion(ccodeid,leftcode,field='rightcode'): ''' converts code using a db-table. converted value is returned. ''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND leftcode = %(leftcode)s''', {'ccodeid':ccodeid, 'leftcode':leftcode, }): return row[field] raise botslib.CodeConversionError(_(u'Value "$value" not in codetconversions, user table "$table".'),value=leftcode,tabel=ccodeid)
def safecodetconversion(ccodeid,leftcode,field='rightcode'): ''' converts code using a db-table. converted value is returned. ''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND leftcode = %(leftcode)s''', {'ccodeid':ccodeid, 'leftcode':leftcode, }): return row[field] return leftcode
def reverse_ccode(ccodeid,rightcode,field='leftcode',safe=False): ''' as ccode but reversed lookup.''' for row in botslib.query(u'''SELECT ''' +field+ ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND rightcode = %(rightcode)s''', {'ccodeid':ccodeid,'rightcode':rightcode}): return row[field] if safe: return rightcode else: raise botslib.CodeConversionError(_(u'Value "%(value)s" not in code-conversion, user table "%(table)s".'), {'value':rightcode,'table':ccodeid})
def getcodeset(ccodeid, leftcode, field='rightcode'): ''' Returns a list of all 'field' values in ccode with right ccodeid and leftcode. ''' terug = [] for row in botslib.query( u'''SELECT ''' + field + ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND leftcode = %(leftcode)s''', { 'ccodeid': ccodeid, 'leftcode': leftcode }): terug.append(row[field]) return terug
def reverse_ccode(ccodeid, rightcode, field='leftcode'): ''' as ccode but reversed lookup.''' for row in botslib.query( u'''SELECT ''' + field + ''' FROM ccode WHERE ccodeid_id = %(ccodeid)s AND rightcode = %(rightcode)s''', { 'ccodeid': ccodeid, 'rightcode': rightcode, }): return row[field] raise botslib.CodeConversionError( _(u'Value "$value" not in code-conversion, user table "$table".'), value=rightcode, table=ccodeid)
def evaluateretryrun(evaluate_type, stuff2evaluate): resultlast = {OPEN: 0, ERROR: 0, OK: 0, DONE: 0} didretry = False for row in botslib.query( '''SELECT idta FROM filereport GROUP BY idta HAVING MAX(statust) != %(statust)s''', {'statust': DONE}): didretry = True for tadict in botslib.query( '''SELECT ''' + TAVARS + ''' FROM ta WHERE idta= %(idta)s ''', {'idta': row['idta']}): break else: #there really should be a corresponding ta raise botslib.PanicError( _(u'MaintenanceRetry: could not find transaction "$txt".'), txt=row['idta']) mytrace = Trace(tadict, stuff2evaluate) resultlast[mytrace.statusttree] += 1 if mytrace.statusttree == DONE: mytrace.errortext = '' #~ mytrace.ta.update(tracestatus=mytrace.statusttree) #ts for retried filereports is tricky: is this the time the file was originally received? best would be to use ts of prepare... #that is quite difficult, so use time of this run rootta = botslib.OldTransaction(stuff2evaluate) rootta.syn('ts') #get the timestamp of this run mytrace.ts = rootta.ts insert_filereport(mytrace) del mytrace.ta_object del mytrace if not didretry: return 0 #no error return finish_evaluation(stuff2evaluate, resultlast, evaluate_type)