def evaluateretryrun(type,stuff2evaluate):
resultlast={OPEN:0,ERROR:0,OK:0,DONE:0}
didretry = False
for row in botslib.query('''SELECT idta
FROM filereport
GROUP BY idta
HAVING MAX(statust) != %(statust)s''',
{'statust':DONE}):
didretry = True
for tadict in botslib.query('''SELECT ''' + tavars + '''
FROM ta
WHERE idta= %(idta)s ''',
{'idta':row['idta']}):
break
else: #there really should be a corresponding ta
raise botslib.PanicError(_(u'MaintenanceRetry: could not find transaction "$txt".'),txt=row['idta'])
mytrace = Trace(tadict,stuff2evaluate)
resultlast[mytrace.statusttree]+=1
if mytrace.statusttree == DONE:
mytrace.errortext = ''
#~ mytrace.ta.update(tracestatus=mytrace.statusttree)
#ts for retried filereports is tricky: is this the time the file was originally received? best would be to use ts of prepare...
#that is quite difficult, so use time of this run
rootta=botslib.OldTransaction(stuff2evaluate)
rootta.syn('ts') #get the timestamp of this run
mytrace.ts = rootta.ts
insert_filereport(mytrace)
del mytrace.ta
del mytrace
if not didretry:
return 0 #no error
return finish_evaluation(stuff2evaluate,resultlast,type)
def run(self):
''' reinjects files with failed communication.
'''
#bots keeps track of last time automaticretrycommunication was done; reason: performance
idta_lastretry = botslib.unique('bots__automaticretrycommunication',updatewith=self.minta4query)
if idta_lastretry == 1:
#this is the first time automaticretrycommunication is run.
#do not do anything this run, in order to avoid sending older files.
return False #no run
for row in botslib.query('''SELECT MIN(idta) AS min_idta
FROM filereport
WHERE idta > %(idta_lastretry)s
AND statust = %(statust)s ''',
{'statust':ERROR,'idta_lastretry':idta_lastretry}):
startidta = row['min_idta']
if not startidta:
return False #no run
do_retransmit = False
for row in botslib.query('''SELECT idta,parent,numberofresends
FROM ta
WHERE idta > %(startidta)s
AND status = %(status)s
AND statust = %(statust)s ''',
{'statust':ERROR,'status':EXTERNOUT,'startidta':startidta}):
do_retransmit = True
ta_outgoing = botslib.OldTransaction(row['idta'])
ta_outgoing.update(retransmit=False,statust=RESEND) #set retransmit back to False
ta_resend = botslib.OldTransaction(row['parent']) #parent ta with status RAWOUT; this is where the outgoing file is kept
ta_externin = ta_resend.copyta(status=EXTERNIN,statust=DONE) #inject; status is DONE so this ta is not used further
ta_externin.copyta(status=FILEOUT,statust=OK,numberofresends=row['numberofresends']) #reinjected file is ready as new input
if do_retransmit:
return super(automaticretrycommunication, self).run()
else:
return False #no run
def finish_evaluation(stuff2evaluate,resultlast,type):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidta)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidta':stuff2evaluate,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidta)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidta':stuff2evaluate,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta=botslib.OldTransaction(stuff2evaluate)
rootta.syn('ts') #get the timestamp of this run
LastReceived=resultlast[DONE]+resultlast[OK]+resultlast[OPEN]+resultlast[ERROR]
status = bool(resultlast[OK]+resultlast[OPEN]+resultlast[ERROR]+processerrors)
botslib.change(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,status,type)
VALUES (%(idta)s,
%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,
%(send)s,%(processerrors)s,%(ts)s,%(lastreceived)s,%(status)s,%(type)s)
''',
{'idta':stuff2evaluate,
'lastopen':resultlast[OPEN],'lasterror':resultlast[ERROR],'lastok':resultlast[OK],'lastdone':resultlast[DONE],
'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':LastReceived,'status':status,'type':type[2:]})
return generate_report(stuff2evaluate) #return report status: 0 (no error) or 1 (error)
def run(self):
''' prepare the files indicated by user to be rereceived. Return: indication if files should be rereceived.
'''
do_retransmit = False
for row in botslib.query('''SELECT idta
FROM filereport
WHERE retransmit = %(retransmit)s ''',
{'retransmit':1}):
do_retransmit = True
botslib.changeq('''UPDATE filereport
SET retransmit = %(retransmit)s
WHERE idta = %(idta)s ''',
{'idta':row['idta'],'retransmit':0})
for row2 in botslib.query('''SELECT idta
FROM ta
WHERE parent = %(parent)s ''',
{'parent':row['idta']}):
ta_rereceive = botslib.OldTransaction(row2['idta'])
ta_externin = ta_rereceive.copyta(status=EXTERNIN,statust=DONE,parent=0) #inject; status is DONE so this ta is not used further
ta_externin.copyta(status=FILEIN,statust=OK) #reinjected file is ready as new input
if do_retransmit:
return super(rereceive, self).run()
else:
return False #no run
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(rootidtaofrun)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]
status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors)
botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors,
ts,lastreceived,status,type,filesize,acceptance)
VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s,
%(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s) ''',
{'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK],
'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived,
'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False))})
#20120830: if new run with nothing received and no process errors: delete ta's.
if command == 'new' and not lastreceived and not processerrors:
botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def email_error_report(rootidtaofrun):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidtaofrun)s''',
{'rootidtaofrun':rootidtaofrun}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':unicode(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':unicode(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext) #log the report texts
# only send email report if there are errors.
# sendreportifprocesserror (in bots.ini): no email reports if only process errors
if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):
# Include details about process errors in the email report; if debug is True: includes trace
if results['processerrors']:
for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
FROM ta
WHERE idta>=%(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
reporttext += '\nProcess error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
# Include details about file errors in the email report; if debug is True: includes trace
if results['lasterror'] or results['lastopen'] or results['lastok']:
for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
FROM filereport
WHERE idta>%(rootidtaofrun)s
AND statust!=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'statust':DONE}):
reporttext += '\nFile error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def email_error_report(rootidtaofrun):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidtaofrun)s''',
{'rootidtaofrun':rootidtaofrun}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext) #log the report texts
# only send email report if there are errors.
# sendreportifprocesserror (in bots.ini): no email reports if only process errors
if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):
# Include details about process errors in the email report; if debug is True: includes trace
if results['processerrors']:
for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
FROM ta
WHERE idta>=%(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
reporttext += '\nProcess error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
# Include details about file errors in the email report; if debug is True: includes trace
if results['lasterror'] or results['lastopen'] or results['lastok']:
for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
FROM filereport
WHERE idta>%(rootidtaofrun)s
AND statust!=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'statust':DONE}):
reporttext += '\nFile error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def preparerecommunication():
#for each out-communication process that went wrong:
retransmit = False #indicate retransmit
for row in botslib.query(
'''SELECT idta,tochannel
FROM ta
WHERE statust!=%(statust)s
AND status=%(status)s
AND retransmit=%(retransmit)s ''', {
'status': PROCESS,
'retransmit': True,
'statust': DONE
}):
run_outgoing = botslib.OldTransaction(row['idta'])
run_outgoing.update(retransmit=False) #set retransmit back to False
#get rootidta of run where communication failed
for row2 in botslib.query(
'''SELECT max(idta) as rootidta
FROM ta
WHERE script=%(script)s
AND idta<%(thisidta)s ''', {
'script': 0,
'thisidta': row['idta']
}):
rootidta = row2['rootidta']
#get endidta of run where communication failed
for row3 in botslib.query(
'''SELECT min(idta) as endidta
FROM ta
WHERE script=%(script)s
AND idta>%(thisidta)s ''', {
'script': 0,
'thisidta': row['idta']
}):
endidta = row3['endidta']
if not endidta:
endidta = sys.maxint - 1
#reinject
for row4 in botslib.query(
'''SELECT idta
FROM ta
WHERE idta<%(endidta)s
AND idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND tochannel=%(tochannel)s ''', {
'statust': OK,
'status': RAWOUT,
'rootidta': rootidta,
'endidta': endidta,
'tochannel': row['tochannel']
}):
retransmit = True
ta_outgoing = botslib.OldTransaction(row4['idta'])
ta_outgoing_copy = ta_outgoing.copyta(status=RAWOUT, statust=OK)
ta_outgoing.update(statust=DONE)
return retransmit
def prepareretransmit():
''' prepare the retransmittable files. Return: indication if files should be retransmitted.'''
retransmit = False #indicate retransmit
#for rereceive
for row in botslib.query(
'''SELECT idta,reportidta
FROM filereport
WHERE retransmit=%(retransmit)s ''',
{'retransmit': True}):
retransmit = True
botslib.change(
'''UPDATE filereport
SET retransmit=%(retransmit)s
WHERE idta=%(idta)s
AND reportidta=%(reportidta)s ''', {
'idta': row['idta'],
'reportidta': row['reportidta'],
'retransmit': False
})
for row2 in botslib.query(
'''SELECT idta
FROM ta
WHERE parent=%(parent)s
AND status=%(status)s''', {
'parent': row['idta'],
'status': RAWIN
}):
ta_rereceive = botslib.OldTransaction(row2['idta'])
ta_externin = ta_rereceive.copyta(
status=EXTERNIN, statust=DONE, parent=0
) #inject; status is DONE so this ta is not used further
ta_raw = ta_externin.copyta(
status=RAWIN,
statust=OK) #reinjected file is ready as new input
#for resend; this one is slow. Can be improved by having a separate list of idta to resend
for row in botslib.query(
'''SELECT idta,parent
FROM ta
WHERE retransmit=%(retransmit)s
AND status=%(status)s''', {
'retransmit': True,
'status': EXTERNOUT
}):
retransmit = True
ta_outgoing = botslib.OldTransaction(row['idta'])
ta_outgoing.update(
retransmit=False) #is reinjected; set retransmit back to False
ta_resend = botslib.OldTransaction(
row['parent']
) #parent ta with status RAWOUT; this is where the outgoing file is kept
ta_externin = ta_resend.copyta(
status=EXTERNIN, statust=DONE,
parent=0) #inject; status is DONE so this ta is not used further
ta_raw = ta_externin.copyta(
status=RAWOUT, statust=OK) #reinjected file is ready as new input
return retransmit
def finish_evaluation(stuff2evaluate, resultlast, evaluate_type):
#count nr files send
for row in botslib.query(
'''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidta)s
AND status=%(status)s
AND statust=%(statust)s ''', {
'status': EXTERNOUT,
'rootidta': stuff2evaluate,
'statust': DONE
}):
send = row['count']
#count process errors
for row in botslib.query(
'''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidta)s
AND status=%(status)s
AND statust=%(statust)s''', {
'status': PROCESS,
'rootidta': stuff2evaluate,
'statust': ERROR
}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(stuff2evaluate)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultlast[DONE] + resultlast[OK] + resultlast[
OPEN] + resultlast[ERROR]
status = bool(resultlast[OK] + resultlast[OPEN] + resultlast[ERROR] +
processerrors)
botslib.change(
u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,status,type)
VALUES (%(idta)s,
%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,
%(send)s,%(processerrors)s,%(ts)s,%(lastreceived)s,%(status)s,%(type)s)
''', {
'idta': stuff2evaluate,
'lastopen': resultlast[OPEN],
'lasterror': resultlast[ERROR],
'lastok': resultlast[OK],
'lastdone': resultlast[DONE],
'send': send,
'processerrors': processerrors,
'ts': rootta.ts,
'lastreceived': lastreceived,
'status': status,
'type': evaluate_type[2:]
})
return generate_report(
stuff2evaluate) #return report status: 0 (no error) or 1 (error)
def _buildevaluationstructure(self, tacurrent):
""" recursive,for each db-ta:
- fill global talist with the children (and children of children, etc)
"""
# gather next steps/ta's for tacurrent;
if tacurrent.child: # find successor by using child relation ship
for row in botslib.query(
"""SELECT """
+ TAVARS
+ """
FROM ta
WHERE idta=%(child)s""",
{"child": tacurrent.child},
):
realdict = dict([(key, row[key]) for key in row.keys()])
tacurrent.talijst = [botslib.OldTransaction(**realdict)]
else: # find successor by using parent-relationship; mostly this relation except for merge operations
talijst = []
for row in botslib.query(
"""SELECT """
+ TAVARS
+ """
FROM ta
WHERE idta > %(currentidta)s
AND parent=%(currentidta)s """, # adding the idta > %(parent)s to selection speeds up a lot.
{"currentidta": tacurrent.idta},
):
realdict = dict([(key, row[key]) for key in row.keys()])
talijst.append(botslib.OldTransaction(**realdict))
# filter:
# one ta might have multiple children; 2 possible reasons for that:
# 1. split up
# 2. error is processing the file; and retried
# Here case 2 (error/retry) is filtered; it is not interesting to evaluate the older errors!
# So: if the same filename and different script: use newest idta
# shortcut: when an error occurs in a split all is turned back.
# so: split up is OK as a whole or because of retries.
# so: if split, and different scripts: split is becaue of retries: use newest idta.
# ~ print tacurrent.talijst
if len(talijst) > 1 and talijst[0].script != talijst[1].script:
# find higest idta
highest_ta = talijst[0]
for ta_object in talijst[1:]:
if ta_object.idta > highest_ta.idta:
highest_ta = ta_object
tacurrent.talijst = [highest_ta]
else:
tacurrent.talijst = talijst
# recursive build:
for child in tacurrent.talijst:
self._buildevaluationstructure(child)
def finish_evaluation(stuff2evaluate, resultlast, evaluate_type):
# count nr files send
for row in botslib.query(
"""SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidta)s
AND status=%(status)s
AND statust=%(statust)s """,
{"status": EXTERNOUT, "rootidta": stuff2evaluate, "statust": DONE},
):
send = row["count"]
# count process errors
for row in botslib.query(
"""SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidta)s
AND status=%(status)s
AND statust=%(statust)s""",
{"status": PROCESS, "rootidta": stuff2evaluate, "statust": ERROR},
):
processerrors = row["count"]
# generate report (in database)
rootta = botslib.OldTransaction(stuff2evaluate)
rootta.syn("ts") # get the timestamp of this run
lastreceived = resultlast[DONE] + resultlast[OK] + resultlast[OPEN] + resultlast[ERROR]
status = bool(resultlast[OK] + resultlast[OPEN] + resultlast[ERROR] + processerrors)
botslib.change(
u"""INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,status,type)
VALUES (%(idta)s,
%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,
%(send)s,%(processerrors)s,%(ts)s,%(lastreceived)s,%(status)s,%(type)s)
""",
{
"idta": stuff2evaluate,
"lastopen": resultlast[OPEN],
"lasterror": resultlast[ERROR],
"lastok": resultlast[OK],
"lastdone": resultlast[DONE],
"send": send,
"processerrors": processerrors,
"ts": rootta.ts,
"lastreceived": lastreceived,
"status": status,
"type": evaluate_type[2:],
},
)
return generate_report(stuff2evaluate) # return report status: 0 (no error) or 1 (error)
def evaluaterun(evaluate_type, stuff2evaluate):
''' traces all received files.
Write a filereport for each file,
and writes a report for the run.
'''
resultlast = {
OPEN: 0,
ERROR: 0,
OK: 0,
DONE: 0
} #gather results of all filereports for runreport
#look at infiles from this run; trace them to determine their tracestatus.
for tadict in botslib.query(
'''SELECT ''' + TAVARS + '''
FROM ta
WHERE idta > %(rootidta)s
AND status=%(status)s ''', {
'status': EXTERNIN,
'rootidta': stuff2evaluate
}):
botsglobal.logger.debug(u'evaluate %s.', tadict['idta'])
mytrace = Trace(tadict, stuff2evaluate)
resultlast[mytrace.statusttree] += 1
insert_filereport(mytrace)
del mytrace.ta_object
del mytrace
return finish_evaluation(stuff2evaluate, resultlast, evaluate_type)
def _cleanprocessnothingreceived():
''' delete all --new runs that received no files; including all process under the run
processes are organised as trees, so recursive.
'''
def core(idta):
#select db-ta's referring to this db-ta
for row in botslib.query('''SELECT idta
FROM ta
WHERE idta > %(idta)s
AND script=%(idta)s''',
{'idta':idta}):
core(row['idta'])
ta=botslib.OldTransaction(idta)
ta.delete()
return
#select root-processes older than hoursnotrefferedarekept
vanaf = datetime.datetime.today() - datetime.timedelta(hours=botsglobal.ini.getint('settings','hoursrunwithoutresultiskept',1))
for row in botslib.query('''SELECT idta
FROM report
WHERE type = 'new'
AND lastreceived=0
AND ts < %(vanaf)s''',
{'vanaf':vanaf}):
core(row['idta'])
#delete report
botslib.change('''DELETE FROM report WHERE idta=%(idta)s ''',{'idta':row['idta']})
def partnerlookup(value,field,field_where_value_is_searched='idpartner',safe=False):
''' lookup via table partner.
lookup value is returned, exception if not there.
when using 'field_where_value_is_searched' with other values as ='idpartner',
partner tabel is only indexed on idpartner (so uniqueness is not guaranteerd).
should work OK if not too many partners.
parameter safe can be:
- True: if not found, return value
- False: if not found throw exception
- None: if not found, return None
'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM partner
WHERE '''+field_where_value_is_searched+ ''' = %(value)s
''',{'value':value}):
if row[field]:
return row[field]
#nothing found in partner table
if safe is None:
return None
elif safe: #if safe is True
return value
else:
raise botslib.CodeConversionError(_(u'No result found for partner lookup; either partner "%(idpartner)s" does not exist or field "%(field)s" has no value.'),
{'idpartner':value,'field':field})
def generate_report(stuff2evaluate):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidta)s''',
{'rootidta':stuff2evaluate}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext)
# sendreportifprocesserror allows blocking of email reports for process errors
if (results['lasterror'] or results['lastopen'] or results['lastok'] or
(results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True))):
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def prepareautomaticrecommunication():
''' reinjects all files for which communication failed (status = RAWOUT)
'''
retransmit = False #indicate retransmit
#bots keeps track of last time automaticretrycommunication was done; reason is mainly performance
startidta = max(
botslib.keeptrackoflastretry('bots__automaticretrycommunication',
botslib.getlastrun()),
botslib.get_idta_last_error())
#reinject
for row4 in botslib.query(
'''SELECT idta
FROM ta
WHERE idta>%(startidta)s
AND status=%(status)s
AND statust=%(statust)s ''', {
'statust': OK,
'status': RAWOUT,
'startidta': startidta
}):
retransmit = True
ta_outgoing = botslib.OldTransaction(row4['idta'])
ta_outgoing_copy = ta_outgoing.copyta(status=RAWOUT, statust=OK)
ta_outgoing.update(statust=DONE)
return retransmit
def postprocess(routedict,function,status=FILEOUT,rootidta=None,**argv):
''' for postprocessing of files.
these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
- zip files.
If errors occur during processing, no ta are left with status FILEOUT !
postprocess is called right before the out-communicatiation
'''
if rootidta is None:
rootidta = botsglobal.currentrun.get_minta4query_routepart()
nr_files = 0
for row in botslib.query(u'''SELECT idta,filename
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND tochannel=%(tochannel)s
''',
{'status':status,'statust':OK,'idroute':routedict['idroute'],'tochannel':routedict['tochannel'],'rootidta':rootidta}):
try:
botsglobal.logger.debug(u'Start postprocessing "%(name)s" for file "%(filename)s".',
{'name':function.__name__,'filename':row['filename']})
ta_from = botslib.OldTransaction(row['idta'])
ta_from.filename = row['filename']
function(ta_from=ta_from,endstatus=status,routedict=routedict,**argv)
except:
txt = botslib.txtexc()
ta_from.update(statust=ERROR,errortext=txt)
ta_from.deletechildren()
else:
botsglobal.logger.debug(u'OK postprocessing "%(name)s" for file "%(filename)s".',
{'name':function.__name__,'filename':row['filename']})
ta_from.update(statust=DONE)
nr_files += 1
return nr_files
def _cleanprocessnothingreceived():
''' delete all --new runs that recieved no files; including all process under the run
processes are organised as trees, so recursive.
'''
def core(idta):
#select db-ta's referring to this db-ta
for row in botslib.query(
'''SELECT idta
FROM ta
WHERE idta > %(idta)s
AND script=%(idta)s''', {'idta': idta}):
core(row['idta'])
ta = botslib.OldTransaction(idta)
ta.delete()
return
#select root-processes older than hoursnotrefferedarekept
vanaf = datetime.datetime.today() - datetime.timedelta(
hours=botsglobal.ini.getint('settings', 'hoursrunwithoutresultiskept',
1))
for row in botslib.query(
'''SELECT idta
FROM report
WHERE type = 'new'
AND lastreceived=0
AND ts < %(vanaf)s''', {'vanaf': vanaf}):
core(row['idta'])
#delete report
botslib.change('''DELETE FROM report WHERE idta=%(idta)s ''',
{'idta': row['idta']})
def partnerlookup(value,
field,
field_where_value_is_searched='idpartner',
safe=False):
''' lookup via table partner.
lookup value is returned, exception if not there.
when using 'field_where_value_is_searched' with other values as ='idpartner',
partner tabel is only indexed on idpartner (so uniqueness is not guaranteerd).
should work OK if not too many partners.
parameter safe can be:
- True: if not found, return value
- False: if not found throw exception
- None: if not found, return None
'''
for row in botslib.query(
u'''SELECT ''' + field + '''
FROM partner
WHERE ''' + field_where_value_is_searched +
''' = %(value)s
''', {'value': value}):
if row[field]:
return row[field]
#nothing found in partner table
if safe is None:
return None
elif safe: #if safe is True
return value
else:
raise botslib.CodeConversionError(
_(u'No result found for partner lookup; either partner "%(idpartner)s" does not exist or field "%(field)s" has no value.'
), {
'idpartner': value,
'field': field
})
def sqlite_database_is_version3():
for row in botslib.query('''PRAGMA table_info(routes)'''):
if row['name'] == 'translateind':
if row['type'] == 'bool':
return False
else:
return True
raise Exception('Could determine version of database')
def sqlite_database_is_version3():
for row in botslib.query('''PRAGMA table_info(routes)'''):
if row['name'] == 'translateind':
if row['type'] == 'bool':
return False
else:
return True
raise Exception('Could not determine version of database')
def preprocess(routedict, function, status=FILEIN, **argv):
''' for pre- and postprocessing of files.
these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
- unzipping zipped files.
- convert excel to csv
- password protected files.
Select files from INFILE -> SET_FOR_PROCESSING using criteria
Than the actual processing function is called.
The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
If errors occur during processing, no ta are left with status FILEIN !
preprocess is called right after the in-communicatiation
'''
nr_files = 0
preprocessnumber = botslib.getpreprocessnumber()
if not botslib.addinfo(change={'status': preprocessnumber},
where={
'status': status,
'idroute': routedict['idroute'],
'fromchannel': routedict['fromchannel']
}): #check if there is something to do
return 0
for row in botslib.query(
u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND fromchannel=%(fromchannel)s
''', {
'status': preprocessnumber,
'statust': OK,
'idroute': routedict['idroute'],
'fromchannel': routedict['fromchannel'],
'rootidta': botslib.get_minta4query()
}):
try:
botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',
function.__name__, row['filename'])
ta_set_for_processing = botslib.OldTransaction(row['idta'])
ta_processing = ta_set_for_processing.copyta(
status=preprocessnumber + 1)
ta_processing.filename = row['filename']
function(ta_from=ta_processing,
endstatus=status,
routedict=routedict,
**argv)
except:
txt = botslib.txtexc()
ta_processing.failure()
ta_processing.update(statust=ERROR, errortext=txt)
else:
botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',
function.__name__, row['filename'])
ta_set_for_processing.update(statust=DONE)
ta_processing.update(statust=DONE)
nr_files += 1
return nr_files
def _buildevaluationstructure(self, tacurrent):
''' recursive,for each db-ta:
- fill global talist with the children (and children of children, etc)
'''
#gather next steps/ta's for tacurrent;
if tacurrent.child: #find successor by using child relation ship
for row in botslib.query(
'''SELECT ''' + TAVARS + '''
FROM ta
WHERE idta=%(child)s''',
{'child': tacurrent.child}):
realdict = dict([(key, row[key]) for key in row.keys()])
tacurrent.talijst = [botslib.OldTransaction(**realdict)]
else: #find successor by using parent-relationship; mostly this relation except for merge operations
talijst = []
for row in botslib.query(
'''SELECT ''' + TAVARS + '''
FROM ta
WHERE idta > %(currentidta)s
AND parent=%(currentidta)s ''', #adding the idta > %(parent)s to selection speeds up a lot.
{'currentidta': tacurrent.idta}):
realdict = dict([(key, row[key]) for key in row.keys()])
talijst.append(botslib.OldTransaction(**realdict))
#filter:
#one ta might have multiple children; 2 possible reasons for that:
#1. split up
#2. error is processing the file; and retried
#Here case 2 (error/retry) is filtered; it is not interesting to evaluate the older errors!
#So: if the same filename and different script: use newest idta
#shortcut: when an error occurs in a split all is turned back.
#so: split up is OK as a whole or because of retries.
#so: if split, and different scripts: split is becaue of retries: use newest idta.
#~ print tacurrent.talijst
if len(talijst) > 1 and talijst[0].script != talijst[1].script:
#find higest idta
highest_ta = talijst[0]
for ta_object in talijst[1:]:
if ta_object.idta > highest_ta.idta:
highest_ta = ta_object
tacurrent.talijst = [highest_ta]
else:
tacurrent.talijst = talijst
#recursive build:
for child in tacurrent.talijst:
self._buildevaluationstructure(child)
def routedispatcher(routestorun, type=None):
''' run all route(s). '''
if type == '--retransmit':
if not prepareretransmit():
return 0
elif type == '--retrycommunication':
if not preparerecommunication():
return 0
elif type == '--automaticretrycommunication':
if not prepareautomaticrecommunication():
return 0
elif type == '--retry':
if not prepareretry():
return 0
stuff2evaluate = botslib.getlastrun()
botslib.set_minta4query()
for route in routestorun:
foundroute = False
botslib.setpreprocessnumber(SET_FOR_PROCESSING)
for routedict in botslib.query(
'''SELECT idroute ,
fromchannel_id as fromchannel,
tochannel_id as tochannel,
fromeditype,
frommessagetype,
alt,
frompartner_id as frompartner,
topartner_id as topartner,
toeditype,
tomessagetype,
seq,
frompartner_tochannel_id,
topartner_tochannel_id,
testindicator,
translateind,
defer
FROM routes
WHERE idroute=%(idroute)s
AND active=%(active)s
ORDER BY seq''', {
'idroute': route,
'active': True
}):
botsglobal.logger.info(_(u'running route %(idroute)s %(seq)s'), {
'idroute': routedict['idroute'],
'seq': routedict['seq']
})
botslib.setrouteid(routedict['idroute'])
foundroute = True
router(routedict)
botslib.setrouteid('')
botsglobal.logger.debug(u'finished route %s %s',
routedict['idroute'], routedict['seq'])
if not foundroute:
botsglobal.logger.warning(_(u'there is no (active) route "%s".'),
route)
return stuff2evaluate
def _cleanarchive():
''' delete all archive directories older than maxdaysarchive days.'''
vanaf = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d')
for row in botslib.query('''SELECT archivepath FROM channel '''):
if row['archivepath']:
vanafdir = botslib.join(row['archivepath'],vanaf)
for dir in glob.glob(botslib.join(row['archivepath'],'*')):
if dir < vanafdir:
shutil.rmtree(dir,ignore_errors=True)
def persist_lookup(domein,botskey):
''' lookup persistent values in db.
'''
for row in botslib.query(u'''SELECT content
FROM persist
WHERE domein=%(domein)s
AND botskey=%(botskey)s''',
{'domein':domein,'botskey':botskey}):
return pickle.loads(str(row['content']))
return None
def getcodeset(ccodeid,leftcode,field='rightcode'):
''' Get a code set
'''
return list(botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND leftcode = %(leftcode)s''',
{'ccodeid':ccodeid,
'leftcode':leftcode,
}))
def persist_lookup(domein,botskey):
''' lookup persistent values in db.
'''
for row in botslib.query(u'''SELECT content
FROM persist
WHERE domein=%(domein)s
AND botskey=%(botskey)s''',
{'domein':domein,'botskey':botskey}):
return pickle.loads(str(row['content']))
return None
def reverse_ccode(ccodeid,rightcode,field='leftcode'):
''' as ccode but reversed lookup.'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND rightcode = %(rightcode)s''',
{'ccodeid':ccodeid,
'rightcode':rightcode,
}):
return row[field]
raise botslib.CodeConversionError(_(u'Value "$value" not in code-conversion, user table "$table".'),value=rightcode,table=ccodeid)
def _cleantransactions():
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30))
lijst = list( botslib.query('''SELECT idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf}))
for rootta in lijst:
botslib.change('''DELETE FROM filereport WHERE reportidta = %(rootta)s''',{'rootta':rootta['idta']})
botslib.change('''DELETE FROM report WHERE idta = %(rootta)s''',{'rootta':rootta['idta']})
#~ botslib.change('''DELETE FROM filereport WHERE idta = %(rootta)s''',{'rootta':rootta['idta']})
for index in range(1,len(lijst)):
botslib.change('''DELETE FROM ta WHERE idta >= %(minrootta)s
AND idta < %(maxrootta)s''',
{'minrootta':lijst[index-1]['idta'],'maxrootta':lijst[index]['idta']})
def rcodetconversion(ccodeid,rightcode,field='leftcode'):
''' as codetconversion but reverses the dictionary first'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND rightcode = %(rightcode)s''',
{'ccodeid':ccodeid,
'rightcode':rightcode,
}):
return row[field]
raise botslib.CodeConversionError(_(u'Value "$value" not in codetconversions, user table "$table".'),value=rightcode,tabel=ccodeid)
def core(idta):
#select db-ta's referring to this db-ta
for row in botslib.query('''SELECT idta
FROM ta
WHERE idta > %(idta)s
AND script=%(idta)s''',
{'idta':idta}):
core(row['idta'])
ta=botslib.OldTransaction(idta)
ta.delete()
return
def safercodetconversion(ccodeid,rightcode,field='leftcode'):
''' as codetconversion but reverses the dictionary first'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND rightcode = %(rightcode)s''',
{'ccodeid':ccodeid,
'rightcode':rightcode,
}):
return row[field]
return rightcode
def getcodeset(ccodeid,leftcode,field='rightcode'):
''' Returns a list of all 'field' values in ccode with right ccodeid and leftcode.
'''
terug = []
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND leftcode = %(leftcode)s''',
{'ccodeid':ccodeid,'leftcode':leftcode}):
terug.append(row[field])
return terug
def _cleanarchive():
''' delete all archive directories older than maxdaysarchive days.'''
vanaf = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d')
for row in botslib.query('''SELECT archivepath FROM channel WHERE archivepath != '' '''):
vanafdir = botslib.join(row['archivepath'],vanaf)
for entry in glob.iglob(botslib.join(row['archivepath'],'*')):
if entry < vanafdir:
if entry.endswith('.zip'):
os.remove(entry)
else:
shutil.rmtree(entry,ignore_errors=True)
def getcodeset(ccodeid,leftcode,field='rightcode'):
'''
Get a code set
'''
return list(botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND leftcode = %(leftcode)s''',
{'ccodeid':ccodeid,
'leftcode':leftcode,
}))
def get_minta4query_route(self):
''' find out where route was started.
if not started in crashed run, value for recovery run will be found.
'''
for row in botslib.query('''SELECT MIN(idta) as route_idta
FROM ta
WHERE idta > %(rootidta_of_current_run)s
AND script = %(rootidta_of_current_run)s
AND idroute = %(idroute)s ''',
{'rootidta_of_current_run':self.get_minta4query(),'idroute':botslib.getrouteid()}):
return row['route_idta']
def core(idta):
#select db-ta's referring to this db-ta
for row in botslib.query(
'''SELECT idta
FROM ta
WHERE idta > %(idta)s
AND script=%(idta)s''', {'idta': idta}):
core(row['idta'])
ta = botslib.OldTransaction(idta)
ta.delete()
return
def run(self):
#get rootidta of crashed run
for row in botslib.query('''SELECT MAX(idta) as crashed_idta
FROM ta
WHERE idta < %(rootidta_of_current_run)s
AND script = 0 ''',
{'rootidta_of_current_run':self.minta4query}):
self.minta4query_crash = row['crashed_idta']
if not self.minta4query_crash:
return False #no run
rootofcrashedrun = botslib.OldTransaction(self.minta4query_crash)
rootofcrashedrun.update(statust=DONE)
#clean up things from crash **********************************
#delete run report
botslib.changeq('''DELETE FROM report WHERE idta = %(rootofcrashedrun)s''',{'rootofcrashedrun':rootofcrashedrun.idta})
#delete file reports
botslib.changeq('''DELETE FROM filereport WHERE idta>%(rootofcrashedrun)s''',{'rootofcrashedrun':rootofcrashedrun.idta})
#delete ta's after ERROR and OK for crashed merges
mergedidtatodelete = set()
for row in botslib.query('''SELECT child FROM ta
WHERE idta > %(rootofcrashedrun)s
AND statust = %(statust)s
AND status != %(status)s
AND child != 0 ''',
{'rootofcrashedrun':rootofcrashedrun.idta,'status':PROCESS,'statust':OK}):
mergedidtatodelete.add(row['child'])
for idta in mergedidtatodelete:
ta_object = botslib.OldTransaction(idta)
ta_object.delete()
#delete ta's after ERROR and OK for other
for row in botslib.query('''SELECT idta FROM ta
WHERE idta > %(rootofcrashedrun)s
AND ( statust = %(statust1)s OR statust = %(statust2)s )
AND status != %(status)s
AND child = 0 ''',
{'rootofcrashedrun':rootofcrashedrun.idta,'status':PROCESS,'statust1':OK,'statust2':ERROR}):
ta_object = botslib.OldTransaction(row['idta'])
ta_object.deletechildren()
return super(crashrecovery, self).run()
def preparerecommunication():
#for each out-communication process that went wrong:
retransmit = False #indicate retransmit
for row in botslib.query('''SELECT idta,tochannel
FROM ta
WHERE statust!=%(statust)s
AND status=%(status)s
AND retransmit=%(retransmit)s ''',
{'status':PROCESS,'retransmit':True,'statust':DONE}):
run_outgoing = botslib.OldTransaction(row['idta'])
run_outgoing.update(retransmit=False) #set retransmit back to False
#get rootidta of run where communication failed
for row2 in botslib.query('''SELECT max(idta) as rootidta
FROM ta
WHERE script=%(script)s
AND idta<%(thisidta)s ''',
{'script':0,'thisidta':row['idta']}):
rootidta = row2['rootidta']
#get endidta of run where communication failed
for row3 in botslib.query('''SELECT min(idta) as endidta
FROM ta
WHERE script=%(script)s
AND idta>%(thisidta)s ''',
{'script':0,'thisidta':row['idta']}):
endidta = row3['endidta']
if not endidta:
endidta = sys.maxint - 1
#reinject
for row4 in botslib.query('''SELECT idta
FROM ta
WHERE idta<%(endidta)s
AND idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND tochannel=%(tochannel)s ''',
{'statust':OK,'status':RAWOUT,'rootidta':rootidta,'endidta':endidta,'tochannel':row['tochannel']}):
retransmit = True
ta_outgoing = botslib.OldTransaction(row4['idta'])
ta_outgoing_copy = ta_outgoing.copyta(status=RAWOUT,statust=OK)
ta_outgoing.update(statust=DONE)
return retransmit
def _cleantransactions():
''' delete records from report, filereport and ta.
best indexes are on idta/reportidta; this should go fast.
'''
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30))
for row in botslib.query('''SELECT MAX(idta) as max_idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf}):
maxidta = row['max_idta']
if maxidta is None: #if there is no maxidta to delete, do nothing
return
botslib.changeq('''DELETE FROM report WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM filereport WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM ta WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
def _cleantransactions():
''' delete records from report, filereport and ta.
best indexes are on idta/reportidta; this should go fast.
'''
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30))
for row in botslib.query('''SELECT MAX(idta) as max_idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf}):
maxidta = row['max_idta']
if maxidta is None: #if there is no maxidta to delete, do nothing
return
botslib.changeq('''DELETE FROM report WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM filereport WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM ta WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
def persist_lookup(domein, botskey):
""" lookup persistent values in db.
"""
for row in botslib.query(
u"""SELECT content
FROM persist
WHERE domein=%(domein)s
AND botskey=%(botskey)s""",
{"domein": domein, "botskey": botskey},
):
return pickle.loads(str(row["content"]))
return None
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(rootidtaofrun)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]
status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors)
#give information about the used command line parameters for each run. Problem is that there is only 35pos for this (in MySQL, PostgreSQL).
#~ commandline =
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
commandline = ' '.join(sys.argv)
else:
commandline = ' '.join([arg for arg in sys.argv[1:] if arg!='-cconfig' and not arg.startswith('--')])[:35]
botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors,
ts,lastreceived,status,type,filesize,acceptance,rsrv1)
VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s,
%(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s,%(rsrv1)s) ''',
{'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK],
'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived,
'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False)),
'rsrv1':commandline})
#20120830: if new run with nothing received and no process errors: delete ta's.
if command == 'new' and not lastreceived and not processerrors:
botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def codetconversion(ccodeid,leftcode,field='rightcode'):
''' converts code using a db-table.
converted value is returned.
'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND leftcode = %(leftcode)s''',
{'ccodeid':ccodeid,
'leftcode':leftcode,
}):
return row[field]
raise botslib.CodeConversionError(_(u'Value "$value" not in codetconversions, user table "$table".'),value=leftcode,tabel=ccodeid)
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(rootidtaofrun)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]
status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors)
#give information about the used command line parameters for each run. Problem is that there is only 35pos for this (in MySQL, PostgreSQL).
#~ commandline =
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
commandline = ' '.join(sys.argv)
else:
commandline = ' '.join([arg for arg in sys.argv[1:] if arg!='-cconfig' and not arg.startswith('--')])[:35]
botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors,
ts,lastreceived,status,type,filesize,acceptance,rsrv1)
VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s,
%(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s,%(rsrv1)s) ''',
{'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK],
'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived,
'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False)),
'rsrv1':commandline})
#20120830: if new run with nothing received and no process errors: delete ta's.
if command == 'new' and not lastreceived and not processerrors:
botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def safecodetconversion(ccodeid,leftcode,field='rightcode'):
''' converts code using a db-table.
converted value is returned.
'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND leftcode = %(leftcode)s''',
{'ccodeid':ccodeid,
'leftcode':leftcode,
}):
return row[field]
return leftcode
def reverse_ccode(ccodeid,rightcode,field='leftcode',safe=False):
''' as ccode but reversed lookup.'''
for row in botslib.query(u'''SELECT ''' +field+ '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND rightcode = %(rightcode)s''',
{'ccodeid':ccodeid,'rightcode':rightcode}):
return row[field]
if safe:
return rightcode
else:
raise botslib.CodeConversionError(_(u'Value "%(value)s" not in code-conversion, user table "%(table)s".'),
{'value':rightcode,'table':ccodeid})
def getcodeset(ccodeid, leftcode, field='rightcode'):
''' Returns a list of all 'field' values in ccode with right ccodeid and leftcode.
'''
terug = []
for row in botslib.query(
u'''SELECT ''' + field + '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND leftcode = %(leftcode)s''', {
'ccodeid': ccodeid,
'leftcode': leftcode
}):
terug.append(row[field])
return terug
def reverse_ccode(ccodeid, rightcode, field='leftcode'):
''' as ccode but reversed lookup.'''
for row in botslib.query(
u'''SELECT ''' + field + '''
FROM ccode
WHERE ccodeid_id = %(ccodeid)s
AND rightcode = %(rightcode)s''', {
'ccodeid': ccodeid,
'rightcode': rightcode,
}):
return row[field]
raise botslib.CodeConversionError(
_(u'Value "$value" not in code-conversion, user table "$table".'),
value=rightcode,
table=ccodeid)
def evaluateretryrun(evaluate_type, stuff2evaluate):
resultlast = {OPEN: 0, ERROR: 0, OK: 0, DONE: 0}
didretry = False
for row in botslib.query(
'''SELECT idta
FROM filereport
GROUP BY idta
HAVING MAX(statust) != %(statust)s''',
{'statust': DONE}):
didretry = True
for tadict in botslib.query(
'''SELECT ''' + TAVARS + '''
FROM ta
WHERE idta= %(idta)s ''',
{'idta': row['idta']}):
break
else: #there really should be a corresponding ta
raise botslib.PanicError(
_(u'MaintenanceRetry: could not find transaction "$txt".'),
txt=row['idta'])
mytrace = Trace(tadict, stuff2evaluate)
resultlast[mytrace.statusttree] += 1
if mytrace.statusttree == DONE:
mytrace.errortext = ''
#~ mytrace.ta.update(tracestatus=mytrace.statusttree)
#ts for retried filereports is tricky: is this the time the file was originally received? best would be to use ts of prepare...
#that is quite difficult, so use time of this run
rootta = botslib.OldTransaction(stuff2evaluate)
rootta.syn('ts') #get the timestamp of this run
mytrace.ts = rootta.ts
insert_filereport(mytrace)
del mytrace.ta_object
del mytrace
if not didretry:
return 0 #no error
return finish_evaluation(stuff2evaluate, resultlast, evaluate_type)
