def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(rootidtaofrun)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]
status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors)
botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors,
ts,lastreceived,status,type,filesize,acceptance)
VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s,
%(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s) ''',
{'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK],
'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived,
'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False))})
#20120830: if new run with nothing received and no process errors: delete ta's.
if command == 'new' and not lastreceived and not processerrors:
botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def run(self):
''' prepare the files indicated by user to be rereceived. Return: indication if files should be rereceived.
'''
do_retransmit = False
for row in botslib.query('''SELECT idta
FROM filereport
WHERE retransmit = %(retransmit)s ''',
{'retransmit':1}):
do_retransmit = True
botslib.changeq('''UPDATE filereport
SET retransmit = %(retransmit)s
WHERE idta = %(idta)s ''',
{'idta':row['idta'],'retransmit':0})
for row2 in botslib.query('''SELECT idta
FROM ta
WHERE parent = %(parent)s ''',
{'parent':row['idta']}):
ta_rereceive = botslib.OldTransaction(row2['idta'])
ta_externin = ta_rereceive.copyta(status=EXTERNIN,statust=DONE,parent=0) #inject; status is DONE so this ta is not used further
ta_externin.copyta(status=FILEIN,statust=OK) #reinjected file is ready as new input
if do_retransmit:
return super(rereceive, self).run()
else:
return False #no run
def persist_delete(domein,botskey):
''' store persistent values in db.
'''
botslib.changeq(u''' DELETE FROM persist
WHERE domein=%(domein)s
AND botskey=%(botskey)s''',
{'domein':domein,'botskey':botskey})
def persist_update(domein,botskey,value):
''' store persistent values in db.
'''
content = pickle.dumps(value)
botslib.changeq(u''' UPDATE persist
SET content=%(content)s
WHERE domein=%(domein)s
AND botskey=%(botskey)s''',
{'domein':domein,'botskey':botskey,'content':content})
def persist_delete(domein, botskey):
''' store persistent values in db.
'''
botslib.changeq(
u''' DELETE FROM persist
WHERE domein=%(domein)s
AND botskey=%(botskey)s''', {
'domein': domein,
'botskey': botskey
})
def persist_add(domein,botskey,value):
''' store persistent values in db.
'''
content = pickle.dumps(value)
try:
botslib.changeq(u''' INSERT INTO persist (domein,botskey,content)
VALUES (%(domein)s,%(botskey)s,%(content)s)''',
{'domein':domein,'botskey':botskey,'content':content})
except:
raise botslib.PersistError(_(u'Failed to add for domein "%(domein)s", botskey "%(botskey)s", value "%(value)s".'),
{'domein':domein,'botskey':botskey,'value':value})
def persist_update(domein, botskey, value):
''' store persistent values in db.
'''
content = pickle.dumps(value).decode('iso-8859-1')
botslib.changeq(
u''' UPDATE persist
SET content=%(content)s
WHERE domein=%(domein)s
AND botskey=%(botskey)s''', {
'domein': domein,
'botskey': botskey,
'content': content
})
def make_file_report(self):
botslib.changeq(u'''INSERT INTO filereport (idta,statust,reportidta,retransmit,idroute,fromchannel,ts,
infilename,tochannel,frompartner,topartner,frommail,
tomail,ineditype,inmessagetype,outeditype,outmessagetype,
incontenttype,outcontenttype,nrmessages,outfilename,errortext,
divtext,outidta,rsrv1,filesize)
VALUES (%(idta)s,%(statust)s,%(reportidta)s,%(retransmit)s,%(idroute)s,%(fromchannel)s,%(ts)s,
%(infilename)s,%(tochannel)s,%(frompartner)s,%(topartner)s,%(frommail)s,
%(tomail)s,%(ineditype)s,%(inmessagetype)s,%(outeditype)s,%(outmessagetype)s,
%(incontenttype)s,%(outcontenttype)s,%(nrmessages)s,%(outfilename)s,%(errortext)s,
%(divtext)s,%(outidta)s,%(rsrv1)s,%(filesize)s )
''',
self.__dict__ )
def _cleanrunsnothingreceived():
''' delete all report off new runs that received no files and no process errors.
#20120830: if new run with nothing received and no process errors: ta's are already deleted in automaticmaintenance.
'''
vanaf = datetime.datetime.today() - datetime.timedelta(hours=botsglobal.ini.getint('settings','hoursrunwithoutresultiskept',1))
onlycheckrunsofoneday = datetime.datetime.today() - datetime.timedelta(hours=25)
botslib.changeq('''DELETE FROM report
WHERE ts < %(vanaf)s
AND ts >= %(onlycheckrunsofoneday)s
AND type = 'new'
AND lastreceived=0
AND processerrors=0 ''',
{'vanaf':vanaf,'onlycheckrunsofoneday':onlycheckrunsofoneday})
def _cleanrunsnothingreceived():
''' delete all report off new runs that received no files and no process errors.
#20120830: if new run with nothing received and no process errors: ta's are already deleted in automaticmaintenance.
'''
vanaf = datetime.datetime.today() - datetime.timedelta(hours=botsglobal.ini.getint('settings','hoursrunwithoutresultiskept',1))
onlycheckrunsofoneday = datetime.datetime.today() - datetime.timedelta(hours=25)
botslib.changeq('''DELETE FROM report
WHERE ts < %(vanaf)s
AND ts >= %(onlycheckrunsofoneday)s
AND type = 'new'
AND lastreceived=0
AND processerrors=0 ''',
{'vanaf':vanaf,'onlycheckrunsofoneday':onlycheckrunsofoneday})
def make_file_report(self):
#20140116: patch for MySQLdb version 1.2.5. This version seems to check all parameters - not just the ones actually used.
tmp_dict = self.__dict__.copy()
tmp_dict.pop('rootofinfile','nep')
botslib.changeq(u'''INSERT INTO filereport (idta,statust,reportidta,retransmit,idroute,fromchannel,ts,
infilename,tochannel,frompartner,topartner,frommail,
tomail,ineditype,inmessagetype,outeditype,outmessagetype,
incontenttype,outcontenttype,nrmessages,outfilename,errortext,
divtext,outidta,rsrv1,filesize)
VALUES (%(idta)s,%(statust)s,%(reportidta)s,%(retransmit)s,%(idroute)s,%(fromchannel)s,%(ts)s,
%(infilename)s,%(tochannel)s,%(frompartner)s,%(topartner)s,%(frommail)s,
%(tomail)s,%(ineditype)s,%(inmessagetype)s,%(outeditype)s,%(outmessagetype)s,
%(incontenttype)s,%(outcontenttype)s,%(nrmessages)s,%(outfilename)s,%(errortext)s,
%(divtext)s,%(outidta)s,%(rsrv1)s,%(filesize)s )
''',
tmp_dict)
def make_file_report(self):
#20140116: patch for MySQLdb version 1.2.5. This version seems to check all parameters - not just the ones actually used.
tmp_dict = self.__dict__.copy()
tmp_dict.pop('rootofinfile','nep')
botslib.changeq(u'''INSERT INTO filereport (idta,statust,reportidta,retransmit,idroute,fromchannel,ts,
infilename,tochannel,frompartner,topartner,frommail,
tomail,ineditype,inmessagetype,outeditype,outmessagetype,
incontenttype,outcontenttype,nrmessages,outfilename,errortext,
divtext,outidta,rsrv1,filesize)
VALUES (%(idta)s,%(statust)s,%(reportidta)s,%(retransmit)s,%(idroute)s,%(fromchannel)s,%(ts)s,
%(infilename)s,%(tochannel)s,%(frompartner)s,%(topartner)s,%(frommail)s,
%(tomail)s,%(ineditype)s,%(inmessagetype)s,%(outeditype)s,%(outmessagetype)s,
%(incontenttype)s,%(outcontenttype)s,%(nrmessages)s,%(outfilename)s,%(errortext)s,
%(divtext)s,%(outidta)s,%(rsrv1)s,%(filesize)s )
''',
tmp_dict)
def persist_add(domein, botskey, value):
''' store persistent values in db.
'''
content = pickle.dumps(value).decode('iso-8859-1')
try:
botslib.changeq(
u''' INSERT INTO persist (domein,botskey,content)
VALUES (%(domein)s,%(botskey)s,%(content)s)''', {
'domein': domein,
'botskey': botskey,
'content': content
})
except:
raise botslib.PersistError(
_(u'Failed to add for domein "%(domein)s", botskey "%(botskey)s", value "%(value)s".'
), {
'domein': domein,
'botskey': botskey,
'value': value
})
def run(self):
#get rootidta of crashed run
for row in botslib.query('''SELECT MAX(idta) as crashed_idta
FROM ta
WHERE idta < %(rootidta_of_current_run)s
AND script = 0 ''',
{'rootidta_of_current_run':self.minta4query}):
self.minta4query_crash = row['crashed_idta']
if not self.minta4query_crash:
return False #no run
rootofcrashedrun = botslib.OldTransaction(self.minta4query_crash)
rootofcrashedrun.update(statust=DONE)
#clean up things from crash **********************************
#delete run report
botslib.changeq('''DELETE FROM report WHERE idta = %(rootofcrashedrun)s''',{'rootofcrashedrun':rootofcrashedrun.idta})
#delete file reports
botslib.changeq('''DELETE FROM filereport WHERE idta>%(rootofcrashedrun)s''',{'rootofcrashedrun':rootofcrashedrun.idta})
#delete ta's after ERROR and OK for crashed merges
mergedidtatodelete = set()
for row in botslib.query('''SELECT child FROM ta
WHERE idta > %(rootofcrashedrun)s
AND statust = %(statust)s
AND status != %(status)s
AND child != 0 ''',
{'rootofcrashedrun':rootofcrashedrun.idta,'status':PROCESS,'statust':OK}):
mergedidtatodelete.add(row['child'])
for idta in mergedidtatodelete:
ta_object = botslib.OldTransaction(idta)
ta_object.delete()
#delete ta's after ERROR and OK for other
for row in botslib.query('''SELECT idta FROM ta
WHERE idta > %(rootofcrashedrun)s
AND ( statust = %(statust1)s OR statust = %(statust2)s )
AND status != %(status)s
AND child = 0 ''',
{'rootofcrashedrun':rootofcrashedrun.idta,'status':PROCESS,'statust1':OK,'statust2':ERROR}):
ta_object = botslib.OldTransaction(row['idta'])
ta_object.deletechildren()
return super(crashrecovery, self).run()
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(rootidtaofrun)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]
status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors)
#give information about the used command line parameters for each run. Problem is that there is only 35pos for this (in MySQL, PostgreSQL).
#~ commandline =
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
commandline = ' '.join(sys.argv)
else:
commandline = ' '.join([arg for arg in sys.argv[1:] if arg!='-cconfig' and not arg.startswith('--')])[:35]
botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors,
ts,lastreceived,status,type,filesize,acceptance,rsrv1)
VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s,
%(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s,%(rsrv1)s) ''',
{'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK],
'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived,
'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False)),
'rsrv1':commandline})
#20120830: if new run with nothing received and no process errors: delete ta's.
if command == 'new' and not lastreceived and not processerrors:
botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def make_run_report(rootidtaofrun,resultsofrun,command,totalfilesize):
#count nr files send
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta > %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'status':EXTERNOUT,'rootidtaofrun':rootidtaofrun,'statust':DONE}):
send = row['count']
#count process errors
for row in botslib.query('''SELECT COUNT(*) as count
FROM ta
WHERE idta >= %(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s''',
{'status':PROCESS,'rootidtaofrun':rootidtaofrun,'statust':ERROR}):
processerrors = row['count']
#generate report (in database)
rootta = botslib.OldTransaction(rootidtaofrun)
rootta.syn('ts') #get the timestamp of this run
lastreceived = resultsofrun[DONE]+resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]
status = bool(resultsofrun[OK]+resultsofrun[OPEN]+resultsofrun[ERROR]+processerrors)
#give information about the used command line parameters for each run. Problem is that there is only 35pos for this (in MySQL, PostgreSQL).
#~ commandline =
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
commandline = ' '.join(sys.argv)
else:
commandline = ' '.join([arg for arg in sys.argv[1:] if arg!='-cconfig' and not arg.startswith('--')])[:35]
botslib.changeq(u'''INSERT INTO report (idta,lastopen,lasterror,lastok,lastdone,send,processerrors,
ts,lastreceived,status,type,filesize,acceptance,rsrv1)
VALUES (%(rootidtaofrun)s,%(lastopen)s,%(lasterror)s,%(lastok)s,%(lastdone)s,%(send)s,%(processerrors)s,
%(ts)s,%(lastreceived)s,%(status)s,%(type)s,%(totalfilesize)s,%(acceptance)s,%(rsrv1)s) ''',
{'rootidtaofrun':rootidtaofrun,'lastopen':resultsofrun[OPEN],'lasterror':resultsofrun[ERROR],'lastok':resultsofrun[OK],
'lastdone':resultsofrun[DONE],'send':send,'processerrors':processerrors,'ts':rootta.ts,'lastreceived':lastreceived,
'status':status,'type':command,'totalfilesize':totalfilesize,'acceptance':int(botsglobal.ini.getboolean('acceptance','runacceptancetest',False)),
'rsrv1':commandline})
#20120830: if new run with nothing received and no process errors: delete ta's.
if command == 'new' and not lastreceived and not processerrors:
botslib.changeq('''DELETE FROM ta WHERE idta>=%(rootidtaofrun)s''',{'rootidtaofrun':rootidtaofrun})
def _cleantransactions():
''' delete records from report, filereport and ta.
best indexes are on idta/reportidta; this should go fast.
'''
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30))
for row in botslib.query('''SELECT MAX(idta) as max_idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf}):
maxidta = row['max_idta']
if maxidta is None: #if there is no maxidta to delete, do nothing
return
botslib.changeq('''DELETE FROM report WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM filereport WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM ta WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
def _cleantransactions():
''' delete records from report, filereport and ta.
best indexes are on idta/reportidta; this should go fast.
'''
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdays',30))
for row in botslib.query('''SELECT MAX(idta) as max_idta FROM report WHERE ts < %(vanaf)s''',{'vanaf':vanaf}):
maxidta = row['max_idta']
if maxidta is None: #if there is no maxidta to delete, do nothing
return
botslib.changeq('''DELETE FROM report WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM filereport WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
botslib.changeq('''DELETE FROM ta WHERE idta < %(maxidta)s''',{'maxidta':maxidta})
def _cleanupsession():
''' delete all expired sessions. Bots-engine starts up much more often than web-server.'''
vanaf = datetime.datetime.today()
botslib.changeq('''DELETE FROM django_session WHERE expire_date < %(vanaf)s''', {'vanaf':vanaf})
def _cleanpersist():
'''delete all persist older than xx days.'''
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdayspersist',30))
botslib.changeq('''DELETE FROM persist WHERE ts < %(vanaf)s''',{'vanaf':vanaf})
def _cleanupsession():
''' delete all expired sessions. Bots-engine starts up much more often than web-server.'''
vanaf = datetime.datetime.today()
botslib.changeq('''DELETE FROM django_session WHERE expire_date < %(vanaf)s''', {'vanaf':vanaf})
def _cleanpersist():
'''delete all persist older than xx days.'''
vanaf = datetime.datetime.today() - datetime.timedelta(days=botsglobal.ini.getint('settings','maxdayspersist',30))
botslib.changeq('''DELETE FROM persist WHERE ts < %(vanaf)s''',{'vanaf':vanaf})