def connect():
#different connect code per tyoe of database
if botsglobal.settings.DATABASE_ENGINE == 'sqlite3':
#sqlite has some more fiddling; in separate file. Mainly because of some other method of parameter passing.
if not os.path.isfile(botsglobal.settings.DATABASE_NAME):
raise botslib.PanicError(
_(u'Could not find database file for SQLite'))
import botssqlite
botsglobal.db = botssqlite.connect(
database=botsglobal.settings.DATABASE_NAME)
elif botsglobal.settings.DATABASE_ENGINE == 'mysql':
import MySQLdb
from MySQLdb import cursors
botsglobal.db = MySQLdb.connect(
host=botsglobal.settings.DATABASE_HOST,
port=int(botsglobal.settings.DATABASE_PORT),
db=botsglobal.settings.DATABASE_NAME,
user=botsglobal.settings.DATABASE_USER,
passwd=botsglobal.settings.DATABASE_PASSWORD,
cursorclass=cursors.DictCursor,
**botsglobal.settings.DATABASE_OPTIONS)
elif botsglobal.settings.DATABASE_ENGINE == 'postgresql_psycopg2':
import psycopg2
import psycopg2.extensions
import psycopg2.extras
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
botsglobal.db = psycopg2.connect(
'host=%s dbname=%s user=%s password=%s' %
(botsglobal.settings.DATABASE_HOST,
botsglobal.settings.DATABASE_NAME,
botsglobal.settings.DATABASE_USER,
botsglobal.settings.DATABASE_PASSWORD),
connection_factory=psycopg2.extras.DictConnection)
botsglobal.db.set_client_encoding('UNICODE')
def email_error_report(rootidtaofrun):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidtaofrun)s''',
{'rootidtaofrun':rootidtaofrun}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':unicode(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':unicode(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext) #log the report texts
# only send email report if there are errors.
# sendreportifprocesserror (in bots.ini): no email reports if only process errors
if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):
# Include details about process errors in the email report; if debug is True: includes trace
if results['processerrors']:
for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
FROM ta
WHERE idta>=%(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
reporttext += '\nProcess error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
# Include details about file errors in the email report; if debug is True: includes trace
if results['lasterror'] or results['lastopen'] or results['lastok']:
for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
FROM filereport
WHERE idta>%(rootidtaofrun)s
AND statust!=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'statust':DONE}):
reporttext += '\nFile error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def connect():
''' connect to database for non-django modules eg engine '''
if botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
#sqlite has some more fiddling; in separate file. Mainly because of some other method of parameter passing.
if not os.path.isfile(
botsglobal.settings.DATABASES['default']['NAME']):
raise botslib.PanicError(
u'Could not find database file for SQLite')
import botssqlite
botsglobal.db = botssqlite.connect(
database=botsglobal.settings.DATABASES['default']['NAME'])
elif botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.mysql':
import MySQLdb
from MySQLdb import cursors
botsglobal.db = MySQLdb.connect(
host=botsglobal.settings.DATABASES['default']['HOST'],
port=int(botsglobal.settings.DATABASES['default']['PORT']),
db=botsglobal.settings.DATABASES['default']['NAME'],
user=botsglobal.settings.DATABASES['default']['USER'],
passwd=botsglobal.settings.DATABASES['default']['PASSWORD'],
cursorclass=cursors.DictCursor,
**botsglobal.settings.DATABASES['default']['OPTIONS'])
elif botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.postgresql_psycopg2':
import psycopg2
import psycopg2.extensions
import psycopg2.extras
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
botsglobal.db = psycopg2.connect(
host=botsglobal.settings.DATABASES['default']['HOST'],
port=botsglobal.settings.DATABASES['default']['PORT'],
database=botsglobal.settings.DATABASES['default']['NAME'],
user=botsglobal.settings.DATABASES['default']['USER'],
password=botsglobal.settings.DATABASES['default']['PASSWORD'],
connection_factory=psycopg2.extras.DictConnection)
botsglobal.db.set_client_encoding('UNICODE')
else:
raise botslib.PanicError(
u'Unknown database engine "%(engine)s".',
{'engine': botsglobal.settings.DATABASES['default']['ENGINE']})
def generate_report(stuff2evaluate):
for results in botslib.query(
'''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidta)s''',
{'rootidta': stuff2evaluate}):
break
else:
raise botslib.PanicError(
_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s') % {
'time': str(results['ts'])[:16]
}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n') % {
'type': results['type'],
'time': str(results['ts'])[:19]
}
reporttext += _(u' %d files received/processed in run.\n') % (
results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n') % (
results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors') % (results['lasterror'])
reporttext += _(u' %d files with errors,\n') % (
results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck') % (results['lastok'])
reporttext += _(u' %d files got stuck,\n') % (results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors') % (results['lastopen'])
reporttext += _(u' %d system errors,\n') % (results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors') % (results['processerrors'])
reporttext += _(u' %d errors in processes.\n') % (
results['processerrors'])
reporttext += _(u' %d files send in run.\n') % (results['send'])
botsglobal.logger.info(reporttext)
# sendreportifprocesserror allows blocking of email reports for process errors
if (results['lasterror'] or results['lastopen'] or results['lastok']
or (results['processerrors'] and botsglobal.ini.getboolean(
'settings', 'sendreportifprocesserror', True))):
botslib.sendbotserrorreport(subject, reporttext)
return int(
results['status']) #return report status: 0 (no error) or 1 (error)
def evaluateretryrun(evaluate_type, stuff2evaluate):
resultlast = {OPEN: 0, ERROR: 0, OK: 0, DONE: 0}
didretry = False
for row in botslib.query(
'''SELECT idta
FROM filereport
GROUP BY idta
HAVING MAX(statust) != %(statust)s''',
{'statust': DONE}):
didretry = True
for tadict in botslib.query(
'''SELECT ''' + TAVARS + '''
FROM ta
WHERE idta= %(idta)s ''',
{'idta': row['idta']}):
break
else: #there really should be a corresponding ta
raise botslib.PanicError(
_(u'MaintenanceRetry: could not find transaction "$txt".'),
txt=row['idta'])
mytrace = Trace(tadict, stuff2evaluate)
resultlast[mytrace.statusttree] += 1
if mytrace.statusttree == DONE:
mytrace.errortext = ''
#~ mytrace.ta.update(tracestatus=mytrace.statusttree)
#ts for retried filereports is tricky: is this the time the file was originally received? best would be to use ts of prepare...
#that is quite difficult, so use time of this run
rootta = botslib.OldTransaction(stuff2evaluate)
rootta.syn('ts') #get the timestamp of this run
mytrace.ts = rootta.ts
insert_filereport(mytrace)
del mytrace.ta_object
del mytrace
if not didretry:
return 0 #no error
return finish_evaluation(stuff2evaluate, resultlast, evaluate_type)
def generalinit(configdir):
##########################################################################
#Configdir: settings.py & bots.ini#########################################
#Configdir MUST be importable. So configdir is relative to PYTHONPATH. Try several options for this import.
try: #first check if is configdir outside bots-directory: import configdir.settings.py
importnameforsettings = os.path.normpath(
os.path.join(configdir, 'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #normal: configdir is in bots directory: import bots.configdir.settings.py
try:
importnameforsettings = os.path.normpath(
os.path.join('bots', configdir,
'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #set pythonpath to config directory first
if not os.path.exists(configdir): #check if configdir exists.
raise botslib.PanicError(
u'In initilisation: path to configuration does not exists: "%(configdir)s".',
{'configdir': configdir})
addtopythonpath = os.path.abspath(os.path.dirname(configdir))
moduletoimport = os.path.basename(configdir)
sys.path.append(addtopythonpath)
importnameforsettings = os.path.normpath(
os.path.join(moduletoimport, 'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
#settings is imported, so now we know where to find settings.py: importnameforsettings
#note: the imported settings.py itself is NOT used, this is doen via django.conf.settings
configdirectory = os.path.abspath(os.path.dirname(settings.__file__))
#Read configuration-file bots.ini.
botsglobal.ini = BotsConfig()
botsglobal.ini.read(os.path.join(configdirectory, 'bots.ini'))
# 'directories','botspath': absolute path for bots directory
botsglobal.ini.set('directories', 'botspath',
os.path.abspath(os.path.dirname(__file__)))
# 'directories','config': absolute path for config directory
botsglobal.ini.set('directories', 'config', configdirectory)
#set config as originally received; used in starting engine via bots-monitor
botsglobal.ini.set('directories', 'config_org', configdir)
############################################################################
#Usersys####################################################################
#usersys MUST be importable. So usersys is relative to PYTHONPATH. Try several options for this import.
usersys = botsglobal.ini.get('directories', 'usersys', 'usersys')
try: #usersys outside bots-directory: import usersys
importnameforusersys = os.path.normpath(usersys).replace(os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #usersys is in bots directory: import bots.usersys
try:
importnameforusersys = os.path.normpath(
os.path.join('bots', usersys)).replace(os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #set pythonpath to usersys directory first
if not os.path.exists(usersys): #check if configdir exists.
raise botslib.PanicError(
u'In initilisation: path to configuration does not exists: "%(usersys)s".',
{'usersys': usersys})
addtopythonpath = os.path.abspath(os.path.dirname(usersys)) #????
moduletoimport = os.path.basename(usersys)
sys.path.append(addtopythonpath)
importnameforusersys = os.path.normpath(usersys).replace(
os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
# 'directories','usersysabs': absolute path for config usersysabs
botsglobal.ini.set('directories', 'usersysabs',
os.path.abspath(
os.path.dirname(importedusersys.__file__))
) #???Find pathname usersys using imported usersys
# botsglobal.usersysimportpath: used for imports from usersys
botsglobal.usersysimportpath = importnameforusersys
botsglobal.ini.set(
'directories', 'templatehtml',
botslib.join(botsglobal.ini.get('directories', 'usersysabs'),
'grammars/templatehtml/templates'))
############################################################################
#Botssys####################################################################
# 'directories','botssys': absolute path for config botssys
botssys = botsglobal.ini.get('directories', 'botssys', 'botssys')
botsglobal.ini.set('directories', 'botssys_org',
botssys) #store original botssys setting
botsglobal.ini.set('directories', 'botssys',
botslib.join(botssys)) #use absolute path
botsglobal.ini.set('directories', 'data', botslib.join(botssys, 'data'))
botsglobal.ini.set('directories', 'logging',
botslib.join(botssys, 'logging'))
############################################################################
#other inits##############################################################
if botsglobal.ini.get(
'webserver', 'environment', 'development'
) != 'development': #values in bots.ini are also used in setting up cherrypy
logging.raiseExceptions = 0 # during production: if errors occurs in writing to log: ignore error. (leads to a missing log line, better than error;-).
botslib.dirshouldbethere(botsglobal.ini.get('directories', 'data'))
botslib.dirshouldbethere(botsglobal.ini.get('directories', 'logging'))
initbotscharsets() #initialise bots charsets
node.Node.checklevel = botsglobal.ini.getint('settings', 'get_checklevel',
1)
botslib.settimeout(botsglobal.ini.getint('settings', 'globaltimeout', 10))
############################################################################
#Init django#################################################################################
os.environ['DJANGO_SETTINGS_MODULE'] = importnameforsettings
import django
if hasattr(django, 'setup'):
django.setup()
from django.conf import settings
botsglobal.settings = settings #settings are accessed using botsglobal
