def connect(self):
"""
recreate a session and db access
"""
# create a session and db access
if self.session is None:
self.session = SafeSession(dbInstance=self.dbInstance)
def testCreation2(self):
"""
creates a single task with N jobs
"""
# log information
logging.info("Test Creation 2: creating a task with %s jobs" % self.N)
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create a task
task = Task()
task['name'] = 'task2'
task['startDirectory'] = '/tmp/startdir'
task['outputDirectory'] = '/tmp/outdir'
task['globalSandbox'] = '/tmp/data_area'
# create template for jobs
template = {
'inputFiles': ['a.txt', 'b.txt'],
'executable': 'production.sh',
'arguments': "-o c.txt",
'outputFiles': ['c.txt'],
'logFile': 'output.log'
}
# create list of N jobs
jobs = []
for index in range(1, self.N + 1):
# set specific parameters
template['name'] = 'job' + str(index)
if index <= int(self.N / 2):
template['logFile'] = 'output.log'
else:
template['logFile'] = 'useful.log'
template['outputFiles'] = ['a.txt', 'b.txt']
# add job to list
jobs.append(Job(template))
# add jobs to task
task.addJobs(jobs)
# save task (and jobs) in single transaction
try:
rows = task.save(db)
session.commit()
except TaskError, msg:
self.fail("Error: " + str(msg))
def testCreation2(self):
"""
creates a single task with N jobs
"""
# log information
logging.info("Test Creation 2: creating a task with %s jobs" % self.N)
start = time()
# create a session and db access
session = SafeSession(dbInstance = self.dbInstance)
db = TrackingDB(session)
# create a task
task = Task()
task['name'] = 'task2'
task['startDirectory'] = '/tmp/startdir'
task['outputDirectory'] = '/tmp/outdir'
task['globalSandbox'] = '/tmp/data_area'
# create template for jobs
template = { 'inputFiles' : ['a.txt', 'b.txt'],
'executable' : 'production.sh',
'arguments' : "-o c.txt",
'outputFiles' : ['c.txt'],
'logFile' : 'output.log' }
# create list of N jobs
jobs = []
for index in range(1, self.N + 1):
# set specific parameters
template['name'] = 'job' + str(index)
if index <= int(self.N / 2):
template['logFile'] = 'output.log'
else:
template['logFile'] = 'useful.log'
template['outputFiles'] = ['a.txt' , 'b.txt']
# add job to list
jobs.append(Job(template))
# add jobs to task
task.addJobs(jobs)
# save task (and jobs) in single transaction
try:
rows = task.save(db)
session.commit()
except TaskError, msg:
self.fail("Error: " + str(msg))
def testLoading3(self):
"""
loading of a single job in the N jobs task
"""
# log information
logging.info("Test Loading 3: load of a single job in a %s jobs task" \
% self.N)
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create template for task
task = Task()
task['name'] = 'task2'
# create template for jobs with particular logFile
job = Job()
job['name'] = 'job1'
# add to task template
task.addJob(job)
# load partial task (jobs with logFile 'output.log')
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
def connect ( self ) :
"""
recreate a session and db access
"""
# create a session and db access
if self.session is None:
self.session = SafeSession(dbInstance = self.dbInstance)
def connect ( self ) :
"""
recreate a session and db access
"""
# create a session and db access
if self.session is None:
self.session = SafeSession(pool = self.pool)
def installMySQL(self, schemaLocation):
"""
install MySQL database
"""
import getpass
from ProdCommon.Database.MysqlInstance import MysqlInstance
# ask for password (optional)
print
userName = raw_input("""
Please provide the mysql user name (typically "root") for updating the
database server (leave empty if not needed): ')
""")
if userName == '':
userName = '******'
print
passwd = getpass.getpass(
"""Please provide mysql passwd associated to this user name for
updating the database server:
""")
# define connection type
from copy import deepcopy
rootConfig = deepcopy(self.dbConfig)
rootConfig.update({
'dbName': 'mysql',
'user': userName,
'passwd': passwd
})
dbInstance = MysqlInstance(rootConfig)
session = SafeSession(dbInstance=dbInstance)
# check if db exists
create = True
query = "show databases like '" + self.dbConfig['dbName'] + "'"
try:
session.execute(query)
session.commit()
results = session.fetchall()
if results[0][0] == self.dbConfig['dbName']:
print "DB ", self.dbConfig['dbName'], "already exists."
print "Installing just BossLite tables.\n"
create = False
except IndexError:
pass
except Exception, msg:
session.close()
raise DbError(str(msg))
def testCreation1(self):
"""
creates a single task with 2 jobs
"""
# log information
print "Creation tests"
logging.info("Test Creation 1: creating a task with two jobs")
start = time()
# create a session and db access
session = SafeSession(dbInstance = self.dbInstance)
db = TrackingDB(session)
# create a task
task = Task()
task['name'] = 'task1'
task['startDirectory'] = '/tmp/startdir'
task['outputDirectory'] = '/tmp/output'
task['globalSandbox'] = '/tmp/inputdata'
# create first job
job1 = Job()
job1['name'] = 'job1'
job1['executable'] = 'test.sh'
job1['inputFiles'] = ['a.txt', 'b.txt']
job1['outputFiles'] = ['c.txt']
# create second job
job2 = Job()
job2['name'] = 'job2'
job2['executable'] = 'production.sh'
job2['arguments'] = "-o c.txt"
# add jobs to task
task.addJobs([job1, job2])
# save task in single transaction
try:
rows = task.save(db)
session.commit()
except TaskError, msg:
self.fail("Error: " + str(msg))
def testCreation1(self):
"""
creates a single task with 2 jobs
"""
# log information
print "Creation tests"
logging.info("Test Creation 1: creating a task with two jobs")
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create a task
task = Task()
task['name'] = 'task1'
task['startDirectory'] = '/tmp/startdir'
task['outputDirectory'] = '/tmp/output'
task['globalSandbox'] = '/tmp/inputdata'
# create first job
job1 = Job()
job1['name'] = 'job1'
job1['executable'] = 'test.sh'
job1['inputFiles'] = ['a.txt', 'b.txt']
job1['outputFiles'] = ['c.txt']
# create second job
job2 = Job()
job2['name'] = 'job2'
job2['executable'] = 'production.sh'
job2['arguments'] = "-o c.txt"
# add jobs to task
task.addJobs([job1, job2])
# save task in single transaction
try:
rows = task.save(db)
session.commit()
except TaskError, msg:
self.fail("Error: " + str(msg))
def installMySQL(self, schemaLocation):
"""
install MySQL database
"""
import getpass
from ProdCommon.Database.MysqlInstance import MysqlInstance
# ask for password (optional)
print
userName = raw_input(
"""
Please provide the mysql user name (typically "root") for updating the
database server (leave empty if not needed): ')
"""
)
if userName == "":
userName = "******"
print
passwd = getpass.getpass(
"""Please provide mysql passwd associated to this user name for
updating the database server:
"""
)
# define connection type
from copy import deepcopy
rootConfig = deepcopy(self.dbConfig)
rootConfig.update({"dbName": "mysql", "user": userName, "passwd": passwd})
dbInstance = MysqlInstance(rootConfig)
session = SafeSession(dbInstance=dbInstance)
# check if db exists
create = True
query = "show databases like '" + self.dbConfig["dbName"] + "'"
try:
session.execute(query)
session.commit()
results = session.fetchall()
if results[0][0] == self.dbConfig["dbName"]:
print "DB ", self.dbConfig["dbName"], "already exists."
print "Installing just BossLite tables.\n"
create = False
except IndexError:
pass
except Exception, msg:
session.close()
raise DbError(str(msg))
def __init__(self, argv):
"""
initialize test instance
"""
# initialize super class
super(jobUnitTests, self).__init__(argv)
# read configuration
self.configure()
# MySQL: get DB configuration from config file
if self.database == "MySQL":
from ProdAgentCore.Configuration import loadProdAgentConfiguration
cfg = loadProdAgentConfiguration()
dbConfig = cfg.getConfig("ProdAgentDB")
dbConfig['dbName'] += '_BOSS'
# create DB instance
self.dbInstance = MysqlInstance(dbConfig)
else:
dbConfig = {}
dbConfig['dbName'] = 'ProdAgentDB_BOSS'
# create DB instance
self.dbInstance = SqliteInstance(dbConfig)
# remove all information from database
session = SafeSession(dbInstance=self.dbInstance)
try:
session.execute("delete from bl_runningjob")
session.execute("delete from bl_job")
session.execute("delete from bl_task")
session.commit()
except DbError, msg:
print "Cannot erase database contents: " + str(msg)
sys.exit(1)
def __init__(self, argv):
"""
initialize test instance
"""
# initialize super class
super(jobUnitTests, self).__init__(argv)
# read configuration
self.configure()
# MySQL: get DB configuration from config file
if self.database == "MySQL":
from ProdAgentCore.Configuration import loadProdAgentConfiguration
cfg = loadProdAgentConfiguration()
dbConfig = cfg.getConfig("ProdAgentDB")
dbConfig['dbName'] += '_BOSS'
# create DB instance
self.dbInstance = MysqlInstance(dbConfig)
else:
dbConfig = {}
dbConfig['dbName'] = 'ProdAgentDB_BOSS'
# create DB instance
self.dbInstance = SqliteInstance(dbConfig)
# remove all information from database
session = SafeSession(dbInstance = self.dbInstance)
try:
session.execute("delete from bl_runningjob")
session.execute("delete from bl_job")
session.execute("delete from bl_task")
session.commit()
except DbError, msg:
print "Cannot erase database contents: " + str(msg)
sys.exit(1)
def testUpdate1(self):
"""
updating task object
"""
# log information
logging.info("Test Updating 1: updating task object")
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create template for task
task = Task()
task['name'] = 'task2'
# load task
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
def testLoading4(self):
"""
checking update of full output path for output files and input files
"""
# log information
logging.info("Test Loading 4: testing full path operations")
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create template for task
task = Task()
task['name'] = 'task2'
# load task
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
def testCreation3(self):
"""
creating a running instance
"""
# log information
logging.info("Test Updating 1: updating task object")
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create template for job
task = Task({'name': 'task1'})
job = Job({'name': 'job1'})
task.addJob(job)
# load information
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
def testLoading1(self):
"""
deep load of a task with 2 jobs and a task with N jobs
"""
# log information
print "\nLoading tests"
logging.info("Test Loading 1: deep load of tasks with 2 and %s jobs" \
% self.N)
start = time()
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create template
task = Task()
task['name'] = 'task1'
# load complete task
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
class BossLiteDB(object):
"""
High level API class for DB queries.
It allows load/operate/update DB using free format queries
"""
dbConfig = {
'dbName': 'BossLiteDB',
'user': '******',
'passwd': 'BossLitePass',
'socketFileLocation': '',
'host': '',
'portNr': '',
'refreshPeriod': 4 * 3600,
'maxConnectionAttempts': 5,
'dbWaitingTime': 10
}
def __init__(self, database, dbConfig):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
"""
# database
self.database = database # "MySQL" or "SQLite"
# MySQL: get DB configuration from config file
if self.database == "MySQL":
# update db config
self.dbConfig['socketFileLocation'] = expandvars(
self.dbConfig['socketFileLocation'])
self.dbConfig.update(dbConfig)
# create DB instance
from ProdCommon.Database.MysqlInstance import MysqlInstance
self.dbInstance = MysqlInstance(self.dbConfig)
else:
# update db config
self.dbConfig = {'dbName': 'BossLiteDB'}
dbConfig['dbName'] = expandvars(dbConfig['dbName'])
self.dbConfig.update(dbConfig)
# create DB instance
from ProdCommon.Database.SqliteInstance import SqliteInstance
self.dbInstance = SqliteInstance(self.dbConfig)
# create a session and db access
self.session = None
##########################################################################
def connect(self):
"""
recreate a session and db access
"""
# create a session and db access
if self.session is None:
self.session = SafeSession(dbInstance=self.dbInstance)
##########################################################################
def close(self):
"""
close session and db access
"""
self.session.close()
self.session = None
##########################################################################
def reset(self):
"""
reset session and db access
"""
self.close()
self.connect()
##########################################################################
def commit(self):
"""
commit
"""
self.session.commit()
##########################################################################
def select(self, query):
"""
execute a query.
"""
# db connect
self.connect()
self.session.execute(query)
# return query results
return self.session.fetchall()
##########################################################################
def selectOne(self, query):
"""
execute a query.with only one result expected
"""
# db connect
self.connect()
self.session.execute(query)
# return query results
return self.session.fetchone()[0]
##########################################################################
def modify(self, query):
"""
execute a query which does not return such as insert/update/delete
"""
# db connect
self.connect()
# return query results
rows = self.session.execute(query)
if self.database == "MySQL":
self.session.commit()
return rows
##########################################################################
def updateDB(self, obj):
"""
update any object table in the DB
works for tasks, jobs, runningJobs
"""
# db connect
self.connect()
# update
obj.update(self.session)
if self.database == "MySQL":
self.session.commit()
##########################################################################
def installDB(self, schemaLocation):
"""
install database
"""
schemaLocation = expandvars(schemaLocation)
if self.database == "MySQL":
self.installMySQL(schemaLocation)
elif self.database == "SQLite":
self.installSQlite(schemaLocation)
else:
raise NotImplementedError
##########################################################################
def installMySQL(self, schemaLocation):
"""
install MySQL database
"""
import getpass
from ProdCommon.Database.MysqlInstance import MysqlInstance
# ask for password (optional)
print
userName = raw_input("""
Please provide the mysql user name (typically "root") for updating the
database server (leave empty if not needed): ')
""")
if userName == '':
userName = '******'
print
passwd = getpass.getpass(
"""Please provide mysql passwd associated to this user name for
updating the database server:
""")
# define connection type
from copy import deepcopy
rootConfig = deepcopy(self.dbConfig)
rootConfig.update({
'dbName': 'mysql',
'user': userName,
'passwd': passwd
})
dbInstance = MysqlInstance(rootConfig)
session = SafeSession(dbInstance=dbInstance)
# check if db exists
create = True
query = "show databases like '" + self.dbConfig['dbName'] + "'"
try:
session.execute(query)
session.commit()
results = session.fetchall()
if results[0][0] == self.dbConfig['dbName']:
print "DB ", self.dbConfig['dbName'], "already exists."
print "Installing just BossLite tables.\n"
create = False
except IndexError:
pass
except Exception, msg:
session.close()
raise DbError(str(msg))
# create db
if create:
query = 'create database ' + self.dbConfig['dbName']
try:
session.execute(query)
session.commit()
except Exception, msg:
session.close()
raise DbError(str(msg))
class BossLiteDB(object):
"""
High level API class for DB queries.
It allows load/operate/update DB using free format queries
"""
dbConfig = {'dbName':'BossLiteDB',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'',
'host':'',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
def __init__(self, database, dbConfig):
"""
initialize the API instance
- database can be both MySQl or SQLite
- dbConfig can be a dictionary with the format
{'dbName':'BossLiteDB',
'host':'localhost',
'user':'******',
'passwd':'BossLitePass',
'socketFileLocation':'/var/run/mysql/mysql.sock',
'portNr':'',
'refreshPeriod' : 4*3600 ,
'maxConnectionAttempts' : 5,
'dbWaitingTime' : 10
}
"""
# database
self.database = database # "MySQL" or "SQLite"
# MySQL: get DB configuration from config file
if self.database == "MySQL":
# update db config
self.dbConfig['socketFileLocation'] = expandvars(
self.dbConfig['socketFileLocation']
)
self.dbConfig.update( dbConfig )
# create DB instance
from ProdCommon.Database.MysqlInstance import MysqlInstance
self.dbInstance = MysqlInstance(self.dbConfig)
else :
# update db config
self.dbConfig = {'dbName':'BossLiteDB'}
dbConfig['dbName'] = expandvars( dbConfig['dbName'] )
self.dbConfig.update( dbConfig )
# create DB instance
from ProdCommon.Database.SqliteInstance import SqliteInstance
self.dbInstance = SqliteInstance(self.dbConfig)
# create a session and db access
self.session = None
##########################################################################
def connect ( self ) :
"""
recreate a session and db access
"""
# create a session and db access
if self.session is None:
self.session = SafeSession(dbInstance = self.dbInstance)
##########################################################################
def close ( self ) :
"""
close session and db access
"""
self.session.close()
self.session = None
##########################################################################
def reset ( self ) :
"""
reset session and db access
"""
self.close()
self.connect()
##########################################################################
def commit ( self ) :
"""
commit
"""
self.session.commit()
##########################################################################
def select(self, query):
"""
execute a query.
"""
# db connect
self.connect()
self.session.execute(query)
# return query results
return self.session.fetchall()
##########################################################################
def selectOne(self, query):
"""
execute a query.with only one result expected
"""
# db connect
self.connect()
self.session.execute(query)
# return query results
return self.session.fetchone()[0]
##########################################################################
def modify(self, query):
"""
execute a query which does not return such as insert/update/delete
"""
# db connect
self.connect()
# return query results
rows = self.session.execute( query )
if self.database == "MySQL":
self.session.commit()
return rows
##########################################################################
def updateDB( self, obj ) :
"""
update any object table in the DB
works for tasks, jobs, runningJobs
"""
# db connect
self.connect()
# update
obj.update(self.session)
if self.database == "MySQL":
self.session.commit()
##########################################################################
def installDB( self, schemaLocation ) :
"""
install database
"""
schemaLocation = expandvars( schemaLocation )
if self.database == "MySQL":
self.installMySQL( schemaLocation )
elif self.database == "SQLite":
self.installSQlite( schemaLocation )
else:
raise NotImplementedError
##########################################################################
def installMySQL( self, schemaLocation ) :
"""
install MySQL database
"""
import getpass
from ProdCommon.Database.MysqlInstance import MysqlInstance
# ask for password (optional)
print
userName = raw_input(
"""
Please provide the mysql user name (typically "root") for updating the
database server (leave empty if not needed): ')
""" )
if userName == '' :
userName = '******'
print
passwd = getpass.getpass(
"""Please provide mysql passwd associated to this user name for
updating the database server:
""" )
# define connection type
from copy import deepcopy
rootConfig = deepcopy( self.dbConfig )
rootConfig.update(
{ 'dbName' : 'mysql', 'user' : userName, 'passwd' : passwd }
)
dbInstance = MysqlInstance( rootConfig )
session = SafeSession( dbInstance = dbInstance )
# check if db exists
create = True
query = "show databases like '" + self.dbConfig['dbName'] + "'"
try:
session.execute( query )
session.commit()
results = session.fetchall()
if results[0][0] == self.dbConfig['dbName'] :
print "DB ", self.dbConfig['dbName'], "already exists."
print "Installing just BossLite tables.\n"
create = False
except IndexError :
pass
except Exception, msg:
session.close()
raise DbError(str(msg))
# create db
if create :
query = 'create database ' + self.dbConfig['dbName']
try:
session.execute( query )
session.commit()
except Exception, msg:
session.close()
raise DbError(str(msg))
def installSQlite( self, schemaLocation ) :
"""
install SQLite database
"""
# create a session and db access
session = SafeSession(dbInstance = self.dbInstance)
# execute check query
query = "select tbl_name from sqlite_master where tbl_name='bl_task'"
try:
# if bl_task exists, no further operations are needed
session.execute( query )
results = session.fetchall()
if results[0][0] == self.dbConfig['dbName'] :
print "DB ", self.dbConfig['dbName'], "already exists.\n"
return
session.close()
return
except IndexError :
pass
except StandardError:
pass
try:
# if bl_task exists, no further operations are needed
session.execute("select count(*) from bl_task")
session.close()
return
except StandardError:
pass
# execute query
queries = open(schemaLocation).read()
try:
for query in queries.split(';') :
session.execute(query)
except Exception, msg:
raise DbError(str(msg))
# update information about task in database
try:
task.update(db)
session.commit
except TaskError, msg:
self.fail("Cannot update task: " + str(msg))
# close session
session.close()
#####################################
# verify operation
#####################################
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
db = TrackingDB(session)
# create template for task
task = Task()
task['name'] = 'task2'
# load task
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
# update all jobs
for job in task.jobs:
if job['standardError'] != 'output.err':
def installSQlite(self, schemaLocation):
"""
install SQLite database
"""
# create a session and db access
session = SafeSession(dbInstance=self.dbInstance)
# execute check query
query = "select tbl_name from sqlite_master where tbl_name='bl_task'"
try:
# if bl_task exists, no further operations are needed
session.execute(query)
results = session.fetchall()
if results[0][0] == self.dbConfig['dbName']:
print "DB ", self.dbConfig['dbName'], "already exists.\n"
return
session.close()
return
except IndexError:
pass
except StandardError:
pass
try:
# if bl_task exists, no further operations are needed
session.execute("select count(*) from bl_task")
session.close()
return
except StandardError:
pass
# execute query
queries = open(schemaLocation).read()
try:
for query in queries.split(';'):
session.execute(query)
except Exception, msg:
raise DbError(str(msg))
# update information about task in database
try:
task.update(db)
session.commit
except TaskError, msg:
self.fail("Cannot update task: " + str(msg))
# close session
session.close()
#####################################
# verify operation
#####################################
# create a session and db access
session = SafeSession(dbInstance = self.dbInstance)
db = TrackingDB(session)
# create template for task
task = Task()
task['name'] = 'task2'
# load task
try:
task.load(db)
except TaskError, msg:
self.fail("Cannot load task: " + str(msg))
# update all jobs
for job in task.jobs:
if job['standardError'] != 'output.err':