def commit(serverUrl=None,method_name=None,componentID=None):
global lastCall
# all other methods that access the database can do that
# with their own private sessions and hence connect imediately
# to the database. Commit is different since if it fails all
# updates in the queues must fail (rollback).
last_session=Session.current_session
try:
Session.connect('ProdMgr')
Session.start_transaction('ProdMgr')
if (serverUrl==None) or (method_name==None) or (componentID==None):
serverUrl,method_name,componentID=lastCall
sqlStr="""UPDATE ws_last_call SET call_state="result_retrieved" WHERE
server_url="%s" AND component_id="%s" AND service_call="%s";
""" %(serverUrl,componentID,method_name)
Session.execute(sqlStr)
Session.commit()
Session.close()
Session.set_session(last_session)
except Exception,ex:
Session.rollback()
Session.close()
Session.set_session(last_session)
raise ProdAgentException("Service commit Error: "+str(ex))
def bulkQueueJobs(listOfSites, *jobSpecDicts):
"""
_bulkQueueJobs_
For a list of jobs all going to the same site(s) add them
to the job queue.
For each job spec a dictionary should be provided containing:
"JobSpecId"
"JobSpecFile"
"JobType"
"WorkflowSpecId"
"WorkflowPriority"
A list of site names or se names should be provided.
All jobs will be queued for that list of sites
"""
try:
Session.connect()
Session.start_transaction()
jobQ = JobQueueDB()
#jobQ.loadSiteMatchData()
jobQ.insertJobSpecsForSites(listOfSites, *jobSpecDicts)
logging.info("Job List Queued for sites: %s" % listOfSites)
Session.commit_all()
Session.close_all()
except Exception, ex:
msg = "Failed to queue JobSpecs:\n"
msg += str(ex)
logging.error(msg)
Session.rollback()
Session.close_all()
def dropMode():
"""
_dropMode_
Remove a site from the ResourceControlDB
"""
if site == None:
msg = "--site option not provided"
raise RuntimeError, msg
msg = "Dropping Site named: %s\n" % site
Session.set_database(dbConfig)
Session.connect()
Session.start_transaction()
resCon = ResourceControlDB()
try:
resCon.dropSite(site)
Session.commit_all()
Session.close_all()
except Exception, ex:
msg += "Error dropping site:\n"
msg += str(ex)
Session.rollback()
Session.close_all()
print msg
sys.exit(1)
def wrapperFuction(*args, **dictArgs):
"""
_wrapperFuction_
"""
try:
Session.connect()
Session.start_transaction()
reValue = dbfunc(*args, **dictArgs)
Session.commit_all()
Session.close_all()
return reValue
except Exception, ex:
msg = "Error: %s\n" % str(ex)
logging.error(msg)
Session.rollback()
Session.close_all()
def removeHoldByWorkflow(workflowID):
"""
_removeHoldByWorkflow_
Change the status of all jobs in the JobQueue with a particular workflow ID
from "held" to "new" so that they will eventually be released.
"""
try:
Session.connect()
Session.start_transaction()
jobQ = JobQueueDB()
jobQ.removeHoldForWorkflow(workflowID)
Session.commit_all()
Session.close_all()
except Exception, ex:
msg = "Failed to remove jobs for workflow %s." % workflowID
msg += str(ex)
logging.error(msg)
Session.rollback()
Session.close_all()
def logCall(serverUrl,method_name,args,componentID="defaultComponent",tag="0"):
global lastCall
last_session=Session.current_session
try:
Session.connect('ProdMgr')
Session.start_transaction('ProdMgr')
sqlStr="""INSERT INTO ws_last_call(server_url,component_id,service_call,service_parameters,call_state,tag)
VALUES("%s","%s","%s","%s","%s","%s") ON DUPLICATE KEY UPDATE
service_parameters="%s", call_state="%s", tag="%s";
""" %(serverUrl,componentID,method_name,base64.encodestring(cPickle.dumps(args)),"call_placed",str(tag),base64.encodestring(cPickle.dumps(args)),"call_placed",tag)
Session.execute(sqlStr)
Session.commit()
lastCall=(serverUrl,method_name,componentID)
Session.close()
Session.set_session(last_session)
except Exception,ex:
Session.rollback()
Session.close()
Session.set_session(last_session)
raise ProdAgentException("Service logging Error: "+str(ex))
def newMode():
"""
_newMode_
Add a new site with some standard default thresholds
"""
if site == None:
msg = "--site option not provided"
raise RuntimeError, msg
if ceName == None:
msg = "--ce-name option not provided. Warning, this is not supported"
msg += " by all ResourceMonitor / Submitter combinations\n"
print(msg)
if seName == None:
msg = "--se-name option not provided"
raise RuntimeError, msg
msg = "Adding New Site named: %s\n" % site
Session.set_database(dbConfig)
Session.connect()
Session.start_transaction()
active = True
if deactivate != None:
active = False
resCon = ResourceControlDB()
try:
siteIndex = resCon.newSite(site, seName, ceName, active)
except Exception, ex:
msg += "Error adding new site:\n%s\n" % str(ex)
Session.rollback()
Session.close_all()
print msg
sys.exit(1)
def retrieve(serverURL=None,method_name=None,componentID=None):
last_session=Session.current_session
try:
Session.connect('ProdMgr')
Session.start_transaction('ProdMgr')
#NOTE: we do several nested queries and assume that the query engine can rewrite them
#NOTE: we should rewrite these queries ourselves.
if serverURL==None and method_name==None and componentID==None:
sqlStr="""SELECT server_url,service_call,component_id,tag FROM ws_last_call WHERE
call_state="call_placed" AND
id in (
SELECT max(id)
FROM ws_last_call
WHERE call_state="call_placed"
AND
log_time IN (
SELECT max(log_time) FROM ws_last_call
WHERE call_state="call_placed" GROUP BY server_url) GROUP BY server_url);
"""
elif serverURL==None and method_name==None and componentID!=None:
sqlStr="""SELECT server_url,service_call,component_id,tag FROM ws_last_call WHERE
component_id="%s" AND call_state="call_placed" AND
id in (
SELECT max(id)
FROM ws_last_call
WHERE component_id="%s" AND call_state="call_placed"
AND log_time IN (
SELECT max(log_time) FROM ws_last_call
WHERE component_id="%s" AND call_state="call_placed" GROUP BY server_url) GROUP BY server_url);
""" %(componentID,componentID,componentID)
elif serverURL==None and method_name!=None and componentID!=None:
sqlStr="""SELECT server_url,service_call,component_id,tag FROM ws_last_call WHERE
component_id="%s" AND service_call="%s" AND call_state="call_placed" AND
id in (
SELECT max(id)
FROM ws_last_call
WHERE component_id="%s" AND service_call="%s" AND call_state="call_placed"
AND log_time IN (
SELECT max(log_time) FROM ws_last_call
WHERE component_id="%s" AND service_call="%s" AND call_state="call_placed" GROUP BY server_url) GROUP BY server_url;
""" %(componentID,method_name,componentID,method_name,componentID,method_name)
elif serverURL!=None and method_name==None and componentID!=None:
sqlStr="""SELECT server_url,service_call,component_id,tag FROM ws_last_call WHERE
component_id="%s" AND server_url="%s" AND call_state="call_placed" AND
id in (
SELECT max(id)
FROM ws_last_call
WHERE component_id="%s" AND server_url="%s" AND call_state="call_placed"
AND log_time IN (
SELECT max(log_time) FROM ws_last_call
WHERE component_id="%s" AND server_url="%s" AND call_state="call_placed" GROUP BY server_url) GROUP BY server_url);
""" %(componentID,serverURL,componentID,serverURL,componentID,serverURL)
elif serverURL!=None and method_name!=None and componentID!=None:
sqlStr="""SELECT server_url,service_call,component_id,tag FROM ws_last_call WHERE
component_id="%s" AND server_url="%s" AND call_state="call_placed" AND service_call="%s"
""" %(componentID,serverURL,method_name)
dbCur.execute(sqlStr)
rows=dbCur.fetchall()
if len(rows)==0:
raise ProdAgentException("No result in local last service call table with componentID :"+\
str(componentID),3000)
server_url=rows[0][0]
service_call=rows[0][1]
component_id=rows[0][2]
tag=rows[0][3]
Session.execute(sqlStr)
Session.commit()
Session.close()
Session.set_session(last_session)
return [server_url,service_call,component_id,tag]
except ProdAgentException:
Session.rollback()
Session.close()
Session.set_session(last_session)
raise
except Exception,ex:
Session.rollback()
Session.close()
Session.set_session(last_session)
raise ProdAgentException("Service commit Error: "+str(ex),3001)
priority = priorityMap.get(jobType, 1)
try:
Session.connect()
Session.start_transaction()
jobQ = JobQueueDB()
jobQ.insertJobSpec(jobSpecId, jobSpecFile, jobType, workflow,
priority, sitesList, status)
logging.info("Job %s Queued with priority %s" % (jobSpecId, priority))
Session.commit_all()
Session.close_all()
except Exception, ex:
msg = "Failed to queue JobSpec:\n%s\n" % jobSpecFile
msg += str(ex)
logging.error(msg)
Session.rollback()
Session.close_all()
return
def bulkQueueJobs(listOfSites, *jobSpecDicts):
"""
_bulkQueueJobs_
For a list of jobs all going to the same site(s) add them
to the job queue.
For each job spec a dictionary should be provided containing:
"JobSpecId"
"JobSpecFile"
"JobType"