def updateEventRanges(self, event_ranges):
""" Update an event range on the Event Server """
pUtil.tolog("Updating event ranges..")
message = ""
#url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
# eventRanges = [{'eventRangeID': '4001396-1800223966-4426028-1-2', 'eventStatus':'running'}, {'eventRangeID': '4001396-1800223966-4426028-2-2','eventStatus':'running'}]
node={}
node['eventRanges']=json.dumps(event_ranges)
# open connection
ret = pUtil.httpConnect(node, url, path=self.__updateEventRangesDir, mode="UPDATEEVENTRANGES")
# response = json.loads(ret[1])
status = ret[0]
if ret[0]: # non-zero return code
message = "Failed to update event range - error code = %d, error: " % (ret[0], ret[1])
else:
response = json.loads(json.dumps(ret[1]))
status = int(response['StatusCode'])
message = json.dumps(response['Returns'])
return status, message
def downloadEventRanges(jobId, jobsetID):
""" Download event ranges from the Event Server """
# Return the server response (instruction to AthenaMP)
# Note: the returned message is a string (of a list of dictionaries). If it needs to be converted back to a list, use json.loads(message)
tolog("Downloading new event ranges for jobId=%s and jobsetID=%s" % (jobId, jobsetID))
# message = "[{u'lastEvent': 2, u'LFN': u'mu_E50_eta0-25.evgen.pool.root',u'eventRangeID': u'130-2068634812-21368-1-1', u'startEvent': 2, u'GUID':u'74DFB3ED-DAA7-E011-8954-001E4F3D9CB1'}]"
message = ""
# url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['pandaID'] = jobId
node['jobsetID'] = jobsetID
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="GETEVENTRANGES")
response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to download event range - error code = %d" % (ret[0])
else:
message = response['eventRanges']
if message == "" or message == "[]":
message = "No more events"
return message
def updateEventRange(self, event_range_id, status='finished'):
""" Update an event range on the Event Server """
pUtil.tolog("Updating an event range..")
message = ""
# url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['eventRangeID'] = event_range_id
# node['cpu'] = eventRangeList[1]
# node['wall'] = eventRangeList[2]
node['eventStatus'] = status
# tolog("node = %s" % str(node))
# open connection
ret = pUtil.httpConnect(node, url, path=self.__updateEventRangesDir, mode="UPDATEEVENTRANGE")
# response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to update event range - error code = %d" % (ret[0])
else:
message = ""
return ret[0], message
def downloadEventRanges(jobId, jobsetID, taskID, numRanges=10):
""" Download event ranges from the Event Server """
# Return the server response (instruction to AthenaMP)
# Note: the returned message is a string (of a list of dictionaries). If it needs to be converted back to a list, use json.loads(message)
tolog(
"Downloading new event ranges for jobId=%s, taskID=%s and jobsetID=%s"
% (jobId, taskID, jobsetID))
# message = "[{u'lastEvent': 2, u'LFN': u'mu_E50_eta0-25.evgen.pool.root',u'eventRangeID': u'130-2068634812-21368-1-1', u'startEvent': 2, u'GUID':u'74DFB3ED-DAA7-E011-8954-001E4F3D9CB1'}]"
message = ""
# url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['pandaID'] = jobId
node['jobsetID'] = jobsetID
node['taskID'] = taskID
node['nRanges'] = numRanges
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="GETEVENTRANGES")
response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to download event range - error code = %d" % (ret[0])
else:
message = response['eventRanges']
if message == "" or message == "[]":
message = "No more events"
return message
def getEventRanges(self, numRanges=2):
""" Download event ranges from the Event Server """
tolog("Server: Downloading new event ranges..")
message = ""
# url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['pandaID'] = self.__job.jobId
node['jobsetID'] = self.__job.jobsetID
node['nRanges'] = numRanges
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="GETEVENTRANGES")
response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to download event range - error code = %d" % (
ret[0])
tolog(message)
return []
else:
message = response['eventRanges']
return json.loads(message)
def updateEventRange(event_range_id, eventRangeList, status='finished'):
""" Update an event range on the Event Server """
tolog("Updating an event range..")
# PanDA dev server: url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['eventRangeID'] = event_range_id
if eventRangeList != []:
pass
# node['cpu'] = eventRangeList[1]
# node['wall'] = eventRangeList[2]
node['eventStatus'] = status
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="UPDATEEVENTRANGE")
# response = ret[1]
if ret[0]: # non-zero return code
message = "Server responded with error code = %d" % (ret[0])
else:
message = ""
return message
def updateEventRanges(event_ranges, pandaProxySecretKey=None, jobId=None, url="https://pandaserver.cern.ch:25443/server/panda", version=0):
""" Update an event range on the Event Server """
tolog("Updating event ranges...")
try:
if pandaProxySecretKey is not None and pandaProxySecretKey != "" :
return updateEventRangesPandaProxy(event_ranges, pandaProxySecretKey, jobId)
message = ""
# eventRanges = [{'eventRangeID': '4001396-1800223966-4426028-1-2', 'eventStatus':'running'}, {'eventRangeID': '4001396-1800223966-4426028-2-2','eventStatus':'running'}]
node={}
node['eventRanges']=json.dumps(event_ranges)
if version:
node['version'] = 1
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="UPDATEEVENTRANGES")
# response = json.loads(ret[1])
status = ret[0]
if ret[0]: # non-zero return code
message = "Failed to update event range - error code = %d, error: %s" % (ret[0], ret[1])
else:
response = json.loads(json.dumps(ret[1]))
status = int(response['StatusCode'])
message = json.dumps(response['Returns'])
return status, message
except:
tolog("Failed to update event ranges: %s" % traceback.format_exc())
return -1, None
def updateEventRange(event_range_id, eventRangeList, jobId, status='finished', os_bucket_id=-1):
""" Update an event range on the Event Server """
tolog("Updating an event range..")
# PanDA dev server: url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['eventRangeID'] = event_range_id
if os_bucket_id != -1:
node['objstoreID'] = os_bucket_id
if eventRangeList != []:
pass
# node['cpu'] = eventRangeList[1]
# node['wall'] = eventRangeList[2]
node['eventStatus'] = status
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="UPDATEEVENTRANGE")
# response = ret[1]
if ret[0]: # non-zero return code
message = "Server responded with error code = %d" % (ret[0])
else:
# is there an instruction in the back channel?
data = ret[1]
tolog("data=%s"%str(data))
from json import loads
try:
d = loads(data['Command'])
except Exception, e:
tolog("No message found in updateEventRange back channel: %s" % (e))
message = ""
else:
def downloadEventRanges(jobId,
jobsetID,
taskID,
numRanges=10,
url="https://pandaserver.cern.ch:25443/server/panda"):
""" Download event ranges from the Event Server """
try:
# url should be '%s:%s/server/panda' % (env['pshttpurl'], str(env['psport']))
if os.environ.has_key('EventRanges') and os.path.exists(
os.environ['EventRanges']):
try:
with open(os.environ['EventRanges']) as json_file:
events = json.load(json_file)
os.rename(os.environ['EventRanges'],
os.environ['EventRanges'] + ".loaded")
tolog(events)
return json.dumps(events)
except:
tolog('Failed to open event ranges json file: %s' %
traceback.format_exc())
# Return the server response (instruction to AthenaMP)
# Note: the returned message is a string (of a list of dictionaries). If it needs to be converted back to a list, use json.loads(message)
tolog(
"Downloading new event ranges for jobId=%s, taskID=%s and jobsetID=%s"
% (jobId, taskID, jobsetID))
# message = "[{u'lastEvent': 2, u'LFN': u'mu_E50_eta0-25.evgen.pool.root',u'eventRangeID': u'130-2068634812-21368-1-1', u'startEvent': 2, u'GUID':u'74DFB3ED-DAA7-E011-8954-001E4F3D9CB1'}]"
if numRanges < 8:
numRanges = 8
message = ""
node = {}
node['pandaID'] = jobId
node['jobsetID'] = jobsetID
node['taskID'] = taskID
node['nRanges'] = numRanges
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="GETEVENTRANGES")
response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to download event range - error code = %d" % (
ret[0])
else:
message = response['eventRanges']
if message == "" or message == "[]":
message = "No more events"
return message
except:
tolog("Failed to download event ranges: %s" % traceback.format_exc())
return None
def getSecurityKey(self, privateKeyName, publicKeyName):
""" Return the key pair """
keyName = privateKeyName + "_" + publicKeyName
if keyName in self.__securityKeys.keys():
return self.__securityKeys[keyName]
else:
try:
node={}
node['privateKeyName'] = privateKeyName
node['publicKeyName'] = publicKeyName
#host = '%s:%s' % (env['pshttpurl'], str(env['psport'])) # The key pair is not set on other panda server
url = 'https://pandaserver.cern.ch:25443/server/panda'
ret = httpConnect(node, url, mode = "GETKEYPAIR", path=os.getcwd())
StatusCode = str(ret[0])
data = ret[1] # dictionary
response = ret[2] # text
if StatusCode == "0":
self.__securityKeys[keyName] = {"publicKey": data["publicKey"], "privateKey": data["privateKey"]}
return self.__securityKeys[keyName]
tolog("!!WARNING!!4444!! Failed to get key from PanDA server:")
tolog("data = %s" % str(data))
except:
_type, value, traceBack = sys.exc_info()
tolog("!!WARNING!!4445!! Failed to getKeyPair for (%s, %s)" % (privateKeyName, publicKeyName))
tolog("ERROR: %s %s" % (_type, value))
tolog("Try to use requests to get key pair")
try:
sslCert = self.getSSLCertificate()
sslKey = sslCert
host = 'pandaserver.cern.ch:25443'
path = '/server/panda/getKeyPair'
import requests
r = requests.post('https://%s%s' % (host, path),
verify=False,
cert=(sslCert, sslKey),
data=urllib.urlencode(node),
timeout=120)
if r and r.status_code == 200:
dic = cgi.parse_qs(r.text)
if dic["StatusCode"][0] == "0":
self.__securityKeys[keyName] = {"publicKey": dic["publicKey"][0], "privateKey": dic["privateKey"][0]}
return self.__securityKeys[keyName]
except:
_type, value, traceBack = sys.exc_info()
tolog("!!WARNING!!4445!! Failed to getKeyPair for (%s, %s)" % (privateKeyName, publicKeyName))
tolog("ERROR: %s %s" % (_type, value))
return {"publicKey": None, "privateKey": None}
def getSecurityKey(self, privateKeyName, publicKeyName):
""" Return the key pair """
keyName = privateKeyName + "_" + publicKeyName
if keyName in self.__securityKeys.keys():
return self.__securityKeys[keyName]
else:
try:
node={}
node['privateKeyName'] = privateKeyName
node['publicKeyName'] = publicKeyName
#host = '%s:%s' % (env['pshttpurl'], str(env['psport'])) # The key pair is not set on other panda server
url = 'https://pandaserver.cern.ch:25443/server/panda'
ret = httpConnect(node, url, mode = "GETKEYPAIR", path=os.getcwd())
StatusCode = str(ret[0])
data = ret[1] # dictionary
response = ret[2] # text
if StatusCode == "0":
self.__securityKeys[keyName] = {"publicKey": data["publicKey"], "privateKey": data["privateKey"]}
return self.__securityKeys[keyName]
tolog("!!WARNING!!4444!! Failed to get key from PanDA server:")
tolog("data = %s" % str(data))
except:
_type, value, traceBack = sys.exc_info()
tolog("!!WARNING!!4445!! Failed to getKeyPair for (%s, %s)" % (privateKeyName, publicKeyName))
tolog("ERROR: %s %s" % (_type, value))
tolog("Try to use requests to get key pair")
try:
sslCert = self.getSSLCertificate()
sslKey = sslCert
host = 'pandaserver.cern.ch:25443'
path = '/server/panda/getKeyPair'
import requests
r = requests.post('https://%s%s' % (host, path),
verify=False,
cert=(sslCert, sslKey),
data=urllib.urlencode(node),
timeout=120)
if r and r.status_code == 200:
dic = cgi.parse_qs(r.text)
if dic["StatusCode"][0] == "0":
self.__securityKeys[keyName] = {"publicKey": dic["publicKey"][0], "privateKey": dic["privateKey"][0]}
return self.__securityKeys[keyName]
except:
_type, value, traceBack = sys.exc_info()
tolog("!!WARNING!!4445!! Failed to getKeyPair for (%s, %s)" % (privateKeyName, publicKeyName))
tolog("ERROR: %s %s" % (_type, value))
return {"publicKey": None, "privateKey": None}
def getNewJob(self, tofile=True):
try:
_maxinputsize = pUtil.getMaxInputSize(MB=True)
_disk = self.__node.disk
pUtil.tolog("Available WN disk space: %d MB" % (_disk))
_diskSpace = min(_disk, _maxinputsize)
pUtil.tolog("Sending disk space %d MB to dispatcher" % (_diskSpace))
# construct a dictionary for passing to jobDispatcher and get the prodSourceLabel
jNode, prodSourceLabel, pilotErrorDiag = self.getDispatcherDictionary(_diskSpace, tofile)
if jNode == {}:
errorText = "!!FAILED!!1200!! %s" % (pilotErrorDiag)
pUtil.tolog(errorText, tofile=tofile)
# send to stderr
print >> sys.stderr, errorText
return None, None, pilotErrorDiag
# get a random server
url = '%s:%s/server/panda' % (self.__env['pshttpurl'], str(self.__env['psport']))
pUtil.tolog("Looking for a primary job (contacting server at %s)" % (url), tofile=tofile)
# make http connection to jobdispatcher
# format: status, parsed response (data), response
ret = pUtil.httpConnect(jNode, url, mode = "GETJOB", path = self.__pilotWorkingDir, experiment = self.__thisExperiment) # connection mode is GETJOB
# get and write the dispatcher status code to file
StatusCode = str(ret[0])
# the original response will be put in a file in this function
data = ret[1] # dictionary
response = ret[2] # text
# write the dispatcher exit code to file
self.writeDispatcherEC(StatusCode)
if ret[0]: # non-zero return
return None, None, pUtil.getDispatcherErrorDiag(ret[0])
if StatusCode != '0':
pilotErrorDiag = "No job received from jobDispatcher, StatusCode: %s" % (StatusCode)
pUtil.tolog("%s" % (pilotErrorDiag), tofile=tofile)
return None, None, pilotErrorDiag
# test if he attempt number was sent
try:
attemptNr = int(data['attemptNr'])
except Exception,e:
pUtil.tolog("!!WARNING!!1200!! Failed to get attempt number from server: %s" % str(e), tofile=tofile)
else:
def downloadEventRanges(jobId, jobsetID, taskID, pandaProxySecretKey=None, numRanges=10, url="https://pandaserver.cern.ch:25443/server/panda"):
""" Download event ranges from the Event Server """
try:
# url should be '%s:%s/server/panda' % (env['pshttpurl'], str(env['psport']))
if os.environ.has_key('EventRanges') and os.path.exists(os.environ['EventRanges']):
try:
with open(os.environ['EventRanges']) as json_file:
events = json.load(json_file)
os.rename(os.environ['EventRanges'], os.environ['EventRanges'] + ".loaded")
tolog(events)
return json.dumps(events)
except:
tolog('Failed to open event ranges json file: %s' % traceback.format_exc())
# Return the server response (instruction to AthenaMP)
# Note: the returned message is a string (of a list of dictionaries). If it needs to be converted back to a list, use json.loads(message)
tolog("Downloading new event ranges for jobId=%s, taskID=%s and jobsetID=%s" % (jobId, taskID, jobsetID))
if pandaProxySecretKey is not None and pandaProxySecretKey != "" :
return downloadEventRangesPandaProxy(jobId, jobsetID, pandaProxySecretKey)
# message = "[{u'lastEvent': 2, u'LFN': u'mu_E50_eta0-25.evgen.pool.root',u'eventRangeID': u'130-2068634812-21368-1-1', u'startEvent': 2, u'GUID':u'74DFB3ED-DAA7-E011-8954-001E4F3D9CB1'}]"
message = ""
node = {}
node['pandaID'] = jobId
node['jobsetID'] = jobsetID
node['taskID'] = taskID
node['nRanges'] = numRanges
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="GETEVENTRANGES")
response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to download event range - error code = %d" % (ret[0])
else:
message = response['eventRanges']
if message == "" or message == "[]":
message = "No more events"
return message
except Exception, e:
tolog("Failed to download event ranges: %s: %s" % (str(e), traceback.format_exc()))
def updateEventRange(event_range_id,
eventRangeList,
jobId,
status='finished',
os_bucket_id=-1):
""" Update an list of event ranges on the Event Server """
tolog("Updating an event range..")
# PanDA dev server: url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['eventRangeID'] = event_range_id
if os_bucket_id != -1:
node['objstoreID'] = os_bucket_id
if eventRangeList != []:
pass
# node['cpu'] = eventRangeList[1]
# node['wall'] = eventRangeList[2]
node['eventStatus'] = status
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="UPDATEEVENTRANGE")
# response = ret[1]
if ret[0]: # non-zero return code
message = "Server responded with error code = %d" % (ret[0])
else:
# is there an instruction in the back channel?
data = ret[1]
tolog("data=%s" % str(data))
from json import loads
try:
d = loads(data['Command'])
except Exception, e:
tolog("No message found in updateEventRange back channel: %s" %
(e))
message = ""
else:
def getEventRanges(self, numRanges=2):
""" Download event ranges from the Event Server """
tolog("Server: Downloading new event ranges..")
message = ""
# url = "https://aipanda007.cern.ch:25443/server/panda"
url = "https://pandaserver.cern.ch:25443/server/panda"
node = {}
node['pandaID'] = self.__job.jobId
node['jobsetID'] = self.__job.jobsetID
node['nRanges'] = numRanges
# open connection
ret = httpConnect(node, url, path=os.getcwd(), mode="GETEVENTRANGES")
response = ret[1]
if ret[0]: # non-zero return code
message = "Failed to download event range - error code = %d" % (ret[0])
tolog(message)
return []
else:
message = response['eventRanges']
return json.loads(message)
def getNewJob(self, tofile=True, nJobs=1):
try:
_maxinputsize = pUtil.getMaxInputSize(MB=True)
_disk = self.__node.disk
pUtil.tolog("Available WN disk space: %d MB" % (_disk))
_diskSpace = min(_disk, _maxinputsize)
pUtil.tolog("Sending disk space %d MB to dispatcher" % (_diskSpace))
# construct a dictionary for passing to jobDispatcher and get the prodSourceLabel
jNode, prodSourceLabel, pilotErrorDiag = self.getDispatcherDictionary(_diskSpace, tofile, nJobs)
if jNode == {}:
errorText = "!!FAILED!!1200!! %s" % (pilotErrorDiag)
pUtil.tolog(errorText, tofile=tofile)
# send to stderr
print >> sys.stderr, errorText
return None, None, pilotErrorDiag
# get a random server
url = '%s:%s/server/panda' % (self.__env['pshttpurl'], str(self.__env['psport']))
pUtil.tolog("Looking for a primary job (contacting server at %s)" % (url), tofile=tofile)
# make http connection to jobdispatcher
# format: status, parsed response (data), response
ret = pUtil.httpConnect(jNode, url, mode = "GETJOB", path = self.__pilotWorkingDir, experiment = self.__thisExperiment) # connection mode is GETJOB
# get and write the dispatcher status code to file
StatusCode = str(ret[0])
# the original response will be put in a file in this function
data = ret[1] # dictionary
response = ret[2] # text
# write the dispatcher exit code to file
self.writeDispatcherEC(StatusCode)
if ret[0]: # non-zero return
return None, None, pUtil.getDispatcherErrorDiag(ret[0])
if StatusCode != '0':
pilotErrorDiag = "No job received from jobDispatcher, StatusCode: %s" % (StatusCode)
pUtil.tolog("%s" % (pilotErrorDiag), tofile=tofile)
return None, None, pilotErrorDiag
# backup response (will be copied to workdir later)
self.backupDispatcherResponse(response, tofile)
if not data.has_key("jobs"):
jobs = [data]
else:
jobs = data['jobs']
newJobs = []
newJobsData = {}
for job in jobs:
# test if he attempt number was sent
try:
attemptNr = int(job['attemptNr'])
except Exception,e:
pUtil.tolog("!!WARNING!!1200!! Failed to get attempt number from server: %s" % str(e), tofile=tofile)
else:
pUtil.tolog("Attempt number from server: %d" % attemptNr)
# should there be a delay before setting running state?
try:
nSent = int(job['nSent'])
except Exception,e:
nSent = 0
else:
