def test_resubmit(self):
self.usercfg.Broker("TEST")
arc.TargetInformationRetrieverPluginTESTControl.targets = [
self.create_test_target("http://test2.nordugrid.org")
]
arc.TargetInformationRetrieverPluginTESTControl.status = arc.EndpointQueryingStatus(
arc.EndpointQueryingStatus.SUCCESSFUL)
js = arc.JobSupervisor(self.usercfg, [
self.create_test_job(
job_id="http://test.nordugrid.org/1234567890test1",
state=arc.JobState.FAILED),
self.create_test_job(
job_id="http://test.nordugrid.org/1234567890test2",
state=arc.JobState.RUNNING)
])
self.expect(js.GetAllJobs()).to_have(2).jobs()
endpoints = [
arc.Endpoint("http://test2.nordugrid.org",
arc.Endpoint.COMPUTINGINFO, "org.nordugrid.tirtest")
]
resubmitted = arc.JobList()
result = js.Resubmit(0, endpoints, resubmitted)
def test_add_job(self):
js = arc.JobSupervisor(self.usercfg, arc.JobList())
self.expect(js.GetAllJobs()).to_be_empty()
job = self.create_test_job(
job_id="http://test.nordugrid.org/1234567890test1")
self.expect(js.AddJob(job)).to_be(
True, message="AddJob was expected to return True")
self.expect(js.GetAllJobs()).not_to_be_empty()
job.JobManagementInterfaceName = ""
self.expect(js.AddJob(job)).to_be(
False, message="AddJob was expected to return False")
self.expect(js.GetAllJobs()).to_have(1).job()
job.JobManagementInterfaceName = "non.existent.interface"
self.expect(js.AddJob(job)).to_be(
False, message="AddJob was expected to return False")
self.expect(js.GetAllJobs()).to_have(1).job()
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ):
""" Method to submit job
"""
# Assume that the ARC queues are always of the format nordugrid-<batchSystem>-<queue>
# And none of our supported batch systems have a "-" in their name
self.arcQueue = self.queue.split("-",2)[2]
result = self._prepareProxy()
self.usercfg.ProxyPath(os.environ['X509_USER_PROXY'])
if not result['OK']:
gLogger.error( 'ARCComputingElement: failed to set up proxy', result['Message'] )
return result
gLogger.verbose( "Executable file path: %s" % executableFile )
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH + stat.S_IXOTH )
batchIDList = []
stampDict = {}
endpoint = arc.Endpoint( self.ceHost + ":2811/jobs", arc.Endpoint.JOBSUBMIT,
"org.nordugrid.gridftpjob")
# Submit jobs iteratively for now. Tentatively easier than mucking around with the JobSupervisor class
for __i in range(numberOfJobs):
# The basic job description
jobdescs = arc.JobDescriptionList()
# Get the job into the ARC way
xrslString, diracStamp = self.__writeXRSL( executableFile )
gLogger.debug("XRSL string submitted : %s" %xrslString)
gLogger.debug("DIRAC stamp for job : %s" %diracStamp)
if not arc.JobDescription_Parse(xrslString, jobdescs):
gLogger.error("Invalid job description")
break
# Submit the job
jobs = arc.JobList() # filled by the submit process
submitter = arc.Submitter(self.usercfg)
result = submitter.Submit(endpoint, jobdescs, jobs)
# Save info or else ..else.
if ( result == arc.SubmissionStatus.NONE ):
# Job successfully submitted
pilotJobReference = jobs[0].JobID
batchIDList.append( pilotJobReference )
stampDict[pilotJobReference] = diracStamp
gLogger.debug("Successfully submitted job %s to CE %s" % (pilotJobReference, self.ceHost))
else:
message = "Failed to submit job because "
if (result.isSet(arc.SubmissionStatus.NOT_IMPLEMENTED) ):
gLogger.warn( "%s feature not implemented on CE? (weird I know - complain to site admins" % message )
if ( result.isSet(arc.SubmissionStatus.NO_SERVICES) ):
gLogger.warn( "%s no services are running on CE? (open GGUS ticket to site admins" % message )
if ( result.isSet(arc.SubmissionStatus.ENDPOINT_NOT_QUERIED) ):
gLogger.warn( "%s endpoint was not even queried. (network ..?)" % message )
if ( result.isSet(arc.SubmissionStatus.BROKER_PLUGIN_NOT_LOADED) ):
gLogger.warn( "%s BROKER_PLUGIN_NOT_LOADED : ARC library installation problem?" % message )
if ( result.isSet(arc.SubmissionStatus.DESCRIPTION_NOT_SUBMITTED) ):
gLogger.warn( "%s Job not submitted - incorrect job description? (missing field in XRSL string?)" % message )
if ( result.isSet(arc.SubmissionStatus.SUBMITTER_PLUGIN_NOT_LOADED) ):
gLogger.warn( "%s SUBMITTER_PLUGIN_NOT_LOADED : ARC library installation problem?" % message )
if ( result.isSet(arc.SubmissionStatus.AUTHENTICATION_ERROR) ):
gLogger.warn( "%s authentication error - screwed up / expired proxy? Renew / upload pilot proxy on machine?" % message )
if ( result.isSet(arc.SubmissionStatus.ERROR_FROM_ENDPOINT) ):
gLogger.warn( "%s some error from the CE - possibly CE problems?" % message )
gLogger.warn( "%s ... maybe above messages will give a hint." % message )
break # Boo hoo *sniff*
if batchIDList:
result = S_OK( batchIDList )
result['PilotStampDict'] = stampDict
else:
result = S_ERROR('No pilot references obtained from the ARC job submission')
return result
''' This regression test tests whether invoking the next method on a Swig iterator obtained from a std::list of ARC C++ objects generates a segmentation fault. That issue was reported in bug 2683. ''' import arc jobs = arc.JobList() jobs.push_back(arc.Job()) itJobs = jobs.__iter__() next(itJobs)
def submitJob(self,
executableFile,
proxy,
numberOfJobs=1,
inputs=None,
outputs=None):
"""Method to submit job"""
# Assume that the ARC queues are always of the format nordugrid-<batchSystem>-<queue>
# And none of our supported batch systems have a "-" in their name
self.arcQueue = self.queue.split("-", 2)[2]
result = self._prepareProxy()
if not result["OK"]:
self.log.error("ARCComputingElement: failed to set up proxy",
result["Message"])
return result
self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"])
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH + stat.S_IXOTH)
executables = None
if self.preamble:
executables = [executableFile]
executableFile = self._bundlePreamble(executableFile)
batchIDList = []
stampDict = {}
if self.endpointType == "Gridftp":
endpoint = arc.Endpoint(str(self.ceHost + ":2811/jobs"),
arc.Endpoint.JOBSUBMIT,
"org.nordugrid.gridftpjob")
else:
endpoint = arc.Endpoint(
str("https://" + self.ceHost + ":8443/arex"),
arc.Endpoint.JOBSUBMIT,
"org.ogf.glue.emies.activitycreation",
)
# Submit jobs iteratively for now. Tentatively easier than mucking around with the JobSupervisor class
for __i in range(numberOfJobs):
# The basic job description
jobdescs = arc.JobDescriptionList()
# Get the job into the ARC way
xrslString, diracStamp = self._writeXRSL(executableFile, inputs,
outputs, executables)
self.log.debug("XRSL string submitted : %s" % xrslString)
self.log.debug("DIRAC stamp for job : %s" % diracStamp)
# The arc bindings don't accept unicode objects in Python 2 so xrslString must be explicitly cast
result = arc.JobDescription_Parse(str(xrslString), jobdescs)
if not result:
self.log.error("Invalid job description",
"%r, message=%s" % (xrslString, result.str()))
break
# Submit the job
jobs = arc.JobList() # filled by the submit process
submitter = arc.Submitter(self.usercfg)
result = submitter.Submit(endpoint, jobdescs, jobs)
# Save info or else ..else.
if result == arc.SubmissionStatus.NONE:
# Job successfully submitted
pilotJobReference = jobs[0].JobID
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
self.log.debug("Successfully submitted job %s to CE %s" %
(pilotJobReference, self.ceHost))
else:
self._analyzeSubmissionError(result)
break # Boo hoo *sniff*
if self.preamble:
os.unlink(executableFile)
if batchIDList:
result = S_OK(batchIDList)
result["PilotStampDict"] = stampDict
else:
result = S_ERROR(
"No pilot references obtained from the ARC job submission")
return result
def submitJob(self,
executableFile,
proxy,
numberOfJobs=1,
inputs=None,
outputs=None):
"""Method to submit job"""
# Assume that the ARC queues are always of the format nordugrid-<batchSystem>-<queue>
# And none of our supported batch systems have a "-" in their name
self.arcQueue = self.queue.split("-", 2)[2]
result = self._prepareProxy()
if not result["OK"]:
self.log.error("ARCComputingElement: failed to set up proxy",
result["Message"])
return result
self.usercfg.ProxyPath(os.environ["X509_USER_PROXY"])
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH + stat.S_IXOTH)
executables = None
if self.preamble:
executables = [executableFile]
executableFile = self._bundlePreamble(executableFile)
batchIDList = []
stampDict = {}
if self.endpointType == "Gridftp":
endpoint = arc.Endpoint(str(self.ceHost + ":2811/jobs"),
arc.Endpoint.JOBSUBMIT,
"org.nordugrid.gridftpjob")
else:
endpoint = arc.Endpoint(
str("https://" + self.ceHost + ":8443/arex"),
arc.Endpoint.JOBSUBMIT,
"org.ogf.glue.emies.activitycreation",
)
# Submit jobs iteratively for now. Tentatively easier than mucking around with the JobSupervisor class
for __i in range(numberOfJobs):
# The basic job description
jobdescs = arc.JobDescriptionList()
# Get the job into the ARC way
xrslString, diracStamp = self.__writeXRSL(executableFile, inputs,
outputs, executables)
self.log.debug("XRSL string submitted : %s" % xrslString)
self.log.debug("DIRAC stamp for job : %s" % diracStamp)
# The arc bindings don't accept unicode objects in Python 2 so xrslString must be explicitly cast
result = arc.JobDescription_Parse(str(xrslString), jobdescs)
if not result:
self.log.error("Invalid job description",
"%r, message=%s" % (xrslString, result.str()))
break
# Submit the job
jobs = arc.JobList() # filled by the submit process
submitter = arc.Submitter(self.usercfg)
result = submitter.Submit(endpoint, jobdescs, jobs)
# Save info or else ..else.
if result == arc.SubmissionStatus.NONE:
# Job successfully submitted
pilotJobReference = jobs[0].JobID
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
self.log.debug("Successfully submitted job %s to CE %s" %
(pilotJobReference, self.ceHost))
else:
message = "Failed to submit job because "
if result.isSet(arc.SubmissionStatus.NOT_IMPLEMENTED): # pylint: disable=no-member
self.log.warn(
"%s feature not implemented on CE? (weird I know - complain to site admins"
% message)
if result.isSet(arc.SubmissionStatus.NO_SERVICES): # pylint: disable=no-member
self.log.warn(
"%s no services are running on CE? (open GGUS ticket to site admins"
% message)
if result.isSet(arc.SubmissionStatus.ENDPOINT_NOT_QUERIED): # pylint: disable=no-member
self.log.warn(
"%s endpoint was not even queried. (network ..?)" %
message)
if result.isSet(arc.SubmissionStatus.BROKER_PLUGIN_NOT_LOADED): # pylint: disable=no-member
self.log.warn(
"%s BROKER_PLUGIN_NOT_LOADED : ARC library installation problem?"
% message)
if result.isSet(
arc.SubmissionStatus.DESCRIPTION_NOT_SUBMITTED): # pylint: disable=no-member
self.log.warn(
"%s Job not submitted - incorrect job description? (missing field in XRSL string?)"
% message)
if result.isSet(
arc.SubmissionStatus.SUBMITTER_PLUGIN_NOT_LOADED): # pylint: disable=no-member
self.log.warn(
"%s SUBMITTER_PLUGIN_NOT_LOADED : ARC library installation problem?"
% message)
if result.isSet(arc.SubmissionStatus.AUTHENTICATION_ERROR): # pylint: disable=no-member
self.log.warn(
"%s authentication error - screwed up / expired proxy? Renew / upload pilot proxy on machine?"
% message)
if result.isSet(arc.SubmissionStatus.ERROR_FROM_ENDPOINT): # pylint: disable=no-member
self.log.warn(
"%s some error from the CE - possibly CE problems?" %
message)
self.log.warn("%s ... maybe above messages will give a hint." %
message)
break # Boo hoo *sniff*
if self.preamble:
os.unlink(executableFile)
if batchIDList:
result = S_OK(batchIDList)
result["PilotStampDict"] = stampDict
else:
result = S_ERROR(
"No pilot references obtained from the ARC job submission")
return result