def submitJob(self, executableFile, proxy, numberOfJobs=1):
# self.log.verbose( "Executable file path: %s" % executableFile )
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose("Setting up proxy for payload")
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
result = self._submitJobToHost(submitFile, numberOfJobs)
if proxy:
os.remove(submitFile)
return result
def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if self.proxy and not proxy:
proxy = self.proxy
if proxy:
self.log.verbose('Setting up proxy for payload')
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
batchDict = {
'Executable': submitFile,
'NJobs': numberOfJobs,
'OutputDir': self.batchOutput,
'ErrorDir': self.batchError,
'SubmitOptions': self.submitOptions,
'ExecutionContext': self.execution,
'JobStamps': jobStamps,
'Queue': self.queue,
'WholeNode': self.wholeNode,
'NumberOfProcessors': self.numberOfProcessors
}
resultSubmit = self.batch.submitJob(**batchDict)
if proxy:
os.remove(submitFile)
if resultSubmit['Status'] == 0:
self.submittedJobs += len(resultSubmit['Jobs'])
# jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ]
# FIXME: It would be more proper to fix pilotCommands.__setFlavour where 'ssh' is hardcoded than
# making this illogical fix, but there is no good way for pilotCommands to know its origin ceType.
# So, the jobIDs here need to start with 'ssh', not ceType, to accomodate
# them to those hardcoded in pilotCommands.__setFlavour
jobIDs = [
'ssh' + self.batchSystem.lower() + '://' + self.ceName + '/' +
_id for _id in resultSubmit['Jobs']
]
result = S_OK(jobIDs)
else:
result = S_ERROR(resultSubmit['Message'])
return result
def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose('Setting up proxy for payload')
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
batchDict = {
'Executable': submitFile,
'NJobs': numberOfJobs,
'OutputDir': self.batchOutput,
'ErrorDir': self.batchError,
'SubmitOptions': self.submitOptions,
'ExecutionContext': self.execution,
'JobStamps': jobStamps
}
resultSubmit = self.batch.submitJob(**batchDict)
if proxy:
os.remove(submitFile)
if resultSubmit['Status'] == 0:
self.submittedJobs += len(resultSubmit['Jobs'])
jobIDs = [
self.ceType.lower() + '://' + self.ceName + '/' + _id
for _id in resultSubmit['Jobs']
]
result = S_OK(jobIDs)
else:
result = S_ERROR(resultSubmit['Message'])
return result
def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
if not os.access(executableFile, 5):
os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose('Setting up proxy for payload')
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
batchDict = {'Executable': submitFile,
'NJobs': numberOfJobs,
'OutputDir': self.batchOutput,
'ErrorDir': self.batchError,
'SubmitOptions': self.submitOptions,
'ExecutionContext': self.execution,
'JobStamps': jobStamps,
'Queue': self.queue}
resultSubmit = self.batch.submitJob(**batchDict)
if proxy:
os.remove(submitFile)
if resultSubmit['Status'] == 0:
self.submittedJobs += len(resultSubmit['Jobs'])
# jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ]
# FIXME: It would be more proper to fix pilotCommands.__setFlavour where 'ssh' is hardcoded than
# making this illogical fix, but there is no good way for pilotCommands to know its origin ceType.
# So, the jobIDs here need to start with 'ssh', not ceType, to accomodate
# them to those hardcoded in pilotCommands.__setFlavour
jobIDs = ['ssh' + self.batchSystem.lower() + '://' + self.ceName + '/' + _id for _id in resultSubmit['Jobs']]
result = S_OK(jobIDs)
else:
result = S_ERROR(resultSubmit['Message'])
return result
def submitJob( self, executableFile, proxy = None, numberOfJobs = 1 ):
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
wrapperContent = bundleProxy( executableFile, proxy )
name = writeScript( wrapperContent, os.getcwd() )
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range( numberOfJobs ):
jobStamps.append( makeGuid()[:8] )
batchDict = { 'Executable': submitFile,
'NJobs': numberOfJobs,
'OutputDir': self.batchOutput,
'ErrorDir': self.batchError,
'SubmitOptions': self.submitOptions,
'ExecutionContext': self.execution,
'JobStamps': jobStamps }
resultSubmit = self.batch.submitJob( **batchDict )
if proxy:
os.remove( submitFile )
if resultSubmit['Status'] == 0:
self.submittedJobs += len( resultSubmit['Jobs'] )
jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ]
result = S_OK( jobIDs )
else:
result = S_ERROR( resultSubmit['Message'] )
return result
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ):
if not os.access( executableFile, 5 ):
os.chmod( executableFile, 0o755 )
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
wrapperContent = bundleProxy( executableFile, proxy )
name = writeScript( wrapperContent, os.getcwd() )
submitFile = name
else: # no proxy
submitFile = executableFile
result = self._submitJob( submitFile, numberOfJobs )
if proxy:
os.remove( submitFile )
return result
def submitJob(self, executableFile, proxy, numberOfJobs=1):
"""Method to submit job"""
# Choose eligible hosts, rank them by the number of available slots
rankHosts = {}
maxSlots = 0
for host in self.sshHost:
thost = host.split("/")
hostName = thost[0]
maxHostJobs = 1
if len(thost) > 1:
maxHostJobs = int(thost[1])
result = self._getHostStatus(hostName)
if not result["OK"]:
continue
slots = maxHostJobs - result["Value"]["Running"]
if slots > 0:
rankHosts.setdefault(slots, [])
rankHosts[slots].append(hostName)
if slots > maxSlots:
maxSlots = slots
if maxSlots == 0:
return S_ERROR("No online node found on queue")
# make it executable
if not os.access(executableFile, 5):
os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose("Setting up proxy for payload")
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
# Submit jobs now
restJobs = numberOfJobs
submittedJobs = []
for slots in range(maxSlots, 0, -1):
if slots not in rankHosts:
continue
for host in rankHosts[slots]:
result = self._submitJobToHost(submitFile, min(slots, restJobs), host)
if not result["OK"]:
continue
else:
nJobs = len(result["Value"])
if nJobs > 0:
submittedJobs.extend(result["Value"])
restJobs = restJobs - nJobs
if restJobs <= 0:
break
if restJobs <= 0:
break
if proxy:
os.remove(submitFile)
return S_OK(submittedJobs)
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ):
""" Method to submit job
"""
# Choose eligible hosts, rank them by the number of available slots
rankHosts = {}
maxSlots = 0
for host in self.sshHost:
thost = host.split( "/" )
hostName = thost[0]
maxHostJobs = 1
if len( thost ) > 1:
maxHostJobs = int( thost[1] )
result = self._getHostStatus( hostName )
if not result['OK']:
continue
slots = maxHostJobs - result['Value']['Running']
if slots > 0:
rankHosts.setdefault(slots,[])
rankHosts[slots].append( hostName )
if slots > maxSlots:
maxSlots = slots
if maxSlots == 0:
return S_ERROR( "No online node found on queue" )
##make it executable
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
wrapperContent = bundleProxy( executableFile, proxy )
name = writeScript( wrapperContent, os.getcwd() )
submitFile = name
else: # no proxy
submitFile = executableFile
# Submit jobs now
restJobs = numberOfJobs
submittedJobs = []
for slots in range(maxSlots,0,-1):
if not slots in rankHosts:
continue
for host in rankHosts[slots]:
result = self._submitJobToHost( submitFile, min( slots, restJobs ), host )
if not result['OK']:
continue
else:
nJobs = len( result['Value'] )
if nJobs > 0:
submittedJobs.extend( result['Value'] )
restJobs = restJobs - nJobs
if restJobs <= 0:
break
if restJobs <= 0:
break
if proxy:
os.remove( submitFile )
return S_OK( submittedJobs )
def _bundlePreamble(self, executableFile):
"""Bundle the preamble with the executable file"""
wrapperContent = "%s\n./%s" % (self.preamble, executableFile)
return writeScript(wrapperContent, os.getcwd())