def export_submitJob( self, jobDesc ):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" )
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR( 'Failed to get job policies' )
policyDict = result['Value']
if not policyDict[ RIGHT_SUBMIT ]:
return S_ERROR( 'Job submission not authorized' )
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd( jobDesc )
parametricJob = False
if jobClassAd.lookupAttribute( 'Parameters' ):
parametricJob = True
if jobClassAd.isAttributeList( 'Parameters' ):
parameterList = jobClassAd.getListFromExpression( 'Parameters' )
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt( 'Parameters' )
if not nParameters:
value = jobClassAd.get_expression( 'Parameters' )
return S_ERROR( 'Illegal value for Parameters JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterStart' ):
value = jobClassAd.get_expression( 'ParameterStart' ).replace( '"', '' )
try:
pStart = int( value )
except:
try:
pStart = float( value )
except:
return S_ERROR( 'Illegal value for ParameterStart JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterStep' ):
pStep = jobClassAd.getAttributeInt( 'ParameterStep' )
if not pStep:
pStep = jobClassAd.getAttributeFloat( 'ParameterStep' )
if not pStep:
value = jobClassAd.get_expression( 'ParameterStep' )
return S_ERROR( 'Illegal value for ParameterStep JDL field: %s' % value )
if jobClassAd.lookupAttribute( 'ParameterFactor' ):
pFactor = jobClassAd.getAttributeInt( 'ParameterFactor' )
if not pFactor:
pFactor = jobClassAd.getAttributeFloat( 'ParameterFactor' )
if not pFactor:
value = jobClassAd.get_expression( 'ParameterFactor' )
return S_ERROR( 'Illegal value for ParameterFactor JDL field: %s' % value )
parameterList = list()
parameterList.append( pStart )
for i in range( nParameters - 1 ):
parameterList.append( parameterList[i] * pFactor + pStep )
if len( parameterList ) > self.maxParametricJobs:
return S_ERROR( 'The number of parametric jobs exceeded the limit of %d' % self.maxParametricJobs )
jobDescList = []
nParam = len(parameterList) - 1
for n,p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters','ParameterStep','ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type( p ) == type ( ' ' ) and p.startswith('{'):
newClassAd.insertAttributeInt( 'Parameter',str(p) )
else:
newClassAd.insertAttributeString( 'Parameter', str( p ) )
newClassAd.insertAttributeInt( 'ParameterNumber', n )
newJDL = newClassAd.asJDL()
jobDescList.append( newJDL )
else:
jobDescList = [ jobDesc ]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup )
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) )
gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' )
jobIDList.append( jobID )
#Set persistency flag
retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup )
if 'Value' not in retVal or not retVal[ 'Value' ]:
gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True )
if parametricJob:
result = S_OK( jobIDList )
else:
result = S_OK( jobIDList[0] )
result['JobID'] = result['Value']
result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind( jobIDList )
return result
def export_submitJob(self, jobDesc):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR("Can't submit using a limited proxy! (bad boy!)")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR('Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR('Job submission not authorized')
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parameteric one
jobClassAd = ClassAd(jobDesc)
parametricJob = False
if jobClassAd.lookupAttribute('Parameters'):
parametricJob = True
if jobClassAd.isAttributeList('Parameters'):
parameterList = jobClassAd.getListFromExpression('Parameters')
else:
pStep = 0
pFactor = 1
pStart = 1
nParameters = jobClassAd.getAttributeInt('Parameters')
if not nParameters:
value = jobClassAd.get_expression('Parameters')
return S_ERROR(
'Illegal value for Parameters JDL field: %s' % value)
if jobClassAd.lookupAttribute('ParameterStart'):
value = jobClassAd.get_expression(
'ParameterStart').replace('"', '')
try:
pStart = int(value)
except:
try:
pStart = float(value)
except:
return S_ERROR(
'Illegal value for ParameterStart JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterStep'):
pStep = jobClassAd.getAttributeInt('ParameterStep')
if not pStep:
pStep = jobClassAd.getAttributeFloat('ParameterStep')
if not pStep:
value = jobClassAd.get_expression('ParameterStep')
return S_ERROR(
'Illegal value for ParameterStep JDL field: %s'
% value)
if jobClassAd.lookupAttribute('ParameterFactor'):
pFactor = jobClassAd.getAttributeInt('ParameterFactor')
if not pFactor:
pFactor = jobClassAd.getAttributeFloat(
'ParameterFactor')
if not pFactor:
value = jobClassAd.get_expression(
'ParameterFactor')
return S_ERROR(
'Illegal value for ParameterFactor JDL field: %s'
% value)
parameterList = list()
parameterList.append(pStart)
for i in range(nParameters - 1):
parameterList.append(parameterList[i] * pFactor + pStep)
if len(parameterList) > self.maxParametricJobs:
return S_ERROR(
'The number of parametric jobs exceeded the limit of %d' %
self.maxParametricJobs)
jobDescList = []
nParam = len(parameterList) - 1
for n, p in enumerate(parameterList):
newJobDesc = jobDesc.replace('%s', str(p)).replace(
'%n',
str(n).zfill(len(str(nParam))))
newClassAd = ClassAd(newJobDesc)
for attr in ['Parameters', 'ParameterStep', 'ParameterFactor']:
newClassAd.deleteAttribute(attr)
if type(p) == type(' ') and p.startswith('{'):
newClassAd.insertAttributeInt('Parameter', str(p))
else:
newClassAd.insertAttributeString('Parameter', str(p))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
else:
jobDescList = [jobDesc]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner,
self.ownerDN, self.ownerGroup,
self.diracSetup)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' %
(jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID,
result['Status'],
result['MinorStatus'],
source='JobManager')
jobIDList.append(jobID)
#Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN,
self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
self.__sendNewJobsToMind(jobIDList)
return result
def generateParametricJobs( jobClassAd ):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute( 'Parameters' ):
return S_OK( [ jobClassAd.asJDL() ] )
result = getParameterVectorLength( jobClassAd )
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL, 'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [ 'Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor' ]:
if attribute.startswith( key ):
seqID = '0' if not '.' in attribute else attribute.split( '.' )[1]
parameterDict.setdefault( seqID, {} )
if key == 'Parameters':
if jobClassAd.isAttributeList( attribute ):
parList = jobClassAd.getListFromExpression( attribute )
if len( parList ) != nParValues:
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
nPar = jobClassAd.getAttributeInt( attribute )
if nPar is None:
value = jobClassAd.get_expression( attribute )
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description: %s=%s' % ( attribute, value ) )
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt( attribute )
if value is None:
value = jobClassAd.getAttributeFloat( attribute )
if value is None:
value = jobClassAd.get_expression( attribute )
return S_ERROR( 'Illegal value for %s JDL field: %s' % ( attribute, value ) )
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get( '0' ):
parameterDict.pop( '0' )
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence( nParValues,
parList = parameterDict[seqID].get( 'ParameterList', [] ),
parStart = parameterDict[seqID].get( 'ParameterStart', 1 ),
parStep = parameterDict[seqID].get( 'ParameterStep', 0 ),
parFactor = parameterDict[seqID].get( 'ParameterFactor', 1 )
)
if not parList:
return S_ERROR( EWMSJDL, 'Inconsistent parametric job description' )
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len( str( nParValues - 1 ) )
for n in range( nParValues ):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace( '%n', str( n ).zfill( zLength ) )
newClassAd = ClassAd( newJobDesc )
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute( newClassAd, attribute, seqID, str( parameter ) )
for seqID in parameterLists:
for attribute in ['Parameters', 'ParameterStart', 'ParameterStep', 'ParameterFactor']:
if seqID == '0':
newClassAd.deleteAttribute( attribute )
else:
newClassAd.deleteAttribute( '%s.%s' % ( attribute, seqID ) )
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance( parameter, basestring) and parameter.startswith( '{' ):
newClassAd.insertAttributeInt( attribute, str( parameter ) )
else:
newClassAd.insertAttributeString( attribute, str( parameter ) )
newClassAd.insertAttributeInt( 'ParameterNumber', n )
newJDL = newClassAd.asJDL()
jobDescList.append( newJDL )
return S_OK( jobDescList )
def generateParametricJobs(jobClassAd):
""" Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute('Parameters'):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
return result
nParValues = result['Value']
if nParValues is None:
return S_ERROR(EWMSJDL,
'Can not determine the number of job parameters')
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if attribute.startswith(key):
seqID = '0' if '.' not in attribute else attribute.split(
'.')[1]
parameterDict.setdefault(seqID, {})
if key == 'Parameters':
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
parameterDict[seqID]['ParameterList'] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description')
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
'Inconsistent parametric job description: %s=%s'
% (attribute, value))
parameterDict[seqID]['Parameters'] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
'Illegal value for %s JDL field: %s' %
(attribute, value))
parameterDict[seqID][key] = value
if '0' in parameterDict and not parameterDict.get('0'):
parameterDict.pop('0')
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get('ParameterList', []),
parStart=parameterDict[seqID].get('ParameterStart', 1),
parStep=parameterDict[seqID].get('ParameterStep', 0),
parFactor=parameterDict[seqID].get('ParameterFactor', 1))
if not parList:
return S_ERROR(EWMSJDL, 'Inconsistent parametric job description')
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace('%n', str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
'Parameters', 'ParameterStart', 'ParameterStep',
'ParameterFactor'
]:
if seqID == '0':
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute('%s.%s' % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == '0':
attribute = 'Parameter'
else:
attribute = 'Parameter.%s' % seqID
if isinstance(parameter,
six.string_types) and parameter.startswith('{'):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt('ParameterNumber', n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)
def generateParametricJobs(jobClassAd):
"""Generate a series of ClassAd job descriptions expanding
job parameters
:param jobClassAd: ClassAd job description object
:return: list of ClassAd job description objects
"""
if not jobClassAd.lookupAttribute("Parameters"):
return S_OK([jobClassAd.asJDL()])
result = getParameterVectorLength(jobClassAd)
if not result["OK"]:
return result
nParValues = result["Value"]
if nParValues is None:
return S_ERROR(EWMSJDL,
"Can not determine the number of job parameters")
parameterDict = {}
attributes = jobClassAd.getAttributes()
for attribute in attributes:
for key in [
"Parameters", "ParameterStart", "ParameterStep",
"ParameterFactor"
]:
if attribute.startswith(key):
seqID = "0" if "." not in attribute else attribute.split(
".")[1]
parameterDict.setdefault(seqID, {})
if key == "Parameters":
if jobClassAd.isAttributeList(attribute):
parList = jobClassAd.getListFromExpression(attribute)
if len(parList) != nParValues:
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description")
parameterDict[seqID]["ParameterList"] = parList
else:
if attribute != "Parameters":
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description")
nPar = jobClassAd.getAttributeInt(attribute)
if nPar is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
EWMSJDL,
"Inconsistent parametric job description: %s=%s"
% (attribute, value))
parameterDict[seqID]["Parameters"] = nPar
else:
value = jobClassAd.getAttributeInt(attribute)
if value is None:
value = jobClassAd.getAttributeFloat(attribute)
if value is None:
value = jobClassAd.get_expression(attribute)
return S_ERROR(
"Illegal value for %s JDL field: %s" %
(attribute, value))
parameterDict[seqID][key] = value
if "0" in parameterDict and not parameterDict.get("0"):
parameterDict.pop("0")
parameterLists = {}
for seqID in parameterDict:
parList = __getParameterSequence(
nParValues,
parList=parameterDict[seqID].get("ParameterList", []),
parStart=parameterDict[seqID].get("ParameterStart", 1),
parStep=parameterDict[seqID].get("ParameterStep", 0),
parFactor=parameterDict[seqID].get("ParameterFactor", 1),
)
if not parList:
return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
parameterLists[seqID] = parList
jobDescList = []
jobDesc = jobClassAd.asJDL()
# Width of the sequential parameter number
zLength = len(str(nParValues - 1))
for n in range(nParValues):
newJobDesc = jobDesc
newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
newClassAd = ClassAd(newJobDesc)
for seqID in parameterLists:
parameter = parameterLists[seqID][n]
for attribute in newClassAd.getAttributes():
__updateAttribute(newClassAd, attribute, seqID, str(parameter))
for seqID in parameterLists:
for attribute in [
"Parameters", "ParameterStart", "ParameterStep",
"ParameterFactor"
]:
if seqID == "0":
newClassAd.deleteAttribute(attribute)
else:
newClassAd.deleteAttribute("%s.%s" % (attribute, seqID))
parameter = parameterLists[seqID][n]
if seqID == "0":
attribute = "Parameter"
else:
attribute = "Parameter.%s" % seqID
if isinstance(parameter, str) and parameter.startswith("{"):
newClassAd.insertAttributeInt(attribute, str(parameter))
else:
newClassAd.insertAttributeString(attribute, str(parameter))
newClassAd.insertAttributeInt("ParameterNumber", n)
newJDL = newClassAd.asJDL()
jobDescList.append(newJDL)
return S_OK(jobDescList)