def __submitPilots( self, taskQueueDict, pilotsToSubmit ):
"""
Try to insert the submission in the corresponding Thread Pool, disable the Thread Pool
until next itration once it becomes full
"""
# Check if an specific MiddleWare is required
if 'SubmitPools' in taskQueueDict:
submitPools = taskQueueDict[ 'SubmitPools' ]
else:
submitPools = self.am_getOption( 'DefaultSubmitPools' )
submitPools = List.randomize( submitPools )
for submitPool in submitPools:
self.log.verbose( 'Trying SubmitPool:', submitPool )
if not submitPool in self.directors or not self.directors[submitPool]['isEnabled']:
self.log.verbose( 'Not Enabled' )
continue
pool = self.pools[self.directors[submitPool]['pool']]
director = self.directors[submitPool]['director']
ret = pool.generateJobAndQueueIt( director.submitPilots,
args = ( taskQueueDict, pilotsToSubmit, self.workDir ),
oCallback = self.callBack,
oExceptionCallback = director.exceptionCallBack,
blocking = False )
if not ret['OK']:
# Disable submission until next iteration
self.directors[submitPool]['isEnabled'] = False
else:
time.sleep( self.am_getOption( 'ThreadStartDelay' ) )
break
return S_OK( pilotsToSubmit )
def _prepareJDL( self, taskQueueDict, workingDirectory, pilotOptions, pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ ):
"""
Write JDL for Pilot Submission
"""
# RB = List.randomize( self.resourceBrokers )[0]
LDs = []
NSs = []
LBs = []
# Select Randomly one RB from the list
RB = List.randomize( self.resourceBrokers )[0]
LDs.append( '"%s:9002"' % RB )
LBs.append( '"%s:9000"' % RB )
for LB in self.loggingServers:
NSs.append( '"%s:7772"' % LB )
LD = ', '.join( LDs )
NS = ', '.join( NSs )
LB = ', '.join( LBs )
vo = getVO()
if privateTQ or vo not in ['lhcb']:
extraReq = "True"
else:
if submitPrivatePilot:
extraReq = "! AllowsGenericPilot"
else:
extraReq = "AllowsGenericPilot"
rbJDL = """
AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment );
Requirements = pilotRequirements && other.GlueCEStateStatus == "Production" && %s;
RetryCount = 0;
ErrorStorage = "%s/pilotError";
OutputStorage = "%s/pilotOutput";
# ListenerPort = 44000;
ListenerStorage = "%s/Storage";
VirtualOrganisation = "lhcb";
LoggingTimeout = 30;
LoggingSyncTimeout = 30;
LoggingDestination = { %s };
# Default NS logger level is set to 0 (null)
# max value is 6 (very ugly)
NSLoggerLevel = 0;
DefaultLogInfoLevel = 0;
DefaultStatusLevel = 0;
NSAddresses = { %s };
LBAddresses = { %s };
MyProxyServer = "no-myproxy.cern.ch";
""" % ( extraReq, workingDirectory, workingDirectory, workingDirectory, LD, NS, LB )
pilotJDL, pilotRequirements = self._JobJDL( taskQueueDict, pilotOptions, ceMask )
jdl = os.path.join( workingDirectory, '%s.jdl' % taskQueueDict['TaskQueueID'] )
jdl = self._writeJDL( jdl, [pilotJDL, rbJDL] )
return {'JDL':jdl, 'Requirements':pilotRequirements + " && " + extraReq, 'Pilots': pilotsToSubmit, 'RB':RB }
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all Grid PilotDirectors
"""
PilotDirector.configure( self, csSection, submitPool )
self.reloadConfiguration( csSection, submitPool )
self.__failingWMSCache.purgeExpired()
self.__ticketsWMSCache.purgeExpired()
for rb in self.__failingWMSCache.getKeys():
if rb in self.resourceBrokers:
try:
self.resourceBrokers.remove( rb )
except:
pass
self.resourceBrokers = List.randomize( self.resourceBrokers )
if self.gridEnv:
self.log.info( ' GridEnv: ', self.gridEnv )
if self.resourceBrokers:
self.log.info( ' ResourceBrokers:', ', '.join( self.resourceBrokers ) )
def configure(self, csSection, submitPool):
"""
Here goes common configuration for all Grid PilotDirectors
"""
PilotDirector.configure(self, csSection, submitPool)
self.reloadConfiguration(csSection, submitPool)
self.__failingWMSCache.purgeExpired()
self.__ticketsWMSCache.purgeExpired()
for rb in self.__failingWMSCache.getKeys():
if rb in self.resourceBrokers:
try:
self.resourceBrokers.remove(rb)
except:
pass
self.resourceBrokers = List.randomize(self.resourceBrokers)
if self.gridEnv:
self.log.info(' GridEnv: ', self.gridEnv)
if self.resourceBrokers:
self.log.info(' ResourceBrokers:', ', '.join(self.resourceBrokers))
def __submitPilots(self, taskQueueDict, pilotsToSubmit):
"""
Try to insert the submission in the corresponding Thread Pool, disable the Thread Pool
until next itration once it becomes full
"""
# Check if an specific MiddleWare is required
if 'SubmitPools' in taskQueueDict:
submitPools = taskQueueDict['SubmitPools']
else:
submitPools = self.am_getOption('DefaultSubmitPools')
submitPools = List.randomize(submitPools)
for submitPool in submitPools:
self.log.verbose('Trying SubmitPool:', submitPool)
if not submitPool in self.directors or not self.directors[
submitPool]['isEnabled']:
self.log.verbose('Not Enabled')
continue
pool = self.pools[self.directors[submitPool]['pool']]
director = self.directors[submitPool]['director']
ret = pool.generateJobAndQueueIt(
director.submitPilots,
args=(taskQueueDict, pilotsToSubmit, self.workDir),
oCallback=self.callBack,
oExceptionCallback=director.exceptionCallBack,
blocking=False)
if not ret['OK']:
# Disable submission until next iteration
self.directors[submitPool]['isEnabled'] = False
else:
time.sleep(self.am_getOption('ThreadStartDelay'))
break
return S_OK(pilotsToSubmit)
def _prepareJDL(self, taskQueueDict, workingDirectory, pilotOptions,
pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ):
"""
Write JDL for Pilot Submission
"""
# RB = List.randomize( self.resourceBrokers )[0]
LDs = []
NSs = []
LBs = []
# Select Randomly one RB from the list
RB = List.randomize(self.resourceBrokers)[0]
LDs.append('"%s:9002"' % RB)
LBs.append('"%s:9000"' % RB)
for LB in self.loggingServers:
NSs.append('"%s:7772"' % LB)
LD = ', '.join(LDs)
NS = ', '.join(NSs)
LB = ', '.join(LBs)
vo = getVO()
if privateTQ or vo not in ['lhcb']:
extraReq = "True"
else:
if submitPrivatePilot:
extraReq = "! AllowsGenericPilot"
else:
extraReq = "AllowsGenericPilot"
rbJDL = """
AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment );
Requirements = pilotRequirements && other.GlueCEStateStatus == "Production" && %s;
RetryCount = 0;
ErrorStorage = "%s/pilotError";
OutputStorage = "%s/pilotOutput";
# ListenerPort = 44000;
ListenerStorage = "%s/Storage";
VirtualOrganisation = "lhcb";
LoggingTimeout = 30;
LoggingSyncTimeout = 30;
LoggingDestination = { %s };
# Default NS logger level is set to 0 (null)
# max value is 6 (very ugly)
NSLoggerLevel = 0;
DefaultLogInfoLevel = 0;
DefaultStatusLevel = 0;
NSAddresses = { %s };
LBAddresses = { %s };
MyProxyServer = "no-myproxy.cern.ch";
""" % (extraReq, workingDirectory, workingDirectory, workingDirectory, LD, NS,
LB)
pilotJDL, pilotRequirements = self._JobJDL(taskQueueDict, pilotOptions,
ceMask)
jdl = os.path.join(workingDirectory,
'%s.jdl' % taskQueueDict['TaskQueueID'])
jdl = self._writeJDL(jdl, [pilotJDL, rbJDL])
return {
'JDL': jdl,
'Requirements': pilotRequirements + " && " + extraReq,
'Pilots': pilotsToSubmit,
'RB': RB
}
def _prepareJDL( self, taskQueueDict, workingDirectory, pilotOptions,
pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ ):
"""
Write JDL for Pilot Submission
"""
rbList = []
# Select Randomly one RB from the list
rb = List.randomize( self.resourceBrokers )[0]
rbList.append( '"https://%s:7443/glite_wms_wmproxy_server"' % rb )
lbList = []
for lb in self.loggingServers:
lbList.append( '"https://%s:9000"' % lb )
lbList = List.randomize( lbList )
nPilots = 1
vo = gConfig.getValue( '/DIRAC/VirtualOrganization', '' )
if privateTQ or vo not in ['lhcb']:
extraReq = "True"
else:
if submitPrivatePilot:
extraReq = "! AllowsGenericPilot"
else:
extraReq = "AllowsGenericPilot"
wmsClientJDL = """
RetryCount = 0;
ShallowRetryCount = 0;
MyProxyServer = "%s";
AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment );
Requirements = pilotRequirements && %s;
WmsClient = [
ErrorStorage = "%s/pilotError";
OutputStorage = "%s/pilotOutput";
# ListenerPort = 44000;
ListenerStorage = "%s/Storage";
RetryCount = 0;
ShallowRetryCount = 0;
WMProxyEndPoints = { %s };
LBEndPoints = { %s };
MyProxyServer = "%s";
EnableServiceDiscovery = false;
JdlDefaultAttributes = [
requirements = ( other.GlueCEStateStatus == "Production" || other.GlueCEStateStatus == "Special" );
AllowZippedISB = true;
SignificantAttributes = {"Requirements", "Rank", "FuzzyRank"};
PerusalFileEnable = false;
];
];
""" % ( self.myProxyServer, extraReq,
workingDirectory, workingDirectory,
workingDirectory, ', '.join( rbList ),
', '.join( lbList ), self.myProxyServer )
if pilotsToSubmit > 1:
wmsClientJDL += """
JobType = "Parametric";
Parameters= %s;
ParameterStep =1;
ParameterStart = 0;
""" % pilotsToSubmit
nPilots = pilotsToSubmit
( pilotJDL , pilotRequirements ) = self._JobJDL( taskQueueDict, pilotOptions, ceMask )
jdl = os.path.join( workingDirectory, '%s.jdl' % taskQueueDict['TaskQueueID'] )
jdl = self._writeJDL( jdl, [pilotJDL, wmsClientJDL] )
return {'JDL':jdl, 'Requirements':pilotRequirements + " && " + extraReq, 'Pilots':nPilots, 'RB':rb }
def _prepareJDL(self, taskQueueDict, workingDirectory, pilotOptions,
pilotsToSubmit, ceMask, submitPrivatePilot, privateTQ):
"""
Write JDL for Pilot Submission
"""
rbList = []
# Select Randomly one RB from the list
rb = List.randomize(self.resourceBrokers)[0]
rbList.append('"https://%s:7443/glite_wms_wmproxy_server"' % rb)
lbList = []
for lb in self.loggingServers:
lbList.append('"https://%s:9000"' % lb)
lbList = List.randomize(lbList)
nPilots = 1
vo = gConfig.getValue('/DIRAC/VirtualOrganization', '')
if privateTQ or vo not in ['lhcb']:
extraReq = "True"
else:
if submitPrivatePilot:
extraReq = "! AllowsGenericPilot"
else:
extraReq = "AllowsGenericPilot"
wmsClientJDL = """
RetryCount = 0;
ShallowRetryCount = 0;
MyProxyServer = "%s";
AllowsGenericPilot = Member( "VO-lhcb-pilot" , other.GlueHostApplicationSoftwareRunTimeEnvironment );
Requirements = pilotRequirements && %s;
WmsClient = [
ErrorStorage = "%s/pilotError";
OutputStorage = "%s/pilotOutput";
# ListenerPort = 44000;
ListenerStorage = "%s/Storage";
RetryCount = 0;
ShallowRetryCount = 0;
WMProxyEndPoints = { %s };
LBEndPoints = { %s };
MyProxyServer = "%s";
EnableServiceDiscovery = false;
JdlDefaultAttributes = [
requirements = ( other.GlueCEStateStatus == "Production" || other.GlueCEStateStatus == "Special" );
AllowZippedISB = true;
SignificantAttributes = {"Requirements", "Rank", "FuzzyRank"};
PerusalFileEnable = false;
];
];
""" % (self.myProxyServer, extraReq, workingDirectory, workingDirectory,
workingDirectory, ', '.join(rbList), ', '.join(lbList),
self.myProxyServer)
if pilotsToSubmit > 1:
wmsClientJDL += """
JobType = "Parametric";
Parameters= %s;
ParameterStep =1;
ParameterStart = 0;
""" % pilotsToSubmit
nPilots = pilotsToSubmit
(pilotJDL, pilotRequirements) = self._JobJDL(taskQueueDict,
pilotOptions, ceMask)
jdl = os.path.join(workingDirectory,
'%s.jdl' % taskQueueDict['TaskQueueID'])
jdl = self._writeJDL(jdl, [pilotJDL, wmsClientJDL])
return {
'JDL': jdl,
'Requirements': pilotRequirements + " && " + extraReq,
'Pilots': nPilots,
'RB': rb
}
