class PilotCommand(Command):
"""
Pilot "master" Command.
"""
def __init__(self, args=None, clients=None):
super(PilotCommand, self).__init__(args, clients)
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis['WMSAdministrator']
else:
self.wmsAdmin = WMSAdministratorClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def _storeCommand(self, result):
"""
Stores the results of doNew method on the database.
"""
for pilotDict in result:
resQuery = self.rmClient.addOrModifyPilotCache(pilotDict['Site'],
pilotDict['CE'],
pilotDict['PilotsPerJob'],
pilotDict['PilotJobEff'],
pilotDict['Status'])
if not resQuery['OK']:
return resQuery
return S_OK()
def _prepareCommand(self):
"""
JobCommand requires one arguments:
- name : <str>
"""
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
name = self.args['name']
if 'element' not in self.args:
return S_ERROR('element is missing')
element = self.args['element']
if element not in ['Site', 'Resource']:
return S_ERROR('"%s" is not Site nor Resource' % element)
return S_OK((element, name))
def doNew(self, masterParams=None):
if masterParams is not None:
element, name = masterParams
else:
params = self._prepareCommand()
if not params['OK']:
return params
element, name = params['Value']
wmsDict = {}
if element == 'Site':
wmsDict = {'GridSite': name}
elif element == 'Resource':
wmsDict = {'ExpandSite': name}
else:
# You should never see this error
return S_ERROR('"%s" is not Site nor Resource' % element)
wmsResults = self.wmsAdmin.getPilotSummaryWeb(wmsDict, [], 0, 0)
if not wmsResults['OK']:
return wmsResults
wmsResults = wmsResults['Value']
if 'ParameterNames' not in wmsResults:
return S_ERROR('Wrong result dictionary, missing "ParameterNames"')
params = wmsResults['ParameterNames']
if 'Records' not in wmsResults:
return S_ERROR('Wrong formed result dictionary, missing "Records"')
records = wmsResults['Records']
uniformResult = []
for record in records:
# This returns a dictionary with the following keys:
# 'Site', 'CE', 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running',
# 'Done', 'Aborted', 'Done_Empty', 'Aborted_Hour', 'Total', 'PilotsPerJob',
# 'PilotJobEff', 'Status', 'InMask'
pilotDict = dict(zip(params, record))
pilotDict['PilotsPerJob'] = float(pilotDict['PilotsPerJob'])
pilotDict['PilotJobEff'] = float(pilotDict['PilotJobEff'])
uniformResult.append(pilotDict)
storeRes = self._storeCommand(uniformResult)
if not storeRes['OK']:
return storeRes
return S_OK(uniformResult)
def doCache(self):
params = self._prepareCommand()
if not params['OK']:
return params
element, name = params['Value']
if element == 'Site':
# WMS returns Site entries with CE = 'Multiple'
site, ce = name, 'Multiple'
elif element == 'Resource':
site, ce = None, name
else:
# You should never see this error
return S_ERROR('"%s" is not Site nor Resource' % element)
result = self.rmClient.selectPilotCache(site, ce)
if result['OK']:
result = S_OK([dict(zip(result['Columns'], res)) for res in result['Value']])
return result
def doMaster(self):
siteNames = getSites()
if not siteNames['OK']:
return siteNames
siteNames = siteNames['Value']
ces = CSHelpers.getComputingElements()
if not ces['OK']:
return ces
ces = ces['Value']
pilotResults = self.doNew(('Site', siteNames))
if not pilotResults['OK']:
self.metrics['failed'].append(pilotResults['Message'])
pilotResults = self.doNew(('Resource', ces))
if not pilotResults['OK']:
self.metrics['failed'].append(pilotResults['Message'])
return S_OK(self.metrics)
class PilotCommand(Command):
"""
Pilot "master" Command.
"""
def __init__(self, args=None, clients=None):
super(PilotCommand, self).__init__(args, clients)
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis['WMSAdministrator']
else:
self.wmsAdmin = WMSAdministratorClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def _storeCommand(self, result):
"""
Stores the results of doNew method on the database.
"""
for pilotDict in result:
resQuery = self.rmClient.addOrModifyPilotCache(
pilotDict['Site'], pilotDict['CE'], pilotDict['PilotsPerJob'],
pilotDict['PilotJobEff'], pilotDict['Status'])
if not resQuery['OK']:
return resQuery
return S_OK()
def _prepareCommand(self):
"""
JobCommand requires one arguments:
- name : <str>
"""
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
name = self.args['name']
if 'element' not in self.args:
return S_ERROR('element is missing')
element = self.args['element']
if element not in ['Site', 'Resource']:
return S_ERROR('"%s" is not Site nor Resource' % element)
return S_OK((element, name))
def doNew(self, masterParams=None):
if masterParams is not None:
element, name = masterParams
else:
params = self._prepareCommand()
if not params['OK']:
return params
element, name = params['Value']
wmsDict = {}
if element == 'Site':
wmsDict = {'GridSite': name}
elif element == 'Resource':
wmsDict = {'ExpandSite': name}
else:
# You should never see this error
return S_ERROR('"%s" is not Site nor Resource' % element)
wmsResults = self.wmsAdmin.getPilotSummaryWeb(wmsDict, [], 0, 0)
if not wmsResults['OK']:
return wmsResults
wmsResults = wmsResults['Value']
if 'ParameterNames' not in wmsResults:
return S_ERROR('Wrong result dictionary, missing "ParameterNames"')
params = wmsResults['ParameterNames']
if 'Records' not in wmsResults:
return S_ERROR('Wrong formed result dictionary, missing "Records"')
records = wmsResults['Records']
uniformResult = []
for record in records:
# This returns a dictionary with the following keys:
# 'Site', 'CE', 'Submitted', 'Ready', 'Scheduled', 'Waiting', 'Running',
# 'Done', 'Aborted', 'Done_Empty', 'Aborted_Hour', 'Total', 'PilotsPerJob',
# 'PilotJobEff', 'Status', 'InMask'
pilotDict = dict(zip(params, record))
pilotDict['PilotsPerJob'] = float(pilotDict['PilotsPerJob'])
pilotDict['PilotJobEff'] = float(pilotDict['PilotJobEff'])
uniformResult.append(pilotDict)
storeRes = self._storeCommand(uniformResult)
if not storeRes['OK']:
return storeRes
return S_OK(uniformResult)
def doCache(self):
params = self._prepareCommand()
if not params['OK']:
return params
element, name = params['Value']
if element == 'Site':
# WMS returns Site entries with CE = 'Multiple'
site, ce = name, 'Multiple'
elif element == 'Resource':
site, ce = None, name
else:
# You should never see this error
return S_ERROR('"%s" is not Site nor Resource' % element)
result = self.rmClient.selectPilotCache(site, ce)
if result['OK']:
result = S_OK(
[dict(zip(result['Columns'], res)) for res in result['Value']])
return result
def doMaster(self):
siteNames = getSites()
if not siteNames['OK']:
return siteNames
siteNames = siteNames['Value']
ces = CSHelpers.getComputingElements()
if not ces['OK']:
return ces
ces = ces['Value']
pilotResults = self.doNew(('Site', siteNames))
if not pilotResults['OK']:
self.metrics['failed'].append(pilotResults['Message'])
pilotResults = self.doNew(('Resource', ces))
if not pilotResults['OK']:
self.metrics['failed'].append(pilotResults['Message'])
return S_OK(self.metrics)
def test_PilotsDB(self):
wmsAdministrator = WMSAdministratorClient()
pilotAgentDB = PilotAgentsDB()
res = wmsAdministrator.addPilotTQReference(['aPilot'], 1, '/a/ownerDN', 'a/owner/Group')
self.assertTrue(res['OK'])
res = wmsAdministrator.getCurrentPilotCounters({})
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {'Submitted': 1})
res = pilotAgentDB.deletePilot('aPilot')
self.assertTrue(res['OK'])
res = wmsAdministrator.getCurrentPilotCounters({})
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {})
res = wmsAdministrator.addPilotTQReference(['anotherPilot'], 1, '/a/ownerDN', 'a/owner/Group')
self.assertTrue(res['OK'])
res = wmsAdministrator.storePilotOutput('anotherPilot', 'This is an output', 'this is an error')
self.assertTrue(res['OK'])
res = wmsAdministrator.getPilotOutput('anotherPilot')
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {'OwnerDN': '/a/ownerDN',
'OwnerGroup': 'a/owner/Group',
'StdErr': 'this is an error',
'FileList': [],
'StdOut': 'This is an output'})
# need a job for the following
# res = wmsAdministrator.getJobPilotOutput( 1 )
# self.assertEqual( res['Value'], {'OwnerDN': '/a/ownerDN', 'OwnerGroup': 'a/owner/Group',
# 'StdErr': 'this is an error', 'FileList': [], 'StdOut': 'This is an output'} )
# self.assertTrue(res['OK'])
res = wmsAdministrator.getPilotInfo('anotherPilot')
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['anotherPilot']['AccountingSent'], 'False')
self.assertEqual(res['Value']['anotherPilot']['PilotJobReference'], 'anotherPilot')
res = wmsAdministrator.selectPilots({})
self.assertTrue(res['OK'])
# res = wmsAdministrator.getPilotLoggingInfo( 'anotherPilot' )
# self.assertTrue(res['OK'])
res = wmsAdministrator.getPilotSummary('', '')
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['Total']['Submitted'], 1)
res = wmsAdministrator.getPilotMonitorWeb({}, [], 0, 100)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['TotalRecords'], 1)
res = wmsAdministrator.getPilotMonitorSelectors()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {'GridType': ['DIRAC'],
'OwnerGroup': ['a/owner/Group'],
'DestinationSite': ['NotAssigned'],
'Broker': ['Unknown'], 'Status': ['Submitted'],
'OwnerDN': ['/a/ownerDN'],
'GridSite': ['Unknown'],
'Owner': []})
res = wmsAdministrator.getPilotSummaryWeb({}, [], 0, 100)
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['TotalRecords'], 1)
res = wmsAdministrator.setAccountingFlag('anotherPilot', 'True')
self.assertTrue(res['OK'])
res = wmsAdministrator.setPilotStatus('anotherPilot', 'Running')
self.assertTrue(res['OK'])
res = wmsAdministrator.getPilotInfo('anotherPilot')
self.assertTrue(res['OK'])
self.assertEqual(res['Value']['anotherPilot']['AccountingSent'], 'True')
self.assertEqual(res['Value']['anotherPilot']['Status'], 'Running')
res = wmsAdministrator.setJobForPilot(123, 'anotherPilot')
self.assertTrue(res['OK'])
res = wmsAdministrator.setPilotBenchmark('anotherPilot', 12.3)
self.assertTrue(res['OK'])
res = wmsAdministrator.countPilots({})
self.assertTrue(res['OK'])
# res = wmsAdministrator.getCounters()
# # getPilotStatistics
res = pilotAgentDB.deletePilot('anotherPilot')
self.assertTrue(res['OK'])
res = wmsAdministrator.getCurrentPilotCounters({})
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], {})
