def __getUniqueKeyValues(self, typeName):
sessionData = self.getSessionData()
cacheKey = (sessionData["user"].get("username", ""),
sessionData["user"].get("group", ""),
sessionData["setup"],
typeName)
data = MonitoringHandler.__keysCache.get(cacheKey)
if not data:
client = MonitoringClient()
retVal = client.listUniqueKeyValues(typeName)
if 'rpcStub' in retVal:
del(retVal['rpcStub'])
if not retVal['OK']:
return retVal
# Site ordering based on TierLevel / alpha
if 'Site' in retVal['Value']:
siteLevel = {}
for siteName in retVal['Value']['Site']:
sitePrefix = siteName.split(".")[0].strip()
level = gConfig.getValue("/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10)
if level not in siteLevel:
siteLevel[level] = []
siteLevel[level].append(siteName)
orderedSites = []
for level in sorted(siteLevel):
orderedSites.extend(sorted(siteLevel[level]))
retVal['Value']['Site'] = orderedSites
data = retVal
MonitoringHandler.__keysCache.add(cacheKey, 300, data)
return data
def _getHistoryData(self, timeSpan, groupToUse):
""" Get history data from ElasticSearch Monitoring database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
monitoringClient = MonitoringClient()
reportCondition = {'Status': ['Running']}
if not groupToUse:
reportGrouping = 'UserGroup'
reportCondition["grouping"] = ["UserGroup"]
else:
reportGrouping = 'User'
reportCondition["UserGroup"] = groupToUse
reportCondition["grouping"] = ["User"]
now = Time.dateTime()
result = monitoringClient.getReport('WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta(seconds=timeSpan), now,
reportCondition, reportGrouping,
{'lastSeconds': timeSpan})
return result
def _getHistoryData(self, timeSpan, groupToUse):
"""Get history data from ElasticSearch Monitoring database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
monitoringClient = MonitoringClient()
reportCondition = {"Status": ["Running"]}
if not groupToUse:
reportGrouping = "UserGroup"
reportCondition["grouping"] = ["UserGroup"]
else:
reportGrouping = "User"
reportCondition["UserGroup"] = groupToUse
reportCondition["grouping"] = ["User"]
now = datetime.datetime.utcnow()
result = monitoringClient.getReport(
"WMSHistory",
"AverageNumberOfJobs",
now - datetime.timedelta(seconds=timeSpan),
now,
reportCondition,
reportGrouping,
{"lastSeconds": timeSpan},
)
return result
def _getHistoryData(self, timeSpan, groupToUse):
""" Get history data from ElasticSearch Monitoring database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
monitoringClient = MonitoringClient()
reportCondition = {'Status': ['Running']}
if not groupToUse:
reportGrouping = 'UserGroup'
reportCondition["grouping"] = ["UserGroup"]
else:
reportGrouping = 'User'
reportCondition["UserGroup"] = groupToUse
reportCondition["grouping"] = ["User"]
now = Time.dateTime()
result = monitoringClient.getReport('WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta(seconds=timeSpan), now,
reportCondition, reportGrouping,
{'lastSeconds': timeSpan})
return result
def __queryForPlot(self):
retVal = self.__parseFormParams()
if not retVal['OK']:
return retVal
params = retVal['Value']
repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring"))
retVal = repClient.generateDelayedPlot(*params)
return retVal
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
# Get unique key values
retVal = yield self.threadTask(self.__getUniqueKeyValues, typeName)
if not retVal['OK']:
self.finish({"success": "false", "result": "", "error": retVal['Message']})
return
records = {}
for record in retVal['Value']: # may have more than 1000 of records.
# do not show all of them in the web portal
length = len(retVal['Value'][record])
if length > 10000:
records[record] = retVal['Value'][record][length - 5000:]
message = "The %s accounting type contains to many rows: %s - > %d. Note: Only 1000 rows are returned!" % (
typeName, record, length)
gLogger.warn(message)
else:
records[record] = retVal['Value'][record]
callback["selectionValues"] = records
# Cache for plotsList?
data = MonitoringHandler.__keysCache.get("reportsList:%s" % typeName)
if not data:
repClient = MonitoringClient()
retVal = yield self.threadTask(repClient.listReports, typeName)
if not retVal['OK']:
self.finish({"success": "false", "result": "", "error": retVal['Message']})
return
data = retVal['Value']
MonitoringHandler.__keysCache.add("reportsList:%s" % typeName, 300, data)
callback["plotsList"] = data
self.finish({"success": "true", "result": callback})
def web_getPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
self.finish(rawData['data'])
def web_getCsvPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = MonitoringClient(
rpcClient=RPCClient("Monitoring/Monitoring"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
# print rawData['data']
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])),
granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
self.set_header('Content-type', 'text/csv')
self.set_header(
'Content-Disposition',
'attachment; filename="%s.csv"' % md5(str(params)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
from DIRAC.tests.Utilities.utils import find_all
from DIRAC.Core.Utilities.DIRACScript import DIRACScript as Script
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient
from DIRAC.Core.Tornado.Client.ClientSelector import TransferClientSelector as TransferClient
from DIRAC.Core.Utilities.JEncode import strToIntDict
#############################################
gLogger.setLevel("DEBUG")
client = MonitoringClient()
# fixture for preparation + teardown
@pytest.fixture
def putAndDelete():
# Find the test data
fj = find_all("WMSHistory_testData.json", "../",
"tests/Integration/Monitoring")[0]
with open(fj) as fp:
data = json.load(fp)
# put
res = client.addRecords("wmshistory_index", "WMSHistory", data)
assert res["OK"]
assert res["Value"] == len(data)
def setUp(self):
gLogger.setLevel('DEBUG')
self.client = MonitoringClient()
self.data = [{
u'Status': u'Waiting',
'Jobs': 2,
u'timestamp': 1458130176,
u'JobSplitType': u'MCStripping',
u'MinorStatus': u'unset',
u'Site': u'LCG.GRIDKA.de',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049848',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458130176,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'LCG.PIC.es',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'olupton',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458130176,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'LCG.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'olupton',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458130176,
u'JobSplitType': u'MCStripping',
u'MinorStatus': u'unset',
u'Site': u'LCG.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049845',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 34,
u'timestamp': 1458141578,
u'JobSplitType': u'DataStripping',
u'MinorStatus': u'unset',
u'Site': u'Group.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050299',
u'UserGroup': u'lhcb_data',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 120,
u'timestamp': 1458141578,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'LCG.CERN.ch',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'mvesteri',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458141578,
u'JobSplitType': u'MCStripping',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049845',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 2,
u'timestamp': 1458141578,
u'JobSplitType': u'MCStripping',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049848',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458141578,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050286',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 95,
u'timestamp': 1458199202,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'Multiple',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'mamartin',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 3,
u'timestamp': 1458199202,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'Multiple',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'olupton',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 129,
u'timestamp': 1458199202,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'Multiple',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049844',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 5,
u'timestamp': 1458217812,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.IHEP.su',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050232',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 7,
u'timestamp': 1458217812,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.IHEP.su',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050234',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 1,
u'timestamp': 1458217812,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.IHEP.su',
u'Reschedules': 1,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050236',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 3,
u'timestamp': 1458217812,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.IHEP.su',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050238',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 2,
u'timestamp': 1458217812,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.IHEP.su',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050248',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 12,
u'timestamp': 1458218413,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050248',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 5,
u'timestamp': 1458218413,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050250',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 4,
u'timestamp': 1458218413,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050251',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 1,
u'timestamp': 1458218413,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.CNAF.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050280',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 24,
u'timestamp': 1458219012,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.NIKHEF.nl',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050248',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 3,
u'timestamp': 1458219012,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.NIKHEF.nl',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050251',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 1,
u'timestamp': 1458222013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.Bologna.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050303',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 7,
u'timestamp': 1458222013,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'LCG.Bristol.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'clangenb',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 2,
u'timestamp': 1458222013,
u'JobSplitType': u'User',
u'MinorStatus': u'unset',
u'Site': u'LCG.Bristol.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'mrwillia',
u'JobGroup': u'lhcb',
u'UserGroup': u'lhcb_user',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 1,
u'timestamp': 1458222013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.Bari.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050244',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 11,
u'timestamp': 1458222013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.Bari.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050246',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 22,
u'timestamp': 1458222013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.Bari.it',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050248',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 23,
u'timestamp': 1458225013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.DESYZN.de',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049844',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 18,
u'timestamp': 1458225013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.DESYZN.de',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00049847',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 1,
u'timestamp': 1458225013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.DESYZN.de',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050238',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Running',
'Jobs': 1,
u'timestamp': 1458225013,
u'JobSplitType': u'MCSimulation',
u'MinorStatus': u'unset',
u'Site': u'LCG.DESYZN.de',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050246',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458226213,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.RRCKI.ru',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050243',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458226213,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.RRCKI.ru',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050251',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458226213,
u'JobSplitType': u'MCStripping',
u'MinorStatus': u'unset',
u'Site': u'LCG.RRCKI.ru',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050256',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458226213,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050229',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458226213,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050241',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 1,
u'timestamp': 1458226213,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050243',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}, {
u'Status': u'Waiting',
'Jobs': 2,
u'timestamp': 1458226213,
u'JobSplitType': u'MCReconstruction',
u'MinorStatus': u'unset',
u'Site': u'LCG.RAL.uk',
u'Reschedules': 0,
u'ApplicationStatus': u'unset',
u'User': u'phicharp',
u'JobGroup': u'00050247',
u'UserGroup': u'lhcb_mc',
u'metric': u'WMSHistory'
}]
def do_show( self, args ):
"""
Show list of components with various related information
usage:
show software - show components for which software is available
show installed - show components installed in the host with runit system
show setup - show components set up for automatic running in the host
show project - show project to install or upgrade
show status - show status of the installed components
show database - show status of the databases
show mysql - show status of the MySQL server
show log <system> <service|agent> [nlines]
- show last <nlines> lines in the component log file
show info - show version of software and setup
show doc <type> <system> <name>
- show documentation for a given service or agent
show host - show host related parameters
show hosts - show all available hosts
show ports [host] - show all ports used by a host. If no host is given, the host currently connected to is used
show installations [ list | current | -n <Name> | -h <Host> | -s <System> | -m <Module> | -t <Type> | -itb <InstallationTime before>
| -ita <InstallationTime after> | -utb <UnInstallationTime before> | -uta <UnInstallationTime after> ]*
- show all the installations of components that match the given parameters
show profile <system> <component> [ -s <size> | -h <host> | -id <initial date DD/MM/YYYY> | -it <initial time hh:mm>
| -ed <end date DD/MM/YYYY | -et <end time hh:mm> ]*
- show <size> log lines of profiling information for a component in the machine <host>
show errors [*|<system> <service|agent>]
- show error count for the given component or all the components
in the last hour and day
"""
argss = args.split()
if not argss:
gLogger.notice( self.do_show.__doc__ )
return
option = argss[0]
del argss[0]
if option == 'software':
client = SystemAdministratorClient( self.host, self.port )
result = client.getSoftwareComponents()
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( '' )
pprint.pprint( result['Value'] )
elif option == 'installed':
client = SystemAdministratorClient( self.host, self.port )
result = client.getInstalledComponents()
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( '' )
pprint.pprint( result['Value'] )
elif option == 'setup':
client = SystemAdministratorClient( self.host, self.port )
result = client.getSetupComponents()
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( '' )
pprint.pprint( result['Value'] )
elif option == 'project':
result = SystemAdministratorClient( self.host, self.port ).getProject()
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( "Current project is %s" % result[ 'Value' ] )
elif option == 'status':
client = SystemAdministratorClient( self.host, self.port )
result = client.getOverallStatus()
if not result['OK']:
self._errMsg( result['Message'] )
else:
fields = ["System",'Name','Module','Type','Setup','Installed','Runit','Uptime','PID']
records = []
rDict = result['Value']
for compType in rDict:
for system in rDict[compType]:
components = rDict[compType][system].keys()
components.sort()
for component in components:
record = []
if rDict[compType][system][component]['Installed']:
module = str( rDict[compType][system][component]['Module'] )
record += [ system,component,module,compType.lower()[:-1]]
if rDict[compType][system][component]['Setup']:
record += ['Setup']
else:
record += ['NotSetup']
if rDict[compType][system][component]['Installed']:
record += ['Installed']
else:
record += ['NotInstalled']
record += [str( rDict[compType][system][component]['RunitStatus'] )]
record += [str( rDict[compType][system][component]['Timeup'] )]
record += [str( rDict[compType][system][component]['PID'] )]
records.append(record)
printTable(fields,records)
elif option == 'database' or option == 'databases':
client = SystemAdministratorClient( self.host, self.port )
if not gComponentInstaller.mysqlPassword:
gComponentInstaller.mysqlPassword = "******"
gComponentInstaller.getMySQLPasswords()
result = client.getDatabases( gComponentInstaller.mysqlRootPwd )
if not result['OK']:
self._errMsg( result['Message'] )
return
resultSW = client.getAvailableDatabases()
if not resultSW['OK']:
self._errMsg( resultSW['Message'] )
return
sw = resultSW['Value']
installed = result['Value']
gLogger.notice( '' )
for db in sw:
if db in installed:
gLogger.notice( db.rjust( 25 ), ': Installed' )
else:
gLogger.notice( db.rjust( 25 ), ': Not installed' )
if not sw:
gLogger.notice( "No database found" )
elif option == 'mysql':
client = SystemAdministratorClient( self.host, self.port )
result = client.getMySQLStatus()
if not result['OK']:
self._errMsg( result['Message'] )
elif result['Value']:
gLogger.notice( '' )
for par, value in result['Value'].items():
gLogger.notice( ( par.rjust( 28 ), ':', value ) )
else:
gLogger.notice( "No MySQL database found" )
elif option == "log":
self.getLog( argss )
elif option == "info":
client = SystemAdministratorClient( self.host, self.port )
result = client.getInfo()
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( '' )
gLogger.notice( "Setup:", result['Value']['Setup'] )
gLogger.notice( "DIRAC version:", result['Value']['DIRAC'] )
if result['Value']['Extensions']:
for e, v in result['Value']['Extensions'].items():
gLogger.notice( "%s version" % e, v )
gLogger.notice( '' )
elif option == "host":
client = SystemAdministratorClient( self.host, self.port )
result = client.getHostInfo()
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( '' )
gLogger.notice( "Host info:" )
gLogger.notice( '' )
fields = ['Parameter','Value']
records = []
for parameter in result['Value'].iteritems():
if parameter[0] == 'Extension':
extensions = parameter[1].split( ',' )
for extension in extensions:
extensionName, extensionVersion = extension.split( ':' )
records.append( [ '%sVersion' % extensionName, str( extensionVersion ) ] )
else:
records.append( [ parameter[0], str( parameter[1] ) ] )
printTable( fields, records )
elif option == "hosts":
client = ComponentMonitoringClient()
result = client.getHosts( {}, False, False )
if not result[ 'OK' ]:
self._errMsg( 'Error retrieving the list of hosts: %s' % ( result[ 'Message' ] ) )
else:
hostList = result[ 'Value' ]
gLogger.notice( '' )
gLogger.notice( ' ' + 'Host'.center( 32 ) + ' ' + 'CPU'.center( 34 ) + ' ' )
gLogger.notice( ( '-' * 69 ) )
for element in hostList:
gLogger.notice( '|' + element[ 'HostName' ].center( 32 ) + '|' + element[ 'CPU' ].center( 34 ) + '|' )
gLogger.notice( ( '-' * 69 ) )
gLogger.notice( '' )
elif option == "ports":
if not argss:
client = SystemAdministratorClient( self.host )
else:
hostname = argss[0]
del argss[0]
client = ComponentMonitoringClient()
result = client.hostExists( { 'HostName': hostname } )
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
else:
if not result[ 'Value' ]:
self._errMsg( 'Given host does not exist' )
return
client = SystemAdministratorClient( hostname )
result = client.getUsedPorts()
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
pprint.pprint( result[ 'Value' ] )
elif option == "errors":
self.getErrors( argss )
elif option == "installations":
self.getInstallations( argss )
elif option == "doc":
if len( argss ) > 2:
if argss[0] in [ 'service', 'agent' ]:
compType = argss[0]
compSystem = argss[1]
compModule = argss[2]
client = SystemAdministratorClient( self.host, self.port )
result = client.getComponentDocumentation( compType, compSystem, compModule )
if result[ 'OK' ]:
gLogger.notice( result[ 'Value' ] )
else:
self._errMsg( result[ 'Message' ] )
else:
gLogger.notice( self.do_show.__doc__ )
else:
gLogger.notice( self.do_show.__doc__ )
elif option == "profile":
if len( argss ) > 1:
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
component = '%s_%s' % ( system, component )
argDict = { '-s': None, '-h': self.host, '-id': None, '-it': '00:00', '-ed': None, '-et': '00:00' }
key = None
for arg in argss:
if not key:
key = arg
else:
argDict[ key ] = arg
key = None
size = None
try:
if argDict[ '-s' ]:
size = int( argDict[ '-s' ] )
except ValueError as _ve:
self._errMsg( 'Argument \'size\' must be an integer' )
return
host = argDict[ '-h' ]
initialDate = argDict[ '-id' ]
initialTime = argDict[ '-it' ]
endingDate = argDict[ '-ed' ]
endingTime = argDict[ '-et' ]
if initialDate:
initialDate = '%s %s' % ( initialDate, initialTime )
else:
initialDate = ''
if endingDate:
endingDate = '%s %s' % ( endingDate, endingTime )
else:
endingDate = ''
client = MonitoringClient()
if size:
result = client.getLimitedData( host, component, size )
else:
result = client.getDataForAGivenPeriod( host, component, initialDate, endingDate )
if result[ 'OK' ]:
text = ''
headers = [result['Value'][0].keys()]
for header in headers:
text += str( header ).ljust( 15 )
gLogger.notice( text )
for record in result[ 'Value' ]:
for metric in record.itervalues():
text += str( metric ).ljust( 15 )
gLogger.notice( text )
else:
self._errMsg( result[ 'Message' ] )
else:
gLogger.notice( self.do_show.__doc__ )
else:
gLogger.notice( "Unknown option:", option )