def main():
#a list of workflows
usage = "python %prog [OPTIONS] WORKFLOWS"
parser = OptionParser(usage=usage)
parser.add_option('-f', '--file', help='Text file with a list of workflows', dest='file')
(options, args) = parser.parse_args()
if options.file:
workflows = [l.strip() for l in open(options.file) if l.strip()]
else:
workflows = args
wMStats = WMStatsClient(url)
print "start to getting job information from %s" % url
#retrieve job information
workflowsWithData = wMStats.getRequestByNames(workflows, jobInfoFlag = True)
print '-'*120
#print workflowsWithData
requestCol = RequestInfoCollection(workflowsWithData)
#print summary for each workflow
for wf, info in requestCol.getData().items():
print '-'*120
print wf
#a table
print '\n'.join( "%10s %10s"%(t,n)
for t, n in info.getJobSummary().getJSONStatus().items())
def main():
# a list of workflows
workflows = sys.argv[1:]
wMStats = WMStatsClient(url)
print "start to getting job information from %s" % url
# retrieve job information
workflowsWithData = wMStats.getRequestByNames(workflows, jobInfoFlag=True)
print "-" * 120
# print workflowsWithData
requestCol = RequestInfoCollection(workflowsWithData)
# print summary for each workflow
for wf, info in requestCol.getData().items():
print "-" * 120
print wf
# a table
print "\n".join("%10s %10s" % (t, n) for t, n in info.getJobSummary().getJSONStatus().items())
def main():
#a list of workflows
workflows = sys.argv[1:]
wMStats = WMStatsClient(url)
print "start to getting job information from %s" % url
#retrieve job information
workflowsWithData = wMStats.getRequestByNames(workflows, jobInfoFlag = True)
print '-'*120
#print workflowsWithData
requestCol = RequestInfoCollection(workflowsWithData)
#print summary for each workflow
for wf, info in requestCol.getData().items():
print '-'*120
print wf
#a table
print '\n'.join( "%10s %10s"%(t,n)
for t,n in info.getJobSummary().getJSONStatus().items())
def getJobSummary(workflow):
"""
Loads job summary from wmstats
"""
#JSON job summary from WMStats view
jdata = {}
try:
jdata = wMStats.getRequestByNames([workflow], jobInfoFlag=True)
requestCol = RequestInfoCollection(jdata)
for wf2, info in requestCol.getData().items():
jdata = info.getJobSummary().getJSONStatus()
except KeyError as e:
print "ERROR, Cannot get job stats for", workflow
print e
except Exception as e:
print "ERROR, Cannot get job stats for", workflow
print e
return jdata
def getJobSummary(workflow):
"""
Loads job summary from wmstats
"""
#JSON job summary from WMStats view
jdata = {}
try:
jdata = wMStats.getRequestByNames([workflow], jobInfoFlag = True)
requestCol = RequestInfoCollection(jdata)
for wf2, info in requestCol.getData().items():
jdata = info.getJobSummary().getJSONStatus()
except KeyError as e:
print "ERROR, Cannot get job stats for", workflow
print e
except Exception as e:
print "ERROR, Cannot get job stats for", workflow
print e
return jdata
def loadJobSummary(rList):
"""
Loads job summary from wmstats
"""
#JSON job summary from WMStats view
wMStats = WMStatsClient("https://cmsweb.cern.ch/couchdb/wmstats")
jlist = {}
print "Getting job information"
for wf in rList:
try:
jdata = wMStats.getRequestByNames([wf], jobInfoFlag = True)
print wf
requestCol = RequestInfoCollection(jdata)
for wf2, info in requestCol.getData().items():
jlist[wf2] = {}
for t,n in info.getJobSummary().getJSONStatus().items():
jlist[wf2][t]=n
except KeyError as e:
print "ERROR, Cannot get job stats for", wf
print e
except Exception as e:
print "ERROR, Cannot get job stats for", wf
print e
return jlist
parser.add_option("-s", "--server", dest="server",
help="CouchDB server to write results to", )
parser.add_option("-d", "--database", dest="database", default='latency_analytics',
help="CouchDB database for results")
(options, args) = parser.parse_args()
analyticsServer = CouchServer(options.server)
couchdb = analyticsServer.connectDatabase(options.database)
url = "https://cmsweb.cern.ch/couchdb/wmstats"
WMStats = WMStatsClient(url)
print "Getting job information from %s. Please wait." % url
requests = WMStats.getRequestByStatus(WMStatsClient.ACTIVE_STATUS, jobInfoFlag = True)
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
requestsDict = requestCollection.getData()
print "Total %s requests retrieved\n" % len(result)
report = {}
for wf in result.keys():
# Store a copy of the CouchDB document so we can compare later before updating
if couchdb.documentExists(wf):
oldCouchDoc = couchdb.document(wf)
wfExists = True
else:
oldCouchDoc = CouchDoc(id=wf)
wfExists = False
newCouchDoc = copy.deepcopy(oldCouchDoc)
from WMCoreService.WMStatsClient import WMStatsClient
from WMCoreService.DataStruct.RequestInfoCollection import RequestInfoCollection
if __name__ == "__main__":
url = "https://cmsweb-testbed.cern.ch/couchdb/wmstats"
testbedWMStats = WMStatsClient(url)
print "start to getting job information from %s" % url
print "will take a while\n"
requests = testbedWMStats.getRequestByStatus(["running-closed"],
jobInfoFlag=True)
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
print result
print "\ntotal %s requests retrieved" % len(result)
requestsDict = requestCollection.getData()
for requestName, requestInfo in requestsDict.items():
print requestName + " :"
print "\ttotalLuims: %s" % requestInfo.getTotalInputLumis()
print "\ttotalEvents: %s" % requestInfo.getTotalInputEvents()
print "\ttotal top level jobs: %s" % requestInfo.getTotalTopLevelJobs()
print "\ttotal top level jobs in wmbs: %s" % requestInfo.getTotalTopLevelJobsInWMBS(
)
print "\tProgress by output dataset:"
summaryDict = requestInfo.getProgressSummaryByOutputDataset()
for oDataset, summary in summaryDict.items():
print "\t %s" % oDataset
print "\t %s" % summary.getReport()
print "\n"
print "done"