def main(): #a list of workflows usage = "python %prog [OPTIONS] WORKFLOWS" parser = OptionParser(usage=usage) parser.add_option('-f', '--file', help='Text file with a list of workflows', dest='file') (options, args) = parser.parse_args() if options.file: workflows = [l.strip() for l in open(options.file) if l.strip()] else: workflows = args wMStats = WMStatsClient(url) print "start to getting job information from %s" % url #retrieve job information workflowsWithData = wMStats.getRequestByNames(workflows, jobInfoFlag = True) print '-'*120 #print workflowsWithData requestCol = RequestInfoCollection(workflowsWithData) #print summary for each workflow for wf, info in requestCol.getData().items(): print '-'*120 print wf #a table print '\n'.join( "%10s %10s"%(t,n) for t, n in info.getJobSummary().getJSONStatus().items())
def main(): # a list of workflows workflows = sys.argv[1:] wMStats = WMStatsClient(url) print "start to getting job information from %s" % url # retrieve job information workflowsWithData = wMStats.getRequestByNames(workflows, jobInfoFlag=True) print "-" * 120 # print workflowsWithData requestCol = RequestInfoCollection(workflowsWithData) # print summary for each workflow for wf, info in requestCol.getData().items(): print "-" * 120 print wf # a table print "\n".join("%10s %10s" % (t, n) for t, n in info.getJobSummary().getJSONStatus().items())
def main(): #a list of workflows workflows = sys.argv[1:] wMStats = WMStatsClient(url) print "start to getting job information from %s" % url #retrieve job information workflowsWithData = wMStats.getRequestByNames(workflows, jobInfoFlag = True) print '-'*120 #print workflowsWithData requestCol = RequestInfoCollection(workflowsWithData) #print summary for each workflow for wf, info in requestCol.getData().items(): print '-'*120 print wf #a table print '\n'.join( "%10s %10s"%(t,n) for t,n in info.getJobSummary().getJSONStatus().items())
def getJobSummary(workflow): """ Loads job summary from wmstats """ #JSON job summary from WMStats view jdata = {} try: jdata = wMStats.getRequestByNames([workflow], jobInfoFlag=True) requestCol = RequestInfoCollection(jdata) for wf2, info in requestCol.getData().items(): jdata = info.getJobSummary().getJSONStatus() except KeyError as e: print "ERROR, Cannot get job stats for", workflow print e except Exception as e: print "ERROR, Cannot get job stats for", workflow print e return jdata
def getJobSummary(workflow): """ Loads job summary from wmstats """ #JSON job summary from WMStats view jdata = {} try: jdata = wMStats.getRequestByNames([workflow], jobInfoFlag = True) requestCol = RequestInfoCollection(jdata) for wf2, info in requestCol.getData().items(): jdata = info.getJobSummary().getJSONStatus() except KeyError as e: print "ERROR, Cannot get job stats for", workflow print e except Exception as e: print "ERROR, Cannot get job stats for", workflow print e return jdata
def loadJobSummary(rList): """ Loads job summary from wmstats """ #JSON job summary from WMStats view wMStats = WMStatsClient("https://cmsweb.cern.ch/couchdb/wmstats") jlist = {} print "Getting job information" for wf in rList: try: jdata = wMStats.getRequestByNames([wf], jobInfoFlag = True) print wf requestCol = RequestInfoCollection(jdata) for wf2, info in requestCol.getData().items(): jlist[wf2] = {} for t,n in info.getJobSummary().getJSONStatus().items(): jlist[wf2][t]=n except KeyError as e: print "ERROR, Cannot get job stats for", wf print e except Exception as e: print "ERROR, Cannot get job stats for", wf print e return jlist
parser.add_option("-d", "--database", dest="database", default='latency_analytics', help="CouchDB database for results") (options, args) = parser.parse_args() analyticsServer = CouchServer(options.server) couchdb = analyticsServer.connectDatabase(options.database) url = "https://cmsweb.cern.ch/couchdb/wmstats" WMStats = WMStatsClient(url) print "Getting job information from %s. Please wait." % url requests = WMStats.getRequestByStatus(WMStatsClient.ACTIVE_STATUS, jobInfoFlag = True) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() requestsDict = requestCollection.getData() print "Total %s requests retrieved\n" % len(result) report = {} for wf in result.keys(): # Store a copy of the CouchDB document so we can compare later before updating if couchdb.documentExists(wf): oldCouchDoc = couchdb.document(wf) wfExists = True else: oldCouchDoc = CouchDoc(id=wf) wfExists = False newCouchDoc = copy.deepcopy(oldCouchDoc) ancientCouchDoc = copy.deepcopy(oldCouchDoc) report[wf] = oldCouchDoc
from WMCoreService.WMStatsClient import WMStatsClient from WMCoreService.DataStruct.RequestInfoCollection import RequestInfoCollection if __name__ == "__main__": url = "https://cmsweb-testbed.cern.ch/couchdb/wmstats" testbedWMStats = WMStatsClient(url) print "start to getting job information from %s" % url print "will take a while\n" requests = testbedWMStats.getRequestByStatus(["running-closed"], jobInfoFlag=True) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() print result print "\ntotal %s requests retrieved" % len(result) requestsDict = requestCollection.getData() for requestName, requestInfo in requestsDict.items(): print requestName + " :" print "\ttotalLuims: %s" % requestInfo.getTotalInputLumis() print "\ttotalEvents: %s" % requestInfo.getTotalInputEvents() print "\ttotal top level jobs: %s" % requestInfo.getTotalTopLevelJobs() print "\ttotal top level jobs in wmbs: %s" % requestInfo.getTotalTopLevelJobsInWMBS( ) print "\tProgress by output dataset:" summaryDict = requestInfo.getProgressSummaryByOutputDataset() for oDataset, summary in summaryDict.items(): print "\t %s" % oDataset print "\t %s" % summary.getReport() print "\n" print "done"