def getJobsInfo(vo_map, cp): xml = runCommand(sge_job_info_cmd) handler = JobInfoParser() parseXmlSax(xml, handler) job_info = handler.getJobInfo() queue_jobs = {} for job in job_info: user = job['JB_owner'] state = job['state'] queue = job.get('queue_name', '') if queue.strip() == '': queue = 'waiting' queue = queue.split('@')[0] try: vo = vo_map[user].lower() except: # Most likely, this means that the user is local and not # associated with a VO, so we skip the job. continue voinfo = queue_jobs.setdefault(queue, {}) info = voinfo.setdefault(vo, {"running":0, "wait":0, "total":0}) if state == "r": info["running"] += 1 else: info["wait"] += 1 info["total"] += 1 info["vo"] = vo log.debug("SGE job info: %s" % str(queue_jobs)) return queue_jobs
def getJobsInfo(vo_map, cp): xml = runCommand(sge_job_info_cmd) handler = JobInfoParser() parseXmlSax(xml, handler) job_info = handler.getJobInfo() queue_jobs = {} for job in job_info: user = job['JB_owner'] state = job['state'] queue = job.get('queue_name', '') if queue.strip() == '': queue = 'waiting' queue = queue.split('@')[0] try: vo = vo_map[user].lower() except: # Most likely, this means that the user is local and not # associated with a VO, so we skip the job. continue voinfo = queue_jobs.setdefault(queue, {}) info = voinfo.setdefault(vo, {"running": 0, "wait": 0, "total": 0}) if state == "r": info["running"] += 1 else: info["wait"] += 1 info["total"] += 1 info["vo"] = vo log.debug("SGE job info: %s" % str(queue_jobs)) return queue_jobs
def writeResultsPage(self): contents = '<?xml version="1.0" encoding="UTF-8"?>\n' contents += '<?xml-stylesheet type="text/xsl" href="index.xsl"?>\n' contents += '<TestRunList>\n' siteTestStatus = "" updateDateTime = time.strftime("%a %b %d %T UTC %Y", time.gmtime()) contents += "<TestRunTime><![CDATA[%s]]></TestRunTime>\n" % updateDateTime for dict in self.output_files: file = dict["file"] type = dict["type"] fullPathItems = file.split("/") display = os.path.basename(file).replace("_", " ")[:-4] contents += "<TestDetail path='%s' type='%s'>%s</TestDetail>\n" % ( os.path.basename(file), type, display) if type == 'critical': xml = open(file, "r") handler = GipResultsParser() parseXmlSax(xml, handler) crit_test_results = handler.getGipResults() cases = crit_test_results["cases"] for case in cases: sitename = case["site"] count = int(case["failure_count"]) + int( case["error_count"]) + int(case["info_count"]) if count > 0: result = "red" else: result = "green" siteTestStatus += "<Site name='%s' test='%s' result='%s' path='%s'/>\n" % ( sitename, os.path.basename(file)[:-4], result, os.path.basename(file)) # need to build site status table here contents += siteTestStatus contents += '</TestRunList>' output_file = "%s/index.xml" % self.output_dir fileOverWrite(output_file, contents) return output_file
def writeResultsPage(self): contents = '<?xml version="1.0" encoding="UTF-8"?>\n' contents += '<?xml-stylesheet type="text/xsl" href="index.xsl"?>\n' contents += '<TestRunList>\n' siteTestStatus = "" updateDateTime = time.strftime("%a %b %d %T UTC %Y", time.gmtime()) contents += "<TestRunTime><![CDATA[%s]]></TestRunTime>\n" % updateDateTime for dict in self.output_files: file = dict["file"] type = dict["type"] fullPathItems = file.split("/") display = os.path.basename(file).replace("_", " ")[:-4] contents += "<TestDetail path='%s' type='%s'>%s</TestDetail>\n" % (os.path.basename(file), type, display) if type == 'critical': xml = open(file, "r") handler = GipResultsParser() parseXmlSax(xml, handler) crit_test_results = handler.getGipResults() cases = crit_test_results["cases"] for case in cases: sitename = case["site"] count = int(case["failure_count"]) + int(case["error_count"]) + int(case["info_count"]) if count > 0: result = "red" else: result = "green" siteTestStatus += "<Site name='%s' test='%s' result='%s' path='%s'/>\n" % (sitename, os.path.basename(file)[:-4], result, os.path.basename(file)) # need to build site status table here contents += siteTestStatus contents += '</TestRunList>' output_file = "%s/index.xml" % self.output_dir fileOverWrite(output_file, contents) return output_file
def getQueueInfo(cp): """ Looks up the queue and job information from SGE. @param cp: Configuration of site. @returns: A dictionary of queue data and a dictionary of job data. """ queue_list = {} xml = runCommand(sge_queue_info_cmd) handler = QueueInfoParser() parseXmlSax(xml, handler) queue_info = handler.getQueueInfo() for queue, qinfo in queue_info.items(): if queue == 'waiting': continue # get queue name name = queue.split("@")[0] q = queue_list.get(name, {'slots_used': 0, 'slots_total': 0, 'slots_free': 0, 'waiting' : 0, 'name' : name}) try: q['slots_used'] += int(qinfo['slots_used']) except: pass try: q['slots_total'] += int(qinfo['slots_total']) except: pass q['slots_free'] = q['slots_total'] - q['slots_used'] if 'arch' in qinfo: q['arch'] = qinfo['arch'] q['max_running'] = q['slots_total'] try: state = queue_info[queue]["state"] if state.find("d") >= 0 or state.find("D") >= 0: status = "Draining" elif state.find("s") >= 0: status = "Closed" else: status = "Production" except: status = "Production" q['status'] = status q['priority'] = 0 # No such thing that I can find for a queue # How do you handle queues with no limit? sqc = SGEQueueConfig(sgeCommand(sge_queue_config_cmd % name, cp)) try: q['priority'] = int(sqc['priority']) except: pass max_wall_hard = convert_time_to_secs(sqc.get('h_rt', 'INFINITY')) max_wall_soft = convert_time_to_secs(sqc.get('s_rt', 'INFINITY')) max_wall = min(max_wall_hard, max_wall_soft) try: q['max_wall'] = min(max_wall, q['max_wall']) except: q['max_wall'] = max_wall user_list = sqc.get('user_lists', 'NONE') if user_list.lower().find('none') >= 0: user_list = re.split('\s*,?\s*', user_list) if 'all' in user_list: user_list = [] q['user_list'] = user_list queue_list[name] = q waiting_jobs = 0 for job in queue_info['waiting']: waiting_jobs += 1 queue_list['waiting'] = {'waiting': waiting_jobs} return queue_list, queue_info
def getQueueInfo(cp): """ Looks up the queue and job information from SGE. @param cp: Configuration of site. @returns: A dictionary of queue data and a dictionary of job data. """ queue_list = {} xml = runCommand(sge_queue_info_cmd) handler = QueueInfoParser() parseXmlSax(xml, handler) queue_info = handler.getQueueInfo() for queue, qinfo in queue_info.items(): if queue == 'waiting': continue # get queue name name = queue.split("@")[0] q = queue_list.get( name, { 'slots_used': 0, 'slots_total': 0, 'slots_free': 0, 'waiting': 0, 'name': name }) try: q['slots_used'] += int(qinfo['slots_used']) except: pass try: q['slots_total'] += int(qinfo['slots_total']) except: pass q['slots_free'] = q['slots_total'] - q['slots_used'] if 'arch' in qinfo: q['arch'] = qinfo['arch'] q['max_running'] = q['slots_total'] try: state = queue_info[queue]["state"] if state.find("d") >= 0 or state.find("D") >= 0: status = "Draining" elif state.find("s") >= 0: status = "Closed" else: status = "Production" except: status = "Production" q['status'] = status q['priority'] = 0 # No such thing that I can find for a queue # How do you handle queues with no limit? sqc = SGEQueueConfig(sgeCommand(sge_queue_config_cmd % name, cp)) try: q['priority'] = int(sqc['priority']) except: pass max_wall_hard = convert_time_to_secs(sqc.get('h_rt', 'INFINITY')) max_wall_soft = convert_time_to_secs(sqc.get('s_rt', 'INFINITY')) max_wall = min(max_wall_hard, max_wall_soft) try: q['max_wall'] = min(max_wall, q['max_wall']) except: q['max_wall'] = max_wall user_list = sqc.get('user_lists', 'NONE') if user_list.lower().find('none') >= 0: user_list = re.split('\s*,?\s*', user_list) if 'all' in user_list: user_list = [] q['user_list'] = user_list queue_list[name] = q waiting_jobs = 0 for job in queue_info['waiting']: waiting_jobs += 1 queue_list['waiting'] = {'waiting': waiting_jobs} return queue_list, queue_info