def getJobsInfo(vo_map, cp):
xml = runCommand(sge_job_info_cmd)
handler = JobInfoParser()
parseXmlSax(xml, handler)
job_info = handler.getJobInfo()
queue_jobs = {}
for job in job_info:
user = job['JB_owner']
state = job['state']
queue = job.get('queue_name', '')
if queue.strip() == '':
queue = 'waiting'
queue = queue.split('@')[0]
try:
vo = vo_map[user].lower()
except:
# Most likely, this means that the user is local and not
# associated with a VO, so we skip the job.
continue
voinfo = queue_jobs.setdefault(queue, {})
info = voinfo.setdefault(vo, {"running":0, "wait":0, "total":0})
if state == "r":
info["running"] += 1
else:
info["wait"] += 1
info["total"] += 1
info["vo"] = vo
log.debug("SGE job info: %s" % str(queue_jobs))
return queue_jobs
def getJobsInfo(vo_map, cp):
xml = runCommand(sge_job_info_cmd)
handler = JobInfoParser()
parseXmlSax(xml, handler)
job_info = handler.getJobInfo()
queue_jobs = {}
for job in job_info:
user = job['JB_owner']
state = job['state']
queue = job.get('queue_name', '')
if queue.strip() == '':
queue = 'waiting'
queue = queue.split('@')[0]
try:
vo = vo_map[user].lower()
except:
# Most likely, this means that the user is local and not
# associated with a VO, so we skip the job.
continue
voinfo = queue_jobs.setdefault(queue, {})
info = voinfo.setdefault(vo, {"running": 0, "wait": 0, "total": 0})
if state == "r":
info["running"] += 1
else:
info["wait"] += 1
info["total"] += 1
info["vo"] = vo
log.debug("SGE job info: %s" % str(queue_jobs))
return queue_jobs
def writeResultsPage(self):
contents = '<?xml version="1.0" encoding="UTF-8"?>\n'
contents += '<?xml-stylesheet type="text/xsl" href="index.xsl"?>\n'
contents += '<TestRunList>\n'
siteTestStatus = ""
updateDateTime = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
contents += "<TestRunTime><![CDATA[%s]]></TestRunTime>\n" % updateDateTime
for dict in self.output_files:
file = dict["file"]
type = dict["type"]
fullPathItems = file.split("/")
display = os.path.basename(file).replace("_", " ")[:-4]
contents += "<TestDetail path='%s' type='%s'>%s</TestDetail>\n" % (
os.path.basename(file), type, display)
if type == 'critical':
xml = open(file, "r")
handler = GipResultsParser()
parseXmlSax(xml, handler)
crit_test_results = handler.getGipResults()
cases = crit_test_results["cases"]
for case in cases:
sitename = case["site"]
count = int(case["failure_count"]) + int(
case["error_count"]) + int(case["info_count"])
if count > 0:
result = "red"
else:
result = "green"
siteTestStatus += "<Site name='%s' test='%s' result='%s' path='%s'/>\n" % (
sitename, os.path.basename(file)[:-4], result,
os.path.basename(file))
# need to build site status table here
contents += siteTestStatus
contents += '</TestRunList>'
output_file = "%s/index.xml" % self.output_dir
fileOverWrite(output_file, contents)
return output_file
def writeResultsPage(self):
contents = '<?xml version="1.0" encoding="UTF-8"?>\n'
contents += '<?xml-stylesheet type="text/xsl" href="index.xsl"?>\n'
contents += '<TestRunList>\n'
siteTestStatus = ""
updateDateTime = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
contents += "<TestRunTime><![CDATA[%s]]></TestRunTime>\n" % updateDateTime
for dict in self.output_files:
file = dict["file"]
type = dict["type"]
fullPathItems = file.split("/")
display = os.path.basename(file).replace("_", " ")[:-4]
contents += "<TestDetail path='%s' type='%s'>%s</TestDetail>\n" % (os.path.basename(file), type, display)
if type == 'critical':
xml = open(file, "r")
handler = GipResultsParser()
parseXmlSax(xml, handler)
crit_test_results = handler.getGipResults()
cases = crit_test_results["cases"]
for case in cases:
sitename = case["site"]
count = int(case["failure_count"]) + int(case["error_count"]) + int(case["info_count"])
if count > 0:
result = "red"
else:
result = "green"
siteTestStatus += "<Site name='%s' test='%s' result='%s' path='%s'/>\n" % (sitename, os.path.basename(file)[:-4], result, os.path.basename(file))
# need to build site status table here
contents += siteTestStatus
contents += '</TestRunList>'
output_file = "%s/index.xml" % self.output_dir
fileOverWrite(output_file, contents)
return output_file
def getQueueInfo(cp):
"""
Looks up the queue and job information from SGE.
@param cp: Configuration of site.
@returns: A dictionary of queue data and a dictionary of job data.
"""
queue_list = {}
xml = runCommand(sge_queue_info_cmd)
handler = QueueInfoParser()
parseXmlSax(xml, handler)
queue_info = handler.getQueueInfo()
for queue, qinfo in queue_info.items():
if queue == 'waiting':
continue
# get queue name
name = queue.split("@")[0]
q = queue_list.get(name, {'slots_used': 0, 'slots_total': 0,
'slots_free': 0, 'waiting' : 0, 'name' : name})
try:
q['slots_used'] += int(qinfo['slots_used'])
except:
pass
try:
q['slots_total'] += int(qinfo['slots_total'])
except:
pass
q['slots_free'] = q['slots_total'] - q['slots_used']
if 'arch' in qinfo:
q['arch'] = qinfo['arch']
q['max_running'] = q['slots_total']
try:
state = queue_info[queue]["state"]
if state.find("d") >= 0 or state.find("D") >= 0:
status = "Draining"
elif state.find("s") >= 0:
status = "Closed"
else:
status = "Production"
except:
status = "Production"
q['status'] = status
q['priority'] = 0 # No such thing that I can find for a queue
# How do you handle queues with no limit?
sqc = SGEQueueConfig(sgeCommand(sge_queue_config_cmd % name, cp))
try:
q['priority'] = int(sqc['priority'])
except:
pass
max_wall_hard = convert_time_to_secs(sqc.get('h_rt', 'INFINITY'))
max_wall_soft = convert_time_to_secs(sqc.get('s_rt', 'INFINITY'))
max_wall = min(max_wall_hard, max_wall_soft)
try:
q['max_wall'] = min(max_wall, q['max_wall'])
except:
q['max_wall'] = max_wall
user_list = sqc.get('user_lists', 'NONE')
if user_list.lower().find('none') >= 0:
user_list = re.split('\s*,?\s*', user_list)
if 'all' in user_list:
user_list = []
q['user_list'] = user_list
queue_list[name] = q
waiting_jobs = 0
for job in queue_info['waiting']:
waiting_jobs += 1
queue_list['waiting'] = {'waiting': waiting_jobs}
return queue_list, queue_info
def getQueueInfo(cp):
"""
Looks up the queue and job information from SGE.
@param cp: Configuration of site.
@returns: A dictionary of queue data and a dictionary of job data.
"""
queue_list = {}
xml = runCommand(sge_queue_info_cmd)
handler = QueueInfoParser()
parseXmlSax(xml, handler)
queue_info = handler.getQueueInfo()
for queue, qinfo in queue_info.items():
if queue == 'waiting':
continue
# get queue name
name = queue.split("@")[0]
q = queue_list.get(
name, {
'slots_used': 0,
'slots_total': 0,
'slots_free': 0,
'waiting': 0,
'name': name
})
try:
q['slots_used'] += int(qinfo['slots_used'])
except:
pass
try:
q['slots_total'] += int(qinfo['slots_total'])
except:
pass
q['slots_free'] = q['slots_total'] - q['slots_used']
if 'arch' in qinfo:
q['arch'] = qinfo['arch']
q['max_running'] = q['slots_total']
try:
state = queue_info[queue]["state"]
if state.find("d") >= 0 or state.find("D") >= 0:
status = "Draining"
elif state.find("s") >= 0:
status = "Closed"
else:
status = "Production"
except:
status = "Production"
q['status'] = status
q['priority'] = 0 # No such thing that I can find for a queue
# How do you handle queues with no limit?
sqc = SGEQueueConfig(sgeCommand(sge_queue_config_cmd % name, cp))
try:
q['priority'] = int(sqc['priority'])
except:
pass
max_wall_hard = convert_time_to_secs(sqc.get('h_rt', 'INFINITY'))
max_wall_soft = convert_time_to_secs(sqc.get('s_rt', 'INFINITY'))
max_wall = min(max_wall_hard, max_wall_soft)
try:
q['max_wall'] = min(max_wall, q['max_wall'])
except:
q['max_wall'] = max_wall
user_list = sqc.get('user_lists', 'NONE')
if user_list.lower().find('none') >= 0:
user_list = re.split('\s*,?\s*', user_list)
if 'all' in user_list:
user_list = []
q['user_list'] = user_list
queue_list[name] = q
waiting_jobs = 0
for job in queue_info['waiting']:
waiting_jobs += 1
queue_list['waiting'] = {'waiting': waiting_jobs}
return queue_list, queue_info
