def get_bhosts_load_info(command='bhosts -l'):
"""
Get "CURRENT LOAD USED FOR SCHEDULING" information with command
====
HOST n212-206-212
STATUS CPUF JL/U MAX NJOBS RUN SSUSP USUSP RSV DISPATCH_WINDOW
ok 15.00 - 48 2 2 0 0 0 -
CURRENT LOAD USED FOR SCHEDULING:
r15s r1m r15m ut pg io ls it tmp swp mem slots
Total 0.0 0.0 0.0 2% 0.0 8 0 14324 1667_g 127.2_g 683_g 46
Reserved 0.0 0.0 0.0 0% 0.0 0 0 0 0_m 0_m 178_g -
====
"""
bhosts_load_dic = collections.OrderedDict()
load_info_mark = False
hostname = ''
head_list = []
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.match('^\s*HOST\s+(.+?)\s*$', line):
my_match = re.match('^\s*HOST\s+(.+?)\s*$', line)
hostname = my_match.group(1)
bhosts_load_dic.setdefault(hostname, {})
load_info_mark = False
elif re.match('^\s*CURRENT LOAD USED FOR SCHEDULING:\s*$', line):
load_info_mark = True
elif load_info_mark:
if re.match('^\s*$', line):
load_info_mark = False
elif re.match('^\s*Total\s+(.+?)\s*$', line):
bhosts_load_dic[hostname].setdefault('Total', {})
my_match = re.match('^\s*Total\s+(.+?)\s*$', line)
total_load_string = my_match.group(1)
total_load_list = total_load_string.split()
for (i, head_name) in enumerate(head_list):
load = re.sub('\*', '', total_load_list[i])
bhosts_load_dic[hostname]['Total'].setdefault(
head_name, load)
elif re.match('^\s*Reserved\s+(.+?)\s*$', line):
bhosts_load_dic[hostname].setdefault('Reserved', {})
my_match = re.match('^\s*Reserved\s+(.+?)\s*$', line)
reserved_load_string = my_match.group(1)
reserved_load_list = reserved_load_string.split()
for (i, head_name) in enumerate(head_list):
load = re.sub('\*', '', reserved_load_list[i])
bhosts_load_dic[hostname]['Reserved'].setdefault(
head_name, load)
else:
head_list = line.split()
return (bhosts_load_dic)
def get_queue_host_info():
"""
Get hosts on (specified) queues.
"""
queue_host_dic = {}
queue_compile = re.compile('^QUEUE:\s*(\S+)\s*$')
hosts_compile = re.compile('^HOSTS:\s*(.*?)\s*$')
queue = ''
command = 'bqueues -l'
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if queue_compile.match(line):
my_match = queue_compile.match(line)
queue = my_match.group(1)
queue_host_dic[queue] = []
if hosts_compile.match(line):
my_match = hosts_compile.match(line)
hosts_string = my_match.group(1)
if hosts_string == 'all':
common.print_warning(
'*Warning* (get_queue_host_info) : queue "' + str(queue) +
'" is not well configured, all of the hosts are on the same queue.'
)
queue_host_dic[queue] = get_host_list()
else:
queue_host_dic.setdefault(queue, [])
hosts_list = hosts_string.split()
for hosts in hosts_list:
if re.match('.+/', hosts):
host_group_name = re.sub('/$', '', hosts)
host_list = get_host_group_members(host_group_name)
if len(host_list) > 0:
queue_host_dic[queue].extend(host_list)
elif re.match('^(.+)\+\d+$', hosts):
my_match = re.match('^(.+)\+\d+$', hosts)
host_group_name = my_match.group(1)
host_list = get_host_group_members(host_group_name)
if len(host_list) == 0:
queue_host_dic[queue].append(hosts)
else:
queue_host_dic[queue].extend(host_list)
else:
queue_host_dic[queue].append(hosts)
return (queue_host_dic)
def get_command_dict(command):
"""
Collect (common) LSF command info into a dict.
It only works with the Title-Item type informations.
"""
my_dic = collections.OrderedDict()
key_list = []
(return_code, stdout, stderr) = common.run_command(command)
i = -1
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if line:
i += 1
# Some speciall preprocess.
if re.search('lsload', command):
line = re.sub('\*', ' ', line)
if i == 0:
key_list = line.split()
for key in key_list:
my_dic[key] = []
else:
command_info = line.split()
if len(command_info) < len(key_list):
common.print_warning(
'*Warning* (get_command_dict) : For command "' +
str(command) +
'", below info line is incomplate/unexpected.')
common.print_warning(' ' + str(line))
for j in range(len(key_list)):
key = key_list[j]
if j < len(command_info):
value = command_info[j]
else:
value = ''
my_dic[key].append(value)
return (my_dic)
def get_tool_name():
"""
Make sure it is lsf or openlava.
"""
command = 'lsid'
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.search('LSF', line):
return ('lsf')
elif re.search('Open_lava', line) or re.search('openlava', line):
return ('openlava')
print('*Warning*: Not sure current cluster is LSF or Openlava.')
return ('')
def get_process_info(self):
process_dic = {
'user': [],
'pid': [],
'cpu': [],
'mem': [],
'stat': [],
'started': [],
'command': [],
}
command = 'ps -o ruser=userForLongName -o pid,%cpu,%mem,stat,start,command -f' + ','.join(self.pid_list)
if self.job:
bsub_command = self.get_bsub_command()
command = str(bsub_command) + " '" + str(command) + "'"
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.match('^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+([a-zA-Z]{3} \d{2}|\d{2}:\d{2}:\d{2})\s(.+)$', line):
my_match = re.match('^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+([a-zA-Z]{3} \d{2}|\d{2}:\d{2}:\d{2})\s(.+)$', line)
user = my_match.group(1)
pid = my_match.group(2)
cpu = my_match.group(3)
mem = my_match.group(4)
stat = my_match.group(5)
started = my_match.group(6)
command = my_match.group(7)
process_dic['user'].append(user)
process_dic['pid'].append(pid)
process_dic['cpu'].append(cpu)
process_dic['mem'].append(mem)
process_dic['stat'].append(stat)
process_dic['started'].append(started)
process_dic['command'].append(command)
else:
continue
return(process_dic)
def check_pid(self, pid):
pid_list = []
command = 'pstree -p ' + str(pid)
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.findall('\((\d+)\)', line):
tmp_pid_list = re.findall('\((\d+)\)', line)
if tmp_pid_list:
pid_list.extend(tmp_pid_list)
if not pid_list:
common.print_error('*Error*: No valid pid was found.')
sys.exit(1)
return(pid_list)
def get_user_group_members(user_group_name):
"""
Get user group members with bugroup.
====
[yanqing.li@nxnode03 lsfMonitor]$ bugroup pd
GROUP_NAME USERS
pd yanqing.li san.zhang si.li
====
"""
user_list = []
command = 'bugroup -r ' + str(user_group_name)
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.match('^' + str(user_group_name) + ' .*$', line):
my_list = line.split()
user_list = my_list[1:]
return (user_list)
def get_host_group_members(host_group_name):
"""
Get host group members with bmgroup.
====
[yanqing.li@nxnode03 lsfMonitor]$ bmgroup pd
GROUP_NAME HOSTS
pd dm006 dm007 dm010 dm009 dm002 dm003 dm005
====
"""
host_list = []
command = 'bmgroup -w -r ' + str(host_group_name)
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.search('No such user/host group', line):
break
elif re.match('^' + str(host_group_name) + ' .*$', line):
my_list = line.split()
host_list = my_list[1:]
return (host_list)
def get_openlava_bjobs_uf_info(command):
"""
Parse job info which are from command 'bjobs -u all -UF'.
====
Job <205>, User <liyanqing>, Project <default>, Status <PEND>, Queue <normal>, Command <sleep 1000>
Sun May 13 18:08:26: Submitted from host <lava_host1>, CWD <$HOME>, 2 Processors Requested, Requested Resources <rusage[mem=1234] span[hosts=1]>;
PENDING REASONS:
New job is waiting for scheduling: 1 host;
SCHEDULING PARAMETERS:
r15s r1m r15m ut pg io ls it tmp swp mem
load_sched - - - - - - - - - - -
load_stop - - - - - - - - - - -
RESOURCE REQUIREMENT DETAILS:
Combined: rusage[mem=1234] span[hosts=1]
Effective: rusage[mem=1234] span[hosts=1]
====
"""
job_compile_dic = {
'job_compile':
re.compile('.*Job <([0-9]+(\[[0-9]+\])?)>.*'),
'job_name_compile':
re.compile('.*Job Name <([^>]+)>.*'),
'user_compile':
re.compile('.*User <([^>]+)>.*'),
'project_compile':
re.compile('.*Project <([^>]+)>.*'),
'status_compile':
re.compile('.*Status <([A-Z]+)>*'),
'queue_compile':
re.compile('.*Queue <([^>]+)>.*'),
'command_compile':
re.compile('.*Command <(.+?\S)>\s*$'),
'submitted_from_compile':
re.compile('.*Submitted from host <([^>]+)>.*'),
'submitted_time_compile':
re.compile('(.*): Submitted from host.*'),
'cwd_compile':
re.compile('.*CWD <([^>]+)>.*'),
'processors_requested_compile':
re.compile('.* ([1-9][0-9]*) Processors Requested.*'),
'requested_resources_compile':
re.compile('.*Requested Resources <(.+)>;.*'),
'span_hosts_compile':
re.compile('.*Requested Resources <.*span\[hosts=([1-9][0-9]*).*>.*'),
'rusage_mem_compile':
re.compile('.*Requested Resources <.*rusage\[mem=([1-9][0-9]*).*>.*'),
'started_on_compile':
re.compile('.*[sS]tarted on ([0-9]+ Hosts/Processors )?([^;,]+).*'),
'started_time_compile':
re.compile('(.*): (\[\d+\])?\s*[sS]tarted on.*'),
'finished_time_compile':
re.compile('(.*): (Done successfully|Exited with).*'),
'exit_code_compile':
re.compile('.*Exited with exit code (\d+)\..*'),
'lsf_signal_compile':
re.compile('.*Exited by LSF signal (\S+?)\..*'),
'term_owner_compile':
re.compile('.*TERM_OWNER: (.+?\.).*'),
'cpu_time_compile':
re.compile('.*The CPU time used is ([1-9][0-9]*) seconds.*'),
'mem_compile':
re.compile('.*MEM: ([1-9][0-9]*) Mbytes.*'),
}
my_dic = collections.OrderedDict()
job = ''
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.match('Job <' + str(job) + '> is not found', line):
continue
else:
if job_compile_dic['job_compile'].match(line):
my_match = job_compile_dic['job_compile'].match(line)
job = my_match.group(1)
# Initialization for my_dic[job].
my_dic[job] = collections.OrderedDict()
my_dic[job]['job_info'] = ''
my_dic[job]['job_id'] = job
my_dic[job]['job_name'] = ''
my_dic[job]['user'] = ''
my_dic[job]['project'] = ''
my_dic[job]['status'] = ''
my_dic[job]['queue'] = ''
my_dic[job]['command'] = ''
my_dic[job]['submitted_from'] = ''
my_dic[job]['submitted_time'] = ''
my_dic[job]['cwd'] = ''
my_dic[job]['processors_requested'] = '1'
my_dic[job]['requested_resources'] = ''
my_dic[job]['span_hosts'] = ''
my_dic[job]['rusage_mem'] = ''
my_dic[job]['started_on'] = ''
my_dic[job]['started_time'] = ''
my_dic[job]['finished_time'] = ''
my_dic[job]['exit_code'] = ''
my_dic[job]['lsf_signal'] = ''
my_dic[job]['term_owner'] = ''
my_dic[job]['cpu_time'] = ''
my_dic[job]['mem'] = ''
my_dic[job]['swap'] = ''
my_dic[job]['run_limit'] = ''
my_dic[job]['pids'] = []
my_dic[job]['max_mem'] = ''
my_dic[job]['avg_mem'] = ''
my_dic[job]['pending_reasons'] = []
if job != '':
if my_dic[job]['job_info']:
my_dic[job]['job_info'] = str(
my_dic[job]['job_info']) + '\n' + str(line)
else:
my_dic[job]['job_info'] = line
if job_compile_dic['job_name_compile'].match(line):
my_match = job_compile_dic['job_name_compile'].match(line)
my_dic[job]['job_name'] = my_match.group(1)
if job_compile_dic['user_compile'].match(line):
my_match = job_compile_dic['user_compile'].match(line)
my_dic[job]['user'] = my_match.group(1)
if job_compile_dic['project_compile'].match(line):
my_match = job_compile_dic['project_compile'].match(line)
my_dic[job]['project'] = my_match.group(1)
if job_compile_dic['status_compile'].match(line):
my_match = job_compile_dic['status_compile'].match(line)
my_dic[job]['status'] = my_match.group(1)
if job_compile_dic['queue_compile'].match(line):
my_match = job_compile_dic['queue_compile'].match(line)
my_dic[job]['queue'] = my_match.group(1)
if job_compile_dic['command_compile'].match(line):
my_match = job_compile_dic['command_compile'].match(line)
my_dic[job]['command'] = my_match.group(1)
if job_compile_dic['submitted_from_compile'].match(line):
my_match = job_compile_dic['submitted_from_compile'].match(
line)
my_dic[job]['submitted_from'] = my_match.group(1)
if job_compile_dic['submitted_time_compile'].match(line):
my_match = job_compile_dic['submitted_time_compile'].match(
line)
my_dic[job]['submitted_time'] = my_match.group(1)
if job_compile_dic['cwd_compile'].match(line):
my_match = job_compile_dic['cwd_compile'].match(line)
my_dic[job]['cwd'] = my_match.group(1)
if job_compile_dic['processors_requested_compile'].match(line):
my_match = job_compile_dic[
'processors_requested_compile'].match(line)
my_dic[job]['processors_requested'] = my_match.group(1)
if job_compile_dic['requested_resources_compile'].match(line):
my_match = job_compile_dic[
'requested_resources_compile'].match(line)
my_dic[job]['requested_resources'] = my_match.group(1)
if job_compile_dic['span_hosts_compile'].match(line):
my_match = job_compile_dic['span_hosts_compile'].match(
line)
my_dic[job]['span_hosts'] = my_match.group(1)
if job_compile_dic['rusage_mem_compile'].match(line):
my_match = job_compile_dic['rusage_mem_compile'].match(
line)
my_dic[job]['rusage_mem'] = my_match.group(1)
if job_compile_dic['started_on_compile'].match(line):
my_match = job_compile_dic['started_on_compile'].match(
line)
started_host = my_match.group(2)
started_host = re.sub('<', '', started_host)
started_host = re.sub('>', '', started_host)
my_dic[job]['started_on'] = started_host
if job_compile_dic['started_time_compile'].match(line):
my_match = job_compile_dic['started_time_compile'].match(
line)
my_dic[job]['started_time'] = my_match.group(1)
if job_compile_dic['finished_time_compile'].match(line):
my_match = job_compile_dic['finished_time_compile'].match(
line)
my_dic[job]['finished_time'] = my_match.group(1)
if job_compile_dic['exit_code_compile'].match(line):
my_match = job_compile_dic['exit_code_compile'].match(line)
my_dic[job]['exit_code'] = my_match.group(1)
if job_compile_dic['lsf_signal_compile'].match(line):
my_match = job_compile_dic['lsf_signal_compile'].match(
line)
my_dic[job]['lsf_signal'] = my_match.group(1)
if job_compile_dic['term_owner_compile'].match(line):
my_match = job_compile_dic['term_owner_compile'].match(
line)
my_dic[job]['term_owner'] = my_match.group(1)
if job_compile_dic['cpu_time_compile'].match(line):
my_match = job_compile_dic['cpu_time_compile'].match(line)
my_dic[job]['cpu_time'] = my_match.group(1)
if job_compile_dic['mem_compile'].match(line):
my_match = job_compile_dic['mem_compile'].match(line)
my_dic[job]['mem'] = my_match.group(1)
return (my_dic)
def get_lsf_bjobs_uf_info(command):
"""
Parse job info which are from command 'bjobs -u all -UF'.
====
Job <101>, Job Name <Tesf for lsfMonitor>, User <liyanqing>, Project <lsf_test>, Status <RUN>, Queue <normal>, Command <sleep 12345>, Share group charged </liyanqing>
Mon Oct 26 17:43:07: Submitted from host <cmp01>, CWD <$HOME>, 2 Task(s), Requested Resources <span[hosts=1] rusage[mem=123]>;
Mon Oct 26 17:43:07: Started 2 Task(s) on Host(s) <2*cmp01>, Allocated 2 Slot(s) on Host(s) <2*cmp01>, Execution Home </home/liyanqing>, Execution CWD </home/liyanqing>;
Mon Oct 26 17:46:17: Resource usage collected. MEM: 2 Mbytes; SWAP: 238 Mbytes; NTHREAD: 4; PGID: 10643; PIDs: 10643 10644 10646;
MEMORY USAGE:
MAX MEM: 2 Mbytes; AVG MEM: 2 Mbytes
SCHEDULING PARAMETERS:
r15s r1m r15m ut pg io ls it tmp swp mem
load_sched - - - - - - - - - - -
load_stop - - - - - - - - - - -
RESOURCE REQUIREMENT DETAILS:
Combined: select[type == local] order[r15s:pg] rusage[mem=123.00] span[hosts=1]
Effective: select[type == local] order[r15s:pg] rusage[mem=123.00] span[hosts=1]
====
"""
job_compile_dic = {
'job_compile':
re.compile('.*Job <([0-9]+(\[[0-9]+\])?)>.*'),
'job_name_compile':
re.compile('.*Job Name <([^>]+)>.*'),
'user_compile':
re.compile('.*User <([^>]+)>.*'),
'project_compile':
re.compile('.*Project <([^>]+)>.*'),
'status_compile':
re.compile('.*Status <([A-Z]+)>*'),
'queue_compile':
re.compile('.*Queue <([^>]+)>.*'),
'command_compile':
re.compile('.*Command <(.+?\S)>.*$'),
'submitted_from_compile':
re.compile('.*Submitted from host <([^>]+)>.*'),
'submitted_time_compile':
re.compile('(.*): Submitted from host.*'),
'cwd_compile':
re.compile('.*CWD <([^>]+)>.*'),
'processors_requested_compile':
re.compile('.* (\d+) Task\(s\).*'),
'requested_resources_compile':
re.compile('.*Requested Resources <(.+)>;.*'),
'span_hosts_compile':
re.compile('.*Requested Resources <.*span\[hosts=([1-9][0-9]*).*>.*'),
'rusage_mem_compile':
re.compile('.*Requested Resources <.*rusage\[mem=([1-9][0-9]*).*>.*'),
'started_on_compile':
re.compile(
'(.*): (\[\d+\] )?[sS]tarted \d+ Task\(s\) on Host\(s\) (.+?), Allocated (\d+) Slot\(s\) on Host\(s\).*'
),
'finished_time_compile':
re.compile(
'(.*): (Done successfully|Exited with exit code|Exited by LSF signal|Completed <exit>).*'
),
'exit_code_compile':
re.compile('.*Exited with exit code (\d+)\..*'),
'lsf_signal_compile':
re.compile('.*Exited by LSF signal (\S+?)\..*'),
'term_owner_compile':
re.compile('.*TERM_OWNER: (.+?\.).*'),
'cpu_time_compile':
re.compile('.*The CPU time used is (\d+(\.\d+)?) seconds.*'),
'mem_compile':
re.compile('.*MEM:\s*(\d+(\.\d+)?)\s*([KMGT]bytes).*'),
'swap_compile':
re.compile('.*SWAP:\s*(\d+(\.\d+)?)\s*([KMGT]bytes).*'),
'run_limit_compile':
re.compile('\s*RUNLIMIT\s*'),
'pids_compile':
re.compile('PIDs:\s+(.+?);'),
'max_mem_compile':
re.compile(
'\s*MAX MEM: (\d+(\.\d+)?) ([KMGT]bytes);\s*AVG MEM: (\d+(\.\d+)?) ([KMGT]bytes)\s*'
),
'pending_reasons_compile':
re.compile('\s*PENDING REASONS:\s*'),
'empty_line_compile':
re.compile('^\s*$'),
}
my_dic = collections.OrderedDict()
job = ''
run_limit_mark = False
pending_mark = False
(return_code, stdout, stderr) = common.run_command(command)
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if re.match('Job <' + str(job) + '> is not found', line):
continue
else:
if job_compile_dic['job_compile'].match(line):
my_match = job_compile_dic['job_compile'].match(line)
job = my_match.group(1)
# Initialization for my_dic[job].
my_dic[job] = collections.OrderedDict()
my_dic[job]['job_info'] = ''
my_dic[job]['job_id'] = job
my_dic[job]['job_name'] = ''
my_dic[job]['user'] = ''
my_dic[job]['project'] = ''
my_dic[job]['status'] = ''
my_dic[job]['queue'] = ''
my_dic[job]['command'] = ''
my_dic[job]['submitted_from'] = ''
my_dic[job]['submitted_time'] = ''
my_dic[job]['cwd'] = ''
my_dic[job]['processors_requested'] = '1'
my_dic[job]['requested_resources'] = ''
my_dic[job]['span_hosts'] = ''
my_dic[job]['rusage_mem'] = ''
my_dic[job]['started_on'] = ''
my_dic[job]['started_time'] = ''
my_dic[job]['finished_time'] = ''
my_dic[job]['exit_code'] = ''
my_dic[job]['lsf_signal'] = ''
my_dic[job]['term_owner'] = ''
my_dic[job]['cpu_time'] = ''
my_dic[job]['mem'] = ''
my_dic[job]['swap'] = ''
my_dic[job]['run_limit'] = ''
my_dic[job]['pids'] = []
my_dic[job]['max_mem'] = ''
my_dic[job]['avg_mem'] = ''
my_dic[job]['pending_reasons'] = []
if job != '':
if my_dic[job]['job_info']:
my_dic[job]['job_info'] = str(
my_dic[job]['job_info']) + '\n' + str(line)
else:
my_dic[job]['job_info'] = line
if job_compile_dic['empty_line_compile'].match(line):
if run_limit_mark:
run_limit_mark = False
if pending_mark:
pending_mark = False
else:
if run_limit_mark:
my_dic[job]['run_limit'] = re.sub('min', '', line)
my_dic[job]['run_limit'] = re.sub(
'\s', '', my_dic[job]['run_limit'])
continue
if pending_mark:
my_dic[job]['pending_reasons'].append(line.strip())
continue
if job_compile_dic['job_name_compile'].match(line):
my_match = job_compile_dic['job_name_compile'].match(
line)
my_dic[job]['job_name'] = my_match.group(1)
if job_compile_dic['user_compile'].match(line):
my_match = job_compile_dic['user_compile'].match(line)
my_dic[job]['user'] = my_match.group(1)
if job_compile_dic['project_compile'].match(line):
my_match = job_compile_dic['project_compile'].match(
line)
my_dic[job]['project'] = my_match.group(1)
if job_compile_dic['status_compile'].match(line):
my_match = job_compile_dic['status_compile'].match(
line)
my_dic[job]['status'] = my_match.group(1)
if job_compile_dic['queue_compile'].match(line):
my_match = job_compile_dic['queue_compile'].match(line)
my_dic[job]['queue'] = my_match.group(1)
if job_compile_dic['command_compile'].match(line):
my_match = job_compile_dic['command_compile'].match(
line)
my_dic[job]['command'] = my_match.group(1)
continue
if job_compile_dic['submitted_time_compile'].match(line):
my_match = job_compile_dic[
'submitted_time_compile'].match(line)
my_dic[job]['submitted_time'] = my_match.group(1)
if job_compile_dic['cwd_compile'].match(line):
my_match = job_compile_dic['cwd_compile'].match(line)
my_dic[job]['cwd'] = my_match.group(1)
if job_compile_dic['processors_requested_compile'].match(
line):
my_match = job_compile_dic[
'processors_requested_compile'].match(line)
my_dic[job]['processors_requested'] = my_match.group(1)
if job_compile_dic['requested_resources_compile'].match(
line):
my_match = job_compile_dic[
'requested_resources_compile'].match(line)
my_dic[job]['requested_resources'] = my_match.group(1)
if job_compile_dic['span_hosts_compile'].match(line):
my_match = job_compile_dic['span_hosts_compile'].match(
line)
my_dic[job]['span_hosts'] = my_match.group(1)
if job_compile_dic['rusage_mem_compile'].match(line):
my_match = job_compile_dic['rusage_mem_compile'].match(
line)
my_dic[job]['rusage_mem'] = my_match.group(1)
if job_compile_dic['submitted_from_compile'].match(line):
my_match = job_compile_dic[
'submitted_from_compile'].match(line)
my_dic[job]['submitted_from'] = my_match.group(1)
continue
if job_compile_dic['started_on_compile'].match(line):
my_match = job_compile_dic['started_on_compile'].match(
line)
my_dic[job]['started_time'] = my_match.group(1)
started_host = my_match.group(3)
started_host = re.sub('<', '', started_host)
started_host = re.sub('>', '', started_host)
started_host = re.sub('\d+\*', '', started_host)
my_dic[job]['started_on'] = started_host
continue
if job_compile_dic['cpu_time_compile'].match(line):
my_match = job_compile_dic['cpu_time_compile'].match(
line)
my_dic[job]['cpu_time'] = my_match.group(1)
if job_compile_dic['mem_compile'].match(line):
my_match = job_compile_dic['mem_compile'].match(line)
my_dic[job]['mem'] = my_match.group(1)
unit = my_match.group(3)
if unit == 'Kbytes':
my_dic[job]['mem'] = float(
my_dic[job]['mem']) / 1024
elif unit == 'Gbytes':
my_dic[job]['mem'] = float(
my_dic[job]['mem']) * 1024
elif unit == 'Tbytes':
my_dic[job]['mem'] = float(
my_dic[job]['mem']) * 1024 * 1024
if job_compile_dic['swap_compile'].match(line):
my_match = job_compile_dic['swap_compile'].match(line)
my_dic[job]['swap'] = my_match.group(1)
unit = my_match.group(3)
if unit == 'Kbytes':
my_dic[job]['swap'] = float(
my_dic[job]['swap']) / 1024
elif unit == 'Gbytes':
my_dic[job]['swap'] = float(
my_dic[job]['swap']) * 1024
elif unit == 'Tbytes':
my_dic[job]['swap'] = float(
my_dic[job]['swap']) * 1024 * 1024
if job_compile_dic['finished_time_compile'].match(line):
my_match = job_compile_dic[
'finished_time_compile'].match(line)
my_dic[job]['finished_time'] = my_match.group(1)
if job_compile_dic['exit_code_compile'].match(line):
my_match = job_compile_dic['exit_code_compile'].match(
line)
my_dic[job]['exit_code'] = my_match.group(1)
if job_compile_dic['lsf_signal_compile'].match(line):
my_match = job_compile_dic['lsf_signal_compile'].match(
line)
my_dic[job]['lsf_signal'] = my_match.group(1)
if job_compile_dic['term_owner_compile'].match(line):
my_match = job_compile_dic['term_owner_compile'].match(
line)
my_dic[job]['term_owner'] = my_match.group(1)
if job_compile_dic['pids_compile'].findall(line):
my_match = job_compile_dic['pids_compile'].findall(
line)
my_string = ' '.join(my_match)
my_dic[job]['pids'] = my_string.split()
continue
if job_compile_dic['max_mem_compile'].match(line):
my_match = job_compile_dic['max_mem_compile'].match(
line)
my_dic[job]['max_mem'] = my_match.group(1)
unit = my_match.group(3)
if unit == 'Kbytes':
my_dic[job]['max_mem'] = float(
my_dic[job]['max_mem']) / 1024
elif unit == 'Gbytes':
my_dic[job]['max_mem'] = float(
my_dic[job]['max_mem']) * 1024
elif unit == 'Tbytes':
my_dic[job]['max_mem'] = float(
my_dic[job]['max_mem']) * 1024 * 1024
my_dic[job]['avg_mem'] = my_match.group(4)
unit = my_match.group(6)
if unit == 'Kbytes':
my_dic[job]['avg_mem'] = float(
my_dic[job]['avg_mem']) / 1024
elif unit == 'Gbytes':
my_dic[job]['avg_mem'] = float(
my_dic[job]['avg_mem']) * 1024
elif unit == 'Tbytes':
my_dic[job]['avg_mem'] = float(
my_dic[job]['avg_mem']) * 1024 * 1024
continue
if job_compile_dic['run_limit_compile'].match(line):
run_limit_mark = True
if job_compile_dic['pending_reasons_compile'].match(line):
pending_mark = True
return (my_dic)
def get_license_info(self):
"""
Get EDA liecnse feature usage and expires information on license_dic.
license_dic format is like below:
license_dic = {
'license_files': '',
'license_server_status': 'UNKNOWN',
'license_server_version': '',
'vendor_daemon': { vendor_daemon: {
'vendor_daemon_status': 'UP',
'vendor_daemon_version': '',
'feature': {feature: {
'issued': 0,
'in_use': 0,
'in_use_info_string': [],
'in_use_info': [],
},
},
'expires': {feature: {
'version': '',
'license': '',
'vendor': '',
'expires': '',
},
},
},
},
}
"""
# Get lmstat output message.
(return_code, stdout, stderr) = common.run_command(self.lmstat_command)
# Parse lmstat output message.
license_dic = self.init_license_dic()
license_server = ''
vendor_daemon = ''
feature = ''
expires_mark = False
vendor_daemon_status_mark = False
license_compile_dic = {
'empty_line':
re.compile('^\s*$'),
'license_server_status':
re.compile('^\s*License server status: (\S+)\s*$'),
'license_files':
re.compile('^\s*License file\(s\) on (\S+): (\S+):\s*$'),
'license_server':
re.compile('^\s*(\S+): license server (\S+?) .* (\S+?)\s*$'),
'vendor_daemon_status':
re.compile('^\s*Vendor daemon status \(on (.+)\):\s*$'),
'vendor_daemon_up':
re.compile('^\s*(\S+): UP (\S+)\s*$'),
'vendor_daemon_down':
re.compile('^\s*(\S+): The desired vendor daemon is down\..*$'),
'users_of_feature':
re.compile(
'^Users of (\S+): \(Total of ([0-9]+) license(s?) issued; Total of ([0-9]+) license(s?) in use\)\s*$'
),
'in_use_info':
re.compile(
'^\s*(\S+)\s+(\S+)\s+(\S+)?\s*(.+)?\s*\((\S+)\)\s+\((\S+)\s+(\d+)\), start (.+?)(,\s+(\d+)\s+licenses)?\s*$'
),
'reservation':
re.compile(
'^\s*(\d+)\s+RESERVATION for HOST\s+(\S+)\s+\((\S+)(\s+(\d+))?\)\s*$'
),
'feature_expires':
re.compile('^Feature .* Expires\s*$'),
'expire_info':
re.compile(
'^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(permanent\(no expiration date\)|[0-9]{1,2}-[a-zA-Z]{3}-[0-9]{4})\s*$'
),
}
for line in str(stdout, 'utf-8').split('\n'):
line = line.strip()
if license_compile_dic['empty_line'].match(line):
continue
elif expires_mark and license_compile_dic['expire_info'].match(
line):
my_match = license_compile_dic['expire_info'].match(line)
feature = my_match.group(1)
expire_dic = {
'version': my_match.group(2),
'license': my_match.group(3),
'vendor': my_match.group(4),
'expires': my_match.group(5),
}
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'expires'].setdefault(feature, [])
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'expires'][feature].append(expire_dic)
elif license_compile_dic['users_of_feature'].match(line):
my_match = license_compile_dic['users_of_feature'].match(line)
feature = my_match.group(1)
issued_num = my_match.group(2)
in_use_num = my_match.group(4)
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'feature'].setdefault(
feature, {
'issued': issued_num,
'in_use': in_use_num,
'in_use_info_string': [],
'in_use_info': [],
})
elif license_compile_dic['in_use_info'].match(line):
my_match = license_compile_dic['in_use_info'].match(line)
usage_dic = {
'user': my_match.group(1),
'execute_host': my_match.group(2),
'submit_host': 'N/A',
'version': my_match.group(5),
'license_server': my_match.group(6),
'start_time': my_match.group(8),
'license_num': '1',
}
display_setting = my_match.group(3)
if display_setting:
if re.match('^(.+):.+$', display_setting):
display_match = re.match('^(.+):.+$', display_setting)
usage_dic['submit_host'] = display_match.group(1)
license_num_setting = my_match.group(9)
if license_num_setting:
usage_dic['license_num'] = my_match.group(10)
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'feature'][feature]['in_use_info_string'].append(
line.strip())
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'feature'][feature]['in_use_info'].append(usage_dic)
elif license_compile_dic['reservation'].match(line):
my_match = license_compile_dic['reservation'].match(line)
usage_dic = {
'user': '******',
'execute_host': my_match.group(2),
'submit_host': 'N/A',
'version': 'N/A',
'license_server': my_match.group(3),
'start_time': 'N/A',
'license_num': my_match.group(1),
}
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'feature'][feature]['in_use_info_string'].append(
line.strip())
license_dic[license_server]['vendor_daemon'][vendor_daemon][
'feature'][feature]['in_use_info'].append(usage_dic)
elif license_compile_dic['license_server_status'].match(line):
my_match = license_compile_dic['license_server_status'].match(
line)
license_server = my_match.group(1)
license_dic.setdefault(
license_server, {
'license_files': '',
'license_server_status': 'UNKNOWN',
'license_server_version': '',
'vendor_daemon': {},
})
expires_mark = False
elif license_compile_dic['license_files'].match(line):
my_match = license_compile_dic['license_files'].match(line)
license_dic[license_server]['license_files'] = my_match.group(
2)
elif license_compile_dic['license_server'].match(line):
# License Manager Status—Status of each license server manager. Click the Update Status button to immediately refresh the status display for the license server.
# • Up—License server is currently running.
# • Down—License server is currently not running.
# • Unknown—License server status is not known.
# • Lost quorum—For license-server triads, this means that the quorum has been lost. A quorum requires that at least two of the three license servers are running and communicating with each other.
my_match = license_compile_dic['license_server'].match(line)
license_dic[license_server][
'license_server_status'] = my_match.group(2)
license_dic[license_server][
'license_server_version'] = my_match.group(3)
elif license_compile_dic['vendor_daemon_status'].match(line):
vendor_daemon_status_mark = True
elif vendor_daemon_status_mark and license_compile_dic[
'vendor_daemon_up'].match(line):
my_match = license_compile_dic['vendor_daemon_up'].match(line)
vendor_daemon = my_match.group(1)
license_dic[license_server]['vendor_daemon'].setdefault(
vendor_daemon, {
'vendor_daemon_status': 'UP',
'vendor_daemon_version': my_match.group(2),
'feature': {},
'expires': {},
})
vendor_daemon_status_mark = False
elif vendor_daemon_status_mark and license_compile_dic[
'vendor_daemon_down'].match(line):
my_match = license_compile_dic['vendor_daemon_down'].match(
line)
vendor_daemon = my_match.group(1)
license_dic[license_server]['vendor_daemon'].setdefault(
vendor_daemon, {
'vendor_daemon_status': 'DOWN',
'vendor_daemon_version': '',
'feature': {},
'expires': {},
})
vendor_daemon_status_mark = False
elif license_compile_dic['feature_expires'].match(line):
expires_mark = True
license_dic[license_server]['vendor_daemon'][
vendor_daemon].setdefault('expires', {})
return (license_dic)