def get_qinfo(self, host):
HEADING()
self.pbs = PBS(self.user, host)
results = self.pbs.qinfo()
for name in results:
element = results[name]
pprint(element)
def pbs_service(self, rack_name=None):
config = cm_config()
username = config.get("cloudmesh.hpc.username")
pbs = PBS(username, "india.futuregrid.org")
dict_pbs_info = pbs.pbsnodes()
dict_data = {}
inventory = Inventory()
racks = inventory.get_clusters()
for rack in racks:
rack_name = rack["cm_cluster"]
dict_data[rack_name] = {}
hosts = rack["cm_value"]
for host in hosts:
(hid, hlabel) = inventory.get_host_id_label(host, "public")
utype = "unknown"
if hlabel in dict_pbs_info:
server = dict_pbs_info[hlabel]
if "note" in server.keys():
note_value = server["note"]
# to compatible with the future change
if type(note_value) is dict:
utype = note_value["service"]
else: # currently is a literal string for note
utype = note_value
dict_data[rack_name][hid] = utype
return dict_data
def test_info(self):
HEADING()
for host in self.hosts:
self.pbs = PBS(self.user, host)
results = self.pbs.qstat()
print host, " =", len(results), "jobs"
class Test:
hosts = ["india.futuregrid.org",
"sierra.futuregrid.org",
"hotel.futuregrid.org",
"alamo.futuregrid.org"]
def setup(self):
# self.configuration = cm_config()
# pprint (self.configuration.__dict__)
self.user = "******"
self.host = "india.futuregrid.org"
# pprint (_create_pbsinfo_dict(data))
def tearDown(self):
pass
def get_qstat(self, host):
HEADING()
self.pbs = PBS(self.user, host)
results = self.pbs.qstat()
for name in results:
element = results[name]
pprint (element)
def get_qinfo(self, host):
HEADING()
self.pbs = PBS(self.user, host)
results = self.pbs.qinfo()
for name in results:
element = results[name]
pprint (element)
def test_all(self):
HEADING()
for host in self.hosts:
banner(host)
self.get_qstat(host)
def test_info(self):
HEADING()
for host in self.hosts:
self.pbs = PBS(self.user, host)
results = self.pbs.qstat()
print host, " =", len(results), "jobs"
def test_qinfo(self):
for host in self.hosts:
banner(host)
self.get_qinfo(host)
class Test:
hosts = [
"india.futuregrid.org", "sierra.futuregrid.org",
"hotel.futuregrid.org", "alamo.futuregrid.org"
]
def setup(self):
# self.configuration = cm_config()
# pprint (self.configuration.__dict__)
self.user = "******"
self.host = "india.futuregrid.org"
# pprint (_create_pbsinfo_dict(data))
def tearDown(self):
pass
def get_qstat(self, host):
HEADING()
self.pbs = PBS(self.user, host)
results = self.pbs.qstat()
for name in results:
element = results[name]
pprint(element)
def get_qinfo(self, host):
HEADING()
self.pbs = PBS(self.user, host)
results = self.pbs.qinfo()
for name in results:
element = results[name]
pprint(element)
def test_all(self):
HEADING()
for host in self.hosts:
banner(host)
self.get_qstat(host)
def test_info(self):
HEADING()
for host in self.hosts:
self.pbs = PBS(self.user, host)
results = self.pbs.qstat()
print host, " =", len(results), "jobs"
def test_qinfo(self):
for host in self.hosts:
banner(host)
self.get_qinfo(host)
def get_qinfo(self, host):
HEADING()
self.pbs = PBS(self.user, host)
results = self.pbs.qinfo()
for name in results:
element = results[name]
pprint(element)
def test_info(self):
HEADING()
for host in self.hosts:
self.pbs = PBS(self.user, host)
results = self.pbs.qstat()
print(host, " =", len(results), "jobs")
def display_status():
msg = ""
status = ""
values = {
'india' : { 'jobs' : 0, 'users' : 0},
'bravo' : { 'jobs' : 0, 'users' : 0},
'echo' : { 'jobs' : 0, 'users' : 0},
'hotel' : { 'jobs' : 0, 'users' : 0},
'sierra' : { 'jobs' : 0, 'users' : 0},
'alamo' : { 'jobs' : 0, 'users' : 0},
'delta' : { 'jobs' : 0, 'users': 0},
'lima' : { 'jobs' : 0, 'users': 0}
}
config = cm_config()
time_now = datetime.now().strftime("%Y-%m-%d %H:%M")
user = config.get("cloudmesh.hpc.username")
services = {}
qinfo = {}
qstat = {}
qstat_uniq_users = {}
for host in ['sierra.futuregrid.org',
'india.futuregrid.org',
]:
pbs = PBS(user, host)
services[host] = pbs.service_distribution()
qinfo[host] = pbs.qinfo()
qstat[host] = pbs.qstat()
qstat_uniq_users[host] = pbs.get_uniq_users()
machines = services.keys()
# print "FFF", machines
#
# collecting all atttributes
#
all_attributes = set()
for machine in machines:
attributes = set(list(services[machine].keys()))
print "P", attributes
all_attributes.update(attributes)
# print "XXX", all_attributes
spider_services = {'machines' : machines,
'categories' : list(all_attributes),
'data' : {}}
#
# seeting all attributes to 0
#
for machine in machines:
ser = []
i = 0
for attribute in all_attributes:
try:
ser.append(services[machine][attribute])
except:
ser.append(0)
i = i + 1
spider_services['data'][machine] = ser
# print "SSS", spider_services
# Users and Jobs
total_jobs = {}
unique_users = {}
for machine in machines:
for qserver in qinfo[machine]:
total_jobs[qserver] = 0
unique_users[qserver] = 0
try:
hostname = qserver.split('.')[0]
except:
hostname = ""
for qname in qinfo[machine][qserver]:
total_jobs[qserver] += qinfo[machine][qserver][qname]['total_jobs']
try:
unique_users[qserver] += len(qstat_uniq_users[machine][qserver])
except KeyError:
pass
values[hostname]['jobs'] = total_jobs[qserver]
values[hostname]['users'] = unique_users[qserver]
return render_template('status/status.html',
services=spider_services,
values=values,
status=status,
show=msg)
Job.list()
name = "job1"
job = Job(name=name, jobid=None, group="group1", user="******", state="defined", ssh_config="india", queue=queue)
job.save()
# print (Job.status(name))
# jobs = Job.objects()
Job.list()
Job.info(name)
j = Job.find(name)
j.name = "a"
j.save()
Job.list()
# # BUG
# Job.rename(name, 'job2')
# Job.info('job2')
banner("PBS")
pbs = PBS("gvonlasz", "india.futuregrid.org")
# pprint (pbs.qinfo())
pprint(pbs.qstat())
def display_status():
msg = ""
status = ""
values = {
'india': {
'jobs': 0,
'users': 0
},
'bravo': {
'jobs': 0,
'users': 0
},
'echo': {
'jobs': 0,
'users': 0
},
'hotel': {
'jobs': 0,
'users': 0
},
'sierra': {
'jobs': 0,
'users': 0
},
'alamo': {
'jobs': 0,
'users': 0
},
'delta': {
'jobs': 0,
'users': 0
},
'lima': {
'jobs': 0,
'users': 0
}
}
config = cm_config()
time_now = datetime.now().strftime("%Y-%m-%d %H:%M")
user = config.get("cloudmesh.hpc.username")
services = {}
qinfo = {}
qstat = {}
qstat_uniq_users = {}
for host in [
'sierra.futuregrid.org',
'india.futuregrid.org',
]:
pbs = PBS(user, host)
services[host] = pbs.service_distribution()
qinfo[host] = pbs.qinfo()
qstat[host] = pbs.qstat()
qstat_uniq_users[host] = pbs.get_uniq_users()
machines = services.keys()
# print "FFF", machines
#
# collecting all atttributes
#
all_attributes = set()
for machine in machines:
attributes = set(list(services[machine].keys()))
print "P", attributes
all_attributes.update(attributes)
# print "XXX", all_attributes
spider_services = {
'machines': machines,
'categories': list(all_attributes),
'data': {}
}
#
# seeting all attributes to 0
#
for machine in machines:
ser = []
i = 0
for attribute in all_attributes:
try:
ser.append(services[machine][attribute])
except:
ser.append(0)
i = i + 1
spider_services['data'][machine] = ser
# print "SSS", spider_services
# Users and Jobs
total_jobs = {}
unique_users = {}
for machine in machines:
for qserver in qinfo[machine]:
total_jobs[qserver] = 0
unique_users[qserver] = 0
try:
hostname = qserver.split('.')[0]
except:
hostname = ""
for qname in qinfo[machine][qserver]:
total_jobs[qserver] += qinfo[machine][qserver][qname][
'total_jobs']
try:
unique_users[qserver] += len(
qstat_uniq_users[machine][qserver])
except KeyError:
pass
values[hostname]['jobs'] = total_jobs[qserver]
values[hostname]['users'] = unique_users[qserver]
return render_template('status/status.html',
services=spider_services,
values=values,
status=status,
show=msg)
def fetch_pbs_nodes_info(self):
pbs = PBS(self.username, self.hostname)
self.pbs_nodes_info = pbs.pbsnodes()
def fetch_pbs_nodes_info(self):
pbs = PBS(self.username, self.hostname)
self.pbs_nodes_info = pbs.pbsnodes()
def activate(self, host, user):
"""activates a specific host to be queried"""
self.pbs_client = PBS(user, host)
self.hosts[host] = self.pbs_client
group='group1',
user='******',
state='defined',
ssh_config='india',
queue=queue)
job.save()
# print (Job.status(name))
# jobs = Job.objects()
Job.list()
Job.info(name)
j = Job.find(name)
j.name = 'a'
j.save()
Job.list()
# # BUG
# Job.rename(name, 'job2')
# Job.info('job2')
banner("PBS")
pbs = PBS("gvonlasz", "india.futuregrid.org")
# pprint (pbs.qinfo())
pprint(pbs.qstat())
def fetch_pbs_nodes_info(self):
'''
fetches the pbs_nodes information from the remote computer
'''
pbs = PBS(self.username, self.hostname)
self.pbs_nodes_info = pbs.pbsnodes()