def update_all_queues(batchserver_name):
""" Update info about all queues for give batchserver.
"""
server,created = getBatchServer(batchserver_name)
if server.queues_lastupdate and (datetime.datetime.now()-server.queues_lastupdate).total_seconds()<GlobalConfiguration.objects.get(pk=1).max_lastupdate:
logging.debug("Queue info is new enough for server: %s" % batchserver_name)
return
conn = pbs.pbs_connect(batchserver_name.encode('iso-8859-1', 'replace'))
if conn==-1:
logging.error("Cannot connect to %s - live data will be missing" % server.name)
return
statqueues = pbs.pbs_statque(conn, "" , [], "")
pbs.pbs_disconnect(conn)
if conn==-1:
logging.error("Cannot connect to %s - live data will be missing" % server.name)
return
for sq in statqueues:
queue,created = getQueue(sq.name, server)
attr_dict = dict([ (x.name,x.value) for x in sq.attribs])
update_one_queue_from_pbs_data(queue, attr_dict)
queue.save()
server.queues_lastupdate = datetime.datetime.now()
server.save()
def _statqueue(self, queue_name='', attrib_list=None):
"""Get the queue config from the pbs server"""
if attrib_list:
self._list_2_attrib(attrib_list)
else:
self.attribs = 'NULL'
self._connect()
queues = pbs.pbs_statque(self.con, queue_name, self.attribs, 'NULL')
self._disconnect()
self._list_2_dict(queues, queue)
def _statqueue(self, queue_name='', attrib_list=None): """Get the queue config from the pbs server""" if attrib_list: self._list_2_attrib(attrib_list) else: self.attribs = 'NULL' self._connect() queues = pbs.pbs_statque(self.con, queue_name, self.attribs, 'NULL') self._disconnect() self._list_2_dict(queues, queue)
def update_all_queues(conn,bs):
"""
Update all queue information in the database using the given pbs connection conn
and batch server bs.
"""
statqueues = pbs.pbs_statque(conn,"",[],"")
for sq in statqueues:
name = sq.name
q,created = Queue.objects.get_or_create(name=name, server=bs)
if created:
log(LOG_INFO, "new queue will be created: %s @ %s" % (name,bs.name))
q.server = bs
dattrs = attribs_to_dict(sq.attribs)
for key,val in dattrs.items():
setattr(q,key,val)
q.save()
def update_one_queue(queue):
""" Update live info about the given queue
"""
conn = pbs.pbs_connect(queue.server.name.encode('iso-8859-1', 'replace'))
if conn==-1:
logging.error("Cannot connect to %s - live data will be missing" % server.name)
return
statqueues = pbs.pbs_statque(conn, queue.name.encode('iso-8859-1', 'replace') , [], "")
pbs.pbs_disconnect(conn)
if len(statqueues)==0:
logging.error("pbs_statque failed for queue: %s" % queue.name)
return
if len(statqueues)>1:
logging.warning("pbs_statque returned more than one records for queue: %s" % queue.name)
attr_dict = dict([ (x.name,x.value) for x in statqueues[0].attribs])
update_one_queue_from_pbs_data(queue, attr_dict)
queue.save()
import pbs
except:
# Running from within the tests directory.
sys.path.append(os.path.abspath("../"))
import pbs
# You need to set the hostname of the PBS Server.
pbsserver = 'hpcnode0'
conn = pbs.pbs_connect(pbsserver)
if conn < 0:
print('Error connecting to PBS server.')
sys.exit(1)
# Returns a batch_status structure.
b = pbs.pbs_statque(conn, '', None, None)
while b != None:
print("\n------ Queue: %s ------" % b.name)
attribs = b.attribs
# print(dir(attribs)) # Uncomment to see what methods are available.
while attribs != None:
if attribs.resource != None:
print(" %s.%s = %s" %
(attribs.name, attribs.resource, attribs.value))
else:
print(" %s = %s" % (attribs.name, attribs.value))
attribs = attribs.next
b = b.next
x[0].name = 'np'
combine = z + x
print combine, len(combine)
#print combine[0].name
#print combine[1].name
#print combine[2].name
nodes = pbs.pbs_statnode(con, "", 'NULL', "NULL")
for node in nodes:
print node.name, ':'
for prop in node.attribs:
print '\t', prop.name, ' = ', prop.value
queues = pbs.pbs_statque(con, "", 'NULL', "")
for queue in queues:
print queue.name
for attrib in queue.attribs:
print '\t', attrib.name, ' = ', attrib.value
jobs = pbs.pbs_statjob(con, "", 'NULL', "")
for job in jobs:
print job.name
for attrib in job.attribs:
print '\t', attrib.name, ' = ', attrib.value
sys.exit(0)
## OLD stuff obselete
##
def get_queues(conn):
'''
Get information on the PBS queues.
This function returns a list of queues, where each queue is a dictionary.
Example: Queue Name = smallq
if attribs.resource == None <== we get the attribs:
name : value
---- -----
queue_type : Execution
total_jobs : 49
state_count : Transit:0 Queued:18 Held:0 Waiting:0 Running:30 Exiting:0 Begun:1
max_run : [u:PBS_GENERIC=12]
enabled : True
started : True
if attribs.resource != None <== we get the attribs:
name : resource = value
---- -------- -----
resources_max : mem = 32gb
resources_max : ncpus = 2
resources_max : walltime = 200:00:00
resources_default : walltime = 24:00:00
resources_assigned : mem = 598gb
resources_assigned : ncpus = 57
resources_assigned : nodect = 29
To make the returned dictionary simpler we rename the name:resource above
to be a key like this:
resources_max : mem => resources_max_mem
resources_max : ncpus => resources_max_ncpus
resources_max : walltime => resources_max_walltime
resources_default : walltime => resources_default_walltime
resources_assigned : mem => resources_assigned_mem
resources_assigned : ncpus => resources_assigned_ncpus
resources_assigned : nodect => resources_assigned_nodect
'''
queues = [] # This will contain a list of dictionaries.
# Some of the attributes are not present for all queues so we list them all
# here and in the loop below set them to None. For instance, a routing queue
# does not have some of these attributes.
attribute_names = ['resources_max_mem','resources_max_ncpus','resources_max_walltime', \
'resources_assigned_mem','resources_assigned_ncpus', \
'resources_default_walltime', 'max_run', 'state_count', 'acl_user_enable']
b = pbs.pbs_statque(conn, '', None, None)
while b != None:
attributes = {} # Init the dictionary to empty.
for name in attribute_names:
attributes[name] = None
attribs = b.attribs
#print('METHODS: ', dir(attribs)) # Uncomment to see what methods are available.
#print('------------ Queue %s ------------' % b.name)
attributes['queue_name'] = b.name
while attribs != None:
if attribs.resource != None:
# The print below here is indented a bit more to distinguish
# resource attributes from non-resource attributes.
#print(' ', attribs.name, ':', attribs.resource, '=', attribs.value)
keyname = '%s_%s' % (attribs.name, attribs.resource)
attributes[keyname] = attribs.value
else:
#print(' ', attribs.name, ':', attribs.value)
# e.g. acl_user_enable : True
attributes[attribs.name] = attribs.value
attribs = attribs.next
# Don't save the defaultq as this is a routing queue.
# TODO move this to reformat?
if attributes['queue_name'] != 'defaultq':
queues.append(attributes)
b = b.next
return queues
