def sm_sync(self):
'''Resynchronize with the script manager'''
# get this cache first -- it's no problem if this data is old, but bad things
# happen when this data is newer than the list of running processes in scriptm
self.lock.acquire()
try:
process_groups_cache = self.process_groups.values()
except:
self.logger.error("error copying process_groups.values()", exc_info=True)
self.lock.release()
try:
pgroups = ComponentProxy("script-manager").get_jobs([{'id':'*', 'state':'running'}])
except (ComponentLookupError, xmlrpclib.Fault):
self.logger.error("Failed to communicate with script manager")
return
live = [item['id'] for item in pgroups]
for each in process_groups_cache:
if each.mode == 'script' and each.script_id not in live:
self.logger.info("Found dead pg for script job %s" % (each.script_id))
result = ComponentProxy("script-manager").wait_jobs([{'id':each.script_id, 'exit_status':'*'}])
self.logger.info("wait returned %r" % result)
for r in result:
which_one = None
if r['id'] == each.script_id:
each.exit_status = r['exit_status']
self.reserve_resources_until(each.location, None, each.jobid)
def q_add(self, *args, **kwargs):
'''Add a reservation to tracking.
Side Efffects:
-Add a queue to be tracked
-If no cqm associated queue, create a reservation queue
-set policies for new queue
-emit numerous creation messages
'''
qm = ComponentProxy("queue-manager")
try:
queues = [spec['name'] for spec in qm.get_queues([{'name': "*"}])]
except ComponentLookupError:
logger.error(
"unable to contact queue manager when adding reservation")
raise
try:
specs = args[0]
for spec in specs:
if "res_id" not in spec or spec['res_id'] == '*':
spec['res_id'] = bgsched_id_gen.get()
reservations = Cobalt.Data.DataDict.q_add(self, *args, **kwargs)
except KeyError, err:
raise ReservationError(
"Error: a reservation named %s already exists" % err)
def event_driver(self):
"""core part that drives the clock"""
if self.go_next:
#only if the go_next tag is true will the clock be incremented. enable scheduler schedule multiple job at the same time stamp
self.clock_increment()
machine = self.get_current_event_machine()
# print "[%s]: %s, machine=%s, event=%s, job=%s" % (
# self.implementation,
# self.get_current_date_time(),
# self.get_current_event_machine(),
# self.get_current_event_type(),
# self.get_current_event_job(),
# )
if machine == INTREPID:
self.bgsched.schedule_jobs()
util = ComponentProxy("queue-manager").get_util()
self.log_info(util, "mira_util_mesh")
if machine == EUREKA:
self.csched.schedule_jobs()
if self.go_next:
ComponentProxy("queue-manager").calc_loss_of_capacity()
def update (self, spec):
if spec.has_key("users"):
qm = ComponentProxy("queue-manager")
try:
qm.set_queues([{'name':self.queue,}], {'users':spec['users']}, "bgsched")
except ComponentLookupError:
logger.error("unable to contact queue manager when updating reservation users")
raise
# try the above first -- if we can't contact the queue-manager, don't update the users
if spec.has_key('cycle') and not self.cycle:
#we have just turned this into a cyclic reservation and need a cycle_id.
spec['cycle_id'] = self.cycle_id_gen.get()
#get the user name of whoever issued the command
user_name = None
if spec.has_key('__cmd_user'):
user_name = spec['__cmd_user']
del spec['__cmd_user']
#if we're defering, pull out the 'defer' entry and send a cobalt db message.
#there really isn't a corresponding field to update
deferred = False
if spec.has_key('defer'):
logger.info("Res %s/%s: Deferring cyclic reservation: %s",
self.res_id, self.cycle_id, self.name)
dbwriter.log_to_db(user_name, "deferred", "reservation", self)
del spec['defer']
deferred = True
Data.update(self, spec)
if not deferred or not self.running:
#we only want this if we aren't defering. If we are, the cycle will
#take care of the new data object creation.
dbwriter.log_to_db(user_name, "modifying", "reservation", self)
def _get_exit_status (self):
try:
running = ComponentProxy("forker").active_list()
except:
self.logger.error("failed to contact forker component for list of running jobs")
return
for each in self.process_groups.itervalues():
if each.head_pid not in running and each.exit_status is None:
# FIXME: i bet we should consider a retry thing here -- if we fail enough times, just
# assume the process is dead? or maybe just say there's no exit code the first time it happens?
# maybe the second choice is better
try:
dead_dict = ComponentProxy("forker").get_status(each.head_pid)
except Queue.Empty:
self.logger.error("failed call for get_status from forker component for pg %s", each.head_pid)
return
if dead_dict is None:
self.logger.info("process group %i: job %s/%s exited with unknown status", each.id, each.jobid, each.user)
each.exit_status = 1234567
else:
each.exit_status = dead_dict["exit_status"]
if dead_dict["signum"] == 0:
self.logger.info("process group %i: job %s/%s exited with status %i",
each.id, each.jobid, each.user, each.exit_status)
else:
if dead_dict["core_dump"]:
core_dump_str = ", core dumped"
else:
core_dump_str = ""
self.logger.info("process group %i: job %s/%s terminated with signal %s%s",
each.id, each.jobid, each.user, dead_dict["signum"], core_dump_str)
class metricmon(Component):
"""metrics mointor: monitors the real time statistics of interested metrics such as average waiting time and system utilization rate"""
implementation = "imon"
name = "imon"
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.mmon_logger = None
def get_current_time_sec(self):
return self.event_manager.get_current_time()
def get_current_time_date(self):
return self.event_manager.get_current_date_time()
def init_mmon_logger(self):
if self.mon_logger == None:
self.mmon_logger = PBSlogger(self.bqsim.get_outputlog_string() + "-mmon")
def metric_monitor(self):
self.bqsim.monitor_metrics()
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
self.total_cost = 0.0
self.time_power_list =[{"unixtime":0, "power":0, "count":0, "utilization":0}]
def fetch_hardware():
'''Autodetect which system we are trying to contact and perform hardware
information fetch'''
global partition_table
global node_state
global indexes
global system_type
# Generate a list of all possible nodecards in all possible partitions
system = ComponentProxy('system', defer=True)
if system_type is None:
# We are self-discovering. Can save this step if we
# already know the system type.
system_type = system.get_implementation()
if system_type in bg_types:
partition_table = dict((part['name'], part['node_card_names'])
for part in system.get_partitions([{'name': '*',
'node_card_names': '*'}]))
node_state = {}
elif system_type in cluster_types:
partition_table = {}
node_state = dict((node[0], node[1]) for node in system.get_node_status())
elif system_type in cray_types:
partition_table = {}
# Using JSON for speed and avoinding the XML-RPC marshaller.
stst = json.loads(system.get_nodes(True, None, alps_system_query_fields, True))
indexes = {}
for idx in stst:
r = stst[idx]
if r['status'] == 'busy':
indexes[idx] = r['name']
node_state = dict((k, v['status']) for k, v in stst.iteritems())
else:
raise RuntimeError('The %s system implementation is not supported by cweb')
return system_type
class metricmon(Component):
"""metrics mointor: monitors the real time statistics of interested metrics such as average waiting time and system utilization rate"""
implementation = "imon"
name = "imon"
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.mmon_logger = None
def get_current_time_sec(self):
return self.event_manager.get_current_time()
def get_current_time_date(self):
return self.event_manager.get_current_date_time()
def init_mmon_logger(self):
if self.mon_logger == None:
self.mmon_logger = PBSlogger(self.bqsim.get_outputlog_string() +
"-mmon")
def metric_monitor(self):
self.bqsim.monitor_metrics()
def q_add (self, *args, **kwargs):
'''Add a reservation to tracking.
Side Efffects:
-Add a queue to be tracked
-If no cqm associated queue, create a reservation queue
-set policies for new queue
-emit numerous creation messages
'''
qm = ComponentProxy("queue-manager")
try:
queues = [spec['name'] for spec in qm.get_queues([{'name':"*"}])]
except ComponentLookupError:
logger.error("unable to contact queue manager when adding reservation")
raise
try:
specs = args[0]
for spec in specs:
if "res_id" not in spec or spec['res_id'] == '*':
spec['res_id'] = bgsched_id_gen.get()
reservations = Cobalt.Data.DataDict.q_add(self, *args, **kwargs)
except KeyError, err:
raise ReservationError("Error: a reservation named %s already exists" % err)
def signal_process_groups(self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
if pg.exit_status is None:
if pg.mode == "script":
try:
ComponentProxy("script-manager").signal_jobs(
[{
'id': pg.script_id
}], signame)
except (ComponentLookupError, xmlrpclib.Fault):
self.logger.error(
"Failed to communicate with script manager when killing job"
)
else:
try:
ComponentProxy("forker").signal(pg.head_pid, signame)
except:
self.logger.error(
"Failed to communicate with forker when signalling job"
)
if signame == "SIGKILL" and not pg.true_mpi_args:
self._mark_partition_for_cleaning(pg.location[0], pg.jobid)
return my_process_groups
def update (self, spec):
if spec.has_key("users"):
qm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
qm.set_queues([{'name':self.queue,}], {'users':spec['users']}, "bgsched")
except ComponentLookupError:
logger.error("unable to contact queue manager when updating reservation users")
raise
# try the above first -- if we can't contact the queue-manager, don't update the users
Data.update(self, spec)
def _get_exit_status(self):
#common to bgsystem
running = []
active_forker_components = []
for forker_component in ['bg_mpirun_forker', 'user_script_forker']:
try:
running.extend(
ComponentProxy(forker_component).active_list(
"process group"))
active_forker_components.append(forker_component)
except:
self.logger.error(
"failed to contact %s component for list of running jobs",
forker_component)
for each in self.process_groups.itervalues():
if each.head_pid not in running and each.exit_status is None and each.forker in active_forker_components:
# FIXME: i bet we should consider a retry thing here -- if we fail enough times, just
# assume the process is dead? or maybe just say there's no exit code the first time it happens?
# maybe the second choice is better
try:
if each.head_pid != None:
dead_dict = ComponentProxy(each.forker).get_status(
each.head_pid)
else:
dead_dict = None
except:
self.logger.error(
"%s: RPC to get_status method in %s component failed",
each.label, each.forker)
return
if dead_dict is None:
self.logger.info("%s: job exited with unknown status",
each.label)
# FIXME: should we use a negative number instead to indicate internal errors? --brt
each.exit_status = 1234567
else:
each.exit_status = dead_dict["exit_status"]
if dead_dict["signum"] == 0:
self.logger.info("%s: job exited with status %i",
each.label, each.exit_status)
else:
if dead_dict["core_dump"]:
core_dump_str = ", core dumped"
else:
core_dump_str = ""
self.logger.info("%s: terminated with signal %s%s",
each.label, dead_dict["signum"],
core_dump_str)
self.reserve_resources_until(each.location, None,
each.jobid)
def q_del (self, *args, **kwargs):
reservations = Cobalt.Data.DataDict.q_del(self, *args, **kwargs)
qm = ComponentProxy('queue-manager')
queues = [spec['name'] for spec in qm.get_queues([{'name':"*"}])]
spec = [{'name': reservation.queue} for reservation in reservations \
if reservation.createdQueue and reservation.queue in queues and \
not self.q_get([{'queue':reservation.queue}])]
try:
qm.set_queues(spec, {'state':"dead"}, "bgsched")
except Exception, e:
logger.error("problem disabling reservation queue (%s)" % e)
def q_del(self, *args, **kwargs):
reservations = Cobalt.Data.DataDict.q_del(self, *args, **kwargs)
qm = ComponentProxy('queue-manager')
queues = [spec['name'] for spec in qm.get_queues([{'name': "*"}])]
spec = [{'name': reservation.queue} for reservation in reservations \
if reservation.createdQueue and reservation.queue in queues and \
not self.q_get([{'queue':reservation.queue}])]
try:
qm.set_queues(spec, {'state': "dead"}, "bgsched")
except Exception, e:
logger.error("problem disabling reservation queue (%s)" % e)
def _start_job(self, job, partition_list):
cqm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
self.logger.info("trying to start job %d on partition %r" % (job.jobid, partition_list))
cqm.run_jobs([{'tag':"job", 'jobid':job.jobid}], partition_list)
except ComponentLookupError:
self.logger.error("failed to connect to queue manager")
return
self.started_jobs[job.jobid] = self.get_current_time()
def _start_job(self, job, partition_list):
cqm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
self.logger.info("trying to start job %d on partition %r" %
(job.jobid, partition_list))
cqm.run_jobs([{'tag': "job", 'jobid': job.jobid}], partition_list)
except ComponentLookupError:
self.logger.error("failed to connect to queue manager")
return
self.started_jobs[job.jobid] = self.get_current_time()
def q_add (self, *args, **kwargs):
qm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
queues = [spec['name'] for spec in qm.get_queues([{'name':"*"}])]
except ComponentLookupError:
logger.error("unable to contact queue manager when adding reservation")
raise
try:
reservations = Cobalt.Data.DataDict.q_add(self, *args, **kwargs)
except KeyError, e:
raise ReservationError("Error: a reservation named %s already exists" % e)
def _start_job(self, job, partition_list, resid=None):
"""Get the queue manager to start a job."""
cqm = ComponentProxy("queue-manager")
try:
self.logger.info("trying to start job %d on partition %r" % (job.jobid, partition_list))
cqm.run_jobs([{'tag':"job", 'jobid':job.jobid}], partition_list, None, resid)
except ComponentLookupError:
self.logger.error("failed to connect to queue manager")
return
self.started_jobs[job.jobid] = self.get_current_time()
def update(self, spec):
if spec.has_key("users"):
qm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
qm.set_queues([{
'name': self.queue,
}], {'users': spec['users']}, "bgsched")
except ComponentLookupError:
logger.error(
"unable to contact queue manager when updating reservation users"
)
raise
# try the above first -- if we can't contact the queue-manager, don't update the users
Data.update(self, spec)
def q_add(self, *args, **kwargs):
qm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
queues = [spec['name'] for spec in qm.get_queues([{'name': "*"}])]
except ComponentLookupError:
logger.error(
"unable to contact queue manager when adding reservation")
raise
try:
reservations = Cobalt.Data.DataDict.q_add(self, *args, **kwargs)
except KeyError, e:
raise ReservationError(
"Error: a reservation named %s already exists" % e)
def check_reservations(self):
ret = ""
reservations = self.reservations.values()
for i in range(len(reservations)):
for j in range(i + 1, len(reservations)):
# if at least one reservation is cyclic, we want *that* reservation to be the one getting its overlaps method
# called
if reservations[i].cycle is not None:
res1 = reservations[i]
res2 = reservations[j]
else:
res1 = reservations[j]
res2 = reservations[i]
# we subtract a little bit because the overlaps method isn't really meant to do this
# it will report warnings when one reservation starts at the same time another ends
if res1.overlaps(res2.start, res2.duration - 0.00001):
# now we need to check for overlap in space
results = ComponentProxy(self.COMP_SYSTEM).get_partitions(
[{
'name': p,
'children': '*',
'parents': '*'
} for p in res2.partitions.split(":")])
for p in res1.partitions.split(":"):
for r in results:
if p == r['name'] or p in r['children'] or p in r[
'parents']:
ret += "Warning: reservation '%s' overlaps reservation '%s'\n" % (
res1.name, res2.name)
return ret
def check_dependencies(dependency_string):
if dependency_string.lower() == 'none':
#we are removing all job dependencies.
print "Removing job dependencies"
return
deps = set(dependency_string.split(":"))
query = []
for dep in deps:
try:
query.append({"jobid": int(dep)})
except:
pass
jobs = ComponentProxy("queue-manager").get_jobs(query)
job_ids = set( [str(j["jobid"]) for j in jobs] )
missing = deps.difference(job_ids)
if missing:
print "WARNING: dependencies %s do not match jobs currently in the "\
"queue" % ":".join(missing)
def add_process_groups(self, specs):
"""Create a process group.
Arguments:
spec -- dictionary hash specifying a process group to start
"""
self.logger.info("add_process_groups(%r)" % (specs))
script_specs = []
other_specs = []
for spec in specs:
if spec.get('mode', False) == "script":
script_specs.append(spec)
else:
other_specs.append(spec)
# start up script jobs
script_pgroups = []
if script_specs:
for spec in script_specs:
try:
self._set_kernel(
spec.get('location')[0], spec.get('kernel', "default"))
except Exception, e:
new_pgroup = self.process_groups.q_add([spec])
pgroup = new_pgroup[0]
pgroup.nodect = self._partitions[pgroup.location[0]].size
pgroup.exit_status = 1
self.logger.info(
"process group %s: job %s/%s failed to set the kernel; %s",
pgroup.id, pgroup.jobid, pgroup.user, e)
else:
try:
script_pgroup = ComponentProxy(
"script-manager").add_jobs([spec])
except (ComponentLookupError, xmlrpclib.Fault):
self._clear_kernel(spec.get('location')[0])
# FIXME: jobs that were already started are not reported
raise ProcessGroupCreationError(
"system::add_process_groups failed to communicate with script-manager"
)
new_pgroup = self.process_groups.q_add([spec])
pgroup = new_pgroup[0]
pgroup.script_id = script_pgroup[0]['id']
pgroup.nodect = self._partitions[pgroup.location[0]].size
self.logger.info(
"job %s/%s: process group %s created to track script",
pgroup.jobid, pgroup.user, pgroup.id)
self.reserve_resources_until(
spec['location'],
time.time() + 60 * float(spec['walltime']),
pgroup.jobid)
if pgroup.kernel != "default":
self.logger.info(
"process group %s: job %s/%s using kernel %s",
pgroup.id, pgroup.jobid, pgroup.user,
pgroup.kernel)
script_pgroups.append(pgroup)
def start(self):
"""Start the process group by forking to _mpirun()"""
try:
data = self.prefork()
self.head_pid = ComponentProxy("forker").fork(data)
except:
self.logger.error(
"problem forking: pg %s did not find a child pid", self.id)
def _start_job(self, job, partition_list, resid=None):
"""Get the queue manager to start a job."""
cqm = ComponentProxy("queue-manager")
try:
self.logger.info("trying to start job %d on partition %r" %
(job.jobid, partition_list))
cqm.run_jobs([{
'tag': "job",
'jobid': job.jobid
}], partition_list, None, resid, job.walltime)
except ComponentLookupError:
self.logger.error("failed to connect to queue manager")
return
self.started_jobs[job.jobid] = self.get_current_time()
def __init__(self, *args, **kwargs):
BGSched.__init__(self, *args, **kwargs)
self.get_current_time = ComponentProxy("event-manager").get_current_time
self.COMP_QUEUE_MANAGER = "cluster-queue-manager"
self.COMP_SYSTEM = "cluster-system"
self.queues = Cobalt.Components.bgsched.QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = Cobalt.Components.bgsched.JobDict(self.COMP_QUEUE_MANAGER)
self.running_job_walltime_prediction = False
def q_add (self, *args, **kwargs):
qm = ComponentProxy("queue-manager")
try:
queues = [spec['name'] for spec in qm.get_queues([{'name':"*"}])]
except ComponentLookupError:
logger.error("unable to contact queue manager when adding reservation")
raise
try:
specs = args[0]
for spec in specs:
if "res_id" not in spec or spec['res_id'] == '*':
spec['res_id'] = bgsched_id_gen.get()
reservations = Cobalt.Data.DataDict.q_add(self, *args, **kwargs)
except KeyError, e:
raise ReservationError("Error: a reservation named %s already exists" % e)
def q_del(self, *args, **kwargs):
'''Delete a reservation from tracking.
Side Effects: Removes a queue from tracking.
Logs that the reservation has terminated.
Emits a terminated database record
Attempts to mark the queue dead in the queue-manager.
'''
reservations = Cobalt.Data.DataDict.q_del(self, *args, **kwargs)
qm = ComponentProxy('queue-manager')
queues = [spec['name'] for spec in qm.get_queues([{'name': "*"}])]
spec = [{'name': reservation.queue} for reservation in reservations \
if reservation.createdQueue and reservation.queue in queues \
and not self.q_get([{'queue':reservation.queue}])]
try:
qm.set_queues(spec, {'state': "dead"}, "bgsched")
except Exception, err:
logger.error("problem disabling reservation queue (%s)" % err)
def unregister_with_slp(self):
try:
name = self.instance.name
except AttributeError:
return
try:
ComponentProxy("service-location").unregister(name)
except Exception, e:
self.logger.error("unregister_with_slp() [%s]" % (e))
def cobalt_query(state):
cqm = ComponentProxy('queue-manager', defer=True)
scheduler = ComponentProxy('scheduler', defer=True)
if state not in ('running', 'queued', 'reservation'):
return None
# Templates for queries to coblat
query_job = dict.fromkeys(job_query_fields, '*')
query_res = dict.fromkeys(reservation_query_fields, '*')
if state == 'reservation':
return scheduler.get_reservations([query_res])
if state == 'running' or state == 'starting':
query_job['state'] = 'running'
query_job['location'] = '*'
if state == 'queued':
query_job['state'] = 'queued'
query_job['score'] = '*'
return cqm.get_jobs([query_job])
class QueueDict(ForeignDataDict):
"""Dictionary for the queue metadata cache.
"""
item_cls = Queue
key = 'name'
__oserror__ = Cobalt.Util.FailureMode("QM Connection (queue)")
__function__ = ComponentProxy("queue-manager").get_queues
__fields__ = ['name', 'state', 'policy', 'priority']
def _run_reservation_jobs(self, reservations_cache):
# handle each reservation separately, as they shouldn't be competing for resources
for cur_res in reservations_cache.itervalues():
#print "trying to run res jobs in", cur_res.name, self.started_jobs
queue = cur_res.queue
if not (self.queues.has_key(queue)
and self.queues[queue].state == 'running'):
continue
temp_jobs = self.jobs.q_get([{
'is_runnable': True,
'queue': queue
}])
active_jobs = []
for j in temp_jobs:
if not self.started_jobs.has_key(
j.jobid) and cur_res.job_within_reservation(j):
active_jobs.append(j)
if not active_jobs:
continue
active_jobs.sort(self.utilitycmp)
job_location_args = []
for job in active_jobs:
job_location_args.append({
'jobid':
str(job.jobid),
'nodes':
job.nodes,
'queue':
job.queue,
'required':
cur_res.partitions.split(":"),
'utility_score':
job.score,
'walltime':
job.walltime,
'attrs':
job.attrs,
'user':
job.user,
})
# there's no backfilling in reservations
try:
best_partition_dict = ComponentProxy(
self.COMP_SYSTEM).find_job_location(job_location_args, [])
except:
self.logger.error("failed to connect to system component")
best_partition_dict = {}
for jobid in best_partition_dict:
job = self.jobs[int(jobid)]
self._start_job(job, best_partition_dict[jobid],
{str(job.jobid): cur_res.res_id})
def q_del (self, *args, **kwargs):
'''Delete a reservation from tracking.
Side Effects: Removes a queue from tracking.
Logs that the reservation has terminated.
Emits a terminated database record
Attempts to mark the queue dead in the queue-manager.
Marks the reservation as dying
'''
reservations = Cobalt.Data.DataDict.q_del(self, *args, **kwargs)
qm = ComponentProxy('queue-manager')
queues = [spec['name'] for spec in qm.get_queues([{'name':"*"}])]
spec = [{'name': reservation.queue} for reservation in reservations \
if reservation.createdQueue and reservation.queue in queues \
and not self.q_get([{'queue':reservation.queue}])]
try:
qm.set_queues(spec, {'state':"dead"}, "bgsched")
except Exception, err:
logger.error("problem disabling reservation queue (%s)" % err)
def q_add(self, *args, **kwargs):
qm = ComponentProxy("queue-manager")
try:
queues = [spec['name'] for spec in qm.get_queues([{'name': "*"}])]
except ComponentLookupError:
logger.error(
"unable to contact queue manager when adding reservation")
raise
try:
specs = args[0]
for spec in specs:
if "res_id" not in spec or spec['res_id'] == '*':
spec['res_id'] = bgsched_id_gen.get()
reservations = Cobalt.Data.DataDict.q_add(self, *args, **kwargs)
except KeyError, e:
raise ReservationError(
"Error: a reservation named %s already exists" % e)
def add_jobs (self, specs):
"""Add a job to the process manager."""
self.logger.info("add_jobs(%r)" % (specs))
jobs = self.jobs.q_add(specs)
system_specs = \
ComponentProxy("system").add_jobs([job.to_rx() for job in jobs])
for system_spec in system_specs:
job = self.jobs[system_spec['id']]
job.state = "running"
return jobs
def signal_process_groups (self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
if pg.exit_status is None:
try:
ComponentProxy("forker").signal(pg.head_pid, signame)
except:
self.logger.error("Failed to communicate with forker when signalling job")
return my_process_groups
def get_mate_jobs_status_local(self, remote_jobid):
'''return mate job status, invoked by local functions'''
status_dict = {}
try:
status_dict = ComponentProxy(REMOTE_QUEUE_MANAGER).get_mate_job_status(remote_jobid)
except:
self.logger.error("failed to connect to remote queue-manager component!")
status_dict = {'status':'notconnected'}
self.dbglog.LogMessage("failed to connect to remote queue-manager component!")
return status_dict
def register_with_slp(self):
try:
name = self.instance.name
except AttributeError:
self.logger.error("register_with_slp() [unknown component]")
return
try:
ComponentProxy("service-location").register(name, self.url)
except Exception, e:
self.logger.error("register_with_slp() [%s]" % (e))
def __init__(self, *args, **kwargs):
BGSched.__init__(self, *args, **kwargs)
self.get_current_time = ComponentProxy("event-manager").get_current_time
predict_scheme = kwargs.get("predict", False)
if predict_scheme:
self.running_job_walltime_prediction = bool(int(predict_scheme[2]))
else:
self.running_job_walltime_prediction = False
def start(self):
"""Start the process group by contact the appropriate forker component"""
try:
data = self.prefork()
self.head_pid = ComponentProxy(self.forker, retry=False).fork([self.executable] + self.args, self.tag,
"Job %s/%s/%s" %(self.jobid, self.user, self.id), self.env, data, self.runid)
except:
_logger.error("Job %s/%s/%s: problem forking; %s did not return a child id", self.jobid, self.user, self.id,
self.forker)
raise
def signal(self, signame="SIGINT"):
"""
Do something with this process group depending on the signal
"""
logstr = "ProcessGroup:signal:"
LOGGER.debug(logstr + "%s:%s" % (self.jobid, signame))
try:
if self.local_id:
ComponentProxy("forker").signal(self.local_id, signame)
except OSError as ose:
LOGGER.exception( logstr + "failure for PG %s: %s" \
% (self.id, err))
def update(self, spec):
if spec.has_key("users"):
qm = ComponentProxy("queue-manager")
try:
qm.set_queues([{
'name': self.queue,
}], {'users': spec['users']}, "bgsched")
except ComponentLookupError:
logger.error(
"unable to contact queue manager when updating reservation users"
)
raise
# try the above first -- if we can't contact the queue-manager,
# don't update the users
if spec.has_key('cycle') and not self.cycle:
#just turned this into a cyclic reservation and need a cycle_id
spec['cycle_id'] = self.cycle_id_gen.get()
#get the user name of whoever issued the command
user_name = None
if spec.has_key('__cmd_user'):
user_name = spec['__cmd_user']
del spec['__cmd_user']
#if we're defering, pull out the 'defer' entry and send a cobalt db
#message. There really isn't a corresponding field to update
deferred = False
if spec.has_key('defer'):
logger.info("Res %s/%s: Deferring cyclic reservation: %s",
self.res_id, self.cycle_id, self.name)
dbwriter.log_to_db(user_name, "deferred", "reservation", self)
del spec['defer']
deferred = True
Data.update(self, spec)
if not deferred or not self.running:
#we only want this if we aren't defering. If we are, the cycle will
#take care of the new data object creation.
dbwriter.log_to_db(user_name, "modifying", "reservation", self)
class PowerMonitor(Component):
""" Power Monitor. Monitor runtime power consumption, logging into output log or provide to scheduler for power-aware job scheduling"""
implementation = "powmon"
name = "powmon"
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
def init_powmon_logger(self):
if self.iomon_logger == None:
self.iomon_logger = PBSlogger(self.bqsim.get_outputlog_string() + "-powmon")
def monitor_power(self):
print "monitor_power"
if __name__ == '__main__':
if '--version' in sys.argv:
print "partadm %s" % __revision__
print "cobalt %s" % __version__
raise SystemExit, 0
try:
(opts, args) = getopt.getopt(sys.argv[1:], 'adlrs:',
['dump', 'free', 'load=', 'enable', 'disable', 'activate', 'deactivate',
'queue=', 'deps=', 'xml', 'diag=', 'fail', 'unfail', 'savestate'])
except getopt.GetoptError, msg:
print msg
print helpmsg
raise SystemExit, 1
try:
system = ComponentProxy("system", defer=False)
except ComponentLookupError:
print "Failed to connect to system component"
raise SystemExit, 1
whoami = getpass.getuser()
if '-r' in sys.argv:
partdata = system.get_partitions([{'tag':'partition', 'name':name, 'children':'*'} for name in args])
parts = args
for part in partdata:
for child in part['children']:
if child not in parts:
parts.append(child)
else:
raise SystemExit, 1
if len(args) < 1:
print usehelp
raise SystemExit, 1
level = 30
if '-d' in sys.argv:
level = 10
user = pwd.getpwuid(os.getuid())[0]
Cobalt.Logging.setup_logging('qmove', to_syslog=False, level=level)
logger = Cobalt.Logging.logging.getLogger('qmove')
CP = ConfigParser.ConfigParser()
CP.read(Cobalt.CONFIG_FILES)
try:
cqm = ComponentProxy("queue-manager", defer=False)
except ComponentLookupError:
print >> sys.stderr, "Failed to connect to queue manager"
sys.exit(1)
for i in range(len(args)):
if args[i] == '*':
continue
try:
args[i] = int(args[i])
except:
logger.error("jobid must be an integer")
raise SystemExit, 1
spec = [{'tag':'job', 'user':user, 'jobid':jobid, 'project':'*', 'notify':'*',
'walltime':'*', 'queue':'*', 'procs':'*', 'nodes':'*'} for jobid in args]
def __init__(self, *args, **kwargs):
ClusterBaseSystem.__init__(self, *args, **kwargs)
self.sleep_interval = kwargs.get("sleep_interval", 0)
self.fraction = kwargs.get("cluster_fraction", 1)
self.sim_start = kwargs.get("c_trace_start", 0)
self.sim_end = kwargs.get("c_trace_end", sys.maxint)
self.anchor = kwargs.get("anchor", 0)
self.workload_file = kwargs.get("cjob")
self.output_log = MACHINE_NAME + "-" + kwargs.get("outputlog", "")
self.bgjob = kwargs.get("bgjob")
self.event_manager = ComponentProxy("event-manager")
walltime_prediction = get_histm_config("walltime_prediction", False) # *AdjEst*
print "walltime_prediction=", walltime_prediction
if walltime_prediction in ["True", "true"]:
self.walltime_prediction = True
else:
self.walltime_prediction = False
self.time_stamps = [('I', '0', 0, {})]
self.cur_time_index = 0
self.queues = SimQueueDict(policy=None)
# self.invisible_job_dict = {} # for jobs not submitted, {jobid:job_instance}
self.unsubmitted_job_spec_dict = {} #{jobid: jobspec}
self.num_running = 0
self.num_waiting = 0
self.num_busy = 0
self.num_end = 0
self.total_job = 0
self.total_nodes = len(self.all_nodes)
self.init_queues()
#initialize PBS-style logger
self.pbslog = PBSlogger(self.output_log)
#initialize debug logger
if self.output_log:
self.dbglog = PBSlogger(self.output_log+"-debug")
else:
self.dbglog = PBSlogger(".debug")
#finish tag
self.finished = False
#register local alias "system" for this component
local_components["cluster-system"] = self
#initialize capacity loss
self.capacity_loss = 0
#starting job(id)s at current time stamp. used for calculating capacity loss
self.starting_jobs = []
self.user_utility_functions = {}
self.builtin_utility_functions = {}
self.define_builtin_utility_functions()
self.define_user_utility_functions()
self.cosched_scheme_tup = kwargs.get("coscheduling", (0,0))
self.cosched_scheme = self.cosched_scheme_tup[1]
self.cosched_scheme_remote = self.cosched_scheme_tup[0]
self.mate_vicinity = kwargs.get("vicinity", 0)
self.mate_ratio = kwargs.get("mate_ratio", 0)
valid_cosched_schemes = ["hold", "yield"]
if self.cosched_scheme in valid_cosched_schemes and self.cosched_scheme_remote in valid_cosched_schemes:
self.coscheduling = True
else:
self.coscheduling = False
if not kwargs.get("bgjob", None):
self.coscheduling = False
self.mate_job_dict = {}
if self.coscheduling:
self.jobid_qtime_pairs = self.init_jobid_qtime_pairs()
try:
self.remote_jobid_qtime_pairs = ComponentProxy(REMOTE_QUEUE_MANAGER).get_jobid_qtime_pairs()
except:
self.logger.error("fail to connect to remote queue-manager component!")
self.coscheduling = False
if self.mate_vicinity:
print "start init mate job dict, vicinity=", self.mate_vicinity
self.init_mate_job_dict_by_vicinity()
elif self.mate_ratio:
print "start init mate job dict, mate_ratio=", self.mate_ratio
self.init_mate_job_dict_by_ratio(self.mate_ratio)
else:
self.logger.error("fail to initialize mate job dict!")
matejobs = len(self.mate_job_dict.keys())
proportion = float(matejobs) / self.total_job
#recording holding job id and holden resource
self.job_hold_dict = {}
#record holding job's holding time jobid:first hold (sec)
self.first_hold_time_dict = {}
#record yield jobs's first yielding time, for calculating the extra waiting time
self.first_yield_hold_time_dict = {}
#record yield job ids. update dynamically
self.yielding_job_list = []
if self.coscheduling:
remote_mate_job_dict = dict((v,k) for k, v in self.mate_job_dict.iteritems())
try:
ComponentProxy(REMOTE_QUEUE_MANAGER).set_mate_job_dict(remote_mate_job_dict)
except:
self.logger.error("failed to connect to remote queue-manager component!")
self.coscheduling = False
print "number of mate job pairs: %s, proportion in cluster jobs: %s%%" \
% (len(self.mate_job_dict.keys()), round(proportion *100, 1) )
self.max_holding_sys_util = DEFAULT_MAX_HOLDING_SYS_UTIL
elif len(attr.split("=")) == 1:
if attr[:3] == "no_":
jobspec["attrs"].update({attr[3:]:"false"})
newoptsattrs.update({attr[3:]:"false"})
else:
jobspec["attrs"].update({attr:"true"})
newoptsattrs.update({attr:"true"})
else:
print "Improperly formatted argument to attrs : %s" % attr
sys.exit(1)
opts['attrs'] = newoptsattrs
else:
opts['attrs'] = {}
try:
try:
system = ComponentProxy("system", defer=False)
except:
print >> sys.stderr, "Failed to contact system component"
sys.exit(1)
opts = system.validate_job(opts)
except xmlrpclib.Fault, flt:
logger.error("Job failed to validate: %s" % (flt.faultString))
if not opts['forcenoval']:
sys.exit(1)
if opts['project']:
jobspec['project'] = opts['project']
if opts['notify']:
jobspec['notify'] = opts['notify']
import Cobalt.Util
import xmlrpclib
from Cobalt.Proxy import ComponentProxy
from Cobalt.Exceptions import ComponentLookupError, NotSupportedError
helpmsg = '''Usage: partlist [--version]'''
if __name__ == '__main__':
if '--version' in sys.argv:
print "partlist %s" % __revision__
print "cobalt %s" % __version__
raise SystemExit, 0
try:
system = ComponentProxy("system", defer=False)
except ComponentLookupError:
print "Failed to connect to system"
raise SystemExit, 1
try:
scheduler = ComponentProxy("scheduler", defer=False)
except ComponentLookupError:
print "Failed to connect to scheduler"
raise SystemExit, 1
spec = [{'tag':'partition', 'name':'*', 'queue':'*', 'state':'*', 'size':'*',
'functional':'*', 'scheduled':'*', 'children':'*', 'backfill_time':"*", 'draining':"*"}]
try:
parts = system.get_partitions(spec)
except xmlrpclib.Fault, flt:
import sys, time, Cobalt.Proxy, Cobalt.Logging
import pwd
import os
from Cobalt.Proxy import ComponentProxy
from Cobalt.Exceptions import ComponentLookupError
import Cobalt.Util
if __name__ == '__main__':
level = 20
if '-d' in sys.argv:
level = 10
Cobalt.Logging.setup_logging('cmd', to_syslog=False, level=0)
user = pwd.getpwuid(os.getuid())[0]
try:
pm = ComponentProxy("process-manager", defer=False)
except ComponentLookupError:
print >> sys.stderr, "Failed to connect to process manager"
sys.exit(1)
r = pm.add_jobs([{'tag':'process-group', 'user':user, 'args':[], 'env':{},
'executable':'/tmp/testscript', 'size':700, 'cwd':'/tmp', 'location':['ANLR00'],
'outputfile':'/tmp/test1-output', 'errorfile':'/tmp/test1-error', 'id': '*'}])
print "jobs : " + `len(r)`
pgid = r[0]['id']
while True:
r = pm.get_jobs([{'tag':'process-group', 'id':pgid, 'state':'*'}])
state = r[0]['state']
if state == 'running':
Cobalt.Util.sleep(5)
continue
custom_header_full = CP.get("cqm", "cqstat_header_full").split(":")
except:
pass
if "CQSTAT_HEADER" in os.environ.keys():
custom_header = os.environ["CQSTAT_HEADER"].split(":")
elif "QSTAT_HEADER" in os.environ.keys():
custom_header = os.environ["QSTAT_HEADER"].split(":")
if "CQSTAT_HEADER_FULL" in os.environ.keys():
custom_header_full = os.environ["CQSTAT_HEADER_FULL"].split(":")
elif "QSTAT_HEADER_FULL" in os.environ.keys():
custom_header_full = os.environ["QSTAT_HEADER_FULL"].split(":")
if opts["header"]:
custom_header = opts["header"].split(":")
try:
cqm = ComponentProxy("queue-manager", defer=False)
except ComponentLookupError:
print >> sys.stderr, "Failed to connect to queue manager"
sys.exit(2)
try:
queues = cqm.get_queues([{"name": "*", "state": "*"}])
except:
print >> sys.stderr, "Failed to get queue list from queue manager"
sys.exit(2)
level = 30
if opts["debug"]:
level = 10
if opts["version"]:
starttime is in format: YYYY_MM_DD-HH:MM
duration may be in minutes or HH:MM:SS
cycle time may be in minutes or DD:HH:MM:SS
queue name is only needed to specify a name other than the default
cycle time, queue name, and user are optional'''
if __name__ == '__main__':
if '--version' in sys.argv:
print "setres %s" % __revision__
print "cobalt %s" % __version__
raise SystemExit, 0
if '-h' in sys.argv or '--help' in sys.argv or len(sys.argv) == 1:
print helpmsg
raise SystemExit, 0
try:
scheduler = ComponentProxy("scheduler", defer=False)
except ComponentLookupError:
print "Failed to connect to scheduler"
raise SystemExit, 1
try:
(opts, args) = getopt.getopt(sys.argv[1:], 'A:c:s:d:mn:p:q:u:axD', ['res_id=', 'cycle_id=', 'force_id'])
except getopt.GetoptError, msg:
print msg
print helpmsg
raise SystemExit, 1
try:
partitions = [opt[1] for opt in opts if opt[0] == '-p'] + args
except ValueError:
if args:
partitions = args
or opts['killq'] or opts['policy'] or opts['unsetq']:
spec = [{'tag':'queue', 'name':qname} for qname in args]
else:
for i in range(len(args)):
if args[i] == '*':
continue
try:
args[i] = int(args[i])
except:
print >> sys.stderr, "jobid must be an integer"
raise SystemExit, 1
spec = [{'tag':'job', 'jobid':jobid} for jobid in args]
try:
cqm = ComponentProxy("queue-manager")
except ComponentLookupError:
print >> sys.stderr, "Failed to connect to queue manager"
sys.exit(1)
kdata = [item for item in ['--kill', '--delete'] if item in sys.argv]
if opts['setjobid']:
try:
response = cqm.set_jobid(int(opts['setjobid']), whoami)
except ValueError:
print "The new jobid must be an integer"
raise SystemExit, 1
except xmlrpclib.Fault, flt:
print flt.faultString
raise SystemExit, 1
elif opts['savestate']:
'RunTime', 'Nodes', 'State', 'Location', 'Mode', 'Procs',
'Preemptable', 'Queue', 'StartTime', 'Index']
long_header = ['JobID', 'JobName', 'User', 'WallTime', 'QueuedTime',
'RunTime', 'RemainingTime', 'Nodes', 'State', 'Location', 'Mode', 'Procs',
'Preemptable', 'User_Hold', 'Admin_Hold', 'Queue',
'StartTime', 'Index', 'SubmitTime', 'Path', 'OutputDir',
'ErrorPath', 'OutputPath',
'Envs', 'Command', 'Args', 'Kernel', 'KernelOptions',
'Project', 'Dependencies', 'short_state', 'Notify',
'Score', 'Maxtasktime', 'attrs', 'dep_frac', 'user_list' ]
header = None
query_dependencies = {'QueuedTime':['SubmitTime', 'StartTime'], 'RunTime':['StartTime'],
'RemainingTime':['StartTime', 'WallTime']}
try:
cqm = ComponentProxy("queue-manager", defer=False)
except ComponentLookupError:
print >> sys.stderr, "Failed to connect to queue manager"
sys.exit(2)
user_name = '*'
if opts['user']:
user_name = opts['user']
if opts['q']: # querying for queues
query = [{'name' :qname, 'users':'*', 'groups':'*',
'mintime':'*', 'maxtime':'*', 'maxrunning':'*',
'maxqueued':'*', 'maxusernodes':'*',
'maxnodehours': '*',
'totalnodes':'*', 'state':'*', 'priority':'*'} for qname in names]
header = ['Name', 'State', 'Users', 'Groups', 'MinTime', 'MaxTime',
custom_header_full = CP.get('cqm', 'cqstat_header_full').split(':')
except:
pass
if 'CQSTAT_HEADER' in os.environ.keys():
custom_header = os.environ['CQSTAT_HEADER'].split(':')
elif 'QSTAT_HEADER' in os.environ.keys():
custom_header = os.environ['QSTAT_HEADER'].split(':')
if 'CQSTAT_HEADER_FULL' in os.environ.keys():
custom_header_full = os.environ['CQSTAT_HEADER_FULL'].split(':')
elif 'QSTAT_HEADER_FULL' in os.environ.keys():
custom_header_full = os.environ['QSTAT_HEADER_FULL'].split(':')
if opts['header']:
custom_header = opts['header'].split(':')
try:
cqm = ComponentProxy("queue-manager", defer=False)
except ComponentLookupError:
print >> sys.stderr, "Failed to connect to queue manager"
sys.exit(2)
try:
queues = cqm.get_queues([{'name':"*", 'state':"*"}])
except:
print >> sys.stderr, "Failed to get queue list from queue manager"
sys.exit(2)
level = 30
if opts['debug']:
level = 10
if opts['version']:
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.event_manager = ComponentProxy("event-manager")
self.bqsim = ComponentProxy("queue-manager")
self.powmon_logger = None
if sys_type == 'bgq':
print_block = print_block_bgq
if __name__ == '__main__':
try:
opts, args = opt_parser.parse_args()
except optparse.OptParseError, msg:
print msg
print helpmsg
raise SystemExit, 1
try:
system = ComponentProxy("system", defer=False)
except ComponentLookupError:
print "Failed to connect to system component"
raise SystemExit, 1
whoami = pwd.getpwuid(os.getuid())[0]
if opts.recursive:
partdata = system.get_partitions([{'tag':'partition', 'name':name, 'children_list':'*'} for name in args])
parts = args
for part in partdata:
for child in part['children']:
if child not in parts:
parts.append(child)
else:
from Cobalt.Exceptions import ComponentLookupError
if __name__ == '__main__':
p = optparse.OptionParser(usage="%prog <reservation name>",
description="This program does things to reservations you are done using. Cyclic reservations are deferred until the next time they repeat, while one time reservations are released.")
if len(sys.argv) == 1:
p.print_help()
sys.exit(1)
opt, args = p.parse_args()
reservation_names = args
try:
scheduler = ComponentProxy("scheduler", defer=False)
except ComponentLookupError:
print "Failed to connect to scheduler"
sys.exit(1)
# Check if reservation exists
spec = [{'name': rname, 'users':"*", 'start':'*', 'cycle':'*', 'duration':'*'} for rname in reservation_names]
try:
result = scheduler.get_reservations(spec)
except:
print "Error communicating with scheduler"
sys.exit(1)
if len(result) and len(result) != len(args):
print "Reservation subset matched"
elif not result:
def check_done_cleaning(self):
"""Check to see if the processes we are using to clean up nodes
post-run are done. If they are, release the nodes back for general
consumption. If the cleanup fails for some reason, then mark the node
down and release it.
"""
if self.cleaning_processes == []:
#don't worry if we have nothing to cleanup
return
for cleaning_process in self.cleaning_processes:
#if we can't reach the forker, we've lost all the cleanup scripts.
#don't try and recover, just assume all nodes that were being
#cleaned are down. --PMR
if cleaning_process['retry'] == True:
continue #skip this. Try anyway, if component came back up.
jobid = cleaning_process['jobid']
user = cleaning_process['user']
try:
exit_status = ComponentProxy("system_script_forker").child_completed(
cleaning_process['cleaning_id'])
ComponentProxy("system_script_forker").child_cleanup(
[cleaning_process['cleaning_id']])
except ComponentLookupError:
self.logger.error("Job %s/%s: Error contacting forker "
"component. Running child processes are "
"unrecoverable." % (jobid, user))
return
if exit_status != None:
#we're done, this node is now free to be scheduled again.
self.running_nodes.discard(cleaning_process["host"])
cleaning_process["completed"] = True
self.cleaning_host_count[jobid] -= 1
else:
#timeout exceeded
if (time.time() - cleaning_process["start_time"] >
float(get_cluster_system_config("epilogue_timeout", 60.0))):
cleaning_process["completed"] = True
try:
forker = ComponentProxy("system_script_forker").signal(
cleaning_process['cleaning_id'], "SIGINT")
child_output = forker.get_child_data(
cleaning_process['cleaning_id'])
forker.child_cleanup([cleaning_process['cleaning_id']])
#mark as dirty and arrange to mark down.
self.down_nodes.add(cleaning_process['host'])
self.running_nodes.discard(cleaning_process['host'].host) # <---????check this!
self.logger.error("Job %s/%s: epilogue timed out on host %s, marking hosts down",
user, jobid, cleaning_process['host'])
self.logger.error("Job %s/%s: stderr from epilogue on host %s: [%s]",
user, jobid,
cleaning_process['host'],
child_output['stderr'].strip())
self.cleaning_host_count[jobid] -= 1
except ComponentLookupError:
self.logger.error("Job %s/%s: Error contacting forker "
"component. Running child processes are "
"unrecoverable." % (jobid, user))
if self.cleaning_host_count[jobid] == 0:
self.del_process_groups(jobid)
#clean up other cleanup-monitoring stuff
self.logger.info("Job %s/%s: job finished on %s",
user, jobid, Cobalt.Util.merge_nodelist(self.locations_by_jobid[jobid]))
del self.locations_by_jobid[jobid]
del self.jobid_to_user[jobid]
self.cleaning_processes = [cleaning_process for cleaning_process in self.cleaning_processes
if cleaning_process["completed"] == False]
class ClusterQsim(ClusterBaseSystem):
'''Cobalt Queue Simulator for cluster systems'''
implementation = "cqsim"
name = "cluster-queue-manager"
alias = "cluster-system"
logger = logging.getLogger(__name__)
def __init__(self, *args, **kwargs):
ClusterBaseSystem.__init__(self, *args, **kwargs)
self.sleep_interval = kwargs.get("sleep_interval", 0)
self.fraction = kwargs.get("cluster_fraction", 1)
self.sim_start = kwargs.get("c_trace_start", 0)
self.sim_end = kwargs.get("c_trace_end", sys.maxint)
self.anchor = kwargs.get("anchor", 0)
self.workload_file = kwargs.get("cjob")
self.output_log = MACHINE_NAME + "-" + kwargs.get("outputlog", "")
self.bgjob = kwargs.get("bgjob")
self.event_manager = ComponentProxy("event-manager")
walltime_prediction = get_histm_config("walltime_prediction", False) # *AdjEst*
print "walltime_prediction=", walltime_prediction
if walltime_prediction in ["True", "true"]:
self.walltime_prediction = True
else:
self.walltime_prediction = False
self.time_stamps = [('I', '0', 0, {})]
self.cur_time_index = 0
self.queues = SimQueueDict(policy=None)
# self.invisible_job_dict = {} # for jobs not submitted, {jobid:job_instance}
self.unsubmitted_job_spec_dict = {} #{jobid: jobspec}
self.num_running = 0
self.num_waiting = 0
self.num_busy = 0
self.num_end = 0
self.total_job = 0
self.total_nodes = len(self.all_nodes)
self.init_queues()
#initialize PBS-style logger
self.pbslog = PBSlogger(self.output_log)
#initialize debug logger
if self.output_log:
self.dbglog = PBSlogger(self.output_log+"-debug")
else:
self.dbglog = PBSlogger(".debug")
#finish tag
self.finished = False
#register local alias "system" for this component
local_components["cluster-system"] = self
#initialize capacity loss
self.capacity_loss = 0
#starting job(id)s at current time stamp. used for calculating capacity loss
self.starting_jobs = []
self.user_utility_functions = {}
self.builtin_utility_functions = {}
self.define_builtin_utility_functions()
self.define_user_utility_functions()
self.cosched_scheme_tup = kwargs.get("coscheduling", (0,0))
self.cosched_scheme = self.cosched_scheme_tup[1]
self.cosched_scheme_remote = self.cosched_scheme_tup[0]
self.mate_vicinity = kwargs.get("vicinity", 0)
self.mate_ratio = kwargs.get("mate_ratio", 0)
valid_cosched_schemes = ["hold", "yield"]
if self.cosched_scheme in valid_cosched_schemes and self.cosched_scheme_remote in valid_cosched_schemes:
self.coscheduling = True
else:
self.coscheduling = False
if not kwargs.get("bgjob", None):
self.coscheduling = False
self.mate_job_dict = {}
if self.coscheduling:
self.jobid_qtime_pairs = self.init_jobid_qtime_pairs()
try:
self.remote_jobid_qtime_pairs = ComponentProxy(REMOTE_QUEUE_MANAGER).get_jobid_qtime_pairs()
except:
self.logger.error("fail to connect to remote queue-manager component!")
self.coscheduling = False
if self.mate_vicinity:
print "start init mate job dict, vicinity=", self.mate_vicinity
self.init_mate_job_dict_by_vicinity()
elif self.mate_ratio:
print "start init mate job dict, mate_ratio=", self.mate_ratio
self.init_mate_job_dict_by_ratio(self.mate_ratio)
else:
self.logger.error("fail to initialize mate job dict!")
matejobs = len(self.mate_job_dict.keys())
proportion = float(matejobs) / self.total_job
#recording holding job id and holden resource
self.job_hold_dict = {}
#record holding job's holding time jobid:first hold (sec)
self.first_hold_time_dict = {}
#record yield jobs's first yielding time, for calculating the extra waiting time
self.first_yield_hold_time_dict = {}
#record yield job ids. update dynamically
self.yielding_job_list = []
if self.coscheduling:
remote_mate_job_dict = dict((v,k) for k, v in self.mate_job_dict.iteritems())
try:
ComponentProxy(REMOTE_QUEUE_MANAGER).set_mate_job_dict(remote_mate_job_dict)
except:
self.logger.error("failed to connect to remote queue-manager component!")
self.coscheduling = False
print "number of mate job pairs: %s, proportion in cluster jobs: %s%%" \
% (len(self.mate_job_dict.keys()), round(proportion *100, 1) )
self.max_holding_sys_util = DEFAULT_MAX_HOLDING_SYS_UTIL
def get_current_time(self):
'''this function overrid the get_current_time in bgsched, bg_base_system, and cluster_base_system'''
return self.event_manager.get_current_time()
def get_current_time_sec(self):
return self.event_manager.get_current_time()
def get_current_time_date(self):
return self.event_manager.get_current_date_time()
def insert_time_stamp(self, timestamp, type, info):
'''insert time stamps in the same order'''
if type not in SET_event:
print "invalid event type,", type
return
evspec = {}
evspec['jobid'] = info.get('jobid', 0)
evspec['type'] = type
evspec['datetime'] = sec_to_date(timestamp)
evspec['unixtime'] = timestamp
evspec['machine'] = MACHINE_ID
self.event_manager.add_event(evspec)
def _get_queuing_jobs(self):
return [job for job in self.queues.get_jobs([{'is_runnable':True}])]
queuing_jobs = property(_get_queuing_jobs)
def _get_running_jobs(self):
return [job for job in self.queues.get_jobs([{'has_resources':True}])]
running_jobs = property(_get_running_jobs)
def add_queues(self, specs):
'''add queues'''
return self.queues.add_queues(specs)
add_queues = exposed(query(add_queues))
def get_queues(self, specs):
'''get queues'''
return self.queues.get_queues(specs)
get_queues = exposed(query(get_queues))
def init_queues(self):
'''parses the work load log file, initializes queues and sorted time
stamp list'''
print "Initializing cluster jobs, one moment please..."
raw_jobs = parse_work_load(self.workload_file)
specs = []
tag = 0
for key in raw_jobs:
spec = {}
tmp = raw_jobs[key]
spec['jobid'] = tmp.get('jobid')
spec['queue'] = tmp.get('queue')
#convert submittime from "%m/%d/%Y %H:%M:%S" to Unix time sec
format_sub_time = tmp.get('submittime')
if format_sub_time:
qtime = date_to_sec(format_sub_time)
if qtime < self.sim_start or qtime > self.sim_end:
continue
spec['submittime'] = qtime
#spec['submittime'] = float(tmp.get('qtime'))
spec['first_subtime'] = spec['submittime'] #set the first submit time
else:
continue
spec['user'] = tmp.get('user')
spec['project'] = tmp.get('account')
#convert walltime from 'hh:mm:ss' to float of minutes
format_walltime = tmp.get('Resource_List.walltime')
spec['walltime'] = 0
if format_walltime:
parts = format_walltime.split(',')
days = 0
if len(parts) > 1: #contain day: 1 day, 11:00:00
days = int(parts[0].split(' ')[0])
minutes_part = parts[1]
else:
minutes_part = parts[0]
segs = minutes_part.split(':')
walltime_minutes = int(segs[0])*60 + int(segs[1])
total_walltime_minutes = walltime_minutes + days * 24 * 60
spec['walltime'] = str(total_walltime_minutes)
else: #invalid job entry, discard
continue
if tmp.get('start') and tmp.get('end'):
act_run_time = float(tmp.get('end')) - float(tmp.get('start'))
if act_run_time / (float(spec['walltime'])*60) > 1.1:
act_run_time = float(spec['walltime'])*60
spec['runtime'] = str(round(act_run_time, 1))
else:
continue
if tmp.get('Resource_List.nodect'):
spec['nodes'] = tmp.get('Resource_List.nodect')
if int(spec['nodes']) == 40960:
continue
else: #invalid job entry, discard
continue
if self.walltime_prediction: #*AdjEst*
ap = self.get_walltime_Ap(spec)
spec['walltime_p'] = float(spec['walltime']) * ap
else:
spec['walltime_p'] = float(spec['walltime'])
spec['state'] = 'invisible'
spec['start_time'] = '0'
spec['end_time'] = '0'
spec['queue'] = "default"
spec['has_resources'] = False
spec['is_runnable'] = False
#add the job spec to the spec list
specs.append(spec)
specs.sort(subtimecmp)
#adjust workload density and simulation start time
if self.fraction != 1 or self.anchor !=0 :
tune_workload(specs, self.fraction, self.anchor)
print "workload adjusted: "
print "first job submitted:", sec_to_date(specs[0].get('submittime'))
print "last job submitted:", sec_to_date(specs[len(specs)-1].get('submittime'))
self.total_job = len(specs)
print "total job number:", self.total_job
#self.add_jobs(specs)
self.unsubmitted_job_spec_dict = self.init_unsubmitted_dict(specs)
self.event_manager.add_init_events(specs, MACHINE_ID)
return 0
def init_unsubmitted_dict(self, specs):
#jobdict = {}
specdict = {}
for spec in specs:
jobid = str(spec['jobid'])
#new_job = Job(spec)
#jobdict[jobid] = new_job
specdict[jobid] = spec
return specdict
def get_walltime_Ap(self, spec): #*AdjEst*
'''get walltime adjusting parameter from history manager component'''
projectname = spec.get('project')
username = spec.get('user')
if prediction_scheme == "paired":
return self.history_manager.get_Ap_by_keypair(username, projectname)
Ap_proj = self.history_manager.get_Ap('project', projectname)
Ap_user = self.history_manager.get_Ap('user', username)
if prediction_scheme == "project":
return Ap_proj
elif prediction_scheme == "user":
print "Ap_user==========", Ap_user
return Ap_user
elif prediction_scheme == "combined":
return (Ap_proj + Ap_user) / 2
else:
return self.history_manager.get_Ap_by_keypair(username, projectname)
def log_job_event(self, eventtype, timestamp, spec):
'''log job events(Queue,Start,End) to PBS-style log'''
def len2 (_input):
_input = str(_input)
if len(_input) == 1:
return "0" + _input
else:
return _input
if eventtype == 'Q': #submitted(queued) for the first time
message = "%s;Q;%s;queue=%s" % (timestamp, spec['jobid'], spec['queue'])
elif eventtype == 'R': #resume running after failure recovery
message = "%s;R;%s" % (timestamp, ":".join(spec['location']))
else:
wall_time = spec['walltime']
walltime_minutes = len2(int(float(wall_time)) % 60)
walltime_hours = len2(int(float(wall_time)) // 60)
log_walltime = "%s:%s:00" % (walltime_hours, walltime_minutes)
if eventtype == 'S': #start running
message = "%s;S;%s;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s start=%s exec_host=%s" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'],
spec['nodes'], log_walltime, spec['start_time'], ":".join(spec['location']))
elif eventtype == 'H': #hold some resource
message = "%s;H;%s;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s exec_host=%s" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'],
spec['nodes'], log_walltime, ":".join(spec['location']))
elif eventtype == "U": #unhold some resources
message = "%s;U;%s;host=%s" % \
(timestamp, spec['jobid'], ":".join(spec['location']))
elif eventtype == 'E': #end
first_yield_hold = self.first_yield_hold_time_dict.get(int(spec['jobid']), 0)
if first_yield_hold > 0:
overhead = spec['start_time'] - first_yield_hold
else:
overhead = 0
message = "%s;E;%s;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s start=%s end=%f exec_host=%s runtime=%s hold=%s overhead=%s" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'], spec['nodes'], log_walltime, spec['start_time'],
round(float(spec['end_time']), 1), ":".join(spec['location']),
spec['runtime'], spec['hold_time'], overhead)
else:
print "---invalid event type, type=", eventtype
return
self.pbslog.LogMessage(message)
def get_live_job_by_id(self, jobid):
'''get waiting or running job instance by jobid'''
job = None
joblist = self.queues.get_jobs([{'jobid':int(jobid)}])
if joblist:
job = joblist[0]
return job
def get_jobs(self, specs):
'''get a list of jobs, each time triggers time stamp increment and job
states update'''
jobs = []
if self.event_manager.get_go_next():
del self.yielding_job_list[:]
cur_event = self.event_manager.get_current_event_type()
if cur_event in ["Q", "E"]:
self.update_job_states(specs, {}, cur_event)
self.compute_utility_scores()
#unhold holding job. MUST be after compute_utility_scores()
if cur_event == "U":
cur_job = self.event_manager.get_current_event_job()
if cur_job in self.job_hold_dict.keys():
self.unhold_job(cur_job)
else:
#if the job not in job_hold_dict, do nothing. the job should have already started
return []
if cur_event == "C":
if self.job_hold_dict.keys():
self.unhold_all()
self.event_manager.set_go_next(True)
jobs = self.queues.get_jobs([{'tag':"job"}])
if self.yielding_job_list:
jobs = [job for job in jobs if job.jobid not in self.yielding_job_list]
return jobs
get_jobs = exposed(query(get_jobs))
def update_job_states(self, specs, updates, cur_event):
'''update the state of the jobs associated to the current time stamp'''
ids_str = str(self.event_manager.get_current_event_job())
ids = ids_str.split(':')
#print "current event=", cur_event, " ", ids
for Id in ids:
if cur_event == "Q": # Job (Id) is submitted
tempspec = self.unsubmitted_job_spec_dict[Id]
tempspec['state'] = "queued" #invisible -> queued
tempspec['is_runnable'] = True #False -> True
self.queues.add_jobs([tempspec])
self.num_waiting += 1
self.log_job_event("Q", self.get_current_time_date(), tempspec)
del self.unsubmitted_job_spec_dict[Id]
elif cur_event=="E": # Job (Id) is completed
joblist = self.queues.get_jobs([{'jobid':int(Id)}])
if joblist:
completed_job = joblist[0]
else:
return 0
#log the job end event
jobspec = completed_job.to_rx()
#print "end jobspec=", jobspec
if jobspec['end_time']:
end = float(jobspec['end_time'])
else:
end = 0
end_datetime = sec_to_date(end)
self.log_job_event("E", end_datetime, jobspec)
#free nodes
self.nodes_up(completed_job.location)
self.num_busy -= len(completed_job.location)
#delete the job instance from self.queues
self.queues.del_jobs([{'jobid':int(Id)}])
self.num_running -= 1
self.num_end += 1
return 0
def run_job_updates(self, jobspec, newattr):
''' return the state updates (including state queued -> running,
setting the start_time, end_time)'''
updates = {}
#print "enter run_job_updates, jobspec=", jobspec
start = self.get_current_time_sec()
updates['start_time'] = start
updates['starttime'] = start
updates['state'] = 'running'
updates['system_state'] = 'running'
updates['is_runnable'] = False
updates['has_resources'] = True
if jobspec['last_hold'] > 0:
updates['hold_time'] = jobspec['hold_time'] + self.get_current_time_sec() - jobspec['last_hold']
#print self.get_current_time_date(), "run job state change, job", jobspec['jobid'], \
# ":", jobspec['state'], "->", updates['state']
#determine whether the job is going to fail before completion
location = newattr['location']
duration = jobspec['remain_time']
end = start + duration
updates['end_time'] = end
self.insert_time_stamp(end, "E", {'jobid':jobspec['jobid']})
updates.update(newattr)
return updates
def add_jobs(self, specs):
'''Add a job'''
response = self.queues.add_jobs(specs)
return response
add_jobs = exposed(query(add_jobs))
def run_jobs(self, specs, nodelist):
'''run a queued job, by updating the job state, start_time and
end_time, invoked by bgsched'''
#print "run job ", specs, " on nodes", nodelist
if specs == None:
return 0
for spec in specs:
action = "start"
dbgmsg = ""
if self.coscheduling:
local_job_id = spec.get('jobid') #int
#check whether there is a mate job
mate_job_id = self.mate_job_dict.get(local_job_id, 0)
#if mate job exists, get the status of the mate job
if mate_job_id > 0:
remote_status = self.get_mate_jobs_status_local(mate_job_id).get('status', "unknown")
dbgmsg1 = "local=%s;mate=%s;mate_status=%s" % (local_job_id, mate_job_id, remote_status)
self.dbglog.LogMessage(dbgmsg1)
if remote_status in ["queuing", "unsubmitted"]:
if self.cosched_scheme == "hold": # hold resource if mate cannot run, favoring job
action = "start_both_or_hold"
if self.cosched_scheme == "yield": # give up if mate cannot run, favoring sys utilization
action = "start_both_or_yield"
if remote_status == "holding":
action = "start_both"
#self.dbglog.LogMessage(dbgmsg)
#to be inserted co-scheduling handling code
else:
pass
if action == "start":
#print "CQSIM-normal: start job %s on nodes %s" % (spec['jobid'], nodelist)
self.start_job([spec], {'location': nodelist})
elif action == "start_both_or_hold":
#print "try to hold job %s on location %s" % (local_job_id, nodelist)
mate_job_can_run = False
#try to invoke a scheduling iteration to see if remote yielding job can run now
try:
mate_job_can_run = ComponentProxy(REMOTE_QUEUE_MANAGER).try_to_run_mate_job(mate_job_id)
except:
self.logger.error("failed to connect to remote queue-manager component!")
if mate_job_can_run:
#now that mate has been started, start local job
self.start_job([spec], {'location': nodelist})
dbgmsg += " ###start both"
else:
self.hold_job(spec, {'location': nodelist})
elif action == "start_both":
#print "start both mated jobs %s and %s" % (local_job_id, mate_job_id)
self.start_job([spec], {'location': nodelist})
ComponentProxy(REMOTE_QUEUE_MANAGER).run_holding_job([{'jobid':mate_job_id}])
elif action == "start_both_or_yield":
mate_job_can_run = False
#try to invoke a scheduling iteration to see if remote yielding job can run now
try:
mate_job_can_run = ComponentProxy(REMOTE_QUEUE_MANAGER).try_to_run_mate_job(mate_job_id)
except:
self.logger.error("failed to connect to remote queue-manager component!")
if mate_job_can_run:
#now that mate has been started, start local job
self.start_job([spec], {'location': nodelist})
dbgmsg += " ###start both"
else:
#mate job cannot run, give up the turn. mark the job as yielding.
job_id = spec.get('jobid')
self.yielding_job_list.append(job_id) #int
#record the first time this job yields
if not self.first_yield_hold_time_dict.has_key(job_id):
self.first_yield_hold_time_dict[job_id] = self.get_current_time_sec()
self.dbglog.LogMessage("%s: job %s first yield" % (self.get_current_time_date(), job_id))
#set tag false, enable scheduling another job at the same time
self.event_manager.set_go_next(False)
#self.print_screen()
return len(specs)
run_jobs = exposed(run_jobs)
def start_job(self, specs, updates):
'''update the job state and start_time and end_time when cqadm --run
is issued to a group of jobs'''
nodelist = updates['location']
self.nodes_down(nodelist)
self.num_busy += len(nodelist)
self.num_running += 1
self.num_waiting -= 1
def _start_job(job, newattr):
'''callback function to update job start/end time'''
temp = job.to_rx()
newattr = self.run_job_updates(temp, newattr)
temp.update(newattr)
job.update(newattr)
self.log_job_event('S', self.get_current_time_date(), temp)
return self.queues.get_jobs(specs, _start_job, updates)
def find_job_location(self, arg_list, end_times):
best_location_dict = {}
winner = arg_list[0]
# first time through, try for starting jobs based on utility scores
for args in arg_list:
location_data = self._find_job_location(args)
if location_data:
best_location_dict.update(location_data)
break
# the next time through, try to backfill, but only if we couldn't find anything to start
if not best_location_dict:
job_end_times = {}
total = 0
for item in sorted(end_times, cmp=self._backfill_cmp):
total += len(item[0])
job_end_times[total] = item[1]
needed = int(winner['nodes']) - len(self._get_available_nodes(winner))
now = self.get_current_time() ##different from super function
backfill_cutoff = 0
for num in sorted(job_end_times):
if needed <= num:
backfill_cutoff = job_end_times[num] - now
for args in arg_list:
if 60*float(args['walltime']) > backfill_cutoff:
continue
location_data = self._find_job_location(args)
if location_data:
best_location_dict.update(location_data)
self.logger.info("backfilling job %s" % args['jobid'])
break
#!!!following two lines must be commented for coscheduling feature because giving up may occur. when
# a job is found location but give up to run, the nodes can't be updated to running status.
# reserve the stuff in the best_partition_dict, as those partitions are allegedly going to
# be running jobs very soon
# for location_list in best_location_dict.itervalues():
# self.running_nodes.update(location_list)
return best_location_dict
find_job_location = exposed(find_job_location)
# order the jobs with biggest utility first
def utilitycmp(self, job1, job2):
return -cmp(job1.score, job2.score)
def compute_utility_scores (self):
utility_scores = []
current_time = self.get_current_time_sec()
for job in self.queues.get_jobs([{'is_runnable':True}]):
utility_name = self.queues[job.queue].policy
args = {'queued_time':current_time - float(job.submittime),
'wall_time': 60*float(job.walltime_p), # *AdjEst*
'size': float(job.nodes),
'user_name': job.user,
'project': job.project,
'queue_priority': int(self.queues[job.queue].priority),
#'machine_size': max_nodes,
'jobid': int(job.jobid),
'score': job.score,
'recovering': job.recovering,
'state': job.state,
}
try:
if utility_name in self.builtin_utility_functions:
utility_func = self.builtin_utility_functions[utility_name]
else:
utility_func = self.user_utility_functions[utility_name]
utility_func.func_globals.update(args)
score = utility_func()
except KeyError:
# do something sensible when the requested utility function doesn't exist
# probably go back to the "default" one
# and if we get here, try to fix it and throw away this scheduling iteration
self.logger.error("cannot find utility function '%s' named by queue '%s'" % (utility_name, job.queue))
self.user_utility_functions[utility_name] = self.builtin_utility_functions["default"]
self.logger.error("falling back to 'default' policy to replace '%s'" % utility_name)
return
except:
# do something sensible when the requested utility function explodes
# probably go back to the "default" one
# and if we get here, try to fix it and throw away this scheduling iteration
self.logger.error("error while executing utility function '%s' named by queue '%s'" % (utility_name, job.queue), \
exc_info=True)
self.user_utility_functions[utility_name] = self.builtin_utility_functions["default"]
self.logger.error("falling back to 'default' policy to replace '%s'" % utility_name)
return
try:
job.score += score
except:
self.logger.error("utility function '%s' named by queue '%s' returned a non-number" % (utility_name, job.queue), \
exc_info=True)
self.user_utility_functions[utility_name] = self.builtin_utility_functions["default"]
self.logger.error("falling back to 'default' policy to replace '%s'" % utility_name)
return
def define_user_utility_functions(self):
self.logger.info("building user utility functions")
self.user_utility_functions.clear()
filename = os.path.expandvars(get_bgsched_config("utility_file", ""))
try:
f = open(filename)
except:
#self.logger.error("Can't read utility function definitions from file %s" % get_bgsched_config("utility_file", ""))
return
str = f.read()
try:
code = compile(str, filename, 'exec')
except:
self.logger.error("Problem compiling utility function definitions.", exc_info=True)
return
globals = {'math':math, 'time':time}
locals = {}
try:
exec code in globals, locals
except:
self.logger.error("Problem executing utility function definitions.", exc_info=True)
for thing in locals.values():
if type(thing) is types.FunctionType:
if thing.func_name in self.builtin_utility_functions:
self.logger.error("Attempting to overwrite builtin utility function '%s'. User version discarded." % \
thing.func_name)
else:
self.user_utility_functions[thing.func_name] = thing
define_user_utility_functions = exposed(define_user_utility_functions)
def define_builtin_utility_functions(self):
self.logger.info("building builtin utility functions")
self.builtin_utility_functions.clear()
# I think this duplicates cobalt's old scheduling policy
# higher queue priorities win, with jobid being the tie breaker
def default0():
val = queue_priority + 0.1
return val
def default1():
'''FCFS'''
val = queued_time
return val
def default():
'''WFP, supporting coordinated job recovery'''
wall_time_sec = wall_time*60
val = ( queued_time / wall_time_sec)**3 * (size/64.0)
return val
def high_prio():
val = 1.0
return val
self.builtin_utility_functions["default"] = default
self.builtin_utility_functions["high_prio"] = high_prio
#####coscheduling stuff
def init_jobid_qtime_pairs(self):
'''initialize mate job dict'''
jobid_qtime_pairs = []
for id, spec in self.unsubmitted_job_spec_dict.iteritems():
qtime = spec['submittime']
jobid_qtime_pairs.append((qtime, int(id)))
def _qtimecmp(tup1, tup2):
return cmp(tup1[0], tup2[0])
jobid_qtime_pairs.sort(_qtimecmp)
return jobid_qtime_pairs
def find_mate_id(self, qtime, threshold):
mate_subtime = 0
ret_id = 0
last = (0,0)
for pair in self.remote_jobid_qtime_pairs:
if pair[0] > qtime:
break
last = pair
mate_subtime = last[0]
mate_id = last[1]
if mate_subtime > 0:
if float(qtime) - mate_subtime < threshold:
ret_id = mate_id
return ret_id
def init_mate_job_dict_by_vicinity(self):
'''init mate job dictionary by vicinity'''
temp_dict = {} #remote_id:local_id
for id, spec in self.unsubmitted_job_spec_dict.iteritems():
id = int(id)
submit_time = spec.get('submittime')
mate_id = self.find_mate_id(submit_time, self.mate_vicinity)
if mate_id > 0:
#self.mate_job_dict[spec['jobid']] = int(mateid)
if temp_dict.has_key(mate_id):
tmp = temp_dict[mate_id]
if id > tmp:
temp_dict[mate_id] = id
else:
temp_dict[mate_id] = id
#reserve dict to local_id:remote_id. (guarentee one-to-one)
self.mate_job_dict = dict((local_id, remote_id) for remote_id, local_id in temp_dict.iteritems())
def init_mate_job_dict_by_ratio(self, ratio):
'''init mate job dictionary by specified ratio'''
if ratio <= 0.5:
step = int(1.0 / ratio)
reverse_step = 1
else:
step = 1
reverse_step = int(1.0/(1-ratio))
print "step=", step
print "reverse_step=", reverse_step
i = 0
temp_dict = {}
for item in self.jobid_qtime_pairs:
remote_item = self.remote_jobid_qtime_pairs[i]
random_number = random.random()
if step > 1:
if i % step == 0:
temp_dict[item[1]] = remote_item[1]
if reverse_step > 1:
if i % reverse_step != 0:
temp_dict[item[1]] = remote_item[1]
i += 1
self.mate_job_dict = temp_dict
def get_mate_job_dict(self):
return self.mate_job_dict
get_mate_job_dict = exposed(get_mate_job_dict)
def hold_job(self, spec, updates):
'''hold a job. a holding job is not started but hold some resources that can run itself in the future
once its mate job in a remote system can be started immediatly. Note, one time hold only one job'''
def _hold_job(job, newattr):
'''callback function to update job start/end time'''
temp = job.to_rx()
newattr = self.hold_job_updates(temp, newattr)
temp.update(newattr)
job.update(newattr)
self.log_job_event('H', self.get_current_time_date(), temp)
current_holden_nodes = 0
for nodelist in self.job_hold_dict.values():
current_holden_nodes += len(nodelist)
nodelist = updates['location']
job_id = spec['jobid']
if current_holden_nodes + len(nodelist) < self.max_holding_sys_util * self.total_nodes:
self.job_hold_dict[job_id] = nodelist
if not self.first_hold_time_dict.has_key(job_id):
self.first_hold_time_dict[job_id] = self.get_current_time_sec()
self.nodes_down(nodelist)
if not self.first_yield_hold_time_dict.has_key(job_id):
self.first_yield_hold_time_dict[job_id] = self.get_current_time_sec()
return self.queues.get_jobs([spec], _hold_job, updates)
else:
#if execeeding the maximum limite of holding nodes, the job will not hold but yield
self.yielding_job_list.append(job_id) #int
#record the first time this job yields
if not self.first_yield_hold_time_dict.has_key(job_id):
self.first_yield_hold_time_dict[job_id] = self.get_current_time_sec()
self.dbglog.LogMessage("%s: job %s first yield" % (self.get_current_time_date(), job_id))
return 0
def hold_job_updates(self, jobspec, newattr):
''' return the state updates (including state queued -> running,
setting the start_time, end_time)'''
updates = {}
updates['is_runnable'] = False
updates['has_resources'] = False
updates['state'] = "holding"
updates['last_hold'] = self.get_current_time_sec()
updates.update(newattr)
if SELF_UNHOLD_INTERVAL > 0:
release_time = self.get_current_time_sec() + SELF_UNHOLD_INTERVAL
self.insert_time_stamp(release_time, "U", {'jobid':jobspec['jobid'], 'location':newattr['location']})
return updates
def unhold_job(self, jobid):
'''if a job holds a partition longer than MAX_HOLD threshold, the job will release the partition and starts yielding'''
nodelist = self.job_hold_dict.get(jobid)
#release holden partitions
if nodelist:
self.nodes_up(nodelist)
else:
print "holding job %s not found in job_hold_dict: " % jobid
def _unholding_job(job, newattr):
'''callback function'''
temp = job.to_rx()
newattr = self.unholding_job_updates(temp, newattr)
temp.update(newattr)
job.update(newattr)
self.log_job_event("U", self.get_current_time_date(), temp)
del self.job_hold_dict[jobid]
return self.queues.get_jobs([{'jobid':jobid}], _unholding_job, {'location':self.job_hold_dict.get(jobid, ["N"])})
def unholding_job_updates(self, jobspec, newattr):
'''unhold job'''
updates = {}
updates['is_runnable'] = True
updates['has_resources'] = False
updates['state'] = "queued"
#set the job to lowest priority at this scheduling point.
#if no other job gets the nodes it released, the unholden job can hold those nodes again
updates['score'] = 0
updates['hold_time'] = jobspec['hold_time'] + self.get_current_time_sec() - jobspec['last_hold']
updates['last_hold'] = 0
updates.update(newattr)
return updates
def unhold_all(self):
'''unhold all jobs. periodically invoked to prevent deadlock'''
for jobid in self.job_hold_dict.keys():
job_hold_time = self.get_current_time_sec() - self.first_hold_time_dict[jobid]
#if a job has holden at least 10 minutes, then periodically unhold it
if job_hold_time > AT_LEAST_HOLD:
self.unhold_job(jobid)
def try_to_run_mate_job(self, _jobid):
'''try to run mate job, start all the jobs that can run. If the started
jobs include the given mate job, return True else return False. _jobid : int
'''
#if the job is not yielding, do not continue; no other job is possibly to be scheduled
if _jobid not in self.yielding_job_list:
return False
mate_job_started = False
#start all the jobs that can run
while True:
running_jobs = [job for job in self.queues.get_jobs([{'has_resources':True}])]
end_times = []
now = self.get_current_time_sec()
for job in running_jobs:
end_time = max(float(job.starttime) + 60 * float(job.walltime), now + 5*60)
end_times.append([job.location, end_time])
active_jobs = [job for job in self.queues.get_jobs([{'is_runnable':True}])] #waiting jobs
active_jobs.sort(self.utilitycmp)
job_location_args = []
for job in active_jobs:
if not job.jobid == _jobid and self.mate_job_dict.get(job.jobid, 0) > 0:
#if a job other than given job (_jobid) has mate, skip it.
continue
job_location_args.append({'jobid': str(job.jobid),
'nodes': job.nodes,
'queue': job.queue,
'forbidden': [],
'utility_score': job.score,
'walltime': job.walltime,
'walltime_p': job.walltime_p, #*AdjEst*
'attrs': job.attrs,
} )
if len(job_location_args) == 0:
break
#print "queue order=", [item['jobid'] for item in job_location_args]
best_partition_dict = self.find_job_location(job_location_args, end_times)
if best_partition_dict:
#print "best_partition_dict=", best_partition_dict
for canrun_jobid in best_partition_dict:
nodelist = best_partition_dict[canrun_jobid]
if str(_jobid) == canrun_jobid:
mate_job_started = True
self.start_job([{'tag':"job", 'jobid':int(canrun_jobid)}], {'location':nodelist})
#print "bqsim.try_to_run_mate, start job jobid ", canrun_jobid
else:
break
return mate_job_started
try_to_run_mate_job = exposed(try_to_run_mate_job)
def run_holding_job(self, specs):
'''start holding job'''
for spec in specs:
jobid = spec.get('jobid')
nodelist = self.job_hold_dict.get(jobid, None)
if nodelist == None:
#print "cannot find holding resources"
return
#print "start holding job %s on location %s" % (spec['jobid'], nodelist)
self.start_job([spec], {'location':nodelist})
del self.job_hold_dict[jobid]
run_holding_job = exposed(run_holding_job)
#coscheduling stuff
def get_mate_job_status(self, jobid):
'''return mate job status, remote function, invoked by remote component'''
ret_dict = {'jobid':jobid}
ret_dict['status'] = self.get_coschedule_status(jobid)
return ret_dict
get_mate_job_status = exposed(get_mate_job_status)
def get_mate_jobs_status_local(self, remote_jobid):
'''return mate job status, invoked by local functions'''
status_dict = {}
try:
status_dict = ComponentProxy(REMOTE_QUEUE_MANAGER).get_mate_job_status(remote_jobid)
except:
self.logger.error("failed to connect to remote queue-manager component!")
status_dict = {'status':'notconnected'}
self.dbglog.LogMessage("failed to connect to remote queue-manager component!")
return status_dict
def test_can_run(self, jobid):
'''test whether a job can start immediately, specifically in following cases:
1. highest utility score and resource is available
2. not with top priority but can start in non-drained partition when top-priority job is draining
3. can be backfilled
'''
return False
def get_coschedule_status(self, jobid):
'''return job status regarding coscheduling,
input: jobid
output: listed as follows:
1. "queuing"
2. "holding"
3. "unsubmitted"
4. "running"
5. "ended"
'''
ret_status = "unknown"
job = self.get_live_job_by_id(jobid)
if job: #queuing or running
has_resources = job.has_resources
is_runnable = job.is_runnable
if is_runnable and not has_resources:
ret_status = "queuing"
if not is_runnable and has_resources:
ret_status = "running"
if not is_runnable and not has_resources:
ret_status = "holding"
else: #unsubmitted or ended
if self.unsubmitted_job_spec_dict.has_key(str(jobid)):
ret_status = "unsubmitted"
else:
ret_status = "unknown" #ended or no such job
del self.mate_job_dict[jobid]
return ret_status
#display stuff
def print_screen(self, cur_event=""):
'''print screen, show number of waiting jobs, running jobs, busy_nodes%'''
print "Cluster"
current_datetime = self.event_manager.get_current_date_time()
print "%s %s" % (current_datetime, cur_event)
print "number of waiting jobs: ", self.num_waiting
waiting_job_bar = REDS
for i in range(self.num_waiting):
waiting_job_bar += "*"
waiting_job_bar += ENDC
print waiting_job_bar
holding_jobs = len(self.job_hold_dict.keys())
holden_nodes = 0
for nodelist in self.job_hold_dict.values():
nodes = len(nodelist)
holden_nodes += nodes
print "number of running jobs: ", self.num_running
running_job_bar = BLUES
for i in range(self.num_running):
running_job_bar += "+"
running_job_bar += ENDC
print running_job_bar
print "number of holding jobs: ", holding_jobs
print "number of holden nodes: ", holden_nodes
print "number of busy nodes: ", self.num_busy
print "system utilization: ", float(self.num_busy) / self.total_nodes
busy_node_bar = GREENS
i = 0
while i < self.num_busy:
busy_node_bar += "x"
i += 1
j = 0
busy_node_bar += ENDC
busy_node_bar += YELLOWS
while j < holden_nodes:
busy_node_bar += '+'
j += 1
i += 1
busy_node_bar += ENDC
for k in range(i, self.total_nodes):
busy_node_bar += "-"
busy_node_bar += REDS
busy_node_bar += "|"
busy_node_bar += ENDC
print busy_node_bar
print "completed jobs/total jobs: %s/%s" % (self.num_end, self.total_job)
progress = 100 * self.num_end / self.total_job
progress_bar = ""
i = 0
while i < progress:
progress_bar += "="
i += 1
for j in range(i, 100):
progress_bar += "-"
progress_bar += "|"
print progress_bar
#print "waiting jobs: ", [(job.jobid, job.nodes) for job in self.queues.get_jobs([{'is_runnable':True}])]
#print "holding jobs: ", self.job_hold_dict.keys()
if self.sleep_interval:
time.sleep(self.sleep_interval)
def post_simulation_handling(self):
'''post screen after simulation completes'''
#print self.first_yield_hold_time_dict
pass
post_simulation_handling = exposed(post_simulation_handling)
