class Qsimulator(Simulator):
'''Cobalt Queue Simulator'''
implementation = "qsim"
name = "queue-manager"
alias = Simulator.name
def __init__(self, *args, **kwargs):
print "kwargs= ", kwargs
#initialize partitions
Simulator.__init__(self, *args, **kwargs)
partnames = self._partitions.keys()
self.init_partition(partnames)
self.part_size_list = []
for part in self.partitions.itervalues():
if int(part.size) not in self.part_size_list:
self.part_size_list.append(int(part.size))
self.part_size_list.sort()
#get command line parameters
self.FAILURE_FREE = True
self.FRACTION = kwargs.get("fraction", 1)
self.workload_file = kwargs.get("workload")
self.output_log = kwargs.get("outputlog")
self.failure_log = kwargs.get('failurelog')
self.weibull = kwargs.get('weibull')
if self.weibull:
self.SCALE = float(kwargs.get('scale'))
if self.SCALE == 0:
self.SCALE = default_SCALE
self.SHAPE = float(kwargs.get('shape'))
if self.SHAPE == 0:
self.SHAPE = default_SHAPE
self.fault_aware = kwargs.get('faultaware')
self.SENSITIVITY = default_SENSITIVITY
self.SPECIFICITY = default_SPECIFICITY
if self.fault_aware:
self.SENSITIVITY = float(kwargs.get('sensitivity', default_SENSITIVITY))
self.SPECIFICITY = float(kwargs.get('specificity', defalt_SPECIFICITY))
if self.failure_log or self.weibull:
self.FAILURE_FREE = False
#initialize time stamps and job queues
#time stamp format: ('EVENT', 'time_stamp_date', time_stamp_second, {'job_id':str(jobid), 'location':[partition1, partition2,...]})
self.time_stamps = [('I', '0', 0, {})]
self.cur_time_index = 0
self.queues = SimQueueDict(policy=kwargs['policy'])
self.init_queues()
self.visible_jobs = []
#initialize failures
self.failure_dict = {}
if not self.FAILURE_FREE:
if self.failure_log:
#if specified failure log, use log trace failure
self.inject_failures()
elif self.weibull:
#else MAKE failures by Weibull distribution
self.make_failures()
#initialize PBS-style logger
self.pbslog = PBSlogger(self.output_log)
#initialize debug logger
self.dbglog = PBSlogger(self.output_log+"-debug")
#finish tag
self.finished = False
#tag for controlling time stamp increment
self.increment_tag = True
#register local alias "system" for this component
local_components["system"] = self
print "Simulation starts:"
def register_alias(self):
'''register alternate name for the Qsimulator, by registering in slp
with another name for the same location. in this case 'system' is the
alternate name'''
try:
slp = Cobalt.Proxy.ComponentProxy("service-location", defer=False)
except ComponentLookupError:
print >> sys.stderr, "unable to find service-location"
qsim_quit()
svc_location = slp.locate(self.name)
if svc_location:
slp.register(self.alias, svc_location)
register_alias = automatic(register_alias, 30)
def is_finished(self):
return self.finished
is_finished = exposed(is_finished)
def init_partition(self, namelist):
'''add all paritions and apply activate and enable'''
func = self.add_partitions
args = ([{'tag':'partition', 'name':partname, 'size':"*",
'functional':False, 'scheduled':False, 'queue':"*",
'deps':[]} for partname in namelist],)
apply(func, args)
func = self.set_partitions
args = ([{'tag':'partition', 'name':partname} for partname in namelist],
{'scheduled':True, 'functional': True})
apply(func, args)
def get_current_time_event(self):
return self.time_stamps[self.cur_time_index][0]
def get_current_time(self):
'''get current time in date format'''
return self.time_stamps[self.cur_time_index][1]
def get_current_time_sec(self):
return self.time_stamps[self.cur_time_index][2]
get_current_time_sec = exposed(get_current_time_sec)
def get_current_time_job(self):
ret = None
if self.time_stamps[self.cur_time_index][3].has_key('jobid'):
ret = self.time_stamps[self.cur_time_index][3]['jobid']
return ret
def get_current_time_partition(self):
if self.get_current_time_event() in set(["R","S"]):
return self.time_stamps[self.cur_time_index][3]['location']
else:
return None
def get_current_time_stamp(self):
'''get current time stamp index'''
return self.cur_time_index
get_current_time_stamp = exposed(get_current_time_stamp)
def get_current_time_stamp_tuple(self):
return self.time_stamps[self.cur_time_index]
def time_increment(self):
'''the current time stamp increments by 1'''
if self.cur_time_index < len(self.time_stamps) - 1:
self.cur_time_index += 1
print " "
print str(self.get_current_time()) + \
" Time stamp is incremented by 1, current time stamp: " + \
str(self.cur_time_index)
else:
print str(self.get_current_time()) +\
" Reached maximum time stamp: %s, simulating finished! " \
% (str(self.cur_time_index))
self.finished = True
self.pbslog.closeLog()
qsim_quit() #simulation completed, exit!!!
return self.cur_time_index
def insert_time_stamp(self, new_time_date, event, info):
'''insert time stamps in the same order'''
if event not in SET_event:
print "invalid event type,", event
return
new_time_sec = date_to_sec(new_time_date)
new_time_tuple = (event, new_time_date, new_time_sec, info)
pos = len(self.time_stamps)
while new_time_sec < self.time_stamps[pos-1][2]:
pos = pos - 1
self.time_stamps.insert(pos, new_time_tuple)
#print "insert time stamp ", new_time_tuple, " at pos ", pos
return pos
def init_queues(self):
'''parses the work load log file, initializes queues and sorted time
stamp list'''
raw_jobs = parse_work_load(self.workload_file)
specs = []
tag = 0
for key in raw_jobs:
spec = {'valid':True}
tmp = raw_jobs[key]
spec['jobid'] = tmp.get('jobid')
spec['queue'] = tmp.get('queue')
#convert submittime from "%m/%d/%Y %H:%M:%S" to Unix time sec
format_sub_time = tmp.get('submittime')
if format_sub_time:
spec['submittime'] = date_to_sec(format_sub_time)
spec['first_subtime'] = spec['submittime'] #set the first submit time
else:
spec['valid'] = False
#convert walltime from 'hh:mm:ss' to float of minutes
format_walltime = tmp.get('Resource_List.walltime')
spec['walltime'] = 0
if format_walltime:
segs = format_walltime.split(':')
spec['walltime'] = str(int(segs[0])*60 + int(segs[1]))
else: #invalid job entry, discard
spec['valid'] = False
if tmp.get('start') and tmp.get('end'):
act_run_time = float(tmp.get('end')) - float(tmp.get('start'))
spec['runtime'] = str(round(act_run_time, 1))
if IDEALWALLTIME:
wtime = (round(act_run_time / 60, 2) + float(spec['walltime']))/2
#wtime = act_run_time / 60
spec['walltime'] = str(round(wtime, 2))
else:
spec['valid'] = False
if tmp.get('Resource_List.nodect'):
spec['nodes'] = tmp.get('Resource_List.nodect')
else: #invalid job entry, discard
spec['valid'] = False
if tmp.get('user'):
spec['user'] = tmp.get('user')
if tmp.get('project'):
spec['project'] = tmp.get('project')
spec['state'] = 'invisible'
spec['start_time'] = '0'
spec['end_time'] = '0'
#add the job spec to the spec list
if spec['valid'] == True:
specs.append(spec)
#adjust workload density
if FRACTION != 1:
tune_workload(specs, FRACTION)
print "workload adjusted: last submit job=", specs[len(specs)-1].get('submittime')
print "Initializing jobs and time stamps list, wait one moment... ..."
for spec in specs:
format_sub_time = sec_to_date(spec['submittime'])
if not self.time_stamps.__contains__(format_sub_time):
self.insert_time_stamp(format_sub_time, 'Q', {'jobid':str(spec['jobid'])})
print "total job number:", len(specs)
self.add_jobs(specs)
return 0
def log_job_event(self, eventtype, timestamp, spec):
'''log job events(Queue,Start,End) to PBS-style log'''
def len2 (_input):
_input = str(_input)
if len(_input) == 1:
return "0" + _input
else:
return _input
if eventtype == 'Q': #submitted(queued) for the first time
message = "%s;Q;%d;queue=%s" % (timestamp, spec['jobid'], spec['queue'])
elif eventtype == 'R': #resume running after failure recovery
message = "%s;R;%s" % (timestamp, ":".join(spec['location']))
else:
wall_time = spec['walltime']
walltime_minutes = len2(int(float(wall_time)) % 60)
walltime_hours = len2(int(float(wall_time)) // 60)
log_walltime = "%s:%s:00" % (walltime_hours, walltime_minutes)
if eventtype == 'S': #start running
message = "%s;S;%d;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s start=%s exec_host=%s" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'],
spec['nodes'], log_walltime, spec['start_time'], ":".join(spec['location']))
elif eventtype == 'E': #end
message = "%s;E;%d;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s start=%s end=%f exec_host=%s runtime=%s" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'], spec['nodes'], log_walltime, spec['start_time'],
round(float(spec['end_time']), 1), ":".join(spec['location']),
spec['runtime'])
elif eventtype == 'F': #failure
frag_runtime = round(float(spec['failure_time']) - float(spec['start_time']), 1) #running time before failure(after the latest start)
message = "%s;F;%d;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s exec_host=%s start=%s frag_runtime=%s complete=%f" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'],
spec['nodes'], log_walltime, ":".join(spec['location']), spec['start_time'],
frag_runtime, round(frag_runtime / float(spec['runtime']), 2)
)
elif eventtype == 'P': #pending
message = "%s;P;%d;queue=%s qtime=%s Resource_List.nodect=%s Resource_List.walltime=%s exec_host=%s start=%s" % \
(timestamp, spec['jobid'], spec['queue'], spec['submittime'],
spec['nodes'], log_walltime, ":".join(spec['location']), spec['start_time'],
)
else:
print "invalid event type, type=", type
return
self.pbslog.LogMessage(message)
def get_new_states(self, jobspec):
'''return the new state updates of a specific job at specific time
stamp, including invisible->queued, running->ended'''
updates = {}
curstate = jobspec['state']
newstate = curstate
job_id = jobspec['jobid']
cur_event = self.get_current_time_event()
#handle job submssion event
if cur_event == 'Q' and curstate == "invisible":
newstate = "queued"
updates['is_runnable'] = True
updates['is_visible'] = True
self.log_job_event('Q', self.get_current_time(), jobspec)
#handle job completion event
elif cur_event == 'E' and curstate == "running":
newstate = "ended"
updates['is_runnable'] = False
updates['has_resources'] = False
updates['is_visible'] = False
#release partition immediately
partitions = jobspec['location']
for partition in partitions:
self.release_partition(partition)
self.queues.del_jobs([{'jobid':job_id}])
#write to output log
if jobspec['end_time']:
end = float(jobspec['end_time'])
else:
end = 0
end_datetime = sec_to_date(end)
self.log_job_event('E', end_datetime, jobspec)
#handle job failure event
elif cur_event == 'F' and curstate == "running":
print "entered failure handling"
#release partition
partitions = jobspec['location']
for partition in partitions:
print "partition %s start repairing" % (partition)
self.start_repair_partition(partition)
#write to output log
if jobspec['failure_time']:
fail = float(jobspec['failure_time'])
else:
fail = 0
failure_datetime = sec_to_date(fail)
self.log_job_event('F', failure_datetime, jobspec)
print self.get_current_time(), " job %d failed at %s!!" % (job_id, ":".join(jobspec['location']))
rec_updates = self.recovery_mgr(jobspec)
if not rec_updates == {}:
updates.update(rec_updates)
updates['has_resources'] = False
if updates.has_key('state'):
newstate = updates['state']
if CHECKPOINT:
print "enter checkpoint handling****"
#runtime before failed after latest start
frag_runtime = float(jobspec['failure_time']) - float(jobspec['start_time'])
updates['remain_time'] = jobspec['remain_time'] - frag_runtime
else:#other event
pass
if updates and not curstate == newstate:
print self.get_current_time(), "state changed, job", job_id, \
":", curstate, "->", newstate
updates['state'] = newstate
return updates
def update_job_states(self, specs, updates):
'''update the state of the jobs associated to the current time stamp'''
def _update_job_states(job, newattr):
'''callback function to update job states'''
temp = job.to_rx()
newattr = self.get_new_states(temp)
if newattr:
temp.update(newattr)
job.update(newattr)
ids_str = self.get_current_time_job()
ids = ids_str.split(':')
cur_event = self.get_current_time_event()
for id in ids:
for spec in specs:
spec['jobid'] = int(id)
ret_jobs = self.queues.get_jobs(specs, _update_job_states, updates)
if cur_event == "Q":
self.visible_jobs.extend(ret_jobs)
elif cur_event=="E":
self.visible_jobs = [j for j in self.visible_jobs if j not in ret_jobs]
return 0
def run_job_updates(self, jobspec, newattr):
''' return the state updates (including state queued -> running,
setting the start_time, end_time)'''
updates = {}
#print "enter run_job_updates, jobspec=", jobspec
start = self.get_current_time_sec()
updates['start_time'] = start
updates['starttime'] = start
updates['state'] = 'running'
updates['system_state'] = 'running'
updates['is_runnable'] = False
updates['has_resources'] = True
print self.get_current_time(), "run job state change, job", jobspec['jobid'], \
":", jobspec['state'], "->", updates['state']
#determine whether the job is going to fail before completion
location = newattr['location']
duration = jobspec['remain_time']
#print "duration=", duration
nearest_failure = self.get_next_failure(location, start, duration)
if (nearest_failure):
updates['failure_time'] = date_to_sec(nearest_failure)
new_time_stamp = nearest_failure
self.insert_time_stamp(new_time_stamp, 'F', {'jobid':str(jobspec['jobid'])})
else: # will complete
end = start + duration
updates['end_time'] = end
new_time_stamp = sec_to_date(end)
#print "new_time_stamp=", new_time_stamp
self.insert_time_stamp(new_time_stamp, 'E', {'jobid':str(jobspec['jobid'])})
updates.update(newattr)
return updates
def start_job(self, specs, updates):
'''update the job state and start_time and end_time when cqadm --run
is issued to a group of jobs'''
partitions = updates['location']
for partition in partitions:
self.reserve_partition(partition)
def _start_job(job, newattr):
'''callback function to update job start/end time'''
temp = job.to_rx()
newattr = self.run_job_updates(temp, newattr)
temp.update(newattr)
job.update(newattr)
self.log_job_event('S', self.get_current_time(), temp)
return self.queues.get_jobs(specs, _start_job, updates)
def add_jobs(self, specs):
'''Add a job, currently for unit test only'''
response = self.queues.add_jobs(specs)
return response
add_jobs = exposed(query(add_jobs))
def get_jobs(self, specs):
'''get a list of jobs, each time triggers time stamp increment and job
states update'''
jobs = []
if self.increment_tag:
self.time_increment()
eventtype = self.get_current_time_event()
print "current event type====", eventtype
if eventtype == "R":
self.release_repaired_partition()
#if the repaired job associated with some pending jobs,
#returen empty list to scheduler, in order to ensure the next
#time stamp will restart the pending job other than scheduling other jobs at this time stamp
#this will avoid run multiple jobs on the same partition(once a bug, solved)
if self.get_current_time_job():
return jobs
elif eventtype == "S":
self.restart_pending_job()
return jobs
else:
self.update_job_states(specs, {})
if len(self.recovering_jobs) > 0:
self.update_recovering_jobs({})
self.increment_tag = True
jobs = self.visible_jobs
# print "running jobs=", [job.jobid for job in self.running_jobs]
# print "queueing jobs=", [job.jobid for job in self.queuing_jobs]
# print "visible jobs=", [job.jobid for job in self.visible_jobs]
# print "return jobs=", len(jobs)
return jobs
get_jobs = exposed(query(get_jobs))
def update_recovering_jobs(self, updates):
print "enter update_recovering_jobs()"
def _update_recovering_jobs(job, newattr):
'''callback function to update job states'''
temp = job.to_rx()
print "temp=", temp
newattr = self.recovery_mgr(temp)
print "update_recovering_jobs newattr=", newattr
print "temp=", temp
if newattr:
temp.update(newattr)
job.update(newattr)
ids = [job.jobid for job in self.recovering_jobs]
print "ids=", ids
ret = self.queues.get_jobs([{'tag':"job", 'state': "recovering"}], _update_recovering_jobs, updates)
return 0
def _get_queuing_jobs(self):
return [job for job in self.visible_jobs if job.is_runnable==True]
queuing_jobs = property(_get_queuing_jobs)
def _get_running_jobs(self):
return [job for job in self.visible_jobs if job.has_resources==True]
running_jobs = property(_get_running_jobs)
def _get_recovering_jobs(self):
return self.queues.get_jobs([{'jobid':"*", 'state':"recovering"}])
recovering_jobs = property(_get_recovering_jobs)
def get_visible_jobs(self):
return self.visible_jobs;
get_visible_jobs = exposed(get_visible_jobs)
def get_running_jobs(self):
return [job for job in self.visible_jobs if job.has_resources==True]
get_running_jobs = exposed(get_running_jobs)
def get_queuing_jobs(self):
return [job for job in self.visible_jobs if job.is_runnable==True]
get_queuing_jobs = exposed(get_queuing_jobs)
def _get_job_by_id(self, jobid):
jobs = self.queues.get_jobs([{'jobid':jobid}])
if len(jobs) == 1:
return jobs[0]
else:
return None
def add_queues(self, specs):
'''add queues'''
return self.queues.add_queues(specs)
add_queues = exposed(query(add_queues))
def get_queues(self, specs):
'''get queues'''
return self.queues.get_queues(specs)
get_queues = exposed(query(get_queues))
def run_jobs(self, specs, nodelist):
'''run a queued job, by updating the job state, start_time and
end_time'''
print "run job specs=", specs, " on partion", nodelist
if specs:
self.start_job(specs, {'location': nodelist})
#set tag false, enable scheduling another job at the same time
self.increment_tag = False
#print "current running jobs=", [job.jobid for job in self.running_jobs]
return self.running_jobs
run_jobs = exposed(query(run_jobs))
def get_midplanes(self, partname):
'''return a list of sub-partitions each contains 512-nodes(midplane)'''
midplane_list = []
partition = self._partitions[partname]
if partition.size == MIDPLANE_SIZE:
midplane_list.append(partname)
elif partition.size > MIDPLANE_SIZE:
children = partition.children
for part in children:
if self._partitions[part].size == MIDPLANE_SIZE:
midplane_list.append(part)
else:
parents = partition.parents
for part in parents:
if self._partitions[part].size == MIDPLANE_SIZE:
midplane_list.append(part)
return midplane_list
def get_next_failure(self, location, now, duration):
'''return the next(closest) failure moment according the partition failure list'''
if (self.FAILURE_FREE):
return None
def _find_next_failure(partname, now):
next = None
failure_list = self.failure_dict[partname]
if failure_list:
for fail_time in failure_list:
if date_to_sec(fail_time) > now:
next = fail_time
break
return next
closest_fail_sec = MAXINT
partitions = location
midplanes = set()
for partition in partitions:
tmp_midplanes = self.get_midplanes(partition)
for item in tmp_midplanes:
if item not in midplanes:
midplanes.add(item)
for midplane in midplanes:
next = _find_next_failure(midplane, now)
if (next):
next_sec = date_to_sec(next)
if next_sec < closest_fail_sec:
closest_fail_sec =next_sec
if closest_fail_sec == MAXINT:
next_failure_date = None
else:
job_end_sec = now + duration
if closest_fail_sec < job_end_sec:
next_failure_date = sec_to_date(closest_fail_sec)
else:
next_failure_date = None
#print "next_failure_date=", next_failure_date
return next_failure_date
def will_job_fail(self, mtbf, nodes, hours):
'''simulate static failure chance, [not used]'''
return False
print "mtbf=%d, nodes=%d, hours=%f" % (mtbf,nodes,hours)
failure_chance = 1 - (1 - hours * 1.0/mtbf) ** nodes
if failure_chance > 0.7 :
failure_chance = 0.7
random_num = random.random()
print "failure chance=%f, random_num=%f" % (failure_chance, random_num)
if random_num < failure_chance:
return True
else:
return False
def nodes_static(self):
'''static the node requested by each job, [not used]'''
jobs = self.queues.get_jobs([{'jobid':"*", 'queue':"*", 'nodes':"*"}])
nodesdict = {}
for job in jobs:
nodes = int(job.nodes)
nodesstr = nodes
if (nodesdict.has_key(nodesstr)):
nodesdict[nodesstr] = nodesdict[nodesstr] + 1
else:
nodesdict[nodesstr] = 1
keys = nodesdict.keys()
keys.sort()
for key in keys:
print key, ":", nodesdict[key]
def gen_failure_list(self, scale, shape, startdate, enddate):
'''generate a synthetic failure time list based on weibull distribution
and start/end date time'''
failure_moments = []
ttf_list = []
start = date_to_sec(startdate)
end = date_to_sec(enddate)
cur_failure = start
while True:
ttf = random.weibullvariate(scale,shape)
cur_failure += ttf
if cur_failure < end:
ttf_list.append(ttf)
failure_moments.append(sec_to_date(cur_failure))
else:
break
return failure_moments, ttf_list
def make_failures(self):
'''generate failure lists for each 512-nodes partition'''
ttf_dict = {}
start = self.time_stamps[1][1]
end = self.time_stamps[len(self.time_stamps)-1][1]
for partition in self._partitions.values():
if partition.size == MIDPLANE_SIZE:
fl, ttfs = self.gen_failure_list(self.SCALE, self.SHAPE, start, end)
self.failure_dict[partition.name] = fl
ttf_dict[partition.name] = ttfs
partnames = self.failure_dict.keys()
partnames.sort()
f = open(default_FAILURE_LOG, "w")
total_f = 0
mtbf = 0
for part in partnames:
f_list = self.failure_dict[part]
print part, " ", f_list
f.write("%s;%s\n" % (part, ";".join(f_list)))
total_f += len(f_list)
ttfs = ttf_dict[part]
if len(ttfs)==0:
mtbf = 0
else:
total = 0
for ttf in ttfs:
total += ttf
mtbf = total / len(ttfs)
start_sec = date_to_sec(start)
end_sec = date_to_sec(end)
f.write("Total=%d\nMTBF=%f" % (total_f, (end_sec-start_sec)/(total_f*3600)))
f.close()
def inject_failures(self):
'''parse failure trace log to make failure list for each 1-midplane partition'''
raw_job_dict = {}
partnames = set(self._partitions.keys())
flog = open(self.failure_log, "r")
self.failure_dict = {}
for line in flog:
print "line=", line
line = line.strip('\n')
parsedline = line.split(";")
print "parsedline=", parsedline
failure_list = []
part = parsedline[0]
if part in partnames:
for i in range(1, len(parsedline)):
failure_moment = parsedline[i]
if len(failure_moment) == 0:
continue
failure_list.append(failure_moment)
self.failure_dict[part] = failure_list
partnames = self.failure_dict.keys()
partnames.sort()
for part in partnames:
f_list = self.failure_dict[part]
print part, " ", f_list
def get_failure_chance(self, location, duration):
now = date_to_sec(self.get_current_time())
next_fail = self.get_next_failure(location, now, duration)
if (next_fail != None):
return self.SENSITIVITY
else:
return 1 - self.SPECIFICITY
get_failure_chance = exposed(get_failure_chance)
def recovery_mgr(self, jobspec):
"""Recovery manager, this function can be extended to support various recovery options.
at this version, the failed job is sent back to the rear of the queue. The extended code
is ready and available at private code branch(wtang)."""
updates = {}
updates = self.handle_reque_rear(jobspec)
recovery_option = jobspec['recovery_opt']
print "rec_opt=", recovery_option
#if_else structure remains room for recovery option extending
if recovery_option == 1:
#resubmit the job
#resubmit the job, the submit time changed to NOW
updates = self.handle_reque_rear(jobspec)
return updates
def handle_reque_rear(self, jobspec):
'''handle option 1 - resubmit the job to rear of waiting queue'''
updates = {}
updates['state'] = "queued"
updates['start_time'] = 0
updates['submittime'] = self.get_current_time_sec()
return updates
def start_repair_partition(self, partname):
'''partition failed, assuming get repaired MTTR seconds later'''
now = self.get_current_time_sec()
time_to_repair = now + MTTR
time_to_repair_date = sec_to_date(time_to_repair)
self.insert_time_stamp(time_to_repair_date, "R", {'location':partname})
def release_repaired_partition(self):
'''enter release_repaired_partition() partition repaired'''
partition = self.get_current_time_partition()
if partition == None:
return False
self.release_partition(partition)
print "partition %s gets repaired" % (partition)
self.log_job_event('R', self.get_current_time(), {'location':partition})
return True
def restart_pending_job(self):
'''restart jobs that pending for the nodes repair'''
partname = self.get_current_time_partition()
print "enter restart_pending_job() partname=", partname
ids_str = self.get_current_time_job()
ids = ids_str.split(':')
jobspecs = []
for id in ids:
spec = {'tag':'job', 'jobid':int(id)}
jobspecs.append(spec)
print "restart pending job ", jobspecs, " on repaired partition ", partname
self.run_jobs(jobspecs, [partname])
def possible_locations(self, job):
'''find the partitions with the size that can right accomodates the job
(returned partions are not necessarily idle)'''
locations = []
proper_partsize = 64
job_nodes = int(job['nodes'])
for psize in self.part_size_list:
if psize >= job_nodes:
proper_partsize = psize
break
for part in self.cached_partitions.itervalues():
if int(part.size) == proper_partsize:
locations.append(part)
return locations
def _find_job_location(self, args, drain_partitions=set(), backfilling=False):
jobid = args['jobid']
nodes = args['nodes']
queue = args['queue']
utility_score = args['utility_score']
walltime = args['walltime']
forbidden = args.get("forbidden", [])
required = args.get("required", [])
best_score = sys.maxint
best_partition = None
# get partitions of proper size as the candidates
candidate_partitions = self.possible_locations(args)
#exclude the partitions already drained
if drain_partitions:
candidate_partitions = [part for part in candidate_partitions if part not in drain_partitions]
now = self.get_current_time_sec()
for partition in candidate_partitions:
#skip partitions that are not "idle"
if partition.state != "idle":
continue
if backfilling:
#skip the partition with too short cutoff to backfill the job
if 60*float(walltime) > (partition.backfill_time - now):
continue
# let's check the impact on partitions that would become blocked
score = 0
for p in partition.parents:
if self.cached_partitions[p].state == "idle" and self.cached_partitions[p].scheduled:
score += 1
for ch in partition.children:
score += 0.01
if (FAULTAWARE):
Pf = 0
Pf = self.get_failure_chance(partition.name, 60*float(walltime))
score += Pf
# the lower the score, the fewer new partitions will be blocked by this selection
if score < best_score:
best_score = score
best_partition = partition
elif score == best_score:
if partition.name > best_partition.name:
best_partition = partition
if best_partition:
#print "return bestpartition=",{jobid: [best_partition.name, best_partition.state]}
return {jobid: [best_partition.name]}
def find_job_location(self, arg_list, end_times):
best_partition_dict = {}
if self.bridge_in_error:
print "bridge_in_error"
return {}
self.cached_partitions = self.partitions
# first, figure out backfilling cutoffs per partition (which we'll also
# use for picking which partition to drain)
job_end_times = {}
for item in end_times:
job_end_times[item[0][0]] = item[1]
now = self.get_current_time_sec()
for p in self.cached_partitions.itervalues():
if p.state == "idle":
p.backfill_time = now
else:
p.backfill_time = now + 5*60
p.draining = False
for p in self.cached_partitions.itervalues():
if p.name in job_end_times:
if job_end_times[p.name] > p.backfill_time:
p.backfill_time = job_end_times[p.name]
for parent_name in p.parents:
parent_partition = self.cached_partitions[parent_name]
if p.backfill_time > parent_partition.backfill_time:
parent_partition.backfill_time = p.backfill_time
for p in self.cached_partitions.itervalues():
if p.backfill_time == now:
continue
for child_name in p.children:
child_partition = self.cached_partitions[child_name]
if child_partition.backfill_time == now or child_partition.backfill_time > p.backfill_time:
child_partition.backfill_time = p.backfill_time
# first time through, try for starting jobs based on utility scores
drain_partitions = set()
# the sets draining_jobs and cannot_start are for efficiency, not correctness
draining_jobs = set()
cannot_start = set()
for idx in range(len(arg_list)):
winning_job = arg_list[idx]
for jj in range(idx, len(arg_list)):
job = arg_list[jj]
# this job isn't good enough!
if job['utility_score'] < winning_job['threshold']:
break
if job['jobid'] not in cannot_start:
partition_name = self._find_job_location(job, drain_partitions)
if partition_name:
best_partition_dict.update(partition_name)
break
cannot_start.add(job['jobid'])
# we already picked a drain location for the winning job
if winning_job['jobid'] in draining_jobs:
continue
location = self._find_drain_partition(winning_job)
if location is not None:
for p_name in location.parents:
drain_partitions.add(self.cached_partitions[p_name])
for p_name in location.children:
drain_partitions.add(self.cached_partitions[p_name])
self.cached_partitions[p_name].draining = True
drain_partitions.add(location)
#self.logger.info("job %s is draining %s" % (winning_job['jobid'], location.name))
#self.dbglog.LogMessage("job %s is draining %s" % (winning_job['jobid'], location.name))
location.draining = True
draining_jobs.add(winning_job['jobid'])
# at this time, we only want to try launching one job at a time
if best_partition_dict:
# msg = "idx=%s, jj=%s, job=%s, partition=%s" % (idx, jj, job['jobid'],best_partition_dict[job['jobid']])
#print msg
# self.dbglog.LogMessage(msg)
break
# the next time through, try to backfill, but only if we couldn't find anything to start
if not best_partition_dict:
# arg_list.sort(self._walltimecmp)
# msg = "try to backfill jobs..."
# self.dbglog.LogMessage(msg)
for args in arg_list:
partition_name = self._find_job_location(args, backfilling=True)
if partition_name:
msg = "backfilling job %s(%s)" % (args['jobid'], args['nodes'])
self.logger.info(msg)
self.dbglog.LogMessage(msg)
best_partition_dict.update(partition_name)
break
# reserve the stuff in the best_partition_dict, as those partitions are allegedly going to
# be running jobs very soon
#
# also, this is the only part of finding a job location where we need to lock anything
#self._partitions_lock.acquire()
try:
for p in self.partitions.itervalues():
# push the backfilling info from the local cache back to the real objects
p.draining = self.cached_partitions[p.name].draining
p.backfill_time = self.cached_partitions[p.name].backfill_time
for partition_list in best_partition_dict.itervalues():
part = self.partitions[partition_list[0]]
##part.reserved_until = self.get_current_time_sec() + 5*60
part.state = "starting job"
for p in part._parents:
if p.state == "idle":
p.state = "blocked by starting job"
for p in part._children:
if p.state == "idle":
p.state = "blocked by starting job"
except:
self.logger.error("error in find_job_location", exc_info=True)
#self._partitions_lock.release()
#print "best_partition_dict=", best_partition_dict
return best_partition_dict
find_job_location = locking(exposed(find_job_location))
class BGBaseSystem(Component):
"""base system class.
Methods:
add_partitions -- tell the system to manage partitions (exposed, query)
get_partitions -- retrieve partitions in the simulator (exposed, query)
del_partitions -- tell the system not to manage partitions (exposed, query)
set_partitions -- change random attributes of partitions (exposed, query)
update_relatives -- should be called when partitions are added and removed from the managed list
"""
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self._partitions = PartitionDict()
self._managed_partitions = set()
self.process_groups = BGProcessGroupDict()
self.node_card_cache = dict()
self._partitions_lock = thread.allocate_lock()
self.pending_diags = dict()
self.failed_diags = list()
self.bridge_in_error = False
self.cached_partitions = None
self.offline_partitions = []
def _get_partitions(self):
return PartitionDict([(partition.name, partition)
for partition in self._partitions.itervalues()
if partition.name in self._managed_partitions])
partitions = property(_get_partitions)
def add_partitions(self, specs, user_name=None):
self.logger.info("%s called add_partitions(%r)", user_name, specs)
specs = [{'name': spec.get("name")} for spec in specs]
self._partitions_lock.acquire()
try:
partitions = [
partition for partition in self._partitions.q_get(specs)
if partition.name not in self._managed_partitions
]
except:
partitions = []
self.logger.error("error in add_partitions", exc_info=True)
self._partitions_lock.release()
self._managed_partitions.update(
[partition.name for partition in partitions])
self.update_relatives()
return partitions
add_partition = exposed(query(add_partitions))
def get_partitions(self, specs):
"""Query partitions on simulator."""
self._partitions_lock.acquire()
try:
partitions = self.partitions.q_get(specs)
except:
partitions = []
self.logger.error("error in get_partitions", exc_info=True)
self._partitions_lock.release()
return partitions
get_partitions = exposed(query(get_partitions))
def verify_locations(self, location_list):
"""Providing a system agnostic interface for making sure a 'location string' is valid"""
parts = self.get_partitions([{'name': l} for l in location_list])
return [p.name for p in parts]
verify_locations = exposed(verify_locations)
def del_partitions(self, specs, user_name=None):
"""Remove partitions from the list of managed partitions"""
self.logger.info("%s called del_partitions(%r)", user_name, specs)
self._partitions_lock.acquire()
try:
partitions = [
partition for partition in self._partitions.q_get(specs)
if partition.name in self._managed_partitions
]
except:
partitions = []
self.logger.error("error in del_partitions", exc_info=True)
self._partitions_lock.release()
self._managed_partitions -= set(
[partition.name for partition in partitions])
self.update_relatives()
return partitions
del_partitions = exposed(query(del_partitions))
def set_partitions(self, specs, updates, user_name=None):
"""Update random attributes on matching partitions"""
def _set_partitions(part, newattr):
self.logger.info("%s updating partition %s: %r", user_name,
part.name, newattr)
part.update(newattr)
self._partitions_lock.acquire()
try:
partitions = self._partitions.q_get(specs, _set_partitions,
updates)
except:
partitions = []
self.logger.error("error in set_partitions", exc_info=True)
self._partitions_lock.release()
return partitions
set_partitions = exposed(query(set_partitions))
def update_relatives(self):
"""Call this method after changing the contents of self._managed_partitions"""
for p_name in self._managed_partitions:
self._partitions[p_name]._parents = set()
self._partitions[p_name]._children = set()
for p in self._partitions.itervalues():
p._all_children = set()
for p_name in self._managed_partitions:
p = self._partitions[p_name]
#Check the wiring dependencies of our children.
#Touching those would be bad. --PMR
# new_parents = []
# for par in p._parents:
# for dep_name in par._wiring_conflicts:
# if dep_name in self._managed_partitions:
# new_parents.append(self._partitions[dep_name])
# p._parents.union(set(new_parents))
#
# for child in p._children:
# for dep_name in child._wiring_conflicts:
# if dep_name in self._managed_partitions:
# p._parents.add(self._partitions[dep_name])
# toss the wiring dependencies in with the parents
for dep_name in p._wiring_conflicts:
if dep_name in self._managed_partitions:
p._parents.add(self._partitions[dep_name])
for other in self._partitions.itervalues():
if p.name == other.name:
continue
p_set = set(p.node_cards)
other_set = set(other.node_cards)
if other.name in self._managed_partitions:
# if p is a subset of other, then p is a child; add other to p's list of managed parent partitions, and p to
# other's list of managed child partitions
if p_set.intersection(other_set) == p_set:
p._parents.add(other)
other._children.add(p)
# if p contains other, then p is a parent; add other to p's list of managed child partitions and p to other's
# list of managed parent partitions
elif p_set.union(other_set) == p_set:
p._children.add(other)
other._parents.add(p)
# if p contains other, then p is a parent; add other to p's list of all child partitions
if p_set.union(other_set) == p_set:
p._all_children.add(other)
#Let's get the wiring conflicts for direct childeren as well,
#we shouldn't be able to run on these either. --PMR
for p_name in self._managed_partitions:
#if p_name != "ANL-R10-R47-32768":
# continue
p = self._partitions[p_name]
for child in p._children:
#print "Child %s:" % child.name
for dep_name in child._wiring_conflicts:
#print "Conflict: %s" % dep_name
if dep_name in self._managed_partitions:
p._parents.add(self._partitions[dep_name])
#we shouldn't be scheduling on the parents of our children either
for par in child._parents:
#print "Parent: %s" % par.name
if ((par.name != p_name)
and (par.name in self._managed_partitions)):
p._parents.add(self._partitions[par.name])
#for p_name in self._managed_partitions:
#if p_name != "ANL-R10-R47-32768":
# continue
# print str(p_name) + ":"
# print "Parents: " + str(":".join([par.name for par in self._partitions[p_name]._parents]))
# print "Children:" + str(":".join([child.name for child in self._partitions[p_name]._children]))
# print "Conflicts:" + str(":".join([con for con in self._partitions[p_name]._wiring_conflicts]))
def validate_job(self, spec):
"""validate a job for submission
Arguments:
spec -- job specification dictionary
"""
# spec has {nodes, walltime*, procs, mode, kernel}
max_nodes = max([int(p.size) for p in self._partitions.values()])
try:
sys_type = CP.get('bgsystem', 'bgtype')
except:
sys_type = 'bgl'
if sys_type == 'bgp':
job_types = ['smp', 'dual', 'vn', 'script']
else:
job_types = ['co', 'vn', 'script']
try:
spec['nodecount'] = int(spec['nodecount'])
except:
raise JobValidationError("Non-integer node count")
if not 0 < spec['nodecount'] <= max_nodes:
raise JobValidationError("Node count out of realistic range")
if float(spec['time']) < 5:
raise JobValidationError("Walltime less than minimum")
if not spec['mode']:
if sys_type == 'bgp':
spec['mode'] = 'smp'
else:
spec['mode'] = 'co'
if spec['mode'] not in job_types:
raise JobValidationError("Invalid mode")
if spec['attrs'].has_key("location"):
p_name = spec['attrs']['location']
if not self.partitions.has_key(p_name):
raise JobValidationError("Partition %s not found" % p_name)
if not spec['proccount']:
if spec.get('mode', 'co') == 'vn':
if sys_type == 'bgl':
spec['proccount'] = str(2 * int(spec['nodecount']))
elif sys_type == 'bgp':
spec['proccount'] = str(4 * int(spec['nodecount']))
else:
self.logger.error("Unknown bgtype %s" % (sys_type))
elif spec.get('mode', 'co') == 'dual':
spec['proccount'] = 2 * int(spec['nodecount'])
else:
spec['proccount'] = spec['nodecount']
else:
try:
spec['proccount'] = int(spec['proccount'])
except:
JobValidationError("non-integer proccount")
if spec['proccount'] < 1:
raise JobValidationError("negative proccount")
if spec['proccount'] > spec['nodecount']:
if spec['mode'] not in ['vn', 'dual']:
raise JobValidationError("proccount too large")
if sys_type == 'bgl' and (spec['proccount'] >
(2 * spec['nodecount'])):
raise JobValidationError("proccount too large")
elif sys_type == ' bgp' and (spec['proccount'] >
(4 * spec['nodecount'])):
raise JobValidationError("proccount too large")
# need to handle kernel
return spec
validate_job = exposed(validate_job)
def run_diags(self, partition_list, test_name, user_name=None):
self.logger.info("%s running diags %s on partitions %s", user_name,
test_name, partition_list)
def size_cmp(left, right):
return -cmp(left.size, right.size)
def _find_covering(partition):
kids = [self._partitions[c_name] for c_name in partition.children]
kids.sort(size_cmp)
n = len(kids)
part_node_cards = set(partition.node_cards)
# generate the power set, but try to use the big partitions first (hence the sort above)
for i in xrange(1, 2**n + 1):
test_cover = [kids[j] for j in range(n) if i & 2**j]
test_node_cards = set()
for t in test_cover:
test_node_cards.update(t.node_cards)
if test_node_cards.issubset(
part_node_cards) and test_node_cards.issuperset(
part_node_cards):
return test_cover
return []
def _run_diags(partition):
covering = _find_covering(partition)
for child in covering:
self.pending_diags[child] = test_name
return [child.name for child in covering]
results = []
for partition_name in partition_list:
p = self._partitions[partition_name]
results.append(_run_diags(p))
return results
run_diags = exposed(run_diags)
def launch_diags(self, partition, test_name):
'''override this method in derived classes!'''
pass
def finish_diags(self, partition, test_name, exit_value):
'''call this method somewhere in your derived class where you deal with the exit values of diags'''
if exit_value == 0:
for dead in self.failed_diags[:]:
if dead == partition.name or dead in partition.children:
self.failed_diags.remove(dead)
self.logger.info("removing %s from failed_diags list" %
dead)
else:
if partition.children:
self.run_diags([partition.name], test_name)
else:
self.failed_diags.append(partition.name)
self.logger.info("adding %s to failed_diags list" %
partition.name)
def handle_pending_diags(self):
for p in self.pending_diags.keys():
if p.state in [
"idle", "blocked by pending diags", "failed diags",
"blocked by failed diags"
]:
self.logger.info("launching diagnostics on %s" % p.name)
self.launch_diags(p, self.pending_diags[p])
del self.pending_diags[p]
handle_pending_diags = automatic(handle_pending_diags)
def fail_partitions(self, specs, user_name=None):
self.logger.info("%s failing partition %s", user_name, specs)
parts = self.get_partitions(specs)
if not parts:
ret = "no matching partitions found\n"
else:
ret = ""
for p in parts:
if self.failed_diags.count(p.name) == 0:
ret += "failing %s\n" % p.name
self.failed_diags.append(p.name)
else:
ret += "%s is already marked as failing\n" % p.name
return ret
fail_partitions = exposed(fail_partitions)
def unfail_partitions(self, specs, user_name=None):
self.logger.info("%s unfailing partition %s", user_name, specs)
parts = self.get_partitions(specs)
if not parts:
ret = "no matching partitions found\n"
else:
ret = ""
for p in self.get_partitions(specs):
if self.failed_diags.count(p.name):
ret += "unfailing %s\n" % p.name
self.failed_diags.remove(p.name)
else:
ret += "%s is not currently failing\n" % p.name
return ret
unfail_partitions = exposed(unfail_partitions)
def _find_job_location(self,
args,
drain_partitions=set(),
backfilling=False):
jobid = args['jobid']
nodes = args['nodes']
queue = args['queue']
utility_score = args['utility_score']
walltime = args['walltime']
walltime_p = args.get('walltime_p', walltime) #*AdjEst*
forbidden = args.get("forbidden", [])
required = args.get("required", [])
if walltime_prediction_enabled: # *Adj_Est*
runtime_estimate = float(walltime_p)
else:
runtime_estimate = float(walltime)
best_score = sys.maxint
best_partition = None
available_partitions = set()
requested_location = None
if args['attrs'].has_key("location"):
requested_location = args['attrs']['location']
if required:
# whittle down the list of required partitions to the ones of the proper size
# this is a lot like the stuff in _build_locations_cache, but unfortunately,
# reservation queues aren't assigned like real queues, so that code doesn't find
# these
for p_name in required:
available_partitions.add(self.cached_partitions[p_name])
available_partitions.update(
self.cached_partitions[p_name]._children)
possible = set()
for p in available_partitions:
possible.add(p.size)
desired_size = 0
job_nodes = int(nodes)
for psize in sorted(possible):
if psize >= job_nodes:
desired_size = psize
break
for p in available_partitions.copy():
if p.size != desired_size:
available_partitions.remove(p)
elif p.name in self._not_functional_set:
available_partitions.remove(p)
elif requested_location and p.name != requested_location:
available_partitions.remove(p)
else:
for p in self.possible_locations(nodes, queue):
skip = False
for bad_name in forbidden:
if p.name == bad_name or bad_name in p.children or bad_name in p.parents:
skip = True
break
if not skip:
if (not requested_location) or (p.name
== requested_location):
available_partitions.add(p)
available_partitions -= drain_partitions
now = time.time()
for partition in available_partitions:
# if the job needs more time than the partition currently has available, look elsewhere
if backfilling:
if partition.reserved_by:
#if the partition is reserved, we don't use predicted walltime to backfill
runtime_estimate = float(walltime)
if 60 * runtime_estimate > (partition.backfill_time -
now): # *Adj_Est*
continue
if 60 * float(walltime) > (partition.backfill_time - now):
continue
if partition.state == "idle":
# let's check the impact on partitions that would become blocked
score = 0
for p in partition.parents:
if self.cached_partitions[
p].state == "idle" and self.cached_partitions[
p].scheduled:
score += 1
# the lower the score, the fewer new partitions will be blocked by this selection
if score < best_score:
best_score = score
best_partition = partition
if best_partition:
return {jobid: [best_partition.name]}
def _find_drain_partition(self, job):
# if the user requested a particular partition, we only try to drain that one
if job['attrs'].has_key("location"):
target_name = job['attrs']['location']
return self.cached_partitions.get(target_name, None)
drain_partition = None
locations = self.possible_locations(job['nodes'], job['queue'])
for p in locations:
if not drain_partition:
drain_partition = p
else:
if p.backfill_time < drain_partition.backfill_time:
drain_partition = p
if drain_partition:
# don't try to drain for an entire weekend
hours = (drain_partition.backfill_time - time.time()) / 3600.0
if hours > max_drain_hours:
drain_partition = None
return drain_partition
def possible_locations(self, job_nodes, q_name):
desired_size = 0
job_nodes = int(job_nodes)
if self._defined_sizes.has_key(q_name):
for psize in self._defined_sizes[q_name]:
if psize >= job_nodes:
desired_size = psize
break
if self._locations_cache.has_key(q_name):
return self._locations_cache[q_name].get(desired_size, [])
else:
return []
# this function builds three things, namely a pair of dictionaries keyed by queue names, and a set of
# partition names which are not functional
#
# self._defined_sizes maps queue names to an ordered list of partition sizes available in that queue
# for all schedulable partitions (even if currently offline and not functional)
# self._locations_cache maps queue names to dictionaries which map partition sizes to partition objects;
# this structure will only contain partitions which are fully online, so we don't try to drain a
# broken partition
# self._not_functional_set contains names of partitions which are not functional (either themselves, or
# a parent or child)
def _build_locations_cache(self):
per_queue = {}
defined_sizes = {}
not_functional_set = set()
for target_partition in self.cached_partitions.itervalues():
usable = True
if target_partition.name in self.offline_partitions:
usable = False
else:
for part in self.cached_partitions.itervalues():
if not part.functional:
not_functional_set.add(part.name)
if target_partition.name in part.children or target_partition.name in part.parents:
usable = False
not_functional_set.add(target_partition.name)
break
for queue_name in target_partition.queue.split(":"):
if not per_queue.has_key(queue_name):
per_queue[queue_name] = {}
if not defined_sizes.has_key(queue_name):
defined_sizes[queue_name] = set()
if target_partition.scheduled:
defined_sizes[queue_name].add(target_partition.size)
if target_partition.scheduled and target_partition.functional and usable:
if not per_queue[queue_name].has_key(
target_partition.size):
per_queue[queue_name][target_partition.size] = []
per_queue[queue_name][target_partition.size].append(
target_partition)
for q_name in defined_sizes:
defined_sizes[q_name] = sorted(defined_sizes[q_name])
self._defined_sizes = defined_sizes
self._locations_cache = per_queue
self._not_functional_set = not_functional_set
def find_job_location(self, arg_list, end_times):
best_partition_dict = {}
if self.bridge_in_error:
return {}
self._partitions_lock.acquire()
try:
self.cached_partitions = copy.deepcopy(self.partitions)
except:
self.logger.error("error in copy.deepcopy", exc_info=True)
return {}
finally:
self._partitions_lock.release()
# build the cached_partitions structure first
self._build_locations_cache()
# first, figure out backfilling cutoffs per partition (which we'll also use for picking which partition to drain)
job_end_times = {}
for item in end_times:
job_end_times[item[0][0]] = item[1]
now = time.time()
for p in self.cached_partitions.itervalues():
if p.state == "idle":
p.backfill_time = now
else:
p.backfill_time = now + 5 * 60
p.draining = False
for p in self.cached_partitions.itervalues():
if p.name in job_end_times:
if job_end_times[p.name] > p.backfill_time:
p.backfill_time = job_end_times[p.name]
for parent_name in p.parents:
parent_partition = self.cached_partitions[parent_name]
if p.backfill_time > parent_partition.backfill_time:
parent_partition.backfill_time = p.backfill_time
for p in self.cached_partitions.itervalues():
if p.backfill_time == now:
continue
for child_name in p.children:
child_partition = self.cached_partitions[child_name]
if child_partition.backfill_time == now or child_partition.backfill_time > p.backfill_time:
child_partition.backfill_time = p.backfill_time
# first time through, try for starting jobs based on utility scores
drain_partitions = set()
for job in arg_list:
partition_name = self._find_job_location(job, drain_partitions)
if partition_name:
best_partition_dict.update(partition_name)
break
location = self._find_drain_partition(job)
if location is not None:
for p_name in location.parents:
drain_partitions.add(self.cached_partitions[p_name])
for p_name in location.children:
drain_partitions.add(self.cached_partitions[p_name])
self.cached_partitions[p_name].draining = True
drain_partitions.add(location)
#self.logger.info("job %s is draining %s" % (winning_job['jobid'], location.name))
location.draining = True
# the next time through, try to backfill, but only if we couldn't find anything to start
if not best_partition_dict:
# arg_list.sort(self._walltimecmp)
for args in arg_list:
partition_name = self._find_job_location(args,
backfilling=True)
if partition_name:
self.logger.info("backfilling job %s" % args['jobid'])
best_partition_dict.update(partition_name)
break
# reserve the stuff in the best_partition_dict, as those partitions are allegedly going to
# be running jobs very soon
#
# also, this is the only part of finding a job location where we need to lock anything
self._partitions_lock.acquire()
try:
for p in self.partitions.itervalues():
# push the backfilling info from the local cache back to the real objects
p.draining = self.cached_partitions[p.name].draining
p.backfill_time = self.cached_partitions[p.name].backfill_time
for jobid, partition_list in best_partition_dict.iteritems():
part = self.partitions[partition_list[0]]
# FIXME: use reserve_resources_until() here? --brt
part.used_by = int(jobid)
part.reserved_until = time.time() + 5 * 60
part.state = "allocated"
for p in part._parents:
if p.state == "idle":
p.state = "blocked (%s)" % (part.name, )
for p in part._children:
if p.state == "idle":
p.state = "blocked (%s)" % (part.name, )
except:
self.logger.error("error in find_job_location", exc_info=True)
self._partitions_lock.release()
return best_partition_dict
find_job_location = locking(exposed(find_job_location))
def _walltimecmp(self, dict1, dict2):
return -cmp(float(dict1['walltime']), float(dict2['walltime']))
def find_queue_equivalence_classes(self, reservation_dict,
active_queue_names):
equiv = []
for part in self.partitions.itervalues():
if part.functional and part.scheduled:
part_active_queues = []
for q in part.queue.split(":"):
if q in active_queue_names:
part_active_queues.append(q)
# go on to the next partition if there are no running
# queues using this partition
if not part_active_queues:
continue
found_a_match = False
for e in equiv:
if e['data'].intersection(part.node_card_names):
e['queues'].update(part_active_queues)
e['data'].update(part.node_card_names)
found_a_match = True
break
if not found_a_match:
equiv.append({
'queues': set(part_active_queues),
'data': set(part.node_card_names),
'reservations': set()
})
real_equiv = []
for eq_class in equiv:
found_a_match = False
for e in real_equiv:
if e['queues'].intersection(eq_class['queues']):
e['queues'].update(eq_class['queues'])
e['data'].update(eq_class['data'])
found_a_match = True
break
if not found_a_match:
real_equiv.append(eq_class)
equiv = real_equiv
for eq_class in equiv:
for res_name in reservation_dict:
skip = True
for p_name in reservation_dict[res_name].split(":"):
p = self.partitions[p_name]
if eq_class['data'].intersection(p.node_card_names):
eq_class['reservations'].add(res_name)
for dep_name in p._wiring_conflicts:
if self.partitions.has_key(dep_name):
if eq_class['data'].intersection(
self.partitions[dep_name].node_card_names):
eq_class['reservations'].add(res_name)
break
for key in eq_class:
eq_class[key] = list(eq_class[key])
del eq_class['data']
return equiv
find_queue_equivalence_classes = exposed(find_queue_equivalence_classes)
def can_run(self, target_partition, node_count, partition_dict):
if target_partition.state != "idle":
return False
desired = sys.maxint
for part in partition_dict.itervalues():
if not part.functional:
if target_partition.name in part.children or target_partition.name in part.parents:
return False
else:
if part.scheduled:
if int(node_count) <= int(part.size) < desired:
desired = int(part.size)
return target_partition.scheduled and target_partition.functional and int(
target_partition.size) == desired
def reserve_resources_until(self, location, new_time, jobid):
rc = False
partition_name = location[0]
pg = self.process_groups.find_by_jobid(jobid)
try:
self._partitions_lock.acquire()
used_by = self.partitions[partition_name].used_by
if used_by == None:
self.partitions[partition_name].used_by = jobid
used_by = jobid
if new_time:
if used_by == jobid:
self.partitions[partition_name].reserved_until = new_time
self.partitions[partition_name].reserved_by = jobid
self.logger.info(
"job %s: partition '%s' now reserved until %s", jobid,
partition_name, time.asctime(time.gmtime(new_time)))
rc = True
else:
self.logger.error(
"job %s wasn't allowed to update the reservation on partition %s (owner=%s)",
jobid, partition_name, used_by)
else:
if used_by == jobid:
self.partitions[partition_name].reserved_until = False
self.partitions[partition_name].reserved_by = None
self.logger.info(
"reservation on partition '%s' has been removed",
partition_name)
rc = True
else:
self.logger.error(
"job %s wasn't allowed to clear the reservation on partition %s (owner=%s)",
jobid, partition_name, used_by)
except:
self.logger.exception(
"an unexpected error occurred will adjusting the partition reservation time"
)
finally:
self._partitions_lock.release()
return rc
reserve_resources_until = exposed(reserve_resources_until)
class OrcmSystem (OrcmBaseSystem):
"""ORCM system component.
Methods:
add_process_groups -- add (start) an mpirun process on the system (exposed, ~query)
get_process_groups -- retrieve mpirun processes (exposed, query)
wait_process_groups -- get process groups that have exited, and remove them from the system (exposed, query)
signal_process_groups -- send a signal to the head process of the specified process groups (exposed, query)
update_partition_state -- update partition state from the bridge API (runs as a thread)
"""
name = "system"
implementation = "orcm_system"
logger = logger
def __init__ (self, *args, **kwargs):
OrcmBaseSystem.__init__(self, *args, **kwargs)
self.process_groups.item_cls = OrcmProcessGroup
def __del__ (self):
OrcmBaseSystem.__del__(self)
def __getstate__(self):
state = {}
state.update(OrcmBaseSystem.__getstate__(self))
# state.update({
# "orcm_system_version": 1 })
return state
def __setstate__(self, state):
OrcmBaseSystem.__setstate__(self, state)
self.process_groups.item_cls = OrcmProcessGroup
def add_process_groups (self, specs):
"""Create a process group.
Arguments:
spec -- dictionary hash specifying a process group to start
"""
self.logger.info("add_process_groups(%r)", specs)
process_groups = self.process_groups.q_add(specs)
for pgroup in process_groups:
self.logger.info("Job %s/%s: process group %s created to track script", pgroup.user, pgroup.jobid, pgroup.id)
#System has started the job. We need remove them from the temp, alloc array
#in orcm_base_system.
self.apg_started = True
for pgroup in process_groups:
for location in pgroup.location:
try:
del self.alloc_only_nodes[location]
except KeyError:
logger.critical("%s already removed from alloc_only_nodes list", location)
return process_groups
add_process_groups = exposed(query(add_process_groups))
def get_process_groups (self, specs):
self._get_exit_status()
return self.process_groups.q_get(specs)
get_process_groups = exposed(query(get_process_groups))
def _get_exit_status (self):
children = {}
cleanup = {}
_get_exit_status = automatic(_get_exit_status,
float(get_orcm_system_config('get_exit_status_interval', 10)))
def wait_process_groups (self, specs):
process_groups = self.process_groups.q_get(specs)
return process_groups
wait_process_groups = locking(exposed(query(wait_process_groups)))
def signal_process_groups (self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
OrcmBaseSystem.cancel_session(self, pg.jobid)
pg.exit_status = 0
for host in pg.location:
self.running_nodes.discard(host)
return my_process_groups
signal_process_groups = exposed(query(signal_process_groups))
def del_process_groups(self, jobid):
'''delete a process group and don't track it anymore.
jobid -- jobid associated with the process group we are removing
'''
del_items = self.process_groups.q_del([{'jobid':jobid}])
if del_items == []:
self.logger.warning("Job %s: Process group not found for this jobid.", jobid)
else:
self.logger.info("Job %s: Process group deleted.", jobid)
return
class BGSched(Component):
implementation = "bgsched"
name = "scheduler"
logger = logging.getLogger("Cobalt.Components.scheduler")
_configfields = ['utility_file']
_config = ConfigParser.ConfigParser()
print Cobalt.CONFIG_FILES
_config.read(Cobalt.CONFIG_FILES)
if not _config._sections.has_key('bgsched'):
print '''"bgsched" section missing from cobalt config file'''
sys.exit(1)
config = _config._sections['bgsched']
mfields = [field for field in _configfields if not config.has_key(field)]
if mfields:
print "Missing option(s) in cobalt config file [bgsched] section: %s" % (
" ".join(mfields))
sys.exit(1)
if config.get("default_reservation_policy"):
global DEFAULT_RESERVATION_POLICY
DEFAULT_RESERVATION_POLICY = config.get("default_reservation_policy")
def __init__(self, *args, **kwargs):
Component.__init__(self, *args, **kwargs)
self.COMP_QUEUE_MANAGER = "queue-manager"
self.COMP_SYSTEM = "system"
self.reservations = ReservationDict()
self.queues = QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = JobDict(self.COMP_QUEUE_MANAGER)
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.active = True
self.get_current_time = time.time
self.id_gen = IncrID()
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
def __getstate__(self):
return {
'reservations': self.reservations,
'version': 1,
'active': self.active,
'next_res_id': self.id_gen.idnum + 1,
'next_cycle_id': self.cycle_id_gen.idnum + 1,
'msg_queue': dbwriter.msg_queue,
'overflow': dbwriter.overflow
}
def __setstate__(self, state):
self.reservations = state['reservations']
if 'active' in state:
self.active = state['active']
else:
self.active = True
self.id_gen = IncrID()
self.id_gen.set(state['next_res_id'])
global bgsched_id_gen
bgsched_id_gen = self.id_gen
self.cycle_id_gen = IncrID()
self.cycle_id_gen.set(state['next_cycle_id'])
global bgsched_cycle_id_gen
bgsched_cycle_id_gen = self.cycle_id_gen
self.queues = QueueDict(self.COMP_QUEUE_MANAGER)
self.jobs = JobDict(self.COMP_QUEUE_MANAGER)
self.started_jobs = {}
self.sync_state = Cobalt.Util.FailureMode("Foreign Data Sync")
self.get_current_time = time.time
self.lock = threading.Lock()
self.statistics = Statistics()
if state.has_key('msg_queue'):
dbwriter.msg_queue = state['msg_queue']
if state.has_key('overflow') and (dbwriter.max_queued != None):
dbwriter.overflow = state['overflow']
# order the jobs with biggest utility first
def utilitycmp(self, job1, job2):
return -cmp(job1.score, job2.score)
def prioritycmp(self, job1, job2):
"""Compare 2 jobs first using queue priority and then first-in, first-out."""
val = cmp(self.queues[job1.queue].priority,
self.queues[job2.queue].priority)
if val == 0:
return self.fifocmp(job1, job2)
else:
# we want the higher priority first
return -val
def fifocmp(self, job1, job2):
"""Compare 2 jobs for first-in, first-out."""
def fifo_value(job):
if job.index is not None:
return int(job.index)
else:
return job.jobid
# Implement some simple variations on FIFO scheduling
# within a particular queue, based on queue policy
fifoval = cmp(fifo_value(job1), fifo_value(job2))
if (job1.queue == job2.queue):
qpolicy = self.queues[job1.queue].policy
sizeval = cmp(int(job1.nodes), int(job2.nodes))
wtimeval = cmp(int(job1.walltime), int(job2.walltime))
if (qpolicy == 'largest-first' and sizeval):
return -sizeval
elif (qpolicy == 'smallest-first' and sizeval):
return sizeval
elif (qpolicy == 'longest-first' and wtimeval):
return -wtimeval
elif (qpolicy == 'shortest-first' and wtimeval):
return wtimeval
else:
return fifoval
else:
return fifoval
return cmp(fifo_value(job1), fifo_value(job2))
def save_me(self):
Component.save(self)
save_me = automatic(save_me)
#user_name in this context is the user setting/modifying the res.
def add_reservations(self, specs, user_name):
self.logger.info("%s adding reservation: %r" % (user_name, specs))
added_reservations = self.reservations.q_add(specs)
for added_reservation in added_reservations:
self.logger.info("Res %s/%s: %s adding reservation: %r" %
(added_reservation.res_id,
added_reservation.cycle_id, user_name, specs))
dbwriter.log_to_db(user_name, "creating", "reservation",
added_reservation)
return added_reservations
add_reservations = exposed(query(add_reservations))
def del_reservations(self, specs, user_name):
self.logger.info("%s releasing reservation: %r" % (user_name, specs))
del_reservations = self.reservations.q_del(specs)
for del_reservation in del_reservations:
self.logger.info("Res %s/%s/: %s releasing reservation: %r" %
(del_reservation.res_id, del_reservation.cycle_id,
user_name, specs))
#dbwriter.log_to_db(user_name, "ending", "reservation", del_reservation)
return del_reservations
del_reservations = exposed(query(del_reservations))
def get_reservations(self, specs):
return self.reservations.q_get(specs)
get_reservations = exposed(query(get_reservations))
def set_reservations(self, specs, updates, user_name):
log_str = "%s modifying reservation: %r with updates %r" % (
user_name, specs, updates)
self.logger.info(log_str)
#handle defers as a special case: have to log these, and not drop a mod record.
def _set_reservations(res, newattr):
res.update(newattr)
updates['__cmd_user'] = user_name
mod_reservations = self.reservations.q_get(specs, _set_reservations,
updates)
for mod_reservation in mod_reservations:
self.logger.info("Res %s/%s: %s modifying reservation: %r" %
(mod_reservation.res_id, mod_reservation.cycle_id,
user_name, specs))
return mod_reservations
set_reservations = exposed(query(set_reservations))
def release_reservations(self, specs, user_name):
self.logger.info("%s requested release of reservation: %r" %
(user_name, specs))
self.logger.info("%s releasing reservation: %r" % (user_name, specs))
rel_res = self.get_reservations(specs)
for res in rel_res:
dbwriter.log_to_db(user_name, "released", "reservation", res)
del_reservations = self.reservations.q_del(specs)
for del_reservation in del_reservations:
self.logger.info("Res %s/%s/: %s releasing reservation: %r" %
(del_reservation.res_id, del_reservation.cycle_id,
user_name, specs))
return del_reservations
release_reservations = exposed(query(release_reservations))
def check_reservations(self):
ret = ""
reservations = self.reservations.values()
for i in range(len(reservations)):
for j in range(i + 1, len(reservations)):
# if at least one reservation is cyclic, we want *that* reservation to be the one getting its overlaps method
# called
if reservations[i].cycle is not None:
res1 = reservations[i]
res2 = reservations[j]
else:
res1 = reservations[j]
res2 = reservations[i]
# we subtract a little bit because the overlaps method isn't really meant to do this
# it will report warnings when one reservation starts at the same time another ends
if res1.overlaps(res2.start, res2.duration - 0.00001):
# now we need to check for overlap in space
results = ComponentProxy(self.COMP_SYSTEM).get_partitions(
[{
'name': p,
'children': '*',
'parents': '*'
} for p in res2.partitions.split(":")])
for p in res1.partitions.split(":"):
for r in results:
if p == r['name'] or p in r['children'] or p in r[
'parents']:
ret += "Warning: reservation '%s' overlaps reservation '%s'\n" % (
res1.name, res2.name)
return ret
check_reservations = exposed(check_reservations)
def sync_data(self):
started = self.get_current_time()
for item in [self.jobs, self.queues]:
try:
item.Sync()
except (ComponentLookupError, xmlrpclib.Fault):
# the ForeignDataDicts already include FailureMode stuff
pass
# print "took %f seconds for sync_data" % (time.time() - started, )
#sync_data = automatic(sync_data)
def _run_reservation_jobs(self, reservations_cache):
# handle each reservation separately, as they shouldn't be competing for resources
for cur_res in reservations_cache.itervalues():
#print "trying to run res jobs in", cur_res.name, self.started_jobs
queue = cur_res.queue
if not (self.queues.has_key(queue)
and self.queues[queue].state == 'running'):
continue
temp_jobs = self.jobs.q_get([{
'is_runnable': True,
'queue': queue
}])
active_jobs = []
for j in temp_jobs:
if not self.started_jobs.has_key(
j.jobid) and cur_res.job_within_reservation(j):
active_jobs.append(j)
if not active_jobs:
continue
active_jobs.sort(self.utilitycmp)
job_location_args = []
for job in active_jobs:
job_location_args.append({
'jobid':
str(job.jobid),
'nodes':
job.nodes,
'queue':
job.queue,
'required':
cur_res.partitions.split(":"),
'utility_score':
job.score,
'walltime':
job.walltime,
'attrs':
job.attrs,
'user':
job.user,
})
# there's no backfilling in reservations
try:
best_partition_dict = ComponentProxy(
self.COMP_SYSTEM).find_job_location(job_location_args, [])
except:
self.logger.error("failed to connect to system component")
best_partition_dict = {}
for jobid in best_partition_dict:
job = self.jobs[int(jobid)]
self._start_job(job, best_partition_dict[jobid],
{str(job.jobid): cur_res.res_id})
def _start_job(self, job, partition_list, resid=None):
"""Get the queue manager to start a job."""
cqm = ComponentProxy(self.COMP_QUEUE_MANAGER)
try:
self.logger.info("trying to start job %d on partition %r" %
(job.jobid, partition_list))
cqm.run_jobs([{
'tag': "job",
'jobid': job.jobid
}], partition_list, None, resid)
except ComponentLookupError:
self.logger.error("failed to connect to queue manager")
return
self.started_jobs[job.jobid] = self.get_current_time()
def schedule_jobs(self):
'''look at the queued jobs, and decide which ones to start'''
started_scheduling = self.get_current_time()
if not self.active:
return
self.sync_data()
# if we're missing information, don't bother trying to schedule jobs
if not (self.queues.__oserror__.status
and self.jobs.__oserror__.status):
self.sync_state.Fail()
return
self.sync_state.Pass()
self.lock.acquire()
try:
# cleanup any reservations which have expired
for res in self.reservations.values():
if res.is_over():
self.logger.info("reservation %s has ended; removing" %
(res.name))
self.logger.info("Res %s/%s: Ending reservation: %r" %
(res.res_id, res.cycle_id, res.name))
#dbwriter.log_to_db(None, "ending", "reservation",
# res)
del_reservations = self.reservations.q_del([{
'name':
res.name
}])
reservations_cache = self.reservations.copy()
except:
# just to make sure we don't keep the lock forever
self.logger.error("error in schedule_jobs", exc_info=True)
self.lock.release()
# clean up the started_jobs cached data
# TODO: Make this tunable.
now = self.get_current_time()
for job_name in self.started_jobs.keys():
if (now - self.started_jobs[job_name]) > 60:
del self.started_jobs[job_name]
active_queues = []
spruce_queues = []
res_queues = set()
for item in reservations_cache.q_get([{'queue': '*'}]):
if self.queues.has_key(item.queue):
if self.queues[item.queue].state == 'running':
res_queues.add(item.queue)
for queue in self.queues.itervalues():
if queue.name not in res_queues and queue.state == 'running':
if queue.policy == "high_prio":
spruce_queues.append(queue)
else:
active_queues.append(queue)
# handle the reservation jobs that might be ready to go
self._run_reservation_jobs(reservations_cache)
# figure out stuff about queue equivalence classes
if __running_mode__ == "simulation":
equiv = [{'reservations': [], 'queues': ['default']}]
else:
res_info = {}
for cur_res in reservations_cache.values():
res_info[cur_res.name] = cur_res.partitions
try:
equiv = ComponentProxy(
self.COMP_SYSTEM).find_queue_equivalence_classes(
res_info,
[q.name for q in active_queues + spruce_queues])
except:
self.logger.error("failed to connect to system component")
return
for eq_class in equiv:
# recall that is_runnable is True for certain types of holds
temp_jobs = self.jobs.q_get([{'is_runnable':True, 'queue':queue.name} for queue in active_queues \
if queue.name in eq_class['queues']])
active_jobs = []
for j in temp_jobs:
if not self.started_jobs.has_key(j.jobid):
active_jobs.append(j)
temp_jobs = self.jobs.q_get([{'is_runnable':True, 'queue':queue.name} for queue in spruce_queues \
if queue.name in eq_class['queues']])
spruce_jobs = []
for j in temp_jobs:
if not self.started_jobs.has_key(j.jobid):
spruce_jobs.append(j)
# if there are any pending jobs in high_prio queues, those are the only ones that can start
if spruce_jobs:
active_jobs = spruce_jobs
# get the cutoff time for backfilling
#
# BRT: should we use 'has_resources' or 'is_active'? has_resources returns to false once the resource epilogue
# scripts have finished running while is_active only returns to false once the job (not just the running task) has
# completely terminated. the difference is likely to be slight unless the job epilogue scripts are heavy weight.
temp_jobs = [
job for job in self.jobs.q_get([{
'has_resources': True
}]) if job.queue in eq_class['queues']
]
end_times = []
for job in temp_jobs:
# take the max so that jobs which have gone overtime and are being killed
# continue to cast a small backfilling shadow (we need this for the case
# that the final job in a drained partition runs overtime -- which otherwise
# allows things to be backfilled into the drained partition)
##*AdjEst*
if running_job_walltime_prediction:
runtime_estimate = float(job.walltime_p)
else:
runtime_estimate = float(job.walltime)
end_time = max(
float(job.starttime) + 60 * runtime_estimate, now + 5 * 60)
end_times.append([job.location, end_time])
for res_name in eq_class['reservations']:
cur_res = reservations_cache[res_name]
if not cur_res.cycle:
end_time = float(cur_res.start) + float(cur_res.duration)
else:
done_after = float(cur_res.duration) - (
(now - float(cur_res.start)) % float(cur_res.cycle))
if done_after < 0:
done_after += cur_res.cycle
end_time = now + done_after
if cur_res.is_active():
for part_name in cur_res.partitions.split(":"):
end_times.append([[part_name], end_time])
if not active_jobs:
continue
active_jobs.sort(self.utilitycmp)
# now smoosh lots of data together to be passed to the allocator in the system component
job_location_args = []
for job in active_jobs:
forbidden_locations = set()
for res_name in eq_class['reservations']:
cur_res = reservations_cache[res_name]
if cur_res.overlaps(self.get_current_time(),
60 * float(job.walltime) + SLOP_TIME):
forbidden_locations.update(
cur_res.partitions.split(":"))
job_location_args.append({
'jobid': str(job.jobid),
'nodes': job.nodes,
'queue': job.queue,
'forbidden': list(forbidden_locations),
'utility_score': job.score,
'walltime': job.walltime,
'walltime_p': job.walltime_p, #*AdjEst*
'attrs': job.attrs,
'user': job.user,
})
try:
best_partition_dict = ComponentProxy(
self.COMP_SYSTEM).find_job_location(
job_location_args, end_times)
except:
self.logger.error("failed to connect to system component",
exc_info=True)
best_partition_dict = {}
for jobid in best_partition_dict:
job = self.jobs[int(jobid)]
self._start_job(job, best_partition_dict[jobid])
# print "took %f seconds for scheduling loop" % (time.time() - started_scheduling, )
schedule_jobs = locking(automatic(schedule_jobs))
def get_resid(self, queue_name):
return None
get_resid = exposed(get_resid)
def enable(self, user_name):
"""Enable scheduling"""
self.logger.info("%s enabling scheduling", user_name)
self.active = True
enable = exposed(enable)
def disable(self, user_name):
"""Disable scheduling"""
self.logger.info("%s disabling scheduling", user_name)
self.active = False
disable = exposed(disable)
def set_res_id(self, id_num):
"""Set the reservation id number."""
self.id_gen.set(id_num)
logger.info("Reset res_id generator to %s." % id_num)
set_res_id = exposed(set_res_id)
def set_cycle_id(self, id_num):
"""Set the cycle id number."""
self.cycle_id_gen.set(id_num)
logger.info("Reset cycle_id generator to %s." % id_num)
set_cycle_id = exposed(set_cycle_id)
def force_res_id(self, id_num):
"""Override the id-generator and change the resid to id_num"""
self.id_gen.idnum = id_num - 1
logger.warning("Forced res_id generator to %s." % id_num)
force_res_id = exposed(force_res_id)
def force_cycle_id(self, id_num):
"""Override the id-generator and change the cycleid to id_num"""
self.cycle_id_gen.idnum = id_num - 1
logger.warning("Forced cycle_id generator to %s." % id_num)
force_cycle_id = exposed(force_cycle_id)
def get_next_res_id(self):
"""Get what the next resid number would be"""
return self.id_gen.idnum + 1
get_next_res_id = exposed(get_next_res_id)
def get_next_cycle_id(self):
"""get what the next cycleid number would be"""
return self.cycle_id_gen.idnum + 1
get_next_cycle_id = exposed(get_next_cycle_id)
def __flush_msg_queue(self):
"""Send queued messages to the database-writer component"""
dbwriter.flush_queue()
__flush_msg_queue = automatic(
__flush_msg_queue, float(get_bgsched_config('db_flush_interval', 10)))
class ClusterSystem (ClusterBaseSystem):
"""cluster system component.
Methods:
configure -- load partitions from the bridge API
add_process_groups -- add (start) an mpirun process on the system (exposed, ~query)
get_process_groups -- retrieve mpirun processes (exposed, query)
wait_process_groups -- get process groups that have exited, and remove them from the system (exposed, query)
signal_process_groups -- send a signal to the head process of the specified process groups (exposed, query)
update_partition_state -- update partition state from the bridge API (runs as a thread)
"""
name = "system"
implementation = "cluster_system"
logger = logger
def __init__ (self, *args, **kwargs):
ClusterBaseSystem.__init__(self, *args, **kwargs)
self.process_groups.item_cls = ClusterProcessGroup
def __setstate__(self, state):
ClusterBaseSystem.__setstate__(self, state)
self.process_groups.item_cls = ClusterProcessGroup
def add_process_groups (self, specs):
"""Create a process group.
Arguments:
spec -- dictionary hash specifying a process group to start
"""
self.logger.info("add_process_groups(%r)", specs)
process_groups = self.process_groups.q_add(specs)
for pgroup in process_groups:
self.logger.info("job %s/%s: process group %s created to track script",
pgroup.jobid, pgroup.user, pgroup.id)
return process_groups
add_process_groups = exposed(query(add_process_groups))
def get_process_groups (self, specs):
self._get_exit_status()
return self.process_groups.q_get(specs)
get_process_groups = exposed(query(get_process_groups))
def _get_exit_status (self):
try:
running = ComponentProxy("forker").active_list()
except:
self.logger.error("failed to contact forker component for list of running jobs")
return
for each in self.process_groups.itervalues():
if each.head_pid not in running and each.exit_status is None:
# FIXME: i bet we should consider a retry thing here -- if we fail enough times, just
# assume the process is dead? or maybe just say there's no exit code the first time it happens?
# maybe the second choice is better
try:
dead_dict = ComponentProxy("forker").get_status(each.head_pid)
except Queue.Empty:
self.logger.error("failed call for get_status from forker component for pg %s", each.head_pid)
return
if dead_dict is None:
self.logger.info("process group %i: job %s/%s exited with unknown status", each.id, each.jobid, each.user)
each.exit_status = 1234567
else:
each.exit_status = dead_dict["exit_status"]
if dead_dict["signum"] == 0:
self.logger.info("process group %i: job %s/%s exited with status %i",
each.id, each.jobid, each.user, each.exit_status)
else:
if dead_dict["core_dump"]:
core_dump_str = ", core dumped"
else:
core_dump_str = ""
self.logger.info("process group %i: job %s/%s terminated with signal %s%s",
each.id, each.jobid, each.user, dead_dict["signum"], core_dump_str)
_get_exit_status = automatic(_get_exit_status)
def wait_process_groups (self, specs):
self._get_exit_status()
process_groups = [pg for pg in self.process_groups.q_get(specs) if pg.exit_status is not None]
for process_group in process_groups:
thread.start_new_thread(self.clean_nodes, (process_group,))
return process_groups
wait_process_groups = locking(exposed(query(wait_process_groups)))
def signal_process_groups (self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
if pg.exit_status is None:
try:
ComponentProxy("forker").signal(pg.head_pid, signame)
except:
self.logger.error("Failed to communicate with forker when signalling job")
return my_process_groups
signal_process_groups = exposed(query(signal_process_groups))
def clean_nodes(self, pg):
try:
tmp_data = pwd.getpwnam(pg.user)
groupid = tmp_data.pw_gid
group_name = grp.getgrgid(groupid)[0]
except KeyError:
group_name = ""
self.logger.error("Job %s/%s unable to determine group name for epilogue" % (pg.jobid, pg.user))
processes = []
for host in pg.location:
h = host.split(":")[0]
try:
p = subprocess.Popen(["/usr/bin/ssh", h, pg.config.get("epilogue"), str(pg.jobid), pg.user, group_name],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.host = h
processes.append(p)
except:
self.logger.error("Job %s/%s failed to run epilogue on host %s", pg.jobid, pg.user, h, exc_info=True)
start = time.time()
dirty_nodes = []
while True:
running = False
for p in processes:
if p.poll() is None:
running = True
break
if not running:
break
if time.time() - start > float(pg.config.get("epilogue_timeout")):
for p in processes:
if p.poll() is None:
try:
os.kill(p.pid, signal.SIGTERM)
dirty_nodes.append(p.host)
self.logger.error("Job %s/%s epilogue timed out on host %s" % (pg.jobid, pg.user, p.host))
except:
self.logger.error("epilogue for %s already terminated" %p.host)
break
else:
time.sleep(5)
for p in processes:
if p.poll() > 0:
self.logger.error("epilogue failed for host %s", p.host)
self.logger.error("stderr from epilogue on host %s: [%s]", p.host, p.stderr.read().strip())
self.lock.acquire()
try:
self.logger.info("job finished on %s", Cobalt.Util.merge_nodelist(pg.location))
for host in pg.location:
self.running_nodes.discard(host)
if dirty_nodes:
for host in dirty_nodes:
self.down_nodes.add(host)
self.logger.info("epilogue timed out, marking host %s down" % host)
p = subprocess.Popen([pg.config.get("epi_epilogue"), str(pg.jobid), pg.user, group_name] + dirty_nodes)
del self.process_groups[pg.id]
except:
self.logger.error("error in clean_nodes", exc_info=True)
self.lock.release()
_get_exit_status = automatic(
_get_exit_status,
float(get_cluster_system_config('get_exit_status_interval', 10)))
def wait_process_groups(self, specs):
self._get_exit_status()
process_groups = [
pg for pg in self.process_groups.q_get(specs)
if pg.exit_status is not None
]
for process_group in process_groups:
self.clean_nodes(process_group.location, process_group.user,
process_group.jobid)
return process_groups
wait_process_groups = locking(exposed(query(wait_process_groups)))
def signal_process_groups(self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
if pg.exit_status is None:
try:
ComponentProxy(pg.forker).signal(pg.head_pid, signame)
except:
self.logger.error(
"Failed to communicate with forker when signalling job"
)
return my_process_groups
signal_process_groups = exposed(query(signal_process_groups))
ComponentProxy(forker).cleanup_children(cleanup[forker])
except ComponentLookupError:
self.logger.error("failed to contact the %s component to cleanup children", forker)
except:
self.logger.error("unexpected exception while requesting that the %s component perform cleanup",
forker, exc_info=True)
_get_exit_status = automatic(_get_exit_status,
float(get_cluster_system_config('get_exit_status_interval', 10)))
def wait_process_groups (self, specs):
self._get_exit_status()
process_groups = [pg for pg in self.process_groups.q_get(specs) if pg.exit_status is not None]
for process_group in process_groups:
self.clean_nodes(process_group.location, process_group.user, process_group.jobid)
return process_groups
wait_process_groups = locking(exposed(query(wait_process_groups)))
def signal_process_groups (self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
if pg.exit_status is None:
try:
ComponentProxy(pg.forker).signal(pg.head_pid, signame)
except:
self.logger.error("Failed to communicate with forker when signalling job")
return my_process_groups
signal_process_groups = exposed(query(signal_process_groups))
def del_process_groups(self, jobid):
'''delete a process group and don't track it anymore.
class ClusterSystem(ClusterBaseSystem):
"""cluster system component.
Methods:
configure -- load partitions from the bridge API
add_process_groups -- add (start) an mpirun process on the system (exposed, ~query)
get_process_groups -- retrieve mpirun processes (exposed, query)
wait_process_groups -- get process groups that have exited, and remove them from the system (exposed, query)
signal_process_groups -- send a signal to the head process of the specified process groups (exposed, query)
update_partition_state -- update partition state from the bridge API (runs as a thread)
"""
name = "system"
implementation = "cluster_system"
logger = logger
def __init__(self, *args, **kwargs):
ClusterBaseSystem.__init__(self, *args, **kwargs)
self.process_groups.item_cls = ClusterProcessGroup
def __setstate__(self, state):
ClusterBaseSystem.__setstate__(self, state)
self.process_groups.item_cls = ClusterProcessGroup
def add_process_groups(self, specs):
"""Create a process group.
Arguments:
spec -- dictionary hash specifying a process group to start
"""
self.logger.info("add_process_groups(%r)", specs)
process_groups = self.process_groups.q_add(specs)
for pgroup in process_groups:
self.logger.info(
"Job %s/%s: process group %s created to track script",
pgroup.user, pgroup.jobid, pgroup.id)
#System has started the job. We need remove them from the temp, alloc array
#in cluster_base_system.
for pg in process_groups:
for location in pg.location:
del self.alloc_only_nodes[location]
return process_groups
add_process_groups = exposed(query(add_process_groups))
def get_process_groups(self, specs):
self._get_exit_status()
return self.process_groups.q_get(specs)
get_process_groups = exposed(query(get_process_groups))
def _get_exit_status(self):
try:
running = ComponentProxy("forker").active_list("process group")
except:
self.logger.error(
"failed to contact forker component for list of running jobs")
return
for each in self.process_groups.itervalues():
if each.head_pid not in running and each.exit_status is None:
# FIXME: i bet we should consider a retry thing here -- if we fail enough times, just
# assume the process is dead? or maybe just say there's no exit code the first time it happens?
# maybe the second choice is better
try:
dead_dict = ComponentProxy("forker").get_status(
each.head_pid)
except Queue.Empty:
self.logger.error(
"failed call for get_status from forker component for pg %s",
each.head_pid)
return
if dead_dict is None:
self.logger.info(
"Job %s/%s: process group %i: exited with unknown status",
each.user, each.jobid, each.id)
each.exit_status = 1234567
else:
each.exit_status = dead_dict["exit_status"]
if dead_dict["signum"] == 0:
self.logger.info(
"process group %i: job %s/%s exited with status %i",
each.id, each.jobid, each.user, each.exit_status)
else:
if dead_dict["core_dump"]:
core_dump_str = ", core dumped"
else:
core_dump_str = ""
self.logger.info(
"process group %i: job %s/%s terminated with signal %s%s",
each.id, each.jobid, each.user,
dead_dict["signum"], core_dump_str)
_get_exit_status = automatic(_get_exit_status)
def wait_process_groups(self, specs):
self._get_exit_status()
process_groups = [
pg for pg in self.process_groups.q_get(specs)
if pg.exit_status is not None
]
for process_group in process_groups:
self.clean_nodes(
pg.location, pg.user, pg.jobid
) #FIXME: This call is a good place to look for problems
return process_groups
wait_process_groups = locking(exposed(query(wait_process_groups)))
def signal_process_groups(self, specs, signame="SIGINT"):
my_process_groups = self.process_groups.q_get(specs)
for pg in my_process_groups:
if pg.exit_status is None:
try:
ComponentProxy("forker").signal(pg.head_pid, signame)
except:
self.logger.error(
"Failed to communicate with forker when signalling job"
)
return my_process_groups
signal_process_groups = exposed(query(signal_process_groups))
def del_process_groups(self, jobid):
'''delete a process group and don't track it anymore.
jobid -- jobid associated with the process group we are removing
'''
del_items = self.process_groups.q_del([{'jobid': jobid}])
if del_items == []:
self.logger.warning(
"Job %s: Process group not found for this jobid.", jobid)
else:
self.logger.info("Job %s: Process group deleted.", jobid)
class ScriptManager(Component):
'''The ScriptManager supports the running of scripts on a BG machine'''
name = 'script-manager'
# A default logger for the class is placed here.
# Assigning an instance-level logger is supported,
# and expected in the case of multiple instances.
logger = logging.getLogger("Cobalt.Components.ScriptManager")
implementation = 'scriptm'
def __init__(self, *args, **kwargs):
"""Initialize a new ServiceLocator.
All arguments are passed to the component constructor.
"""
Component.__init__(self, *args, **kwargs)
self.ignore = []
self.lastwait = 0
self.pgroups = ProcessGroupDict()
self.zombie_mpi = {}
def manage_children(self):
for pgroup in self.zombie_mpi.keys():
if pgroup.FinishProcess():
del self.zombie_mpi[pgroup]
self.lock.acquire()
try:
if (time.time() - self.lastwait) > 6:
while True:
try:
self.lastwait = time.time()
(pid, stat) = os.waitpid(-1, os.WNOHANG)
except OSError:
break
if pid == 0:
break
pgrps = [
pgrp for pgrp in self.pgroups.itervalues()
if pgrp.pid == pid
]
if len(pgrps) == 0:
self.logger.error(
"Failed to locate process group for pid %s" %
(pid))
elif len(pgrps) == 1:
pgroup = pgrps[0]
pgroup.exit_status = stat
self.logger.info("Job %s/%s: ProcessGroup %s Finished with exit code %d. pid %s" % \
(pgroup.jobid, pgroup.user, pgroup.jobid, int(stat)/256, pgroup.pid))
if os.WIFSIGNALED(stat):
self.logger.info("Job %s/%s: ProcessGroup %s received signal %s" % \
(pgroup.jobid, pgroup.user, pgroup.jobid, os.WTERMSIG(stat)))
try:
err = open(pgroup.cobalt_log_file, 'a')
print >> err, "The script job exited after receiving signal %s" % os.WTERMSIG(
stat)
err.close()
except IOError:
self.logger.error(
"Job %s/%s: ProcessGroup %s failed to update .error file"
%
(pgroup.jobid, pgroup.user, pgroup.jobid))
self.zombie_mpi[pgroup] = True
else:
self.logger.error(
"Got more than one match for pid %s" % (pid))
except:
# just to make sure we don't keep the lock forever
self.logger.error("error in manage_children", exc_info=True)
self.lock.release()
manage_children = locking(automatic(manage_children))
def add_jobs(self, specs):
'''Create new process group element'''
self.logger.info("creating process group %r" % specs)
return self.pgroups.q_add(specs)
add_jobs = exposed(query(add_jobs))
def get_jobs(self, specs):
'''query existing process group'''
return self.pgroups.q_get(specs)
get_jobs = exposed(query(get_jobs))
def wait_jobs(self, specs):
'''Remove completed process group'''
self.logger.info("removing process group %r" % specs)
return self.pgroups.q_del(specs)
wait_jobs = exposed(query(wait_jobs))
def signal_jobs(self, specs, sig):
'''signal existing process group with specified signal'''
ret = []
for spec in specs:
self.logger.info("signaling process group %r with signal %r" %
(spec, sig))
for pg in self.pgroups.itervalues():
if pg.id == int(spec['id']):
ret.append(pg.Signal(sig))
# could not find pg, so return False
return ret
signal_jobs = exposed(signal_jobs)
def SigChildHand(self, sig, frame):
'''Dont Handle SIGCHLDs'''
pass
def invoke_mpi_from_script(self, spec):
'''Invoke the real mpirun on behalf of a script being executed by the script manager.'''
self.lock.acquire()
try:
jobs = self.pgroups.q_get([{
'jobid': spec['jobid'],
'user': spec['user']
}])
except:
# just make sure we don't keep the lock forever
self.logger.error("error in invoke_mpi_from_script", exc_info=True)
self.lock.release()
if len(jobs) != 1:
self.logger.error(
"invoke_mpi_from_script matched more than one job with spec %r"
% spec)
return -1
else:
jobs[0].invoke_mpi_from_script(spec)
return jobs[0].mpi_system_id
invoke_mpi_from_script = locking(exposed(invoke_mpi_from_script))