class NPScheduler:
def __init__(self, N, policy='SJF'):
self.N = N # number of time steps to schedule
self.running = None
self.clock = 0
self.policy = policy
# instantiate the readyQueue, which may be a FIFO or MinHeap
# you may need additional queues for
# - tasks that have been added but not released yet
# - tasks that have been completed
# - the Gantt chart
self.readyQueue = MinHeap()
self.notReadyQueue = FIFO()
self.doneQueue = list()
self.ganttChart = list()
def addTask(self, task):
# if the release time of the new task is not in the future, then
# put it in ready queue; otherwise, put into not-ready queue.
# you may need to copy the scheduler policy into the task
task.setPriorityScheme(self.policy)
if task.releaseTime() <= self.clock: #release
self.readyQueue.put(task)
else:
self.notReadyQueue.put(task)
def dispatch(self, task):
# dispatch here means assign the chosen task as the one to run
# in the current time step.
# the task should be removed from ready-queue by caller;
# The task may be empty (None).
# This method will make an entry into the Gantt chart and perform
# bookkeeping, including
## - recording the last dispatched time of this task,
# - increment the wait times of those tasks not scheduled
# but in the ready queue
if task is None:
self.ganttChart.append(task)
return
self.ganttChart.append(task)
task.decrRemaining()
task.last_dispatched = self.clock
for task_wait in self.readyQueue:
task_wait.incrWaitTime()
def releaseTasks(self):
'''
this is called at the beginning of scheduling each time step to see
if new tasks became ready to be released to ready queue, when their
release time is no later than the current clock.
'''
while True:
r = self.notReadyQueue.head()
# assuming the notReadyQueue outputs by release time
if r is None or r.releaseTime() > self.clock:
break
r = self.notReadyQueue.get()
r.setPriorityScheme(self.policy)
self.readyQueue.put(r)
def checkTaskCompletion(self):
# if there is a current running task, check if it has just finished.
# (i.e., decrement remaining time and see if it has more work to do. ??
# If so, perform bookkeeping for completing the task, ???
# - move task to done-queue, set its completion time and lastrun time ???
# set the scheduler running task to None, and return True
# (so that a new task may be picked.)
# but if not completed, return False.
# If there is no current running task, also return True.
if self.running is None:
return True
# your code here
task = self.running
if task.done():
task.setCompletionTime(self.clock)
self.doneQueue.append(task)
self.running = None
return True
# for future use
if task is None: return True
return False
def schedule(self):
# scheduler that handles nonpreemptive scheduling.
# the policy such as RR, SJF, or FCFS is handled by the task as it
# defines the attribute to compare (in its __iter__() method)
# first, check if added but unreleased tasks may now be released
# (i.e., added to ready queue)
self.releaseTasks()
if self.checkTaskCompletion() == False: # get task status
# There is a current running task and it is not done yet!
# the same task will continue running to its completion.
# simply redispatch the current running task.
## redispatch the current running task, we have current ruuing task
self.dispatch(self.running)
else:
# task completed or no running task.
# get the next task from priority queue and dispatch it.
# add None
if len(self.readyQueue) == 0:
self.dispatch(None)
# pop finishing
if len(self.readyQueue) > 0:
task = self.readyQueue.get()
# another turn to run
self.running = task
self.dispatch(self.running)
def clockGen(self):
for self.clock in range(self.N):
# now run scheduler here
self.schedule()
yield self.clock
def getSchedule(self):
return '-'.join(map(str, self.ganttChart))
# return self.ganttChart
def getThroughput(self):
# calculate and return throughput as a tuple
numerator = len(self.doneQueue)
denominator = self.doneQueue[-1].completion
return (numerator, denominator)
def getWaitTime(self):
# calculate and return
numerator = sum([task.wait_time for task in self.doneQueue])
denominator = len(self.doneQueue)
return (numerator, denominator)
def getTurnaroundTime(self):
# calculate the turnaround time in terms of a tuple with
# separate turnaround times, #processes
#print([task.turnaroundTime() for task in self.doneQueue])
numerator = sum([task.turnaroundTime() for task in self.doneQueue])
denominator = len(self.doneQueue)
return (numerator, denominator)
class BatchPriorityQueue(MergingRecordFactory):
"""
A subclass of RecordFactory that maintains two on-disk FIFOs of
records:
* _pQ is ordered by records' priority_field lowest-first.
get(max_priority) retrieves next record from this queue if its
priority is lower than max_priority.
* _dumpQ is periodically sorted by unique_field, merged by the
accumulate function, and sorted again by priority_field in
order to populate the priority queue. put() adds records to
the dump queue. Calling get() also puts a copy of the record
into the dump queue, so during the merge, the accumulate
function will see the previous record and any new records.
For this reason, records put back into the queue after
processing a record gotten from the queue should present *only
changes* relative to the gotten record, so the accumulate
function can simply add them.
"""
def __init__(self,
record_class,
template,
data_path,
unique_key,
priority_key,
defaults={},
delimiter="|",
cache_size=2**16):
"""
See RecordFactory for record_class, 'template', 'defaults',
and 'delimiter'.
'data_path' and 'cache_size' are for _pQ and _dumpQ
unique_key and priority_key are integer indexes into 'fields'
indicating which to attributes of records to use as unique
keys and priorities.
"""
RecordFactory.__init__(self, record_class, template, defaults,
delimiter)
self._unique_key = unique_key
self._priority_key = priority_key
self._cache_size = cache_size
self._pQ_path = os.path.join(data_path, "pQ")
self._pQ_sync = os.path.join(data_path, "pQ_sync")
self._pQ = None
self._dumpQ_path = os.path.join(data_path, "dumpQ")
self._dumpQ_sync = os.path.join(data_path, "dumpQ_sync")
self._dumpQ = FIFO(self._dumpQ_path, self._cache_size)
# the next deserialized value to return via get
self._next = None
self._sync_pending_path = os.path.join(data_path, "sync_pending")
self._sync_pending = Mutex(self._sync_pending_path)
self._lock = Mutex(os.path.join(data_path, "lock_file"))
def close(self):
"""
Acquires lock, then raises self.Syncing if a sync is in
progress, otherwise closes internal FIFOs.
"""
self._lock.acquire()
try:
if not self._sync_pending.available():
raise self.Syncing
if self._pQ:
self._pQ.close()
self._dumpQ.close()
finally:
self._lock.release()
class NotYet(Exception):
"next record excluded by max_priority"
pass
class Blocked(Exception):
"another process has the mutex"
pass
class Syncing(Exception):
"sync in progress"
pass
class ReadyToSync(Exception):
"pQ empty but records exist in dumpQ"
pass
def get(self, max_priority=None, block=True):
"""
If block=False and cannot acquire lock, raises self.Blocked.
If a sync is in progress, raises self.Syncing.
If both _pQ and _dumpQ are empty, then raise Queue.Empty.
If empty _pQ but not empty _dumpQ, raise self.ReadyToSync.
If next item in _pQ has a priority less than max_priority,
then pops it from queue and returns record.
"""
acquired = self._lock.acquire(block)
if not acquired:
raise self.Blocked
try:
if not self._sync_pending.available():
raise self.Syncing
if self._pQ is None:
self._pQ = FIFO(self._pQ_path, self._cache_size)
if self._next is None:
# instantiate next record without removing from pQ, raises
# Queue.Empty when no lines in FIFO
try:
line = self._pQ.next()
self._next = self.loads(line)
except Queue.Empty:
if len(self._dumpQ) == 0:
raise Queue.Empty
else:
raise self.ReadyToSync
if max_priority is None or \
self._next[self._priority_key] < max_priority:
# Remove this line from _pQ and put into _dumpQ. There
# should be no risk of this raising Queue.Empty.
self._dumpQ.put(self._pQ.get())
ret_next = self._next
self._next = None
# This is only place that get() returns:
return ret_next
elif max_priority is not None:
raise self.NotYet
else:
raise Exception("Should never get here.")
finally:
self._lock.release()
def put(self, record=None, values=None, attrs=None, block=True):
"""
If record=None, then values or attrs is passed to
self.create() to obtain a record.
record is put into _dumpQ.
If block=False and cannot acquire lock, raises self.Blocked.
"""
acquired = self._lock.acquire(block)
if not acquired:
raise self.Blocked
if record is None:
if values is not None:
record = self.create(*values)
elif attrs is not None:
record = self.create(**attrs)
else:
raise Exception("put without record, values, or attrs")
self._dumpQ.put(self.dumps(record))
self._lock.release()
def sync(self, block=True):
"""
Removes all records from _dumpQ and _pQ and performs sort on
unique_key, accumulate, sort on priority_key before putting
all records into _pQ.
If block=False and cannot acquire lock, raises self.Blocked.
"""
acquired = self._lock.acquire(block)
if not acquired:
raise self.Blocked
try:
acquired = self._sync_pending.acquire(block=False)
if not acquired:
raise self.Syncing
# move queues to the side
if self._pQ:
self._pQ.close()
self._dumpQ.close()
os.rename(self._pQ_path, self._pQ_sync)
os.rename(self._dumpQ_path, self._dumpQ_sync)
# set pQ to None while syncing, and reopen dumpQ
self._pQ = None
self._dumpQ = FIFO(self._dumpQ_path, self._cache_size)
# Release sync lock momentarily, so merger can acquire it.
# Get is blocked by _lock, so it won't get confused.
self._sync_pending.release()
# launch a child to sort, accumulate, sort
merger = self.start_merger()
# loop until merger acquires _sync_pending
while merger.is_alive() and self._sync_pending.available():
sleep(0.1)
# now get back to normal operation
return merger
finally:
self._lock.release()
def start_merger(self):
"""
defines, instantiates, and starts a multiprocessing.Process
for sorting, accumulating, and sorting _dumpQ into new _pQ
"""
class Merger(multiprocessing.Process):
"manages the sort, accumulate, sort"
name = "SortAccumulateSort"
_queue = self
def run(self):
"executes sort, accumulate, sort"
try:
openlog(self.name, LOG_NDELAY | LOG_CONS | LOG_PID,
LOG_LOCAL0)
q = self._queue
_sync_pending = Mutex(q._sync_pending_path)
_sync_pending.acquire()
pQ = FIFO(q._pQ_path, q._cache_size)
assert len(pQ) == 0
pQ_data = os.path.join(q._pQ_sync, "data")
dumpQ_data = os.path.join(q._dumpQ_sync, "data")
in_files = [
os.path.join(pQ_data, cache_file)
for cache_file in os.listdir(pQ_data)
]
in_files += [
os.path.join(dumpQ_data, cache_file)
for cache_file in os.listdir(dumpQ_data)
]
start = time()
# fast version of chained generators
map(
pQ.put,
q.sort(
q.accumulate(
q.mergefiles(in_files, (q._unique_key, ))),
(q._priority_key, )))
# slow version of this generator chain:
#merged_lines = []
#for x in q.mergefiles(in_files, q._unique_key):
# syslog("out of mergefiles: %s" % repr(x))
# merged_lines.append(x)
#accumulated_lines = []
#for x in q.accumulate(merged_lines):
# syslog("out of accumulator: %s" % repr(x))
# accumulated_lines.append(x)
#for x in q.sort(accumulated_lines, q._priority_key):
# syslog("out of sort: %s" % repr(x))
# pQ.put(x)
end = time()
pQ.close()
syslog(LOG_INFO, "merge took %.1f seconds" % (end - start))
shutil.rmtree(q._pQ_sync)
shutil.rmtree(q._dumpQ_sync)
_sync_pending.release()
except Exception, exc:
map(lambda line: syslog(LOG_NOTICE, line),
traceback.format_exc(exc).splitlines())
merger = Merger()
merger.start()
return merger
def get(self, block=True, timeout=None):
"gets line from FIFO and returns deserialized record"
return self.loads(FIFO.get(self, block=block, timeout=timeout))
class NPScheduler:
def __init__(self, N, policy='SJF'):
self.N = N # number of time steps to schedule
self.running = None
self.clock = 0
self.policy = policy
self.readyQueue = MinHeap()
self.notReadyQueue = FIFO()
self.ganttChart = FIFO()
# instantiate the readyQueue, which may be a FIFO or MinHeap
# you may need additional queues for
# - tasks that have been added but not released yet
# - tasks that have been completed
# - the Gantt chart
self.lastruntime = 0
self.numOfTask = 0
def addTask(self, task):
task.scheme = self.policy
# for statistic
self.numOfTask = self.numOfTask + 1
if task.releaseTime() <= self.clock:
self.readyQueue.put(task)
else:
self.notReadyQueue.put(task)
# if the release time of the new task is not in the future, then
# put it in ready queue; otherwise, put into not-ready queue.
# you may need to copy the scheduler policy into the task
def dispatch(self, task):
r = self.readyQueue.head()
if r == None:
self.ganttChart.put(None)
return
self.running = task
self.readyQueue.get().lastDispatchedTime = self.clock
self.ganttChart.put(r)
for i in self.readyQueue:
i.incrWaitTime()
# dispatch here means assign the chosen task as the one to run
# in the current time step.
# the task should be removed from ready-queue by caller;
# The task may be empty (None).
# This method will make an entry into the Gantt chart and perform
# bookkeeping, including
# - recording the last dispatched time of this task,
# - increment the wait times of those tasks not scheduled
# but in the ready queue
def releaseTasks(self):
'''
this is called at the beginning of scheduling each time step to see
if new tasks became ready to be released to ready queue, when their
release time is no later than the current clock.
'''
while True:
r = self.notReadyQueue.head()
# assuming the notReadyQueue outputs by release time
if r is None or r.releaseTime() > self.clock:
break
r = self.notReadyQueue.get()
#r.setPriorityScheme(self.policy) little weird in my opinion
r.scheme = self.policy
if r:
r.fifo_prio = self.clock
self.readyQueue.put(r)
def checkTaskCompletion(self):
# if there is a current running task, check if it has just finished.
# (i.e., decrement remaining time and see if it has more work to do.
# If so, perform bookkeeping for completing the task,
# - move task to done-queue, set its completion time and lastrun time
# set the scheduler running task to None, and return True
# (so that a new task may be picked.)
# but if not completed, return False.
# If there is no current running task, also return True.
if self.running is None:
return True
# your code here
self.running.decrRemaining()
if self.running.done() == True:
self.running.completionTime = self.clock
self.running.lastDispatchedTime = self.clock
#for statistic
self.lastruntime = self.clock
return True
else:
return False
def schedule(self):
# scheduler that handles nonpreemptive scheduling.
# the policy such as RR, SJF, or FCFS is handled by the task as it
# defines the attribute to compare (in its __iter__() method)
# first, check if added but unreleased tasks may now be released
# (i.e., added to ready queue)
self.releaseTasks()
if self.checkTaskCompletion() == False:
self.ganttChart.put(self.running)
for i in self.readyQueue:
i.incrWaitTime()
# There is a current running task and it is not done yet!
# the same task will continue running to its completion.
# simply redispatch the current running task.
else:
self.running = self.readyQueue.head()
self.dispatch(self.readyQueue.head())
# task completed or no running task.
# get the next task from priority queue and dispatch it.
def clockGen(self):
for self.clock in range(self.N):
# now run scheduler here
self.schedule()
yield self.clock
def getSchedule(self):
return '-'.join(map(str, self.ganttChart))
# return self.ganttChart
def getThroughput(self):
return (self.numOfTask, self.lastruntime)
# calculate and return throughput as a tuple
def getWaitTime(self):
# calculate and return
s = set(self.ganttChart)
total_waiting_time = 0
for i in s:
if i:
total_waiting_time = total_waiting_time + i.waitingTime
return (total_waiting_time, self.numOfTask)
def getTurnaroundTime(self):
# calculate the turnaround time in terms of a tuple with
# separate turnaround times, #processes
s = set(self.ganttChart)
total_turnaround_time = 0
for i in s:
if i:
total_turnaround_time = total_turnaround_time + i.completionTime - i.release
return (total_turnaround_time, self.numOfTask)
