def put(self, *values, **attrs): """ If values is a namedtuple, then it is used as the record. Otherwise, a record is created from values or attrs. This then serializes record before putting it in FIFO. """ if not (len(values) == 1 and isinstance(values[0], self._class)): record = self.create(*values, **attrs) else: # FIXME: needs test. record = values[0] FIFO.put(self, self.dumps(record))
class NPScheduler: def __init__(self, N, policy='SJF'): self.N = N # number of time steps to schedule self.running = None self.clock = 0 self.policy = policy # instantiate the readyQueue, which may be a FIFO or MinHeap # you may need additional queues for # - tasks that have been added but not released yet # - tasks that have been completed # - the Gantt chart self.readyQueue = MinHeap() self.notReadyQueue = FIFO() self.doneQueue = list() self.ganttChart = list() def addTask(self, task): # if the release time of the new task is not in the future, then # put it in ready queue; otherwise, put into not-ready queue. # you may need to copy the scheduler policy into the task task.setPriorityScheme(self.policy) if task.releaseTime() <= self.clock: #release self.readyQueue.put(task) else: self.notReadyQueue.put(task) def dispatch(self, task): # dispatch here means assign the chosen task as the one to run # in the current time step. # the task should be removed from ready-queue by caller; # The task may be empty (None). # This method will make an entry into the Gantt chart and perform # bookkeeping, including ## - recording the last dispatched time of this task, # - increment the wait times of those tasks not scheduled # but in the ready queue if task is None: self.ganttChart.append(task) return self.ganttChart.append(task) task.decrRemaining() task.last_dispatched = self.clock for task_wait in self.readyQueue: task_wait.incrWaitTime() def releaseTasks(self): ''' this is called at the beginning of scheduling each time step to see if new tasks became ready to be released to ready queue, when their release time is no later than the current clock. ''' while True: r = self.notReadyQueue.head() # assuming the notReadyQueue outputs by release time if r is None or r.releaseTime() > self.clock: break r = self.notReadyQueue.get() r.setPriorityScheme(self.policy) self.readyQueue.put(r) def checkTaskCompletion(self): # if there is a current running task, check if it has just finished. # (i.e., decrement remaining time and see if it has more work to do. ?? # If so, perform bookkeeping for completing the task, ??? # - move task to done-queue, set its completion time and lastrun time ??? # set the scheduler running task to None, and return True # (so that a new task may be picked.) # but if not completed, return False. # If there is no current running task, also return True. if self.running is None: return True # your code here task = self.running if task.done(): task.setCompletionTime(self.clock) self.doneQueue.append(task) self.running = None return True # for future use if task is None: return True return False def schedule(self): # scheduler that handles nonpreemptive scheduling. # the policy such as RR, SJF, or FCFS is handled by the task as it # defines the attribute to compare (in its __iter__() method) # first, check if added but unreleased tasks may now be released # (i.e., added to ready queue) self.releaseTasks() if self.checkTaskCompletion() == False: # get task status # There is a current running task and it is not done yet! # the same task will continue running to its completion. # simply redispatch the current running task. ## redispatch the current running task, we have current ruuing task self.dispatch(self.running) else: # task completed or no running task. # get the next task from priority queue and dispatch it. # add None if len(self.readyQueue) == 0: self.dispatch(None) # pop finishing if len(self.readyQueue) > 0: task = self.readyQueue.get() # another turn to run self.running = task self.dispatch(self.running) def clockGen(self): for self.clock in range(self.N): # now run scheduler here self.schedule() yield self.clock def getSchedule(self): return '-'.join(map(str, self.ganttChart)) # return self.ganttChart def getThroughput(self): # calculate and return throughput as a tuple numerator = len(self.doneQueue) denominator = self.doneQueue[-1].completion return (numerator, denominator) def getWaitTime(self): # calculate and return numerator = sum([task.wait_time for task in self.doneQueue]) denominator = len(self.doneQueue) return (numerator, denominator) def getTurnaroundTime(self): # calculate the turnaround time in terms of a tuple with # separate turnaround times, #processes #print([task.turnaroundTime() for task in self.doneQueue]) numerator = sum([task.turnaroundTime() for task in self.doneQueue]) denominator = len(self.doneQueue) return (numerator, denominator)
class BatchPriorityQueue(MergingRecordFactory): """ A subclass of RecordFactory that maintains two on-disk FIFOs of records: * _pQ is ordered by records' priority_field lowest-first. get(max_priority) retrieves next record from this queue if its priority is lower than max_priority. * _dumpQ is periodically sorted by unique_field, merged by the accumulate function, and sorted again by priority_field in order to populate the priority queue. put() adds records to the dump queue. Calling get() also puts a copy of the record into the dump queue, so during the merge, the accumulate function will see the previous record and any new records. For this reason, records put back into the queue after processing a record gotten from the queue should present *only changes* relative to the gotten record, so the accumulate function can simply add them. """ def __init__(self, record_class, template, data_path, unique_key, priority_key, defaults={}, delimiter="|", cache_size=2**16): """ See RecordFactory for record_class, 'template', 'defaults', and 'delimiter'. 'data_path' and 'cache_size' are for _pQ and _dumpQ unique_key and priority_key are integer indexes into 'fields' indicating which to attributes of records to use as unique keys and priorities. """ RecordFactory.__init__(self, record_class, template, defaults, delimiter) self._unique_key = unique_key self._priority_key = priority_key self._cache_size = cache_size self._pQ_path = os.path.join(data_path, "pQ") self._pQ_sync = os.path.join(data_path, "pQ_sync") self._pQ = None self._dumpQ_path = os.path.join(data_path, "dumpQ") self._dumpQ_sync = os.path.join(data_path, "dumpQ_sync") self._dumpQ = FIFO(self._dumpQ_path, self._cache_size) # the next deserialized value to return via get self._next = None self._sync_pending_path = os.path.join(data_path, "sync_pending") self._sync_pending = Mutex(self._sync_pending_path) self._lock = Mutex(os.path.join(data_path, "lock_file")) def close(self): """ Acquires lock, then raises self.Syncing if a sync is in progress, otherwise closes internal FIFOs. """ self._lock.acquire() try: if not self._sync_pending.available(): raise self.Syncing if self._pQ: self._pQ.close() self._dumpQ.close() finally: self._lock.release() class NotYet(Exception): "next record excluded by max_priority" pass class Blocked(Exception): "another process has the mutex" pass class Syncing(Exception): "sync in progress" pass class ReadyToSync(Exception): "pQ empty but records exist in dumpQ" pass def get(self, max_priority=None, block=True): """ If block=False and cannot acquire lock, raises self.Blocked. If a sync is in progress, raises self.Syncing. If both _pQ and _dumpQ are empty, then raise Queue.Empty. If empty _pQ but not empty _dumpQ, raise self.ReadyToSync. If next item in _pQ has a priority less than max_priority, then pops it from queue and returns record. """ acquired = self._lock.acquire(block) if not acquired: raise self.Blocked try: if not self._sync_pending.available(): raise self.Syncing if self._pQ is None: self._pQ = FIFO(self._pQ_path, self._cache_size) if self._next is None: # instantiate next record without removing from pQ, raises # Queue.Empty when no lines in FIFO try: line = self._pQ.next() self._next = self.loads(line) except Queue.Empty: if len(self._dumpQ) == 0: raise Queue.Empty else: raise self.ReadyToSync if max_priority is None or \ self._next[self._priority_key] < max_priority: # Remove this line from _pQ and put into _dumpQ. There # should be no risk of this raising Queue.Empty. self._dumpQ.put(self._pQ.get()) ret_next = self._next self._next = None # This is only place that get() returns: return ret_next elif max_priority is not None: raise self.NotYet else: raise Exception("Should never get here.") finally: self._lock.release() def put(self, record=None, values=None, attrs=None, block=True): """ If record=None, then values or attrs is passed to self.create() to obtain a record. record is put into _dumpQ. If block=False and cannot acquire lock, raises self.Blocked. """ acquired = self._lock.acquire(block) if not acquired: raise self.Blocked if record is None: if values is not None: record = self.create(*values) elif attrs is not None: record = self.create(**attrs) else: raise Exception("put without record, values, or attrs") self._dumpQ.put(self.dumps(record)) self._lock.release() def sync(self, block=True): """ Removes all records from _dumpQ and _pQ and performs sort on unique_key, accumulate, sort on priority_key before putting all records into _pQ. If block=False and cannot acquire lock, raises self.Blocked. """ acquired = self._lock.acquire(block) if not acquired: raise self.Blocked try: acquired = self._sync_pending.acquire(block=False) if not acquired: raise self.Syncing # move queues to the side if self._pQ: self._pQ.close() self._dumpQ.close() os.rename(self._pQ_path, self._pQ_sync) os.rename(self._dumpQ_path, self._dumpQ_sync) # set pQ to None while syncing, and reopen dumpQ self._pQ = None self._dumpQ = FIFO(self._dumpQ_path, self._cache_size) # Release sync lock momentarily, so merger can acquire it. # Get is blocked by _lock, so it won't get confused. self._sync_pending.release() # launch a child to sort, accumulate, sort merger = self.start_merger() # loop until merger acquires _sync_pending while merger.is_alive() and self._sync_pending.available(): sleep(0.1) # now get back to normal operation return merger finally: self._lock.release() def start_merger(self): """ defines, instantiates, and starts a multiprocessing.Process for sorting, accumulating, and sorting _dumpQ into new _pQ """ class Merger(multiprocessing.Process): "manages the sort, accumulate, sort" name = "SortAccumulateSort" _queue = self def run(self): "executes sort, accumulate, sort" try: openlog(self.name, LOG_NDELAY | LOG_CONS | LOG_PID, LOG_LOCAL0) q = self._queue _sync_pending = Mutex(q._sync_pending_path) _sync_pending.acquire() pQ = FIFO(q._pQ_path, q._cache_size) assert len(pQ) == 0 pQ_data = os.path.join(q._pQ_sync, "data") dumpQ_data = os.path.join(q._dumpQ_sync, "data") in_files = [ os.path.join(pQ_data, cache_file) for cache_file in os.listdir(pQ_data) ] in_files += [ os.path.join(dumpQ_data, cache_file) for cache_file in os.listdir(dumpQ_data) ] start = time() # fast version of chained generators map( pQ.put, q.sort( q.accumulate( q.mergefiles(in_files, (q._unique_key, ))), (q._priority_key, ))) # slow version of this generator chain: #merged_lines = [] #for x in q.mergefiles(in_files, q._unique_key): # syslog("out of mergefiles: %s" % repr(x)) # merged_lines.append(x) #accumulated_lines = [] #for x in q.accumulate(merged_lines): # syslog("out of accumulator: %s" % repr(x)) # accumulated_lines.append(x) #for x in q.sort(accumulated_lines, q._priority_key): # syslog("out of sort: %s" % repr(x)) # pQ.put(x) end = time() pQ.close() syslog(LOG_INFO, "merge took %.1f seconds" % (end - start)) shutil.rmtree(q._pQ_sync) shutil.rmtree(q._dumpQ_sync) _sync_pending.release() except Exception, exc: map(lambda line: syslog(LOG_NOTICE, line), traceback.format_exc(exc).splitlines()) merger = Merger() merger.start() return merger
class NPScheduler: def __init__(self, N, policy='SJF'): self.N = N # number of time steps to schedule self.running = None self.clock = 0 self.policy = policy self.readyQueue = MinHeap() self.notReadyQueue = FIFO() self.ganttChart = FIFO() # instantiate the readyQueue, which may be a FIFO or MinHeap # you may need additional queues for # - tasks that have been added but not released yet # - tasks that have been completed # - the Gantt chart self.lastruntime = 0 self.numOfTask = 0 def addTask(self, task): task.scheme = self.policy # for statistic self.numOfTask = self.numOfTask + 1 if task.releaseTime() <= self.clock: self.readyQueue.put(task) else: self.notReadyQueue.put(task) # if the release time of the new task is not in the future, then # put it in ready queue; otherwise, put into not-ready queue. # you may need to copy the scheduler policy into the task def dispatch(self, task): r = self.readyQueue.head() if r == None: self.ganttChart.put(None) return self.running = task self.readyQueue.get().lastDispatchedTime = self.clock self.ganttChart.put(r) for i in self.readyQueue: i.incrWaitTime() # dispatch here means assign the chosen task as the one to run # in the current time step. # the task should be removed from ready-queue by caller; # The task may be empty (None). # This method will make an entry into the Gantt chart and perform # bookkeeping, including # - recording the last dispatched time of this task, # - increment the wait times of those tasks not scheduled # but in the ready queue def releaseTasks(self): ''' this is called at the beginning of scheduling each time step to see if new tasks became ready to be released to ready queue, when their release time is no later than the current clock. ''' while True: r = self.notReadyQueue.head() # assuming the notReadyQueue outputs by release time if r is None or r.releaseTime() > self.clock: break r = self.notReadyQueue.get() #r.setPriorityScheme(self.policy) little weird in my opinion r.scheme = self.policy if r: r.fifo_prio = self.clock self.readyQueue.put(r) def checkTaskCompletion(self): # if there is a current running task, check if it has just finished. # (i.e., decrement remaining time and see if it has more work to do. # If so, perform bookkeeping for completing the task, # - move task to done-queue, set its completion time and lastrun time # set the scheduler running task to None, and return True # (so that a new task may be picked.) # but if not completed, return False. # If there is no current running task, also return True. if self.running is None: return True # your code here self.running.decrRemaining() if self.running.done() == True: self.running.completionTime = self.clock self.running.lastDispatchedTime = self.clock #for statistic self.lastruntime = self.clock return True else: return False def schedule(self): # scheduler that handles nonpreemptive scheduling. # the policy such as RR, SJF, or FCFS is handled by the task as it # defines the attribute to compare (in its __iter__() method) # first, check if added but unreleased tasks may now be released # (i.e., added to ready queue) self.releaseTasks() if self.checkTaskCompletion() == False: self.ganttChart.put(self.running) for i in self.readyQueue: i.incrWaitTime() # There is a current running task and it is not done yet! # the same task will continue running to its completion. # simply redispatch the current running task. else: self.running = self.readyQueue.head() self.dispatch(self.readyQueue.head()) # task completed or no running task. # get the next task from priority queue and dispatch it. def clockGen(self): for self.clock in range(self.N): # now run scheduler here self.schedule() yield self.clock def getSchedule(self): return '-'.join(map(str, self.ganttChart)) # return self.ganttChart def getThroughput(self): return (self.numOfTask, self.lastruntime) # calculate and return throughput as a tuple def getWaitTime(self): # calculate and return s = set(self.ganttChart) total_waiting_time = 0 for i in s: if i: total_waiting_time = total_waiting_time + i.waitingTime return (total_waiting_time, self.numOfTask) def getTurnaroundTime(self): # calculate the turnaround time in terms of a tuple with # separate turnaround times, #processes s = set(self.ganttChart) total_turnaround_time = 0 for i in s: if i: total_turnaround_time = total_turnaround_time + i.completionTime - i.release return (total_turnaround_time, self.numOfTask)