def __run_tasks(self): already_waiting = lu.waiting_tasks() self.log("Starting the programs") for e in self.executables: try: e.execute() except: raise Exception("Executable failed: %s" % e) self.log("Sleeping until tasks are ready for release...") start = time.clock() while (lu.waiting_tasks() - already_waiting) < len(self.executables): if time.clock() - start > 30.0: raise Exception("Too much time has passed waiting for tasks!") time.sleep(1) # Overhead tracer must be started right after release or overhead # measurements will be full of irrelevant records if self.overhead_trace: self.log("Starting overhead trace") self.overhead_trace.start_tracing() self.log("Releasing %d tasks" % len(self.executables)) released = lu.release_tasks() ret = True if released != len(self.executables): # Some tasks failed to release, kill all tasks and fail # Need to re-release non-released tasks before we can kill them though self.log("Failed to release {} tasks! Re-releasing and killing".format( len(self.executables) - released, len(self.executables))) time.sleep(5) released = lu.release_tasks() self.log("Re-released %d tasks" % released) time.sleep(5) self.log("Killing all tasks") map(methodcaller('kill'), self.executables) ret = False self.log("Waiting for program to finish...") for e in self.executables: if not e.wait(): ret = False # And it must be stopped here for the same reason if self.overhead_trace: self.log("Stopping overhead trace") self.overhead_trace.stop_tracing() if not ret: raise ExperimentFailed(self.name)
def __run_tasks(self): self.log("Starting %d tasks" % len(self.executables)) for i,e in enumerate(self.executables): try: e.execute() except: raise Exception("Executable failed: %s" % e) self.log("Sleeping until tasks are ready for release...") start = time.time() while lu.waiting_tasks() < len(self.executables): if time.time() - start > 30.0: s = "waiting: %d, submitted: %d" %\ (lu.waiting_tasks(), len(self.executables)) raise Exception("Too much time spent waiting for tasks! %s" % s) time.sleep(1) # Exact tracers (like overheads) must be started right after release or # measurements will be full of irrelevant records self.log("Starting %d released tracers" % len(self.exact_tracers)) map(methodcaller('start_tracing'), self.exact_tracers) time.sleep(1) try: self.log("Releasing %d tasks" % len(self.executables)) released = lu.release_tasks() if released != len(self.executables): # Some tasks failed to release, kill all tasks and fail # Need to release non-released tasks before they can be killed raise Exception("Released %s tasks, expected %s tasks" % (released, len(self.executables))) self.log("Waiting for program to finish...") for e in self.executables: if not e.wait(): raise Exception("Executable %s failed to complete!" % e) finally: # And these must be stopped here for the same reason self.log("Stopping exact tracers") map(methodcaller('stop_tracing'), self.exact_tracers)
def __wait_for_ready(self): self.log("Sleeping until tasks are ready for release...") wait_start = time.time() num_ready = lu.waiting_tasks() while num_ready < len(self.executables): # Quit if too much time passes without a task becoming ready if time.time() - wait_start > 180.0: s = "waiting: %d, submitted: %d" %\ (lu.waiting_tasks(), len(self.executables)) raise Exception("Too much time spent waiting for tasks! %s" % s) time.sleep(1) # Quit if any tasks fail self.__check_tasks_status() # Reset the waiting time whenever more tasks become ready now_ready = lu.waiting_tasks() if now_ready != num_ready: wait_start = time.time() num_ready = now_ready
def __wait_for_ready(self): self.log("Sleeping until tasks are ready for release...") wait_start = time.time() num_ready = lu.waiting_tasks() while num_ready < len(self.executables): # Quit if too much time passes without a task becoming ready if time.time() - wait_start > 10.0: s = "waiting: %d, submitted: %d" %\ (lu.waiting_tasks(), len(self.executables)) raise Exception("Too much time spent waiting for tasks! %s" % s) time.sleep(1) # Quit if any tasks fail self.__check_tasks_status() # Reset the waiting time whenever more tasks become ready now_ready = lu.waiting_tasks() if now_ready != num_ready: wait_start = time.time() num_ready = now_ready
def __try_kill_all(self): try: if lu.waiting_tasks(): released = lu.release_tasks() self.log("Re-released %d tasks" % released) time.sleep(1) self.log("Killing all tasks") for e in self.executables: try: e.kill() except: pass time.sleep(1) except: self.log("Failed to kill all tasks.")