def __run_tasks(self):
already_waiting = lu.waiting_tasks()
self.log("Starting the programs")
for e in self.executables:
try:
e.execute()
except:
raise Exception("Executable failed: %s" % e)
self.log("Sleeping until tasks are ready for release...")
start = time.clock()
while (lu.waiting_tasks() - already_waiting) < len(self.executables):
if time.clock() - start > 30.0:
raise Exception("Too much time has passed waiting for tasks!")
time.sleep(1)
# Overhead tracer must be started right after release or overhead
# measurements will be full of irrelevant records
if self.overhead_trace:
self.log("Starting overhead trace")
self.overhead_trace.start_tracing()
self.log("Releasing %d tasks" % len(self.executables))
released = lu.release_tasks()
ret = True
if released != len(self.executables):
# Some tasks failed to release, kill all tasks and fail
# Need to re-release non-released tasks before we can kill them though
self.log("Failed to release {} tasks! Re-releasing and killing".format(
len(self.executables) - released, len(self.executables)))
time.sleep(5)
released = lu.release_tasks()
self.log("Re-released %d tasks" % released)
time.sleep(5)
self.log("Killing all tasks")
map(methodcaller('kill'), self.executables)
ret = False
self.log("Waiting for program to finish...")
for e in self.executables:
if not e.wait():
ret = False
# And it must be stopped here for the same reason
if self.overhead_trace:
self.log("Stopping overhead trace")
self.overhead_trace.stop_tracing()
if not ret:
raise ExperimentFailed(self.name)
def __run_tasks(self):
self.log("Starting %d tasks" % len(self.executables))
for i,e in enumerate(self.executables):
try:
e.execute()
except:
raise Exception("Executable failed: %s" % e)
self.log("Sleeping until tasks are ready for release...")
start = time.time()
while lu.waiting_tasks() < len(self.executables):
if time.time() - start > 30.0:
s = "waiting: %d, submitted: %d" %\
(lu.waiting_tasks(), len(self.executables))
raise Exception("Too much time spent waiting for tasks! %s" % s)
time.sleep(1)
# Exact tracers (like overheads) must be started right after release or
# measurements will be full of irrelevant records
self.log("Starting %d released tracers" % len(self.exact_tracers))
map(methodcaller('start_tracing'), self.exact_tracers)
time.sleep(1)
try:
self.log("Releasing %d tasks" % len(self.executables))
released = lu.release_tasks()
if released != len(self.executables):
# Some tasks failed to release, kill all tasks and fail
# Need to release non-released tasks before they can be killed
raise Exception("Released %s tasks, expected %s tasks" %
(released, len(self.executables)))
self.log("Waiting for program to finish...")
for e in self.executables:
if not e.wait():
raise Exception("Executable %s failed to complete!" % e)
finally:
# And these must be stopped here for the same reason
self.log("Stopping exact tracers")
map(methodcaller('stop_tracing'), self.exact_tracers)
def __wait_for_ready(self):
self.log("Sleeping until tasks are ready for release...")
wait_start = time.time()
num_ready = lu.waiting_tasks()
while num_ready < len(self.executables):
# Quit if too much time passes without a task becoming ready
if time.time() - wait_start > 180.0:
s = "waiting: %d, submitted: %d" %\
(lu.waiting_tasks(), len(self.executables))
raise Exception("Too much time spent waiting for tasks! %s" % s)
time.sleep(1)
# Quit if any tasks fail
self.__check_tasks_status()
# Reset the waiting time whenever more tasks become ready
now_ready = lu.waiting_tasks()
if now_ready != num_ready:
wait_start = time.time()
num_ready = now_ready
def __wait_for_ready(self):
self.log("Sleeping until tasks are ready for release...")
wait_start = time.time()
num_ready = lu.waiting_tasks()
while num_ready < len(self.executables):
# Quit if too much time passes without a task becoming ready
if time.time() - wait_start > 10.0:
s = "waiting: %d, submitted: %d" %\
(lu.waiting_tasks(), len(self.executables))
raise Exception("Too much time spent waiting for tasks! %s" %
s)
time.sleep(1)
# Quit if any tasks fail
self.__check_tasks_status()
# Reset the waiting time whenever more tasks become ready
now_ready = lu.waiting_tasks()
if now_ready != num_ready:
wait_start = time.time()
num_ready = now_ready
def __try_kill_all(self):
try:
if lu.waiting_tasks():
released = lu.release_tasks()
self.log("Re-released %d tasks" % released)
time.sleep(1)
self.log("Killing all tasks")
for e in self.executables:
try:
e.kill()
except:
pass
time.sleep(1)
except:
self.log("Failed to kill all tasks.")
def __try_kill_all(self):
try:
if lu.waiting_tasks():
released = lu.release_tasks()
self.log("Re-released %d tasks" % released)
time.sleep(1)
self.log("Killing all tasks")
for e in self.executables:
try:
e.kill()
except:
pass
time.sleep(1)
except:
self.log("Failed to kill all tasks.")
