def empty_queue(self):
"""Constantly empties the queue if there is something in it, or run the base mission otherwise."""
# TODO CHECK IF ALL MARKERS PLACED
self.start_time = self.nh.get_time()
self._monitor_timeouts()
while len(self.tree) != 0:
TimeoutManager.generate_timeouts(self._get_time_left(), self.missions_left)
m = yield self._get_closest_mission()
yield self._run_mission(m)
yield self.nh.sleep(1)
fprint("MISSIONS COMPLETE, TOTAL RUN TIME: {}".format((self.nh.get_time() - self.start_time).to_sec()), msg_color="green")
fprint("MISSIONS COMPLETE, TOTAL POINTS: {}".format((self.points), msg_color="green"))
def _err_mission(self, err):
if hasattr(
err, "type"
) and err.type == defer.CancelledError: # This means there was a timeout
fprint(self.current_mission.name,
msg_color="red",
title="MISSION TIMEOUT")
yield self.publish("TimingOut")
if (TimeoutManager.can_repeat(
self.missions_left, self._get_time_left(),
self.current_mission)) or len(self.missions_left) == 1:
yield self.publish("Retrying")
self.failed = False
self.current_mission.timeout = self.current_mission.min_time
yield self._run_mission(self.current_mission, redo=True)
defer.returnValue(False)
else:
fprint(err,
msg_color="red",
title="{} MISSION ERROR: ".format(
self.current_mission.name))
yield self.publish("Failing")
yield self.current_mission.safe_exit(self.navigator, err, self,
self.module)
self._mission_complete()
def _err_mission(self, err):
if err.type == MissingPerceptionObject or err.type == defer.CancelledError:
fprint("{} thrown".format(err.type), msg_color="red",
title="{} MISSION ERROR: ".format(self.current_mission.name))
if TimeoutManager.can_repeat(self.missions_left, self._get_time_left(), self.current_mission):
if err.type == MissingPerceptionObject:
new_obj_found = yield self._run_base_mission(self.current_mission.item_dep)
if new_obj_found:
self.pub_msn_info.publish(String("Retrying Mission {}".format(self.current_mission.name)))
self.current_mission.timeout = self.current_mission.min_time
yield self._run_mission(self.current_mission, redo=True)
defer.returnValue(False)
else:
fprint("NEW OBJECT NOT FOUND, KILLING MISSION", msg_color="red")
else:
self.pub_msn_info.publish(String("Retrying Mission {}".format(self.current_mission.name)))
self.current_mission.timeout = self.current_mission.min_time
yield self._run_mission(self.current_mission)
defer.returnValue(False)
else:
fprint(err, msg_color="red", title="{} MISSION ERROR: ".format(self.current_mission.name))
self.pub_msn_info.publish(String("Failing Mission {}".format(self.current_mission.name)))
yield self.current_mission.safe_exit(self.navigator, err, self, self.module)
self._mission_complete(self.current_mission)
def run_test(self, test):
stime = datetime.now()
cmd = self.prepare_run(test)
if cmd == None: return 89, stime.strftime("%Y-%m-%d-%H:%M:%S.%f")
# open log files
self.open_logfile(test)
tattr = self.tconfig.get_test_attributes(test) #= returns dict
#-- let the user know test is about to start
self.logger.debug('Executing {0} on remote node(s)'.format(cmd))
self.logger.info(
'{0} started on {1} node(s) at {2}. It might take up to {3}s.'.
format(test, len(self.target), stime, tattr['timeout']))
tm = TimeoutManager(
self.logger,
timedelta(seconds=int(self.mattr['tty_progress_interval'])),
timedelta(seconds=int(self.mattr['watch_output_progress'])))
if self.mattr['watch_output_progress'] == '0':
tm_thread = start_thread(tm.tick_thread)
else:
tm_thread = start_thread(tm.alert_thread)
#-- initiate the timeout manager with known nodes
if self.proxy:
tm.activity(self.proxy.node)
else:
for node in self.target:
tm.activity(node)
#-- let's execute!!
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
threads = [start_thread(self.store_node_stdout, proc.stdout, tm)]
threads += [start_thread(self.print_log_stderr, proc.stderr, tm)]
proc.wait()
for t in threads:
t.join()
tm.done()
while tm_thread.isAlive():
self.logger.debug('Waiting for the timeout manager thread to end.')
time.sleep(0.05)
etime = datetime.now()
self.logger.info(
'{0} ended on {1} node(s) at {2}, rc= {3}, elapsed time: {4}'.
format(test, len(self.target), etime, proc.returncode,
etime - stime))
self.close_logfile()
if not self.mgmt_mode or (self.proxy and self.proxy.type == 'mgt'):
self.proxy.rc = proc.returncode
del tm # delete the TimeoutMamager
return 0, stime.strftime("%Y-%m-%d %H:%M:%S.%f")
def empty_queue(self):
"""Constantly empties the queue if there is something in it, or run the base mission otherwise."""
# TODO CHECK IF ALL MARKERS PLACED
self.start_time = self.nh.get_time()
self._monitor_timeouts()
while len(self.tree) != 0:
time_left = self._get_time_left()
real_time_left = self._get_real_timeleft()
if real_time_left < 0:
break
TimeoutManager.generate_timeouts(time_left, real_time_left,
self.missions_left)
m = yield self._get_closest_mission()
print m.name
yield self._run_mission(m)
fprint("MISSIONS COMPLETE, TOTAL RUN TIME: {}".format(
(self.nh.get_time() - self.start_time).to_sec()),
msg_color="green")
fprint("MISSIONS COMPLETE, TOTAL POINTS: {}".format((self.points),
msg_color="green"))
def _err_mission(self, err):
if hasattr(err, "type") and err.type == defer.CancelledError: # This means there was a timeout
fprint(self.current_mission.name, msg_color="red", title="MISSION TIMEOUT")
yield self.publish("TimingOut")
if (TimeoutManager.can_repeat(self.missions_left, self._get_time_left(), self.current_mission)) or len(self.missions_left) == 1:
yield self.publish("Retrying")
self.failed = False
self.current_mission.timeout = self.current_mission.min_time
yield self._run_mission(self.current_mission, redo=True)
defer.returnValue(False)
else:
fprint(err, msg_color="red", title="{} MISSION ERROR: ".format(self.current_mission.name))
yield self.publish("Failing")
yield self.current_mission.safe_exit(self.navigator, err, self, self.module)
self._mission_complete()
