def process_queue(self):
# loop until we read a 'quit' for worker, or an error
# When all workers are ready, start the tests
# If workers are not ready after at time, kill them and quit
start_time = time.time()
continue_test = True
while continue_test:
# Catch timeout
if (time.time() - start_time) > self.startup_timeout:
self.log.error("[!!] Exceeded startup_timeout. End all worker")
for worker in self.workers:
self.clean_up(worker)
# When the number of ready messages == workers, start test
if self.ready_counter == len(self.workers):
self.log.info("All Workers ready, start test")
# Send the message to start testing
for worker in self.workers:
worker.queue_from_boss.put(
message_queue.QuitMessage('BOSS', {'BeginTest': True}))
self.ready_counter = False
if self.quit_counter == len(self.workers):
self.log.info("All Workers quit, end program")
continue_test = False
continue
if not self.queue.empty():
try:
msg = self.queue.get(timeout=0.001)
msg.handle(self)
except Queue.Empty as e:
self.log.warning("Queue unexpectedly Empty. {}".format(e))
def test(self):
"""Toggle plug on and off, and check state after each operaiton
If a change fails to take place, terminate test"""
self.log.info('Run Test')
try:
# This is how many times we toggle the plug
for iteration in range(1, self.cycles + 1):
self.log.info(
("=====Plug: {:>2}, Iteration:{:>3}, Error Count:{:>3}===="
).format(self.plug_id, iteration, self.error_count))
self.log.info('Power off')
self.queue_to_boss.put(
message_queue.LogMessage(self.plug_id, "INFO", "off"))
# Change state
try:
self.netbooter.plug_off([self.plug_id])
# Read current state
status = self.netbooter.port_status([self.plug_id])
# If plug not in expected state, count error
if status != [0]:
self.log.critical("PLUG {} NOT OFF.".format(
self.plug_id))
# send quit message
self.error_count += 1
except retrying.RetryError as e:
self.log.critical("FAILED. plug_off() exception:{}".format(
str(e)))
self.error_count += 1
self.log.info('Power on')
self.queue_to_boss.put(
message_queue.LogMessage(self.plug_id, "INFO", "on"))
# Change state
try:
self.netbooter.plug_on([self.plug_id])
# If plug not in expected state, count error
status = self.netbooter.port_status([self.plug_id])
# If plug not in expected state, count error
if status != [1]:
self.log.critical("PLUG {} NOT ON.".format(
self.plug_id))
# send quit message
self.error_count += 1
except retrying.RetryError as e:
self.log.critical("FAILED plug_on(). exception:{}".format(
str(e)))
self.error_count += 1
self.queue_to_boss.put(
message_queue.QuitMessage(self.plug_id, self.error_count))
except Exception as e:
self.log.exception("Exception occurred: {}".format(e))
raise
def run(self):
"""Initialize Logger and Netbooter,
message Boss it is ready,
and poll for a start from the boss"""
#
# Setup log
#
log_file_name = "{}-worker-plug{}.log".format(self.netbooter_ipv4,
self.plug_id)
rh = logging.FileHandler(log_file_name)
fmt = logging.Formatter("%(asctime)s (%(levelname)-8s): %(message)s")
rh.setFormatter(fmt)
self.log = logging.getLogger()
self.log.setLevel('INFO')
self.log.addHandler(rh)
self.log.info("Log Initialized")
#
# Setup Netbooter snmp object
#
self.log.info('Creating instance of Netbooter')
self.netbooter = netbooter.NetBooter(host=self.netbooter_ipv4,
logger=self.log)
#
# Empty Multiprocessing queue
#
while not self.queue_from_boss.empty():
self.log.info('Inside clear() loop -- getting items from queue')
self.queue_from_boss.get()
#
# Tell boss Worker is ready
#
self.queue_to_boss.put(
message_queue.StatusMessage(self.plug_id, {'ready': True}))
#
# Poll messages from Boss to start test
#
self.process_queue()
#
# send quit message
#
self.log.info("Send Quit to Boss")
self.queue_to_boss.put(
message_queue.QuitMessage(self.plug_id, self.error_count))
def test(self):
"""Do command
If command fails, terminate test"""
self.log.info('Run Test')
try:
# This is how many times we run the command
for iteration in range(1, self.cycles + 1):
self.log.info((
"=====Worker: {:>2}, Iteration:{:>3}, Error Count:{:>3}===="
).format(self.worker_id, iteration, self.error_count))
self.log.info('Run Command')
self.queue_to_boss.put(
message_queue.LogMessage(self.worker_id, "INFO",
"run{: }".format(iteration)))
try:
self.log.info("ipv4:{}".format(self.ipv4))
stdout_value, elapsed_time = ping(self.ipv4,
4,
timeout=0.2)
self.log.info("stdout_value: {}".format(stdout_value))
if stdout_value != 0:
self.log.critical("Failed")
self.error_count += 1
except retrying.RetryError as e:
self.log.critical(
"Retry FAILED. Iteration:{:>3}, Exception:{}".format(
iteration, str(e)))
message_queue.LogMessage(
self.worker_id, "ERROR",
"Retry FAILED. Iteration:{:>3}, Exception:{}".format(
iteration, str(e)))
self.error_count += 1
# End For loop
self.queue_to_boss.put(
message_queue.QuitMessage(self.worker_id, self.error_count))
except Exception as e:
self.log.exception("Exception occurred: {}".format(e))
raise
def run(self):
""""Initialize Logger and notify Boss that Worker is ready"""
#
# Setup log
#
log_file_name = "Worker_{}.log".format(self.worker_id)
rh = logging.FileHandler(log_file_name)
fmt = logging.Formatter("%(asctime)s (%(levelname)-8s): %(message)s")
rh.setFormatter(fmt)
self.log = logging.getLogger()
self.log.setLevel('DEBUG')
self.log.addHandler(rh)
self.log.info("Log Initialized")
#
# Empty Multiprocessing queue
#
while not self.queue_from_boss.empty():
self.log.debug('Inside clear() loop -- getting items from queue')
self.queue_from_boss.get()
#
# Tell boss Worker is ready
#
self.log.info("Send Ready o Boss")
self.queue_to_boss.put(
message_queue.StatusMessage(self.worker_id, {'Ready': True}))
#
# Poll messages from Boss to start test
#
self.process_queue()
#
# Tell boss Worker is complete
#
self.log.info("Send Complete to Boss")
self.queue_to_boss.put(
message_queue.StatusMessage(self.worker_id, {'Complete': True}))
#
# send quit message
#
self.log.info("Send Quit to Boss")
self.queue_to_boss.put(
message_queue.QuitMessage(self.worker_id, self.error_count))