class SetForceQueueTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def set_force_helper(self, force): success = False for x in range(10): if self._e.set_force(force) == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully run set_force" # Test that the FORCE value in the JSON can be correctly toggled def test_set_force(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}") self.set_force_helper(False) val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertFalse(val.get("FORCE")) self.set_force_helper(True) val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertTrue(val.get("FORCE")) self.set_force_helper(False) val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertFalse(val.get("FORCE"))
class RemoveFromQueueSuccessTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def remove_from_queue_helper(self): success = False for x in range(10): if self._e.remove_from_queue(True) == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully remove the node from the queue" # Test that marking a node as successful moves it to the COMPLETED list @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_success(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(1, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("COMPLETED")[0]["ID"]) self.assertEqual("DONE", val.get("COMPLETED")[0]["STATUS"]) # Test that marking a node as successful when it is still in the queue doesn't move it to the COMPLETED list (but does take out the first entry) @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_success_still_in_queue(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that calling this method when the node isn't at the front of the # doesn't change the queue @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_success_not_front_of_queue(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("PROCESSING", val.get("QUEUED")[0]["STATUS"])
class RemoveFromQueueSuccessTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def remove_from_queue_helper(self): success = False for x in range(10): if self._e.remove_from_queue(True) == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully remove the node from the queue" # Test that marking a node as successful moves it to the COMPLETED list @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_success(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(1, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("COMPLETED")[0]["ID"]) self.assertEqual("DONE", val.get("COMPLETED")[0]["STATUS"]) # Test that marking a node as successful when it is still in the queue doesn't move it to the COMPLETED list (but does take out the first entry) @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_success_still_in_queue(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that calling this method when the node isn't at the front of the # doesn't change the queue @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_success_not_front_of_queue(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("PROCESSING", val.get("QUEUED")[0]["STATUS"])
class SetForceQueueTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def set_force_helper(self, force): success = False for x in range(10): if self._e.set_force(force) == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully run set_force" # Test that the FORCE value in the JSON can be correctly toggled def test_set_force(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}" ) self.set_force_helper(False) val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertFalse(val.get("FORCE")) self.set_force_helper(True) val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertTrue(val.get("FORCE")) self.set_force_helper(False) val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertFalse(val.get("FORCE"))
def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0
class AddToQueueTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def add_to_queue(self): success = False for x in range(10): if self._e.add_to_queue() == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully add the node to the queue" # Test that adding to an empty queue simply adds the new node to the QUEUED array @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that adding a node when its already in the queue in the processing state adds the new node to the QUEUED array @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_already_processing(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node when its already in the queue in the queued state doesn't add the new node to the QUEUED array @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_already_queued(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node when its not already in the queue in the queued state adds the node (with more nodes in the queue already) @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_with_other_nodes_and_already_queued(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(3, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[2]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[2]["STATUS"]) # Test that adding the node succeeds for a non-empty queue that its not already in @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_with_other_nodes(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node to an empty queue with an unresponsive node doesn't add the unresponsive node to the queue @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue_and_other_node_unresponsive(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": []}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that adding a node to an empty queue with an failed node adds the failed node to the front of the queue @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue_and_other_node_failed(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[0]["STATUS"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node to an empty queue with this node marked as failed node only adds the node to the queue once @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue_and_this_node_failed(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that adding a node when it's marked as completed removes the node from the completed list @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_while_completed(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node that's marked as errored when it's not at the front of the queue doesn't change the errored state @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_while_errored(self): self.set_initial_val( "{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}" ) self.add_to_queue() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
logfile = "/var/log/clearwater-queue-manager/queue_operation.log" logging.basicConfig(filename=logfile, format="%(asctime)s.%(msecs)03d UTC %(levelname)s %(filename)s:%(lineno)d: %(message)s", datefmt="%d-%m-%Y %H:%M:%S", level=logging.DEBUG) operation = sys.argv[1] local_ip = sys.argv[2] site = sys.argv[3] node_type = sys.argv[4] clearwater_key = sys.argv[5] queue_key = sys.argv[6] logging.info("Using etcd key %s" % queue_key) queue_syncer = EtcdSynchronizer(NullPlugin(queue_key), local_ip, site, clearwater_key, node_type) if operation == "add": logging.debug("Adding %s to queue to restart" % (local_ip + "-" + node_type)) while queue_syncer.add_to_queue() != WriteToEtcdStatus.SUCCESS: sleep(2) logging.debug("Node successfully added to restart queue") elif operation == "remove_success": logging.debug("Removing %s from front of queue" % (local_ip + "-" + node_type)) while queue_syncer.remove_from_queue(True) != WriteToEtcdStatus.SUCCESS: sleep(2) logging.debug("Node successfully removed")
def main(args): syslog.openlog("queue-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise local_ip = arguments['--local-ip'] local_site = arguments['--local-site'] etcd_key = arguments['--etcd-key'] node_type = arguments['--node-type'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) wait_plugin_complete = arguments['--wait-plugin-complete'] stdout_err_log = os.path.join(log_dir, "queue-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-queue-mgr") logging_config.configure_logging(log_level, log_dir, "queue-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("queue-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/" plugins = load_plugins_in_dir(plugins_dir, PluginParams(wait_plugin_complete=wait_plugin_complete)) plugins.sort(key=lambda x: x.key()) synchronizers = [] threads = [] for plugin in plugins: syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key, node_type) syncer.start_thread() synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) utils.install_sigterm_handler(synchronizers) while any([thr.isAlive() for thr in threads]): for thr in threads: if thr.isAlive(): thr.join(1) while not utils.should_quit: sleep(1) _log.info("Clearwater Queue Manager shutting down") pdlogs.EXITING.log() syslog.closelog()
def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0
class RemoveFromQueueFailureTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def remove_from_queue_helper(self): success = False for x in range(10): if self._e.remove_from_queue(False) == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully remove the node from the queue" # Tests that marking a node as failed moves it to the ERRORED list @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(0, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"]) self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"]) # Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure_no_force(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(0, len(val.get("QUEUED"))) # Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure_force(self): self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.4-node\",\"STATUS\":\"UNRESPONSIVE\"}, {\"ID\":\"10.0.0.5-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(2, len(val.get("ERRORED"))) self.assertEqual(2, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) @patch("etcd.Client", new=EtcdFactory) # Tests that marking a node as failed when it is in the queued list but not the next node does move it to the ERRORED list def test_remove_from_queue_after_failure_not_next_in_queue_force(self): self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"]) self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"]) # Tests that marking a node as failed when it isn't the front of the queue doesn't change the JSON @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure_not_front_of_queue_force(self): self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.remove_from_queue_helper() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED")))
class RemoveFromQueueFailureTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def remove_from_queue_helper(self): success = False for x in range(10): if self._e.remove_from_queue(False) == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully remove the node from the queue" # Tests that marking a node as failed moves it to the ERRORED list @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}" ) self.remove_from_queue_helper() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(0, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"]) self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"]) # Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure_no_force(self): self.set_initial_val( "{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}" ) self.remove_from_queue_helper() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(0, len(val.get("QUEUED"))) # Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure_force(self): self.set_initial_val( "{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.4-node\",\"STATUS\":\"UNRESPONSIVE\"}, {\"ID\":\"10.0.0.5-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}" ) self.remove_from_queue_helper() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(2, len(val.get("ERRORED"))) self.assertEqual(2, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) @patch("etcd.Client", new=EtcdFactory) # Tests that marking a node as failed when it is in the queued list but not the next node does move it to the ERRORED list def test_remove_from_queue_after_failure_not_next_in_queue_force(self): self.set_initial_val( "{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}" ) self.remove_from_queue_helper() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"]) self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"]) # Tests that marking a node as failed when it isn't the front of the queue doesn't change the JSON @patch("etcd.Client", new=EtcdFactory) def test_remove_from_queue_after_failure_not_front_of_queue_force(self): self.set_initial_val( "{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}" ) self.remove_from_queue_helper() val = json.loads( self._e._client.read( "/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED")))
log_format = logging.Formatter(fmt="%(asctime)s.%(msecs)03d UTC %(levelname)s %(filename)s:%(lineno)d: %(message)s", datefmt="%d-%m-%Y %H:%M:%S") log_format.converter = gmtime handler.setFormatter(log_format) _log.addHandler(handler) operation = sys.argv[1] local_ip = sys.argv[2] site = sys.argv[3] node_type = sys.argv[4] clearwater_key = sys.argv[5] queue_key = sys.argv[6] _log.info("Using etcd key %s" % queue_key) queue_syncer = EtcdSynchronizer(NullPlugin(queue_key), local_ip, site, clearwater_key, node_type) if operation == "add": _log.debug("Adding %s to queue to restart" % (local_ip + "-" + node_type)) while queue_syncer.add_to_queue() != WriteToEtcdStatus.SUCCESS: sleep(2) _log.debug("Node successfully added to restart queue") elif operation == "remove_success": _log.debug("Removing %s from front of queue" % (local_ip + "-" + node_type)) while queue_syncer.remove_from_queue(True) != WriteToEtcdStatus.SUCCESS: sleep(2) _log.debug("Node successfully removed")
def setUp(self): alarms_patch.start() self._p = TestNoTimerDelayPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
def main(args): syslog.openlog("queue-manager", syslog.LOG_PID) pdlogs.STARTUP.log() try: arguments = docopt(__doc__, argv=args) except DocoptExit: pdlogs.EXITING_BAD_CONFIG.log() raise local_ip = arguments['--local-ip'] local_site = arguments['--local-site'] etcd_key = arguments['--etcd-key'] node_type = arguments['--node-type'] log_dir = arguments['--log-directory'] log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG) wait_plugin_complete = arguments['--wait-plugin-complete'] stdout_err_log = os.path.join(log_dir, "queue-manager.output.log") if not arguments['--foreground']: utils.daemonize(stdout_err_log) # Process names are limited to 15 characters, so abbreviate prctl.prctl(prctl.NAME, "cw-queue-mgr") logging_config.configure_logging(log_level, log_dir, "queue-manager", show_thread=True) # urllib3 logs a WARNING log whenever it recreates a connection, but our # etcd usage does this frequently (to allow watch timeouts), so deliberately # ignore this log urllib_logger = logging.getLogger('urllib3') urllib_logger.setLevel(logging.ERROR) utils.install_sigusr1_handler("queue-manager") # Drop a pidfile. We must keep a reference to the file object here, as this keeps # the file locked and provides extra protection against two processes running at # once. pidfile_lock = None try: pidfile_lock = utils.lock_and_write_pid_file( arguments['--pidfile']) # noqa except IOError: # We failed to take the lock - another process is already running exit(1) plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/" plugins = load_plugins_in_dir( plugins_dir, PluginParams(wait_plugin_complete=wait_plugin_complete)) plugins.sort(key=lambda x: x.key()) synchronizers = [] threads = [] # Load the plugins, but don't start them until we've installed the SIGTERM # handler, as that handler will gracefully shut down any running # synchronizers on receiving a SIGTERM for plugin in plugins: syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key, node_type) synchronizers.append(syncer) threads.append(syncer.thread) _log.info("Loaded plugin %s" % plugin) utils.install_sigterm_handler(synchronizers) # Now start the plugin threads for syncer in synchronizers: syncer.start_thread() _log.info("Started thread for plugin %s" % syncer._plugin) while any([thr.isAlive() for thr in threads]): for thr in threads: if thr.isAlive(): thr.join(1) while not utils.should_quit: sleep(1) _log.info("Clearwater Queue Manager shutting down") pdlogs.EXITING.log() syslog.closelog()
class AddToQueueTest(BaseQueueTest): @patch("etcd.Client", new=EtcdFactory) def setUp(self): alarms_patch.start() self._p = TestPlugin() self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node") self._e.WAIT_FOR_TIMER_POP = 0 def add_to_queue(self): success = False for x in range(10): if self._e.add_to_queue() == WriteToEtcdStatus.SUCCESS: success = True break sleep(1) if not success: print "Failed to successfully add the node to the queue" # Test that adding to an empty queue simply adds the new node to the QUEUED array @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that adding a node when its already in the queue in the processing state adds the new node to the QUEUED array @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_already_processing(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node when its already in the queue in the queued state doesn't add the new node to the QUEUED array @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_already_queued(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node when its not already in the queue in the queued state adds the node (with more nodes in the queue already) @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_with_other_nodes_and_already_queued(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(3, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[2]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[2]["STATUS"]) # Test that adding the node succeeds for a non-empty queue that its not already in @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_with_other_nodes(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node to an empty queue with an unresponsive node doesn't add the unresponsive node to the queue @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue_and_other_node_unresponsive(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": []}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that adding a node to an empty queue with an failed node adds the failed node to the front of the queue @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue_and_other_node_failed(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[0]["STATUS"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node to an empty queue with this node marked as failed node only adds the node to the queue once @patch("etcd.Client", new=EtcdFactory) def test_add_to_empty_queue_and_this_node_failed(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(1, len(val.get("QUEUED"))) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"]) # Test that adding a node when it's marked as completed removes the node from the completed list @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_while_completed(self): self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(0, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"]) # Test that adding a node that's marked as errored when it's not at the front of the queue doesn't change the errored state @patch("etcd.Client", new=EtcdFactory) def test_add_to_queue_while_errored(self): self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}") self.add_to_queue() val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value) self.assertEqual(1, len(val.get("ERRORED"))) self.assertEqual(0, len(val.get("COMPLETED"))) self.assertEqual(2, len(val.get("QUEUED"))) self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"]) self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"]) self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])