class SetForceQueueTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
self._e.WAIT_FOR_TIMER_POP = 0
def set_force_helper(self, force):
success = False
for x in range(10):
if self._e.set_force(force) == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully run set_force"
# Test that the FORCE value in the JSON can be correctly toggled
def test_set_force(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}")
self.set_force_helper(False)
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertFalse(val.get("FORCE"))
self.set_force_helper(True)
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertTrue(val.get("FORCE"))
self.set_force_helper(False)
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertFalse(val.get("FORCE"))
class RemoveFromQueueSuccessTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
self._e.WAIT_FOR_TIMER_POP = 0
def remove_from_queue_helper(self):
success = False
for x in range(10):
if self._e.remove_from_queue(True) == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully remove the node from the queue"
# Test that marking a node as successful moves it to the COMPLETED list
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_success(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(1, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("COMPLETED")[0]["ID"])
self.assertEqual("DONE", val.get("COMPLETED")[0]["STATUS"])
# Test that marking a node as successful when it is still in the queue doesn't move it to the COMPLETED list (but does take out the first entry)
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_success_still_in_queue(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that calling this method when the node isn't at the front of the
# doesn't change the queue
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_success_not_front_of_queue(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("PROCESSING", val.get("QUEUED")[0]["STATUS"])
class RemoveFromQueueSuccessTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
self._e.WAIT_FOR_TIMER_POP = 0
def remove_from_queue_helper(self):
success = False
for x in range(10):
if self._e.remove_from_queue(True) == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully remove the node from the queue"
# Test that marking a node as successful moves it to the COMPLETED list
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_success(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(1, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("COMPLETED")[0]["ID"])
self.assertEqual("DONE", val.get("COMPLETED")[0]["STATUS"])
# Test that marking a node as successful when it is still in the queue doesn't move it to the COMPLETED list (but does take out the first entry)
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_success_still_in_queue(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that calling this method when the node isn't at the front of the
# doesn't change the queue
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_success_not_front_of_queue(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("PROCESSING", val.get("QUEUED")[0]["STATUS"])
class SetForceQueueTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater",
"node")
self._e.WAIT_FOR_TIMER_POP = 0
def set_force_helper(self, force):
success = False
for x in range(10):
if self._e.set_force(force) == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully run set_force"
# Test that the FORCE value in the JSON can be correctly toggled
def test_set_force(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}"
)
self.set_force_helper(False)
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertFalse(val.get("FORCE"))
self.set_force_helper(True)
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertTrue(val.get("FORCE"))
self.set_force_helper(False)
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertFalse(val.get("FORCE"))
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater",
"node")
self._e.WAIT_FOR_TIMER_POP = 0
class AddToQueueTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater",
"node")
self._e.WAIT_FOR_TIMER_POP = 0
def add_to_queue(self):
success = False
for x in range(10):
if self._e.add_to_queue() == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully add the node to the queue"
# Test that adding to an empty queue simply adds the new node to the QUEUED array
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that adding a node when its already in the queue in the processing state adds the new node to the QUEUED array
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_already_processing(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node when its already in the queue in the queued state doesn't add the new node to the QUEUED array
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_already_queued(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node when its not already in the queue in the queued state adds the node (with more nodes in the queue already)
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_with_other_nodes_and_already_queued(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(3, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[2]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[2]["STATUS"])
# Test that adding the node succeeds for a non-empty queue that its not already in
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_with_other_nodes(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node to an empty queue with an unresponsive node doesn't add the unresponsive node to the queue
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue_and_other_node_unresponsive(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": []}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that adding a node to an empty queue with an failed node adds the failed node to the front of the queue
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue_and_other_node_failed(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[0]["STATUS"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node to an empty queue with this node marked as failed node only adds the node to the queue once
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue_and_this_node_failed(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that adding a node when it's marked as completed removes the node from the completed list
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_while_completed(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node that's marked as errored when it's not at the front of the queue doesn't change the errored state
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_while_errored(self):
self.set_initial_val(
"{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}"
)
self.add_to_queue()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
logfile = "/var/log/clearwater-queue-manager/queue_operation.log"
logging.basicConfig(filename=logfile,
format="%(asctime)s.%(msecs)03d UTC %(levelname)s %(filename)s:%(lineno)d: %(message)s",
datefmt="%d-%m-%Y %H:%M:%S",
level=logging.DEBUG)
operation = sys.argv[1]
local_ip = sys.argv[2]
site = sys.argv[3]
node_type = sys.argv[4]
clearwater_key = sys.argv[5]
queue_key = sys.argv[6]
logging.info("Using etcd key %s" % queue_key)
queue_syncer = EtcdSynchronizer(NullPlugin(queue_key), local_ip, site, clearwater_key, node_type)
if operation == "add":
logging.debug("Adding %s to queue to restart" % (local_ip + "-" + node_type))
while queue_syncer.add_to_queue() != WriteToEtcdStatus.SUCCESS:
sleep(2)
logging.debug("Node successfully added to restart queue")
elif operation == "remove_success":
logging.debug("Removing %s from front of queue" % (local_ip + "-" + node_type))
while queue_syncer.remove_from_queue(True) != WriteToEtcdStatus.SUCCESS:
sleep(2)
logging.debug("Node successfully removed")
def main(args):
syslog.openlog("queue-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
local_ip = arguments['--local-ip']
local_site = arguments['--local-site']
etcd_key = arguments['--etcd-key']
node_type = arguments['--node-type']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
wait_plugin_complete = arguments['--wait-plugin-complete']
stdout_err_log = os.path.join(log_dir, "queue-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-queue-mgr")
logging_config.configure_logging(log_level, log_dir, "queue-manager", show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("queue-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/"
plugins = load_plugins_in_dir(plugins_dir,
PluginParams(wait_plugin_complete=wait_plugin_complete))
plugins.sort(key=lambda x: x.key())
synchronizers = []
threads = []
for plugin in plugins:
syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key, node_type)
syncer.start_thread()
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
utils.install_sigterm_handler(synchronizers)
while any([thr.isAlive() for thr in threads]):
for thr in threads:
if thr.isAlive():
thr.join(1)
while not utils.should_quit:
sleep(1)
_log.info("Clearwater Queue Manager shutting down")
pdlogs.EXITING.log()
syslog.closelog()
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
self._e.WAIT_FOR_TIMER_POP = 0
class RemoveFromQueueFailureTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
self._e.WAIT_FOR_TIMER_POP = 0
def remove_from_queue_helper(self):
success = False
for x in range(10):
if self._e.remove_from_queue(False) == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully remove the node from the queue"
# Tests that marking a node as failed moves it to the ERRORED list
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(0, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"])
self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"])
# Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure_no_force(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(0, len(val.get("QUEUED")))
# Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure_force(self):
self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.4-node\",\"STATUS\":\"UNRESPONSIVE\"}, {\"ID\":\"10.0.0.5-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(2, len(val.get("ERRORED")))
self.assertEqual(2, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
@patch("etcd.Client", new=EtcdFactory)
# Tests that marking a node as failed when it is in the queued list but not the next node does move it to the ERRORED list
def test_remove_from_queue_after_failure_not_next_in_queue_force(self):
self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"])
self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"])
# Tests that marking a node as failed when it isn't the front of the queue doesn't change the JSON
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure_not_front_of_queue_force(self):
self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.remove_from_queue_helper()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
class RemoveFromQueueFailureTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater",
"node")
self._e.WAIT_FOR_TIMER_POP = 0
def remove_from_queue_helper(self):
success = False
for x in range(10):
if self._e.remove_from_queue(False) == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully remove the node from the queue"
# Tests that marking a node as failed moves it to the ERRORED list
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}"
)
self.remove_from_queue_helper()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(0, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"])
self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"])
# Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure_no_force(self):
self.set_initial_val(
"{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}"
)
self.remove_from_queue_helper()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(0, len(val.get("QUEUED")))
# Tests that marking a node as failed but when it is also the next node in the queue doesn't set it as errored
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure_force(self):
self.set_initial_val(
"{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.4-node\",\"STATUS\":\"UNRESPONSIVE\"}, {\"ID\":\"10.0.0.5-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [{\"ID\":\"10.0.0.3-node\",\"STATUS\":\"DONE\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}"
)
self.remove_from_queue_helper()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(2, len(val.get("ERRORED")))
self.assertEqual(2, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
@patch("etcd.Client", new=EtcdFactory)
# Tests that marking a node as failed when it is in the queued list but not the next node does move it to the ERRORED list
def test_remove_from_queue_after_failure_not_next_in_queue_force(self):
self.set_initial_val(
"{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}"
)
self.remove_from_queue_helper()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("ERRORED")[0]["ID"])
self.assertEqual("FAILURE", val.get("ERRORED")[0]["STATUS"])
# Tests that marking a node as failed when it isn't the front of the queue doesn't change the JSON
@patch("etcd.Client", new=EtcdFactory)
def test_remove_from_queue_after_failure_not_front_of_queue_force(self):
self.set_initial_val(
"{\"FORCE\": true, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"},{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}"
)
self.remove_from_queue_helper()
val = json.loads(
self._e._client.read(
"/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
log_format = logging.Formatter(fmt="%(asctime)s.%(msecs)03d UTC %(levelname)s %(filename)s:%(lineno)d: %(message)s",
datefmt="%d-%m-%Y %H:%M:%S")
log_format.converter = gmtime
handler.setFormatter(log_format)
_log.addHandler(handler)
operation = sys.argv[1]
local_ip = sys.argv[2]
site = sys.argv[3]
node_type = sys.argv[4]
clearwater_key = sys.argv[5]
queue_key = sys.argv[6]
_log.info("Using etcd key %s" % queue_key)
queue_syncer = EtcdSynchronizer(NullPlugin(queue_key), local_ip, site, clearwater_key, node_type)
if operation == "add":
_log.debug("Adding %s to queue to restart" % (local_ip + "-" + node_type))
while queue_syncer.add_to_queue() != WriteToEtcdStatus.SUCCESS:
sleep(2)
_log.debug("Node successfully added to restart queue")
elif operation == "remove_success":
_log.debug("Removing %s from front of queue" % (local_ip + "-" + node_type))
while queue_syncer.remove_from_queue(True) != WriteToEtcdStatus.SUCCESS:
sleep(2)
_log.debug("Node successfully removed")
def setUp(self):
alarms_patch.start()
self._p = TestNoTimerDelayPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater",
"node")
def main(args):
syslog.openlog("queue-manager", syslog.LOG_PID)
pdlogs.STARTUP.log()
try:
arguments = docopt(__doc__, argv=args)
except DocoptExit:
pdlogs.EXITING_BAD_CONFIG.log()
raise
local_ip = arguments['--local-ip']
local_site = arguments['--local-site']
etcd_key = arguments['--etcd-key']
node_type = arguments['--node-type']
log_dir = arguments['--log-directory']
log_level = LOG_LEVELS.get(arguments['--log-level'], logging.DEBUG)
wait_plugin_complete = arguments['--wait-plugin-complete']
stdout_err_log = os.path.join(log_dir, "queue-manager.output.log")
if not arguments['--foreground']:
utils.daemonize(stdout_err_log)
# Process names are limited to 15 characters, so abbreviate
prctl.prctl(prctl.NAME, "cw-queue-mgr")
logging_config.configure_logging(log_level,
log_dir,
"queue-manager",
show_thread=True)
# urllib3 logs a WARNING log whenever it recreates a connection, but our
# etcd usage does this frequently (to allow watch timeouts), so deliberately
# ignore this log
urllib_logger = logging.getLogger('urllib3')
urllib_logger.setLevel(logging.ERROR)
utils.install_sigusr1_handler("queue-manager")
# Drop a pidfile. We must keep a reference to the file object here, as this keeps
# the file locked and provides extra protection against two processes running at
# once.
pidfile_lock = None
try:
pidfile_lock = utils.lock_and_write_pid_file(
arguments['--pidfile']) # noqa
except IOError:
# We failed to take the lock - another process is already running
exit(1)
plugins_dir = "/usr/share/clearwater/clearwater-queue-manager/plugins/"
plugins = load_plugins_in_dir(
plugins_dir, PluginParams(wait_plugin_complete=wait_plugin_complete))
plugins.sort(key=lambda x: x.key())
synchronizers = []
threads = []
# Load the plugins, but don't start them until we've installed the SIGTERM
# handler, as that handler will gracefully shut down any running
# synchronizers on receiving a SIGTERM
for plugin in plugins:
syncer = EtcdSynchronizer(plugin, local_ip, local_site, etcd_key,
node_type)
synchronizers.append(syncer)
threads.append(syncer.thread)
_log.info("Loaded plugin %s" % plugin)
utils.install_sigterm_handler(synchronizers)
# Now start the plugin threads
for syncer in synchronizers:
syncer.start_thread()
_log.info("Started thread for plugin %s" % syncer._plugin)
while any([thr.isAlive() for thr in threads]):
for thr in threads:
if thr.isAlive():
thr.join(1)
while not utils.should_quit:
sleep(1)
_log.info("Clearwater Queue Manager shutting down")
pdlogs.EXITING.log()
syslog.closelog()
class AddToQueueTest(BaseQueueTest):
@patch("etcd.Client", new=EtcdFactory)
def setUp(self):
alarms_patch.start()
self._p = TestPlugin()
self._e = EtcdSynchronizer(self._p, "10.0.0.1", "local", "clearwater", "node")
self._e.WAIT_FOR_TIMER_POP = 0
def add_to_queue(self):
success = False
for x in range(10):
if self._e.add_to_queue() == WriteToEtcdStatus.SUCCESS:
success = True
break
sleep(1)
if not success:
print "Failed to successfully add the node to the queue"
# Test that adding to an empty queue simply adds the new node to the QUEUED array
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": []}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that adding a node when its already in the queue in the processing state adds the new node to the QUEUED array
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_already_processing(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}]}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node when its already in the queue in the queued state doesn't add the new node to the QUEUED array
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_already_queued(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.1-node\",\"STATUS\":\"QUEUED\"}]}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node when its not already in the queue in the queued state adds the node (with more nodes in the queue already)
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_with_other_nodes_and_already_queued(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"PROCESSING\"}, {\"ID\":\"10.0.0.2-node\",\"STATUS\":\"QUEUED\"}]}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(3, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[2]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[2]["STATUS"])
# Test that adding the node succeeds for a non-empty queue that its not already in
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_with_other_nodes(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node to an empty queue with an unresponsive node doesn't add the unresponsive node to the queue
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue_and_other_node_unresponsive(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"UNRESPONSIVE\"}], \"COMPLETED\": [], \"QUEUED\": []}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that adding a node to an empty queue with an failed node adds the failed node to the front of the queue
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue_and_other_node_failed(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[0]["STATUS"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node to an empty queue with this node marked as failed node only adds the node to the queue once
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_empty_queue_and_this_node_failed(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": []}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(1, len(val.get("QUEUED")))
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[0]["ID"])
# Test that adding a node when it's marked as completed removes the node from the completed list
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_while_completed(self):
self.set_initial_val("{\"FORCE\": false, \"ERRORED\": [], \"COMPLETED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"DONE\"}], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(0, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])
# Test that adding a node that's marked as errored when it's not at the front of the queue doesn't change the errored state
@patch("etcd.Client", new=EtcdFactory)
def test_add_to_queue_while_errored(self):
self.set_initial_val("{\"FORCE\": true, \"ERRORED\": [{\"ID\":\"10.0.0.1-node\",\"STATUS\":\"FAILURE\"}], \"COMPLETED\": [], \"QUEUED\": [{\"ID\":\"10.0.0.2-node\",\"STATUS\":\"PROCESSING\"}]}")
self.add_to_queue()
val = json.loads(self._e._client.read("/clearwater/local/configuration/queue_test").value)
self.assertEqual(1, len(val.get("ERRORED")))
self.assertEqual(0, len(val.get("COMPLETED")))
self.assertEqual(2, len(val.get("QUEUED")))
self.assertEqual("10.0.0.2-node", val.get("QUEUED")[0]["ID"])
self.assertEqual("10.0.0.1-node", val.get("QUEUED")[1]["ID"])
self.assertEqual("QUEUED", val.get("QUEUED")[1]["STATUS"])