def test_lock(): '''This test verifies that a second scheduler fails to startup when an existing scheduler is running. Without locking, the scheduler would fail during registration, but after writing its config to ZK. So in order to verify that the scheduler fails immediately, we ensure that the ZK config state is unmodified.''' marathon_client = dcos.marathon.create_client() # Get ZK state from running framework zk_path = "dcos-service-{}/ConfigTarget".format(PACKAGE_NAME) zk_config_old = shakedown.get_zk_node_data(zk_path) # Get marathon app app_id = "/{}".format(PACKAGE_NAME) app = marathon_client.get_app(app_id) old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None) # Scale to 2 instances labels = app["labels"] labels.pop("MARATHON_SINGLE_INSTANCE_APP") marathon_client.update_app(app_id, {"labels": labels}) shakedown.deployment_wait() marathon_client.update_app(app_id, {"instances": 2}) # Wait for second scheduler to fail def fn(): timestamp = marathon_client.get_app(app_id).get("lastTaskFailure", {}).get("timestamp", None) return timestamp != old_timestamp spin.time_wait_noisy(lambda: fn()) # Verify ZK is unchanged zk_config_new = shakedown.get_zk_node_data(zk_path) assert zk_config_old == zk_config_new
def test_lock(): '''This test verifies that a second scheduler fails to startup when an existing scheduler is running. Without locking, the scheduler would fail during registration, but after writing its config to ZK. So in order to verify that the scheduler fails immediately, we ensure that the ZK config state is unmodified.''' foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) marathon_client = dcos.marathon.create_client() # Get ZK state from running framework zk_path = "dcos-service-{}/ConfigTarget".format(foldered_name) zk_config_old = shakedown.get_zk_node_data(zk_path) # Get marathon app app = marathon_client.get_app(foldered_name) old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None) # Scale to 2 instances labels = app["labels"] original_labels = labels.copy() labels.pop("MARATHON_SINGLE_INSTANCE_APP") marathon_client.update_app(foldered_name, {"labels": labels}) shakedown.deployment_wait() marathon_client.update_app(foldered_name, {"instances": 2}) @retrying.retry(wait_fixed=1000, stop_max_delay=120 * 1000, retry_on_result=lambda res: not res) def wait_for_second_scheduler_to_fail(): timestamp = marathon_client.get_app(foldered_name).get( "lastTaskFailure", {}).get("timestamp", None) return timestamp != old_timestamp wait_for_second_scheduler_to_fail() # Verify ZK is unchanged zk_config_new = shakedown.get_zk_node_data(zk_path) assert zk_config_old == zk_config_new # In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation marathon_client.update_app(foldered_name, { "labels": original_labels, "instances": 1 }, force=True) shakedown.deployment_wait()
def test_lock(): '''This test verifies that a second scheduler fails to startup when an existing scheduler is running. Without locking, the scheduler would fail during registration, but after writing its config to ZK. So in order to verify that the scheduler fails immediately, we ensure that the ZK config state is unmodified.''' foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME) marathon_client = dcos.marathon.create_client() # Get ZK state from running framework zk_path = "dcos-service-{}/ConfigTarget".format(foldered_name) zk_config_old = shakedown.get_zk_node_data(zk_path) # Get marathon app app = marathon_client.get_app(foldered_name) old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None) # Scale to 2 instances labels = app["labels"] original_labels = labels.copy() labels.pop("MARATHON_SINGLE_INSTANCE_APP") marathon_client.update_app(foldered_name, {"labels": labels}) shakedown.deployment_wait() marathon_client.update_app(foldered_name, {"instances": 2}) @retrying.retry( wait_fixed=1000, stop_max_delay=120*1000, retry_on_result=lambda res: not res) def wait_for_second_scheduler_to_fail(): timestamp = marathon_client.get_app(foldered_name).get("lastTaskFailure", {}).get("timestamp", None) return timestamp != old_timestamp wait_for_second_scheduler_to_fail() # Verify ZK is unchanged zk_config_new = shakedown.get_zk_node_data(zk_path) assert zk_config_old == zk_config_new # In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation marathon_client.update_app(foldered_name, {"labels": original_labels, "instances": 1}, force=True) shakedown.deployment_wait()
def test_lock(): '''This test verifies that a second scheduler fails to startup when an existing scheduler is running. Without locking, the scheduler would fail during registration, but after writing its config to ZK. So in order to verify that the scheduler fails immediately, we ensure that the ZK config state is unmodified.''' marathon_client = dcos.marathon.create_client() # Get ZK state from running framework zk_path = "dcos-service-{}/ConfigTarget".format(FOLDERED_SERVICE_NAME) zk_config_old = shakedown.get_zk_node_data(zk_path) # Get marathon app app = marathon_client.get_app(FOLDERED_SERVICE_NAME) old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None) # Scale to 2 instances labels = app["labels"] original_labels = labels.copy() labels.pop("MARATHON_SINGLE_INSTANCE_APP") marathon_client.update_app(FOLDERED_SERVICE_NAME, {"labels": labels}) shakedown.deployment_wait() marathon_client.update_app(FOLDERED_SERVICE_NAME, {"instances": 2}) # Wait for second scheduler to fail def fn(): timestamp = marathon_client.get_app(FOLDERED_SERVICE_NAME).get("lastTaskFailure", {}).get("timestamp", None) return timestamp != old_timestamp shakedown.wait_for(lambda: fn()) # Verify ZK is unchanged zk_config_new = shakedown.get_zk_node_data(zk_path) assert zk_config_old == zk_config_new # In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation marathon_client.update_app(FOLDERED_SERVICE_NAME, {"labels": original_labels, "instances": 1}, force=True) shakedown.deployment_wait()