def test_lock():
'''This test verifies that a second scheduler fails to startup when
an existing scheduler is running. Without locking, the scheduler
would fail during registration, but after writing its config to ZK.
So in order to verify that the scheduler fails immediately, we ensure
that the ZK config state is unmodified.'''
marathon_client = dcos.marathon.create_client()
# Get ZK state from running framework
zk_path = "dcos-service-{}/ConfigTarget".format(PACKAGE_NAME)
zk_config_old = shakedown.get_zk_node_data(zk_path)
# Get marathon app
app_id = "/{}".format(PACKAGE_NAME)
app = marathon_client.get_app(app_id)
old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None)
# Scale to 2 instances
labels = app["labels"]
labels.pop("MARATHON_SINGLE_INSTANCE_APP")
marathon_client.update_app(app_id, {"labels": labels})
shakedown.deployment_wait()
marathon_client.update_app(app_id, {"instances": 2})
# Wait for second scheduler to fail
def fn():
timestamp = marathon_client.get_app(app_id).get("lastTaskFailure", {}).get("timestamp", None)
return timestamp != old_timestamp
spin.time_wait_noisy(lambda: fn())
# Verify ZK is unchanged
zk_config_new = shakedown.get_zk_node_data(zk_path)
assert zk_config_old == zk_config_new
def test_lock():
'''This test verifies that a second scheduler fails to startup when
an existing scheduler is running. Without locking, the scheduler
would fail during registration, but after writing its config to ZK.
So in order to verify that the scheduler fails immediately, we ensure
that the ZK config state is unmodified.'''
marathon_client = dcos.marathon.create_client()
# Get ZK state from running framework
zk_path = "dcos-service-{}/ConfigTarget".format(PACKAGE_NAME)
zk_config_old = shakedown.get_zk_node_data(zk_path)
# Get marathon app
app_id = "/{}".format(PACKAGE_NAME)
app = marathon_client.get_app(app_id)
old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None)
# Scale to 2 instances
labels = app["labels"]
labels.pop("MARATHON_SINGLE_INSTANCE_APP")
marathon_client.update_app(app_id, {"labels": labels})
shakedown.deployment_wait()
marathon_client.update_app(app_id, {"instances": 2})
# Wait for second scheduler to fail
def fn():
timestamp = marathon_client.get_app(app_id).get("lastTaskFailure", {}).get("timestamp", None)
return timestamp != old_timestamp
spin.time_wait_noisy(lambda: fn())
# Verify ZK is unchanged
zk_config_new = shakedown.get_zk_node_data(zk_path)
assert zk_config_old == zk_config_new
def test_lock():
'''This test verifies that a second scheduler fails to startup when
an existing scheduler is running. Without locking, the scheduler
would fail during registration, but after writing its config to ZK.
So in order to verify that the scheduler fails immediately, we ensure
that the ZK config state is unmodified.'''
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
marathon_client = dcos.marathon.create_client()
# Get ZK state from running framework
zk_path = "dcos-service-{}/ConfigTarget".format(foldered_name)
zk_config_old = shakedown.get_zk_node_data(zk_path)
# Get marathon app
app = marathon_client.get_app(foldered_name)
old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None)
# Scale to 2 instances
labels = app["labels"]
original_labels = labels.copy()
labels.pop("MARATHON_SINGLE_INSTANCE_APP")
marathon_client.update_app(foldered_name, {"labels": labels})
shakedown.deployment_wait()
marathon_client.update_app(foldered_name, {"instances": 2})
@retrying.retry(wait_fixed=1000,
stop_max_delay=120 * 1000,
retry_on_result=lambda res: not res)
def wait_for_second_scheduler_to_fail():
timestamp = marathon_client.get_app(foldered_name).get(
"lastTaskFailure", {}).get("timestamp", None)
return timestamp != old_timestamp
wait_for_second_scheduler_to_fail()
# Verify ZK is unchanged
zk_config_new = shakedown.get_zk_node_data(zk_path)
assert zk_config_old == zk_config_new
# In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation
marathon_client.update_app(foldered_name, {
"labels": original_labels,
"instances": 1
},
force=True)
shakedown.deployment_wait()
def test_lock():
'''This test verifies that a second scheduler fails to startup when
an existing scheduler is running. Without locking, the scheduler
would fail during registration, but after writing its config to ZK.
So in order to verify that the scheduler fails immediately, we ensure
that the ZK config state is unmodified.'''
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
marathon_client = dcos.marathon.create_client()
# Get ZK state from running framework
zk_path = "dcos-service-{}/ConfigTarget".format(foldered_name)
zk_config_old = shakedown.get_zk_node_data(zk_path)
# Get marathon app
app = marathon_client.get_app(foldered_name)
old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None)
# Scale to 2 instances
labels = app["labels"]
original_labels = labels.copy()
labels.pop("MARATHON_SINGLE_INSTANCE_APP")
marathon_client.update_app(foldered_name, {"labels": labels})
shakedown.deployment_wait()
marathon_client.update_app(foldered_name, {"instances": 2})
@retrying.retry(
wait_fixed=1000,
stop_max_delay=120*1000,
retry_on_result=lambda res: not res)
def wait_for_second_scheduler_to_fail():
timestamp = marathon_client.get_app(foldered_name).get("lastTaskFailure", {}).get("timestamp", None)
return timestamp != old_timestamp
wait_for_second_scheduler_to_fail()
# Verify ZK is unchanged
zk_config_new = shakedown.get_zk_node_data(zk_path)
assert zk_config_old == zk_config_new
# In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation
marathon_client.update_app(foldered_name, {"labels": original_labels, "instances": 1}, force=True)
shakedown.deployment_wait()
def test_lock():
'''This test verifies that a second scheduler fails to startup when
an existing scheduler is running. Without locking, the scheduler
would fail during registration, but after writing its config to ZK.
So in order to verify that the scheduler fails immediately, we ensure
that the ZK config state is unmodified.'''
marathon_client = dcos.marathon.create_client()
# Get ZK state from running framework
zk_path = "dcos-service-{}/ConfigTarget".format(FOLDERED_SERVICE_NAME)
zk_config_old = shakedown.get_zk_node_data(zk_path)
# Get marathon app
app = marathon_client.get_app(FOLDERED_SERVICE_NAME)
old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None)
# Scale to 2 instances
labels = app["labels"]
original_labels = labels.copy()
labels.pop("MARATHON_SINGLE_INSTANCE_APP")
marathon_client.update_app(FOLDERED_SERVICE_NAME, {"labels": labels})
shakedown.deployment_wait()
marathon_client.update_app(FOLDERED_SERVICE_NAME, {"instances": 2})
# Wait for second scheduler to fail
def fn():
timestamp = marathon_client.get_app(FOLDERED_SERVICE_NAME).get("lastTaskFailure", {}).get("timestamp", None)
return timestamp != old_timestamp
shakedown.wait_for(lambda: fn())
# Verify ZK is unchanged
zk_config_new = shakedown.get_zk_node_data(zk_path)
assert zk_config_old == zk_config_new
# In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation
marathon_client.update_app(FOLDERED_SERVICE_NAME, {"labels": original_labels, "instances": 1}, force=True)
shakedown.deployment_wait()
def test_lock():
'''This test verifies that a second scheduler fails to startup when
an existing scheduler is running. Without locking, the scheduler
would fail during registration, but after writing its config to ZK.
So in order to verify that the scheduler fails immediately, we ensure
that the ZK config state is unmodified.'''
marathon_client = dcos.marathon.create_client()
# Get ZK state from running framework
zk_path = "dcos-service-{}/ConfigTarget".format(FOLDERED_SERVICE_NAME)
zk_config_old = shakedown.get_zk_node_data(zk_path)
# Get marathon app
app = marathon_client.get_app(FOLDERED_SERVICE_NAME)
old_timestamp = app.get("lastTaskFailure", {}).get("timestamp", None)
# Scale to 2 instances
labels = app["labels"]
original_labels = labels.copy()
labels.pop("MARATHON_SINGLE_INSTANCE_APP")
marathon_client.update_app(FOLDERED_SERVICE_NAME, {"labels": labels})
shakedown.deployment_wait()
marathon_client.update_app(FOLDERED_SERVICE_NAME, {"instances": 2})
# Wait for second scheduler to fail
def fn():
timestamp = marathon_client.get_app(FOLDERED_SERVICE_NAME).get("lastTaskFailure", {}).get("timestamp", None)
return timestamp != old_timestamp
shakedown.wait_for(lambda: fn())
# Verify ZK is unchanged
zk_config_new = shakedown.get_zk_node_data(zk_path)
assert zk_config_old == zk_config_new
# In order to prevent the second scheduler instance from obtaining a lock, we undo the "scale-up" operation
marathon_client.update_app(FOLDERED_SERVICE_NAME, {"labels": original_labels, "instances": 1}, force=True)
shakedown.deployment_wait()
