def test_deploy():
wait_time_in_seconds = 600
sdk_plan.wait_for_kicked_off_deployment(config.SERVICE_NAME)
# taskcfg.yml will initially fail to deploy because several options are missing in the default
# sdk_marathon.json.mustache. verify that the tasks are failing before continuing.
task_name = 'hello-0-server'
log.info('Checking that {} is failing to launch within {}s'.format(task_name, wait_time_in_seconds))
original_state_history = _get_state_history(task_name)
# wait for new TASK_FAILEDs to appear:
@retrying.retry(
wait_fixed=1000,
stop_max_delay=1000 * wait_time_in_seconds,
retry_on_result=lambda res: not res)
def wait_for_new_failures():
new_state_history = _get_state_history(task_name)
assert len(new_state_history) >= len(original_state_history)
added_state_history = new_state_history[len(original_state_history) :]
log.info("Added {} state history: {}".format(task_name, ", ".join(added_state_history)))
return "TASK_FAILED" in added_state_history
wait_for_new_failures()
# add the needed envvars in marathon and confirm that the deployment succeeds:
marathon_config = sdk_marathon.get_config(config.SERVICE_NAME)
env = marathon_config["env"]
del env["SLEEP_DURATION"]
env["TASKCFG_ALL_OUTPUT_FILENAME"] = "output"
env["TASKCFG_ALL_SLEEP_DURATION"] = "1000"
sdk_marathon.update_app(marathon_config)
config.check_running()
def update_service(package_name: str, service_name: str, options: dict):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
options_path = f.name
log.info("Writing updated options to %s", options_path)
json.dump(options, f)
f.flush()
cmd = ["update", "start", "--options={}".format(options_path)]
sdk_cmd.svc_cli(package_name, service_name, " ".join(cmd))
# An update plan is a deploy plan
sdk_plan.wait_for_kicked_off_deployment(service_name)
sdk_plan.wait_for_completed_deployment(service_name)
def test_adding_data_node_only_restarts_masters():
initial_master_task_ids = sdk_tasks.get_task_ids(foldered_name, "master")
initial_data_task_ids = sdk_tasks.get_task_ids(foldered_name, "data")
initial_coordinator_task_ids = sdk_tasks.get_task_ids(foldered_name, "coordinator")
# Get service configuration.
_, svc_config, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME, foldered_name, "describe", parse_json=True
)
data_nodes_count = get_in(["data_nodes", "count"], svc_config)
global current_expected_task_count
# Increase the data nodes count by 1.
sdk_service.update_configuration(
config.PACKAGE_NAME,
foldered_name,
{"data_nodes": {"count": data_nodes_count + 1}},
current_expected_task_count,
# As of 2018-12-14, sdk_upgrade's `wait_for_deployment` has different behavior than
# sdk_install's (which is what we wanted here), so don't use it. Check manually afterwards
# with `sdk_tasks.check_running`.
wait_for_deployment=False,
)
sdk_plan.wait_for_kicked_off_deployment(foldered_name)
sdk_plan.wait_for_completed_deployment(foldered_name)
_, new_data_pod_info, _ = sdk_cmd.svc_cli(
config.PACKAGE_NAME,
foldered_name,
"pod info data-{}".format(data_nodes_count),
parse_json=True,
)
# Get task ID for new data node task.
new_data_task_id = get_in([0, "info", "taskId", "value"], new_data_pod_info)
# Should be running 1 task more.
current_expected_task_count += 1
sdk_tasks.check_running(foldered_name, current_expected_task_count)
# Master nodes should restart.
sdk_tasks.check_tasks_updated(foldered_name, "master", initial_master_task_ids)
# Data node tasks should be the initial ones plus the new one.
sdk_tasks.check_tasks_not_updated(
foldered_name, "data", initial_data_task_ids + [new_data_task_id]
)
# Coordinator tasks should not restart.
sdk_tasks.check_tasks_not_updated(foldered_name, "coordinator", initial_coordinator_task_ids)
def update_service(package_name: str, service_name: str, options: dict):
with tempfile.NamedTemporaryFile("w", suffix=".json") as f:
options_path = f.name
log.info("Writing updated options to %s", options_path)
json.dump(options, f)
f.flush()
cmd = ["update", "start", "--options={}".format(options_path)]
sdk_cmd.svc_cli(package_name, service_name, " ".join(cmd))
# An update plan is a deploy plan
sdk_plan.wait_for_kicked_off_deployment(service_name)
sdk_plan.wait_for_completed_deployment(service_name)
def test_adding_data_node_only_restarts_masters() -> None:
initial_master_task_ids = sdk_tasks.get_task_ids(service_name, "master")
initial_data_task_ids = sdk_tasks.get_task_ids(service_name, "data")
initial_coordinator_task_ids = sdk_tasks.get_task_ids(service_name, "coordinator")
# Get service configuration.
_, svc_config, _ = sdk_cmd.svc_cli(package_name, service_name, "describe", parse_json=True)
data_nodes_count = get_in(["data_nodes", "count"], svc_config)
global current_expected_task_count
# Increase the data nodes count by 1.
sdk_service.update_configuration(
package_name,
service_name,
{"data_nodes": {"count": data_nodes_count + 1}},
current_expected_task_count,
# As of 2018-12-14, sdk_upgrade's `wait_for_deployment` has different behavior than
# sdk_install's (which is what we wanted here), so don't use it. Check manually afterwards
# with `sdk_tasks.check_running`.
wait_for_deployment=False,
)
sdk_plan.wait_for_kicked_off_deployment(service_name)
sdk_plan.wait_for_completed_deployment(service_name)
_, new_data_pod_info, _ = sdk_cmd.svc_cli(
package_name, service_name, "pod info data-{}".format(data_nodes_count), parse_json=True
)
# Get task ID for new data node task.
new_data_task_id = get_in([0, "info", "taskId", "value"], new_data_pod_info)
# Should be running 1 task more.
current_expected_task_count += 1
sdk_tasks.check_running(service_name, current_expected_task_count)
# Master nodes should restart.
sdk_tasks.check_tasks_updated(service_name, "master", initial_master_task_ids)
# Data node tasks should be the initial ones plus the new one.
sdk_tasks.check_tasks_not_updated(
service_name, "data", initial_data_task_ids + [new_data_task_id]
)
# Coordinator tasks should not restart.
sdk_tasks.check_tasks_not_updated(service_name, "coordinator", initial_coordinator_task_ids)