def test_deploy(): wait_time_in_seconds = 600 sdk_plan.wait_for_kicked_off_deployment(config.SERVICE_NAME) # taskcfg.yml will initially fail to deploy because several options are missing in the default # sdk_marathon.json.mustache. verify that the tasks are failing before continuing. task_name = 'hello-0-server' log.info('Checking that {} is failing to launch within {}s'.format(task_name, wait_time_in_seconds)) original_state_history = _get_state_history(task_name) # wait for new TASK_FAILEDs to appear: @retrying.retry( wait_fixed=1000, stop_max_delay=1000 * wait_time_in_seconds, retry_on_result=lambda res: not res) def wait_for_new_failures(): new_state_history = _get_state_history(task_name) assert len(new_state_history) >= len(original_state_history) added_state_history = new_state_history[len(original_state_history) :] log.info("Added {} state history: {}".format(task_name, ", ".join(added_state_history))) return "TASK_FAILED" in added_state_history wait_for_new_failures() # add the needed envvars in marathon and confirm that the deployment succeeds: marathon_config = sdk_marathon.get_config(config.SERVICE_NAME) env = marathon_config["env"] del env["SLEEP_DURATION"] env["TASKCFG_ALL_OUTPUT_FILENAME"] = "output" env["TASKCFG_ALL_SLEEP_DURATION"] = "1000" sdk_marathon.update_app(marathon_config) config.check_running()
def update_service(package_name: str, service_name: str, options: dict): with tempfile.NamedTemporaryFile("w", suffix=".json") as f: options_path = f.name log.info("Writing updated options to %s", options_path) json.dump(options, f) f.flush() cmd = ["update", "start", "--options={}".format(options_path)] sdk_cmd.svc_cli(package_name, service_name, " ".join(cmd)) # An update plan is a deploy plan sdk_plan.wait_for_kicked_off_deployment(service_name) sdk_plan.wait_for_completed_deployment(service_name)
def test_adding_data_node_only_restarts_masters(): initial_master_task_ids = sdk_tasks.get_task_ids(foldered_name, "master") initial_data_task_ids = sdk_tasks.get_task_ids(foldered_name, "data") initial_coordinator_task_ids = sdk_tasks.get_task_ids(foldered_name, "coordinator") # Get service configuration. _, svc_config, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "describe", parse_json=True ) data_nodes_count = get_in(["data_nodes", "count"], svc_config) global current_expected_task_count # Increase the data nodes count by 1. sdk_service.update_configuration( config.PACKAGE_NAME, foldered_name, {"data_nodes": {"count": data_nodes_count + 1}}, current_expected_task_count, # As of 2018-12-14, sdk_upgrade's `wait_for_deployment` has different behavior than # sdk_install's (which is what we wanted here), so don't use it. Check manually afterwards # with `sdk_tasks.check_running`. wait_for_deployment=False, ) sdk_plan.wait_for_kicked_off_deployment(foldered_name) sdk_plan.wait_for_completed_deployment(foldered_name) _, new_data_pod_info, _ = sdk_cmd.svc_cli( config.PACKAGE_NAME, foldered_name, "pod info data-{}".format(data_nodes_count), parse_json=True, ) # Get task ID for new data node task. new_data_task_id = get_in([0, "info", "taskId", "value"], new_data_pod_info) # Should be running 1 task more. current_expected_task_count += 1 sdk_tasks.check_running(foldered_name, current_expected_task_count) # Master nodes should restart. sdk_tasks.check_tasks_updated(foldered_name, "master", initial_master_task_ids) # Data node tasks should be the initial ones plus the new one. sdk_tasks.check_tasks_not_updated( foldered_name, "data", initial_data_task_ids + [new_data_task_id] ) # Coordinator tasks should not restart. sdk_tasks.check_tasks_not_updated(foldered_name, "coordinator", initial_coordinator_task_ids)
def update_service(package_name: str, service_name: str, options: dict): with tempfile.NamedTemporaryFile("w", suffix=".json") as f: options_path = f.name log.info("Writing updated options to %s", options_path) json.dump(options, f) f.flush() cmd = ["update", "start", "--options={}".format(options_path)] sdk_cmd.svc_cli(package_name, service_name, " ".join(cmd)) # An update plan is a deploy plan sdk_plan.wait_for_kicked_off_deployment(service_name) sdk_plan.wait_for_completed_deployment(service_name)
def test_adding_data_node_only_restarts_masters() -> None: initial_master_task_ids = sdk_tasks.get_task_ids(service_name, "master") initial_data_task_ids = sdk_tasks.get_task_ids(service_name, "data") initial_coordinator_task_ids = sdk_tasks.get_task_ids(service_name, "coordinator") # Get service configuration. _, svc_config, _ = sdk_cmd.svc_cli(package_name, service_name, "describe", parse_json=True) data_nodes_count = get_in(["data_nodes", "count"], svc_config) global current_expected_task_count # Increase the data nodes count by 1. sdk_service.update_configuration( package_name, service_name, {"data_nodes": {"count": data_nodes_count + 1}}, current_expected_task_count, # As of 2018-12-14, sdk_upgrade's `wait_for_deployment` has different behavior than # sdk_install's (which is what we wanted here), so don't use it. Check manually afterwards # with `sdk_tasks.check_running`. wait_for_deployment=False, ) sdk_plan.wait_for_kicked_off_deployment(service_name) sdk_plan.wait_for_completed_deployment(service_name) _, new_data_pod_info, _ = sdk_cmd.svc_cli( package_name, service_name, "pod info data-{}".format(data_nodes_count), parse_json=True ) # Get task ID for new data node task. new_data_task_id = get_in([0, "info", "taskId", "value"], new_data_pod_info) # Should be running 1 task more. current_expected_task_count += 1 sdk_tasks.check_running(service_name, current_expected_task_count) # Master nodes should restart. sdk_tasks.check_tasks_updated(service_name, "master", initial_master_task_ids) # Data node tasks should be the initial ones plus the new one. sdk_tasks.check_tasks_not_updated( service_name, "data", initial_data_task_ids + [new_data_task_id] ) # Coordinator tasks should not restart. sdk_tasks.check_tasks_not_updated(service_name, "coordinator", initial_coordinator_task_ids)