distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import json
import os
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
import datetime, sys, socket
import resource_management.libraries.functions
@patch.object(resource_management.libraries.functions,
"get_unique_id_and_date",
new=MagicMock(return_value=''))
@patch("socket.socket")
@patch("time.time", new=MagicMock(return_value=1431110511.43))
class TestServiceCheck(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "HIVE/0.12.0.2.0/package"
STACK_VERSION = "2.0.6"
def test_service_check_default(self, socket_mock):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/service_check.py",
classname="HiveServiceCheck",
command="service_check",
config_file="default.json",
hdp_stack_version=self.STACK_VERSION,
target=RMFTestCase.TARGET_COMMON_SERVICES)
def accesszone_module_mock(self, mocker):
mocker.patch(MockAccessZoneApi.MODULE_UTILS_PATH + '.ApiException',
new=MockApiException)
az_module_mock = PowerScaleAccessZone()
az_module_mock.module = MagicMock()
return az_module_mock
from ambari_commons import inet_utils, OSCheck
from resource_management import Script, ConfigDictionary
from mock.mock import patch
from mock.mock import MagicMock
from unittest import TestCase
from check_host import CheckHost
from only_for_platform import get_platform, not_for_platform, only_for_platform, os_distro_value, PLATFORM_WINDOWS
from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
@patch.object(HostCheckReportFileHandler,
"writeHostChecksCustomActionsFile",
new=MagicMock())
class TestCheckHost(TestCase):
current_dir = os.path.dirname(os.path.realpath(__file__))
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("os.path.isfile")
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch("resource_management.libraries.script.Script.put_structured_out")
def testJavaHomeAvailableCheck(self, structured_out_mock, get_tmp_dir_mock,
mock_config, os_isfile_mock):
# test, java home exists
os_isfile_mock.return_value = True
get_tmp_dir_mock.return_value = "/tmp"
class TestActionQueue(TestCase):
def setUp(self):
# save original open() method for later use
self.original_open = open
def tearDown(self):
sys.stdout = sys.__stdout__
logger = logging.getLogger()
datanode_install_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'hostLevelParams': {},
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v1'
}
}
}
datanode_auto_start_command = {
'commandType': 'AUTO_EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'START',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'hostLevelParams': {},
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v1'
}
}
}
datanode_upgrade_command = {
'commandId': 17,
'role': "role",
'taskId': "taskId",
'clusterName': "clusterName",
'serviceName': "serviceName",
'roleCommand': 'UPGRADE',
'hostname': "localhost.localdomain",
'hostLevelParams': {},
'clusterHostInfo': "clusterHostInfo",
'commandType': "EXECUTION_COMMAND",
'configurations': {
'global': {}
},
'roleParams': {},
'commandParams': {
'source_stack_version': 'HDP-1.2.1',
'target_stack_version': 'HDP-1.3.0'
}
}
namenode_install_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'NAMENODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 4,
'clusterName': u'cc',
'serviceName': u'HDFS',
'hostLevelParams': {}
}
snamenode_install_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'SECONDARY_NAMENODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 5,
'clusterName': u'cc',
'serviceName': u'HDFS',
'hostLevelParams': {}
}
hbase_install_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'HBASE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 7,
'clusterName': u'cc',
'serviceName': u'HDFS',
'hostLevelParams': {}
}
status_command = {
"serviceName": 'HDFS',
"commandType": "STATUS_COMMAND",
"clusterName": "",
"componentName": "DATANODE",
'configurations': {},
'hostLevelParams': {}
}
datanode_restart_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'CUSTOM_COMMAND',
'commandId': '1-1',
'taskId': 9,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'hostLevelParams': {
'custom_command': 'RESTART',
'clientsToUpdateConfigs': []
}
}
datanode_restart_command_no_clients_update = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'CUSTOM_COMMAND',
'commandId': '1-1',
'taskId': 9,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'hostLevelParams': {
'custom_command': 'RESTART'
}
}
status_command_for_alerts = {
"serviceName": 'FLUME',
"commandType": "STATUS_COMMAND",
"clusterName": "",
"componentName": "FLUME_HANDLER",
'configurations': {},
'hostLevelParams': {}
}
retryable_command = {
'commandType': 'EXECUTION_COMMAND',
'role': 'NAMENODE',
'roleCommand': 'INSTALL',
'commandId': '1-1',
'taskId': 19,
'clusterName': 'c1',
'serviceName': 'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'commandParams': {
'script_type': 'PYTHON',
'script': 'script.py',
'command_timeout': '600',
'jdk_location': '.',
'service_package_folder': '.',
'command_retry_enabled': 'true',
'max_duration_for_retries': '5'
},
'hostLevelParams': {}
}
background_command = {
'commandType': 'BACKGROUND_EXECUTION_COMMAND',
'role': 'NAMENODE',
'roleCommand': 'CUSTOM_COMMAND',
'commandId': '1-1',
'taskId': 19,
'clusterName': 'c1',
'serviceName': 'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'hostLevelParams': {
'custom_command': 'REBALANCE_HDFS'
},
'commandParams': {
'script_type': 'PYTHON',
'script': 'script.py',
'command_timeout': '600',
'jdk_location': '.',
'service_package_folder': '.'
}
}
cancel_background_command = {
'commandType': 'EXECUTION_COMMAND',
'role': 'NAMENODE',
'roleCommand': 'ACTIONEXECUTE',
'commandId': '1-1',
'taskId': 20,
'clusterName': 'c1',
'serviceName': 'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {}
},
'hostLevelParams': {},
'commandParams': {
'script_type': 'PYTHON',
'script': 'cancel_background_task.py',
'before_system_hook_function': 'fetch_bg_pid_by_taskid',
'jdk_location': '.',
'command_timeout': '600',
'service_package_folder': '.',
'cancel_policy': 'SIGKILL',
'cancel_task_id': "19",
}
}
@patch.object(AmbariConfig, "get_parallel_exec_option")
@patch.object(ActionQueue, "process_command")
@patch.object(Queue, "get")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock,
get_mock, process_command_mock,
get_parallel_exec_option_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
get_parallel_exec_option_mock.return_value = 0
config.get_parallel_exec_option = get_parallel_exec_option_mock
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.start()
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
self.assertTrue(process_command_mock.call_count > 1)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("traceback.print_exc")
@patch.object(ActionQueue, "execute_command")
@patch.object(ActionQueue, "execute_status_command")
def test_process_command(self, execute_status_command_mock,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
config = AmbariConfig()
config.set('agent', 'tolerate_download_failures', "true")
actionQueue = ActionQueue(config, dummy_controller)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType': ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType': "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
actionQueue.process_command(status_command)
self.assertFalse(execute_command_mock.called)
self.assertTrue(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
execute_status_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("__builtin__.open")
@patch.object(ActionQueue, "status_update_callback")
def test_auto_execute_command(self, status_update_callback_mock,
open_mock):
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
dummy_controller.recovery_manager.update_config(
5, 5, 1, 11, True, False, "")
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(command,
tmpoutfile,
tmperrfile,
override_output_files=True,
retry=False):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.process_command(command)
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
self.assertFalse(actionQueue.tasks_in_progress_or_pending())
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_auto_start_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
if actionQueue.tasks_in_progress_or_pending():
break
# Continue command execution
unfreeze_flag.set()
# wait until ready
check_queue = True
while check_queue:
report = actionQueue.result()
if not actionQueue.tasks_in_progress_or_pending():
break
time.sleep(0.1)
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_auto_start_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
while check_queue:
report = actionQueue.result()
if not actionQueue.tasks_in_progress_or_pending():
break
time.sleep(0.1)
self.assertEqual(len(report['reports']), 0)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("__builtin__.open")
@patch.object(ActionQueue, "status_update_callback")
def test_execute_command(self, status_update_callback_mock, open_mock):
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(command,
tmpoutfile,
tmperrfile,
override_output_files=True,
retry=False):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
report = actionQueue.result()
if len(report['reports']) != 0:
break
expected = {
'status':
'IN_PROGRESS',
'stderr':
'Read from {0}'.format(os.path.join(tempdir, "errors-3.txt")),
'stdout':
'Read from {0}'.format(os.path.join(tempdir, "output-3.txt")),
'structuredOut':
'Read from {0}'.format(
os.path.join(tempdir, "structured-out-3.json")),
'clusterName':
u'cc',
'roleCommand':
u'INSTALL',
'serviceName':
u'HDFS',
'role':
u'DATANODE',
'actionId':
'1-1',
'taskId':
3,
'exitCode':
777
}
self.assertEqual(report['reports'][0], expected)
self.assertTrue(actionQueue.tasks_in_progress_or_pending())
# Continue command execution
unfreeze_flag.set()
# wait until ready
while report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
configname = os.path.join(tempdir, 'config.json')
expected = {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'exitCode': 0
}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
# Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
self.assertEqual(status_update_callback_mock.call_count, 2)
os.remove(configname)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {
'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 13
}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
### Test upgrade command ###
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_upgrade_command, ))
execution_thread.start()
unfreeze_flag.set()
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'structuredOut': '""',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitCode': 0
}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(CustomServiceOrchestrator, "runCommand")
@patch("CommandStatusDict.CommandStatusDict")
@patch.object(ActionQueue, "status_update_callback")
def test_store_configuration_tags(self, status_update_callback_mock,
command_status_dict_mock,
cso_runCommand_mock):
custom_service_orchestrator_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'exitcode': 0
}
cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.execute_command(self.datanode_restart_command)
report = actionQueue.result()
expected = {
'status': 'COMPLETED',
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'CUSTOM_COMMAND',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 9,
'customCommand': 'RESTART',
'exitCode': 0
}
# Agent caches configurationTags if custom_command RESTART completed
self.assertEqual(len(report['reports']), 1)
self.assertEqual(expected, report['reports'][0])
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(ActualConfigHandler, "write_client_components")
@patch.object(CustomServiceOrchestrator, "runCommand")
@patch("CommandStatusDict.CommandStatusDict")
@patch.object(ActionQueue, "status_update_callback")
def test_store_configuration_tags_no_clients(self,
status_update_callback_mock,
command_status_dict_mock,
cso_runCommand_mock,
write_client_components_mock):
custom_service_orchestrator_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'exitcode': 0
}
cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.execute_command(
self.datanode_restart_command_no_clients_update)
report = actionQueue.result()
expected = {
'status': 'COMPLETED',
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'CUSTOM_COMMAND',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 9,
'customCommand': 'RESTART',
'exitCode': 0
}
# Agent caches configurationTags if custom_command RESTART completed
self.assertEqual(len(report['reports']), 1)
self.assertEqual(expected, report['reports'][0])
self.assertFalse(write_client_components_mock.called)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(ActionQueue, "status_update_callback")
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "requestComponentStatus")
@patch.object(CustomServiceOrchestrator, "requestComponentSecurityState")
@patch.object(ActionQueue, "execute_command")
@patch.object(LiveStatus, "build")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentSecurityState_mock,
requestComponentStatus_mock,
read_stack_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
build_mock.return_value = {'dummy report': ''}
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0}
requestComponentSecurityState_mock.reset_mock()
requestComponentSecurityState_mock.return_value = 'UNKNOWN'
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = {'dummy report': '', 'securityState': 'UNKNOWN'}
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(ActionQueue, "status_update_callback")
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "requestComponentStatus")
@patch.object(CustomServiceOrchestrator, "requestComponentSecurityState")
@patch.object(ActionQueue, "execute_command")
@patch.object(LiveStatus, "build")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_status_command_with_alerts(
self, CustomServiceOrchestrator_mock,
requestComponentSecurityState_mock, build_mock,
execute_command_mock, requestComponentStatus_mock,
read_stack_version_mock, status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'err',
'structuredOut': {
'alerts': [{
'name': 'flume_alert'
}]
}
}
build_mock.return_value = {'somestatusresult': 'aresult'}
actionQueue.execute_status_command(self.status_command_for_alerts)
report = actionQueue.result()
self.assertTrue(requestComponentStatus_mock.called)
self.assertEqual(len(report['componentStatus']), 1)
self.assertTrue(report['componentStatus'][0].has_key('alerts'))
@patch.object(AmbariConfig, "get_parallel_exec_option")
@patch.object(ActionQueue, "process_command")
@patch.object(Queue, "get")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_reset_queue(self, CustomServiceOrchestrator_mock, get_mock,
process_command_mock, gpeo_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
config = MagicMock()
gpeo_mock.return_value = 0
config.get_parallel_exec_option = gpeo_mock
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.start()
actionQueue.put(
[self.datanode_install_command, self.hbase_install_command])
self.assertEqual(2, actionQueue.commandQueue.qsize())
self.assertTrue(actionQueue.tasks_in_progress_or_pending())
actionQueue.reset()
self.assertTrue(actionQueue.commandQueue.empty())
self.assertFalse(actionQueue.tasks_in_progress_or_pending())
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
@patch.object(AmbariConfig, "get_parallel_exec_option")
@patch.object(ActionQueue, "process_command")
@patch.object(Queue, "get")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_cancel(self, CustomServiceOrchestrator_mock, get_mock,
process_command_mock, gpeo_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
gpeo_mock.return_value = 0
config.get_parallel_exec_option = gpeo_mock
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.start()
actionQueue.put(
[self.datanode_install_command, self.hbase_install_command])
self.assertEqual(2, actionQueue.commandQueue.qsize())
actionQueue.reset()
self.assertTrue(actionQueue.commandQueue.empty())
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
@patch.object(AmbariConfig, "get_parallel_exec_option")
@patch.object(ActionQueue, "process_command")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_parallel_exec(self, CustomServiceOrchestrator_mock,
process_command_mock, gpeo_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
gpeo_mock.return_value = 1
config.get_parallel_exec_option = gpeo_mock
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.put(
[self.datanode_install_command, self.hbase_install_command])
self.assertEqual(2, actionQueue.commandQueue.qsize())
actionQueue.start()
time.sleep(1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
self.assertEqual(2, process_command_mock.call_count)
process_command_mock.assert_any_calls([
call(self.datanode_install_command),
call(self.hbase_install_command)
])
@not_for_platform(PLATFORM_LINUX)
@patch("time.sleep")
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_retryable_command(self, CustomServiceOrchestrator_mock,
read_stack_version_mock, sleep_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
def side_effect(command,
tmpoutfile,
tmperrfile,
override_output_files=True,
retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(3, runCommand_mock.call_count)
self.assertEqual(2, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(2), call(3)], False)
runCommand_mock.assert_has_calls([
call(command,
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'errors-19.txt',
override_output_files=True,
retry=False),
call(command,
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'errors-19.txt',
override_output_files=False,
retry=True),
call(command,
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'errors-19.txt',
override_output_files=False,
retry=True)
])
@patch("time.time")
@patch("time.sleep")
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_retryable_command_with_time_lapse(
self, CustomServiceOrchestrator_mock, read_stack_version_mock,
sleep_mock, time_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
times_arr = [8, 10, 14, 18, 22]
if self.logger.isEnabledFor(logging.INFO):
times_arr.insert(0, 4)
time_mock.side_effect = times_arr
def side_effect(command,
tmpoutfile,
tmperrfile,
override_output_files=True,
retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(2)], False)
runCommand_mock.assert_has_calls([
call(command,
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'errors-19.txt',
override_output_files=True,
retry=False),
call(command,
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep +
'errors-19.txt',
override_output_files=False,
retry=True)
])
#retryable_command
@not_for_platform(PLATFORM_LINUX)
@patch("time.sleep")
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_retryable_command_fail_and_succeed(
self, CustomServiceOrchestrator_mock, read_stack_version_mock,
sleep_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execution_result_fail_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
execution_result_succ_dict = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'COMPLETED'
}
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = [
execution_result_fail_dict, execution_result_succ_dict
]
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_any_call(2)
@not_for_platform(PLATFORM_LINUX)
@patch("time.sleep")
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_retryable_command_succeed(self,
CustomServiceOrchestrator_mock,
read_stack_version_mock,
sleep_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execution_result_succ_dict = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'COMPLETED'
}
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = [execution_result_succ_dict]
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertFalse(sleep_mock.called)
self.assertEqual(1, runCommand_mock.call_count)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "runCommand")
@patch.object(CustomServiceOrchestrator, "__init__")
def test_execute_background_command(self, CustomServiceOrchestrator_mock,
runCommand_mock,
read_stack_version_mock):
CustomServiceOrchestrator_mock.return_value = None
CustomServiceOrchestrator.runCommand.return_value = {
'exitcode': 0,
'stdout': 'out-11',
'stderr': 'err-13'
}
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execute_command = copy.deepcopy(self.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
#assert that python execturor start
self.assertTrue(runCommand_mock.called)
runningCommand = actionQueue.commandStatuses.current_state.get(
execute_command['taskId'])
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand[1]['status'],
ActionQueue.IN_PROGRESS_STATUS)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
@patch.object(CustomServiceOrchestrator, "get_py_executor")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(StackVersionsFileHandler, "read_stack_version")
def test_execute_python_executor(self, read_stack_version_mock,
resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir,
actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock(
)
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {
'condensed_result':
copy.copy(process_condensed_result),
'handle':
copy.copy(handle),
'command_status':
actionQueue.commandStatuses.get_command_status(
handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(
actionQueue.on_background_command_complete_callback, None,
command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'],
ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(
self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0]['stdout'], 'process_out')
# self.assertEqual(report['reports'][0]['structuredOut'],'{"a": "b."}')
cancel_background_command = {
"commandType": "CANCEL_COMMAND",
"role": "AMBARI_SERVER_ACTION",
"roleCommand": "ABORT",
"commandId": "2--1",
"taskId": 20,
"clusterName": "c1",
"serviceName": "",
"hostname": "c6401",
"roleParams": {
"cancelTaskIdTargets": "13,14"
},
}
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import os
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
import datetime, sys, socket
import resource_management.libraries.functions
@patch.object(resource_management.libraries.functions,
"get_unique_id_and_date",
new=MagicMock(return_value=''))
@patch("socket.socket", new=MagicMock())
class TestServiceCheck(RMFTestCase):
@patch("sys.exit")
def test_service_check_default(self, sys_exit_mock):
self.executeScript(
"2.0.6/services/HIVE/package/scripts/service_check.py",
classname="HiveServiceCheck",
command="service_check",
config_file="default.json")
self.assertResourceCalled(
'File',
'/tmp/hcatSmoke.sh',
content=StaticFile('hcatSmoke.sh'),
mode=0755,
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
import os
origin_exists = os.path.exists
@patch.object(os.path,
"exists",
new=MagicMock(side_effect=lambda *args: origin_exists(args[0])
if args[0][-2:] == "j2" else True))
class TestResourceManager(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
STACK_VERSION = "2.0.6"
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/resourcemanager.py",
classname="Resourcemanager",
command="configure",
config_file="default.json",
hdp_stack_version=self.STACK_VERSION,
target=RMFTestCase.TARGET_COMMON_SERVICES)
self.assert_configure_default()
self.assertNoMoreResources()
def open_subprocess_files_win(fout, ferr, f):
return MagicMock(), MagicMock()
class TestDatanode(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
STACK_VERSION = "2.0.6"
CONFIG_OVERRIDES = {"serviceName":"HDFS", "role":"DATANODE"}
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "configure",
config_file = "default.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default()
self.assertNoMoreResources()
def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "start",
config_file = "default.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default()
self.assertResourceCalled('Directory', '/var/run/hadoop',
owner = 'hdfs',
group = 'hadoop',
mode = 0755
)
self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertResourceCalled('Execute', "ambari-sudo.sh su hdfs -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ulimit -c unlimited ; /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode'",
environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertNoMoreResources()
@patch('time.sleep')
@patch("os.path.exists", new = MagicMock(return_value=False))
@patch("resource_management.core.shell.checked_call")
def test_stop_default(self, checked_call_mock, time_mock):
def side_effect(arg):
if '-D ipc.client.connect.max.retries=5 -D ipc.client.connect.retry.interval=1000 -getDatanodeInfo' in arg :
raise Fail()
return
checked_call_mock.side_effect = side_effect
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "stop",
config_file = "default.json",
stack_version = self.STACK_VERSION,
checked_call_mocks = side_effect,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', "ambari-sudo.sh su hdfs -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ulimit -c unlimited ; /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf stop datanode'",
environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
only_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid")
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'])
self.assertNoMoreResources()
def test_configure_secured(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "configure",
config_file = "secured.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured()
self.assertNoMoreResources()
def test_start_secured(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "start",
config_file = "secured.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured()
self.assertResourceCalled('Directory', '/var/run/hadoop',
owner = 'hdfs',
group = 'hadoop',
mode = 0755
)
self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertResourceCalled('Execute', 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode',
environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertNoMoreResources()
def test_start_secured_HDP22_root(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
with open(config_file, "r") as f:
secured_json = json.load(f)
secured_json['hostLevelParams']['stack_version']= '2.2'
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "start",
config_dict = secured_json,
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured("2.2", snappy_enabled=False)
self.assertResourceCalled('Directory', '/var/run/hadoop',
owner = 'hdfs',
group = 'hadoop',
mode = 0755
)
self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertResourceCalled('Execute', 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E /usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /usr/hdp/current/hadoop-client/conf start datanode',
environment = {'HADOOP_LIBEXEC_DIR': '/usr/hdp/current/hadoop-client/libexec'},
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertNoMoreResources()
def test_start_secured_HDP22_non_root_https_only(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
with open(config_file, "r") as f:
secured_json = json.load(f)
secured_json['hostLevelParams']['stack_version']= '2.2'
secured_json['configurations']['hdfs-site']['dfs.http.policy']= 'HTTPS_ONLY'
secured_json['configurations']['hdfs-site']['dfs.datanode.address']= '0.0.0.0:10000'
secured_json['configurations']['hdfs-site']['dfs.datanode.https.address']= '0.0.0.0:50000'
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "start",
config_dict = secured_json,
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured("2.2", snappy_enabled=False)
self.assertResourceCalled('Directory', '/var/run/hadoop',
owner = 'hdfs',
group = 'hadoop',
mode = 0755
)
self.assertResourceCalled('Directory', '/var/run/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('Directory', '/var/log/hadoop/hdfs',
owner = 'hdfs',
group = 'hadoop',
create_parents = True,
)
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid',
action = ['delete'],
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertResourceCalled('Execute', "ambari-sudo.sh su hdfs -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ulimit -c unlimited ; /usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /usr/hdp/current/hadoop-client/conf start datanode'",
environment = {'HADOOP_LIBEXEC_DIR': '/usr/hdp/current/hadoop-client/libexec'},
not_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid",
)
self.assertNoMoreResources()
@patch('time.sleep')
@patch("os.path.exists", new = MagicMock(return_value=False))
@patch("resource_management.core.shell.checked_call")
def test_stop_secured(self, checked_call_mock, time_mock):
def side_effect(arg):
if '-D ipc.client.connect.max.retries=5 -D ipc.client.connect.retry.interval=1000 -getDatanodeInfo' in arg :
raise Fail()
return
checked_call_mock.side_effect = side_effect
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "stop",
config_file = "secured.json",
stack_version = self.STACK_VERSION,
checked_call_mocks = side_effect,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf stop datanode',
environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
only_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid")
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'])
self.assertNoMoreResources()
@patch('time.sleep')
@patch("os.path.exists", new = MagicMock(return_value=False))
@patch("resource_management.core.shell.checked_call")
def test_stop_secured_HDP22_root(self, checked_call_mock, time_mock):
def side_effect(arg):
if '-D ipc.client.connect.max.retries=5 -D ipc.client.connect.retry.interval=1000 -getDatanodeInfo' in arg :
raise Fail()
return
checked_call_mock.side_effect = side_effect
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
with open(config_file, "r") as f:
secured_json = json.load(f)
secured_json['hostLevelParams']['stack_version']= '2.2'
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "stop",
config_dict = secured_json,
stack_version = self.STACK_VERSION,
checked_call_mocks = side_effect,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', 'ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E /usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /usr/hdp/current/hadoop-client/conf stop datanode',
environment = {'HADOOP_LIBEXEC_DIR': '/usr/hdp/current/hadoop-client/libexec'},
only_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid")
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'])
self.assertNoMoreResources()
@patch('time.sleep')
@patch("os.path.exists", new = MagicMock(return_value=False))
@patch("resource_management.core.shell.checked_call")
def test_stop_secured_HDP22_non_root_https_only(self, checked_call_mock, time_mock):
def side_effect(arg):
if '-D ipc.client.connect.max.retries=5 -D ipc.client.connect.retry.interval=1000 -getDatanodeInfo' in arg :
raise Fail()
return
checked_call_mock.side_effect = side_effect
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/secured.json"
with open(config_file, "r") as f:
secured_json = json.load(f)
secured_json['hostLevelParams']['stack_version']= '2.2'
secured_json['configurations']['hdfs-site']['dfs.http.policy']= 'HTTPS_ONLY'
secured_json['configurations']['hdfs-site']['dfs.datanode.address']= '0.0.0.0:10000'
secured_json['configurations']['hdfs-site']['dfs.datanode.https.address']= '0.0.0.0:50000'
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "stop",
config_dict = secured_json,
stack_version = self.STACK_VERSION,
checked_call_mocks = side_effect,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', "ambari-sudo.sh su hdfs -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ulimit -c unlimited ; /usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /usr/hdp/current/hadoop-client/conf stop datanode'",
environment = {'HADOOP_LIBEXEC_DIR': '/usr/hdp/current/hadoop-client/libexec'},
only_if = "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E test -f /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid && ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E pgrep -F /var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid")
self.assertResourceCalled('File', '/var/run/hadoop/hdfs/hadoop-hdfs-datanode.pid', action = ['delete'])
self.assertNoMoreResources()
def assert_configure_default(self):
self.assertResourceCalled('Directory', '/usr/lib/hadoop/lib/native/Linux-i386-32',
create_parents = True,
)
self.assertResourceCalled('Directory', '/usr/lib/hadoop/lib/native/Linux-amd64-64',
create_parents = True,
)
self.assertResourceCalled('Link', '/usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so',
to = '/usr/lib/hadoop/lib/libsnappy.so',
)
self.assertResourceCalled('Link', '/usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so',
to = '/usr/lib/hadoop/lib64/libsnappy.so',
)
self.assertResourceCalled('Directory', '/etc/security/limits.d',
owner = 'root',
group = 'root',
create_parents = True,
)
self.assertResourceCalled('File', '/etc/security/limits.d/hdfs.conf',
content = Template('hdfs.conf.j2'),
owner = 'root',
group = 'root',
mode = 0644,
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'core-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
mode = 0644
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
content = Template('slaves.j2'),
owner = 'hdfs',
)
self.assertResourceCalled('Directory', '/var/lib/hadoop-hdfs',
owner = 'hdfs',
group = 'hadoop',
mode = 0751,
create_parents = True,
)
self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/datanode',
mode = 0755,
create_parents = True
)
self.assertResourceCalled('Directory', '/hadoop/hdfs/data',
owner = 'hdfs',
ignore_failures = True,
group = 'hadoop',
mode = 0750,
create_parents = True,
cd_access='a'
)
content = resource_management.libraries.functions.mounted_dirs_helper.DIR_TO_MOUNT_HEADER
self.assertResourceCalled('File', '/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist',
owner = 'hdfs',
group = 'hadoop',
mode = 0644,
content = content
)
def assert_configure_secured(self, stackVersion=STACK_VERSION, snappy_enabled=True):
conf_dir = '/etc/hadoop/conf'
if stackVersion != self.STACK_VERSION:
conf_dir = '/usr/hdp/current/hadoop-client/conf'
if snappy_enabled:
self.assertResourceCalled('Directory', '/usr/lib/hadoop/lib/native/Linux-i386-32',
create_parents = True,
)
self.assertResourceCalled('Directory', '/usr/lib/hadoop/lib/native/Linux-amd64-64',
create_parents = True,
)
self.assertResourceCalled('Link', '/usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so',
to = '/usr/lib/hadoop/lib/libsnappy.so',
)
self.assertResourceCalled('Link', '/usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so',
to = '/usr/lib/hadoop/lib64/libsnappy.so',
)
self.assertResourceCalled('Directory', '/etc/security/limits.d',
owner = 'root',
group = 'root',
create_parents = True,
)
self.assertResourceCalled('File', '/etc/security/limits.d/hdfs.conf',
content = Template('hdfs.conf.j2'),
owner = 'root',
group = 'root',
mode = 0644,
)
self.assertResourceCalled('File', conf_dir + '/hdfs_dn_jaas.conf',
content = Template('hdfs_dn_jaas.conf.j2'),
owner = 'hdfs',
group = 'hadoop',
)
self.assertResourceCalled('File', conf_dir + '/hdfs_nn_jaas.conf',
content = Template('hdfs_nn_jaas.conf.j2'),
owner = 'hdfs',
group = 'hadoop',
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = conf_dir,
configurations = self.getConfig()['configurations']['hdfs-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'core-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = conf_dir,
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
mode = 0644
)
self.assertResourceCalled('File', conf_dir + '/slaves',
content = Template('slaves.j2'),
owner = 'root',
)
self.assertResourceCalled('Directory', '/var/lib/hadoop-hdfs',
owner = 'hdfs',
group = 'hadoop',
mode = 0751,
create_parents = True,
)
self.assertResourceCalled('Directory', '/var/lib/ambari-agent/data/datanode',
mode = 0755,
create_parents = True
)
self.assertResourceCalled('Directory', '/hadoop/hdfs/data',
owner = 'hdfs',
ignore_failures = True,
group = 'hadoop',
mode = 0750,
create_parents = True,
cd_access='a'
)
content = resource_management.libraries.functions.mounted_dirs_helper.DIR_TO_MOUNT_HEADER
self.assertResourceCalled('File', '/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist',
owner = 'hdfs',
group = 'hadoop',
mode = 0644,
content = content
)
def test_pre_upgrade_restart(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.2.1.0-3242'
json_content['commandParams']['version'] = version
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "pre_upgrade_restart",
config_dict = json_content,
config_overrides = self.CONFIG_OVERRIDES,
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES)
self.assertResourceCalled('Execute',
('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-hdfs-datanode', version), sudo=True,)
self.assertNoMoreResources()
@patch("resource_management.core.shell.call")
def test_pre_upgrade_restart_23(self, call_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "pre_upgrade_restart",
config_dict = json_content,
config_overrides = self.CONFIG_OVERRIDES,
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
mocks_dict = mocks_dict)
self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'hadoop-hdfs-datanode', version), sudo=True,)
self.assertNoMoreResources()
@patch("socket.gethostbyname")
@patch('time.sleep')
def test_post_upgrade_restart(self, time_mock, socket_gethostbyname_mock):
shell_call_output = """
Live datanodes (2):
Name: 192.168.64.102:50010 (c6401.ambari.apache.org)
Hostname: c6401.ambari.apache.org
Decommission Status : Normal
Configured Capacity: 524208947200 (488.21 GB)
DFS Used: 193069056 (184.13 MB)
Non DFS Used: 29264986112 (27.26 GB)
DFS Remaining: 494750892032 (460.77 GB)
DFS Used%: 0.04%
DFS Remaining%: 94.38%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 2
Last contact: Fri Dec 12 20:47:21 UTC 2014
"""
mocks_dict = {}
socket_gethostbyname_mock.return_value = "test_host"
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "post_upgrade_restart",
config_file = "default.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, shell_call_output)],
mocks_dict = mocks_dict
)
self.assertTrue(mocks_dict['call'].called)
self.assertEqual(mocks_dict['call'].call_count,1)
@patch("socket.gethostbyname")
@patch('time.sleep')
def test_post_upgrade_restart_datanode_not_ready(self, time_mock, socket_gethostbyname_mock):
mocks_dict = {}
socket_gethostbyname_mock.return_value = "test_host"
try:
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "post_upgrade_restart",
config_file = "default.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, 'There are no DataNodes here!')] * 30,
mocks_dict = mocks_dict
)
self.fail('Missing DataNode should have caused a failure')
except Fail,fail:
self.assertTrue(mocks_dict['call'].called)
self.assertEqual(mocks_dict['call'].call_count,30)
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import json
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
@patch("os.path.exists", new = MagicMock(return_value=True))
class TestHookAfterInstall(RMFTestCase):
def test_hook_default(self):
self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py",
classname="AfterInstallHook",
command="hook",
config_file="default.json"
)
self.assertResourceCalled('XmlConfig', 'core-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import json
from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
from only_for_platform import not_for_platform, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
@patch("resource_management.libraries.functions.get_hdp_version", new=MagicMock(return_value="2.3.2.0-1597"))
class TestSparkThriftServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
STACK_VERSION = "2.3"
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
def test_configure_default(self, copy_to_hdfs_mock):
copy_to_hdfs_mock = True
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_thrift_server.py",
classname = "SparkThriftServer",
command = "configure",
config_file="spark_default.json",
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default()
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
from stacks.utils.RMFTestCase import *
import json
from mock.mock import MagicMock, patch
from resource_management.libraries.script.script import Script
from resource_management.core import shell
import itertools
from resource_management.core.exceptions import Fail
import resource_management.libraries.functions.mounted_dirs_helper
@patch.object(resource_management.libraries.functions, 'check_process_status', new = MagicMock())
@patch.object(Script, 'format_package_name', new = MagicMock())
class TestDatanode(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
STACK_VERSION = "2.0.6"
CONFIG_OVERRIDES = {"serviceName":"HDFS", "role":"DATANODE"}
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/datanode.py",
classname = "DataNode",
command = "configure",
config_file = "default.json",
stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
class TestHardware(TestCase):
@patch.object(Hardware, "osdisks", new=MagicMock(return_value=[]))
@patch.object(Hardware,
"_chk_writable_mount",
new=MagicMock(return_value=True))
@patch.object(FacterLinux,
"get_ip_address_by_ifname",
new=MagicMock(return_value=None))
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
def test_build(self, get_os_version_mock, get_os_type_mock):
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
hardware = Hardware()
result = hardware.get()
osdisks = hardware.osdisks()
for dev_item in result['mounts']:
self.assertTrue(dev_item['available'] >= 0)
self.assertTrue(dev_item['used'] >= 0)
self.assertTrue(dev_item['percent'] is not None)
self.assertTrue(dev_item['device'] is not None)
self.assertTrue(dev_item['mountpoint'] is not None)
self.assertTrue(dev_item['type'] is not None)
self.assertTrue(dev_item['size'] > 0)
for os_disk_item in osdisks:
self.assertTrue(os_disk_item['available'] >= 0)
self.assertTrue(os_disk_item['used'] >= 0)
self.assertTrue(os_disk_item['percent'] is not None)
self.assertTrue(os_disk_item['device'] is not None)
self.assertTrue(os_disk_item['mountpoint'] is not None)
self.assertTrue(os_disk_item['type'] is not None)
self.assertTrue(os_disk_item['size'] > 0)
self.assertTrue(len(result['mounts']) == len(osdisks))
@patch.object(Hardware, "_chk_writable_mount")
@patch("ambari_agent.Hardware.path_isfile")
@patch("resource_management.core.shell.call")
def test_osdisks_parsing(self, shell_call_mock, isfile_mock,
chk_writable_mount_mock):
df_output =\
"""Filesystem Type 1024-blocks Used Available Capacity Mounted on
/dev/mapper/docker-253:0-4980899-d45c264d37ab18c8ed14f890f4d59ac2b81e1c52919eb36a79419787209515f3 xfs 31447040 1282384 30164656 5% /
tmpfs tmpfs 32938336 4 32938332 1% /dev
tmpfs tmpfs 32938336 0 32938336 0% /sys/fs/cgroup
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/resolv.conf
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hostname
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hosts
shm tmpfs 65536 0 65536 0% /dev/shm
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /run/secrets
"""
def isfile_side_effect(path):
assume_files = ["/etc/resolv.conf", "/etc/hostname", "/etc/hosts"]
return path in assume_files
def chk_writable_mount_side_effect(path):
assume_read_only = ["/run/secrets"]
return path not in assume_read_only
isfile_mock.side_effect = isfile_side_effect
chk_writable_mount_mock.side_effect = chk_writable_mount_side_effect
shell_call_mock.return_value = (0, df_output, '')
result = Hardware(cache_info=False).osdisks()
self.assertEquals(1, len(result))
expected_mounts_left = ["/"]
mounts_left = [item["mountpoint"] for item in result]
self.assertEquals(expected_mounts_left, mounts_left)
@patch.object(Hardware, "_chk_writable_mount")
@patch("ambari_agent.Hardware.path_isfile")
@patch("resource_management.core.shell.call")
def test_osdisks_no_ignore_property(self, shell_call_mock, isfile_mock,
chk_writable_mount_mock):
df_output = \
"""Filesystem Type 1024-blocks Used Available Capacity Mounted on
/dev/mapper/docker-253:0-4980899-d45c264d37ab18c8ed14f890f4d59ac2b81e1c52919eb36a79419787209515f3 xfs 31447040 1282384 30164656 5% /
"""
isfile_mock.return_value = False
chk_writable_mount_mock.return_value = True
shell_call_mock.return_value = (0, df_output, '')
config = AmbariConfig()
# check, that config do not define ignore_mount_points property
self.assertEquals(
"test", config.get('agent', 'ignore_mount_points', default="test"))
result = Hardware(config=config, cache_info=False).osdisks()
self.assertEquals(1, len(result))
expected_mounts_left = ["/"]
mounts_left = [item["mountpoint"] for item in result]
self.assertEquals(expected_mounts_left, mounts_left)
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
@patch("resource_management.core.shell.call")
def test_osdisks_remote(self, shell_call_mock, get_os_version_mock,
get_os_type_mock):
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
Hardware(cache_info=False).osdisks()
timeout = 10
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
config = AmbariConfig()
Hardware(config=config, cache_info=False).osdisks()
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
config.add_section(AmbariConfig.AMBARI_PROPERTIES_CATEGORY)
config.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY,
Hardware.CHECK_REMOTE_MOUNTS_KEY, "true")
Hardware(config=config, cache_info=False).osdisks()
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
config.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY,
Hardware.CHECK_REMOTE_MOUNTS_KEY, "false")
Hardware(config=config, cache_info=False).osdisks()
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT", "-l"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
config.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY,
Hardware.CHECK_REMOTE_MOUNTS_TIMEOUT_KEY, "0")
Hardware(config=config, cache_info=False).osdisks()
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT", "-l"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
timeout = 1
config.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY,
Hardware.CHECK_REMOTE_MOUNTS_TIMEOUT_KEY, str(timeout))
Hardware(config=config, cache_info=False).osdisks()
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT", "-l"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
timeout = 2
config.set(AmbariConfig.AMBARI_PROPERTIES_CATEGORY,
Hardware.CHECK_REMOTE_MOUNTS_TIMEOUT_KEY, str(timeout))
Hardware(config=config, cache_info=False).osdisks()
shell_call_mock.assert_called_with(
['timeout', str(timeout), "df", "-kPT", "-l"],
stdout=subprocess32.PIPE,
stderr=subprocess32.PIPE,
timeout=timeout,
quiet=True)
def test_parse_df_line(self):
df_line_sample = "device type size used available percent mountpoint"
samples = [{
"sample":
df_line_sample,
"expected":
dict(zip(df_line_sample.split(), df_line_sample.split()))
}, {
"sample": "device type size used available percent",
"expected": None,
}, {
"sample":
"device type size used available percent mountpoint info",
"expected": None,
}, {
"sample": "",
"expected": None
}]
for sample in samples:
try:
result = Hardware(cache_info=False)._parse_df(
[sample["sample"]]).next()
except StopIteration:
result = None
self.assertEquals(
result, sample["expected"],
"Failed with sample: '{0}', expected: {1}, got: {2}".format(
sample["sample"], sample["expected"], result))
@patch.object(FacterLinux,
"get_ip_address_by_ifname",
new=MagicMock(return_value=None))
@patch.object(hostname, "hostname")
@patch.object(FacterLinux, "getFqdn")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
def test_fqdnDomainHostname(self, get_os_version_mock, get_os_type_mock,
facter_getFqdn_mock, hostname_mock):
facter_getFqdn_mock.return_value = "ambari.apache.org"
hostname_mock.return_value = 'ambari'
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
config = None
result = Facter(config).facterInfo()
self.assertEquals(result['hostname'], "ambari")
self.assertEquals(result['domain'], "apache.org")
self.assertEquals(result['fqdn'],
(result['hostname'] + '.' + result['domain']))
@patch.object(FacterLinux,
"get_ip_address_by_ifname",
new=MagicMock(return_value=None))
@patch.object(FacterLinux, "setDataUpTimeOutput")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
def test_uptimeSecondsHoursDays(self, get_os_version_mock,
get_os_type_mock,
facter_setDataUpTimeOutput_mock):
# 3 days + 1 hour + 13 sec
facter_setDataUpTimeOutput_mock.return_value = "262813.00 123.45"
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
config = None
result = Facter(config).facterInfo()
self.assertEquals(result['uptime_seconds'], '262813')
self.assertEquals(result['uptime_hours'], '73')
self.assertEquals(result['uptime_days'], '3')
@patch.object(FacterLinux,
"get_ip_address_by_ifname",
new=MagicMock(return_value=None))
@patch.object(FacterLinux, "setMemInfoOutput")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
@patch.object(FacterLinux, "getSystemResourceOverrides")
def test_facterMemInfoOutput(self, getSystemResourceOverridesMock,
get_os_version_mock, get_os_type_mock,
facter_setMemInfoOutput_mock):
getSystemResourceOverridesMock.return_value = {}
facter_setMemInfoOutput_mock.return_value = '''
MemTotal: 1832392 kB
MemFree: 868648 kB
HighTotal: 0 kB
HighFree: 0 kB
LowTotal: 1832392 kB
LowFree: 868648 kB
SwapTotal: 2139592 kB
SwapFree: 1598676 kB
'''
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
config = None
result = Facter(config).facterInfo()
self.assertEquals(result['memorysize'], 1832392)
self.assertEquals(result['memorytotal'], 1832392)
self.assertEquals(result['memoryfree'], 868648)
self.assertEquals(result['swapsize'], '2.04 GB')
self.assertEquals(result['swapfree'], '1.52 GB')
@patch("fcntl.ioctl")
@patch("socket.socket")
@patch("struct.pack")
@patch("socket.inet_ntoa")
@patch.object(FacterLinux, "get_ip_address_by_ifname")
@patch.object(Facter, "getIpAddress")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
def test_facterDataIfConfigOutput(self, get_os_version_mock,
get_os_type_mock, getIpAddress_mock,
get_ip_address_by_ifname_mock,
inet_ntoa_mock, struct_pack_mock,
socket_socket_mock, fcntl_ioctl_mock):
getIpAddress_mock.return_value = "10.0.2.15"
get_ip_address_by_ifname_mock.return_value = "10.0.2.15"
inet_ntoa_mock.return_value = "255.255.255.0"
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
config = None
result = Facter(config).facterInfo()
self.assertTrue(inet_ntoa_mock.called)
self.assertTrue(get_ip_address_by_ifname_mock.called)
self.assertTrue(getIpAddress_mock.called)
self.assertEquals(result['ipaddress'], '10.0.2.15')
self.assertEquals(result['netmask'], '255.255.255.0')
self.assertEquals(result['interfaces'], 'eth0,eth1,eth2,lo')
@patch("fcntl.ioctl")
@patch("socket.socket")
@patch("struct.pack")
@patch("socket.inet_ntoa")
@patch.object(FacterLinux, "get_ip_address_by_ifname")
@patch.object(Facter, "getIpAddress")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
def test_facterDataIfConfigOutputNone(self, get_os_version_mock,
get_os_type_mock, getIpAddress_mock,
get_ip_address_by_ifname_mock,
inet_ntoa_mock, struct_pack_mock,
socket_socket_mock,
fcntl_ioctl_mock):
getIpAddress_mock.return_value = "10.0.2.15"
get_ip_address_by_ifname_mock.return_value = ""
inet_ntoa_mock.return_value = "255.255.255.0"
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
config = None
result = Facter(config).facterInfo()
self.assertTrue(get_ip_address_by_ifname_mock.called)
self.assertEquals(result['netmask'], None)
@patch.object(FacterLinux,
"get_ip_address_by_ifname",
new=MagicMock(return_value=None))
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_family")
@patch.object(OSCheck, "get_os_version")
def test_facterDataOperatingsystemVsFamily(self, get_os_version_mock,
get_os_family_mock,
get_os_type_mock):
get_os_type_mock.return_value = "some_type_of_os"
get_os_version_mock.return_value = "11"
get_os_family_mock.return_value = "redhat"
config = None
result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'redhat')
get_os_family_mock.return_value = "ubuntu"
result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'ubuntu')
get_os_family_mock.return_value = "suse"
result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'suse')
get_os_family_mock.return_value = "My_new_family"
result = Facter(config).facterInfo()
self.assertEquals(result['operatingsystem'], 'some_type_of_os')
self.assertEquals(result['osfamily'], 'My_new_family')
@patch("os.path.exists")
@patch("os.path.isdir")
@patch("json.loads")
@patch("glob.glob")
@patch("__builtin__.open")
@patch.object(OSCheck, "get_os_type")
@patch.object(OSCheck, "get_os_version")
@patch.object(FacterLinux, "resolve_ambari_config")
def test_system_resource_overrides(self, resolve_ambari_config,
get_os_version_mock, get_os_type_mock,
open_mock, glob_mock, json_mock, isdir,
exists):
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
config = MagicMock()
config.get.return_value = '/etc/custom_resource_overrides'
config.has_option.return_value = True
resolve_ambari_config.return_value = config
isdir.return_value = True
exists.return_value = True
open_mock.return_value.read = "1"
file_handle = open_mock.return_value.__enter__.return_value
file_handle.read.return_value = '1'
glob_mock.side_effect = \
[
[
"/etc/custom_resource_overrides/1.json",
"/etc/custom_resource_overrides/2.json"
]
]
json_data = json_mock.return_value
json_data.items.return_value = [('key', 'value')]
json_data.__getitem__.return_value = 'value'
facter = Facter(config)
result = facter.getSystemResourceOverrides()
isdir.assert_called_with('/etc/custom_resource_overrides')
exists.assert_called_with('/etc/custom_resource_overrides')
glob_mock.assert_called_with('/etc/custom_resource_overrides/*.json')
self.assertTrue(config.has_option.called)
self.assertTrue(config.get.called)
self.assertTrue(glob_mock.called)
self.assertEquals(2, file_handle.read.call_count)
self.assertEquals(2, open_mock.call_count)
self.assertEquals(2, json_mock.call_count)
self.assertEquals('value', result['key'])
@patch.object(Hardware, "_chk_writable_mount")
@patch("ambari_agent.Hardware.path_isfile")
@patch("resource_management.core.shell.call")
def test_osdisks_blacklist(self, shell_call_mock, isfile_mock,
chk_writable_mount_mock):
df_output = \
"""Filesystem Type 1024-blocks Used Available Capacity Mounted on
/dev/mapper/docker-253:0-4980899-d45c264d37ab18c8ed14f890f4d59ac2b81e1c52919eb36a79419787209515f3 xfs 31447040 1282384 30164656 5% /
tmpfs tmpfs 32938336 4 32938332 1% /dev
tmpfs tmpfs 32938336 0 32938336 0% /sys/fs/cgroup
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/resolv.conf
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hostname
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /etc/hosts
shm tmpfs 65536 0 65536 0% /dev/shm
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /run/secrets
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /mnt/blacklisted_mount
/dev/mapper/fedora-root ext4 224161316 12849696 199901804 7% /mnt/blacklisted_mount/sub-dir
"""
def isfile_side_effect(path):
assume_files = ["/etc/resolv.conf", "/etc/hostname", "/etc/hosts"]
return path in assume_files
def chk_writable_mount_side_effect(path):
assume_read_only = ["/run/secrets"]
return path not in assume_read_only
isfile_mock.side_effect = isfile_side_effect
chk_writable_mount_mock.side_effect = chk_writable_mount_side_effect
config_dict = {
"agent": {
"ignore_mount_points": "/mnt/blacklisted_mount"
}
}
shell_call_mock.return_value = (0, df_output, '')
def conf_get(section, key, default=""):
if section in config_dict and key in config_dict[section]:
return config_dict[section][key]
return default
def has_option(section, key):
return section in config_dict and key in config_dict[section]
conf = Mock()
attr = {
'get.side_effect': conf_get,
'has_option.side_effect': has_option
}
conf.configure_mock(**attr)
result = Hardware(config=conf, cache_info=False).osdisks()
self.assertEquals(1, len(result))
expected_mounts_left = ["/"]
mounts_left = [item["mountpoint"] for item in result]
self.assertEquals(expected_mounts_left, mounts_left)
import unittest
import platform
import socket
from ambari_commons import subprocess32
from only_for_platform import not_for_platform, PLATFORM_WINDOWS
from ambari_agent import hostname
from ambari_agent.Hardware import Hardware
from ambari_agent.AmbariConfig import AmbariConfig
from ambari_agent.Facter import Facter, FacterLinux
from ambari_commons import OSCheck
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(platform,
"linux_distribution",
new=MagicMock(return_value=('Suse', '11', 'Final')))
@patch.object(socket,
"getfqdn",
new=MagicMock(return_value="ambari.apache.org"))
@patch.object(socket,
"gethostbyname",
new=MagicMock(return_value="192.168.1.1"))
@patch.object(
FacterLinux,
"setDataIfConfigShortOutput",
new=MagicMock(
return_value=
'''Iface MTU Met RX-OK RX-ERR RX-DRP RX-OVR TX-OK TX-ERR TX-DRP TX-OVR Flg
eth0 1500 0 9986 0 0 0 5490 0 0 0 BMRU
eth1 1500 0 0 0 0 0 6 0 0 0 BMRU
eth2 1500 0 0 0 0 0 6 0 0 0 BMRU
def test_alert(self, conn_mock):
connection = MagicMock()
response = MagicMock()
response.status = 200
connection.getresponse.return_value = response
conn_mock.return_value = connection
response.read.return_value = '{"metrics":[{"metricname":"metric1","metrics":{"1459966360838":1,"1459966370838":3}}]}'
# OK, but no datapoints above the minimum threshold
[status, messages] = alert.execute(configurations=configs,
parameters=parameters)
self.assertEqual(status, RESULT_STATE_OK)
self.assertTrue(messages is not None and len(messages) == 1)
self.assertEquals(
'No datapoints found above the minimum threshold of 30 seconds',
messages[0])
# Unable to calculate the standard deviation for 1 data point
response.read.return_value = '{"metrics":[{"metricname":"metric1","metrics":{"1459966360838":40000}}]}'
[status, messages] = alert.execute(configurations=configs,
parameters=parameters)
self.assertEqual(status, RESULT_STATE_SKIPPED)
self.assertTrue(messages is not None and len(messages) == 1)
self.assertEquals(
'Unable to calculate the standard deviation for 1 datapoints',
messages[0])
# OK
response.read.return_value = '{"metrics":[{"metricname":"metric1","metrics":{"1459966360838":40000,"1459966370838":50000}}]}'
[status, messages] = alert.execute(configurations=configs,
parameters=parameters)
self.assertEqual(status, RESULT_STATE_OK)
self.assertTrue(messages is not None and len(messages) == 1)
self.assertTrue(
'OK. Percentage standard deviation value is' in messages[0])
# Warning
response.read.return_value = '{"metrics":[{"metricname":"metric1","metrics":{"1459966360838":40000,"1459966370838":1000000}}]}'
[status, messages] = alert.execute(configurations=configs,
parameters=parameters)
self.assertEqual(status, RESULT_STATE_WARNING)
self.assertTrue(messages is not None and len(messages) == 1)
self.assertTrue(
'WARNING. Percentage standard deviation' in messages[0])
# HTTP request to AMS failed
response.read.return_value = ''
response.status = 501
[status, messages] = alert.execute(configurations=configs,
parameters=parameters)
self.assertEqual(status, RESULT_STATE_UNKNOWN)
self.assertTrue(messages is not None and len(messages) == 1)
self.assertEquals('Unable to retrieve metrics from AMS.', messages[0])
# Unable to connect to AMS
conn_mock.side_effect = Exception('Unable to connect to AMS')
[status, messages] = alert.execute(configurations=configs,
parameters=parameters)
self.assertEqual(status, RESULT_STATE_UNKNOWN)
self.assertTrue(messages is not None and len(messages) == 1)
self.assertEquals('Unable to retrieve metrics from AMS.', messages[0])
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import json
from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
from only_for_platform import not_for_platform, PLATFORM_WINDOWS
@not_for_platform(PLATFORM_WINDOWS)
@patch("resource_management.libraries.functions.get_hdp_version",
new=MagicMock(return_value="2.3.0.0-1597"))
class TestSparkClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
STACK_VERSION = "2.2"
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/spark_client.py",
classname="SparkClient",
command="configure",
config_file="default.json",
hdp_stack_version=self.STACK_VERSION,
target=RMFTestCase.TARGET_COMMON_SERVICES)
self.assert_configure_default()
self.assertNoMoreResources()
class TestHookAfterInstall(RMFTestCase):
def test_hook_default(self):
self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py",
classname="AfterInstallHook",
command="hook",
config_file="default.json"
)
self.assertResourceCalled('XmlConfig', 'core-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /etc/hadoop/conf")
self.assertNoMoreResources()
@patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1243"))
@patch("resource_management.libraries.functions.conf_select.create")
@patch("resource_management.libraries.functions.conf_select.select")
@patch("os.symlink")
@patch("shutil.rmtree")
def test_hook_default_conf_select(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock):
def mocked_conf_select(arg1, arg2, arg3, dry_run = False):
return "/etc/{0}/{1}/0".format(arg2, arg3)
conf_select_create_mock.side_effect = mocked_conf_select
config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
json_content['hostLevelParams']['stack_version'] = "2.3"
self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py",
classname="AfterInstallHook",
command="hook",
config_dict = json_content)
self.assertResourceCalled("Execute",
"ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E touch /var/lib/ambari-agent/data/hdp-select-set-all.performed ; " \
"ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^2.3 | tail -1`",
only_if = "ls -d /usr/hdp/2.3*",
not_if = "test -f /var/lib/ambari-agent/data/hdp-select-set-all.performed")
self.assertResourceCalled('XmlConfig', 'core-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = "/usr/hdp/current/hadoop-client/conf",
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/kms/conf', '/etc/ranger/kms/conf.install'),
not_if = 'test -e /etc/ranger/kms/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/ranger/kms/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/ranger/kms/conf',
to = '/usr/hdp/current/ranger-kms/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/zookeeper/conf', '/etc/zookeeper/conf.install'),
not_if = 'test -e /etc/zookeeper/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/zookeeper/conf',
to = '/usr/hdp/current/zookeeper-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf', '/etc/pig/conf.install'),
not_if = 'test -e /etc/pig/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/pig/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/pig/conf',
to = '/usr/hdp/current/pig-client/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/tez/conf', '/etc/tez/conf.install'),
not_if = 'test -e /etc/tez/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/tez/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/tez/conf',
to = '/usr/hdp/current/tez-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.install'),
not_if = 'test -e /etc/hive-webhcat/conf.install',
sudo = True,)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive-hcatalog/conf', '/etc/hive-hcatalog/conf.install'),
not_if = 'test -e /etc/hive-hcatalog/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/hive-webhcat/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/hive-webhcat/conf',
to = '/usr/hdp/current/hive-webhcat/etc/webhcat',)
self.assertResourceCalled('Directory', '/etc/hive-hcatalog/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/hive-hcatalog/conf',
to = '/usr/hdp/current/hive-webhcat/etc/hcatalog',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hbase/conf', '/etc/hbase/conf.install'),
not_if = 'test -e /etc/hbase/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/hbase/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/hbase/conf',
to = '/usr/hdp/current/hbase-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf', '/etc/knox/conf.install'),
not_if = 'test -e /etc/knox/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/knox/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/knox/conf',
to = '/usr/hdp/current/knox-server/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.install'),
not_if = 'test -e /etc/ranger/usersync/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
to = '/usr/hdp/current/ranger-usersync/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hadoop/conf', '/etc/hadoop/conf.install'),
not_if = 'test -e /etc/hadoop/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/hadoop/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/hadoop/conf',
to = '/usr/hdp/current/hadoop-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/mahout/conf', '/etc/mahout/conf.install'),
not_if = 'test -e /etc/mahout/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/mahout/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/mahout/conf',
to = '/usr/hdp/current/mahout-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm/conf', '/etc/storm/conf.install'),
not_if = 'test -e /etc/storm/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/storm/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/storm/conf',
to = '/usr/hdp/current/storm-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/admin/conf', '/etc/ranger/admin/conf.install'),
not_if = 'test -e /etc/ranger/admin/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/ranger/admin/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/ranger/admin/conf',
to = '/usr/hdp/current/ranger-admin/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/flume/conf', '/etc/flume/conf.install'),
not_if = 'test -e /etc/flume/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/flume/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/flume/conf',
to = '/usr/hdp/current/flume-server/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf', '/etc/sqoop/conf.install'),
not_if = 'test -e /etc/sqoop/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/sqoop/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/sqoop/conf',
to = '/usr/hdp/current/sqoop-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/accumulo/conf', '/etc/accumulo/conf.install'),
not_if = 'test -e /etc/accumulo/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/accumulo/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/accumulo/conf',
to = '/usr/hdp/current/accumulo-client/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/phoenix/conf', '/etc/phoenix/conf.install'),
not_if = 'test -e /etc/phoenix/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/phoenix/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/phoenix/conf',
to = '/usr/hdp/current/phoenix-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm-slider-client/conf', '/etc/storm-slider-client/conf.install'),
not_if = 'test -e /etc/storm-slider-client/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/storm-slider-client/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/storm-slider-client/conf',
to = '/usr/hdp/current/storm-slider-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/slider/conf', '/etc/slider/conf.install'),
not_if = 'test -e /etc/slider/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/slider/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/slider/conf',
to = '/usr/hdp/current/slider-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/oozie/conf', '/etc/oozie/conf.install'),
not_if = 'test -e /etc/oozie/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/oozie/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/oozie/conf',
to = '/usr/hdp/current/oozie-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/falcon/conf', '/etc/falcon/conf.install'),
not_if = 'test -e /etc/falcon/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/falcon/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/falcon/conf',
to = '/usr/hdp/current/falcon-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/spark/conf', '/etc/spark/conf.install'),
not_if = 'test -e /etc/spark/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/spark/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/spark/conf',
to = '/usr/hdp/current/spark-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf', '/etc/kafka/conf.install'),
not_if = 'test -e /etc/kafka/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/kafka/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/kafka/conf',
to = '/usr/hdp/current/kafka-broker/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf', '/etc/hive/conf.install'),
not_if = 'test -e /etc/hive/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/hive/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/hive/conf',
to = '/usr/hdp/current/hive-client/conf')
self.assertNoMoreResources()
@patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1243"))
@patch("resource_management.libraries.functions.conf_select.create")
@patch("resource_management.libraries.functions.conf_select.select")
@patch("os.symlink")
@patch("shutil.rmtree")
def test_hook_default_conf_select_with_error(self, rmtree_mock, symlink_mock, conf_select_select_mock, conf_select_create_mock):
def mocked_conf_select(arg1, arg2, arg3, dry_run = False):
if arg2 == "pig" and not dry_run:
raise Exception("whoops")
return "/etc/{0}/{1}/0".format(arg2, arg3)
conf_select_create_mock.side_effect = mocked_conf_select
conf_select_select_mock.side_effect = mocked_conf_select
config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
json_content['hostLevelParams']['stack_version'] = "2.3"
self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py",
classname="AfterInstallHook",
command="hook",
config_dict = json_content)
self.assertResourceCalled("Execute",
"ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E touch /var/lib/ambari-agent/data/hdp-select-set-all.performed ; " \
"ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select versions | grep ^2.3 | tail -1`",
only_if = "ls -d /usr/hdp/2.3*",
not_if = "test -f /var/lib/ambari-agent/data/hdp-select-set-all.performed")
self.assertResourceCalled('XmlConfig', 'core-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = "/usr/hdp/current/hadoop-client/conf",
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
only_if="ls /usr/hdp/current/hadoop-client/conf")
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/kms/conf', '/etc/ranger/kms/conf.install'),
not_if = 'test -e /etc/ranger/kms/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/ranger/kms/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/ranger/kms/conf',
to = '/usr/hdp/current/ranger-kms/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/zookeeper/conf', '/etc/zookeeper/conf.install'),
not_if = 'test -e /etc/zookeeper/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/zookeeper/conf',
to = '/usr/hdp/current/zookeeper-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/pig/conf', '/etc/pig/conf.install'),
not_if = 'test -e /etc/pig/conf.install',
sudo = True,)
# pig fails, so no Directory/Link combo
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/tez/conf', '/etc/tez/conf.install'),
not_if = 'test -e /etc/tez/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/tez/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/tez/conf',
to = '/usr/hdp/current/tez-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive-webhcat/conf', '/etc/hive-webhcat/conf.install'),
not_if = 'test -e /etc/hive-webhcat/conf.install',
sudo = True,)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive-hcatalog/conf', '/etc/hive-hcatalog/conf.install'),
not_if = 'test -e /etc/hive-hcatalog/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/hive-webhcat/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/hive-webhcat/conf',
to = '/usr/hdp/current/hive-webhcat/etc/webhcat',)
self.assertResourceCalled('Directory', '/etc/hive-hcatalog/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/hive-hcatalog/conf',
to = '/usr/hdp/current/hive-webhcat/etc/hcatalog',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hbase/conf', '/etc/hbase/conf.install'),
not_if = 'test -e /etc/hbase/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/hbase/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/hbase/conf',
to = '/usr/hdp/current/hbase-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/knox/conf', '/etc/knox/conf.install'),
not_if = 'test -e /etc/knox/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/knox/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/knox/conf',
to = '/usr/hdp/current/knox-server/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/usersync/conf', '/etc/ranger/usersync/conf.install'),
not_if = 'test -e /etc/ranger/usersync/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/ranger/usersync/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/ranger/usersync/conf',
to = '/usr/hdp/current/ranger-usersync/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hadoop/conf', '/etc/hadoop/conf.install'),
not_if = 'test -e /etc/hadoop/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/hadoop/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/hadoop/conf',
to = '/usr/hdp/current/hadoop-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/mahout/conf', '/etc/mahout/conf.install'),
not_if = 'test -e /etc/mahout/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/mahout/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/mahout/conf',
to = '/usr/hdp/current/mahout-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm/conf', '/etc/storm/conf.install'),
not_if = 'test -e /etc/storm/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/storm/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/storm/conf',
to = '/usr/hdp/current/storm-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/ranger/admin/conf', '/etc/ranger/admin/conf.install'),
not_if = 'test -e /etc/ranger/admin/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/ranger/admin/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/ranger/admin/conf',
to = '/usr/hdp/current/ranger-admin/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/flume/conf', '/etc/flume/conf.install'),
not_if = 'test -e /etc/flume/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/flume/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/flume/conf',
to = '/usr/hdp/current/flume-server/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/sqoop/conf', '/etc/sqoop/conf.install'),
not_if = 'test -e /etc/sqoop/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/sqoop/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/sqoop/conf',
to = '/usr/hdp/current/sqoop-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/accumulo/conf', '/etc/accumulo/conf.install'),
not_if = 'test -e /etc/accumulo/conf.install',
sudo = True,)
self.assertResourceCalled('Directory', '/etc/accumulo/conf',
action = ['delete'],)
self.assertResourceCalled('Link', '/etc/accumulo/conf',
to = '/usr/hdp/current/accumulo-client/conf',)
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/phoenix/conf', '/etc/phoenix/conf.install'),
not_if = 'test -e /etc/phoenix/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/phoenix/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/phoenix/conf',
to = '/usr/hdp/current/phoenix-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/storm-slider-client/conf', '/etc/storm-slider-client/conf.install'),
not_if = 'test -e /etc/storm-slider-client/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/storm-slider-client/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/storm-slider-client/conf',
to = '/usr/hdp/current/storm-slider-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/slider/conf', '/etc/slider/conf.install'),
not_if = 'test -e /etc/slider/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/slider/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/slider/conf',
to = '/usr/hdp/current/slider-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/oozie/conf', '/etc/oozie/conf.install'),
not_if = 'test -e /etc/oozie/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/oozie/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/oozie/conf',
to = '/usr/hdp/current/oozie-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/falcon/conf', '/etc/falcon/conf.install'),
not_if = 'test -e /etc/falcon/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/falcon/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/falcon/conf',
to = '/usr/hdp/current/falcon-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/spark/conf', '/etc/spark/conf.install'),
not_if = 'test -e /etc/spark/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/spark/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/spark/conf',
to = '/usr/hdp/current/spark-client/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/kafka/conf', '/etc/kafka/conf.install'),
not_if = 'test -e /etc/kafka/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/kafka/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/kafka/conf',
to = '/usr/hdp/current/kafka-broker/conf')
self.assertResourceCalled('Execute', ('cp', '-R', '-p', '/etc/hive/conf', '/etc/hive/conf.install'),
not_if = 'test -e /etc/hive/conf.install',
sudo = True)
self.assertResourceCalled('Directory', '/etc/hive/conf',
action = ['delete'])
self.assertResourceCalled('Link', '/etc/hive/conf',
to = '/usr/hdp/current/hive-client/conf')
self.assertNoMoreResources()
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import json
from mock.mock import MagicMock, call, patch
from stacks.utils.RMFTestCase import *
@patch("os.path.exists", new = MagicMock(return_value=True))
@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
class TestZookeeperClient(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "ZOOKEEPER/3.4.5.2.0/package"
STACK_VERSION = "2.0.6"
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/zookeeper_client.py",
classname = "ZookeeperClient",
command = "configure",
config_file = "default.json",
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/etc/zookeeper/conf',
def test_create_wallet_testnet_created(self):
plebnet_settings.Init.wallets_testnet_created = MagicMock(
return_value=True)
assert walletcontroller.create_wallet('TBTC')
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import json
from mock.mock import MagicMock, patch
import tempfile
import tarfile
import contextlib
from resource_management import *
from stacks.utils.RMFTestCase import *
@patch("platform.linux_distribution", new=MagicMock(return_value="Linux"))
@patch.object(tarfile, "open", new=MagicMock())
@patch.object(tempfile, "mkdtemp", new=MagicMock(return_value='/tmp/123'))
@patch.object(contextlib, "closing", new=MagicMock())
@patch("os.path.exists", new=MagicMock(return_value=True))
class Test(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "HDFS/2.1.0.2.0/package"
STACK_VERSION = "2.0.6"
def test_generate_configs_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/hdfs_client.py",
classname="HdfsClient",
command="generate_configs",
config_file="default.json",
hdp_stack_version=self.STACK_VERSION,
def test_create_wallet_no_market(self):
marketcontroller.is_market_running = MagicMock(return_value=False)
self.assertFalse(walletcontroller.create_wallet('TBTC'))
def test_auto_execute_command(self, status_update_callback_mock,
open_mock):
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
dummy_controller.recovery_manager.update_config(
5, 5, 1, 11, True, False, "")
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(command,
tmpoutfile,
tmperrfile,
override_output_files=True,
retry=False):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.process_command(command)
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
self.assertFalse(actionQueue.tasks_in_progress_or_pending())
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_auto_start_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
if actionQueue.tasks_in_progress_or_pending():
break
# Continue command execution
unfreeze_flag.set()
# wait until ready
check_queue = True
while check_queue:
report = actionQueue.result()
if not actionQueue.tasks_in_progress_or_pending():
break
time.sleep(0.1)
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_auto_start_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
while check_queue:
report = actionQueue.result()
if not actionQueue.tasks_in_progress_or_pending():
break
time.sleep(0.1)
self.assertEqual(len(report['reports']), 0)
class TestRepositoryResource(TestCase):
@patch.object(System, "os_family", new='redhat')
@patch("resource_management.libraries.providers.repository.File")
def test_create_repo_redhat(self, file_mock):
with Environment('/') as env:
with patch.object(
repository,
"Template",
new=DummyTemplate.create(RHEL_SUSE_DEFAULT_TEMPLATE)):
Repository(
'hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository',
repo_template='dummy.j2')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository'][
'hadoop'].arguments
expected_arguments = {
'repo_template': 'dummy.j2',
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
expected_template_arguments = {
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[0][0],
'/etc/yum.repos.d/Repository.repo')
template_item = file_mock.call_args[1]['content']
template = str(template_item.name)
expected_template_arguments.update({'repo_id': 'hadoop'})
self.assertEqual(expected_template_arguments,
template_item.context._dict)
self.assertEqual('dummy.j2', template)
@patch.object(System, "os_family", new='suse')
@patch("resource_management.libraries.providers.repository.File")
def test_create_repo_suse(self, file_mock):
with Environment('/') as env:
with patch.object(
repository,
"Template",
new=DummyTemplate.create(RHEL_SUSE_DEFAULT_TEMPLATE)):
Repository(
'hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_template="dummy.j2",
repo_file_name='Repository')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository'][
'hadoop'].arguments
expected_arguments = {
'repo_template': 'dummy.j2',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'base_url': 'http://download.base_url.org/rpm/',
'repo_file_name': 'Repository'
}
expected_template_arguments = {
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'base_url': 'http://download.base_url.org/rpm/',
'repo_file_name': 'Repository'
}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[0][0],
'/etc/zypp/repos.d/Repository.repo')
template_item = file_mock.call_args[1]['content']
template = str(template_item.name)
expected_template_arguments.update({'repo_id': 'hadoop'})
self.assertEqual(expected_template_arguments,
template_item.context._dict)
self.assertEqual('dummy.j2', template)
@patch("resource_management.libraries.providers.repository.checked_call")
@patch.object(tempfile, "NamedTemporaryFile")
@patch("resource_management.libraries.providers.repository.Execute")
@patch("resource_management.libraries.providers.repository.File")
@patch("os.path.isfile", new=MagicMock(return_value=True))
@patch("filecmp.cmp", new=MagicMock(return_value=False))
@patch.object(System, "os_release_name", new='precise')
@patch.object(System, "os_family", new='ubuntu')
def test_create_repo_ubuntu_repo_exists(self, file_mock, execute_mock,
tempfile_mock, checked_call_mock):
tempfile_mock.return_value = MagicMock(spec=file)
tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD"
with Environment('/') as env:
with patch.object(
repository,
"Template",
new=DummyTemplate.create(DEBIAN_DEFAUTL_TEMPLATE)):
Repository('HDP',
base_url='http://download.base_url.org/rpm/',
repo_file_name='HDP',
repo_template="dummy.j2",
components=['a', 'b', 'c'])
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']
self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content,
'deb http://download.base_url.org/rpm/ a b c\n')
copy_item = str(file_mock.call_args_list[1])
self.assertEqual(
copy_item,
"call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))"
)
#'apt-get update -qq -o Dir::Etc::sourcelist="sources.list.d/HDP.list" -o APT::Get::List-Cleanup="0"')
execute_command_item = execute_mock.call_args_list[0][0][0]
self.assertEqual(checked_call_mock.call_args_list[0][0][0], [
'apt-get', 'update', '-qq', '-o',
'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o',
'APT::Get::List-Cleanup=0'
])
self.assertEqual(
execute_command_item,
'apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 123ABCD')
@patch("resource_management.libraries.providers.repository.checked_call")
@patch.object(tempfile, "NamedTemporaryFile")
@patch("resource_management.libraries.providers.repository.Execute")
@patch("resource_management.libraries.providers.repository.File")
@patch("os.path.isfile", new=MagicMock(return_value=True))
@patch("filecmp.cmp", new=MagicMock(return_value=False))
@patch.object(System, "os_release_name", new='precise')
@patch.object(System, "os_family", new='ubuntu')
def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock,
execute_mock,
tempfile_mock,
checked_call_mock):
"""
Checks that GPG key is extracted from output without \r sign
"""
tempfile_mock.return_value = MagicMock(spec=file)
tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n"
with Environment('/') as env:
with patch.object(
repository,
"Template",
new=DummyTemplate.create(DEBIAN_DEFAUTL_TEMPLATE)):
Repository('HDP',
base_url='http://download.base_url.org/rpm/',
repo_file_name='HDP',
repo_template="dummy.j2",
components=['a', 'b', 'c'])
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']
self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content,
'deb http://download.base_url.org/rpm/ a b c\n')
copy_item = str(file_mock.call_args_list[1])
self.assertEqual(
copy_item,
"call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))"
)
execute_command_item = execute_mock.call_args_list[0][0][0]
self.assertEqual(checked_call_mock.call_args_list[0][0][0], [
'apt-get', 'update', '-qq', '-o',
'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o',
'APT::Get::List-Cleanup=0'
])
self.assertEqual(
execute_command_item,
'apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 123ABCD')
@patch.object(tempfile, "NamedTemporaryFile")
@patch("resource_management.libraries.providers.repository.Execute")
@patch("resource_management.libraries.providers.repository.File")
@patch("os.path.isfile", new=MagicMock(return_value=True))
@patch("filecmp.cmp", new=MagicMock(return_value=True))
@patch.object(System, "os_release_name", new='precise')
@patch.object(System, "os_family", new='ubuntu')
def test_create_repo_ubuntu_doesnt_repo_exist(self, file_mock,
execute_mock, tempfile_mock):
tempfile_mock.return_value = MagicMock(spec=file)
tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
with Environment('/') as env:
with patch.object(
repository,
"Template",
new=DummyTemplate.create(DEBIAN_DEFAUTL_TEMPLATE)):
Repository('HDP',
base_url='http://download.base_url.org/rpm/',
repo_file_name='HDP',
repo_template="dummy.j2",
components=['a', 'b', 'c'])
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']
self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content,
'deb http://download.base_url.org/rpm/ a b c\n')
self.assertEqual(file_mock.call_count, 1)
self.assertEqual(execute_mock.call_count, 0)
@patch("os.path.isfile", new=MagicMock(return_value=True))
@patch.object(System, "os_family", new='ubuntu')
@patch("resource_management.libraries.providers.repository.Execute")
@patch("resource_management.libraries.providers.repository.File")
def test_remove_repo_ubuntu_repo_exist(self, file_mock, execute_mock):
with Environment('/') as env:
Repository('HDP', action="remove", repo_file_name='HDP')
self.assertEqual(
str(file_mock.call_args),
"call('/etc/apt/sources.list.d/HDP.list', action='delete')")
self.assertEqual(execute_mock.call_args[0][0], [
'apt-get', 'update', '-qq', '-o',
'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o',
'APT::Get::List-Cleanup=0'
])
@patch("os.path.isfile", new=MagicMock(return_value=False))
@patch.object(System, "os_family", new='ubuntu')
@patch("resource_management.libraries.providers.repository.Execute")
@patch("resource_management.libraries.providers.repository.File")
def test_remove_repo_ubuntu_repo_doenst_exist(self, file_mock,
execute_mock):
with Environment('/') as env:
Repository('HDP', action="remove", repo_file_name='HDP')
self.assertEqual(file_mock.call_count, 0)
self.assertEqual(execute_mock.call_count, 0)
@patch.object(System, "os_family", new='redhat')
@patch("resource_management.libraries.providers.repository.File")
def test_remove_repo_redhat(self, file_mock):
with Environment('/') as env:
Repository(
'hadoop',
action='remove',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
expected_arguments = {
'action': ['remove'],
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[1]['action'], 'delete')
self.assertEqual(file_mock.call_args[0][0],
'/etc/yum.repos.d/Repository.repo')
@patch.object(System, "os_family", new='suse')
@patch("resource_management.libraries.providers.repository.File")
def test_remove_repo_suse(self, file_mock):
with Environment('/') as env:
Repository(
'hadoop',
action='remove',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
expected_arguments = {
'action': ['remove'],
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[1]['action'], 'delete')
self.assertEqual(file_mock.call_args[0][0],
'/etc/zypp/repos.d/Repository.repo')
def test_execute_command(self, status_update_callback_mock, open_mock):
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(command,
tmpoutfile,
tmperrfile,
override_output_files=True,
retry=False):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(CustomServiceOrchestrator,
"runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
report = actionQueue.result()
if len(report['reports']) != 0:
break
expected = {
'status':
'IN_PROGRESS',
'stderr':
'Read from {0}'.format(os.path.join(tempdir, "errors-3.txt")),
'stdout':
'Read from {0}'.format(os.path.join(tempdir, "output-3.txt")),
'structuredOut':
'Read from {0}'.format(
os.path.join(tempdir, "structured-out-3.json")),
'clusterName':
u'cc',
'roleCommand':
u'INSTALL',
'serviceName':
u'HDFS',
'role':
u'DATANODE',
'actionId':
'1-1',
'taskId':
3,
'exitCode':
777
}
self.assertEqual(report['reports'][0], expected)
self.assertTrue(actionQueue.tasks_in_progress_or_pending())
# Continue command execution
unfreeze_flag.set()
# wait until ready
while report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
configname = os.path.join(tempdir, 'config.json')
expected = {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'exitCode': 0
}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
# Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
self.assertEqual(status_update_callback_mock.call_count, 2)
os.remove(configname)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {
'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 13
}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
### Test upgrade command ###
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_upgrade_command, ))
execution_thread.start()
unfreeze_flag.set()
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'structuredOut': '""',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitCode': 0
}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
def test_build_long_result(self, result_mock):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
actionQueue = ActionQueue(config, dummy_controller)
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 777},
{'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitCode': 0},
{'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 13},
{'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {'global': {'tag': 'v1'}},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 0}
],
'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE'},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE'},
],
}
heartbeat = Heartbeat(actionQueue)
hb = heartbeat.build(10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
expected = {'nodeStatus':
{'status': 'HEALTHY',
'cause': 'NONE'},
'recoveryReport': {'summary': 'DISABLED'},
'recoveryTimestamp': -1,
'timestamp': 'timestamp', 'hostname': 'hostname',
'responseId': 10, 'reports': [
{'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc',
'taskId': 3, 'exitCode': 777},
{'status': 'COMPLETED', 'roleCommand': 'UPGRADE',
'serviceName': 'serviceName', 'role': 'role', 'actionId': 17,
'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName',
'taskId': 'taskId', 'exitCode': 0},
{'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS',
'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr',
'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitCode': 13},
{'status': 'COMPLETED', 'stdout': 'out',
'configurationTags': {'global': {'tag': 'v1'}}, 'taskId': 3,
'exitCode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'stderr'}], 'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE'},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE'}]}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
ANSIBLE_METADATA = {
'metadata_version': '1.1',
'status': ['preview'],
'supported_by': 'community'
}
import pytest
from mock.mock import MagicMock
from ansible_collections.dellemc.powerscale.plugins.module_utils.storage.dell \
import dellemc_ansible_powerscale_utils as utils
utils.get_logger = MagicMock()
utils.isi_sdk = MagicMock()
from ansible.module_utils import basic
basic.AnsibleModule = MagicMock()
from ansible_collections.dellemc.powerscale.plugins.modules.dellemc_powerscale_accesszone import PowerScaleAccessZone
from ansible_collections.dellemc.powerscale.tests.unit.plugins.\
module_utils import mock_accesszone_api as MockAccessZoneApi
from ansible_collections.dellemc.powerscale.tests.unit.plugins.module_utils.mock_sdk_response \
import MockSDKResponse
from ansible_collections.dellemc.powerscale.tests.unit.plugins.module_utils.mock_api_exception \
import MockApiException
class TestPowerScaleAccessZone():
get_access_zone_args = {
class TestHeartbeat(TestCase):
def setUp(self):
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
def tearDown(self):
# enable stdout
sys.stdout = sys.__stdout__
def test_build(self):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
dummy_controller.recovery_manager.recovery_timestamp = -1
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue)
result = heartbeat.build(100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True, "hostname should not be empty")
self.assertEquals(result['responseId'], 100)
self.assertEquals(result['componentStatus'] is not None, True, "Heartbeat should contain componentStatus")
self.assertEquals(result['reports'] is not None, True, "Heartbeat should contain reports")
self.assertEquals(result['timestamp'] >= 1353679373880L, True)
self.assertEquals(result['recoveryTimestamp'], -1)
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 7) or (len(result) is 8), True)
self.assertEquals(not heartbeat.reports, True, "Heartbeat should not contain task in progress")
@patch("subprocess.Popen")
@patch.object(Hardware, "_chk_writable_mount", new = MagicMock(return_value=True))
@patch.object(ActionQueue, "result")
@patch.object(HostInfoLinux, "register")
def test_no_mapping(self, register_mock, result_mock, Popen_mock):
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 777}],
'componentStatus': [{'status': 'HEALTHY', 'componentName': 'NAMENODE'}]
}
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue)
hb = heartbeat.build(id = 10, add_state=True, componentsMapped=True)
self.assertEqual(register_mock.call_args_list[0][0][1], True)
register_mock.reset_mock()
hb = heartbeat.build(id = 0, add_state=True, componentsMapped=True)
self.assertEqual(register_mock.call_args_list[0][0][1], False)
@patch.object(ActionQueue, "result")
def test_build_long_result(self, result_mock):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
actionQueue = ActionQueue(config, dummy_controller)
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 777},
{'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitCode': 0},
{'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 13},
{'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {'global': {'tag': 'v1'}},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 0}
],
'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE'},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE'},
],
}
heartbeat = Heartbeat(actionQueue)
hb = heartbeat.build(10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
expected = {'nodeStatus':
{'status': 'HEALTHY',
'cause': 'NONE'},
'recoveryReport': {'summary': 'DISABLED'},
'recoveryTimestamp': -1,
'timestamp': 'timestamp', 'hostname': 'hostname',
'responseId': 10, 'reports': [
{'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc',
'taskId': 3, 'exitCode': 777},
{'status': 'COMPLETED', 'roleCommand': 'UPGRADE',
'serviceName': 'serviceName', 'role': 'role', 'actionId': 17,
'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName',
'taskId': 'taskId', 'exitCode': 0},
{'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS',
'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr',
'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitCode': 13},
{'status': 'COMPLETED', 'stdout': 'out',
'configurationTags': {'global': {'tag': 'v1'}}, 'taskId': 3,
'exitCode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'stderr'}], 'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE'},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE'}]}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
@patch("subprocess.Popen")
@patch.object(Hardware, "_chk_writable_mount", new = MagicMock(return_value=True))
@patch.object(HostInfoLinux, 'register')
def test_heartbeat_no_host_check_cmd_in_queue(self, register_mock, Popen_mock):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.statusCommandQueue = multiprocessing.Queue()
statusCommand = {
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "c1",
"componentName" : "DATANODE",
"role" : "DATANODE",
'configurations':{'global' : {}}
}
actionQueue.put_status([statusCommand])
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertFalse(args[2])
self.assertFalse(args[1])
@patch("subprocess.Popen")
@patch.object(Hardware, "_chk_writable_mount", new = MagicMock(return_value=True))
@patch.object(HostInfoLinux, 'register')
def test_heartbeat_host_check_no_cmd(self, register_mock, Popen_mock):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertFalse(args[1])
self.assertFalse(args[2])
import json
import os
from stacks.utils.RMFTestCase import *
from mock.mock import MagicMock, patch
from resource_management.libraries import functions
from resource_management.core.logger import Logger
from resource_management.libraries.script.config_dictionary import UnknownConfiguration
from hive_server_interactive import HiveServerInteractiveDefault
from resource_management.libraries.script.script import Script
from resource_management.core import shell
@patch("resource_management.libraries.Script.get_tmp_dir",
new=MagicMock(return_value=('/var/lib/ambari-agent/tmp')))
@patch.object(functions,
"get_stack_version",
new=MagicMock(return_value="2.0.0.0-1234"))
@patch("resource_management.libraries.functions.check_thrift_port_sasl",
new=MagicMock())
@patch(
"resource_management.libraries.functions.get_user_call_output.get_user_call_output",
new=MagicMock(return_value=(0, '123', '')))
class TestHiveServerInteractive(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "HIVE/0.12.0.2.0/package"
STACK_VERSION = "2.0.6"
UPGRADE_STACK_VERSION = "2.2"
DEFAULT_IMMUTABLE_PATHS = [
'/apps/hive/warehouse', '/apps/falcon', '/mr-history/done',
'/app-logs', '/tmp'
def setUp(self):
HostCleanup.logger = MagicMock()
self.hostcleanup = HostCleanup.HostCleanup()
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
class TestCheckHost(TestCase):
current_dir = os.path.dirname(os.path.realpath(__file__))
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("os.path.isfile")
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch("resource_management.libraries.script.Script.put_structured_out")
def testJavaHomeAvailableCheck(self, structured_out_mock, get_tmp_dir_mock,
mock_config, os_isfile_mock):
# test, java home exists
os_isfile_mock.return_value = True
get_tmp_dir_mock.return_value = "/tmp"
mock_config.return_value = {
"commandParams": {
"check_execute_list": "java_home_check",
"java_home": "test_java_home"
}
}
checkHost = CheckHost()
checkHost.actionexecute(None)
self.assertEquals(structured_out_mock.call_args[0][0], {
'java_home_check': {
'message': 'Java home exists!',
'exit_code': 0
}
})
# test, java home doesn't exist
os_isfile_mock.reset_mock()
os_isfile_mock.return_value = False
checkHost.actionexecute(None)
self.assertEquals(
structured_out_mock.call_args[0][0], {
'java_home_check': {
"message": "Java home doesn't exist!",
"exit_code": 1
}
})
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch("check_host.download_file")
@patch("resource_management.libraries.script.Script.put_structured_out")
@patch("check_host.format")
@patch("os.path.isfile")
@patch("resource_management.core.shell.call")
def testDBConnectionCheck(self, shell_call_mock, isfile_mock, format_mock,
structured_out_mock, download_file_mock,
get_tmp_dir_mock, mock_config):
# test, download DBConnectionVerification.jar failed
mock_config.return_value = {
"commandParams": {
"check_execute_list": "db_connection_check",
"java_home": "test_java_home",
"ambari_server_host": "test_host",
"jdk_location": "test_jdk_location",
"db_name": "mysql",
"db_connection_url": "test_db_connection_url",
"user_name": "test_user_name",
"user_passwd": "test_user_passwd",
"jdk_name": "test_jdk_name"
},
"hostLevelParams": {
"agentCacheDir": "/nonexistent_tmp"
}
}
get_tmp_dir_mock.return_value = "/tmp"
download_file_mock.side_effect = Exception("test exception")
isfile_mock.return_value = True
checkHost = CheckHost()
checkHost.actionexecute(None)
self.assertEquals(structured_out_mock.call_args[0][0], {'db_connection_check': {'message': 'Error downloading ' \
'DBConnectionVerification.jar from Ambari Server resources. Check network access to Ambari ' \
'Server.\ntest exception', 'exit_code': 1}})
# test, download jdbc driver failed
mock_config.return_value = {
"commandParams": {
"check_execute_list": "db_connection_check",
"java_home": "test_java_home",
"ambari_server_host": "test_host",
"jdk_location": "test_jdk_location",
"db_name": "oracle",
"db_connection_url": "test_db_connection_url",
"user_name": "test_user_name",
"user_passwd": "test_user_passwd",
"jdk_name": "test_jdk_name"
},
"hostLevelParams": {
"agentCacheDir": "/nonexistent_tmp"
}
}
format_mock.reset_mock()
download_file_mock.reset_mock()
p = MagicMock()
download_file_mock.side_effect = [p, Exception("test exception")]
checkHost.actionexecute(None)
self.assertEquals(
format_mock.call_args[0][0],
'Error: Ambari Server cannot download the database JDBC driver '
'and is unable to test the database connection. You must run ambari-server setup '
'--jdbc-db={db_name} --jdbc-driver=/path/to/your/{db_name}/driver.jar on the Ambari '
'Server host to make the JDBC driver available for download and to enable testing '
'the database connection.\n')
self.assertEquals(
structured_out_mock.call_args[0][0]['db_connection_check']
['exit_code'], 1)
# test, no connection to remote db
mock_config.return_value = {
"commandParams": {
"check_execute_list": "db_connection_check",
"java_home": "test_java_home",
"ambari_server_host": "test_host",
"jdk_location": "test_jdk_location",
"db_name": "postgres",
"db_connection_url": "test_db_connection_url",
"user_name": "test_user_name",
"user_passwd": "test_user_passwd",
"jdk_name": "test_jdk_name"
},
"hostLevelParams": {
"agentCacheDir": "/nonexistent_tmp"
}
}
format_mock.reset_mock()
download_file_mock.reset_mock()
download_file_mock.side_effect = [p, p]
shell_call_mock.return_value = (1, "test message")
checkHost.actionexecute(None)
self.assertEquals(structured_out_mock.call_args[0][0], {
'db_connection_check': {
'message': 'test message',
'exit_code': 1
}
})
self.assertEquals(
format_mock.call_args[0][0],
'{java_exec} -cp {check_db_connection_path}{class_path_delimiter}'
'{jdbc_jar_path} -Djava.library.path={java_library_path} org.apache.ambari.server.DBConnectionVerification'
' "{db_connection_url}" {user_name} {user_passwd!p} {jdbc_driver_class}'
)
# test, db connection success
download_file_mock.reset_mock()
download_file_mock.side_effect = [p, p]
shell_call_mock.return_value = (0, "test message")
checkHost.actionexecute(None)
self.assertEquals(
structured_out_mock.call_args[0][0], {
'db_connection_check': {
'message': 'DB connection check completed successfully!',
'exit_code': 0
}
})
#test jdk_name and java home are not available
mock_config.return_value = {
"commandParams": {
"check_execute_list": "db_connection_check",
"java_home": "test_java_home",
"ambari_server_host": "test_host",
"jdk_location": "test_jdk_location",
"db_connection_url": "test_db_connection_url",
"user_name": "test_user_name",
"user_passwd": "test_user_passwd",
"db_name": "postgres"
},
"hostLevelParams": {
"agentCacheDir": "/nonexistent_tmp"
}
}
isfile_mock.return_value = False
checkHost.actionexecute(None)
self.assertEquals(structured_out_mock.call_args[0][0], {'db_connection_check': {'message': 'Custom java is not ' \
'available on host. Please install it. Java home should be the same as on server. \n', 'exit_code': 1}})
pass
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("socket.gethostbyname")
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch("resource_management.libraries.script.Script.put_structured_out")
def testHostResolution(self, structured_out_mock, get_tmp_dir_mock,
mock_config, mock_socket):
mock_socket.return_value = "192.168.1.1"
jsonFilePath = os.path.join(
TestCheckHost.current_dir + "/../../resources/custom_actions",
"check_host_ip_addresses.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
mock_config.return_value = ConfigDictionary(jsonPayload)
get_tmp_dir_mock.return_value = "/tmp"
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(structured_out_mock.called)
structured_out_mock.assert_called_with({
'host_resolution_check': {
'failures': [],
'message': 'All hosts resolved to an IP address.',
'failed_count': 0,
'success_count': 5,
'exit_code': 0
}
})
# try it now with errors
mock_socket.side_effect = socket.error
checkHost.actionexecute(None)
structured_out_mock.assert_called_with({
'host_resolution_check': {
'failures': [{
'cause': (),
'host': u'c6401.ambari.apache.org',
'type': 'FORWARD_LOOKUP'
}, {
'cause': (),
'host': u'c6402.ambari.apache.org',
'type': 'FORWARD_LOOKUP'
}, {
'cause': (),
'host': u'c6403.ambari.apache.org',
'type': 'FORWARD_LOOKUP'
}, {
'cause': (),
'host': u'foobar',
'type': 'FORWARD_LOOKUP'
}, {
'cause': (),
'host': u'!!!',
'type': 'FORWARD_LOOKUP'
}],
'message':
'There were 5 host(s) that could not resolve to an IP address.',
'failed_count':
5,
'success_count':
0,
'exit_code':
0
}
})
pass
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch("resource_management.libraries.script.Script.put_structured_out")
def testInvalidCheck(self, structured_out_mock, get_tmp_dir_mock,
mock_config):
jsonFilePath = os.path.join(
TestCheckHost.current_dir + "/../../resources/custom_actions",
"invalid_check.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
mock_config.return_value = ConfigDictionary(jsonPayload)
get_tmp_dir_mock.return_value = "tmp"
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(structured_out_mock.called)
structured_out_mock.assert_called_with({})
pass
@not_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("platform.system")
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch('resource_management.libraries.script.Script.put_structured_out')
@patch('ambari_agent.HostInfo.HostInfoLinux.javaProcs')
@patch('ambari_agent.HostInfo.HostInfoLinux.checkLiveServices')
@patch('ambari_agent.HostInfo.HostInfoLinux.getUMask')
@patch('ambari_agent.HostInfo.HostInfoLinux.getTransparentHugePage')
@patch('ambari_agent.HostInfo.HostInfoLinux.checkFirewall')
@patch('ambari_agent.HostInfo.HostInfoLinux.checkReverseLookup')
@patch('time.time')
def testLastAgentEnv(self, time_mock, checkReverseLookup_mock,
checkFirewall_mock, getTransparentHugePage_mock,
getUMask_mock, checkLiveServices_mock, javaProcs_mock,
put_structured_out_mock, get_tmp_dir_mock,
get_config_mock, systemmock):
jsonFilePath = os.path.join(
TestCheckHost.current_dir + "/../../resources/custom_actions",
"check_last_agent_env.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
get_config_mock.return_value = ConfigDictionary(jsonPayload)
get_tmp_dir_mock.return_value = "/tmp"
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(time_mock.called)
self.assertTrue(checkReverseLookup_mock.called)
self.assertTrue(checkFirewall_mock.called)
self.assertTrue(getTransparentHugePage_mock.called)
self.assertTrue(getUMask_mock.called)
self.assertTrue(checkLiveServices_mock.called)
self.assertTrue(javaProcs_mock.called)
self.assertTrue(put_structured_out_mock.called)
# ensure the correct keys are in the result map
last_agent_env_check_result = put_structured_out_mock.call_args[0][0]
self.assertTrue('last_agent_env_check' in last_agent_env_check_result)
self.assertTrue('hostHealth' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('firewallRunning' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('firewallName' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('reverseLookup' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('alternatives' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue(
'umask' in last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('stackFoldersAndFiles' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('existingUsers' in
last_agent_env_check_result['last_agent_env_check'])
# try it now with errors
javaProcs_mock.side_effect = Exception("test exception")
checkHost.actionexecute(None)
#ensure the correct response is returned
put_structured_out_mock.assert_called_with({
'last_agent_env_check': {
'message': 'test exception',
'exit_code': 1
}
})
pass
@only_for_platform(PLATFORM_WINDOWS)
@patch.object(OSCheck,
"os_distribution",
new=MagicMock(return_value=os_distro_value))
@patch("platform.system")
@patch.object(Script, 'get_config')
@patch.object(Script, 'get_tmp_dir')
@patch('resource_management.libraries.script.Script.put_structured_out')
@patch('ambari_agent.HostInfo.HostInfoWindows.javaProcs')
@patch('ambari_agent.HostInfo.HostInfoWindows.checkLiveServices')
@patch('ambari_agent.HostInfo.HostInfoWindows.getUMask')
@patch('ambari_agent.HostInfo.HostInfoWindows.checkFirewall')
@patch('ambari_agent.HostInfo.HostInfoWindows.checkReverseLookup')
@patch('time.time')
def testLastAgentEnv(self, time_mock, checkReverseLookup_mock,
checkFirewall_mock, getUMask_mock,
checkLiveServices_mock, javaProcs_mock,
put_structured_out_mock, get_tmp_dir_mock,
get_config_mock, systemmock):
jsonFilePath = os.path.join(TestCheckHost.current_dir, "..", "..",
"resources", "custom_actions",
"check_last_agent_env.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
get_config_mock.return_value = ConfigDictionary(jsonPayload)
get_tmp_dir_mock.return_value = "/tmp"
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(time_mock.called)
self.assertTrue(checkReverseLookup_mock.called)
self.assertTrue(checkFirewall_mock.called)
self.assertTrue(getUMask_mock.called)
self.assertTrue(checkLiveServices_mock.called)
self.assertTrue(javaProcs_mock.called)
self.assertTrue(put_structured_out_mock.called)
# ensure the correct keys are in the result map
last_agent_env_check_result = put_structured_out_mock.call_args[0][0]
self.assertTrue('last_agent_env_check' in last_agent_env_check_result)
self.assertTrue('hostHealth' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('firewallRunning' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('firewallName' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('alternatives' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue(
'umask' in last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('stackFoldersAndFiles' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('existingUsers' in
last_agent_env_check_result['last_agent_env_check'])
# try it now with errors
javaProcs_mock.side_effect = Exception("test exception")
checkHost.actionexecute(None)
#ensure the correct response is returned
put_structured_out_mock.assert_called_with({
'last_agent_env_check': {
'message': 'test exception',
'exit_code': 1
}
})
pass
@patch.object(HostCheckReportFileHandler, "resolve_ambari_config")
@patch("resource_management.libraries.script.Script.put_structured_out")
@patch.object(Script, 'get_tmp_dir')
@patch.object(Script, 'get_config')
@patch("os.path.isfile")
@patch('__builtin__.open')
def testTransparentHugePage(self, open_mock, os_path_isfile_mock,
mock_config, get_tmp_dir_mock,
structured_out_mock, resolve_config_mock):
context_manager_mock = MagicMock()
open_mock.return_value = context_manager_mock
file_mock = MagicMock()
file_mock.read.return_value = "[never] always"
enter_mock = MagicMock()
enter_mock.return_value = file_mock
enter_mock = MagicMock()
enter_mock.return_value = file_mock
exit_mock = MagicMock()
setattr(context_manager_mock, '__enter__', enter_mock)
setattr(context_manager_mock, '__exit__', exit_mock)
os_path_isfile_mock.return_value = True
get_tmp_dir_mock.return_value = "/tmp"
mock_config.return_value = {
"commandParams": {
"check_execute_list": "transparentHugePage"
}
}
checkHost = CheckHost()
checkHost.actionexecute(None)
self.assertEquals(
structured_out_mock.call_args[0][0],
{'transparentHugePage': {
'message': 'never',
'exit_code': 0
}})
# case 2, file not exists
os_path_isfile_mock.return_value = False
checkHost.actionexecute(None)
self.assertEquals(
structured_out_mock.call_args[0][0],
{'transparentHugePage': {
'message': '',
'exit_code': 0
}})
def test_runCommand_with_config(self, run_file_mock,
resolve_script_path_mock, unlink_mock,
isfile_mock, hostname_mock):
hostname_mock.return_value = "test.hst"
isfile_mock.return_value = True
command = {
'role': 'REGION_SERVER',
'hostLevelParams': {
'stack_name': 'HDP',
'stack_version': '2.0.7',
'jdk_location': 'some_location'
},
'commandParams': {
'script_type': 'PYTHON',
'script': 'scripts/hbase_regionserver.py',
'command_timeout': '600',
'service_package_folder': 'HBASE'
},
'configurations': {
"hbase-site": {
"hbase.log": "${AGENT_LOG_ROOT}",
"hbase.number": "10485760"
},
"hbase-log4j": {
"a": "b"
}
},
'taskId': '3',
'roleCommand': 'INSTALL',
'commandType': 'EXECUTION_COMMAND',
'commandId': '1-1'
}
command_get = {
'roleCommand': 'GET_CONFIG',
'commandType': 'STATUS_COMMAND'
}
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
resolve_script_path_mock.return_value = "/basedir/scriptpath"
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
# normal run case
run_file_mock.return_value = {
'stdout': 'sss',
'stderr': 'eee',
'exitcode': 0,
}
expected = {
'hbase-site': {
'hbase.log': tempdir,
'hbase.number': '10485760'
},
'hbase-log4j': {
'a': 'b'
}
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt", True,
True)
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
self.assertEqual(orchestrator.applied_configs, expected)
ret = orchestrator.requestComponentStatus(command_get)
self.assertEqual(ret['configurations'], expected)
pass
