def test_add_reg_listener_to_controller(self, FileCache_mock):
FileCache_mock.return_value = None
dummy_controller = MagicMock()
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
CustomServiceOrchestrator(config, dummy_controller)
self.assertTrue(dummy_controller.registration_listeners.append.called)
def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock,
decompress_cluster_host_info_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations':{'global' : {}},
'configurationTags':{'global' : { 'tag': 'v1' }},
'clusterHostInfo':{'namenode_host' : ['1'],
'slave_hosts' : ['0', '1'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']},
'hostLevelParams':{}
}
decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670', '8670']}
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
self.assertTrue(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType']='STATUS_COMMAND'
decompress_cluster_host_info_mock.reset_mock()
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
self.assertFalse(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
def resolve_ambari_config(self):
try:
config = AmbariConfig()
if os.path.exists(AmbariConfig.getConfigFile()):
config.read(AmbariConfig.getConfigFile())
else:
raise Exception("No config found, use default")
except Exception, err:
logger.warn(err)
def test_configureEnviron(self, hrdware_facterinfo_mock, subproc_popen, os_path_exists_mock):
config = AmbariConfig().getConfig()
config.set("puppet", "ruby_home", AmbariConfig().getConfig().get("stack", "installprefix"))
hardware = Hardware(config)
os_path_exists_mock.return_value = True
result = hardware.configureEnviron({'PATH': ""})
self.assertEquals(result['PATH'], AmbariConfig().getConfig().get("stack", "installprefix") + "/bin:")
self.assertEquals(result['MY_RUBY_HOME'], AmbariConfig().getConfig().get("stack", "installprefix"))
config.remove_option("puppet", "ruby_home")
def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,
kill_process_with_children_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
# shell.kill_process_with_children = MagicMock()
dummy_controller = MagicMock()
cfg = AmbariConfig().getConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f (a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
TestActionQueue.patch_output_file(orchestrator.python_executor)
orchestrator.python_executor.prepare_process_result = MagicMock()
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited']= ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19,'')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
def test_facterInfo(self, os_path_exists_mock, hardware_facterLib_mock, subprocess_popen_mock):
config = AmbariConfig().getConfig()
config.set("puppet", "facter_home", AmbariConfig().getConfig().get("stack", "installprefix"))
hardware = Hardware(config)
facter = MagicMock()
facter.communicate.return_value = ["memoryfree => 1 GB\n memorysize => 25 MB\n memorytotal => 300 KB\n "
+ "physicalprocessorcount => 25\n is_virtual => true\n", "no errors"]
facter.returncode = 0
os.environ['RUBYLIB'] = AmbariConfig().getConfig().get("stack", "installprefix");
subprocess_popen_mock.return_value = facter
os_path_exists_mock.return_value = True
hardware_facterLib_mock.return_value = "bla bla bla"
facterInfo = hardware.facterInfo()
self.assertEquals(facterInfo['memoryfree'], 1048576L)
self.assertEquals(facterInfo['memorysize'], 25600L)
self.assertEquals(facterInfo['memorytotal'], 300L)
self.assertEquals(facterInfo['physicalprocessorcount'], 25)
self.assertTrue(facterInfo['is_virtual'])
self.assertEquals(subprocess_popen_mock.call_args[1]['env']['RUBYLIB'],
AmbariConfig().getConfig().get("stack", "installprefix") + ":" + "bla bla bla")
facter.communicate.return_value = ["memoryfree => 1 G\n memorysize => 25 M\n memorytotal => 300 K\n "
+ "someinfo => 12 Byte\n ssh_name_key => Aa06Fdd\n", "no errors"]
facterInfo = hardware.facterInfo()
facter.returncode = 1
self.assertEquals(facterInfo['memoryfree'], 1048576L)
self.assertEquals(facterInfo['memorysize'], 25600L)
self.assertEquals(facterInfo['memorytotal'], 300L)
self.assertEquals(facterInfo['someinfo'], '12 Byte')
self.assertFalse(facterInfo.has_key('ssh_name_key'))
facter.communicate.return_value = ["memoryfree => 1024 M B\n memorytotal => 1024 Byte" , "no errors"]
facterInfo = hardware.facterInfo()
self.assertEquals(facterInfo['memoryfree'], 1L)
self.assertEquals(facterInfo['memorytotal'], 1L)
os_path_exists_mock.return_value = False
facterInfo = hardware.facterInfo()
self.assertEquals(facterInfo, {})
def test_run_command(self, runPuppetFileMock, isJavaAvailableMock):
tmpdir = AmbariConfig().getConfig().get("stack", "installprefix")
puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir,
AmbariConfig().getConfig())
jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
jsonStr = jsonFile.read()
parsedJson = json.loads(jsonStr)
parsedJson["taskId"] = 1
def side_effect1(puppetFile, result, puppetEnv, tmpoutfile,
tmperrfile):
result["exitcode"] = 0
runPuppetFileMock.side_effect = side_effect1
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = True
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 0)
self.assertTrue(puppetInstance.reposInstalled)
def side_effect2(puppetFile, result, puppetEnv, tmpoutfile,
tmperrfile):
result["exitcode"] = 999
runPuppetFileMock.side_effect = side_effect2
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = True
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 999)
self.assertFalse(puppetInstance.reposInstalled)
os.unlink(tmpdir + os.sep + 'site-' + str(parsedJson["taskId"]) +
'.pp')
def side_effect2(puppetFile, result, puppetEnv, tmpoutfile,
tmperrfile):
result["exitcode"] = 0
runPuppetFileMock.side_effect = side_effect2
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = False
parsedJson['roleCommand'] = "START"
parsedJson['configurations'] = {
'global': {
'java64_home': '/usr/jdk/jdk123'
}
}
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java"
errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123')
self.assertEquals(res["exitcode"], 1)
self.assertEquals(res["stderr"], errMsg)
self.assertFalse(puppetInstance.reposInstalled)
parsedJson['configurations'] = {'random': {'name1': 'value2'}}
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 1)
self.assertEquals(
res["stderr"],
"Cannot access JDK! Make sure java64_home is specified in global config"
)
def setUp(self):
self.config = AmbariConfig()
def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock,
decompress_cluster_host_info_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'clusterHostInfo': {
'namenode_host': ['1'],
'slave_hosts': ['0', '1'],
'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']
}
}
decompress_cluster_host_info_mock.return_value = {
'namenode_host': ['h2.hortonworks.com'],
'slave_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670', '8670']
}
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
self.assertTrue(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType'] = 'STATUS_COMMAND'
decompress_cluster_host_info_mock.reset_mock()
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
self.assertFalse(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
import os
import time
import ConfigParser
import ProcessHelper
from Controller import Controller
from AmbariConfig import AmbariConfig
from NetUtil import NetUtil
from PingPortListener import PingPortListener
import hostname
from DataCleaner import DataCleaner
import socket
logger = logging.getLogger()
formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()
config = AmbariConfig()
configFile = config.CONFIG_FILE
two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY
if 'AMBARI_LOG_DIR' in os.environ:
logfile = os.environ['AMBARI_LOG_DIR'] + "/ambari-agent.log"
else:
logfile = "/var/log/ambari-agent/ambari-agent.log"
def signal_handler(signum, frame):
#we want the handler to run only for the agent process and not
#for the children (e.g. namenode, etc.)
if os.getpid() != agentPid:
os._exit(0)
logger.info('signal received, exiting.')
ProcessHelper.stopAgent()
def test_facterLib(self):
hardware = Hardware(AmbariConfig().getConfig())
facterLib = hardware.facterLib("/home")
self.assertEquals(facterLib, "/home/lib/")
def test_run_command(self, runPuppetFileMock, isJavaAvailableMock,
generateManifestMock):
tmpdir = tempfile.gettempdir()
puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir,
AmbariConfig().getConfig())
jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
jsonStr = jsonFile.read()
parsedJson = json.loads(jsonStr)
parsedJson["taskId"] = 1
def side_effect1(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile,
timeout):
result["exitcode"] = 0
runPuppetFileMock.side_effect = side_effect1
generateManifestMock.return_value = ''
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = True
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 0)
self.assertTrue(puppetInstance.reposInstalled)
def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile,
timeout):
result["exitcode"] = 999
runPuppetFileMock.side_effect = side_effect2
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = True
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 999)
self.assertFalse(puppetInstance.reposInstalled)
generateManifestMock.return_value = 'error during manifest generation'
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertTrue(generateManifestMock.called)
self.assertEquals(res["exitcode"], 1)
generateManifestMock.return_value = ''
def side_effect2(puppetFile, result, puppetEnv, tmpoutfile,
tmperrfile):
result["exitcode"] = 0
runPuppetFileMock.side_effect = side_effect2
puppetInstance.reposInstalled = False
isJavaAvailableMock.return_value = False
parsedJson['roleCommand'] = "START"
parsedJson['hostLevelParams'] = {'java_home': '/usr/jdk/jdk123'}
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java"
errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123')
self.assertEquals(res["exitcode"], 1)
self.assertEquals(res["stderr"], errMsg)
self.assertFalse(puppetInstance.reposInstalled)
parsedJson['hostLevelParams'] = {'random': {'name1': 'value2'}}
res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
tmpdir + '/err.txt')
self.assertEquals(res["exitcode"], 1)
self.assertEquals(
res["stderr"],
"Cannot access JDK! Make sure java_home is specified in hostLevelParams"
)
def test_cancel_backgound_command(self, read_stack_version_mock,
resolve_hook_script_path_mock,
resolve_script_path_mock, FileCache_mock,
kill_process_with_children_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
# shell.kill_process_with_children = MagicMock()
dummy_controller = MagicMock()
cfg = AmbariConfig().getConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f(a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
TestActionQueue.patch_output_file(orchestrator.python_executor)
orchestrator.python_executor.prepare_process_result = MagicMock()
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited'] = ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(
actionQueue.on_background_command_complete_callback,
command_complete_w, None)
execute_command = copy.deepcopy(
TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19, '')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)