def test_add_reg_listener_to_controller(self, FileCache_mock): FileCache_mock.return_value = None dummy_controller = MagicMock() config = AmbariConfig().getConfig() tempdir = tempfile.gettempdir() config.set('agent', 'prefix', tempdir) CustomServiceOrchestrator(config, dummy_controller) self.assertTrue(dummy_controller.registration_listeners.append.called)
def test_dump_command_to_json(self, FileCache_mock, unlink_mock, isfile_mock, hostname_mock, decompress_cluster_host_info_mock): FileCache_mock.return_value = None hostname_mock.return_value = "test.hst" command = { 'commandType': 'EXECUTION_COMMAND', 'role': u'DATANODE', 'roleCommand': u'INSTALL', 'commandId': '1-1', 'taskId': 3, 'clusterName': u'cc', 'serviceName': u'HDFS', 'configurations':{'global' : {}}, 'configurationTags':{'global' : { 'tag': 'v1' }}, 'clusterHostInfo':{'namenode_host' : ['1'], 'slave_hosts' : ['0', '1'], 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], 'all_ping_ports': ['8670:0,1']}, 'hostLevelParams':{} } decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'], 'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], 'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'], 'all_ping_ports': ['8670', '8670']} config = AmbariConfig().getConfig() tempdir = tempfile.gettempdir() config.set('agent', 'prefix', tempdir) dummy_controller = MagicMock() orchestrator = CustomServiceOrchestrator(config, dummy_controller) isfile_mock.return_value = True # Test dumping EXECUTION_COMMAND json_file = orchestrator.dump_command_to_json(command) self.assertTrue(os.path.exists(json_file)) self.assertTrue(os.path.getsize(json_file) > 0) if get_platform() != PLATFORM_WINDOWS: self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') self.assertTrue(json_file.endswith("command-3.json")) self.assertTrue(decompress_cluster_host_info_mock.called) os.unlink(json_file) # Test dumping STATUS_COMMAND command['commandType']='STATUS_COMMAND' decompress_cluster_host_info_mock.reset_mock() json_file = orchestrator.dump_command_to_json(command) self.assertTrue(os.path.exists(json_file)) self.assertTrue(os.path.getsize(json_file) > 0) if get_platform() != PLATFORM_WINDOWS: self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') self.assertTrue(json_file.endswith("status_command.json")) self.assertFalse(decompress_cluster_host_info_mock.called) os.unlink(json_file) # Testing side effect of dump_command_to_json self.assertEquals(command['public_hostname'], "test.hst") self.assertTrue(unlink_mock.called)
def resolve_ambari_config(self): try: config = AmbariConfig() if os.path.exists(AmbariConfig.getConfigFile()): config.read(AmbariConfig.getConfigFile()) else: raise Exception("No config found, use default") except Exception, err: logger.warn(err)
def test_configureEnviron(self, hrdware_facterinfo_mock, subproc_popen, os_path_exists_mock): config = AmbariConfig().getConfig() config.set("puppet", "ruby_home", AmbariConfig().getConfig().get("stack", "installprefix")) hardware = Hardware(config) os_path_exists_mock.return_value = True result = hardware.configureEnviron({'PATH': ""}) self.assertEquals(result['PATH'], AmbariConfig().getConfig().get("stack", "installprefix") + "/bin:") self.assertEquals(result['MY_RUBY_HOME'], AmbariConfig().getConfig().get("stack", "installprefix")) config.remove_option("puppet", "ruby_home")
def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock): FileCache_mock.return_value = None FileCache_mock.cache_dir = MagicMock() resolve_hook_script_path_mock.return_value = None # shell.kill_process_with_children = MagicMock() dummy_controller = MagicMock() cfg = AmbariConfig().getConfig() cfg.set('agent', 'tolerate_download_failures', 'true') cfg.set('agent', 'prefix', '.') cfg.set('agent', 'cache_dir', 'background_tasks') actionQueue = ActionQueue(cfg, dummy_controller) dummy_controller.actionQueue = actionQueue orchestrator = CustomServiceOrchestrator(cfg, dummy_controller) orchestrator.file_cache = MagicMock() def f (a, b): return "" orchestrator.file_cache.get_service_base_dir = f actionQueue.customServiceOrchestrator = orchestrator import TestActionQueue import copy TestActionQueue.patch_output_file(orchestrator.python_executor) orchestrator.python_executor.prepare_process_result = MagicMock() orchestrator.dump_command_to_json = MagicMock() lock = threading.RLock() complete_done = threading.Condition(lock) complete_was_called = {} def command_complete_w(process_condenced_result, handle): with lock: complete_was_called['visited']= '' complete_done.wait(3) actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None) execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command) actionQueue.put([execute_command]) actionQueue.processBackgroundQueueSafeEmpty() time.sleep(.1) orchestrator.cancel_command(19,'') self.assertTrue(kill_process_with_children_mock.called) kill_process_with_children_mock.assert_called_with(33) with lock: complete_done.notifyAll() with lock: self.assertTrue(complete_was_called.has_key('visited')) time.sleep(.1) runningCommand = actionQueue.commandStatuses.get_command_status(19) self.assertTrue(runningCommand is not None) self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
def test_facterInfo(self, os_path_exists_mock, hardware_facterLib_mock, subprocess_popen_mock): config = AmbariConfig().getConfig() config.set("puppet", "facter_home", AmbariConfig().getConfig().get("stack", "installprefix")) hardware = Hardware(config) facter = MagicMock() facter.communicate.return_value = ["memoryfree => 1 GB\n memorysize => 25 MB\n memorytotal => 300 KB\n " + "physicalprocessorcount => 25\n is_virtual => true\n", "no errors"] facter.returncode = 0 os.environ['RUBYLIB'] = AmbariConfig().getConfig().get("stack", "installprefix"); subprocess_popen_mock.return_value = facter os_path_exists_mock.return_value = True hardware_facterLib_mock.return_value = "bla bla bla" facterInfo = hardware.facterInfo() self.assertEquals(facterInfo['memoryfree'], 1048576L) self.assertEquals(facterInfo['memorysize'], 25600L) self.assertEquals(facterInfo['memorytotal'], 300L) self.assertEquals(facterInfo['physicalprocessorcount'], 25) self.assertTrue(facterInfo['is_virtual']) self.assertEquals(subprocess_popen_mock.call_args[1]['env']['RUBYLIB'], AmbariConfig().getConfig().get("stack", "installprefix") + ":" + "bla bla bla") facter.communicate.return_value = ["memoryfree => 1 G\n memorysize => 25 M\n memorytotal => 300 K\n " + "someinfo => 12 Byte\n ssh_name_key => Aa06Fdd\n", "no errors"] facterInfo = hardware.facterInfo() facter.returncode = 1 self.assertEquals(facterInfo['memoryfree'], 1048576L) self.assertEquals(facterInfo['memorysize'], 25600L) self.assertEquals(facterInfo['memorytotal'], 300L) self.assertEquals(facterInfo['someinfo'], '12 Byte') self.assertFalse(facterInfo.has_key('ssh_name_key')) facter.communicate.return_value = ["memoryfree => 1024 M B\n memorytotal => 1024 Byte" , "no errors"] facterInfo = hardware.facterInfo() self.assertEquals(facterInfo['memoryfree'], 1L) self.assertEquals(facterInfo['memorytotal'], 1L) os_path_exists_mock.return_value = False facterInfo = hardware.facterInfo() self.assertEquals(facterInfo, {})
def test_run_command(self, runPuppetFileMock, isJavaAvailableMock): tmpdir = AmbariConfig().getConfig().get("stack", "installprefix") puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, AmbariConfig().getConfig()) jsonFile = open('../../main/python/ambari_agent/test.json', 'r') jsonStr = jsonFile.read() parsedJson = json.loads(jsonStr) parsedJson["taskId"] = 1 def side_effect1(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile): result["exitcode"] = 0 runPuppetFileMock.side_effect = side_effect1 puppetInstance.reposInstalled = False isJavaAvailableMock.return_value = True res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertEquals(res["exitcode"], 0) self.assertTrue(puppetInstance.reposInstalled) def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile): result["exitcode"] = 999 runPuppetFileMock.side_effect = side_effect2 puppetInstance.reposInstalled = False isJavaAvailableMock.return_value = True res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertEquals(res["exitcode"], 999) self.assertFalse(puppetInstance.reposInstalled) os.unlink(tmpdir + os.sep + 'site-' + str(parsedJson["taskId"]) + '.pp') def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile): result["exitcode"] = 0 runPuppetFileMock.side_effect = side_effect2 puppetInstance.reposInstalled = False isJavaAvailableMock.return_value = False parsedJson['roleCommand'] = "START" parsedJson['configurations'] = { 'global': { 'java64_home': '/usr/jdk/jdk123' } } res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java" errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123') self.assertEquals(res["exitcode"], 1) self.assertEquals(res["stderr"], errMsg) self.assertFalse(puppetInstance.reposInstalled) parsedJson['configurations'] = {'random': {'name1': 'value2'}} res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertEquals(res["exitcode"], 1) self.assertEquals( res["stderr"], "Cannot access JDK! Make sure java64_home is specified in global config" )
def setUp(self): self.config = AmbariConfig()
def test_dump_command_to_json(self, FileCache_mock, unlink_mock, isfile_mock, hostname_mock, decompress_cluster_host_info_mock): FileCache_mock.return_value = None hostname_mock.return_value = "test.hst" command = { 'commandType': 'EXECUTION_COMMAND', 'role': u'DATANODE', 'roleCommand': u'INSTALL', 'commandId': '1-1', 'taskId': 3, 'clusterName': u'cc', 'serviceName': u'HDFS', 'configurations': { 'global': {} }, 'configurationTags': { 'global': { 'tag': 'v1' } }, 'clusterHostInfo': { 'namenode_host': ['1'], 'slave_hosts': ['0', '1'], 'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'], 'all_ping_ports': ['8670:0,1'] } } decompress_cluster_host_info_mock.return_value = { 'namenode_host': ['h2.hortonworks.com'], 'slave_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'], 'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'], 'all_ping_ports': ['8670', '8670'] } config = AmbariConfig().getConfig() tempdir = tempfile.gettempdir() config.set('agent', 'prefix', tempdir) dummy_controller = MagicMock() orchestrator = CustomServiceOrchestrator(config, dummy_controller) isfile_mock.return_value = True # Test dumping EXECUTION_COMMAND json_file = orchestrator.dump_command_to_json(command) self.assertTrue(os.path.exists(json_file)) self.assertTrue(os.path.getsize(json_file) > 0) self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') self.assertTrue(json_file.endswith("command-3.json")) self.assertTrue(decompress_cluster_host_info_mock.called) os.unlink(json_file) # Test dumping STATUS_COMMAND command['commandType'] = 'STATUS_COMMAND' decompress_cluster_host_info_mock.reset_mock() json_file = orchestrator.dump_command_to_json(command) self.assertTrue(os.path.exists(json_file)) self.assertTrue(os.path.getsize(json_file) > 0) self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600') self.assertTrue(json_file.endswith("status_command.json")) self.assertFalse(decompress_cluster_host_info_mock.called) os.unlink(json_file) # Testing side effect of dump_command_to_json self.assertEquals(command['public_hostname'], "test.hst") self.assertTrue(unlink_mock.called)
import os import time import ConfigParser import ProcessHelper from Controller import Controller from AmbariConfig import AmbariConfig from NetUtil import NetUtil from PingPortListener import PingPortListener import hostname from DataCleaner import DataCleaner import socket logger = logging.getLogger() formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s" agentPid = os.getpid() config = AmbariConfig() configFile = config.CONFIG_FILE two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY if 'AMBARI_LOG_DIR' in os.environ: logfile = os.environ['AMBARI_LOG_DIR'] + "/ambari-agent.log" else: logfile = "/var/log/ambari-agent/ambari-agent.log" def signal_handler(signum, frame): #we want the handler to run only for the agent process and not #for the children (e.g. namenode, etc.) if os.getpid() != agentPid: os._exit(0) logger.info('signal received, exiting.') ProcessHelper.stopAgent()
def test_facterLib(self): hardware = Hardware(AmbariConfig().getConfig()) facterLib = hardware.facterLib("/home") self.assertEquals(facterLib, "/home/lib/")
def test_run_command(self, runPuppetFileMock, isJavaAvailableMock, generateManifestMock): tmpdir = tempfile.gettempdir() puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir, AmbariConfig().getConfig()) jsonFile = open('../../main/python/ambari_agent/test.json', 'r') jsonStr = jsonFile.read() parsedJson = json.loads(jsonStr) parsedJson["taskId"] = 1 def side_effect1(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile, timeout): result["exitcode"] = 0 runPuppetFileMock.side_effect = side_effect1 generateManifestMock.return_value = '' puppetInstance.reposInstalled = False isJavaAvailableMock.return_value = True res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertEquals(res["exitcode"], 0) self.assertTrue(puppetInstance.reposInstalled) def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile, timeout): result["exitcode"] = 999 runPuppetFileMock.side_effect = side_effect2 puppetInstance.reposInstalled = False isJavaAvailableMock.return_value = True res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertEquals(res["exitcode"], 999) self.assertFalse(puppetInstance.reposInstalled) generateManifestMock.return_value = 'error during manifest generation' res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertTrue(generateManifestMock.called) self.assertEquals(res["exitcode"], 1) generateManifestMock.return_value = '' def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile): result["exitcode"] = 0 runPuppetFileMock.side_effect = side_effect2 puppetInstance.reposInstalled = False isJavaAvailableMock.return_value = False parsedJson['roleCommand'] = "START" parsedJson['hostLevelParams'] = {'java_home': '/usr/jdk/jdk123'} res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java" errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123') self.assertEquals(res["exitcode"], 1) self.assertEquals(res["stderr"], errMsg) self.assertFalse(puppetInstance.reposInstalled) parsedJson['hostLevelParams'] = {'random': {'name1': 'value2'}} res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt', tmpdir + '/err.txt') self.assertEquals(res["exitcode"], 1) self.assertEquals( res["stderr"], "Cannot access JDK! Make sure java_home is specified in hostLevelParams" )
def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock): FileCache_mock.return_value = None FileCache_mock.cache_dir = MagicMock() resolve_hook_script_path_mock.return_value = None # shell.kill_process_with_children = MagicMock() dummy_controller = MagicMock() cfg = AmbariConfig().getConfig() cfg.set('agent', 'tolerate_download_failures', 'true') cfg.set('agent', 'prefix', '.') cfg.set('agent', 'cache_dir', 'background_tasks') actionQueue = ActionQueue(cfg, dummy_controller) dummy_controller.actionQueue = actionQueue orchestrator = CustomServiceOrchestrator(cfg, dummy_controller) orchestrator.file_cache = MagicMock() def f(a, b): return "" orchestrator.file_cache.get_service_base_dir = f actionQueue.customServiceOrchestrator = orchestrator import TestActionQueue import copy TestActionQueue.patch_output_file(orchestrator.python_executor) orchestrator.python_executor.prepare_process_result = MagicMock() orchestrator.dump_command_to_json = MagicMock() lock = threading.RLock() complete_done = threading.Condition(lock) complete_was_called = {} def command_complete_w(process_condenced_result, handle): with lock: complete_was_called['visited'] = '' complete_done.wait(3) actionQueue.on_background_command_complete_callback = TestActionQueue.wraped( actionQueue.on_background_command_complete_callback, command_complete_w, None) execute_command = copy.deepcopy( TestActionQueue.TestActionQueue.background_command) actionQueue.put([execute_command]) actionQueue.processBackgroundQueueSafeEmpty() time.sleep(.1) orchestrator.cancel_command(19, '') self.assertTrue(kill_process_with_children_mock.called) kill_process_with_children_mock.assert_called_with(33) with lock: complete_done.notifyAll() with lock: self.assertTrue(complete_was_called.has_key('visited')) time.sleep(.1) runningCommand = actionQueue.commandStatuses.get_command_status(19) self.assertTrue(runningCommand is not None) self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)