def test_add_reg_listener_to_controller(self, FileCache_mock):
   FileCache_mock.return_value = None
   dummy_controller = MagicMock()
   config = AmbariConfig().getConfig()
   tempdir = tempfile.gettempdir()
   config.set('agent', 'prefix', tempdir)
   CustomServiceOrchestrator(config, dummy_controller)
   self.assertTrue(dummy_controller.registration_listeners.append.called)
 def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
                               isfile_mock, hostname_mock,
                               decompress_cluster_host_info_mock):
   FileCache_mock.return_value = None
   hostname_mock.return_value = "test.hst"
   command = {
     'commandType': 'EXECUTION_COMMAND',
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'commandId': '1-1',
     'taskId': 3,
     'clusterName': u'cc',
     'serviceName': u'HDFS',
     'configurations':{'global' : {}},
     'configurationTags':{'global' : { 'tag': 'v1' }},
     'clusterHostInfo':{'namenode_host' : ['1'],
                        'slave_hosts'   : ['0', '1'],
                        'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_ping_ports': ['8670:0,1']},
     'hostLevelParams':{}
   }
   
   decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
                        'slave_hosts'   : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_ping_ports': ['8670', '8670']}
   
   config = AmbariConfig().getConfig()
   tempdir = tempfile.gettempdir()
   config.set('agent', 'prefix', tempdir)
   dummy_controller = MagicMock()
   orchestrator = CustomServiceOrchestrator(config, dummy_controller)
   isfile_mock.return_value = True
   # Test dumping EXECUTION_COMMAND
   json_file = orchestrator.dump_command_to_json(command)
   self.assertTrue(os.path.exists(json_file))
   self.assertTrue(os.path.getsize(json_file) > 0)
   if get_platform() != PLATFORM_WINDOWS:
     self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
   self.assertTrue(json_file.endswith("command-3.json"))
   self.assertTrue(decompress_cluster_host_info_mock.called)
   os.unlink(json_file)
   # Test dumping STATUS_COMMAND
   command['commandType']='STATUS_COMMAND'
   decompress_cluster_host_info_mock.reset_mock()
   json_file = orchestrator.dump_command_to_json(command)
   self.assertTrue(os.path.exists(json_file))
   self.assertTrue(os.path.getsize(json_file) > 0)
   if get_platform() != PLATFORM_WINDOWS:
     self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
   self.assertTrue(json_file.endswith("status_command.json"))
   self.assertFalse(decompress_cluster_host_info_mock.called)
   os.unlink(json_file)
   # Testing side effect of dump_command_to_json
   self.assertEquals(command['public_hostname'], "test.hst")
   self.assertTrue(unlink_mock.called)
  def resolve_ambari_config(self):
    try:
      config = AmbariConfig()
      if os.path.exists(AmbariConfig.getConfigFile()):
        config.read(AmbariConfig.getConfigFile())
      else:
        raise Exception("No config found, use default")

    except Exception, err:
      logger.warn(err)
  def test_configureEnviron(self, hrdware_facterinfo_mock, subproc_popen, os_path_exists_mock):
    config = AmbariConfig().getConfig()
    config.set("puppet", "ruby_home", AmbariConfig().getConfig().get("stack", "installprefix"))
    hardware = Hardware(config)
    os_path_exists_mock.return_value = True
    result = hardware.configureEnviron({'PATH': ""})

    self.assertEquals(result['PATH'], AmbariConfig().getConfig().get("stack", "installprefix") + "/bin:")
    self.assertEquals(result['MY_RUBY_HOME'], AmbariConfig().getConfig().get("stack", "installprefix"))
    config.remove_option("puppet", "ruby_home")
  def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,
                                      kill_process_with_children_mock):
    FileCache_mock.return_value = None
    FileCache_mock.cache_dir = MagicMock()
    resolve_hook_script_path_mock.return_value = None
#     shell.kill_process_with_children = MagicMock()
    dummy_controller = MagicMock()
    cfg = AmbariConfig().getConfig()
    cfg.set('agent', 'tolerate_download_failures', 'true')
    cfg.set('agent', 'prefix', '.')
    cfg.set('agent', 'cache_dir', 'background_tasks')

    actionQueue = ActionQueue(cfg, dummy_controller)

    dummy_controller.actionQueue = actionQueue
    orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
    orchestrator.file_cache = MagicMock()
    def f (a, b):
      return ""
    orchestrator.file_cache.get_service_base_dir = f
    actionQueue.customServiceOrchestrator = orchestrator

    import TestActionQueue
    import copy

    TestActionQueue.patch_output_file(orchestrator.python_executor)
    orchestrator.python_executor.prepare_process_result = MagicMock()
    orchestrator.dump_command_to_json = MagicMock()

    lock = threading.RLock()
    complete_done = threading.Condition(lock)

    complete_was_called = {}
    def command_complete_w(process_condenced_result, handle):
      with lock:
        complete_was_called['visited']= ''
        complete_done.wait(3)

    actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
    execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
    actionQueue.put([execute_command])
    actionQueue.processBackgroundQueueSafeEmpty()

    time.sleep(.1)

    orchestrator.cancel_command(19,'')
    self.assertTrue(kill_process_with_children_mock.called)
    kill_process_with_children_mock.assert_called_with(33)

    with lock:
      complete_done.notifyAll()

    with lock:
      self.assertTrue(complete_was_called.has_key('visited'))

    time.sleep(.1)

    runningCommand = actionQueue.commandStatuses.get_command_status(19)
    self.assertTrue(runningCommand is not None)
    self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
  def test_facterInfo(self, os_path_exists_mock, hardware_facterLib_mock, subprocess_popen_mock):
    config = AmbariConfig().getConfig()
    config.set("puppet", "facter_home", AmbariConfig().getConfig().get("stack", "installprefix"))
    hardware = Hardware(config)
    facter = MagicMock()
    facter.communicate.return_value = ["memoryfree => 1 GB\n memorysize => 25 MB\n memorytotal => 300 KB\n "
                                        + "physicalprocessorcount => 25\n is_virtual => true\n", "no errors"]
    facter.returncode = 0
    os.environ['RUBYLIB'] = AmbariConfig().getConfig().get("stack", "installprefix");
    subprocess_popen_mock.return_value = facter
    os_path_exists_mock.return_value = True
    hardware_facterLib_mock.return_value = "bla bla bla"
    facterInfo = hardware.facterInfo()

    self.assertEquals(facterInfo['memoryfree'], 1048576L)
    self.assertEquals(facterInfo['memorysize'], 25600L)
    self.assertEquals(facterInfo['memorytotal'], 300L)
    self.assertEquals(facterInfo['physicalprocessorcount'], 25)
    self.assertTrue(facterInfo['is_virtual'])
    self.assertEquals(subprocess_popen_mock.call_args[1]['env']['RUBYLIB'],
                                      AmbariConfig().getConfig().get("stack", "installprefix") + ":" + "bla bla bla")

    facter.communicate.return_value = ["memoryfree => 1 G\n memorysize => 25 M\n memorytotal => 300 K\n "
                                         + "someinfo => 12 Byte\n ssh_name_key => Aa06Fdd\n", "no errors"]
    facterInfo = hardware.facterInfo()
    facter.returncode = 1
    self.assertEquals(facterInfo['memoryfree'], 1048576L)
    self.assertEquals(facterInfo['memorysize'], 25600L)
    self.assertEquals(facterInfo['memorytotal'], 300L)
    self.assertEquals(facterInfo['someinfo'], '12 Byte')
    self.assertFalse(facterInfo.has_key('ssh_name_key'))

    facter.communicate.return_value = ["memoryfree => 1024 M B\n memorytotal => 1024 Byte" , "no errors"]

    facterInfo = hardware.facterInfo()

    self.assertEquals(facterInfo['memoryfree'], 1L)
    self.assertEquals(facterInfo['memorytotal'], 1L)

    os_path_exists_mock.return_value = False
    facterInfo = hardware.facterInfo()

    self.assertEquals(facterInfo, {})
Exemplo n.º 7
0
    def test_run_command(self, runPuppetFileMock, isJavaAvailableMock):
        tmpdir = AmbariConfig().getConfig().get("stack", "installprefix")
        puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir,
                                        AmbariConfig().getConfig())
        jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
        jsonStr = jsonFile.read()
        parsedJson = json.loads(jsonStr)
        parsedJson["taskId"] = 1

        def side_effect1(puppetFile, result, puppetEnv, tmpoutfile,
                         tmperrfile):
            result["exitcode"] = 0

        runPuppetFileMock.side_effect = side_effect1
        puppetInstance.reposInstalled = False
        isJavaAvailableMock.return_value = True
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertEquals(res["exitcode"], 0)
        self.assertTrue(puppetInstance.reposInstalled)

        def side_effect2(puppetFile, result, puppetEnv, tmpoutfile,
                         tmperrfile):
            result["exitcode"] = 999

        runPuppetFileMock.side_effect = side_effect2
        puppetInstance.reposInstalled = False
        isJavaAvailableMock.return_value = True
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertEquals(res["exitcode"], 999)
        self.assertFalse(puppetInstance.reposInstalled)
        os.unlink(tmpdir + os.sep + 'site-' + str(parsedJson["taskId"]) +
                  '.pp')

        def side_effect2(puppetFile, result, puppetEnv, tmpoutfile,
                         tmperrfile):
            result["exitcode"] = 0

        runPuppetFileMock.side_effect = side_effect2
        puppetInstance.reposInstalled = False
        isJavaAvailableMock.return_value = False
        parsedJson['roleCommand'] = "START"
        parsedJson['configurations'] = {
            'global': {
                'java64_home': '/usr/jdk/jdk123'
            }
        }
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')

        JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java"
        errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123')
        self.assertEquals(res["exitcode"], 1)
        self.assertEquals(res["stderr"], errMsg)
        self.assertFalse(puppetInstance.reposInstalled)

        parsedJson['configurations'] = {'random': {'name1': 'value2'}}
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertEquals(res["exitcode"], 1)
        self.assertEquals(
            res["stderr"],
            "Cannot access JDK! Make sure java64_home is specified in global config"
        )
Exemplo n.º 8
0
 def setUp(self):
     self.config = AmbariConfig()
    def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
                                  isfile_mock, hostname_mock,
                                  decompress_cluster_host_info_mock):
        FileCache_mock.return_value = None
        hostname_mock.return_value = "test.hst"
        command = {
            'commandType': 'EXECUTION_COMMAND',
            'role': u'DATANODE',
            'roleCommand': u'INSTALL',
            'commandId': '1-1',
            'taskId': 3,
            'clusterName': u'cc',
            'serviceName': u'HDFS',
            'configurations': {
                'global': {}
            },
            'configurationTags': {
                'global': {
                    'tag': 'v1'
                }
            },
            'clusterHostInfo': {
                'namenode_host': ['1'],
                'slave_hosts': ['0', '1'],
                'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
                'all_ping_ports': ['8670:0,1']
            }
        }

        decompress_cluster_host_info_mock.return_value = {
            'namenode_host': ['h2.hortonworks.com'],
            'slave_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
            'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
            'all_ping_ports': ['8670', '8670']
        }

        config = AmbariConfig().getConfig()
        tempdir = tempfile.gettempdir()
        config.set('agent', 'prefix', tempdir)
        dummy_controller = MagicMock()
        orchestrator = CustomServiceOrchestrator(config, dummy_controller)
        isfile_mock.return_value = True
        # Test dumping EXECUTION_COMMAND
        json_file = orchestrator.dump_command_to_json(command)
        self.assertTrue(os.path.exists(json_file))
        self.assertTrue(os.path.getsize(json_file) > 0)
        self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
        self.assertTrue(json_file.endswith("command-3.json"))
        self.assertTrue(decompress_cluster_host_info_mock.called)
        os.unlink(json_file)
        # Test dumping STATUS_COMMAND
        command['commandType'] = 'STATUS_COMMAND'
        decompress_cluster_host_info_mock.reset_mock()
        json_file = orchestrator.dump_command_to_json(command)
        self.assertTrue(os.path.exists(json_file))
        self.assertTrue(os.path.getsize(json_file) > 0)
        self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
        self.assertTrue(json_file.endswith("status_command.json"))
        self.assertFalse(decompress_cluster_host_info_mock.called)
        os.unlink(json_file)
        # Testing side effect of dump_command_to_json
        self.assertEquals(command['public_hostname'], "test.hst")
        self.assertTrue(unlink_mock.called)
Exemplo n.º 10
0
import os
import time
import ConfigParser
import ProcessHelper
from Controller import Controller
from AmbariConfig import AmbariConfig
from NetUtil import NetUtil
from PingPortListener import PingPortListener
import hostname
from DataCleaner import DataCleaner
import socket

logger = logging.getLogger()
formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()
config = AmbariConfig()
configFile = config.CONFIG_FILE
two_way_ssl_property = config.TWO_WAY_SSL_PROPERTY

if 'AMBARI_LOG_DIR' in os.environ:
  logfile = os.environ['AMBARI_LOG_DIR'] + "/ambari-agent.log"
else:
  logfile = "/var/log/ambari-agent/ambari-agent.log"

def signal_handler(signum, frame):
  #we want the handler to run only for the agent process and not
  #for the children (e.g. namenode, etc.)
  if os.getpid() != agentPid:
    os._exit(0)
  logger.info('signal received, exiting.')
  ProcessHelper.stopAgent()
Exemplo n.º 11
0
 def test_facterLib(self):
     hardware = Hardware(AmbariConfig().getConfig())
     facterLib = hardware.facterLib("/home")
     self.assertEquals(facterLib, "/home/lib/")
Exemplo n.º 12
0
    def test_run_command(self, runPuppetFileMock, isJavaAvailableMock,
                         generateManifestMock):
        tmpdir = tempfile.gettempdir()
        puppetInstance = PuppetExecutor("/tmp", "/x", "/y", tmpdir,
                                        AmbariConfig().getConfig())
        jsonFile = open('../../main/python/ambari_agent/test.json', 'r')
        jsonStr = jsonFile.read()
        parsedJson = json.loads(jsonStr)
        parsedJson["taskId"] = 1

        def side_effect1(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile,
                         timeout):
            result["exitcode"] = 0

        runPuppetFileMock.side_effect = side_effect1
        generateManifestMock.return_value = ''
        puppetInstance.reposInstalled = False
        isJavaAvailableMock.return_value = True
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertEquals(res["exitcode"], 0)
        self.assertTrue(puppetInstance.reposInstalled)

        def side_effect2(puppetFile, result, puppetEnv, tmpoutfile, tmperrfile,
                         timeout):
            result["exitcode"] = 999

        runPuppetFileMock.side_effect = side_effect2
        puppetInstance.reposInstalled = False
        isJavaAvailableMock.return_value = True
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertEquals(res["exitcode"], 999)
        self.assertFalse(puppetInstance.reposInstalled)

        generateManifestMock.return_value = 'error during manifest generation'
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertTrue(generateManifestMock.called)
        self.assertEquals(res["exitcode"], 1)
        generateManifestMock.return_value = ''

        def side_effect2(puppetFile, result, puppetEnv, tmpoutfile,
                         tmperrfile):
            result["exitcode"] = 0

        runPuppetFileMock.side_effect = side_effect2
        puppetInstance.reposInstalled = False
        isJavaAvailableMock.return_value = False
        parsedJson['roleCommand'] = "START"
        parsedJson['hostLevelParams'] = {'java_home': '/usr/jdk/jdk123'}
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')

        JAVANOTVALID_MSG = "Cannot access JDK! Make sure you have permission to execute {0}/bin/java"
        errMsg = JAVANOTVALID_MSG.format('/usr/jdk/jdk123')
        self.assertEquals(res["exitcode"], 1)
        self.assertEquals(res["stderr"], errMsg)
        self.assertFalse(puppetInstance.reposInstalled)

        parsedJson['hostLevelParams'] = {'random': {'name1': 'value2'}}
        res = puppetInstance.runCommand(parsedJson, tmpdir + '/out.txt',
                                        tmpdir + '/err.txt')
        self.assertEquals(res["exitcode"], 1)
        self.assertEquals(
            res["stderr"],
            "Cannot access JDK! Make sure java_home is specified in hostLevelParams"
        )
    def test_cancel_backgound_command(self, read_stack_version_mock,
                                      resolve_hook_script_path_mock,
                                      resolve_script_path_mock, FileCache_mock,
                                      kill_process_with_children_mock):
        FileCache_mock.return_value = None
        FileCache_mock.cache_dir = MagicMock()
        resolve_hook_script_path_mock.return_value = None
        #     shell.kill_process_with_children = MagicMock()
        dummy_controller = MagicMock()
        cfg = AmbariConfig().getConfig()
        cfg.set('agent', 'tolerate_download_failures', 'true')
        cfg.set('agent', 'prefix', '.')
        cfg.set('agent', 'cache_dir', 'background_tasks')

        actionQueue = ActionQueue(cfg, dummy_controller)

        dummy_controller.actionQueue = actionQueue
        orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
        orchestrator.file_cache = MagicMock()

        def f(a, b):
            return ""

        orchestrator.file_cache.get_service_base_dir = f
        actionQueue.customServiceOrchestrator = orchestrator

        import TestActionQueue
        import copy

        TestActionQueue.patch_output_file(orchestrator.python_executor)
        orchestrator.python_executor.prepare_process_result = MagicMock()
        orchestrator.dump_command_to_json = MagicMock()

        lock = threading.RLock()
        complete_done = threading.Condition(lock)

        complete_was_called = {}

        def command_complete_w(process_condenced_result, handle):
            with lock:
                complete_was_called['visited'] = ''
                complete_done.wait(3)

        actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(
            actionQueue.on_background_command_complete_callback,
            command_complete_w, None)
        execute_command = copy.deepcopy(
            TestActionQueue.TestActionQueue.background_command)
        actionQueue.put([execute_command])
        actionQueue.processBackgroundQueueSafeEmpty()

        time.sleep(.1)

        orchestrator.cancel_command(19, '')
        self.assertTrue(kill_process_with_children_mock.called)
        kill_process_with_children_mock.assert_called_with(33)

        with lock:
            complete_done.notifyAll()

        with lock:
            self.assertTrue(complete_was_called.has_key('visited'))

        time.sleep(.1)

        runningCommand = actionQueue.commandStatuses.get_command_status(19)
        self.assertTrue(runningCommand is not None)
        self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)