Exemplo n.º 1
0
    def init(self):
        """
    Initialize properties
    """
        self.config = AmbariConfig.get_resolved_config()

        self.is_registered = False

        self.metadata_cache = ClusterMetadataCache(
            self.config.cluster_cache_dir)
        self.topology_cache = ClusterTopologyCache(
            self.config.cluster_cache_dir, self.config)
        self.host_level_params_cache = ClusterHostLevelParamsCache(
            self.config.cluster_cache_dir)
        self.configurations_cache = ClusterConfigurationCache(
            self.config.cluster_cache_dir)
        self.alert_definitions_cache = ClusterAlertDefinitionsCache(
            self.config.cluster_cache_dir)
        self.configuration_builder = ConfigurationBuilder(self)
        self.stale_alerts_monitor = StaleAlertsMonitor(self)

        self.file_cache = FileCache(self.config)

        self.customServiceOrchestrator = CustomServiceOrchestrator(self)

        self.recovery_manager = RecoveryManager(self.config.recovery_cache_dir)
        self.commandStatuses = CommandStatusDict(self)
        self.action_queue = ActionQueue(self)
        self.alert_scheduler_handler = AlertSchedulerHandler(self)
Exemplo n.º 2
0
    def init(self):
        """
    Initialize properties
    """
        self.is_registered = False

        self.metadata_cache = ClusterMetadataCache(
            self.config.cluster_cache_dir)
        self.topology_cache = ClusterTopologyCache(
            self.config.cluster_cache_dir, self.config)
        self.host_level_params_cache = ClusterHostLevelParamsCache(
            self.config.cluster_cache_dir)
        self.configurations_cache = ClusterConfigurationCache(
            self.config.cluster_cache_dir)
        self.alert_definitions_cache = ClusterAlertDefinitionsCache(
            self.config.cluster_cache_dir)
        self.configuration_builder = ConfigurationBuilder(self)
        self.stale_alerts_monitor = StaleAlertsMonitor(self)
        self.server_responses_listener = ServerResponsesListener()
        self.file_cache = FileCache(self.config)
        self.customServiceOrchestrator = CustomServiceOrchestrator(self)
        self.recovery_manager = RecoveryManager()
        self.commandStatuses = CommandStatusDict(self)

        self.init_threads()
 def test_size_approved(self):
     # as json: '{"status": "IN_PROGRESS", "structuredOut": "structured_out.tmp", "taskId": 5}', length=77
     command_in_progress_report = {
         'status': 'IN_PROGRESS',
         'taskId': 5,
         'structuredOut': 'structured_out.tmp',
     }
     mock = MagicMock()
     command_statuses = CommandStatusDict(mock)
     self.assertEqual(
         command_statuses.size_approved(command_in_progress_report, 78),
         True)
     self.assertEqual(
         command_statuses.size_approved(command_in_progress_report, 77),
         True)
     self.assertEqual(
         command_statuses.size_approved(command_in_progress_report, 76),
         False)
Exemplo n.º 4
0
 def test_put_command_status(self):
   execution_command = {
     'commandType': 'EXECUTION_COMMAND',
     'commandId': '1-1',
     'clusterName': u'cc',
     'exitCode': 777,
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'serviceName': u'HDFS',
     'taskId': 5
   }
   status_command = {
     'componentName': 'DATANODE',
     'commandType': 'STATUS_COMMAND',
     }
   callback_mock = MagicMock()
   commandStatuses = CommandStatusDict(callback_action = callback_mock)
   commandStatuses.put_command_status(status_command, None)
   self.assertEqual(callback_mock.call_count, 0)
   commandStatuses.put_command_status(execution_command, None)
   self.assertEqual(callback_mock.call_count, 1)
Exemplo n.º 5
0
 def test_structured_output(self, open_mock):
     callback_mock = MagicMock()
     commandStatuses = CommandStatusDict(callback_action=callback_mock)
     file_mock = MagicMock(name='file_mock')
     file_mock.__enter__.return_value = file_mock
     file_mock.read.return_value = '{"var1":"test1", "var2":"test2"}'
     open_mock.return_value = file_mock
     command_in_progress1 = {
         'commandType': 'EXECUTION_COMMAND',
         'commandId': '1-1',
         'clusterName': u'cc',
         'exitCode': 777,
         'role': u'DATANODE',
         'roleCommand': u'INSTALL',
         'serviceName': u'HDFS',
         'stderr': '',
         'stdout':
         "notice: /Stage[1]/Hdp::Iptables/Service[iptables]/ensure: ensure changed 'running' to 'stopped'\nnotice: /Stage[1]/Hdp/File[/tmp/changeUid.sh]/ensure: defined content as '{md5}32b994a2e970f8acc3c91c198b484654'\nnotice: /Stage[1]/Hdp::Snappy::Package/Hdp::Package[snappy]/Hdp::Package::Process_pkg[snappy]/Package[snappy]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::Group[nagios_group]/Group[nagios_group]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::User[nagios_user]/User[nagios]/ensure: created\nnotice: /Stage[1]/Hdp::Snmp/Hdp::Package[snmp]/Hdp::Package::Process_pkg[snmp]/Package[net-snmp-utils]/ensure: created",
         'taskId': 5
     }
     command_in_progress1_report = {
         'status': 'IN_PROGRESS',
         'taskId': 5,
         'structuredOut': 'structured_out.tmp',
     }
     commandStatuses.put_command_status(command_in_progress1,
                                        command_in_progress1_report)
     report = commandStatuses.generate_report()
     expected = \
       {'componentStatus': [],
        'reports': [{'status': 'IN_PROGRESS', 'stderr': '...',
                     'stdout': '...', 'clusterName': u'cc',
                     'structuredOut': '{"var1":"test1", "var2":"test2"}',
                     'roleCommand': u'INSTALL',
                     'serviceName': u'HDFS',
                     'role': u'DATANODE',
                     'actionId': '1-1', 'taskId': 5, 'exitCode': 777}]
       }
     self.assertEquals(report, expected)
Exemplo n.º 6
0
 def test_structured_output(self, open_mock):
   callback_mock = MagicMock()
   commandStatuses = CommandStatusDict(callback_action = callback_mock)
   file_mock = MagicMock(name = 'file_mock')
   file_mock.__enter__.return_value = file_mock
   file_mock.read.return_value = '{"var1":"test1", "var2":"test2"}'
   open_mock.return_value = file_mock
   command_in_progress1 = {
     'commandType': 'EXECUTION_COMMAND',
     'commandId': '1-1',
     'clusterName': u'cc',
     'exitCode': 777,
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'serviceName': u'HDFS',
     'stderr': '',
     'stdout': "notice: /Stage[1]/Hdp::Iptables/Service[iptables]/ensure: ensure changed 'running' to 'stopped'\nnotice: /Stage[1]/Hdp/File[/tmp/changeUid.sh]/ensure: defined content as '{md5}32b994a2e970f8acc3c91c198b484654'\nnotice: /Stage[1]/Hdp::Snappy::Package/Hdp::Package[snappy]/Hdp::Package::Process_pkg[snappy]/Package[snappy]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::Group[nagios_group]/Group[nagios_group]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::User[nagios_user]/User[nagios]/ensure: created\nnotice: /Stage[1]/Hdp::Snmp/Hdp::Package[snmp]/Hdp::Package::Process_pkg[snmp]/Package[net-snmp-utils]/ensure: created",
     'taskId': 5
   }
   command_in_progress1_report = {
     'status': 'IN_PROGRESS',
     'taskId': 5,
     'structuredOut' : 'structured_out.tmp',
     }
   commandStatuses.put_command_status(command_in_progress1, command_in_progress1_report)
   report = commandStatuses.generate_report()
   expected = \
     {'componentStatus': [],
      'reports': [{'status': 'IN_PROGRESS', 'stderr': '...',
                   'stdout': '...', 'clusterName': u'cc',
                   'structuredOut': '{"var1":"test1", "var2":"test2"}',
                   'roleCommand': u'INSTALL',
                   'serviceName': u'HDFS',
                   'role': u'DATANODE',
                   'actionId': '1-1', 'taskId': 5, 'exitCode': 777}]
     }
   self.assertEquals(report, expected)
 def test_put_command_status(self):
   execution_command = {
     'commandType': 'EXECUTION_COMMAND',
     'commandId': '1-1',
     'clusterName': u'cc',
     'exitCode': 777,
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'serviceName': u'HDFS',
     'taskId': 5
   }
   status_command = {
     'componentName': 'DATANODE',
     'commandType': 'STATUS_COMMAND',
     }
   callback_mock = MagicMock()
   commandStatuses = CommandStatusDict(callback_action = callback_mock)
   commandStatuses.put_command_status(status_command, None)
   self.assertEqual(callback_mock.call_count, 0)
   commandStatuses.put_command_status(execution_command, None)
   self.assertEqual(callback_mock.call_count, 1)
Exemplo n.º 8
0
 def test_put_and_generate(self):
   callback_mock = MagicMock()
   commandStatuses = CommandStatusDict(callback_action = callback_mock)
   command_in_progress1 = {
     'commandType': 'EXECUTION_COMMAND',
     'commandId': '1-1',
     'clusterName': u'cc',
     'exitCode': 777,
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'serviceName': u'HDFS',
     'stderr': '',
     'stdout': "notice: /Stage[1]/Hdp::Iptables/Service[iptables]/ensure: ensure changed 'running' to 'stopped'\nnotice: /Stage[1]/Hdp/File[/tmp/changeUid.sh]/ensure: defined content as '{md5}32b994a2e970f8acc3c91c198b484654'\nnotice: /Stage[1]/Hdp::Snappy::Package/Hdp::Package[snappy]/Hdp::Package::Process_pkg[snappy]/Package[snappy]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::Group[nagios_group]/Group[nagios_group]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::User[nagios_user]/User[nagios]/ensure: created\nnotice: /Stage[1]/Hdp::Snmp/Hdp::Package[snmp]/Hdp::Package::Process_pkg[snmp]/Package[net-snmp-utils]/ensure: created",
     'taskId': 5
   }
   command_in_progress1_report = {
     'status': 'IN_PROGRESS',
     'taskId': 5
   }
   command_in_progress2 = {
     'commandType': 'EXECUTION_COMMAND',
     'commandId': '1-1',
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'taskId': 6,
     'clusterName': u'cc',
     'serviceName': u'HDFS',
   }
   command_in_progress2_report = {
     'status': 'IN_PROGRESS',
     'taskId': 6
   }
   finished_command = {
     'commandType': 'EXECUTION_COMMAND',
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'commandId': '1-1',
     'taskId': 4,
     'clusterName': u'cc',
     'serviceName': u'HDFS',
   }
   finished_command_report = {
     'status': 'COMPLETE',
     'taskId': 4,
   }
   failed_command = {
     'commandType': 'EXECUTION_COMMAND',
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'commandId': '1-1',
     'taskId': 3,
     'clusterName': u'cc',
     'serviceName': u'HDFS',
   }
   failed_command_report = {
     'status': 'FAILED',
     'taskId': 3,
   }
   status_command = {
     'componentName': 'DATANODE',
     'commandType': 'STATUS_COMMAND',
   }
   status_command_report = {
     'componentName': 'DATANODE',
     'status': 'HEALTHY'
   }
   commandStatuses.put_command_status(command_in_progress1, command_in_progress1_report)
   commandStatuses.put_command_status(command_in_progress2, command_in_progress2_report)
   commandStatuses.put_command_status(finished_command, finished_command_report)
   commandStatuses.put_command_status(failed_command, failed_command_report)
   commandStatuses.put_command_status(status_command, status_command_report)
   report = commandStatuses.generate_report()
   expected = \
     {'componentStatus': [{'status': 'HEALTHY', 'componentName': 'DATANODE'}],
      'reports': [{'status': 'FAILED', 'taskId': 3},
                  {'status': 'COMPLETE', 'taskId': 4},
                  {'status': 'IN_PROGRESS', 'stderr': '...',
                   'stdout': '...', 'clusterName': u'cc',
                   'structuredOut' : '{}',
                   'roleCommand': u'INSTALL', 'serviceName': u'HDFS',
                   'role': u'DATANODE', 'actionId': '1-1', 'taskId': 5,
                   'exitCode': 777},
                  {'status': 'IN_PROGRESS',
                   'stderr': '...',
                   'stdout': '...',
                   'structuredOut' : '{}',
                   'clusterName': u'cc',
                   'roleCommand': u'INSTALL',
                   'serviceName': u'HDFS',
                   'role': u'DATANODE',
                   'actionId': '1-1',
                   'taskId': 6,
                   'exitCode': 777}]
     }
   self.assertEquals(report, expected)
Exemplo n.º 9
0
 def test_put_and_generate(self):
     callback_mock = MagicMock()
     commandStatuses = CommandStatusDict(callback_action=callback_mock)
     command_in_progress1 = {
         'commandType': 'EXECUTION_COMMAND',
         'commandId': '1-1',
         'clusterName': u'cc',
         'exitCode': 777,
         'role': u'DATANODE',
         'roleCommand': u'INSTALL',
         'serviceName': u'HDFS',
         'stderr': '',
         'stdout':
         "notice: /Stage[1]/Hdp::Iptables/Service[iptables]/ensure: ensure changed 'running' to 'stopped'\nnotice: /Stage[1]/Hdp/File[/tmp/changeUid.sh]/ensure: defined content as '{md5}32b994a2e970f8acc3c91c198b484654'\nnotice: /Stage[1]/Hdp::Snappy::Package/Hdp::Package[snappy]/Hdp::Package::Process_pkg[snappy]/Package[snappy]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::Group[nagios_group]/Group[nagios_group]/ensure: created\nnotice: /Stage[1]/Hdp/Hdp::User[nagios_user]/User[nagios]/ensure: created\nnotice: /Stage[1]/Hdp::Snmp/Hdp::Package[snmp]/Hdp::Package::Process_pkg[snmp]/Package[net-snmp-utils]/ensure: created",
         'taskId': 5
     }
     command_in_progress1_report = {'status': 'IN_PROGRESS', 'taskId': 5}
     command_in_progress2 = {
         'commandType': 'EXECUTION_COMMAND',
         'commandId': '1-1',
         'role': u'DATANODE',
         'roleCommand': u'INSTALL',
         'taskId': 6,
         'clusterName': u'cc',
         'serviceName': u'HDFS',
     }
     command_in_progress2_report = {'status': 'IN_PROGRESS', 'taskId': 6}
     finished_command = {
         'commandType': 'EXECUTION_COMMAND',
         'role': u'DATANODE',
         'roleCommand': u'INSTALL',
         'commandId': '1-1',
         'taskId': 4,
         'clusterName': u'cc',
         'serviceName': u'HDFS',
     }
     finished_command_report = {
         'status': 'COMPLETE',
         'taskId': 4,
     }
     failed_command = {
         'commandType': 'EXECUTION_COMMAND',
         'role': u'DATANODE',
         'roleCommand': u'INSTALL',
         'commandId': '1-1',
         'taskId': 3,
         'clusterName': u'cc',
         'serviceName': u'HDFS',
     }
     failed_command_report = {
         'status': 'FAILED',
         'taskId': 3,
     }
     status_command = {
         'componentName': 'DATANODE',
         'commandType': 'STATUS_COMMAND',
     }
     status_command_report = {
         'componentName': 'DATANODE',
         'status': 'HEALTHY'
     }
     commandStatuses.put_command_status(command_in_progress1,
                                        command_in_progress1_report)
     commandStatuses.put_command_status(command_in_progress2,
                                        command_in_progress2_report)
     commandStatuses.put_command_status(finished_command,
                                        finished_command_report)
     commandStatuses.put_command_status(failed_command,
                                        failed_command_report)
     commandStatuses.put_command_status(status_command,
                                        status_command_report)
     report = commandStatuses.generate_report()
     expected = \
       {'componentStatus': [{'status': 'HEALTHY', 'componentName': 'DATANODE'}],
        'reports': [{'status': 'FAILED', 'taskId': 3},
                    {'status': 'COMPLETE', 'taskId': 4},
                    {'status': 'IN_PROGRESS', 'stderr': '...',
                     'stdout': '...', 'clusterName': u'cc',
                     'structuredOut' : '{}',
                     'roleCommand': u'INSTALL', 'serviceName': u'HDFS',
                     'role': u'DATANODE', 'actionId': '1-1', 'taskId': 5,
                     'exitCode': 777},
                    {'status': 'IN_PROGRESS',
                     'stderr': '...',
                     'stdout': '...',
                     'structuredOut' : '{}',
                     'clusterName': u'cc',
                     'roleCommand': u'INSTALL',
                     'serviceName': u'HDFS',
                     'role': u'DATANODE',
                     'actionId': '1-1',
                     'taskId': 6,
                     'exitCode': 777}]
       }
     self.assertEquals(report, expected)
    def test_split_reports(self):
        # 4 reports for each cluster, general size in json = 295
        generated_reports = \
          {'1': [{'status': 'FAILED', 'taskId': 3},
                       {'status': 'FAILED', 'taskId': 4},
                       {'status': 'FAILED', 'taskId': 5},
                       {'status': 'FAILED', 'taskId': 6}],
           '2': [{'status': 'FAILED', 'taskId': 7},
                 {'status': 'FAILED', 'taskId': 8},
                 {'status': 'FAILED', 'taskId': 9},
                 {'status': 'FAILED', 'taskId': 10}],
          }
        mock = MagicMock()
        command_statuses = CommandStatusDict(mock)

        # all reports will be send at once
        splitted_reports = []
        for report in command_statuses.split_reports(generated_reports, 295):
            splitted_reports.append(report)

        self.assertEqual(len(splitted_reports), 1)
        self.assertEqual(len(splitted_reports[0]), 2)
        self.assertEqual(len(splitted_reports[0]['1']), 4)
        self.assertEqual(len(splitted_reports[0]['2']), 4)

        # all reports will be divided between two parts
        # {'1': [{3}, {4}, {5}, {6}], '2': [{7}, {8}, {9}]}
        # {'2': [{10}]}
        splitted_reports = []
        for report in command_statuses.split_reports(generated_reports, 294):
            splitted_reports.append(report)

        self.assertEqual(len(splitted_reports), 2)
        self.assertEqual(len(splitted_reports[0]), 2)
        self.assertEqual(len(splitted_reports[0]['1']), 4)
        self.assertEqual(len(splitted_reports[0]['2']), 3)
        self.assertEqual(len(splitted_reports[1]), 1)
        self.assertEqual(len(splitted_reports[1]['2']), 1)

        # all reports will be divided between 8 parts
        # {'1': [{3}]}
        #...
        # {'2': [{10}]}
        splitted_reports = []
        for report in command_statuses.split_reports(generated_reports, 73):
            splitted_reports.append(report)

        self.assertEqual(len(splitted_reports), 8)
        self.assertEqual(len(splitted_reports[0]), 1)
        self.assertEqual(len(splitted_reports[0]['1']), 1)
        self.assertEqual(len(splitted_reports[1]), 1)
        self.assertEqual(len(splitted_reports[1]['1']), 1)
        self.assertEqual(len(splitted_reports[2]), 1)
        self.assertEqual(len(splitted_reports[2]['1']), 1)
        self.assertEqual(len(splitted_reports[3]), 1)
        self.assertEqual(len(splitted_reports[3]['1']), 1)
        self.assertEqual(len(splitted_reports[4]), 1)
        self.assertEqual(len(splitted_reports[4]['2']), 1)
        self.assertEqual(len(splitted_reports[5]), 1)
        self.assertEqual(len(splitted_reports[5]['2']), 1)
        self.assertEqual(len(splitted_reports[6]), 1)
        self.assertEqual(len(splitted_reports[6]['2']), 1)
        self.assertEqual(len(splitted_reports[7]), 1)
        self.assertEqual(len(splitted_reports[7]['2']), 1)