Exemple #1
0
 def set_hostname(self, hostname):
     """
     Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
     Due to a bug in systemd in Centos-7.0, if this call fails, fallback
     to hostname.
     """
     hostnamectl_cmd = "hostnamectl set-hostname {0} --static".format(hostname)
     if shellutil.run(hostnamectl_cmd, chk_err=False) != 0:
         logger.warn("[{0}] failed, attempting fallback".format(hostnamectl_cmd))
         DefaultOSUtil.set_hostname(self, hostname)
Exemple #2
0
 def set_hostname(self, hostname):
     """
     Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
     Due to a bug in systemd in Centos-7.0, if this call fails, fallback
     to hostname.
     """
     hostnamectl_cmd = ['hostnamectl', 'set-hostname', hostname, '--static']
     if self._run_command_without_raising(hostnamectl_cmd,
                                          log_error=False) != 0:
         logger.warn("[{0}] failed, attempting fallback".format(
             ' '.join(hostnamectl_cmd)))
         DefaultOSUtil.set_hostname(self, hostname)
Exemple #3
0
 def set_hostname(self, hostname):
     """
     Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
     Due to a bug in systemd in Centos-7.0, if this call fails, fallback
     to hostname.
     """
     hostnamectl_cmd = ["hostnamectl", "set-hostname", hostname, "--static"]
     try:
         shellutil.run_command(hostnamectl_cmd)
     except Exception as e:
         logger.warn("[{0}] failed with error: {1}, attempting fallback".format(' '.join(hostnamectl_cmd), ustr(e)))
         DefaultOSUtil.set_hostname(self, hostname)
Exemple #4
0
    def setUp(self):
        AgentTestCase.setUp(self)

        # save the original run_command so that mocks can reference it
        self.shellutil_run_command = shellutil.run_command

        # save an instance of the original DefaultOSUtil so that mocks can reference it
        self.default_osutil = DefaultOSUtil()

        # AgentTestCase.setUp mocks osutil.factory._get_osutil; we override that mock for this class with a new mock
        # that always returns the default implementation.
        self.mock_get_osutil = patch(
            "azurelinuxagent.common.osutil.factory._get_osutil",
            return_value=DefaultOSUtil())
        self.mock_get_osutil.start()
Exemple #5
0
    def test_mount_cgroups_should_handle_errors_when_mounting_an_individual_controller(
            self):
        original_run_get_output = shellutil.run_get_output

        def mock_run_get_output(cmd, *args, **kwargs):
            if cmd.startswith('mount '):
                if 'memory' in cmd:
                    raise Exception(
                        'A test exception mounting the memory controller')
                return 0, None
            return original_run_get_output(cmd, *args, **kwargs)

        with patch(
                "azurelinuxagent.common.osutil.default.shellutil.run_get_output",
                side_effect=mock_run_get_output) as patch_run_get_output:
            with patch("azurelinuxagent.common.cgroupconfigurator.logger.warn"
                       ) as mock_logger_warn:
                DefaultOSUtil().mount_cgroups()

                # the cgroup filesystem and the cpu controller should still have been mounted
                mount_commands = DefaultOsUtilTestCase._get_mount_commands(
                    patch_run_get_output)

                self.assertRegex(mount_commands, ';mount.* cgroup_root ',
                                 'The cgroups file system was not mounted')
                self.assertRegex(mount_commands, ';mount.* cpu,cpuacct ',
                                 'The cpu controller was not mounted')

                # A warning should have been logged for the memory controller
                args, kwargs = mock_logger_warn.call_args
                self.assertIn(
                    'A test exception mounting the memory controller', args)
Exemple #6
0
    def setUpClass(cls):
        AgentTestCase.setUpClass()

        # Use the file system implementation of CGroupsApi (FileSystemCgroupsApi)
        cls.mock_is_systemd = patch(
            "azurelinuxagent.common.cgroupapi.CGroupsApi._is_systemd",
            return_value=False)
        cls.mock_is_systemd.start()

        # Use the default implementation of osutil
        cls.mock_get_osutil = patch(
            "azurelinuxagent.common.cgroupconfigurator.get_osutil",
            return_value=DefaultOSUtil())
        cls.mock_get_osutil.start()

        # Currently osutil.is_cgroups_supported() returns False on Travis runs. We need to revisit this design; in the
        # meanwhile mock the method to return True
        cls.mock_is_cgroups_supported = patch(
            "azurelinuxagent.common.osutil.default.DefaultOSUtil.is_cgroups_supported",
            return_value=True)
        cls.mock_is_cgroups_supported.start()

        # Mounting the cgroup filesystem requires root privileges. Since these tests do not perform any actual operation on cgroups, make it a noop.
        cls.mock_mount_cgroups = patch(
            "azurelinuxagent.common.osutil.default.DefaultOSUtil.mount_cgroups"
        )
        cls.mock_mount_cgroups.start()
Exemple #7
0
    def test_mount_cgroups_should_not_mount_the_cgroups_file_system_when_it_already_exists(
            self):
        os.mkdir(self.cgroups_file_system_root)

        original_run_get_output = shellutil.run_get_output

        def mock_run_get_output(cmd, *args, **kwargs):
            if cmd.startswith('mount '):
                return 0, None
            return original_run_get_output(cmd, *args, **kwargs)

        with patch(
                "azurelinuxagent.common.osutil.default.shellutil.run_get_output",
                side_effect=mock_run_get_output) as patch_run_get_output:
            DefaultOSUtil().mount_cgroups()

            mount_commands = DefaultOsUtilTestCase._get_mount_commands(
                patch_run_get_output)

            self.assertNotIn(
                'cgroup_root', mount_commands,
                'The cgroups file system should not have been mounted')
            self.assertRegex(mount_commands, ';mount.* cpu,cpuacct ',
                             'The cpu controller was not mounted')
            self.assertRegex(mount_commands, ';mount.* memory ',
                             'The memory controller was not mounted')
Exemple #8
0
    def test_mount_cgroups_should_mount_the_cpu_and_memory_controllers(self):
        # the mount command requires root privileges; make it a no op and check only for file existence
        original_run_get_output = shellutil.run_get_output

        def mock_run_get_output(cmd, *args, **kwargs):
            if cmd.startswith('mount '):
                return 0, None
            return original_run_get_output(cmd, *args, **kwargs)

        with patch(
                "azurelinuxagent.common.osutil.default.shellutil.run_get_output",
                side_effect=mock_run_get_output) as patch_run_get_output:
            DefaultOSUtil().mount_cgroups()

            # the directories for the controllers should have been created
            for controller in ['cpu', 'memory', 'cpuacct', 'cpu,cpuacct']:
                directory = os.path.join(self.cgroups_file_system_root,
                                         controller)
                self.assertTrue(
                    os.path.exists(directory),
                    "A directory for controller {0} was not created".format(
                        controller))

            # the cgroup filesystem and the cpu and memory controllers should have been mounted
            mount_commands = DefaultOsUtilTestCase._get_mount_commands(
                patch_run_get_output)

            self.assertRegex(mount_commands, ';mount.* cgroup_root ',
                             'The cgroups file system was not mounted')
            self.assertRegex(mount_commands, ';mount.* cpu,cpuacct ',
                             'The cpu controller was not mounted')
            self.assertRegex(mount_commands, ';mount.* memory ',
                             'The memory controller was not mounted')
Exemple #9
0
    def test_mount_cgroups_should_raise_when_the_cgroups_filesystem_fails_to_mount(
            self):
        original_run_get_output = shellutil.run_get_output

        def mock_run_get_output(cmd, *args, **kwargs):
            if cmd.startswith('mount '):
                if 'cgroup_root' in cmd:
                    raise Exception(
                        'A test exception mounting the cgroups file system')
                return 0, None
            return original_run_get_output(cmd, *args, **kwargs)

        with patch(
                "azurelinuxagent.common.osutil.default.shellutil.run_get_output",
                side_effect=mock_run_get_output) as patch_run_get_output:
            with self.assertRaises(Exception) as context_manager:
                DefaultOSUtil().mount_cgroups()

            self.assertRegex(
                str(context_manager.exception),
                'A test exception mounting the cgroups file system')

            mount_commands = DefaultOsUtilTestCase._get_mount_commands(
                patch_run_get_output)
            self.assertNotIn(
                'memory', mount_commands,
                'The memory controller should not have been mounted')
            self.assertNotIn(
                'cpu', mount_commands,
                'The cpu controller should not have been mounted')
Exemple #10
0
    def _get_persist_firewall_rules_handler(self, systemd=True):

        osutil = DefaultOSUtil()
        osutil.get_firewall_will_wait = MagicMock(
            return_value=self.__test_wait)
        osutil.get_agent_bin_path = MagicMock(
            return_value=self.__agent_bin_dir)
        osutil.get_systemd_unit_file_install_path = MagicMock(
            return_value=self.__systemd_dir)

        self._expected_service_name = PersistFirewallRulesHandler._AGENT_NETWORK_SETUP_NAME_FORMAT.format(
            osutil.get_service_name())

        self._network_service_unit_file = os.path.join(
            self.__systemd_dir, self._expected_service_name)
        self._binary_file = os.path.join(
            conf.get_lib_dir(), PersistFirewallRulesHandler.BINARY_FILE_NAME)

        # Just for these tests, ignoring the mode of mkdir to allow non-sudo tests
        orig_mkdir = fileutil.mkdir
        with patch(
                "azurelinuxagent.common.persist_firewall_rules.fileutil.mkdir",
                side_effect=lambda path, **mode: orig_mkdir(path)):
            with patch(
                    "azurelinuxagent.common.persist_firewall_rules.get_osutil",
                    return_value=osutil):
                with patch('azurelinuxagent.common.osutil.systemd.is_systemd',
                           return_value=systemd):
                    with patch(
                            "azurelinuxagent.common.utils.shellutil.subprocess.Popen",
                            side_effect=self.__mock_popen):
                        yield PersistFirewallRulesHandler(
                            self.__test_dst_ip, self.__test_uid)
    def test_mount_cgroups_should_not_create_symbolic_links_when_the_cpu_controller_fails_to_mount(self):
        original_run_get_output = shellutil.run_get_output

        def mock_run_get_output(cmd, *args, **kwargs):
            if cmd.startswith('mount '):
                if 'cpu,cpuacct' in cmd:
                    raise Exception('A test exception mounting the cpu controller')
                return 0, None
            return original_run_get_output(cmd, *args, **kwargs)

        with patch("azurelinuxagent.common.osutil.default.shellutil.run_get_output", side_effect=mock_run_get_output):
            with patch("azurelinuxagent.common.osutil.default.os.symlink") as patch_symlink:
                DefaultOSUtil().mount_cgroups()

                self.assertEquals(patch_symlink.call_count, 0, 'A symbolic link should not have been created')
    def test_mount_cgroups_should_raise_when_all_controllers_fail_to_mount(self):
        original_run_get_output = shellutil.run_get_output

        def mock_run_get_output(cmd, *args, **kwargs):
            if cmd.startswith('mount '):
                if 'memory' in cmd or 'cpu,cpuacct' in cmd:
                    raise Exception('A test exception mounting a cgroup controller')
                return 0, None
            return original_run_get_output(cmd, *args, **kwargs)

        with patch("azurelinuxagent.common.osutil.default.shellutil.run_get_output", side_effect=mock_run_get_output):
            with self.assertRaises(Exception) as context_manager:
                DefaultOSUtil().mount_cgroups()

            self.assertRegex(str(context_manager.exception), 'A test exception mounting a cgroup controller')
Exemple #13
0
    def test_provisioning_is_skipped_when_not_enabled(self, mock_conf):  # pylint: disable=unused-argument
        ph = ProvisionHandler()  # pylint: disable=invalid-name
        ph.osutil = DefaultOSUtil()
        ph.osutil.get_instance_id = Mock(
            return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502')

        ph.check_provisioned_file = Mock()
        ph.report_ready = Mock()
        ph.write_provisioned = Mock()

        ph.run()

        self.assertEqual(0, ph.check_provisioned_file.call_count)
        self.assertEqual(1, ph.report_ready.call_count)
        self.assertEqual(1, ph.write_provisioned.call_count)
Exemple #14
0
    def test_provisioning_is_skipped_when_not_enabled(self, mock_conf):
        ph = ProvisionHandler()
        ph.osutil = DefaultOSUtil()
        ph.osutil.get_instance_id = Mock(
                        return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502')

        ph.is_provisioned = Mock()
        ph.report_ready = Mock()
        ph.write_provisioned = Mock()

        ph.run()

        ph.is_provisioned.assert_not_called()
        ph.report_ready.assert_called_once()
        ph.write_provisioned.assert_called_once()
Exemple #15
0
    def get_list_of_routes(route_table):
        """
        Construct a list of all network routes known to this system.

        :param list(str) route_table: List of text entries from route table, including headers
        :return: a list of network routes
        :rtype: list(RouteEntry)
        """
        route_list = []
        count = len(route_table)

        if count < 1:
            logger.error("netstat -rn -f inet is missing headers")
        elif count == 1:
            logger.error("netstat -rn -f inet contains no routes")
        else:
            route_list = DefaultOSUtil._build_route_list(route_table)
        return route_list
Exemple #16
0
 def set_hostname(self, hostname):
     """
     Set /etc/hostname
     Unlike redhat 6.x, redhat 7.x will set hostname to /etc/hostname
     """
     DefaultOSUtil.set_hostname(self, hostname)
Exemple #17
0
 def test_customdata(self):
     base64data = 'Q3VzdG9tRGF0YQ=='
     data = DefaultOSUtil().decode_customdata(base64data)
     fileutil.write_file(tempfile.mktemp(), data)
 def test_default_service_name(self):
     self.assertEquals(DefaultOSUtil().get_service_name(), "waagent")
Exemple #19
0
 def openssl_to_openssh(self, input_file, output_file):
     DefaultOSUtil.openssl_to_openssh(self, input_file, output_file)
Exemple #20
0
 def openssl_to_openssh(self, input_file, output_file):
     DefaultOSUtil.openssl_to_openssh(self, input_file, output_file)
Exemple #21
0
 def __init__(self):
     self._default = DefaultOSUtil()
     self.jit_enabled = False # ga/remoteaccess.py
class TestCGroupsTelemetry(AgentTestCase):
    TestProcessIds = ["1000", "1001", "1002"]
    TestProcStatmMemoryUsed = 1234
    TestProcComm = "python"
    TestProcCommandLine = "python -u bin/WALinuxAgent-2.2.45-py2.7.egg -run-exthandlers"
    NumSummarizationValues = 7

    @classmethod
    def setUpClass(cls):
        AgentTestCase.setUpClass()

        # Use the default value for memory used from proc_statm
        cls.mock_get_memory_usage_from_proc_statm = patch(
            "azurelinuxagent.common.resourceusage.MemoryResourceUsage."
            "get_memory_usage_from_proc_statm",
            return_value=TestCGroupsTelemetry.TestProcStatmMemoryUsed)
        cls.mock_get_memory_usage_from_proc_statm.start()

        # Use the default value for memory used from proc_statm
        cls.mock_get_tracked_processes = patch(
            "azurelinuxagent.common.cgroup.CGroup.get_tracked_processes",
            return_value=TestCGroupsTelemetry.TestProcessIds)
        cls.mock_get_tracked_processes.start()

        cls.mock_get_proc_name = patch(
            "azurelinuxagent.common.resourceusage.ProcessInfo.get_proc_name",
            return_value=TestCGroupsTelemetry.TestProcComm)
        cls.mock_get_proc_name.start()

        cls.mock_get_proc_cmdline = patch(
            "azurelinuxagent.common.resourceusage.ProcessInfo.get_proc_cmdline",
            return_value=TestCGroupsTelemetry.TestProcCommandLine)
        cls.mock_get_proc_cmdline.start()

        # CPU Cgroups compute usage based on /proc/stat and /sys/fs/cgroup/.../cpuacct.stat; use mock data for those
        # files
        original_read_file = fileutil.read_file

        def mock_read_file(filepath, **args):
            if filepath == "/proc/stat":
                filepath = os.path.join(data_dir, "cgroups", "proc_stat_t0")
            elif filepath.endswith("/cpuacct.stat"):
                filepath = os.path.join(data_dir, "cgroups", "cpuacct.stat_t0")
            return original_read_file(filepath, **args)

        cls._mock_read_cpu_cgroup_file = patch(
            "azurelinuxagent.common.utils.fileutil.read_file",
            side_effect=mock_read_file)
        cls._mock_read_cpu_cgroup_file.start()

    @classmethod
    def tearDownClass(cls):
        cls.mock_get_memory_usage_from_proc_statm.stop()
        cls.mock_get_tracked_processes.stop()
        cls.mock_get_proc_name.stop()
        cls.mock_get_proc_cmdline.stop()
        cls._mock_read_cpu_cgroup_file.stop()

        AgentTestCase.tearDownClass()

    def setUp(self):
        AgentTestCase.setUp(self)
        CGroupsTelemetry.reset()

    def tearDown(self):
        AgentTestCase.tearDown(self)
        CGroupsTelemetry.reset()

    @staticmethod
    def _track_new_extension_cgroups(num_extensions):
        for i in range(num_extensions):
            dummy_cpu_cgroup = CGroup.create("dummy_cpu_path_{0}".format(i),
                                             "cpu",
                                             "dummy_extension_{0}".format(i))
            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)

            dummy_memory_cgroup = CGroup.create(
                "dummy_memory_path_{0}".format(i), "memory",
                "dummy_extension_{0}".format(i))
            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)

    def _assert_cgroups_are_tracked(self, num_extensions):
        for i in range(num_extensions):
            self.assertTrue(
                CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
            self.assertTrue(
                CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))

    def _assert_calculated_resource_metrics_equal(self,
                                                  cpu_usage,
                                                  memory_usage,
                                                  max_memory_usage,
                                                  memory_statm_memory_usage,
                                                  proc_ids=None):
        if not proc_ids:
            proc_ids = TestCGroupsTelemetry.TestProcessIds

        processes_instances = [
            CGroupsTelemetry.get_process_info_summary(pid) for pid in proc_ids
        ]
        for _, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
            self.assertListEqual(cgroup_metric.get_memory_metrics()._data,
                                 memory_usage)
            self.assertListEqual(cgroup_metric.get_max_memory_metrics()._data,
                                 max_memory_usage)
            self.assertListEqual(cgroup_metric.get_cpu_metrics()._data,
                                 cpu_usage)
            for kv_pair in cgroup_metric.get_proc_statm_memory_metrics():
                self.assertIn(kv_pair.pid_name_cmdline, processes_instances)
                self.assertListEqual(kv_pair.resource_metric._data,
                                     memory_statm_memory_usage)

    def _assert_polled_metrics_equal(self,
                                     metrics,
                                     cpu_metric_value,
                                     memory_metric_value,
                                     max_memory_metric_value,
                                     proc_stat_memory_usage_value,
                                     pids=None):
        for metric in metrics:
            self.assertIn(metric.category, ["Process", "Memory"])
            if metric.category == "Process":
                self.assertEqual(metric.counter, "% Processor Time")
                self.assertEqual(metric.value, cpu_metric_value)
            if metric.category == "Memory":
                self.assertIn(metric.counter, [
                    "Total Memory Usage", "Max Memory Usage",
                    "Memory Used by Process"
                ])
                if metric.counter == "Total Memory Usage":
                    self.assertEqual(metric.value, memory_metric_value)
                elif metric.counter == "Max Memory Usage":
                    self.assertEqual(metric.value, max_memory_metric_value)
                elif metric.counter == "Memory Used by Process":
                    if pids:
                        processes_instances = [
                            CGroupsTelemetry.get_process_info_summary(pid)
                            for pid in pids
                        ]
                    else:
                        processes_instances = [
                            CGroupsTelemetry.get_process_info_summary(pid)
                            for pid in TestCGroupsTelemetry.TestProcessIds
                        ]
                    self.assertIn(metric.instance, processes_instances)
                    self.assertEqual(metric.value,
                                     proc_stat_memory_usage_value)

    def _assert_extension_metrics_data(self,
                                       collected_metrics,
                                       num_extensions,
                                       cpu_percent_values,
                                       proc_stat_memory_usage_values,
                                       memory_usage_values,
                                       max_memory_usage_values,
                                       is_cpu_present=True,
                                       is_memory_present=True):
        num_summarization_values = TestCGroupsTelemetry.NumSummarizationValues

        if not (is_cpu_present or is_memory_present):
            self.assertEquals(collected_metrics, {})
            return
        else:
            for i in range(num_extensions):
                name = "dummy_extension_{0}".format(i)

                if is_memory_present:
                    self.assertIn(name, collected_metrics)
                    self.assertIn("memory", collected_metrics[name])
                    self.assertIn("cur_mem", collected_metrics[name]["memory"])
                    self.assertIn("max_mem", collected_metrics[name]["memory"])
                    self.assertEqual(
                        num_summarization_values,
                        len(collected_metrics[name]["memory"]["cur_mem"]))
                    self.assertEqual(
                        num_summarization_values,
                        len(collected_metrics[name]["memory"]["max_mem"]))

                    self.assertIn("proc_statm_memory", collected_metrics[name])
                    self.assertEqual(
                        3, len(collected_metrics[name]["proc_statm_memory"]
                               ))  # number of processes added
                    for tracked_process in collected_metrics[name][
                            "proc_statm_memory"]:
                        self.assertEqual(
                            num_summarization_values,
                            len(collected_metrics[name]["proc_statm_memory"]
                                [tracked_process]))
                        self.assertListEqual(
                            generate_metric_list(
                                proc_stat_memory_usage_values),
                            collected_metrics[name]["proc_statm_memory"]
                            [tracked_process][0:5])

                    self.assertListEqual(
                        generate_metric_list(memory_usage_values),
                        collected_metrics[name]["memory"]["cur_mem"][0:5])
                    self.assertListEqual(
                        generate_metric_list(max_memory_usage_values),
                        collected_metrics[name]["memory"]["max_mem"][0:5])

                if is_cpu_present:
                    self.assertIn("cpu", collected_metrics[name])
                    self.assertIn("cur_cpu", collected_metrics[name]["cpu"])
                    self.assertEqual(
                        num_summarization_values,
                        len(collected_metrics[name]["cpu"]["cur_cpu"]))
                    self.assertListEqual(
                        generate_metric_list(cpu_percent_values),
                        collected_metrics[name]["cpu"]["cur_cpu"][0:5])

    def test_telemetry_polling_with_active_cgroups(self, *args):
        num_extensions = 3

        self._track_new_extension_cgroups(num_extensions)

        with patch(
                "azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage"
        ) as patch_get_memory_max_usage:
            with patch(
                    "azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage"
            ) as patch_get_memory_usage:
                with patch(
                        "azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage"
                ) as patch_get_cpu_usage:
                    with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
                               ) as patch_is_active:
                        patch_is_active.return_value = True

                        current_cpu = 30
                        current_memory = 209715200
                        current_max_memory = 471859200
                        current_proc_statm = TestCGroupsTelemetry.TestProcStatmMemoryUsed

                        # 1 CPU metric + 1 Current Memory + 1 Max memor + num_processes * memory from statm
                        num_of_metrics_per_extn_expected = 1 + 1 + 1 + 3 * 1
                        patch_get_cpu_usage.return_value = current_cpu
                        patch_get_memory_usage.return_value = current_memory  # example 200 MB
                        patch_get_memory_max_usage.return_value = current_max_memory  # example 450 MB
                        num_polls = 10

                        for data_count in range(1, num_polls + 1):
                            metrics = CGroupsTelemetry.poll_all_tracked()

                            self.assertEqual(
                                len(CGroupsTelemetry._cgroup_metrics),
                                num_extensions)
                            self._assert_calculated_resource_metrics_equal(
                                cpu_usage=[current_cpu] * data_count,
                                memory_usage=[current_memory] * data_count,
                                max_memory_usage=[current_max_memory] *
                                data_count,
                                proc_ids=TestCGroupsTelemetry.TestProcessIds,
                                memory_statm_memory_usage=[current_proc_statm
                                                           ] * data_count)
                            self.assertEqual(
                                len(metrics), num_extensions *
                                num_of_metrics_per_extn_expected)
                            self._assert_polled_metrics_equal(
                                metrics, current_cpu, current_memory,
                                current_max_memory, current_proc_statm)

        collected_metrics = CGroupsTelemetry.report_all_tracked()

        self._assert_extension_metrics_data(
            collected_metrics,
            num_extensions, [current_cpu] * num_polls,
            [TestCGroupsTelemetry.TestProcStatmMemoryUsed] * num_polls,
            [current_memory] * num_polls, [current_max_memory] * num_polls,
            is_cpu_present=False)

        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                         num_extensions)
        self._assert_calculated_resource_metrics_equal([], [], [], [], [])

    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage",
           side_effect=raise_ioerror)
    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
           side_effect=raise_ioerror)
    @patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
           side_effect=raise_ioerror)
    @patch("azurelinuxagent.common.cgroup.CGroup.is_active",
           return_value=False)
    def test_telemetry_polling_with_inactive_cgroups(self, *_):
        num_extensions = 5
        no_extensions_expected = 0

        self._track_new_extension_cgroups(num_extensions)
        self._assert_cgroups_are_tracked(num_extensions)

        metrics = CGroupsTelemetry.poll_all_tracked()

        for i in range(num_extensions):
            self.assertFalse(
                CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
            self.assertFalse(
                CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))

        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                         num_extensions)
        self._assert_calculated_resource_metrics_equal([], [], [], [],
                                                       proc_ids=None)
        self.assertEqual(len(metrics), 0)

        collected_metrics = CGroupsTelemetry.report_all_tracked()
        self._assert_extension_metrics_data(collected_metrics,
                                            num_extensions, [], [], [], [],
                                            is_cpu_present=False,
                                            is_memory_present=False)
        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                         no_extensions_expected)
        self._assert_calculated_resource_metrics_equal([], [], [], [], [])

    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage")
    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage")
    @patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage")
    @patch("azurelinuxagent.common.cgroup.CGroup.is_active")
    @patch(
        "azurelinuxagent.common.resourceusage.MemoryResourceUsage.get_memory_usage_from_proc_statm"
    )
    def test_telemetry_polling_with_changing_cgroups_state(
            self, patch_get_statm, patch_is_active, patch_get_cpu_usage,
            patch_get_mem, patch_get_max_mem, *args):
        num_extensions = 5
        self._track_new_extension_cgroups(num_extensions)

        patch_is_active.return_value = True

        no_extensions_expected = 0
        expected_data_count = 1

        current_cpu = 30
        current_memory = 209715200
        current_max_memory = 471859200
        current_proc_statm = 20000000

        patch_get_cpu_usage.return_value = current_cpu
        patch_get_mem.return_value = current_memory  # example 200 MB
        patch_get_max_mem.return_value = current_max_memory  # example 450 MB
        patch_get_statm.return_value = current_proc_statm

        self._assert_cgroups_are_tracked(num_extensions)
        CGroupsTelemetry.poll_all_tracked()

        self._assert_cgroups_are_tracked(num_extensions)

        patch_is_active.return_value = False
        patch_get_cpu_usage.side_effect = raise_ioerror
        patch_get_mem.side_effect = raise_ioerror
        patch_get_max_mem.side_effect = raise_ioerror
        patch_get_statm.side_effect = raise_ioerror

        CGroupsTelemetry.poll_all_tracked()

        for i in range(num_extensions):
            self.assertFalse(
                CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
            self.assertFalse(
                CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))

        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                         num_extensions)
        self._assert_calculated_resource_metrics_equal(
            cpu_usage=[current_cpu] * expected_data_count,
            memory_usage=[current_memory] * expected_data_count,
            max_memory_usage=[current_max_memory] * expected_data_count,
            proc_ids=TestCGroupsTelemetry.TestProcessIds,
            memory_statm_memory_usage=[current_proc_statm] *
            expected_data_count)

        CGroupsTelemetry.report_all_tracked()

        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                         no_extensions_expected)
        self._assert_calculated_resource_metrics_equal([], [], [], [], [])

    # mocking get_proc_stat to make it run on Mac and other systems. This test does not need to read the values of the
    # /proc/stat file on the filesystem.
    @patch("azurelinuxagent.common.logger.periodic_warn")
    def test_telemetry_polling_to_not_generate_transient_logs_ioerror_file_not_found(
            self, patch_periodic_warn):
        num_extensions = 1
        self._track_new_extension_cgroups(num_extensions)
        self.assertEqual(0, patch_periodic_warn.call_count)

        # Not expecting logs present for io_error with errno=errno.ENOENT
        io_error_2 = IOError()
        io_error_2.errno = errno.ENOENT

        with patch("azurelinuxagent.common.utils.fileutil.read_file",
                   side_effect=io_error_2):
            poll_count = 1
            for data_count in range(poll_count, 10):
                CGroupsTelemetry.poll_all_tracked()
                self.assertEqual(0, patch_periodic_warn.call_count)

    @patch("azurelinuxagent.common.logger.periodic_warn")
    def test_telemetry_polling_to_generate_transient_logs_ioerror_permission_denied(
            self, patch_periodic_warn):
        num_extensions = 1
        num_controllers = 2
        is_active_check_per_controller = 2
        self._track_new_extension_cgroups(num_extensions)

        self.assertEqual(0, patch_periodic_warn.call_count)

        # Expecting logs to be present for different kind of errors
        io_error_3 = IOError()
        io_error_3.errno = errno.EPERM

        with patch("azurelinuxagent.common.utils.fileutil.read_file",
                   side_effect=io_error_3):
            poll_count = 1
            expected_count_per_call = num_controllers + is_active_check_per_controller
            # each collect per controller would generate a log statement, and each cgroup would invoke a
            # is active check raising an exception

            for data_count in range(poll_count, 10):
                CGroupsTelemetry.poll_all_tracked()
                self.assertEqual(poll_count * expected_count_per_call,
                                 patch_periodic_warn.call_count)

    def test_telemetry_polling_to_generate_transient_logs_index_error(self):
        num_extensions = 1
        self._track_new_extension_cgroups(num_extensions)

        # Generating a different kind of error (non-IOError) to check the logging.
        # Trying to invoke IndexError during the getParameter call
        with patch("azurelinuxagent.common.utils.fileutil.read_file",
                   return_value=''):
            with patch("azurelinuxagent.common.logger.periodic_warn"
                       ) as patch_periodic_warn:
                expected_call_count = 2  # 1 periodic warning for the cpu cgroups, and 1 for memory
                for data_count in range(1, 10):
                    CGroupsTelemetry.poll_all_tracked()
                    self.assertEqual(expected_call_count,
                                     patch_periodic_warn.call_count)

    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage")
    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage")
    @patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage")
    @patch("azurelinuxagent.common.cgroup.CGroup.is_active")
    @patch(
        "azurelinuxagent.common.resourceusage.MemoryResourceUsage.get_memory_usage_from_proc_statm"
    )
    def test_telemetry_calculations(self, patch_get_statm, patch_is_active,
                                    patch_get_cpu_usage,
                                    patch_get_memory_usage,
                                    patch_get_memory_max_usage, *args):
        num_polls = 10
        num_extensions = 1

        cpu_percent_values = [random.randint(0, 100) for _ in range(num_polls)]

        # only verifying calculations and not validity of the values.
        memory_usage_values = [
            random.randint(0, 8 * 1024**3) for _ in range(num_polls)
        ]
        max_memory_usage_values = [
            random.randint(0, 8 * 1024**3) for _ in range(num_polls)
        ]
        proc_stat_memory_usage_values = [
            random.randint(0, 8 * 1024**3) for _ in range(num_polls)
        ]

        self._track_new_extension_cgroups(num_extensions)
        self.assertEqual(2 * num_extensions, len(CGroupsTelemetry._tracked))

        for i in range(num_polls):
            patch_is_active.return_value = True
            patch_get_cpu_usage.return_value = cpu_percent_values[i]
            patch_get_memory_usage.return_value = memory_usage_values[
                i]  # example 200 MB
            patch_get_memory_max_usage.return_value = max_memory_usage_values[
                i]  # example 450 MB
            patch_get_statm.return_value = proc_stat_memory_usage_values[i]

            metrics = CGroupsTelemetry.poll_all_tracked()

            # 1 CPU metric + 1 Current Memory + 1 Max memory + num_processes (3) * memory from statm
            self.assertEqual(len(metrics), 6 * num_extensions)
            self._assert_polled_metrics_equal(metrics, cpu_percent_values[i],
                                              memory_usage_values[i],
                                              max_memory_usage_values[i],
                                              proc_stat_memory_usage_values[i])

        collected_metrics = CGroupsTelemetry.report_all_tracked()
        self._assert_extension_metrics_data(collected_metrics, num_extensions,
                                            cpu_percent_values,
                                            proc_stat_memory_usage_values,
                                            memory_usage_values,
                                            max_memory_usage_values)

    def test_cgroup_tracking(self, *args):
        num_extensions = 5
        num_controllers = 2
        self._track_new_extension_cgroups(num_extensions)
        self._assert_cgroups_are_tracked(num_extensions)
        self.assertEqual(num_extensions * num_controllers,
                         len(CGroupsTelemetry._tracked))

    def test_cgroup_pruning(self, *args):
        num_extensions = 5
        num_controllers = 2
        self._track_new_extension_cgroups(num_extensions)
        self._assert_cgroups_are_tracked(num_extensions)
        self.assertEqual(num_extensions * num_controllers,
                         len(CGroupsTelemetry._tracked))

        CGroupsTelemetry.prune_all_tracked()
        for i in range(num_extensions):
            self.assertFalse(
                CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
            self.assertFalse(
                CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))

        self.assertEqual(0, len(CGroupsTelemetry._tracked))

    def test_cgroup_is_tracked(self, *args):
        num_extensions = 5
        self._track_new_extension_cgroups(num_extensions)
        self._assert_cgroups_are_tracked(num_extensions)
        self.assertFalse(
            CGroupsTelemetry.is_tracked("not_present_cpu_dummy_path"))
        self.assertFalse(
            CGroupsTelemetry.is_tracked("not_present_memory_dummy_path"))

    @patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
           side_effect=raise_ioerror)
    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
           side_effect=raise_ioerror)
    def test_process_cgroup_metric_with_incorrect_cgroups_mounted(self, *args):
        num_extensions = 5
        self._track_new_extension_cgroups(num_extensions)

        for data_count in range(1, 10):
            metrics = CGroupsTelemetry.poll_all_tracked()
            self.assertEqual(len(metrics), 0)

        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                         num_extensions)

        collected_metrics = {}
        for name, cgroup_metrics in CGroupsTelemetry._cgroup_metrics.items():
            collected_metrics[name] = CGroupsTelemetry._process_cgroup_metric(
                cgroup_metrics)
            self.assertEqual(collected_metrics[name], {})  # empty

    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
           side_effect=raise_ioerror)
    def test_process_cgroup_metric_with_no_memory_cgroup_mounted(self, *args):
        num_extensions = 5
        self._track_new_extension_cgroups(num_extensions)

        with patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage"
                   ) as patch_get_cpu_usage:
            with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
                       ) as patch_is_active:
                patch_is_active.return_value = True

                current_cpu = 30
                patch_get_cpu_usage.return_value = current_cpu

                poll_count = 1

                for data_count in range(poll_count, 10):
                    metrics = CGroupsTelemetry.poll_all_tracked()

                    self.assertEqual(
                        CGroupsTelemetry._cgroup_metrics.__len__(),
                        num_extensions)
                    self._assert_calculated_resource_metrics_equal(
                        cpu_usage=[current_cpu] * data_count,
                        memory_usage=[],
                        max_memory_usage=[],
                        proc_ids=[],
                        memory_statm_memory_usage=[])
                    self.assertEqual(len(metrics),
                                     num_extensions * 1)  # Only CPU populated
                    self._assert_polled_metrics_equal(metrics, current_cpu, 0,
                                                      0, 0)

                CGroupsTelemetry.report_all_tracked()

                self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
                                 num_extensions)
                self._assert_calculated_resource_metrics_equal([], [], [], [],
                                                               [])

    @patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
           side_effect=raise_ioerror)
    def test_process_cgroup_metric_with_no_cpu_cgroup_mounted(self, *args):
        num_extensions = 5

        self._track_new_extension_cgroups(num_extensions)

        with patch(
                "azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage"
        ) as patch_get_memory_max_usage:
            with patch(
                    "azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage"
            ) as patch_get_memory_usage:
                with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
                           ) as patch_is_active:
                    patch_is_active.return_value = True

                    current_memory = 209715200
                    current_max_memory = 471859200

                    patch_get_memory_usage.return_value = current_memory  # example 200 MB
                    patch_get_memory_max_usage.return_value = current_max_memory  # example 450 MB
                    num_polls = 10
                    for data_count in range(1, num_polls + 1):
                        metrics = CGroupsTelemetry.poll_all_tracked()
                        self.assertEqual(len(CGroupsTelemetry._cgroup_metrics),
                                         num_extensions)
                        self._assert_calculated_resource_metrics_equal(
                            cpu_usage=[],
                            memory_usage=[current_memory] * data_count,
                            max_memory_usage=[current_max_memory] * data_count,
                            memory_statm_memory_usage=[
                                TestCGroupsTelemetry.TestProcStatmMemoryUsed
                            ] * data_count,
                            proc_ids=TestCGroupsTelemetry.TestProcessIds)
                        # Memory is only populated, CPU is not. Thus 5 metrics per cgroup.
                        self.assertEqual(len(metrics), num_extensions * 5)
                        self._assert_polled_metrics_equal(
                            metrics, 0, current_memory, current_max_memory,
                            TestCGroupsTelemetry.TestProcStatmMemoryUsed)

                    collected_metrics = CGroupsTelemetry.report_all_tracked()
                    self._assert_extension_metrics_data(
                        collected_metrics,
                        num_extensions, [],
                        [TestCGroupsTelemetry.TestProcStatmMemoryUsed] *
                        num_polls, [current_memory] * num_polls,
                        [current_max_memory] * num_polls,
                        is_cpu_present=False)

                    self.assertEqual(len(CGroupsTelemetry._cgroup_metrics),
                                     num_extensions)
                    self._assert_calculated_resource_metrics_equal([], [], [],
                                                                   [], [])

    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
           side_effect=raise_ioerror)
    @patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage",
           side_effect=raise_ioerror)
    @patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
           side_effect=raise_ioerror)
    def test_extension_telemetry_not_sent_for_empty_perf_metrics(self, *args):
        num_extensions = 5
        self._track_new_extension_cgroups(num_extensions)

        with patch("azurelinuxagent.common.cgroupstelemetry.CGroupsTelemetry._process_cgroup_metric") as \
                patch_process_cgroup_metric:
            with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
                       ) as patch_is_active:

                patch_is_active.return_value = False
                patch_process_cgroup_metric.return_value = {}
                poll_count = 1

                for data_count in range(poll_count, 10):
                    metrics = CGroupsTelemetry.poll_all_tracked()
                    self.assertEqual(0, len(metrics))

                collected_metrics = CGroupsTelemetry.report_all_tracked()
                self.assertEqual(0, len(collected_metrics))

    @skip_if_predicate_true(
        lambda: True,
        "Skipping this test currently: We need two different tests - one for "
        "FileSystemCgroupAPI based test and one for SystemDCgroupAPI based test. @vrdmr will "
        "be splitting this test in subsequent PRs")
    @skip_if_predicate_false(are_cgroups_enabled,
                             "Does not run when Cgroups are not enabled")
    @skip_if_predicate_true(is_trusty_in_travis,
                            "Does not run on Trusty in Travis")
    @attr('requires_sudo')
    @patch("azurelinuxagent.common.cgroupconfigurator.get_osutil",
           return_value=DefaultOSUtil())
    @patch("azurelinuxagent.common.cgroupapi.CGroupsApi._is_systemd",
           return_value=False)
    def test_telemetry_with_tracked_cgroup(self, *_):
        self.assertTrue(i_am_root(), "Test does not run when non-root")
        CGroupConfigurator._instance = None

        max_num_polls = 30
        time_to_wait = 3
        extn_name = "foobar-1.0.0"
        num_summarization_values = 7

        cgs = make_new_cgroup(extn_name)
        self.assertEqual(len(cgs), 2)

        ext_handler_properties = ExtHandlerProperties()
        ext_handler_properties.version = "1.0.0"
        self.ext_handler = ExtHandler(name='foobar')
        self.ext_handler.properties = ext_handler_properties
        self.ext_handler_instance = ExtHandlerInstance(
            ext_handler=self.ext_handler, protocol=None)

        command = self.create_script(
            "keep_cpu_busy_and_consume_memory_for_5_seconds", '''
nohup python -c "import time

for i in range(5):
    x = [1, 2, 3, 4, 5] * (i * 1000)
    time.sleep({0})
    x *= 0
    print('Test loop')" &
'''.format(time_to_wait))

        self.log_dir = os.path.join(self.tmp_dir, "log")

        with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_base_dir", lambda *_: self.tmp_dir) as \
                patch_get_base_dir:
            with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_log_dir", lambda *_: self.log_dir) as \
                    patch_get_log_dir:
                self.ext_handler_instance.launch_command(command)

        self.assertTrue(
            CGroupsTelemetry.is_tracked(
                os.path.join(BASE_CGROUPS, "cpu", "walinuxagent.extensions",
                             "foobar_1.0.0")))
        self.assertTrue(
            CGroupsTelemetry.is_tracked(
                os.path.join(BASE_CGROUPS, "memory", "walinuxagent.extensions",
                             "foobar_1.0.0")))

        for i in range(max_num_polls):
            CGroupsTelemetry.poll_all_tracked()
            time.sleep(0.5)

        collected_metrics = CGroupsTelemetry.report_all_tracked()

        self.assertIn("memory", collected_metrics[extn_name])
        self.assertIn("cur_mem", collected_metrics[extn_name]["memory"])
        self.assertIn("max_mem", collected_metrics[extn_name]["memory"])
        self.assertEqual(
            len(collected_metrics[extn_name]["memory"]["cur_mem"]),
            num_summarization_values)
        self.assertEqual(
            len(collected_metrics[extn_name]["memory"]["max_mem"]),
            num_summarization_values)

        self.assertIsInstance(
            collected_metrics[extn_name]["memory"]["cur_mem"][5], str)
        self.assertIsInstance(
            collected_metrics[extn_name]["memory"]["cur_mem"][6], str)
        self.assertIsInstance(
            collected_metrics[extn_name]["memory"]["max_mem"][5], str)
        self.assertIsInstance(
            collected_metrics[extn_name]["memory"]["max_mem"][6], str)

        self.assertIn("cpu", collected_metrics[extn_name])
        self.assertIn("cur_cpu", collected_metrics[extn_name]["cpu"])
        self.assertEqual(len(collected_metrics[extn_name]["cpu"]["cur_cpu"]),
                         num_summarization_values)

        self.assertIsInstance(
            collected_metrics[extn_name]["cpu"]["cur_cpu"][5], str)
        self.assertIsInstance(
            collected_metrics[extn_name]["cpu"]["cur_cpu"][6], str)

        for i in range(5):
            self.assertGreater(
                collected_metrics[extn_name]["memory"]["cur_mem"][i], 0)
            self.assertGreater(
                collected_metrics[extn_name]["memory"]["max_mem"][i], 0)
            self.assertGreaterEqual(
                collected_metrics[extn_name]["cpu"]["cur_cpu"][i], 0)
Exemple #23
0
class PexOSUtil(object):
    def __init__(self):
        self._default = DefaultOSUtil()
        self.jit_enabled = False # ga/remoteaccess.py


    def get_agent_conf_file_path(self):
        # agent.py
        return self._default.get_agent_conf_file_path()

    def start_agent_service(self):
        # agent.py
        pass

    def stop_agent_service(self):
        # agent.py
        pass

    def register_agent_service(self):
        # agent.py
        pass


    def is_cgroups_supported(self):
        # common/cgroups.py
        return False


    def is_dhcp_available(self):
        # common/dhcp.py common/protocol/util.py
        return (False, '')


    def check_pid_alive(self, pid):
        # daemon/main.py ga/env.py ga/update.py
        return self._default.check_pid_alive(pid)


    def get_instance_id(self):
        # pa/provision/default.py
        return ''


    def get_dhcp_pid(self):
        # ga/env.py
        return None

    def get_hostname_record(self):
        # ga/env.py
        return None

    def remove_rules_files(self):
        # ga/env.py
        pass


    def get_firewall_dropped_packets(self, dst_ip=None):
        # ga/monitor.py
        return 0

    def read_route_table(self):
        # ga/monitor.py
        return []

    def get_list_of_routes(self, route_table):
        # ga/monitor.py
        return []

    def get_nic_state(self):
        # ga/monitor.py
        return {}

    def get_total_mem(self):
        # ga/monitor.py
        return self._default.get_total_mem()

    def get_processor_cores(self):
        # ga/monitor.py cgroups.py
        return self._default.get_processor_cores()


    # For tests only: runtime configuration ensures these aren't used
    def device_for_ide_port(self, port_id):
        return None

    def try_load_atapiix_mod(self):
        return

    def mount_dvd(self, **kwargs):
        return self._default.mount_dvd(**kwargs)

    def umount_dvd(self, **kwargs):
        return self._default.umount_dvd(**kwargs)

    def set_hostname(self, hostname):
        return

    def set_dhcp_hostname(self, hostname):
        return

    def del_root_password(self):
        return
Exemple #24
0
 def set_hostname(self, hostname):
     """
     Set /etc/hostname
     Unlike redhat 6.x, redhat 7.x will set hostname to /etc/hostname
     """
     DefaultOSUtil.set_hostname(self, hostname)