def set_hostname(self, hostname):
"""
Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
Due to a bug in systemd in Centos-7.0, if this call fails, fallback
to hostname.
"""
hostnamectl_cmd = "hostnamectl set-hostname {0} --static".format(hostname)
if shellutil.run(hostnamectl_cmd, chk_err=False) != 0:
logger.warn("[{0}] failed, attempting fallback".format(hostnamectl_cmd))
DefaultOSUtil.set_hostname(self, hostname)
def set_hostname(self, hostname):
"""
Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
Due to a bug in systemd in Centos-7.0, if this call fails, fallback
to hostname.
"""
hostnamectl_cmd = ['hostnamectl', 'set-hostname', hostname, '--static']
if self._run_command_without_raising(hostnamectl_cmd,
log_error=False) != 0:
logger.warn("[{0}] failed, attempting fallback".format(
' '.join(hostnamectl_cmd)))
DefaultOSUtil.set_hostname(self, hostname)
def set_hostname(self, hostname):
"""
Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
Due to a bug in systemd in Centos-7.0, if this call fails, fallback
to hostname.
"""
hostnamectl_cmd = ["hostnamectl", "set-hostname", hostname, "--static"]
try:
shellutil.run_command(hostnamectl_cmd)
except Exception as e:
logger.warn("[{0}] failed with error: {1}, attempting fallback".format(' '.join(hostnamectl_cmd), ustr(e)))
DefaultOSUtil.set_hostname(self, hostname)
def setUp(self):
AgentTestCase.setUp(self)
# save the original run_command so that mocks can reference it
self.shellutil_run_command = shellutil.run_command
# save an instance of the original DefaultOSUtil so that mocks can reference it
self.default_osutil = DefaultOSUtil()
# AgentTestCase.setUp mocks osutil.factory._get_osutil; we override that mock for this class with a new mock
# that always returns the default implementation.
self.mock_get_osutil = patch(
"azurelinuxagent.common.osutil.factory._get_osutil",
return_value=DefaultOSUtil())
self.mock_get_osutil.start()
def test_mount_cgroups_should_handle_errors_when_mounting_an_individual_controller(
self):
original_run_get_output = shellutil.run_get_output
def mock_run_get_output(cmd, *args, **kwargs):
if cmd.startswith('mount '):
if 'memory' in cmd:
raise Exception(
'A test exception mounting the memory controller')
return 0, None
return original_run_get_output(cmd, *args, **kwargs)
with patch(
"azurelinuxagent.common.osutil.default.shellutil.run_get_output",
side_effect=mock_run_get_output) as patch_run_get_output:
with patch("azurelinuxagent.common.cgroupconfigurator.logger.warn"
) as mock_logger_warn:
DefaultOSUtil().mount_cgroups()
# the cgroup filesystem and the cpu controller should still have been mounted
mount_commands = DefaultOsUtilTestCase._get_mount_commands(
patch_run_get_output)
self.assertRegex(mount_commands, ';mount.* cgroup_root ',
'The cgroups file system was not mounted')
self.assertRegex(mount_commands, ';mount.* cpu,cpuacct ',
'The cpu controller was not mounted')
# A warning should have been logged for the memory controller
args, kwargs = mock_logger_warn.call_args
self.assertIn(
'A test exception mounting the memory controller', args)
def setUpClass(cls):
AgentTestCase.setUpClass()
# Use the file system implementation of CGroupsApi (FileSystemCgroupsApi)
cls.mock_is_systemd = patch(
"azurelinuxagent.common.cgroupapi.CGroupsApi._is_systemd",
return_value=False)
cls.mock_is_systemd.start()
# Use the default implementation of osutil
cls.mock_get_osutil = patch(
"azurelinuxagent.common.cgroupconfigurator.get_osutil",
return_value=DefaultOSUtil())
cls.mock_get_osutil.start()
# Currently osutil.is_cgroups_supported() returns False on Travis runs. We need to revisit this design; in the
# meanwhile mock the method to return True
cls.mock_is_cgroups_supported = patch(
"azurelinuxagent.common.osutil.default.DefaultOSUtil.is_cgroups_supported",
return_value=True)
cls.mock_is_cgroups_supported.start()
# Mounting the cgroup filesystem requires root privileges. Since these tests do not perform any actual operation on cgroups, make it a noop.
cls.mock_mount_cgroups = patch(
"azurelinuxagent.common.osutil.default.DefaultOSUtil.mount_cgroups"
)
cls.mock_mount_cgroups.start()
def test_mount_cgroups_should_not_mount_the_cgroups_file_system_when_it_already_exists(
self):
os.mkdir(self.cgroups_file_system_root)
original_run_get_output = shellutil.run_get_output
def mock_run_get_output(cmd, *args, **kwargs):
if cmd.startswith('mount '):
return 0, None
return original_run_get_output(cmd, *args, **kwargs)
with patch(
"azurelinuxagent.common.osutil.default.shellutil.run_get_output",
side_effect=mock_run_get_output) as patch_run_get_output:
DefaultOSUtil().mount_cgroups()
mount_commands = DefaultOsUtilTestCase._get_mount_commands(
patch_run_get_output)
self.assertNotIn(
'cgroup_root', mount_commands,
'The cgroups file system should not have been mounted')
self.assertRegex(mount_commands, ';mount.* cpu,cpuacct ',
'The cpu controller was not mounted')
self.assertRegex(mount_commands, ';mount.* memory ',
'The memory controller was not mounted')
def test_mount_cgroups_should_mount_the_cpu_and_memory_controllers(self):
# the mount command requires root privileges; make it a no op and check only for file existence
original_run_get_output = shellutil.run_get_output
def mock_run_get_output(cmd, *args, **kwargs):
if cmd.startswith('mount '):
return 0, None
return original_run_get_output(cmd, *args, **kwargs)
with patch(
"azurelinuxagent.common.osutil.default.shellutil.run_get_output",
side_effect=mock_run_get_output) as patch_run_get_output:
DefaultOSUtil().mount_cgroups()
# the directories for the controllers should have been created
for controller in ['cpu', 'memory', 'cpuacct', 'cpu,cpuacct']:
directory = os.path.join(self.cgroups_file_system_root,
controller)
self.assertTrue(
os.path.exists(directory),
"A directory for controller {0} was not created".format(
controller))
# the cgroup filesystem and the cpu and memory controllers should have been mounted
mount_commands = DefaultOsUtilTestCase._get_mount_commands(
patch_run_get_output)
self.assertRegex(mount_commands, ';mount.* cgroup_root ',
'The cgroups file system was not mounted')
self.assertRegex(mount_commands, ';mount.* cpu,cpuacct ',
'The cpu controller was not mounted')
self.assertRegex(mount_commands, ';mount.* memory ',
'The memory controller was not mounted')
def test_mount_cgroups_should_raise_when_the_cgroups_filesystem_fails_to_mount(
self):
original_run_get_output = shellutil.run_get_output
def mock_run_get_output(cmd, *args, **kwargs):
if cmd.startswith('mount '):
if 'cgroup_root' in cmd:
raise Exception(
'A test exception mounting the cgroups file system')
return 0, None
return original_run_get_output(cmd, *args, **kwargs)
with patch(
"azurelinuxagent.common.osutil.default.shellutil.run_get_output",
side_effect=mock_run_get_output) as patch_run_get_output:
with self.assertRaises(Exception) as context_manager:
DefaultOSUtil().mount_cgroups()
self.assertRegex(
str(context_manager.exception),
'A test exception mounting the cgroups file system')
mount_commands = DefaultOsUtilTestCase._get_mount_commands(
patch_run_get_output)
self.assertNotIn(
'memory', mount_commands,
'The memory controller should not have been mounted')
self.assertNotIn(
'cpu', mount_commands,
'The cpu controller should not have been mounted')
def _get_persist_firewall_rules_handler(self, systemd=True):
osutil = DefaultOSUtil()
osutil.get_firewall_will_wait = MagicMock(
return_value=self.__test_wait)
osutil.get_agent_bin_path = MagicMock(
return_value=self.__agent_bin_dir)
osutil.get_systemd_unit_file_install_path = MagicMock(
return_value=self.__systemd_dir)
self._expected_service_name = PersistFirewallRulesHandler._AGENT_NETWORK_SETUP_NAME_FORMAT.format(
osutil.get_service_name())
self._network_service_unit_file = os.path.join(
self.__systemd_dir, self._expected_service_name)
self._binary_file = os.path.join(
conf.get_lib_dir(), PersistFirewallRulesHandler.BINARY_FILE_NAME)
# Just for these tests, ignoring the mode of mkdir to allow non-sudo tests
orig_mkdir = fileutil.mkdir
with patch(
"azurelinuxagent.common.persist_firewall_rules.fileutil.mkdir",
side_effect=lambda path, **mode: orig_mkdir(path)):
with patch(
"azurelinuxagent.common.persist_firewall_rules.get_osutil",
return_value=osutil):
with patch('azurelinuxagent.common.osutil.systemd.is_systemd',
return_value=systemd):
with patch(
"azurelinuxagent.common.utils.shellutil.subprocess.Popen",
side_effect=self.__mock_popen):
yield PersistFirewallRulesHandler(
self.__test_dst_ip, self.__test_uid)
def test_mount_cgroups_should_not_create_symbolic_links_when_the_cpu_controller_fails_to_mount(self):
original_run_get_output = shellutil.run_get_output
def mock_run_get_output(cmd, *args, **kwargs):
if cmd.startswith('mount '):
if 'cpu,cpuacct' in cmd:
raise Exception('A test exception mounting the cpu controller')
return 0, None
return original_run_get_output(cmd, *args, **kwargs)
with patch("azurelinuxagent.common.osutil.default.shellutil.run_get_output", side_effect=mock_run_get_output):
with patch("azurelinuxagent.common.osutil.default.os.symlink") as patch_symlink:
DefaultOSUtil().mount_cgroups()
self.assertEquals(patch_symlink.call_count, 0, 'A symbolic link should not have been created')
def test_mount_cgroups_should_raise_when_all_controllers_fail_to_mount(self):
original_run_get_output = shellutil.run_get_output
def mock_run_get_output(cmd, *args, **kwargs):
if cmd.startswith('mount '):
if 'memory' in cmd or 'cpu,cpuacct' in cmd:
raise Exception('A test exception mounting a cgroup controller')
return 0, None
return original_run_get_output(cmd, *args, **kwargs)
with patch("azurelinuxagent.common.osutil.default.shellutil.run_get_output", side_effect=mock_run_get_output):
with self.assertRaises(Exception) as context_manager:
DefaultOSUtil().mount_cgroups()
self.assertRegex(str(context_manager.exception), 'A test exception mounting a cgroup controller')
def test_provisioning_is_skipped_when_not_enabled(self, mock_conf): # pylint: disable=unused-argument
ph = ProvisionHandler() # pylint: disable=invalid-name
ph.osutil = DefaultOSUtil()
ph.osutil.get_instance_id = Mock(
return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502')
ph.check_provisioned_file = Mock()
ph.report_ready = Mock()
ph.write_provisioned = Mock()
ph.run()
self.assertEqual(0, ph.check_provisioned_file.call_count)
self.assertEqual(1, ph.report_ready.call_count)
self.assertEqual(1, ph.write_provisioned.call_count)
def test_provisioning_is_skipped_when_not_enabled(self, mock_conf):
ph = ProvisionHandler()
ph.osutil = DefaultOSUtil()
ph.osutil.get_instance_id = Mock(
return_value='B9F3C233-9913-9F42-8EB3-BA656DF32502')
ph.is_provisioned = Mock()
ph.report_ready = Mock()
ph.write_provisioned = Mock()
ph.run()
ph.is_provisioned.assert_not_called()
ph.report_ready.assert_called_once()
ph.write_provisioned.assert_called_once()
def get_list_of_routes(route_table):
"""
Construct a list of all network routes known to this system.
:param list(str) route_table: List of text entries from route table, including headers
:return: a list of network routes
:rtype: list(RouteEntry)
"""
route_list = []
count = len(route_table)
if count < 1:
logger.error("netstat -rn -f inet is missing headers")
elif count == 1:
logger.error("netstat -rn -f inet contains no routes")
else:
route_list = DefaultOSUtil._build_route_list(route_table)
return route_list
def set_hostname(self, hostname):
"""
Set /etc/hostname
Unlike redhat 6.x, redhat 7.x will set hostname to /etc/hostname
"""
DefaultOSUtil.set_hostname(self, hostname)
def test_customdata(self):
base64data = 'Q3VzdG9tRGF0YQ=='
data = DefaultOSUtil().decode_customdata(base64data)
fileutil.write_file(tempfile.mktemp(), data)
def test_default_service_name(self):
self.assertEquals(DefaultOSUtil().get_service_name(), "waagent")
def openssl_to_openssh(self, input_file, output_file):
DefaultOSUtil.openssl_to_openssh(self, input_file, output_file)
def openssl_to_openssh(self, input_file, output_file):
DefaultOSUtil.openssl_to_openssh(self, input_file, output_file)
def __init__(self):
self._default = DefaultOSUtil()
self.jit_enabled = False # ga/remoteaccess.py
class TestCGroupsTelemetry(AgentTestCase):
TestProcessIds = ["1000", "1001", "1002"]
TestProcStatmMemoryUsed = 1234
TestProcComm = "python"
TestProcCommandLine = "python -u bin/WALinuxAgent-2.2.45-py2.7.egg -run-exthandlers"
NumSummarizationValues = 7
@classmethod
def setUpClass(cls):
AgentTestCase.setUpClass()
# Use the default value for memory used from proc_statm
cls.mock_get_memory_usage_from_proc_statm = patch(
"azurelinuxagent.common.resourceusage.MemoryResourceUsage."
"get_memory_usage_from_proc_statm",
return_value=TestCGroupsTelemetry.TestProcStatmMemoryUsed)
cls.mock_get_memory_usage_from_proc_statm.start()
# Use the default value for memory used from proc_statm
cls.mock_get_tracked_processes = patch(
"azurelinuxagent.common.cgroup.CGroup.get_tracked_processes",
return_value=TestCGroupsTelemetry.TestProcessIds)
cls.mock_get_tracked_processes.start()
cls.mock_get_proc_name = patch(
"azurelinuxagent.common.resourceusage.ProcessInfo.get_proc_name",
return_value=TestCGroupsTelemetry.TestProcComm)
cls.mock_get_proc_name.start()
cls.mock_get_proc_cmdline = patch(
"azurelinuxagent.common.resourceusage.ProcessInfo.get_proc_cmdline",
return_value=TestCGroupsTelemetry.TestProcCommandLine)
cls.mock_get_proc_cmdline.start()
# CPU Cgroups compute usage based on /proc/stat and /sys/fs/cgroup/.../cpuacct.stat; use mock data for those
# files
original_read_file = fileutil.read_file
def mock_read_file(filepath, **args):
if filepath == "/proc/stat":
filepath = os.path.join(data_dir, "cgroups", "proc_stat_t0")
elif filepath.endswith("/cpuacct.stat"):
filepath = os.path.join(data_dir, "cgroups", "cpuacct.stat_t0")
return original_read_file(filepath, **args)
cls._mock_read_cpu_cgroup_file = patch(
"azurelinuxagent.common.utils.fileutil.read_file",
side_effect=mock_read_file)
cls._mock_read_cpu_cgroup_file.start()
@classmethod
def tearDownClass(cls):
cls.mock_get_memory_usage_from_proc_statm.stop()
cls.mock_get_tracked_processes.stop()
cls.mock_get_proc_name.stop()
cls.mock_get_proc_cmdline.stop()
cls._mock_read_cpu_cgroup_file.stop()
AgentTestCase.tearDownClass()
def setUp(self):
AgentTestCase.setUp(self)
CGroupsTelemetry.reset()
def tearDown(self):
AgentTestCase.tearDown(self)
CGroupsTelemetry.reset()
@staticmethod
def _track_new_extension_cgroups(num_extensions):
for i in range(num_extensions):
dummy_cpu_cgroup = CGroup.create("dummy_cpu_path_{0}".format(i),
"cpu",
"dummy_extension_{0}".format(i))
CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
dummy_memory_cgroup = CGroup.create(
"dummy_memory_path_{0}".format(i), "memory",
"dummy_extension_{0}".format(i))
CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
def _assert_cgroups_are_tracked(self, num_extensions):
for i in range(num_extensions):
self.assertTrue(
CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
self.assertTrue(
CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))
def _assert_calculated_resource_metrics_equal(self,
cpu_usage,
memory_usage,
max_memory_usage,
memory_statm_memory_usage,
proc_ids=None):
if not proc_ids:
proc_ids = TestCGroupsTelemetry.TestProcessIds
processes_instances = [
CGroupsTelemetry.get_process_info_summary(pid) for pid in proc_ids
]
for _, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
self.assertListEqual(cgroup_metric.get_memory_metrics()._data,
memory_usage)
self.assertListEqual(cgroup_metric.get_max_memory_metrics()._data,
max_memory_usage)
self.assertListEqual(cgroup_metric.get_cpu_metrics()._data,
cpu_usage)
for kv_pair in cgroup_metric.get_proc_statm_memory_metrics():
self.assertIn(kv_pair.pid_name_cmdline, processes_instances)
self.assertListEqual(kv_pair.resource_metric._data,
memory_statm_memory_usage)
def _assert_polled_metrics_equal(self,
metrics,
cpu_metric_value,
memory_metric_value,
max_memory_metric_value,
proc_stat_memory_usage_value,
pids=None):
for metric in metrics:
self.assertIn(metric.category, ["Process", "Memory"])
if metric.category == "Process":
self.assertEqual(metric.counter, "% Processor Time")
self.assertEqual(metric.value, cpu_metric_value)
if metric.category == "Memory":
self.assertIn(metric.counter, [
"Total Memory Usage", "Max Memory Usage",
"Memory Used by Process"
])
if metric.counter == "Total Memory Usage":
self.assertEqual(metric.value, memory_metric_value)
elif metric.counter == "Max Memory Usage":
self.assertEqual(metric.value, max_memory_metric_value)
elif metric.counter == "Memory Used by Process":
if pids:
processes_instances = [
CGroupsTelemetry.get_process_info_summary(pid)
for pid in pids
]
else:
processes_instances = [
CGroupsTelemetry.get_process_info_summary(pid)
for pid in TestCGroupsTelemetry.TestProcessIds
]
self.assertIn(metric.instance, processes_instances)
self.assertEqual(metric.value,
proc_stat_memory_usage_value)
def _assert_extension_metrics_data(self,
collected_metrics,
num_extensions,
cpu_percent_values,
proc_stat_memory_usage_values,
memory_usage_values,
max_memory_usage_values,
is_cpu_present=True,
is_memory_present=True):
num_summarization_values = TestCGroupsTelemetry.NumSummarizationValues
if not (is_cpu_present or is_memory_present):
self.assertEquals(collected_metrics, {})
return
else:
for i in range(num_extensions):
name = "dummy_extension_{0}".format(i)
if is_memory_present:
self.assertIn(name, collected_metrics)
self.assertIn("memory", collected_metrics[name])
self.assertIn("cur_mem", collected_metrics[name]["memory"])
self.assertIn("max_mem", collected_metrics[name]["memory"])
self.assertEqual(
num_summarization_values,
len(collected_metrics[name]["memory"]["cur_mem"]))
self.assertEqual(
num_summarization_values,
len(collected_metrics[name]["memory"]["max_mem"]))
self.assertIn("proc_statm_memory", collected_metrics[name])
self.assertEqual(
3, len(collected_metrics[name]["proc_statm_memory"]
)) # number of processes added
for tracked_process in collected_metrics[name][
"proc_statm_memory"]:
self.assertEqual(
num_summarization_values,
len(collected_metrics[name]["proc_statm_memory"]
[tracked_process]))
self.assertListEqual(
generate_metric_list(
proc_stat_memory_usage_values),
collected_metrics[name]["proc_statm_memory"]
[tracked_process][0:5])
self.assertListEqual(
generate_metric_list(memory_usage_values),
collected_metrics[name]["memory"]["cur_mem"][0:5])
self.assertListEqual(
generate_metric_list(max_memory_usage_values),
collected_metrics[name]["memory"]["max_mem"][0:5])
if is_cpu_present:
self.assertIn("cpu", collected_metrics[name])
self.assertIn("cur_cpu", collected_metrics[name]["cpu"])
self.assertEqual(
num_summarization_values,
len(collected_metrics[name]["cpu"]["cur_cpu"]))
self.assertListEqual(
generate_metric_list(cpu_percent_values),
collected_metrics[name]["cpu"]["cur_cpu"][0:5])
def test_telemetry_polling_with_active_cgroups(self, *args):
num_extensions = 3
self._track_new_extension_cgroups(num_extensions)
with patch(
"azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage"
) as patch_get_memory_max_usage:
with patch(
"azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage"
) as patch_get_memory_usage:
with patch(
"azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage"
) as patch_get_cpu_usage:
with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
) as patch_is_active:
patch_is_active.return_value = True
current_cpu = 30
current_memory = 209715200
current_max_memory = 471859200
current_proc_statm = TestCGroupsTelemetry.TestProcStatmMemoryUsed
# 1 CPU metric + 1 Current Memory + 1 Max memor + num_processes * memory from statm
num_of_metrics_per_extn_expected = 1 + 1 + 1 + 3 * 1
patch_get_cpu_usage.return_value = current_cpu
patch_get_memory_usage.return_value = current_memory # example 200 MB
patch_get_memory_max_usage.return_value = current_max_memory # example 450 MB
num_polls = 10
for data_count in range(1, num_polls + 1):
metrics = CGroupsTelemetry.poll_all_tracked()
self.assertEqual(
len(CGroupsTelemetry._cgroup_metrics),
num_extensions)
self._assert_calculated_resource_metrics_equal(
cpu_usage=[current_cpu] * data_count,
memory_usage=[current_memory] * data_count,
max_memory_usage=[current_max_memory] *
data_count,
proc_ids=TestCGroupsTelemetry.TestProcessIds,
memory_statm_memory_usage=[current_proc_statm
] * data_count)
self.assertEqual(
len(metrics), num_extensions *
num_of_metrics_per_extn_expected)
self._assert_polled_metrics_equal(
metrics, current_cpu, current_memory,
current_max_memory, current_proc_statm)
collected_metrics = CGroupsTelemetry.report_all_tracked()
self._assert_extension_metrics_data(
collected_metrics,
num_extensions, [current_cpu] * num_polls,
[TestCGroupsTelemetry.TestProcStatmMemoryUsed] * num_polls,
[current_memory] * num_polls, [current_max_memory] * num_polls,
is_cpu_present=False)
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
num_extensions)
self._assert_calculated_resource_metrics_equal([], [], [], [], [])
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage",
side_effect=raise_ioerror)
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
side_effect=raise_ioerror)
@patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
side_effect=raise_ioerror)
@patch("azurelinuxagent.common.cgroup.CGroup.is_active",
return_value=False)
def test_telemetry_polling_with_inactive_cgroups(self, *_):
num_extensions = 5
no_extensions_expected = 0
self._track_new_extension_cgroups(num_extensions)
self._assert_cgroups_are_tracked(num_extensions)
metrics = CGroupsTelemetry.poll_all_tracked()
for i in range(num_extensions):
self.assertFalse(
CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
self.assertFalse(
CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
num_extensions)
self._assert_calculated_resource_metrics_equal([], [], [], [],
proc_ids=None)
self.assertEqual(len(metrics), 0)
collected_metrics = CGroupsTelemetry.report_all_tracked()
self._assert_extension_metrics_data(collected_metrics,
num_extensions, [], [], [], [],
is_cpu_present=False,
is_memory_present=False)
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
no_extensions_expected)
self._assert_calculated_resource_metrics_equal([], [], [], [], [])
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage")
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage")
@patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage")
@patch("azurelinuxagent.common.cgroup.CGroup.is_active")
@patch(
"azurelinuxagent.common.resourceusage.MemoryResourceUsage.get_memory_usage_from_proc_statm"
)
def test_telemetry_polling_with_changing_cgroups_state(
self, patch_get_statm, patch_is_active, patch_get_cpu_usage,
patch_get_mem, patch_get_max_mem, *args):
num_extensions = 5
self._track_new_extension_cgroups(num_extensions)
patch_is_active.return_value = True
no_extensions_expected = 0
expected_data_count = 1
current_cpu = 30
current_memory = 209715200
current_max_memory = 471859200
current_proc_statm = 20000000
patch_get_cpu_usage.return_value = current_cpu
patch_get_mem.return_value = current_memory # example 200 MB
patch_get_max_mem.return_value = current_max_memory # example 450 MB
patch_get_statm.return_value = current_proc_statm
self._assert_cgroups_are_tracked(num_extensions)
CGroupsTelemetry.poll_all_tracked()
self._assert_cgroups_are_tracked(num_extensions)
patch_is_active.return_value = False
patch_get_cpu_usage.side_effect = raise_ioerror
patch_get_mem.side_effect = raise_ioerror
patch_get_max_mem.side_effect = raise_ioerror
patch_get_statm.side_effect = raise_ioerror
CGroupsTelemetry.poll_all_tracked()
for i in range(num_extensions):
self.assertFalse(
CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
self.assertFalse(
CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
num_extensions)
self._assert_calculated_resource_metrics_equal(
cpu_usage=[current_cpu] * expected_data_count,
memory_usage=[current_memory] * expected_data_count,
max_memory_usage=[current_max_memory] * expected_data_count,
proc_ids=TestCGroupsTelemetry.TestProcessIds,
memory_statm_memory_usage=[current_proc_statm] *
expected_data_count)
CGroupsTelemetry.report_all_tracked()
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
no_extensions_expected)
self._assert_calculated_resource_metrics_equal([], [], [], [], [])
# mocking get_proc_stat to make it run on Mac and other systems. This test does not need to read the values of the
# /proc/stat file on the filesystem.
@patch("azurelinuxagent.common.logger.periodic_warn")
def test_telemetry_polling_to_not_generate_transient_logs_ioerror_file_not_found(
self, patch_periodic_warn):
num_extensions = 1
self._track_new_extension_cgroups(num_extensions)
self.assertEqual(0, patch_periodic_warn.call_count)
# Not expecting logs present for io_error with errno=errno.ENOENT
io_error_2 = IOError()
io_error_2.errno = errno.ENOENT
with patch("azurelinuxagent.common.utils.fileutil.read_file",
side_effect=io_error_2):
poll_count = 1
for data_count in range(poll_count, 10):
CGroupsTelemetry.poll_all_tracked()
self.assertEqual(0, patch_periodic_warn.call_count)
@patch("azurelinuxagent.common.logger.periodic_warn")
def test_telemetry_polling_to_generate_transient_logs_ioerror_permission_denied(
self, patch_periodic_warn):
num_extensions = 1
num_controllers = 2
is_active_check_per_controller = 2
self._track_new_extension_cgroups(num_extensions)
self.assertEqual(0, patch_periodic_warn.call_count)
# Expecting logs to be present for different kind of errors
io_error_3 = IOError()
io_error_3.errno = errno.EPERM
with patch("azurelinuxagent.common.utils.fileutil.read_file",
side_effect=io_error_3):
poll_count = 1
expected_count_per_call = num_controllers + is_active_check_per_controller
# each collect per controller would generate a log statement, and each cgroup would invoke a
# is active check raising an exception
for data_count in range(poll_count, 10):
CGroupsTelemetry.poll_all_tracked()
self.assertEqual(poll_count * expected_count_per_call,
patch_periodic_warn.call_count)
def test_telemetry_polling_to_generate_transient_logs_index_error(self):
num_extensions = 1
self._track_new_extension_cgroups(num_extensions)
# Generating a different kind of error (non-IOError) to check the logging.
# Trying to invoke IndexError during the getParameter call
with patch("azurelinuxagent.common.utils.fileutil.read_file",
return_value=''):
with patch("azurelinuxagent.common.logger.periodic_warn"
) as patch_periodic_warn:
expected_call_count = 2 # 1 periodic warning for the cpu cgroups, and 1 for memory
for data_count in range(1, 10):
CGroupsTelemetry.poll_all_tracked()
self.assertEqual(expected_call_count,
patch_periodic_warn.call_count)
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage")
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage")
@patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage")
@patch("azurelinuxagent.common.cgroup.CGroup.is_active")
@patch(
"azurelinuxagent.common.resourceusage.MemoryResourceUsage.get_memory_usage_from_proc_statm"
)
def test_telemetry_calculations(self, patch_get_statm, patch_is_active,
patch_get_cpu_usage,
patch_get_memory_usage,
patch_get_memory_max_usage, *args):
num_polls = 10
num_extensions = 1
cpu_percent_values = [random.randint(0, 100) for _ in range(num_polls)]
# only verifying calculations and not validity of the values.
memory_usage_values = [
random.randint(0, 8 * 1024**3) for _ in range(num_polls)
]
max_memory_usage_values = [
random.randint(0, 8 * 1024**3) for _ in range(num_polls)
]
proc_stat_memory_usage_values = [
random.randint(0, 8 * 1024**3) for _ in range(num_polls)
]
self._track_new_extension_cgroups(num_extensions)
self.assertEqual(2 * num_extensions, len(CGroupsTelemetry._tracked))
for i in range(num_polls):
patch_is_active.return_value = True
patch_get_cpu_usage.return_value = cpu_percent_values[i]
patch_get_memory_usage.return_value = memory_usage_values[
i] # example 200 MB
patch_get_memory_max_usage.return_value = max_memory_usage_values[
i] # example 450 MB
patch_get_statm.return_value = proc_stat_memory_usage_values[i]
metrics = CGroupsTelemetry.poll_all_tracked()
# 1 CPU metric + 1 Current Memory + 1 Max memory + num_processes (3) * memory from statm
self.assertEqual(len(metrics), 6 * num_extensions)
self._assert_polled_metrics_equal(metrics, cpu_percent_values[i],
memory_usage_values[i],
max_memory_usage_values[i],
proc_stat_memory_usage_values[i])
collected_metrics = CGroupsTelemetry.report_all_tracked()
self._assert_extension_metrics_data(collected_metrics, num_extensions,
cpu_percent_values,
proc_stat_memory_usage_values,
memory_usage_values,
max_memory_usage_values)
def test_cgroup_tracking(self, *args):
num_extensions = 5
num_controllers = 2
self._track_new_extension_cgroups(num_extensions)
self._assert_cgroups_are_tracked(num_extensions)
self.assertEqual(num_extensions * num_controllers,
len(CGroupsTelemetry._tracked))
def test_cgroup_pruning(self, *args):
num_extensions = 5
num_controllers = 2
self._track_new_extension_cgroups(num_extensions)
self._assert_cgroups_are_tracked(num_extensions)
self.assertEqual(num_extensions * num_controllers,
len(CGroupsTelemetry._tracked))
CGroupsTelemetry.prune_all_tracked()
for i in range(num_extensions):
self.assertFalse(
CGroupsTelemetry.is_tracked("dummy_cpu_path_{0}".format(i)))
self.assertFalse(
CGroupsTelemetry.is_tracked("dummy_memory_path_{0}".format(i)))
self.assertEqual(0, len(CGroupsTelemetry._tracked))
def test_cgroup_is_tracked(self, *args):
num_extensions = 5
self._track_new_extension_cgroups(num_extensions)
self._assert_cgroups_are_tracked(num_extensions)
self.assertFalse(
CGroupsTelemetry.is_tracked("not_present_cpu_dummy_path"))
self.assertFalse(
CGroupsTelemetry.is_tracked("not_present_memory_dummy_path"))
@patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
side_effect=raise_ioerror)
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
side_effect=raise_ioerror)
def test_process_cgroup_metric_with_incorrect_cgroups_mounted(self, *args):
num_extensions = 5
self._track_new_extension_cgroups(num_extensions)
for data_count in range(1, 10):
metrics = CGroupsTelemetry.poll_all_tracked()
self.assertEqual(len(metrics), 0)
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
num_extensions)
collected_metrics = {}
for name, cgroup_metrics in CGroupsTelemetry._cgroup_metrics.items():
collected_metrics[name] = CGroupsTelemetry._process_cgroup_metric(
cgroup_metrics)
self.assertEqual(collected_metrics[name], {}) # empty
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
side_effect=raise_ioerror)
def test_process_cgroup_metric_with_no_memory_cgroup_mounted(self, *args):
num_extensions = 5
self._track_new_extension_cgroups(num_extensions)
with patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage"
) as patch_get_cpu_usage:
with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
) as patch_is_active:
patch_is_active.return_value = True
current_cpu = 30
patch_get_cpu_usage.return_value = current_cpu
poll_count = 1
for data_count in range(poll_count, 10):
metrics = CGroupsTelemetry.poll_all_tracked()
self.assertEqual(
CGroupsTelemetry._cgroup_metrics.__len__(),
num_extensions)
self._assert_calculated_resource_metrics_equal(
cpu_usage=[current_cpu] * data_count,
memory_usage=[],
max_memory_usage=[],
proc_ids=[],
memory_statm_memory_usage=[])
self.assertEqual(len(metrics),
num_extensions * 1) # Only CPU populated
self._assert_polled_metrics_equal(metrics, current_cpu, 0,
0, 0)
CGroupsTelemetry.report_all_tracked()
self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(),
num_extensions)
self._assert_calculated_resource_metrics_equal([], [], [], [],
[])
@patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
side_effect=raise_ioerror)
def test_process_cgroup_metric_with_no_cpu_cgroup_mounted(self, *args):
num_extensions = 5
self._track_new_extension_cgroups(num_extensions)
with patch(
"azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage"
) as patch_get_memory_max_usage:
with patch(
"azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage"
) as patch_get_memory_usage:
with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
) as patch_is_active:
patch_is_active.return_value = True
current_memory = 209715200
current_max_memory = 471859200
patch_get_memory_usage.return_value = current_memory # example 200 MB
patch_get_memory_max_usage.return_value = current_max_memory # example 450 MB
num_polls = 10
for data_count in range(1, num_polls + 1):
metrics = CGroupsTelemetry.poll_all_tracked()
self.assertEqual(len(CGroupsTelemetry._cgroup_metrics),
num_extensions)
self._assert_calculated_resource_metrics_equal(
cpu_usage=[],
memory_usage=[current_memory] * data_count,
max_memory_usage=[current_max_memory] * data_count,
memory_statm_memory_usage=[
TestCGroupsTelemetry.TestProcStatmMemoryUsed
] * data_count,
proc_ids=TestCGroupsTelemetry.TestProcessIds)
# Memory is only populated, CPU is not. Thus 5 metrics per cgroup.
self.assertEqual(len(metrics), num_extensions * 5)
self._assert_polled_metrics_equal(
metrics, 0, current_memory, current_max_memory,
TestCGroupsTelemetry.TestProcStatmMemoryUsed)
collected_metrics = CGroupsTelemetry.report_all_tracked()
self._assert_extension_metrics_data(
collected_metrics,
num_extensions, [],
[TestCGroupsTelemetry.TestProcStatmMemoryUsed] *
num_polls, [current_memory] * num_polls,
[current_max_memory] * num_polls,
is_cpu_present=False)
self.assertEqual(len(CGroupsTelemetry._cgroup_metrics),
num_extensions)
self._assert_calculated_resource_metrics_equal([], [], [],
[], [])
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_memory_usage",
side_effect=raise_ioerror)
@patch("azurelinuxagent.common.cgroup.MemoryCgroup.get_max_memory_usage",
side_effect=raise_ioerror)
@patch("azurelinuxagent.common.cgroup.CpuCgroup.get_cpu_usage",
side_effect=raise_ioerror)
def test_extension_telemetry_not_sent_for_empty_perf_metrics(self, *args):
num_extensions = 5
self._track_new_extension_cgroups(num_extensions)
with patch("azurelinuxagent.common.cgroupstelemetry.CGroupsTelemetry._process_cgroup_metric") as \
patch_process_cgroup_metric:
with patch("azurelinuxagent.common.cgroup.CGroup.is_active"
) as patch_is_active:
patch_is_active.return_value = False
patch_process_cgroup_metric.return_value = {}
poll_count = 1
for data_count in range(poll_count, 10):
metrics = CGroupsTelemetry.poll_all_tracked()
self.assertEqual(0, len(metrics))
collected_metrics = CGroupsTelemetry.report_all_tracked()
self.assertEqual(0, len(collected_metrics))
@skip_if_predicate_true(
lambda: True,
"Skipping this test currently: We need two different tests - one for "
"FileSystemCgroupAPI based test and one for SystemDCgroupAPI based test. @vrdmr will "
"be splitting this test in subsequent PRs")
@skip_if_predicate_false(are_cgroups_enabled,
"Does not run when Cgroups are not enabled")
@skip_if_predicate_true(is_trusty_in_travis,
"Does not run on Trusty in Travis")
@attr('requires_sudo')
@patch("azurelinuxagent.common.cgroupconfigurator.get_osutil",
return_value=DefaultOSUtil())
@patch("azurelinuxagent.common.cgroupapi.CGroupsApi._is_systemd",
return_value=False)
def test_telemetry_with_tracked_cgroup(self, *_):
self.assertTrue(i_am_root(), "Test does not run when non-root")
CGroupConfigurator._instance = None
max_num_polls = 30
time_to_wait = 3
extn_name = "foobar-1.0.0"
num_summarization_values = 7
cgs = make_new_cgroup(extn_name)
self.assertEqual(len(cgs), 2)
ext_handler_properties = ExtHandlerProperties()
ext_handler_properties.version = "1.0.0"
self.ext_handler = ExtHandler(name='foobar')
self.ext_handler.properties = ext_handler_properties
self.ext_handler_instance = ExtHandlerInstance(
ext_handler=self.ext_handler, protocol=None)
command = self.create_script(
"keep_cpu_busy_and_consume_memory_for_5_seconds", '''
nohup python -c "import time
for i in range(5):
x = [1, 2, 3, 4, 5] * (i * 1000)
time.sleep({0})
x *= 0
print('Test loop')" &
'''.format(time_to_wait))
self.log_dir = os.path.join(self.tmp_dir, "log")
with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_base_dir", lambda *_: self.tmp_dir) as \
patch_get_base_dir:
with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_log_dir", lambda *_: self.log_dir) as \
patch_get_log_dir:
self.ext_handler_instance.launch_command(command)
self.assertTrue(
CGroupsTelemetry.is_tracked(
os.path.join(BASE_CGROUPS, "cpu", "walinuxagent.extensions",
"foobar_1.0.0")))
self.assertTrue(
CGroupsTelemetry.is_tracked(
os.path.join(BASE_CGROUPS, "memory", "walinuxagent.extensions",
"foobar_1.0.0")))
for i in range(max_num_polls):
CGroupsTelemetry.poll_all_tracked()
time.sleep(0.5)
collected_metrics = CGroupsTelemetry.report_all_tracked()
self.assertIn("memory", collected_metrics[extn_name])
self.assertIn("cur_mem", collected_metrics[extn_name]["memory"])
self.assertIn("max_mem", collected_metrics[extn_name]["memory"])
self.assertEqual(
len(collected_metrics[extn_name]["memory"]["cur_mem"]),
num_summarization_values)
self.assertEqual(
len(collected_metrics[extn_name]["memory"]["max_mem"]),
num_summarization_values)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["cur_mem"][5], str)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["cur_mem"][6], str)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["max_mem"][5], str)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["max_mem"][6], str)
self.assertIn("cpu", collected_metrics[extn_name])
self.assertIn("cur_cpu", collected_metrics[extn_name]["cpu"])
self.assertEqual(len(collected_metrics[extn_name]["cpu"]["cur_cpu"]),
num_summarization_values)
self.assertIsInstance(
collected_metrics[extn_name]["cpu"]["cur_cpu"][5], str)
self.assertIsInstance(
collected_metrics[extn_name]["cpu"]["cur_cpu"][6], str)
for i in range(5):
self.assertGreater(
collected_metrics[extn_name]["memory"]["cur_mem"][i], 0)
self.assertGreater(
collected_metrics[extn_name]["memory"]["max_mem"][i], 0)
self.assertGreaterEqual(
collected_metrics[extn_name]["cpu"]["cur_cpu"][i], 0)
class PexOSUtil(object):
def __init__(self):
self._default = DefaultOSUtil()
self.jit_enabled = False # ga/remoteaccess.py
def get_agent_conf_file_path(self):
# agent.py
return self._default.get_agent_conf_file_path()
def start_agent_service(self):
# agent.py
pass
def stop_agent_service(self):
# agent.py
pass
def register_agent_service(self):
# agent.py
pass
def is_cgroups_supported(self):
# common/cgroups.py
return False
def is_dhcp_available(self):
# common/dhcp.py common/protocol/util.py
return (False, '')
def check_pid_alive(self, pid):
# daemon/main.py ga/env.py ga/update.py
return self._default.check_pid_alive(pid)
def get_instance_id(self):
# pa/provision/default.py
return ''
def get_dhcp_pid(self):
# ga/env.py
return None
def get_hostname_record(self):
# ga/env.py
return None
def remove_rules_files(self):
# ga/env.py
pass
def get_firewall_dropped_packets(self, dst_ip=None):
# ga/monitor.py
return 0
def read_route_table(self):
# ga/monitor.py
return []
def get_list_of_routes(self, route_table):
# ga/monitor.py
return []
def get_nic_state(self):
# ga/monitor.py
return {}
def get_total_mem(self):
# ga/monitor.py
return self._default.get_total_mem()
def get_processor_cores(self):
# ga/monitor.py cgroups.py
return self._default.get_processor_cores()
# For tests only: runtime configuration ensures these aren't used
def device_for_ide_port(self, port_id):
return None
def try_load_atapiix_mod(self):
return
def mount_dvd(self, **kwargs):
return self._default.mount_dvd(**kwargs)
def umount_dvd(self, **kwargs):
return self._default.umount_dvd(**kwargs)
def set_hostname(self, hostname):
return
def set_dhcp_hostname(self, hostname):
return
def del_root_password(self):
return
def set_hostname(self, hostname):
"""
Set /etc/hostname
Unlike redhat 6.x, redhat 7.x will set hostname to /etc/hostname
"""
DefaultOSUtil.set_hostname(self, hostname)