def test_start_extension_command_should_use_fallback_option_if_systemd_fails(
self, _):
original_popen = subprocess.Popen
def mock_popen(*args, **kwargs):
# Inject a syntax error to the call
systemd_command = args[0].replace('systemd-run',
'systemd-run syntax_error')
new_args = (systemd_command, )
return original_popen(new_args, **kwargs)
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.add_event"
) as mock_add_event:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", side_effect=mock_popen) \
as patch_mock_popen:
# We expect this call to fail because of the syntax error
extension_cgroups, process_output = SystemdCgroupsApi(
).start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command="date",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
args, kwargs = mock_add_event.call_args
self.assertIn(
"Failed to run systemd-run for unit Microsoft.Compute.TestExtension_1.2.3",
kwargs['message'])
self.assertIn(
"Failed to find executable syntax_error: No such file or directory",
kwargs['message'])
self.assertEquals(False, kwargs['is_success'])
self.assertEquals('InvokeCommandUsingSystemd',
kwargs['op'])
# We expect two calls to Popen, first for the systemd-run call, second for the fallback option
self.assertEquals(2, patch_mock_popen.call_count)
first_call_args = patch_mock_popen.mock_calls[0][1][0]
second_call_args = patch_mock_popen.mock_calls[1][1][0]
self.assertIn("systemd-run --unit", first_call_args)
self.assertNotIn("systemd-run --unit",
second_call_args)
# No cgroups should have been created
self.assertEquals(extension_cgroups, [])
def test_cleanup_legacy_cgroups_should_remove_legacy_cgroups(self):
# Set up a mock /var/run/waagent.pid file
daemon_pid_file = os.path.join(self.tmp_dir, "waagent.pid")
fileutil.write_file(daemon_pid_file, "42\n")
# Set up old controller cgroups, but do not add the daemon's PID to them
legacy_cpu_cgroup = CGroupsTools.create_legacy_agent_cgroup(self.cgroups_file_system_root, "cpu", '')
legacy_memory_cgroup = CGroupsTools.create_legacy_agent_cgroup(self.cgroups_file_system_root, "memory", '')
with patch("azurelinuxagent.common.cgroupapi.get_agent_pid_file_path", return_value=daemon_pid_file):
legacy_cgroups = SystemdCgroupsApi().cleanup_legacy_cgroups()
self.assertEqual(legacy_cgroups, 2, "cleanup_legacy_cgroups() did not find all the expected cgroups")
self.assertFalse(os.path.exists(legacy_cpu_cgroup), "cleanup_legacy_cgroups() did not remove the CPU legacy cgroup")
self.assertFalse(os.path.exists(legacy_memory_cgroup), "cleanup_legacy_cgroups() did not remove the memory legacy cgroup")
def initialize(self):
try:
if self._initialized:
return
# check whether cgroup monitoring is supported on the current distro
self._cgroups_supported = CGroupsApi.cgroups_supported()
if not self._cgroups_supported:
logger.info("Cgroup monitoring is not supported on {0}", get_distro())
return
# check that systemd is detected correctly
self._cgroups_api = SystemdCgroupsApi()
if not systemd.is_systemd():
_log_cgroup_warning("systemd was not detected on {0}", get_distro())
return
_log_cgroup_info("systemd version: {0}", systemd.get_version())
# This is temporarily disabled while we analyze telemetry. Likely it will be removed.
# self.__collect_azure_unit_telemetry()
# self.__collect_agent_unit_files_telemetry()
if not self.__check_no_legacy_cgroups():
return
agent_unit_name = systemd.get_agent_unit_name()
agent_slice = systemd.get_unit_property(agent_unit_name, "Slice")
if agent_slice not in (_AZURE_SLICE, "system.slice"):
_log_cgroup_warning("The agent is within an unexpected slice: {0}", agent_slice)
return
self.__setup_azure_slice()
cpu_controller_root, memory_controller_root = self.__get_cgroup_controllers()
self._agent_cpu_cgroup_path, self._agent_memory_cgroup_path = self.__get_agent_cgroups(agent_slice, cpu_controller_root, memory_controller_root)
if self._agent_cpu_cgroup_path is not None:
_log_cgroup_info("Agent CPU cgroup: {0}", self._agent_cpu_cgroup_path)
self.enable()
CGroupsTelemetry.track_cgroup(CpuCgroup(AGENT_NAME_TELEMETRY, self._agent_cpu_cgroup_path))
_log_cgroup_info('Cgroups enabled: {0}', self._cgroups_enabled)
except Exception as exception:
_log_cgroup_warning("Error initializing cgroups: {0}", ustr(exception))
finally:
self._initialized = True
def test_start_extension_command_should_use_fallback_option_if_systemd_times_out(
self, _):
# Systemd has its own internal timeout which is shorter than what we define for extension operation timeout.
# When systemd times out, it will write a message to stderr and exit with exit code 1.
# In that case, we will internally recognize the failure due to the non-zero exit code, not as a timeout.
original_popen = subprocess.Popen
systemd_timeout_command = "echo 'Failed to start transient scope unit: Connection timed out' >&2 && exit 1"
def mock_popen(*args, **kwargs):
# If trying to invoke systemd, mock what would happen if systemd timed out internally:
# write failure to stderr and exit with exit code 1.
new_args = args
if "systemd-run" in args[0]:
new_args = (systemd_timeout_command, )
return original_popen(new_args, **kwargs)
expected_output = "[stdout]\n{0}\n\n\n[stderr]\n"
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", side_effect=mock_popen) \
as patch_mock_popen:
extension_cgroups, process_output = SystemdCgroupsApi(
).start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="echo 'success'",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
# We expect two calls to Popen, first for the systemd-run call, second for the fallback option
self.assertEquals(2, patch_mock_popen.call_count)
first_call_args = patch_mock_popen.mock_calls[0][1][0]
second_call_args = patch_mock_popen.mock_calls[1][1][0]
self.assertIn("systemd-run --unit", first_call_args)
self.assertNotIn("systemd-run --unit", second_call_args)
self.assertEquals(extension_cgroups, [])
self.assertEquals(expected_output.format("success"),
process_output)
def test_start_extension_command_should_execute_the_command_in_a_cgroup(self, _):
with mock_cgroup_environment(self.tmp_dir):
SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="test command",
cmd_name="test",
shell=False,
timeout=300,
cwd=self.tmp_dir,
env={},
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
tracked = CGroupsTelemetry._tracked
self.assertTrue(
any(cg for cg in tracked.values() if cg.name == 'Microsoft.Compute.TestExtension-1.2.3' and 'cpu' in cg.path),
"The extension's CPU is not being tracked")
def test_start_extension_command_should_use_systemd_to_execute_the_command(self, _):
with mock_cgroup_environment(self.tmp_dir):
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", wraps=subprocess.Popen) as popen_patch:
SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="the-test-extension-command",
cmd_name="test",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
extension_calls = [args[0] for (args, _) in popen_patch.call_args_list if "the-test-extension-command" in args[0]]
self.assertEqual(1, len(extension_calls), "The extension should have been invoked exactly once")
self.assertIn("systemd-run", extension_calls[0], "The extension should have been invoked using systemd")
def test_start_extension_command_should_use_systemd_if_not_failing(self):
original_popen = subprocess.Popen
def mock_popen(*args, **kwargs):
return original_popen(*args, **kwargs)
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", side_effect=mock_popen) as patch_mock_popen:
process, extension_cgroups = SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="date",
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
process.wait()
ret = process.poll()
# We should have invoked the extension command only once and succeeded
self.assertEquals(0, ret)
self.assertEquals(1, patch_mock_popen.call_count)
# Assert that the extension's cgroups were created as well
self.assertEqual(len(extension_cgroups), 2, 'start_extension_command did not return the expected number of cgroups')
cpu_found = memory_found = False
for cgroup in extension_cgroups:
match = re.match(r'^/sys/fs/cgroup/(cpu|memory)/system.slice/Microsoft.Compute.TestExtension_1\.2\.3\_([a-f0-9-]+)\.scope$', cgroup.path)
self.assertTrue(match is not None, "Unexpected path for cgroup: {0}".format(cgroup.path))
if match.group(1) == 'cpu':
cpu_found = True
if match.group(1) == 'memory':
memory_found = True
self.assertTrue(cpu_found, 'start_extension_command did not return a cpu cgroup')
self.assertTrue(memory_found, 'start_extension_command did not return a memory cgroup')
def test_start_extension_command_should_not_use_systemd_if_failing(self):
original_popen = subprocess.Popen
def mock_popen(*args, **kwargs):
# Inject a syntax error to the call
systemd_command = args[0].replace('systemd-run', 'systemd-run syntax_error')
new_args = (systemd_command,)
return original_popen(new_args, **kwargs)
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.logger.warn") as mock_logger_warn:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", side_effect=mock_popen) as patch_mock_popen:
# We expect this call to fail because of the syntax error
process, extension_cgroups = SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="date",
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
process.wait()
args, kwargs = mock_logger_warn.call_args
message = args[0]
self.assertIn("Failed to find executable syntax_error: No such file or directory", message)
self.assertIn("Failed to run systemd-run for unit Microsoft.Compute.TestExtension_1.2.3", message)
# We expect the process to ultimately succeed since the fallback option worked
ret = process.poll()
self.assertEquals(0, ret)
# We expect two calls to Popen, first for the systemd-run call, second for the fallback option
self.assertEquals(2, patch_mock_popen.call_count)
# No cgroups should have been created
self.assertEquals(extension_cgroups, [])
def test_start_extension_command_should_create_extension_scopes(self, _):
original_popen = subprocess.Popen
def mock_popen(*args, **kwargs):
return original_popen("date", **kwargs)
# we mock subprocess.Popen to execute a dummy command (date), so no actual cgroups are created; their paths
# should be computed properly, though
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen",
mock_popen):
extension_cgroups, process_output = SystemdCgroupsApi(
).start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="date",
shell=False,
timeout=300,
cwd=self.tmp_dir,
env={},
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.assert_cgroups_created(extension_cgroups)
def test_create_agent_cgroups_should_create_cgroups_on_all_controllers(
self, patch_read_file):
mock__base_cgroups = patch(
"azurelinuxagent.common.cgroupapi.CGROUPS_FILE_SYSTEM_ROOT",
self.cgroups_file_system_root)
mock__base_cgroups.start()
mock_proc_self_cgroup = '''12:blkio:/system.slice/walinuxagent.service
11:memory:/system.slice/walinuxagent.service
10:perf_event:/
9:hugetlb:/
8:freezer:/
7:net_cls,net_prio:/
6:devices:/system.slice/walinuxagent.service
5:cpuset:/
4:cpu,cpuacct:/system.slice/walinuxagent.service
3:pids:/system.slice/walinuxagent.service
2:rdma:/
1:name=systemd:/system.slice/walinuxagent.service
0::/system.slice/walinuxagent.service
'''
patch_read_file.return_value = mock_proc_self_cgroup
agent_cgroups = SystemdCgroupsApi().create_agent_cgroups()
def assert_cgroup_created(controller):
expected_cgroup_path = os.path.join(self.cgroups_file_system_root,
controller, "system.slice",
VM_AGENT_CGROUP_NAME)
self.assertTrue(
any(cgroups.path == expected_cgroup_path
for cgroups in agent_cgroups))
self.assertTrue(
any(cgroups.name == VM_AGENT_CGROUP_NAME
for cgroups in agent_cgroups))
assert_cgroup_created("cpu")
assert_cgroup_created("memory")
mock__base_cgroups.stop()
def test_start_extension_command_should_return_the_command_output(self, _):
original_popen = subprocess.Popen
def mock_popen(command, *args, **kwargs):
if command.startswith('systemd-run --unit'):
command = "echo TEST_OUTPUT"
return original_popen(command, *args, **kwargs)
with mock_cgroup_environment(self.tmp_dir):
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as output_file:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", side_effect=mock_popen) as popen_patch: # pylint: disable=unused-variable
command_output = SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="A_TEST_COMMAND",
cmd_name="test",
shell=True,
timeout=300,
cwd=self.tmp_dir,
env={},
stdout=output_file,
stderr=output_file)
self.assertIn("[stdout]\nTEST_OUTPUT\n", command_output, "The test output was not captured")
def test_start_extension_command_should_capture_only_the_last_subprocess_output(
self, _):
original_popen = subprocess.Popen
def mock_popen(*args, **kwargs):
# Inject a syntax error to the call
systemd_command = args[0].replace('systemd-run',
'systemd-run syntax_error')
new_args = (systemd_command, )
return original_popen(new_args, **kwargs)
expected_output = "[stdout]\n{0}\n\n\n[stderr]\n"
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.add_event"):
with patch(
"azurelinuxagent.common.cgroupapi.subprocess.Popen",
side_effect=mock_popen):
# We expect this call to fail because of the syntax error
extension_cgroups, process_output = SystemdCgroupsApi(
).start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command="echo 'very specific test message'",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
self.assertEquals(
expected_output.format(
"very specific test message"), process_output)
self.assertEquals(extension_cgroups, [])
def test_start_extension_command_should_create_extension_scopes(self):
original_popen = subprocess.Popen
def mock_popen(*args, **kwargs):
return original_popen("date", **kwargs)
# we mock subprocess.Popen to execute a dummy command (date), so no actual cgroups are created; their paths
# should be computed properly, though
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", mock_popen):
process, extension_cgroups = SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="date",
shell=False,
cwd=self.tmp_dir,
env={},
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process.wait()
self.assertEqual(len(extension_cgroups), 2, 'start_extension_command did not return the expected number of cgroups')
cpu_found = memory_found = False
for cgroup in extension_cgroups:
match = re.match(r'^/sys/fs/cgroup/(cpu|memory)/system.slice/Microsoft.Compute.TestExtension_1\.2\.3\_([a-f0-9-]+)\.scope$', cgroup.path)
self.assertTrue(match is not None, "Unexpected path for cgroup: {0}".format(cgroup.path))
if match.group(1) == 'cpu':
cpu_found = True
if match.group(1) == 'memory':
memory_found = True
self.assertTrue(cpu_found, 'start_extension_command did not return a cpu cgroup')
self.assertTrue(memory_found, 'start_extension_command did not return a memory cgroup')
def test_get_cpu_and_memory_mount_points_should_return_the_cgroup_mount_points(self):
with mock_cgroup_environment(self.tmp_dir):
cpu, memory = SystemdCgroupsApi().get_cgroup_mount_points()
self.assertEqual(cpu, '/sys/fs/cgroup/cpu,cpuacct', "The mount point for the CPU controller is incorrect")
self.assertEqual(memory, '/sys/fs/cgroup/memory', "The mount point for the memory controller is incorrect")
def test_get_extensions_slice_root_name_should_return_the_root_slice_for_extensions(
self):
root_slice_name = SystemdCgroupsApi()._get_extensions_slice_root_name()
self.assertEqual(root_slice_name,
"system-walinuxagent.extensions.slice")
def test_it_should_return_extension_slice_name(self):
extension_name = "Microsoft.Azure.DummyExtension-1.0"
extension_slice_name = SystemdCgroupsApi()._get_extension_slice_name(extension_name)
self.assertEqual(extension_slice_name, "system-walinuxagent.extensions-Microsoft.Azure.DummyExtension_1.0.slice")
def test_start_extension_command_should_invoke_the_command_directly_if_systemd_fails(
self, _):
original_popen = subprocess.Popen
def mock_popen(command, *args, **kwargs):
if command.startswith('systemd-run'):
# Inject a syntax error to the call
command = command.replace('systemd-run',
'systemd-run syntax_error')
return original_popen(command, *args, **kwargs)
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as output_file:
with patch("azurelinuxagent.common.cgroupapi.add_event"
) as mock_add_event:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen",
side_effect=mock_popen) as popen_patch:
CGroupsTelemetry.reset()
command = "echo TEST_OUTPUT"
command_output = SystemdCgroupsApi(
).start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command=command,
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=output_file,
stderr=output_file)
args, kwargs = mock_add_event.call_args
self.assertIn(
"Failed to run systemd-run for unit Microsoft.Compute.TestExtension_1.2.3",
kwargs['message'])
self.assertIn(
"Failed to find executable syntax_error: No such file or directory",
kwargs['message'])
self.assertEquals(False, kwargs['is_success'])
self.assertEquals('InvokeCommandUsingSystemd',
kwargs['op'])
extension_calls = [
args[0] for (args, _) in popen_patch.call_args_list
if command in args[0]
]
self.assertEquals(
2, len(extension_calls),
"The extension should have been invoked exactly twice")
self.assertIn(
"systemd-run --unit=Microsoft.Compute.TestExtension_1.2.3",
extension_calls[0],
"The first call to the extension should have used systemd"
)
self.assertEquals(
command, extension_calls[1],
"The second call to the extension should not have used systemd"
)
self.assertEquals(len(CGroupsTelemetry._tracked), 0,
"No cgroups should have been created")
self.assertIn("TEST_OUTPUT\n", command_output,
"The test output was not captured")