def test_start_extension_command_should_not_use_fallback_option_if_extension_fails(
self, *args):
self.assertTrue(i_am_root(), "Test does not run when non-root")
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", wraps=subprocess.Popen) \
as patch_mock_popen:
with self.assertRaises(ExtensionError) as context_manager:
SystemdCgroupsApi().start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command="ls folder_does_not_exist",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
# We should have invoked the extension command only once, in the systemd-run case
self.assertEquals(1, patch_mock_popen.call_count)
args = patch_mock_popen.call_args[0][0]
self.assertIn("systemd-run --unit", args)
self.assertEquals(
context_manager.exception.code,
ExtensionErrorCodes.PluginUnknownFailure)
self.assertIn("Non-zero exit code",
ustr(context_manager.exception))
def test_start_extension_command_should_not_use_fallback_option_if_extension_times_out(
self, *args):
self.assertTrue(i_am_root(), "Test does not run when non-root")
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch(
"azurelinuxagent.common.utils.extensionprocessutil.wait_for_process_completion_or_timeout",
return_value=[True, None]):
with patch(
"azurelinuxagent.common.cgroupapi.SystemdCgroupsApi._is_systemd_failure",
return_value=False):
with self.assertRaises(
ExtensionError) as context_manager:
SystemdCgroupsApi().start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command="date",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
self.assertEquals(
context_manager.exception.code,
ExtensionErrorCodes.PluginHandlerScriptTimedout)
self.assertIn("Timeout",
ustr(context_manager.exception))
def test_start_extension_command_should_use_systemd_and_not_the_fallback_option_if_successful(
self, _):
self.assertTrue(i_am_root(), "Test does not run when non-root")
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen", wraps=subprocess.Popen) \
as patch_mock_popen:
extension_cgroups, process_output = SystemdCgroupsApi(
).start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="date",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
# We should have invoked the extension command only once and succeeded
self.assertEquals(1, patch_mock_popen.call_count)
args = patch_mock_popen.call_args[0][0]
self.assertIn("systemd-run --unit", args)
self.assert_cgroups_created(extension_cgroups)
def test_create_extension_cgroups_should_create_extension_slice(self):
self.assertTrue(i_am_root(), "Test does not run when non-root")
extension_name = "Microsoft.Azure.DummyExtension-1.0"
cgroups = SystemdCgroupsApi().create_extension_cgroups(extension_name)
cpu_cgroup, memory_cgroup = cgroups[0], cgroups[1]
self.assertEqual(
cpu_cgroup.path,
"/sys/fs/cgroup/cpu/system.slice/Microsoft.Azure.DummyExtension_1.0"
)
self.assertEqual(
memory_cgroup.path,
"/sys/fs/cgroup/memory/system.slice/Microsoft.Azure.DummyExtension_1.0"
)
unit_name = SystemdCgroupsApi()._get_extension_slice_name(
extension_name)
self.assertEqual(
"system-walinuxagent.extensions-Microsoft.Azure.DummyExtension_1.0.slice",
unit_name)
_, status = shellutil.run_get_output(
"systemctl status {0}".format(unit_name))
self.assertIn("Loaded: loaded", status)
self.assertIn("Active: active", status)
shellutil.run_get_output("systemctl stop {0}".format(unit_name))
shellutil.run_get_output("systemctl disable {0}".format(unit_name))
os.remove("/etc/systemd/system/{0}".format(unit_name))
shellutil.run_get_output("systemctl daemon-reload")
def test_start_extension_command_should_not_use_fallback_option_if_extension_times_out(
self, *args): # pylint: disable=unused-argument
self.assertTrue(i_am_root(), "Test does not run when non-root")
with self._get_cgroup_configurator() as configurator:
pass # release the mocks used to create the test CGroupConfigurator so that they do not conflict the mock Popen below
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch(
"azurelinuxagent.common.utils.extensionprocessutil.wait_for_process_completion_or_timeout",
return_value=[True, None]):
with patch(
"azurelinuxagent.common.cgroupapi.SystemdCgroupsApi._is_systemd_failure",
return_value=False):
with self.assertRaises(
ExtensionError) as context_manager:
configurator.start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command="date",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
self.assertEqual(
context_manager.exception.code,
ExtensionErrorCodes.PluginHandlerScriptTimedout)
self.assertIn("Timeout",
ustr(context_manager.exception))
def test_start_extension_command_should_not_use_fallback_option_if_extension_fails_with_long_output(
self, *args):
self.assertTrue(i_am_root(), "Test does not run when non-root")
with self._get_cgroup_configurator() as configurator:
pass # release the mocks used to create the test CGroupConfigurator so that they do not conflict the mock Popen below
long_output = "a" * 20 # large enough to ensure both stdout and stderr are truncated
long_stdout_stderr_command = "echo {0} && echo {0} >&2 && ls folder_does_not_exist".format(
long_output)
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen",
wraps=subprocess.Popen) as popen_patch:
with self.assertRaises(ExtensionError) as context_manager:
configurator.start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command=long_stdout_stderr_command,
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
extension_calls = [
args[0] for (args, _) in popen_patch.call_args_list
if long_stdout_stderr_command in args[0]
]
self.assertEqual(
1, len(extension_calls),
"The extension should have been invoked exactly once")
self.assertIn(
"systemd-run --unit=Microsoft.Compute.TestExtension_1.2.3",
extension_calls[0],
"The first call to the extension should have used systemd"
)
self.assertEqual(context_manager.exception.code,
ExtensionErrorCodes.PluginUnknownFailure)
self.assertIn("Non-zero exit code",
ustr(context_manager.exception))
# stdout and stderr should have been truncated, so the scope name doesn't appear in stderr
# even though systemd-run ran
self.assertNotIn("Microsoft.Compute.TestExtension_1.2.3",
ustr(context_manager.exception))
def test_create_extension_cgroups_root_should_create_extensions_root_slice(
self):
self.assertTrue(i_am_root(), "Test does not run when non-root")
SystemdCgroupsApi().create_extension_cgroups_root()
unit_name = SystemdCgroupsApi()._get_extensions_slice_root_name()
_, status = shellutil.run_get_output(
"systemctl status {0}".format(unit_name))
self.assertIn("Loaded: loaded", status)
self.assertIn("Active: active", status)
shellutil.run_get_output("systemctl stop {0}".format(unit_name))
shellutil.run_get_output("systemctl disable {0}".format(unit_name))
os.remove("/etc/systemd/system/{0}".format(unit_name))
shellutil.run_get_output("systemctl daemon-reload")
def test_telemetry_with_tracked_cgroup(self, *_):
self.assertTrue(i_am_root(), "Test does not run when non-root")
CGroupConfigurator._instance = None
max_num_polls = 30
time_to_wait = 3
extn_name = "foobar-1.0.0"
num_summarization_values = 7
cgs = make_new_cgroup(extn_name)
self.assertEqual(len(cgs), 2)
ext_handler_properties = ExtHandlerProperties()
ext_handler_properties.version = "1.0.0"
self.ext_handler = ExtHandler(name='foobar')
self.ext_handler.properties = ext_handler_properties
self.ext_handler_instance = ExtHandlerInstance(
ext_handler=self.ext_handler, protocol=None)
command = self.create_script(
"keep_cpu_busy_and_consume_memory_for_5_seconds", '''
nohup python -c "import time
for i in range(5):
x = [1, 2, 3, 4, 5] * (i * 1000)
time.sleep({0})
x *= 0
print('Test loop')" &
'''.format(time_to_wait))
self.log_dir = os.path.join(self.tmp_dir, "log")
with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_base_dir", lambda *_: self.tmp_dir) as \
patch_get_base_dir:
with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_log_dir", lambda *_: self.log_dir) as \
patch_get_log_dir:
self.ext_handler_instance.launch_command(command)
self.assertTrue(
CGroupsTelemetry.is_tracked(
os.path.join(BASE_CGROUPS, "cpu", "walinuxagent.extensions",
"foobar_1.0.0")))
self.assertTrue(
CGroupsTelemetry.is_tracked(
os.path.join(BASE_CGROUPS, "memory", "walinuxagent.extensions",
"foobar_1.0.0")))
for i in range(max_num_polls):
CGroupsTelemetry.poll_all_tracked()
time.sleep(0.5)
collected_metrics = CGroupsTelemetry.report_all_tracked()
self.assertIn("memory", collected_metrics[extn_name])
self.assertIn("cur_mem", collected_metrics[extn_name]["memory"])
self.assertIn("max_mem", collected_metrics[extn_name]["memory"])
self.assertEqual(
len(collected_metrics[extn_name]["memory"]["cur_mem"]),
num_summarization_values)
self.assertEqual(
len(collected_metrics[extn_name]["memory"]["max_mem"]),
num_summarization_values)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["cur_mem"][5], str)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["cur_mem"][6], str)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["max_mem"][5], str)
self.assertIsInstance(
collected_metrics[extn_name]["memory"]["max_mem"][6], str)
self.assertIn("cpu", collected_metrics[extn_name])
self.assertIn("cur_cpu", collected_metrics[extn_name]["cpu"])
self.assertEqual(len(collected_metrics[extn_name]["cpu"]["cur_cpu"]),
num_summarization_values)
self.assertIsInstance(
collected_metrics[extn_name]["cpu"]["cur_cpu"][5], str)
self.assertIsInstance(
collected_metrics[extn_name]["cpu"]["cur_cpu"][6], str)
for i in range(5):
self.assertGreater(
collected_metrics[extn_name]["memory"]["cur_mem"][i], 0)
self.assertGreater(
collected_metrics[extn_name]["memory"]["max_mem"][i], 0)
self.assertGreaterEqual(
collected_metrics[extn_name]["cpu"]["cur_cpu"][i], 0)
def test_telemetry_with_tracked_cgroup(self):
self.assertTrue(i_am_root(), "Test does not run when non-root")
# This test has some timing issues when systemd is managing cgroups, so we force the file system API
# by creating a new instance of the CGroupConfigurator
with patch("azurelinuxagent.common.cgroupapi.CGroupsApi._is_systemd", return_value=False):
cgroup_configurator_instance = CGroupConfigurator._instance
CGroupConfigurator._instance = None
try:
max_num_polls = 30
time_to_wait = 3
extn_name = "foobar-1.0.0"
num_summarization_values = 7
cgs = make_new_cgroup(extn_name)
self.assertEqual(len(cgs), 2)
ext_handler_properties = ExtHandlerProperties()
ext_handler_properties.version = "1.0.0"
self.ext_handler = ExtHandler(name='foobar')
self.ext_handler.properties = ext_handler_properties
self.ext_handler_instance = ExtHandlerInstance(ext_handler=self.ext_handler, protocol=None)
command = self.create_script("keep_cpu_busy_and_consume_memory_for_5_seconds", '''
nohup python -c "import time
for i in range(5):
x = [1, 2, 3, 4, 5] * (i * 1000)
time.sleep({0})
x *= 0
print('Test loop')" &
'''.format(time_to_wait))
self.log_dir = os.path.join(self.tmp_dir, "log")
with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_base_dir", lambda *_: self.tmp_dir) as \
patch_get_base_dir:
with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_log_dir", lambda *_: self.log_dir) as \
patch_get_log_dir:
self.ext_handler_instance.launch_command(command)
#
# If the test is made to run using the systemd API, then the paths of the cgroups need to be checked differently:
#
# self.assertEquals(len(CGroupsTelemetry._tracked), 2)
# cpu = os.path.join(BASE_CGROUPS, "cpu", "system.slice", r"foobar_1.0.0_.*\.scope")
# self.assertTrue(any(re.match(cpu, tracked.path) for tracked in CGroupsTelemetry._tracked))
# memory = os.path.join(BASE_CGROUPS, "memory", "system.slice", r"foobar_1.0.0_.*\.scope")
# self.assertTrue(any(re.match(memory, tracked.path) for tracked in CGroupsTelemetry._tracked))
#
self.assertTrue(CGroupsTelemetry.is_tracked(os.path.join(
BASE_CGROUPS, "cpu", "walinuxagent.extensions", "foobar_1.0.0")))
self.assertTrue(CGroupsTelemetry.is_tracked(os.path.join(
BASE_CGROUPS, "memory", "walinuxagent.extensions", "foobar_1.0.0")))
for i in range(max_num_polls):
CGroupsTelemetry.poll_all_tracked()
time.sleep(0.5)
collected_metrics = CGroupsTelemetry.report_all_tracked()
self.assertIn("memory", collected_metrics[extn_name])
self.assertIn("cur_mem", collected_metrics[extn_name]["memory"])
self.assertIn("max_mem", collected_metrics[extn_name]["memory"])
self.assertEqual(len(collected_metrics[extn_name]["memory"]["cur_mem"]), num_summarization_values)
self.assertEqual(len(collected_metrics[extn_name]["memory"]["max_mem"]), num_summarization_values)
self.assertIsInstance(collected_metrics[extn_name]["memory"]["cur_mem"][5], str)
self.assertIsInstance(collected_metrics[extn_name]["memory"]["cur_mem"][6], str)
self.assertIsInstance(collected_metrics[extn_name]["memory"]["max_mem"][5], str)
self.assertIsInstance(collected_metrics[extn_name]["memory"]["max_mem"][6], str)
self.assertIn("cpu", collected_metrics[extn_name])
self.assertIn("cur_cpu", collected_metrics[extn_name]["cpu"])
self.assertEqual(len(collected_metrics[extn_name]["cpu"]["cur_cpu"]), num_summarization_values)
self.assertIsInstance(collected_metrics[extn_name]["cpu"]["cur_cpu"][5], str)
self.assertIsInstance(collected_metrics[extn_name]["cpu"]["cur_cpu"][6], str)
for i in range(5):
self.assertGreater(collected_metrics[extn_name]["memory"]["cur_mem"][i], 0)
self.assertGreater(collected_metrics[extn_name]["memory"]["max_mem"][i], 0)
self.assertGreaterEqual(collected_metrics[extn_name]["cpu"]["cur_cpu"][i], 0)
# Equal because CPU could be zero for minimum value.
finally:
CGroupConfigurator._instance = cgroup_configurator_instance