def disable(self, reason):
self._cgroups_enabled = False
message = "[CGW] Disabling resource usage monitoring. Reason: {0}".format(reason)
logger.info(message) # log as INFO for now, in the future it should be logged as WARNING
add_event(op=WALAEventOperation.CGroupsDisabled, message=message, is_success=False, log_event=False)
self.__reset_cpu_quota()
CGroupsTelemetry.reset()
def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_directly_if_systemd_times_out(
self, _):
with self._get_cgroup_configurator() as configurator:
# Systemd has its own internal timeout which is shorter than what we define for extension operation timeout.
# When systemd times out, it will write a message to stderr and exit with exit code 1.
# In that case, we will internally recognize the failure due to the non-zero exit code, not as a timeout.
configurator.mocks.add_command(
MockCommand(
"systemd-run",
return_value=1,
stdout='',
stderr=
'Failed to start transient scope unit: Connection timed out'
))
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("subprocess.Popen",
wraps=subprocess.Popen) as popen_patch:
CGroupsTelemetry.reset()
configurator.start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command="echo 'success'",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
self.assertFalse(configurator.enabled(),
"Cgroups should have been disabled")
extension_calls = [
args[0] for (args, _) in popen_patch.call_args_list
if "echo 'success'" in args[0]
]
self.assertEqual(
2, len(extension_calls),
"The extension should have been called twice. Got: {0}"
.format(extension_calls))
self.assertIn(
"systemd-run --unit=Microsoft.Compute.TestExtension_1.2.3",
extension_calls[0],
"The first call to the extension should have used systemd"
)
self.assertNotIn(
"systemd-run", extension_calls[1],
"The second call to the extension should not have used systemd"
)
self.assertEqual(
len(CGroupsTelemetry._tracked), 0,
"No cgroups should have been created")
def disable_cgroups(exception):
self.disable()
CGroupsTelemetry.reset()
add_event(AGENT_NAME,
version=CURRENT_VERSION,
op=WALAEventOperation.CGroupsCleanUp,
is_success=False,
log_event=False,
message='{0} {1}'.format(message, ustr(exception)))
def setUp(self):
AgentTestCase.setUp(self)
event.init_event_logger(os.path.join(self.tmp_dir, EVENTS_DIRECTORY))
CGroupsTelemetry.reset()
clear_singleton_instances(ProtocolUtil)
protocol = WireProtocol('endpoint')
protocol.update_goal_state = MagicMock()
self.get_protocol = patch('azurelinuxagent.common.protocol.util.ProtocolUtil.get_protocol', return_value=protocol)
self.get_protocol.start()
def test_start_extension_command_should_invoke_the_command_directly_if_systemd_times_out(
self, _):
# Systemd has its own internal timeout which is shorter than what we define for extension operation timeout.
# When systemd times out, it will write a message to stderr and exit with exit code 1.
# In that case, we will internally recognize the failure due to the non-zero exit code, not as a timeout.
original_popen = subprocess.Popen
systemd_timeout_command = "echo 'Failed to start transient scope unit: Connection timed out' >&2 && exit 1"
def mock_popen(*args, **kwargs):
# If trying to invoke systemd, mock what would happen if systemd timed out internally:
# write failure to stderr and exit with exit code 1.
new_args = args
if "systemd-run" in args[0]:
new_args = (systemd_timeout_command, )
return original_popen(new_args, **kwargs)
expected_output = "[stdout]\n{0}\n\n\n[stderr]\n"
with tempfile.TemporaryFile(dir=self.tmp_dir, mode="w+b") as stdout:
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as stderr:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen",
side_effect=mock_popen) as popen_patch:
CGroupsTelemetry.reset()
SystemdCgroupsApi().start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command="echo 'success'",
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=stdout,
stderr=stderr)
extension_calls = [
args[0] for (args, _) in popen_patch.call_args_list
if "echo 'success'" in args[0]
]
self.assertEquals(
2, len(extension_calls),
"The extension should have been invoked exactly twice")
self.assertIn(
"systemd-run --unit=Microsoft.Compute.TestExtension_1.2.3",
extension_calls[0],
"The first call to the extension should have used systemd"
)
self.assertEquals(
"echo 'success'", extension_calls[1],
"The second call to the extension should not have used systemd"
)
self.assertEquals(len(CGroupsTelemetry._tracked), 0,
"No cgroups should have been created")
def disable(self, reason, disableCgroups):
# Todo: disable/reset extension when ext quotas introduced
if disableCgroups == DisableCgroups.ALL: # disable all
self._agent_cgroups_enabled = False
self._extensions_cgroups_enabled = False
self.__reset_agent_cpu_quota()
CGroupsTelemetry.reset()
elif disableCgroups == DisableCgroups.AGENT: # disable agent
self._agent_cgroups_enabled = False
self.__reset_agent_cpu_quota()
CGroupsTelemetry.stop_tracking(CpuCgroup(AGENT_NAME_TELEMETRY, self._agent_cpu_cgroup_path))
elif disableCgroups == DisableCgroups.EXTENSIONS: # disable extensions
self._extensions_cgroups_enabled = False
message = "[CGW] Disabling resource usage monitoring. Reason: {0}".format(reason)
logger.info(message) # log as INFO for now, in the future it should be logged as WARNING
add_event(op=WALAEventOperation.CGroupsDisabled, message=message, is_success=False, log_event=False)
def _get_cgroup_configurator(self,
initialize=True,
enable=True,
mock_commands=None):
CGroupConfigurator._instance = None
configurator = CGroupConfigurator.get_instance()
CGroupsTelemetry.reset()
with mock_cgroup_environment(self.tmp_dir) as mock_environment:
if mock_commands is not None:
for command in mock_commands:
mock_environment.add_command(command)
configurator.mocks = mock_environment
if initialize:
if not enable:
with patch.object(configurator, "enable"):
configurator.initialize()
else:
configurator.initialize()
yield configurator
def tearDown(self):
AgentTestCase.tearDown(self)
CGroupsTelemetry.reset()
self.get_protocol.stop()
def disable(self):
self._enabled = False
CGroupsTelemetry.reset()
def tearDown(self):
AgentTestCase.tearDown(self)
CGroupsTelemetry.reset()
def setUp(self):
AgentTestCase.setUp(self)
CGroupsTelemetry.reset()
def test_start_extension_command_should_invoke_the_command_directly_if_systemd_fails(
self, _):
original_popen = subprocess.Popen
def mock_popen(command, *args, **kwargs):
if command.startswith('systemd-run'):
# Inject a syntax error to the call
command = command.replace('systemd-run',
'systemd-run syntax_error')
return original_popen(command, *args, **kwargs)
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as output_file:
with patch("azurelinuxagent.common.cgroupapi.add_event"
) as mock_add_event:
with patch("azurelinuxagent.common.cgroupapi.subprocess.Popen",
side_effect=mock_popen) as popen_patch:
CGroupsTelemetry.reset()
command = "echo TEST_OUTPUT"
command_output = SystemdCgroupsApi(
).start_extension_command(
extension_name="Microsoft.Compute.TestExtension-1.2.3",
command=command,
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=output_file,
stderr=output_file)
args, kwargs = mock_add_event.call_args
self.assertIn(
"Failed to run systemd-run for unit Microsoft.Compute.TestExtension_1.2.3",
kwargs['message'])
self.assertIn(
"Failed to find executable syntax_error: No such file or directory",
kwargs['message'])
self.assertEquals(False, kwargs['is_success'])
self.assertEquals('InvokeCommandUsingSystemd',
kwargs['op'])
extension_calls = [
args[0] for (args, _) in popen_patch.call_args_list
if command in args[0]
]
self.assertEquals(
2, len(extension_calls),
"The extension should have been invoked exactly twice")
self.assertIn(
"systemd-run --unit=Microsoft.Compute.TestExtension_1.2.3",
extension_calls[0],
"The first call to the extension should have used systemd"
)
self.assertEquals(
command, extension_calls[1],
"The second call to the extension should not have used systemd"
)
self.assertEquals(len(CGroupsTelemetry._tracked), 0,
"No cgroups should have been created")
self.assertIn("TEST_OUTPUT\n", command_output,
"The test output was not captured")
def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_directly_if_systemd_fails(
self, _):
with self._get_cgroup_configurator() as configurator:
original_popen = subprocess.Popen
def mock_popen(command, *args, **kwargs):
if 'systemd-run' in command:
# Inject a syntax error to the call
command = command.replace('systemd-run',
'systemd-run syntax_error')
return original_popen(command, *args, **kwargs)
with tempfile.TemporaryFile(dir=self.tmp_dir,
mode="w+b") as output_file:
with patch(
"azurelinuxagent.common.cgroupconfigurator.add_event"
) as mock_add_event:
with patch(
"azurelinuxagent.common.cgroupapi.subprocess.Popen",
side_effect=mock_popen) as popen_patch:
CGroupsTelemetry.reset()
command = "echo TEST_OUTPUT"
command_output = configurator.start_extension_command(
extension_name=
"Microsoft.Compute.TestExtension-1.2.3",
command=command,
timeout=300,
shell=True,
cwd=self.tmp_dir,
env={},
stdout=output_file,
stderr=output_file)
self.assertFalse(configurator.enabled(),
"Cgroups should have been disabled")
disabled_events = [
kwargs
for _, kwargs in mock_add_event.call_args_list if
kwargs['op'] == WALAEventOperation.CGroupsDisabled
]
self.assertTrue(
len(disabled_events) == 1,
"Exactly one CGroupsDisabled telemetry event should have been issued. Found: {0}"
.format(disabled_events))
self.assertIn(
"Failed to start Microsoft.Compute.TestExtension-1.2.3 using systemd-run",
disabled_events[0]['message'],
"The systemd-run failure was not included in the telemetry message"
)
self.assertEqual(
False, disabled_events[0]['is_success'],
"The telemetry event should indicate a failure")
extension_calls = [
args[0] for (args, _) in popen_patch.call_args_list
if command in args[0]
]
self.assertEqual(
2, len(extension_calls),
"The extension should have been invoked exactly twice"
)
self.assertIn(
"systemd-run --unit=Microsoft.Compute.TestExtension_1.2.3",
extension_calls[0],
"The first call to the extension should have used systemd"
)
self.assertEqual(
command, extension_calls[1],
"The second call to the extension should not have used systemd"
)
self.assertEqual(
len(CGroupsTelemetry._tracked), 0,
"No cgroups should have been created")
self.assertIn("TEST_OUTPUT\n", command_output,
"The test output was not captured")