def test_run_command_should_raise_when_both_the_input_and_stdin_parameters_are_specified(
self):
with tempfile.TemporaryFile() as input_file:
with self.assertRaises(ValueError):
shellutil.run_command(["cat"],
input='0123456789ABCDEF',
stdin=input_file)
def test_run_command_it_should_log_an_error_when_log_error_is_set(self):
command = ["ls", "-d", "/etc", "nonexistent_file"]
with patch("azurelinuxagent.common.utils.shellutil.logger.error") as mock_log_error:
try:
shellutil.run_command(command, log_error=True)
except:
pass
self.assertEquals(mock_log_error.call_count, 1)
args, kwargs = mock_log_error.call_args
self.assertIn("ls -d /etc nonexistent_file", args, msg="The command was not logged")
self.assertIn(2, args, msg="The command's return code was not logged")
self.assertIn("/etc\n", args, msg="The command's stdout was not logged")
self.assertTrue(any("No such file or directory" in str(a) for a in args), msg="The command's stderr was not logged")
command = "nonexistent_command"
with patch("azurelinuxagent.common.utils.shellutil.logger.error") as mock_log_error:
try:
shellutil.run_command(command, log_error=True)
except:
pass
self.assertEquals(mock_log_error.call_count, 1)
args, kwargs = mock_log_error.call_args
self.assertIn(command, args, msg="The command was not logged")
self.assertTrue(any("No such file or directory" in str(a) for a in args), msg="The command's stderr was not logged")
def set_extension_services_cpu_memory_quota(self, services_list):
"""
Each extension service will have name, systemd path and it's quotas.
This method ensures that drop-in files are created under service.d folder if quotas given.
ex: /lib/systemd/system/extension.service.d/11-CPUAccounting.conf
TODO: set cpu and memory quotas
"""
if self.enabled() and services_list is not None:
for service in services_list:
service_name = service.get('name', None)
unit_file_path = service.get('path', None)
if service_name is not None and unit_file_path is not None:
files_to_create = []
drop_in_path = os.path.join(unit_file_path, "{0}.d".format(service_name))
drop_in_file_cpu_accounting = os.path.join(drop_in_path,
_DROP_IN_FILE_CPU_ACCOUNTING)
files_to_create.append((drop_in_file_cpu_accounting, _DROP_IN_FILE_CPU_ACCOUNTING_CONTENTS))
self.__create_all_files(files_to_create)
# reload the systemd configuration; the new unit will be used once the service restarts
try:
logger.info("Executing systemctl daemon-reload...")
shellutil.run_command(["systemctl", "daemon-reload"])
except Exception as exception:
_log_cgroup_warning("daemon-reload failed (create service unit files): {0}", ustr(exception))
def __collect_azure_unit_telemetry():
azure_units = []
try:
units = shellutil.run_command(['systemctl', 'list-units', 'azure*', '-all'])
for line in units.split('\n'):
match = re.match(r'\s?(azure[^\s]*)\s?', line, re.IGNORECASE)
if match is not None:
azure_units.append((match.group(1), line))
except shellutil.CommandError as command_error:
_log_cgroup_warning("Failed to list systemd units: {0}", ustr(command_error))
for unit_name, unit_description in azure_units:
unit_slice = "Unknown"
try:
unit_slice = systemd.get_unit_property(unit_name, "Slice")
except Exception as exception:
_log_cgroup_warning("Failed to query Slice for {0}: {1}", unit_name, ustr(exception))
_log_cgroup_info("Found an Azure unit under slice {0}: {1}", unit_slice, unit_description)
if len(azure_units) == 0:
try:
cgroups = shellutil.run_command('systemd-cgls')
for line in cgroups.split('\n'):
if re.match(r'[^\x00-\xff]+azure\.slice\s*', line, re.UNICODE):
logger.info(ustr("Found a cgroup for azure.slice\n{0}").format(cgroups))
# Don't add the output of systemd-cgls to the telemetry, since currently it does not support Unicode
add_event(op=WALAEventOperation.CGroupsInfo, message="Found a cgroup for azure.slice")
except shellutil.CommandError as command_error:
_log_cgroup_warning("Failed to list systemd units: {0}", ustr(command_error))
def __set_service_unit_file(self):
service_unit_file = self.get_service_file_path()
binary_path = os.path.join(conf.get_lib_dir(), self.BINARY_FILE_NAME)
try:
fileutil.write_file(
service_unit_file,
self.__SERVICE_FILE_CONTENT.format(binary_path=binary_path,
py_path=sys.executable,
version=self._UNIT_VERSION))
fileutil.chmod(service_unit_file, 0o644)
# Finally enable the service. This is needed to ensure the service is started on system boot
cmd = ["systemctl", "enable", self._network_setup_service_name]
try:
shellutil.run_command(cmd)
except CommandError as error:
msg = ustr(
"Unable to enable service: {0}; deleting service file: {1}. Command: {2}, Exit-code: {3}.\nstdout: {4}\nstderr: {5}"
).format(self._network_setup_service_name, service_unit_file,
' '.join(cmd), error.returncode, error.stdout,
error.stderr)
raise Exception(msg)
except Exception:
self.__remove_file_without_raising(service_unit_file)
raise
def test_run_command_should_raise_an_exception_when_it_cannot_execute_the_command(self):
command = "nonexistent_command"
with self.assertRaises(Exception) as context_manager:
shellutil.run_command(command)
exception = context_manager.exception
self.assertIn("No such file or directory", str(exception))
def create_and_start_unit(unit_filename, unit_contents):
try:
unit_path = os.path.join(UNIT_FILES_FILE_SYSTEM_PATH, unit_filename)
fileutil.write_file(unit_path, unit_contents)
shellutil.run_command(["systemctl", "daemon-reload"])
shellutil.run_command(["systemctl", "start", unit_filename])
except Exception as e:
raise CGroupsException("Failed to create and start {0}. Error: {1}".format(unit_filename, ustr(e)))
def _collect_logs():
logger.info("Starting log collection...")
# Invoke the command line tool in the agent to collect logs, with resource limits on CPU and memory (RAM).
scope_name = "collect-logs-{0}.scope".format(
ustr(int(time.time() * 1000000)))
systemd_cmd = [
"systemd-run", "--unit={0}".format(scope_name), "--scope"
]
# More info on resource limits properties in systemd here:
# https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/resource_management_guide/sec-modifying_control_groups
cpu_limit, memory_limit = CollectLogsHandler._get_resource_limits()
resource_limits = [
"--property=CPUAccounting=1",
"--property=CPUQuota={0}".format(cpu_limit),
"--property=MemoryAccounting=1",
"--property=MemoryLimit={0}".format(memory_limit)
]
# The log tool is invoked from the current agent's egg with the command line option
collect_logs_cmd = [sys.executable, "-u", sys.argv[0], "-collect-logs"]
final_command = systemd_cmd + resource_limits + collect_logs_cmd
start_time = datetime.datetime.utcnow()
success = False
msg = None
try:
shellutil.run_command(final_command, log_error=True)
duration = elapsed_milliseconds(start_time)
archive_size = os.path.getsize(COMPRESSED_ARCHIVE_PATH)
msg = "Successfully collected logs. Archive size: {0} b, elapsed time: {1} ms.".format(
archive_size, duration)
logger.info(msg)
success = True
return True
except Exception as e: # pylint: disable=C0103
duration = elapsed_milliseconds(start_time)
if isinstance(e, CommandError):
exception_message = ustr("[stderr] %s", e.stderr) # pylint: disable=no-member
else:
exception_message = ustr(e)
msg = "Failed to collect logs. Elapsed time: {0} ms. Error: {1}".format(
duration, exception_message)
# No need to log to the local log since we ran run_command with logging errors as enabled
return False
finally:
add_event(name=AGENT_NAME,
version=CURRENT_VERSION,
op=WALAEventOperation.LogCollection,
is_success=success,
message=msg,
log_event=False)
def __execute_cmd(cmd):
try:
shellutil.run_command(cmd)
except CommandError as error:
msg = "Command {0} failed with exit-code: {1}\nStdout: {2}\nStderr: {3}".format(' '.join(cmd),
error.returncode,
ustr(error.stdout),
ustr(error.stderr))
raise Exception(msg)
def test_it_should_execute_binary_file_successfully(self):
# A bare-bone test to ensure no simple syntactical errors in the binary file as its generated dynamically
self.__replace_popen_cmd = TestPersistFirewallRulesHandler.__mock_network_setup_service_disabled
with self._get_persist_firewall_rules_handler() as handler:
self.assertFalse(os.path.exists(self._binary_file), "Binary file should not be there")
handler.setup()
self.assertTrue(os.path.exists(self._binary_file), "Binary file not set properly")
shellutil.run_command([sys.executable, self._binary_file])
def __reload_systemd_conf(self):
try:
logger.info(
"Executing systemctl daemon-reload for setting up {0}".format(
self._network_setup_service_name))
shellutil.run_command(["systemctl", "daemon-reload"])
except Exception as exception:
logger.warn(
"Unable to reload systemctl configurations: {0}".format(
ustr(exception)))
def update_iboip_interfaces(self, mac_ip_array):
net_dir = "/sys/class/net"
nics = os.listdir(net_dir)
count = 0
for nic in nics:
# look for IBoIP interface of format ibXXX
if not re.match(r"ib\w+", nic):
continue
mac_addr = None
with open(os.path.join(net_dir, nic, "address")) as address_file:
mac_addr = address_file.read()
if not mac_addr:
logger.error("RDMA: can't read address for device {0}".format(nic))
continue
mac_addr = mac_addr.upper()
match = re.match(r".+(\w\w):(\w\w):(\w\w):\w\w:\w\w:(\w\w):(\w\w):(\w\w)\n", mac_addr)
if not match:
logger.error("RDMA: failed to parse address for device {0} address {1}".format(nic, mac_addr))
continue
# format an MAC address without :
mac_addr = ""
mac_addr = mac_addr.join(match.groups(0))
for mac_ip in mac_ip_array:
if mac_ip[0] == mac_addr:
ret = 0
try:
# bring up the interface and set its IP address
ip_command = ["ip", "link", "set", nic, "up"]
shellutil.run_command(ip_command)
ip_command = ["ip", "addr", "add", "{0}/16".format(mac_ip[1]), "dev", nic]
shellutil.run_command(ip_command)
except shellutil.CommandError as error:
ret = error.returncode
if ret == 0:
logger.info("RDMA: set address {0} to device {1}".format(mac_ip[1], nic))
if ret and ret != 2:
# return value 2 means the address is already set
logger.error("RDMA: failed to set IP address {0} on device {1}".format(mac_ip[1], nic))
else:
count += 1
break
return count
def test_run_command_should_raise_an_exception_when_the_command_fails(self):
command = ["ls", "-d", "/etc", "nonexistent_file"]
with self.assertRaises(shellutil.CommandError) as context_manager:
shellutil.run_command(command)
exception = context_manager.exception
self.assertEquals(str(exception), "'ls' failed: 2")
self.assertEquals(exception.stdout, "/etc\n")
self.assertIn("No such file or directory", exception.stderr)
self.assertEquals(exception.returncode, 2)
def test_run_command_should_log_an_error_when_log_error_is_set(self):
self.__it_should_log_an_error_when_log_error_is_set(
lambda: shellutil.run_command(["ls", "file-does-not-exist"],
log_error=True
), # Raises a CommandError
command="ls")
self.__it_should_log_an_error_when_log_error_is_set(
lambda: shellutil.run_command("command-does-not-exist",
log_error=True
), # Raises a CommandError
command="command-does-not-exist")
def ifup(self, ifname, retries=3, wait=5):
logger.info('Interface {0} bounce with ifup'.format(ifname))
retry_limit = retries + 1
for attempt in range(1, retry_limit):
try:
shellutil.run_command(['ifup', ifname], log_error=True)
except Exception:
if attempt < retry_limit:
logger.info("retrying in {0} seconds".format(wait))
time.sleep(wait)
else:
logger.warn("exceeded restart retries")
def gen_transport_cert(self, prv_file, crt_file):
"""
Create ssl certificate for https communication with endpoint server.
"""
cmd = [self.openssl_cmd, "req", "-x509", "-nodes", "-subj", "/CN=LinuxTransport",
"-days", "730", "-newkey", "rsa:2048", "-keyout", prv_file, "-out", crt_file]
try:
shellutil.run_command(cmd)
except shellutil.CommandError as cmd_err:
msg = "Failed to create {0} and {1} certificates.\n[stdout]\n{2}\n\n[stderr]\n{3}\n"\
.format(prv_file, crt_file, cmd_err.stdout, cmd_err.stderr)
logger.error(msg)
def set_hostname(self, hostname):
"""
Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
Due to a bug in systemd in Centos-7.0, if this call fails, fallback
to hostname.
"""
hostnamectl_cmd = ["hostnamectl", "set-hostname", hostname, "--static"]
try:
shellutil.run_command(hostnamectl_cmd)
except Exception as e:
logger.warn("[{0}] failed with error: {1}, attempting fallback".format(' '.join(hostnamectl_cmd), ustr(e)))
DefaultOSUtil.set_hostname(self, hostname)
def set_hostname(self, hostname):
"""
Unlike redhat 6.x, redhat 7.x will set hostname via hostnamectl
Due to a bug in systemd in Centos-7.0, if this call fails, fallback
to hostname.
"""
hostnamectl_cmd = ['hostnamectl', 'set-hostname', hostname, '--static']
try:
shellutil.run_command(hostnamectl_cmd, log_error=False)
except shellutil.CommandError:
logger.warn("[{0}] failed, attempting fallback".format(' '.join(hostnamectl_cmd)))
DefaultOSUtil.set_hostname(self, hostname)
def assert_no_message_logged(command):
try:
shellutil.run_command(command)
except: # pylint: disable=bare-except
pass
self.assertEqual(mock_logger.info.call_count, 0)
self.assertEqual(mock_logger.verbose.call_count, 0)
self.assertEqual(mock_logger.warn.call_count, 0)
self.assertEqual(mock_logger.error.call_count, 0)
assert_no_message_logged(["ls", "nonexistent_file"])
assert_no_message_logged("nonexistent_command")
def remove_extension_cgroups(self, extension_name):
# For transient units, cgroups are released automatically when the unit stops, so it is sufficient
# to call stop on them. Persistent cgroups are released when the unit is disabled and its configuration
# file is deleted.
# The assumption is that this method is called after the extension has been uninstalled. For now, since
# we're running extensions within transient scopes which clean up after they finish running, no removal
# of units is needed. In the future, when the extension is running under its own slice,
# the following clean up is needed.
unit_filename = self._get_extension_slice_name(extension_name)
try:
unit_path = os.path.join(UNIT_FILES_FILE_SYSTEM_PATH, unit_filename)
shellutil.run_command(["systemctl", "stop", unit_filename])
fileutil.rm_files(unit_path)
shellutil.run_command(["systemctl", "daemon-reload"])
except Exception as e:
raise CGroupsException("Failed to remove {0}. Error: {1}".format(unit_filename, ustr(e)))
def get_cgroup_mount_points(self):
"""
Returns a tuple with the mount points for the cpu and memory controllers; the values can be None
if the corresponding controller is not mounted
"""
# the output of mount is similar to
# $ mount -t cgroup
# cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,name=systemd)
# cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
# cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
# etc
#
if self._cgroup_mountpoints is None:
cpu = None
memory = None
for line in shellutil.run_command(['mount', '-t', 'cgroup']).splitlines():
match = re.search(r'on\s+(?P<path>/\S+(memory|cpuacct))\s', line)
if match is not None:
path = match.group('path')
if 'cpuacct' in path:
cpu = path
else:
memory = path
self._cgroup_mountpoints = {'cpu': cpu, 'memory': memory}
return self._cgroup_mountpoints['cpu'], self._cgroup_mountpoints['memory']
def get_systemd_version():
# the output is similar to
# $ systemctl --version
# systemd 245 (245.4-4ubuntu3)
# +PAM +AUDIT +SELINUX +IMA +APPARMOR +SMACK +SYSVINIT +UTMP etc
#
return shellutil.run_command(['systemctl', '--version'])
def get_pubkey_from_prv(self, file_name):
if not os.path.exists(file_name):
raise IOError(errno.ENOENT, "File not found", file_name)
else:
cmd = [self.openssl_cmd, "rsa", "-in", file_name, "-pubout"]
pub = shellutil.run_command(cmd, log_error=True)
return pub
def get_lis_version():
"""
This uses the Linux kernel's 'modinfo' command to retrieve the
"version" field for the "hv_vmbus" kernel module (the LIS
drivers). This is the documented method to retrieve the LIS module
version. Every Linux guest on Hyper-V will have this driver, but
it may not be installed as a module (it could instead be built
into the kernel). In that case, this will return "Absent" instead
of the version, indicating the driver version can be deduced from
the kernel version. It will only return "Failed" in the presence
of an exception.
This function is used to generate telemetry for the version of the
LIS drivers installed on the VM. The function and associated
telemetry can be removed after a few releases.
"""
try:
modinfo_output = shellutil.run_command(
["modinfo", "-F", "version", "hv_vmbus"])
if modinfo_output:
return modinfo_output
# If the system doesn't have LIS drivers, 'modinfo' will
# return nothing on stdout, which will cause 'run_command'
# to return an empty string.
return COMMAND_ABSENT
except Exception:
# Ignore almost every possible exception because this is in a
# critical code path. Unfortunately the logger isn't already
# imported in this module or we'd log this too.
return COMMAND_FAILED
def __log_network_setup_service_logs(self):
# Get logs from journalctl - https://www.freedesktop.org/software/systemd/man/journalctl.html
cmd = [
"journalctl", "-u", self._network_setup_service_name, "-b", "--utc"
]
service_failed = self.__verify_network_setup_service_failed()
try:
stdout = shellutil.run_command(cmd)
msg = ustr("Logs from the {0} since system boot:\n {1}").format(
self._network_setup_service_name, stdout)
logger.info(msg)
except CommandError as error:
msg = "Unable to fetch service logs, Command: {0} failed with ExitCode: {1}\nStdout: {2}\nStderr: {3}".format(
' '.join(cmd), error.returncode, error.stdout, error.stderr)
logger.warn(msg)
except Exception as e:
msg = "Ran into unexpected error when getting logs for {0} service. Error: {1}".format(
self._network_setup_service_name, textutil.format_exception(e))
logger.warn(msg)
# Log service status and logs if we can fetch them from journalctl and send it to Kusto,
# else just log the error of the failure of fetching logs
add_event(op=WALAEventOperation.PersistFirewallRules,
is_success=(not service_failed),
message=msg,
log_event=False)
def get_pubkey_from_crt(self, file_name):
if not os.path.exists(file_name): # pylint: disable=R1720
raise IOError(errno.ENOENT, "File not found", file_name)
else:
cmd = [self.openssl_cmd, "x509", "-in", file_name, "-pubkey", "-noout"]
pub = shellutil.run_command(cmd, log_error=True)
return pub
def test_run_command_should_read_stdin_from_the_input_parameter_when_it_is_a_sequence_of_bytes(
self):
command_input = 'TEST BYTES'
output = shellutil.run_command(["cat"], input=command_input)
self.assertEqual(
output, command_input,
"The command did not process its input correctly; the output should match the input"
)
def test_run_command_should_return_a_string_by_default(self):
output = shellutil.run_command(self.__create_tee_script(),
input="TEST STRING")
self.assertTrue(
isinstance(output, ustr),
"The return value should be a string. Got: '{0}'".format(
type(output)))
def get_unit_property(unit_name, property_name):
output = shellutil.run_command(
["systemctl", "show", unit_name, "--property", property_name])
match = re.match("[^=]+=(?P<value>.+)", output)
if match is None:
raise ValueError("Can't find property {0} of {1}", property_name,
unit_name) # pylint: disable=W0715
return match.group('value')
def get_thumbprint_from_crt(self, file_name):
if not os.path.exists(file_name): # pylint: disable=R1720
raise IOError(errno.ENOENT, "File not found", file_name)
else:
cmd = [self.openssl_cmd, "x509", "-in", file_name, "-fingerprint", "-noout"]
thumbprint = shellutil.run_command(cmd)
thumbprint = thumbprint.rstrip().split('=')[1].replace(':', '').upper()
return thumbprint
