def mount_resource_disk(self, mount_point, fs):
device = self.osutil.device_for_ide_port(1)
if device is None:
raise ResourceDiskError("unable to detect disk topology")
device = "/dev/" + device
mountlist = shellutil.run_get_output("mount")[1]
existing = self.osutil.get_mount_point(mountlist, device)
if(existing):
logger.info("Resource disk {0}1 is already mounted", device)
return existing
fileutil.mkdir(mount_point, mode=0o755)
logger.info("Detect GPT...")
partition = device + "1"
ret = shellutil.run_get_output("parted {0} print".format(device))
if ret[0]:
raise ResourceDiskError("({0}) {1}".format(device, ret[1]))
if "gpt" in ret[1]:
logger.info("GPT detected")
logger.info("Get GPT partitions")
parts = [x for x in ret[1].split("\n") if re.match("^\s*[0-9]+", x)]
logger.info("Found more than {0} GPT partitions.", len(parts))
if len(parts) > 1:
logger.info("Remove old GPT partitions")
for i in range(1, len(parts) + 1):
logger.info("Remove partition: {0}", i)
shellutil.run("parted {0} rm {1}".format(device, i))
logger.info("Create a new GPT partition using entire disk space")
shellutil.run("parted {0} mkpart primary 0% 100%".format(device))
logger.info("Format partition: {0} with fstype {1}",partition,fs)
shellutil.run("mkfs." + fs + " " + partition + " -F")
else:
logger.info("GPT not detected")
logger.info("Check fstype")
ret = shellutil.run_get_output("sfdisk -q -c {0} 1".format(device))
if ret[1].rstrip() == "7" and fs != "ntfs":
logger.info("The partition is formatted with ntfs")
logger.info("Format partition: {0} with fstype {1}",partition,fs)
shellutil.run("sfdisk -c {0} 1 83".format(device))
shellutil.run("mkfs." + fs + " " + partition + " -F")
logger.info("Mount resource disk")
ret = shellutil.run("mount {0} {1}".format(partition, mount_point),
chk_err=False)
if ret:
logger.warn("Failed to mount resource disk. Retry mounting")
shellutil.run("mkfs." + fs + " " + partition + " -F")
ret = shellutil.run("mount {0} {1}".format(partition, mount_point))
if ret:
raise ResourceDiskError("({0}) {1}".format(partition, ret))
logger.info("Resource disk ({0}) is mounted at {1} with fstype {2}",
device, mount_point, fs)
return mount_point
def test_it_should_log_unexpected_errors_as_errors(self):
logger_delta = str("logger.EVERY_FIFTEEN_MINUTES")
return_code = 99
command = "exit {0}".format(return_code)
with patch("azurelinuxagent.common.utils.shellutil.logger",
autospec=True) as mock_logger:
shellutil.run_get_output(command,
log_cmd=False,
expected_errors=[return_code + 1])
self.assertEquals(mock_logger.periodic_error.call_count, 1)
args, kwargs = mock_logger.periodic_error.call_args
time_delta = str(args[0])
self.assertIn(logger_delta, time_delta)
message = args[
1] # message is similar to "Command: [exit 99], return code: [99], result: []"
self.assertIn("[{0}]".format(command), message)
self.assertIn("[{0}]".format(return_code), message)
self.assertEquals(mock_logger.info.call_count, 0)
self.assertEquals(mock_logger.verbose.call_count, 0)
self.assertEquals(mock_logger.warn.call_count, 0)
def __init__(self):
super(NSBSDOSUtil, self).__init__()
if self.resolver is None:
# NSBSD doesn't have a system resolver, configure a python one
try:
import dns.resolver
except ImportError:
raise OSUtilError(
"Python DNS resolver not available. Cannot proceed!")
self.resolver = dns.resolver.Resolver()
servers = []
cmd = "getconf /usr/Firewall/ConfigFiles/dns Servers | tail -n +2"
ret, output = shellutil.run_get_output(cmd)
for server in output.split("\n"):
if server == '':
break
server = server[:-1] # remove last '='
cmd = "grep '{}' /etc/hosts".format(
server) + " | awk '{print $1}'"
ret, ip = shellutil.run_get_output(cmd)
servers.append(ip)
self.resolver.nameservers = servers
dns.resolver.override_system_resolver(self.resolver)
def test_it_should_log_command_failures_as_errors(self):
return_code = 99
command = "exit {0}".format(return_code)
with patch("azurelinuxagent.common.utils.shellutil.logger",
autospec=True) as mock_logger:
shellutil.run_get_output(command, log_cmd=False)
self.assertEqual(mock_logger.error.call_count, 1)
args, _ = mock_logger.error.call_args
message = args[
0] # message is similar to "Command: [exit 99], return code: [99], result: []"
self.assertIn("[{0}]".format(command), message)
self.assertIn("[{0}]".format(return_code), message)
self.assertEqual(
mock_logger.info.call_count, 0,
"Did not expect any info messages. Got: {0}".format(
mock_logger.info.call_args_list))
self.assertEqual(
mock_logger.warn.call_count, 0,
"Did not expect any warnings. Got: {0}".format(
mock_logger.warn.call_args_list))
def _get_net_info():
"""
There is no SIOCGIFCONF
on freeBSD - just parse ifconfig.
Returns strings: iface, inet4_addr, and mac
or 'None,None,None' if unable to parse.
We will sleep and retry as the network must be up.
"""
iface = ''
inet = ''
mac = ''
err, output = shellutil.run_get_output('ifconfig -l ether',
chk_err=False)
if err:
raise OSUtilError("Can't find ether interface:{0}".format(output))
ifaces = output.split()
if not ifaces:
raise OSUtilError("Can't find ether interface.")
iface = ifaces[0]
err, output = shellutil.run_get_output('ifconfig ' + iface,
chk_err=False)
if err:
raise OSUtilError("Can't get info for interface:{0}".format(iface))
for line in output.split('\n'):
if line.find('inet ') != -1:
inet = line.split()[1]
elif line.find('ether ') != -1:
mac = line.split()[1]
logger.verbose("Interface info: ({0},{1},{2})", iface, inet, mac)
return iface, inet, mac
def change_partition_type(self, suppress_message, option_str):
"""
use sfdisk to change partition type.
First try with --part-type; if fails, fall back to -c
"""
option_to_use = '--part-type'
command = "sfdisk {0} {1} {2}".format(option_to_use,
'-f' if suppress_message else '',
option_str)
err_code, output = shellutil.run_get_output(command,
chk_err=False,
log_cmd=True)
# fall back to -c
if err_code != 0:
logger.info(
"sfdisk with --part-type failed [{0}], retrying with -c",
err_code)
option_to_use = '-c'
command = "sfdisk {0} {1} {2}".format(
option_to_use, '-f' if suppress_message else '', option_str)
err_code, output = shellutil.run_get_output(command, log_cmd=True)
if err_code == 0:
logger.info('{0} succeeded', command)
else:
logger.error('{0} failed [{1}: {2}]', command, err_code, output)
return err_code, output
def _get_net_info():
"""
There is no SIOCGIFCONF
on freeBSD - just parse ifconfig.
Returns strings: iface, inet4_addr, and mac
or 'None,None,None' if unable to parse.
We will sleep and retry as the network must be up.
"""
iface = ''
inet = ''
mac = ''
err, output = shellutil.run_get_output('ifconfig -l ether', chk_err=False)
if err:
raise OSUtilError("Can't find ether interface:{0}".format(output))
ifaces = output.split()
if not ifaces:
raise OSUtilError("Can't find ether interface.")
iface = ifaces[0]
err, output = shellutil.run_get_output('ifconfig ' + iface, chk_err=False)
if err:
raise OSUtilError("Can't get info for interface:{0}".format(iface))
for line in output.split('\n'):
if line.find('inet ') != -1:
inet = line.split()[1]
elif line.find('ether ') != -1:
mac = line.split()[1]
logger.verbose("Interface info: ({0},{1},{2})", iface, inet, mac)
return iface, inet, mac
def change_partition_type(self, suppress_message, option_str):
"""
use sfdisk to change partition type.
First try with --part-type; if fails, fall back to -c
"""
command_to_use = '--part-type'
input = "sfdisk {0} {1} {2}".format(command_to_use, '-f' if suppress_message else '', option_str)
err_code, output = shellutil.run_get_output(input, chk_err=False, log_cmd=True)
# fall back to -c
if err_code != 0:
logger.info("sfdisk with --part-type failed [{0}], retrying with -c", err_code)
command_to_use = '-c'
input = "sfdisk {0} {1} {2}".format(command_to_use, '-f' if suppress_message else '', option_str)
err_code, output = shellutil.run_get_output(input, log_cmd=True)
if err_code == 0:
logger.info('{0} succeeded',
input)
else:
logger.error('{0} failed [{1}: {2}]',
input,
err_code,
output)
return err_code, output
def test_run_get_output(self):
output = shellutil.run_get_output(u"ls /")
self.assertNotEquals(None, output)
self.assertEquals(0, output[0])
err = shellutil.run_get_output(u"ls /not-exists")
self.assertNotEquals(0, err[0])
err = shellutil.run_get_output(u"ls 我")
self.assertNotEquals(0, err[0])
def test_run_get_output(self):
output = shellutil.run_get_output(u"ls /")
self.assertNotEquals(None, output)
self.assertEquals(0, output[0])
err = shellutil.run_get_output(u"ls /not-exists")
self.assertNotEquals(0, err[0])
err = shellutil.run_get_output(u"ls 我")
self.assertNotEquals(0, err[0])
def test_it_should_log_the_command(self):
command = "echo hello world!"
with patch("azurelinuxagent.common.utils.shellutil.logger", autospec=True) as mock_logger:
shellutil.run_get_output(command)
self.assertEquals(mock_logger.verbose.call_count, 1)
args, kwargs = mock_logger.verbose.call_args
command_in_message = args[1]
self.assertEqual(command_in_message, command)
def create_and_start_unit(unit_filename, unit_contents):
try:
unit_path = os.path.join(UNIT_FILES_FILE_SYSTEM_PATH,
unit_filename)
fileutil.write_file(unit_path, unit_contents)
shellutil.run_get_output("systemctl daemon-reload")
shellutil.run_get_output(
"systemctl start {0}".format(unit_filename))
except Exception as e:
raise CGroupsException(
"Failed to create and start {0}. Error: {1}".format(
unit_filename, ustr(e)))
def process(self):
try:
RDMADeviceHandler.update_dat_conf(dapl_config_paths,
self.ipv4_addr)
skip_rdma_device = False
module_name = "hv_network_direct"
retcode, out = shellutil.run_get_output("modprobe -R %s" %
module_name,
chk_err=False)
if retcode == 0:
module_name = out.strip()
else:
logger.info(
"RDMA: failed to resolve module name. Use original name")
retcode, out = shellutil.run_get_output("modprobe %s" %
module_name)
if retcode != 0:
logger.error("RDMA: failed to load module %s" % module_name)
return
retcode, out = shellutil.run_get_output("modinfo %s" % module_name)
if retcode == 0:
version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out,
re.IGNORECASE)
if version:
v1 = int(version.groups(0)[0])
v2 = int(version.groups(0)[1])
if v1 > 4 or v1 == 4 and v2 > 0:
logger.info(
"Skip setting /dev/hvnd_rdma on 4.1 or later")
skip_rdma_device = True
else:
logger.info(
"RDMA: hv_network_direct driver version not present, assuming 4.0.x or older."
)
else:
logger.warn(
"RDMA: failed to get module info on hv_network_direct.")
if not skip_rdma_device:
RDMADeviceHandler.wait_rdma_device(
self.rdma_dev, self.device_check_timeout_sec,
self.device_check_interval_sec)
RDMADeviceHandler.write_rdma_config_to_device(
self.rdma_dev, self.ipv4_addr, self.mac_addr)
RDMADeviceHandler.update_network_interface(self.mac_addr,
self.ipv4_addr)
except Exception as e:
logger.error("RDMA: device processing failed: {0}".format(e))
def _get_net_info():
iface = ''
inet = ''
mac = ''
err, output = shellutil.run_get_output('dladm show-ether -p -o LINK',
chk_err=False)
if err:
raise OSUtilError("Can't find ether interface:{0}".format(output))
ifaces = output.split()
if not ifaces:
raise OSUtilError("Can't find ether interface.")
iface = ifaces[0]
err, output = shellutil.run_get_output(
'dladm show-phys -m -p -o address ' + iface, chk_err=False)
if err:
raise OSUtilError(
"Can't get mac address for interface:{0}".format(iface))
macs = output.split()
if not macs:
raise OSUtilError("Can't find mac address.")
mac = macs[0]
#
# It's possible for the output from "dladm show-phys" to output
# a mac address, such that each octet is not two characters
# (e.g. "2:dc:0:0:23:ff"). Certain parts of the agent expect
# each octet of the mac address to be two hex characters long,
# so we're forcing the address returned by this function to
# always have two character long octets.
#
mac = ":".join(
map(lambda x: "{0:02x}".format(int(x, 16)), mac.split(":")))
err, output = shellutil.run_get_output('ipadm show-addr -p -o addr ' +
iface + '/',
chk_err=False)
if err:
raise OSUtilError(
"Can't get ip address for interface:{0}".format(iface))
ips = output.split()
if not ips:
raise OSUtilError("Can't find ip address.")
ip = ips[0].split('/')[0]
logger.verbose("Interface info: ({0},{1},{2})", iface, ip, mac)
return iface, ip, mac
def chpasswd(self, username, password, crypt_id=6, salt_len=10):
if self.is_sys_user(username):
raise OSUtilError(("User {0} is a system user. "
"Will not set passwd.").format(username))
cmd = "echo -n {0}|encrypt".format(password)
ret, output = shellutil.run_get_output(cmd, log_cmd=False)
if ret != 0:
raise OSUtilError(("Failed to encrypt password for {0}: {1}"
"").format(username, output))
passwd_hash = output.strip()
cmd = "usermod -p '{0}' {1}".format(passwd_hash, username)
ret, output = shellutil.run_get_output(cmd, log_cmd=False)
if ret != 0:
raise OSUtilError(("Failed to set password for {0}: {1}"
"").format(username, output))
def chpasswd(self, username, password, crypt_id=6, salt_len=10):
if self.is_sys_user(username):
raise OSUtilError(("User {0} is a system user. "
"Will not set passwd.").format(username))
cmd = "echo -n {0}|encrypt".format(password)
ret, output = shellutil.run_get_output(cmd, log_cmd=False)
if ret != 0:
raise OSUtilError(("Failed to encrypt password for {0}: {1}"
"").format(username, output))
passwd_hash = output.strip()
cmd = "usermod -p '{0}' {1}".format(passwd_hash, username)
ret, output = shellutil.run_get_output(cmd, log_cmd=False)
if ret != 0:
raise OSUtilError(("Failed to set password for {0}: {1}"
"").format(username, output))
def mount_dvd(self, max_retry=6, chk_err=True, dvd_device=None, mount_point=None):
if dvd_device is None:
dvd_device = self.get_dvd_device()
if mount_point is None:
mount_point = conf.get_dvd_mount_point()
mountlist = shellutil.run_get_output("mount")[1]
existing = self.get_mount_point(mountlist, dvd_device)
if existing is not None: #Already mounted
logger.info("{0} is already mounted at {1}", dvd_device, existing)
return
if not os.path.isdir(mount_point):
os.makedirs(mount_point)
for retry in range(0, max_retry):
retcode = self.mount(dvd_device, mount_point, option="-o ro -t udf,iso9660",
chk_err=chk_err)
if retcode == 0:
logger.info("Successfully mounted dvd")
return
if retry < max_retry - 1:
logger.warn("Mount dvd failed: retry={0}, ret={1}", retry,
retcode)
time.sleep(5)
if chk_err:
raise OSUtilError("Failed to mount dvd.")
def get_firewall_dropped_packets(self, dst_ip=None):
# If a previous attempt failed, do not retry
global _enable_firewall
if not _enable_firewall:
return 0
try:
wait = self.get_firewall_will_wait()
rc, output = shellutil.run_get_output(FIREWALL_PACKETS.format(wait), log_cmd=False)
if rc == 3:
# Transient error that we ignore. This code fires every loop
# of the daemon (60m), so we will get the value eventually.
return 0
if rc != 0:
return -1
pattern = re.compile(PACKET_PATTERN.format(dst_ip))
for line in output.split('\n'):
m = pattern.match(line)
if m is not None:
return int(m.group(1))
return 0
except Exception as e:
_enable_firewall = False
logger.warn("Unable to retrieve firewall packets dropped"
"{0}".format(ustr(e)))
return -1
def mount(self, dvd, mount_point, option="", chk_err=True):
cmd = "mount {0} {1} {2}".format(option, dvd, mount_point)
retcode, err = shellutil.run_get_output(cmd, chk_err)
if retcode != 0:
detail = "[{0}] returned {1}: {2}".format(cmd, retcode, err)
err = detail
return retcode, err
def get_firewall_dropped_packets(self, dst_ip=None):
# If a previous attempt failed, do not retry
global _enable_firewall
if not _enable_firewall:
return 0
try:
wait = self.get_firewall_will_wait()
rc, output = shellutil.run_get_output(
FIREWALL_PACKETS.format(wait))
if rc != 0:
return -1
pattern = re.compile(PACKET_PATTERN.format(dst_ip))
for line in output.split('\n'):
m = pattern.match(line)
if m is not None:
return int(m.group(1))
return 0
except Exception as e:
_enable_firewall = False
logger.warn("Unable to retrieve firewall packets dropped"
"{0}".format(ustr(e)))
return -1
def restart_if(self, ifname, retries=None, wait=None):
logger.info(
'restarting {} (sort of, actually SIGHUPing dhcpcd)'.format(
ifname))
pid = self.get_dhcp_pid()
if pid != None: # pylint: disable=C0121
ret = shellutil.run_get_output('kill -HUP {}'.format(pid)) # pylint: disable=W0612
def process(self):
try:
RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
skip_rdma_device = False
retcode,out = shellutil.run_get_output("modinfo hv_network_direct")
if retcode == 0:
version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE)
if version:
v1 = int(version.groups(0)[0])
v2 = int(version.groups(0)[1])
if v1>4 or v1==4 and v2>0:
logger.info("Skip setting /dev/hvnd_rdma on 4.1 or later")
skip_rdma_device = True
else:
logger.info("RDMA: hv_network_direct driver version not present, assuming 4.0.x or older.")
else:
logger.warn("RDMA: failed to get module info on hv_network_direct.")
if not skip_rdma_device:
RDMADeviceHandler.wait_rdma_device(
self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
RDMADeviceHandler.write_rdma_config_to_device(
self.rdma_dev, self.ipv4_addr, self.mac_addr)
RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
except Exception as e:
logger.error("RDMA: device processing failed: {0}".format(e))
def restart_if(self, ifname):
logger.info(
'restarting {} (sort of, actually SIGHUPing dhcpcd)'.format(
ifname))
pid = self.get_dhcp_pid()
if pid != None:
ret = shellutil.run_get_output('kill -HUP {}'.format(pid))
def get_total_mem(self):
cmd = "grep MemTotal /proc/meminfo |awk '{print $2}'"
ret = shellutil.run_get_output(cmd)
if ret[0] == 0:
return int(ret[1])/1024
else:
raise OSUtilError("Failed to get total memory: {0}".format(ret[1]))
def update_iboip_interface(ipv4_addr, timeout_sec, check_interval_sec):
logger.info("Wait for ib0 become available")
total_retries = timeout_sec / check_interval_sec
n = 0
found_ib0 = None
while not found_ib0 and n < total_retries:
ret, output = shellutil.run_get_output("ifconfig -a")
if ret != 0:
raise Exception("Failed to list network interfaces")
found_ib0 = re.search("ib0", output, re.IGNORECASE)
if found_ib0:
break
time.sleep(check_interval_sec)
n += 1
if not found_ib0:
raise Exception("ib0 is not available")
netmask = 16
logger.info(
"RDMA: configuring IPv4 addr and netmask on ipoib interface")
addr = '{0}/{1}'.format(ipv4_addr, netmask)
if shellutil.run("ifconfig ib0 {0}".format(addr)) != 0:
raise Exception("Could set addr to {0} on ib0".format(addr))
logger.info("RDMA: ipoib address and netmask configured on interface")
def useradd(self, username, expiration=None, comment=None):
"""Create user account using tmsh
Our policy is to create two accounts when booting a BIG-IP instance.
The first account is the one that the user specified when they did
the instance creation. The second one is the admin account that is,
or should be, built in to the system.
:param username: The username that you want to add to the system
:param expiration: The expiration date to use. We do not use this
value.
:param comment: description of the account. We do not use this value.
"""
if self.get_userentry(username):
logger.info("User {0} already exists, skip useradd", username)
return None
cmd = "/usr/bin/tmsh create auth user %s partition-access add { all-partitions { role admin } } shell bash" % (username)
retcode, out = shellutil.run_get_output(cmd, log_cmd=True, chk_err=True)
if retcode != 0:
raise OSUtilError(
"Failed to create user account:{0}, retcode:{1}, output:{2}".format(username, retcode, out)
)
self._save_sys_config()
return retcode
def get_nic_state(self):
"""
Capture NIC state (IPv4 and IPv6 addresses plus link state).
:return: Dictionary of NIC state objects, with the NIC name as key
:rtype: dict(str,NetworkInformationCard)
"""
state = {}
status, output = shellutil.run_get_output("ip -o link", chk_err=False, log_cmd=False)
if status != 0:
logger.verbose("Could not fetch NIC link info; status {0}, {1}".format(status, output))
return {}
for entry in output.splitlines():
result = self.ip_command_output.match(entry)
if result:
name = result.group(1)
state[name] = NetworkInterfaceCard(name, result.group(2))
self._update_nic_state(state, "ip -o -f inet address", NetworkInterfaceCard.add_ipv4, "an IPv4 address")
self._update_nic_state(state, "ip -o -f inet6 address", NetworkInterfaceCard.add_ipv6, "an IPv6 address")
return state
def get_firewall_dropped_packets(self, dst_ip=None):
# If a previous attempt failed, do not retry
global _enable_firewall
if not _enable_firewall:
return 0
try:
wait = self.get_firewall_will_wait()
rc, output = shellutil.run_get_output(FIREWALL_PACKETS.format(wait), log_cmd=False)
if rc == 3:
# Transient error that we ignore. This code fires every loop
# of the daemon (60m), so we will get the value eventually.
return 0
if rc != 0:
return -1
pattern = re.compile(PACKET_PATTERN.format(dst_ip))
for line in output.split('\n'):
m = pattern.match(line)
if m is not None:
return int(m.group(1))
return 0
except Exception as e:
_enable_firewall = False
logger.warn("Unable to retrieve firewall packets dropped"
"{0}".format(ustr(e)))
return -1
def useradd(self, username, expiration=None, comment=None):
"""Create user account using tmsh
Our policy is to create two accounts when booting a BIG-IP instance.
The first account is the one that the user specified when they did
the instance creation. The second one is the admin account that is,
or should be, built in to the system.
:param username: The username that you want to add to the system
:param expiration: The expiration date to use. We do not use this
value.
:param comment: description of the account. We do not use this value.
"""
if self.get_userentry(username):
logger.info("User {0} already exists, skip useradd", username)
return None
cmd = "/usr/bin/tmsh create auth user %s partition-access add { all-partitions { role admin } } shell bash" % (
username)
retcode, out = shellutil.run_get_output(cmd,
log_cmd=True,
chk_err=True)
if retcode != 0:
raise OSUtilError(
"Failed to create user account:{0}, retcode:{1}, output:{2}".
format(username, retcode, out))
self._save_sys_config()
return retcode
def mount_dvd(self,
max_retry=6,
chk_err=True,
dvd_device=None,
mount_point=None):
if dvd_device is None:
dvd_device = self.get_dvd_device()
if mount_point is None:
mount_point = conf.get_dvd_mount_point()
mountlist = shellutil.run_get_output("mount")[1]
existing = self.get_mount_point(mountlist, dvd_device)
if existing is not None: #Already mounted
logger.info("{0} is already mounted at {1}", dvd_device, existing)
return
if not os.path.isdir(mount_point):
os.makedirs(mount_point)
for retry in range(0, max_retry):
retcode = self.mount(dvd_device,
mount_point,
option="-o ro -t udf,iso9660",
chk_err=chk_err)
if retcode == 0:
logger.info("Successfully mounted dvd")
return
if retry < max_retry - 1:
logger.warn("Mount dvd failed: retry={0}, ret={1}", retry,
retcode)
time.sleep(5)
if chk_err:
raise OSUtilError("Failed to mount dvd.")
def mount_dvd(self,
max_retry=6,
chk_err=True,
dvd_device=None,
mount_point=None,
sleep_time=5):
if dvd_device is None:
dvd_device = self.get_dvd_device()
if mount_point is None:
mount_point = conf.get_dvd_mount_point()
if not os.path.isdir(mount_point):
os.makedirs(mount_point)
for retry in range(0, max_retry):
retcode = self.mount(dvd_device,
mount_point,
option="-o ro -t udf",
chk_err=False)
if retcode == 0:
logger.info("Successfully mounted DVD")
return
if retry < max_retry - 1:
mountlist = shellutil.run_get_output("/sbin/mount")[1]
existing = self.get_mount_point(mountlist, dvd_device)
if existing is not None:
logger.info("{0} is mounted at {1}", dvd_device, existing)
return
logger.warn("Mount DVD failed: retry={0}, ret={1}", retry,
retcode)
time.sleep(sleep_time)
if chk_err:
raise OSUtilError("Failed to mount DVD.")
def test_download_ext_handler_pkg_stream(self, *args):
ext_uri = 'https://dcrdata.blob.core.windows.net/files/packer.zip'
tmp = tempfile.mkdtemp()
destination = os.path.join(tmp,
'test_download_ext_handler_pkg_stream.zip')
success = WireProtocol(wireserver_url).download_ext_handler_pkg(
ext_uri, destination)
self.assertTrue(success)
self.assertTrue(os.path.exists(destination))
# verify size
self.assertEqual(33193077, os.stat(destination).st_size)
# verify unzip
zipfile.ZipFile(destination).extractall(tmp)
packer = os.path.join(tmp, 'packer')
self.assertTrue(os.path.exists(packer))
fileutil.chmod(packer, os.stat(packer).st_mode | stat.S_IXUSR)
# verify unpacked size
self.assertEqual(105552030, os.stat(packer).st_size)
# execute, verify result
packer_version = '{0} --version'.format(packer)
rc, stdout = run_get_output(packer_version)
self.assertEqual(0, rc)
self.assertEqual('1.3.5\n', stdout)
def test_download_ext_handler_pkg_stream(self, *args):
ext_uri = 'https://dcrdata.blob.core.windows.net/files/packer.zip'
tmp = tempfile.mkdtemp()
destination = os.path.join(tmp, 'test_download_ext_handler_pkg_stream.zip')
success = WireProtocol(wireserver_url).download_ext_handler_pkg(ext_uri, destination)
self.assertTrue(success)
self.assertTrue(os.path.exists(destination))
# verify size
self.assertEqual(18380915, os.stat(destination).st_size)
# verify unzip
zipfile.ZipFile(destination).extractall(tmp)
packer = os.path.join(tmp, 'packer')
self.assertTrue(os.path.exists(packer))
fileutil.chmod(packer, os.stat(packer).st_mode | stat.S_IXUSR)
# verify unpacked size
self.assertEqual(87393596, os.stat(packer).st_size)
# execute, verify result
packer_version = '{0} --version'.format(packer)
rc, stdout = run_get_output(packer_version)
self.assertEqual(0, rc)
self.assertEqual('1.2.5\n', stdout)
def useradd(self, username, expiration=None, comment=None):
"""
Create user account with 'username'
"""
userentry = self.get_userentry(username)
if userentry is not None:
logger.info("User {0} already exists, skip useradd", username)
return
if expiration is not None:
cmd = "useradd -m {0} -s /bin/ash -e {1}".format(
username, expiration)
else:
cmd = "useradd -m {0} -s /bin/ash".format(username)
if not os.path.exists("/home"):
os.mkdir("/home")
if comment is not None:
cmd += " -c {0}".format(comment)
retcode, out = shellutil.run_get_output(cmd)
if retcode != 0:
raise OSUtilError(("Failed to create user account:{0}, "
"retcode:{1}, "
"output:{2}").format(username, retcode, out))
def load_atapiix_mod(self):
if self.is_atapiix_mod_loaded():
return
ret, kern_version = shellutil.run_get_output("uname -r")
if ret != 0:
raise Exception("Failed to call uname -r")
mod_path = os.path.join('/lib/modules', kern_version.strip('\n'),
'kernel/drivers/ata/ata_piix.ko')
if not os.path.isfile(mod_path):
raise Exception("Can't find module file:{0}".format(mod_path))
ret, output = shellutil.run_get_output("insmod " + mod_path)
if ret != 0:
raise Exception("Error calling insmod for ATAPI CD-ROM driver")
if not self.is_atapiix_mod_loaded(max_retry=3):
raise Exception("Failed to load ATAPI CD-ROM driver")
def mount(self, dvd, mount_point, option="", chk_err=True):
cmd = "mount {0} {1} {2}".format(option, dvd, mount_point)
retcode, err = shellutil.run_get_output(cmd, chk_err)
if retcode != 0:
detail = "[{0}] returned {1}: {2}".format(cmd, retcode, err)
err = detail
return retcode, err
def get_processor_cores(self):
ret = shellutil.run_get_output(
"grep 'processor.*:' /proc/cpuinfo |wc -l")
if ret[0] == 0:
return int(ret[1])
else:
raise OSUtilError("Failed to get processor cores")
def mount_dvd(self,
max_retry=6,
chk_err=True,
dvd_device=None,
mount_point=None,
sleep_time=5):
if dvd_device is None:
dvd_device = self.get_dvd_device()
if mount_point is None:
mount_point = conf.get_dvd_mount_point()
if not os.path.isdir(mount_point):
os.makedirs(mount_point)
for retry in range(0, max_retry):
retcode = self.mount(dvd_device,
mount_point,
option="-o ro -t udf",
chk_err=False)
if retcode == 0:
logger.info("Successfully mounted DVD")
return
if retry < max_retry - 1:
mountlist = shellutil.run_get_output("/sbin/mount")[1]
existing = self.get_mount_point(mountlist, dvd_device)
if existing is not None:
logger.info("{0} is mounted at {1}", dvd_device, existing)
return
logger.warn("Mount DVD failed: retry={0}, ret={1}", retry,
retcode)
time.sleep(sleep_time)
if chk_err:
raise OSUtilError("Failed to mount DVD.")
def get_total_mem(self):
cmd = "grep MemTotal /proc/meminfo |awk '{print $2}'"
ret = shellutil.run_get_output(cmd)
if ret[0] == 0:
return int(ret[1]) / 1024
else:
raise OSUtilError("Failed to get total memory: {0}".format(ret[1]))
def enable_firewall(self, dst_ip=None, uid=None):
# If a previous attempt failed, do not retry
global _enable_firewall
if not _enable_firewall:
return False
try:
if dst_ip is None or uid is None:
msg = "Missing arguments to enable_firewall"
logger.warn(msg)
raise Exception(msg)
wait = self.get_firewall_will_wait()
# If the DROP rule exists, make no changes
drop_rule = FIREWALL_DROP.format(wait, "C", dst_ip)
rc = shellutil.run(drop_rule, chk_err=False)
if rc == 0:
logger.verbose("Firewall appears established")
return True
elif rc == 2:
self.remove_firewall(dst_ip, uid)
msg = "please upgrade iptables to a version that supports the -C option"
logger.warn(msg)
raise Exception(msg)
# Otherwise, append both rules
accept_rule = FIREWALL_ACCEPT.format(wait, "A", dst_ip, uid)
drop_rule = FIREWALL_DROP.format(wait, "A", dst_ip)
if shellutil.run(accept_rule) != 0:
msg = "Unable to add ACCEPT firewall rule '{0}'".format(
accept_rule)
logger.warn(msg)
raise Exception(msg)
if shellutil.run(drop_rule) != 0:
msg = "Unable to add DROP firewall rule '{0}'".format(
drop_rule)
logger.warn(msg)
raise Exception(msg)
logger.info("Successfully added Azure fabric firewall rules")
rc, output = shellutil.run_get_output(FIREWALL_LIST.format(wait))
if rc == 0:
logger.info("Firewall rules:\n{0}".format(output))
else:
logger.warn(
"Listing firewall rules failed: {0}".format(output))
return True
except Exception as e:
_enable_firewall = False
logger.info("Unable to establish firewall -- "
"no further attempts will be made: "
"{0}".format(ustr(e)))
return False
def get_ssh_host_key_thumbprint(self, keypair_type):
cmd = "ssh-keygen -lf /etc/ssh/ssh_host_{0}_key.pub".format(keypair_type)
ret = shellutil.run_get_output(cmd)
if ret[0] == 0:
return ret[1].rstrip().split()[1].replace(':', '')
else:
raise ProvisionError(("Failed to generate ssh host key: "
"ret={0}, out= {1}").format(ret[0], ret[1]))
def get_total_mem(self):
ret, output = shellutil.run_get_output("sysctl -n hw.physmem")
if ret:
raise OSUtilError("Failed to get total memory: {0}".format(output))
try:
return int(output)/1024/1024
except ValueError:
raise OSUtilError("Failed to get total memory: {0}".format(output))
def set_scsi_disks_timeout(self, timeout):
if self._scsi_disks_timeout_set:
return
ret, output = shellutil.run_get_output('sysctl kern.cam.da.default_timeout={0}'.format(timeout))
if ret:
raise OSUtilError("Failed set SCSI disks timeout: {0}".format(output))
self._scsi_disks_timeout_set = True
def get_ssh_host_key_thumbprint(self, chk_err=True):
cmd = "ssh-keygen -lf {0}".format(conf.get_ssh_key_public_path())
ret = shellutil.run_get_output(cmd, chk_err=chk_err)
if ret[0] == 0:
return ret[1].rstrip().split()[1].replace(':', '')
else:
raise ProvisionError(("Failed to generate ssh host key: "
"ret={0}, out= {1}").format(ret[0], ret[1]))
def get_pubkey_from_crt(self, file_name):
if not os.path.exists(file_name):
raise IOError(errno.ENOENT, "File not found", file_name)
else:
cmd = "{0} x509 -in {1} -pubkey -noout".format(self.openssl_cmd,
file_name)
pub = shellutil.run_get_output(cmd)[1]
return pub
def get_ssh_host_key_thumbprint(self, chk_err=True):
cmd = "ssh-keygen -lf {0}".format(conf.get_ssh_key_public_path())
ret = shellutil.run_get_output(cmd, chk_err=chk_err)
if ret[0] == 0:
return ret[1].rstrip().split()[1].replace(':', '')
else:
raise ProvisionError(("Failed to generate ssh host key: "
"ret={0}, out= {1}").format(ret[0], ret[1]))
def get_pubkey_from_crt(self, file_name):
if not os.path.exists(file_name):
raise IOError("File not found: {0}", file_name)
cmd = "{0} x509 -in {1} -pubkey -noout".format(self.openssl_cmd,
file_name)
pub = shellutil.run_get_output(cmd)[1]
return pub
def get_pubkey_from_prv(self, file_name):
if not os.path.exists(file_name):
raise IOError("File not found: {0}", file_name)
cmd = "{0} rsa -in {1} -pubout 2>/dev/null".format(
self.openssl_cmd, file_name)
pub = shellutil.run_get_output(cmd)[1]
return pub
def load_atapiix_mod(self):
if self.is_atapiix_mod_loaded():
return
ret, kern_version = shellutil.run_get_output("uname -r")
if ret != 0:
raise Exception("Failed to call uname -r")
mod_path = os.path.join('/lib/modules',
kern_version.strip('\n'),
'kernel/drivers/ata/ata_piix.ko')
if not os.path.isfile(mod_path):
raise Exception("Can't find module file:{0}".format(mod_path))
ret, output = shellutil.run_get_output("insmod " + mod_path)
if ret != 0:
raise Exception("Error calling insmod for ATAPI CD-ROM driver")
if not self.is_atapiix_mod_loaded(max_retry=3):
raise Exception("Failed to load ATAPI CD-ROM driver")
def enable_firewall(self, dst_ip=None, uid=None):
# If a previous attempt failed, do not retry
global _enable_firewall
if not _enable_firewall:
return False
try:
if dst_ip is None or uid is None:
msg = "Missing arguments to enable_firewall"
logger.warn(msg)
raise Exception(msg)
wait = self.get_firewall_will_wait()
# If the DROP rule exists, make no changes
drop_rule = FIREWALL_DROP.format(wait, "C", dst_ip)
rc = shellutil.run(drop_rule, chk_err=False)
if rc == 0:
logger.verbose("Firewall appears established")
return True
elif rc == 2:
self.remove_firewall(dst_ip, uid)
msg = "please upgrade iptables to a version that supports the -C option"
logger.warn(msg)
raise Exception(msg)
# Otherwise, append both rules
accept_rule = FIREWALL_ACCEPT.format(wait, "A", dst_ip, uid)
drop_rule = FIREWALL_DROP.format(wait, "A", dst_ip)
if shellutil.run(accept_rule) != 0:
msg = "Unable to add ACCEPT firewall rule '{0}'".format(
accept_rule)
logger.warn(msg)
raise Exception(msg)
if shellutil.run(drop_rule) != 0:
msg = "Unable to add DROP firewall rule '{0}'".format(
drop_rule)
logger.warn(msg)
raise Exception(msg)
logger.info("Successfully added Azure fabric firewall rules")
rc, output = shellutil.run_get_output(FIREWALL_LIST.format(wait))
if rc == 0:
logger.info("Firewall rules:\n{0}".format(output))
else:
logger.warn("Listing firewall rules failed: {0}".format(output))
return True
except Exception as e:
_enable_firewall = False
logger.info("Unable to establish firewall -- "
"no further attempts will be made: "
"{0}".format(ustr(e)))
return False
def get_thumbprint_from_crt(self, file_name):
if not os.path.exists(file_name):
raise IOError(errno.ENOENT, "File not found", file_name)
else:
cmd = "{0} x509 -in {1} -fingerprint -noout".format(self.openssl_cmd,
file_name)
thumbprint = shellutil.run_get_output(cmd)[1]
thumbprint = thumbprint.rstrip().split('=')[1].replace(':', '').upper()
return thumbprint
def get_total_mem(self):
cmd = "sysctl hw.physmem |awk '{print $2}'"
ret, output = shellutil.run_get_output(cmd)
if ret:
raise OSUtilError("Failed to get total memory: {0}".format(output))
try:
return int(output)/1024/1024
except ValueError:
raise OSUtilError("Failed to get total memory: {0}".format(output))
def get_rdma_package_info(self):
"""
Returns the installed rdma package name or None
"""
ret, output = shellutil.run_get_output(
'rpm -q %s' % self.rdma_user_mode_package_name, chk_err=False)
if ret != 0:
return None
return output
def is_driver_loaded(self):
"""Check if the network module is loaded in kernel space"""
cmd = 'lsmod | grep ^%s' % self.driver_module_name
status, loaded_modules = shellutil.run_get_output(cmd)
logger.info('RDMA: Checking if the module loaded.')
if loaded_modules:
logger.info('RDMA: module loaded.')
return True
logger.info('RDMA: module not loaded.')
def get_processor_cores(self):
ret, output = shellutil.run_get_output("sysctl hw.ncpu |awk '{print $2}'")
if ret:
raise OSUtilError("Failed to get processor cores.")
try:
return int(output)
except ValueError:
raise OSUtilError("Failed to get total memory: {0}".format(output))
def is_selinux_enforcing(self):
"""
Calls shell command 'getenforce' and returns True if 'Enforcing'.
"""
if self.is_selinux_system():
output = shellutil.run_get_output("getenforce")[1]
return output.startswith("Enforcing")
else:
return False
def chpasswd(self, username, password, crypt_id=6, salt_len=10):
if self.is_sys_user(username):
raise OSUtilError(("User {0} is a system user. "
"Will not set passwd.").format(username))
passwd_hash = textutil.gen_password_hash(password, crypt_id, salt_len)
cmd = "echo '{0}'|pw usermod {1} -H 0 ".format(passwd_hash, username)
ret, output = shellutil.run_get_output(cmd, log_cmd=False)
if ret != 0:
raise OSUtilError(("Failed to set password for {0}: {1}"
"").format(username, output))
def chpasswd(self, username, password, crypt_id=6, salt_len=10):
cmd = "/usr/Firewall/sbin/fwpasswd -p {0}".format(password)
ret, output = shellutil.run_get_output(cmd, log_cmd=False)
if ret != 0:
raise OSUtilError(("Failed to set password for admin: {0}"
"").format(output))
# password set, activate webadmin and ssh access
shellutil.run('setconf /usr/Firewall/ConfigFiles/webadmin ACL any && ensl',
chk_err=False)
def test_get_total_mem(self):
"""
Validate the returned value matches to the one retrieved by invoking shell command
"""
cmd = "grep MemTotal /proc/meminfo |awk '{print $2}'"
ret = shellutil.run_get_output(cmd)
if ret[0] == 0:
self.assertEqual(int(ret[1]) / 1024, get_osutil().get_total_mem())
else:
self.fail("Cannot retrieve total memory using shell command.")
def test_get_processor_cores(self):
"""
Validate the returned value matches to the one retrieved by invoking shell command
"""
cmd = "grep 'processor.*:' /proc/cpuinfo |wc -l"
ret = shellutil.run_get_output(cmd)
if ret[0] == 0:
self.assertEqual(int(ret[1]), get_osutil().get_processor_cores())
else:
self.fail("Cannot retrieve number of process cores using shell command.")
