def collect_and_send_events(self):
event_list = TelemetryEventList()
event_dir = os.path.join(conf.get_lib_dir(), "events")
event_files = os.listdir(event_dir)
for event_file in event_files:
if not event_file.endswith(".tld"):
continue
event_file_path = os.path.join(event_dir, event_file)
try:
data_str = self.collect_event(event_file_path)
except EventError as e:
logger.error("{0}", e)
continue
try:
event = parse_event(data_str)
self.add_sysinfo(event)
event_list.events.append(event)
except (ValueError, ProtocolError) as e:
logger.warn("Failed to decode event file: {0}", e)
continue
if len(event_list.events) == 0:
return
try:
protocol = self.protocol_util.get_protocol()
protocol.report_event(event_list)
except ProtocolError as e:
logger.error("{0}", e)
def run(self):
# if provisioning is already done, return
provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
if os.path.isfile(provisioned):
logger.info("Provisioning already completed, skipping.")
return
thumbprint = None
# If provision is not enabled, report ready and then return
if not conf.get_provision_enabled():
logger.info("Provisioning is disabled, skipping.")
else:
logger.info("Running default provisioning handler")
try:
if not self.validate_cloud_init(is_expected=False):
raise ProvisionError("cloud-init appears to be running, "
"this is not expected, cannot continue")
logger.info("Copying ovf-env.xml")
ovf_env = self.protocol_util.copy_ovf_env()
self.protocol_util.get_protocol_by_file()
self.report_not_ready("Provisioning", "Starting")
logger.info("Starting provisioning")
self.provision(ovf_env)
thumbprint = self.reg_ssh_host_key()
self.osutil.restart_ssh_service()
self.report_event("Provision succeed", is_success=True)
except (ProtocolError, ProvisionError) as e:
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(ustr(e))
logger.error("Provisioning failed: {0}", ustr(e))
return
# write out provisioned file and report Ready
fileutil.write_file(provisioned, "")
self.report_ready(thumbprint)
logger.info("Provisioning complete")
def put_vm_status(self, status_blob, sas_url, config_blob_type=None):
"""
Try to upload the VM status via the host plugin /status channel
:param sas_url: the blob SAS url to pass to the host plugin
:param config_blob_type: the blob type from the extension config
:type status_blob: StatusBlob
"""
if not self.ensure_initialized():
raise ProtocolError("HostGAPlugin: HostGAPlugin is not available")
if status_blob is None or status_blob.vm_status is None:
raise ProtocolError("HostGAPlugin: Status blob was not provided")
logger.verbose("HostGAPlugin: Posting VM status")
try:
blob_type = status_blob.type if status_blob.type else config_blob_type
if blob_type == "BlockBlob":
self._put_block_blob_status(sas_url, status_blob)
else:
self._put_page_blob_status(sas_url, status_blob)
if not HostPluginProtocol.is_default_channel():
logger.info("HostGAPlugin: Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
except Exception as e:
message = "HostGAPlugin: Exception Put VM status: {0}".format(e)
logger.error(message)
from azurelinuxagent.common.event import WALAEventOperation, report_event
report_event(op=WALAEventOperation.ReportStatus,
is_success=False,
message=message)
logger.warn("HostGAPlugin: resetting default channel")
HostPluginProtocol.set_default_channel(False)
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing file.
To allow efficient execution, fallocate will be tried first. This includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate`` command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes < 0:
raise ValueError(nbytes)
if os.path.isfile(filename):
os.remove(filename)
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
with open(filename, 'w') as f:
try:
os.posix_fallocate(f.fileno(), 0, nbytes)
return 0
except:
# Not confident with this thing, just keep trying...
pass
# fallocate command
fn_sh = shellutil.quote((filename,))
ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(nbytes, fn_sh))
if ret == 0:
return ret
logger.info("fallocate unsuccessful, falling back to dd")
# dd fallback
dd_maxbs = 64 * 1024 ** 2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))
if ret == 0:
logger.info("dd successful")
else:
logger.error("dd unsuccessful")
return ret
def run(self):
# If provision is enabled, run default provision handler
if conf.get_provision_enabled():
logger.warn("Provisioning flag is enabled, this is not typical"
"in Ubuntu, please ensure your config is correct.")
super(UbuntuProvisionHandler, self).run()
return
provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
if os.path.isfile(provisioned):
logger.info("Provisioning already completed, skipping.")
return
logger.info("Running Ubuntu provisioning handler")
self.wait_for_ovfenv()
self.protocol_util.get_protocol()
self.report_not_ready("Provisioning", "Starting")
try:
thumbprint = self.wait_for_ssh_host_key()
fileutil.write_file(provisioned, "")
logger.info("Finished provisioning")
except ProvisionError as e:
logger.error("Provisioning failed: {0}", ustr(e))
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(ustr(e))
return
self.report_ready(thumbprint)
self.report_event("Provision succeed", is_success=True)
def get_status_file_path(self, extension=None):
path = None
seq_no = self.get_largest_seq_no()
# Issue 1116: use the sequence number from goal state where possible
if extension is not None and extension.sequenceNumber is not None:
try:
gs_seq_no = int(extension.sequenceNumber)
if gs_seq_no != seq_no:
add_event(AGENT_NAME,
version=CURRENT_VERSION,
op=WALAEventOperation.SequenceNumberMismatch,
is_success=False,
message="Goal state: {0}, disk: {1}".format(gs_seq_no, seq_no),
log_event=False)
seq_no = gs_seq_no
except ValueError:
logger.error('Sequence number [{0}] does not appear to be valid'.format(extension.sequenceNumber))
if seq_no > -1:
path = os.path.join(
self.get_status_dir(),
"{0}.status".format(seq_no))
return seq_no, path
def add_user(self, username, encrypted_password, account_expiration):
try:
expiration_date = (account_expiration + timedelta(days=1)).strftime(DATE_FORMAT)
logger.verbose("Adding user {0} with expiration date {1}"
.format(username, expiration_date))
self.os_util.useradd(username, expiration_date, REMOTE_ACCESS_ACCOUNT_COMMENT)
except OSError as oe:
logger.error("Error adding user {0}. {1}"
.format(username, oe.strerror))
return
except Exception as e:
logger.error("Error adding user {0}. {1}".format(username, ustr(e)))
return
try:
prv_key = os.path.join(conf.get_lib_dir(), TRANSPORT_PRIVATE_CERT)
pwd = self.cryptUtil.decrypt_secret(encrypted_password, prv_key)
self.os_util.chpasswd(username, pwd, conf.get_password_cryptid(), conf.get_password_crypt_salt_len())
self.os_util.conf_sudoer(username)
logger.info("User '{0}' added successfully with expiration in {1}"
.format(username, expiration_date))
return
except OSError as oe:
self.handle_failed_create(username, oe.strerror)
except Exception as e:
self.handle_failed_create(username, ustr(e))
def init_cgroups():
# Track metrics for the roll-up cgroup and for the agent cgroup
try:
CGroupsTelemetry.track_cgroup(CGroups.for_extension(""))
CGroupsTelemetry.track_agent()
except Exception as e:
logger.error("monitor: Exception tracking wrapper and agent: {0} [{1}]", e, traceback.format_exc())
def run(self):
# If provision is enabled, run default provision handler
if conf.get_provision_enabled():
logger.warn("Provisioning flag is enabled, which overrides using "
"cloud-init; running the default provisioning code")
super(CloudInitProvisionHandler, self).run()
return
try:
if super(CloudInitProvisionHandler, self).is_provisioned():
logger.info("Provisioning already completed, skipping.")
return
utc_start = datetime.utcnow()
logger.info("Running CloudInit provisioning handler")
self.wait_for_ovfenv()
self.protocol_util.get_protocol()
self.report_not_ready("Provisioning", "Starting")
thumbprint = self.wait_for_ssh_host_key()
self.write_provisioned()
logger.info("Finished provisioning")
self.report_ready(thumbprint)
self.report_event("Provisioning with cloud-init succeeded ({0}s)".format(self._get_uptime_seconds()),
is_success=True,
duration=elapsed_milliseconds(utc_start))
except ProvisionError as e:
msg = "Provisioning with cloud-init failed: {0} ({1}s)".format(ustr(e), self._get_uptime_seconds())
logger.error(msg)
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(msg)
return
def main(args=[]):
"""
Parse command line arguments, exit with usage() on error.
Invoke different methods according to different command
"""
if len(args) <= 0:
args = sys.argv[1:]
command, force, verbose = parse_args(args)
if command == "version":
version()
elif command == "help":
usage()
elif command == "start":
start()
else:
try:
agent = Agent(verbose)
if command == "deprovision+user":
agent.deprovision(force, deluser=True)
elif command == "deprovision":
agent.deprovision(force, deluser=False)
elif command == "provision":
agent.provision()
elif command == "register-service":
agent.register_service()
elif command == "daemon":
agent.daemon()
elif command == "run-exthandlers":
agent.run_exthandlers()
except Exception:
logger.error(u"Failed to run '{0}': {1}",
command,
traceback.format_exc())
def enable_swap(self, mount_point):
logger.info("Enable swap")
try:
size_mb = conf.get_resourcedisk_swap_size_mb()
self.create_swap_space(mount_point, size_mb)
except ResourceDiskError as e:
logger.error("Failed to enable swap {0}", e)
def add_metric(self, category, counter, instance, value, log_event=False):
"""
Create and save an event which contains a telemetry event.
:param str category: The category of metric (e.g. "cpu", "memory")
:param str counter: The specific metric within the category (e.g. "%idle")
:param str instance: For instanced metrics, the instance identifier (filesystem name, cpu core#, etc.)
:param value: Value of the metric
:param bool log_event: If true, log the collected metric in the agent log
"""
if log_event:
from azurelinuxagent.common.version import AGENT_NAME
message = "Metric {0}/{1} [{2}] = {3}".format(category, counter, instance, value)
_log_event(AGENT_NAME, "METRIC", message, 0)
event = TelemetryEvent(4, "69B669B9-4AF8-4C50-BDC4-6006FA76E975")
event.parameters.append(TelemetryEventParam('Category', category))
event.parameters.append(TelemetryEventParam('Counter', counter))
event.parameters.append(TelemetryEventParam('Instance', instance))
event.parameters.append(TelemetryEventParam('Value', value))
data = get_properties(event)
try:
self.save_event(json.dumps(data))
except EventError as e:
logger.error("{0}", e)
def reboot_system(self):
"""Reboot the system. This is required as the kernel module for
the rdma driver cannot be unloaded with rmmod"""
logger.info('RDMA: Rebooting system.')
ret = shellutil.run('shutdown -r now')
if ret != 0:
logger.error('RDMA: Failed to reboot the system')
def validate_dhcp_resp(request, response):
bytes_recv = len(response)
if bytes_recv < 0xF6:
logger.error("HandleDhcpResponse: Too few bytes received:{0}",
bytes_recv)
return False
logger.verbose("BytesReceived:{0}", hex(bytes_recv))
logger.verbose("DHCP response:{0}", hex_dump(response, bytes_recv))
# check transactionId, cookie, MAC address cookie should never mismatch
# transactionId and MAC address may mismatch if we see a response
# meant from another machine
if not compare_bytes(request, response, 0xEC, 4):
logger.verbose("Cookie not match:\nsend={0},\nreceive={1}",
hex_dump3(request, 0xEC, 4),
hex_dump3(response, 0xEC, 4))
raise DhcpError("Cookie in dhcp respones doesn't match the request")
if not compare_bytes(request, response, 4, 4):
logger.verbose("TransactionID not match:\nsend={0},\nreceive={1}",
hex_dump3(request, 4, 4),
hex_dump3(response, 4, 4))
raise DhcpError("TransactionID in dhcp respones "
"doesn't match the request")
if not compare_bytes(request, response, 0x1C, 6):
logger.verbose("Mac Address not match:\nsend={0},\nreceive={1}",
hex_dump3(request, 0x1C, 6),
hex_dump3(response, 0x1C, 6))
raise DhcpError("Mac Addr in dhcp respones "
"doesn't match the request")
def mount_cgroups(self):
try:
if not os.path.exists(_cgroup_path()):
fileutil.mkdir(_cgroup_path())
self.mount(device='cgroup_root',
mount_point=_cgroup_path(),
option="-t tmpfs",
chk_err=False)
elif not os.path.isdir(_cgroup_path()):
logger.error("Could not mount cgroups: ordinary file at {0}".format(_cgroup_path()))
return
for metric_hierarchy in ['cpu,cpuacct', 'memory']:
target_path = _cgroup_path(metric_hierarchy)
if not os.path.exists(target_path):
fileutil.mkdir(target_path)
self.mount(device=metric_hierarchy,
mount_point=target_path,
option="-t cgroup -o {0}".format(metric_hierarchy),
chk_err=False)
for metric_hierarchy in ['cpu', 'cpuacct']:
target_path = _cgroup_path(metric_hierarchy)
if not os.path.exists(target_path):
os.symlink(_cgroup_path('cpu,cpuacct'), target_path)
except Exception as e:
logger.error("Could not mount cgroups: {0}", ustr(e))
def change_partition_type(self, suppress_message, option_str):
"""
use sfdisk to change partition type.
First try with --part-type; if fails, fall back to -c
"""
command_to_use = '--part-type'
input = "sfdisk {0} {1} {2}".format(command_to_use, '-f' if suppress_message else '', option_str)
err_code, output = shellutil.run_get_output(input, chk_err=False, log_cmd=True)
# fall back to -c
if err_code != 0:
logger.info("sfdisk with --part-type failed [{0}], retrying with -c", err_code)
command_to_use = '-c'
input = "sfdisk {0} {1} {2}".format(command_to_use, '-f' if suppress_message else '', option_str)
err_code, output = shellutil.run_get_output(input, log_cmd=True)
if err_code == 0:
logger.info('{0} succeeded',
input)
else:
logger.error('{0} failed [{1}: {2}]',
input,
err_code,
output)
return err_code, output
def wireserver_route_exists(self):
"""
Determine whether a route to the known wireserver
ip already exists, and if so use that as the endpoint.
This is true when running in a virtual network.
:return: True if a route to KNOWN_WIRESERVER_IP exists.
"""
route_exists = False
logger.info("Test for route to {0}".format(KNOWN_WIRESERVER_IP))
try:
route_file = '/proc/net/route'
if os.path.exists(route_file) and \
KNOWN_WIRESERVER_IP_ENTRY in open(route_file).read():
# reset self.gateway and self.routes
# we do not need to alter the routing table
self.endpoint = KNOWN_WIRESERVER_IP
self.gateway = None
self.routes = None
route_exists = True
logger.info("Route to {0} exists".format(KNOWN_WIRESERVER_IP))
else:
logger.warn("No route exists to {0}".format(KNOWN_WIRESERVER_IP))
except Exception as e:
logger.error(
"Could not determine whether route exists to {0}: {1}".format(
KNOWN_WIRESERVER_IP, e))
return route_exists
def run(self):
#If provision is enabled, run default provision handler
if conf.get_provision_enabled():
super(UbuntuProvisionHandler, self).run()
return
logger.info("run Ubuntu provision handler")
provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
if os.path.isfile(provisioned):
return
logger.info("Waiting cloud-init to copy ovf-env.xml.")
self.wait_for_ovfenv()
protocol = self.protocol_util.get_protocol()
self.report_not_ready("Provisioning", "Starting")
logger.info("Sleep 15 seconds to prevent throttling")
time.sleep(15) #Sleep to prevent throttling
try:
logger.info("Wait for ssh host key to be generated.")
thumbprint = self.wait_for_ssh_host_key()
fileutil.write_file(provisioned, "")
logger.info("Finished provisioning")
except ProvisionError as e:
logger.error("Provision failed: {0}", e)
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(ustr(e))
return
self.report_ready(thumbprint)
self.report_event("Provision succeed", is_success=True)
def process(self):
try:
RDMADeviceHandler.update_dat_conf(dapl_config_paths, self.ipv4_addr)
skip_rdma_device = False
retcode,out = shellutil.run_get_output("modinfo hv_network_direct")
if retcode == 0:
version = re.search("version:\s+(\d+)\.(\d+)\.(\d+)\D", out, re.IGNORECASE)
if version:
v1 = int(version.groups(0)[0])
v2 = int(version.groups(0)[1])
if v1>4 or v1==4 and v2>0:
logger.info("Skip setting /dev/hvnd_rdma on 4.1 or later")
skip_rdma_device = True
else:
logger.info("RDMA: hv_network_direct driver version not present, assuming 4.0.x or older.")
else:
logger.warn("RDMA: failed to get module info on hv_network_direct.")
if not skip_rdma_device:
RDMADeviceHandler.wait_rdma_device(
self.rdma_dev, self.device_check_timeout_sec, self.device_check_interval_sec)
RDMADeviceHandler.write_rdma_config_to_device(
self.rdma_dev, self.ipv4_addr, self.mac_addr)
RDMADeviceHandler.update_network_interface(self.mac_addr, self.ipv4_addr)
except Exception as e:
logger.error("RDMA: device processing failed: {0}".format(e))
def get_rdma_version():
"""Retrieve the firmware version information from the system.
This depends on information provided by the Linux kernel."""
kvp_key_size = 512
kvp_value_size = 2048
driver_info_source = '/var/lib/hyperv/.kvp_pool_0'
base_kernel_err_msg = 'Kernel does not provide the necessary '
base_kernel_err_msg += 'information or the kvp daemon is not running.'
if not os.path.isfile(driver_info_source):
error_msg = 'RDMA: Source file "%s" does not exist. '
error_msg += base_kernel_err_msg
logger.error(error_msg % driver_info_source)
return
f = open(driver_info_source)
while True :
key = f.read(kvp_key_size)
value = f.read(kvp_value_size)
if key and value :
key_0 = key.split("\x00")[0]
value_0 = value.split("\x00")[0]
if key_0 == "NdDriverVersion" :
f.close()
return value_0
else :
break
f.close()
error_msg = 'RDMA: NdDriverVersion not found in "%s"'
logger.error(error_msg % driver_info_source)
return
def get_dhcp_lease_endpoint(self):
"""
OpenBSD has a sligthly different lease file format.
"""
endpoint = None
pathglob = '/var/db/dhclient.leases.{}'.format(self.get_first_if()[0])
HEADER_LEASE = "lease"
HEADER_OPTION = "option option-245"
HEADER_EXPIRE = "expire"
FOOTER_LEASE = "}"
FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S %Z"
logger.info("looking for leases in path [{0}]".format(pathglob))
for lease_file in glob.glob(pathglob):
leases = open(lease_file).read()
if HEADER_OPTION in leases:
cached_endpoint = None
has_option_245 = False
expired = True # assume expired
for line in leases.splitlines():
if line.startswith(HEADER_LEASE):
cached_endpoint = None
has_option_245 = False
expired = True
elif HEADER_OPTION in line:
try:
ipaddr = line.split(" ")[-1].strip(";").split(":")
cached_endpoint = \
".".join(str(int(d, 16)) for d in ipaddr)
has_option_245 = True
except ValueError:
logger.error("could not parse '{0}'".format(line))
elif HEADER_EXPIRE in line:
if "never" in line:
expired = False
else:
try:
expire_string = line.split(
" ", 4)[-1].strip(";")
expire_date = datetime.datetime.strptime(
expire_string, FORMAT_DATETIME)
if expire_date > datetime.datetime.utcnow():
expired = False
except ValueError:
logger.error("could not parse expiry token "
"'{0}'".format(line))
elif FOOTER_LEASE in line:
logger.info("dhcp entry:{0}, 245:{1}, expired: {2}"
.format(cached_endpoint, has_option_245, expired))
if not expired and cached_endpoint is not None and has_option_245:
endpoint = cached_endpoint
logger.info("found endpoint [{0}]".format(endpoint))
# we want to return the last valid entry, so
# keep searching
if endpoint is not None:
logger.info("cached endpoint found [{0}]".format(endpoint))
else:
logger.info("cached endpoint not found")
return endpoint
def del_account(self, username):
if self.is_sys_user(username):
logger.error("{0} is a system user. Will not delete it.",
username)
shellutil.run("> /var/run/utmp")
shellutil.run("userdel -r " + username)
self.conf_sudoer(username, remove=True)
def run_get_output(cmd, chk_err=True, log_cmd=True, expected_errors=[]):
"""
Wrapper for subprocess.check_output.
Execute 'cmd'. Returns return code and STDOUT, trapping expected
exceptions.
Reports exceptions to Error if chk_err parameter is True
"""
if log_cmd:
logger.verbose(u"Command: [{0}]", cmd)
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True)
output = ustr(output,
encoding='utf-8',
errors="backslashreplace")
except subprocess.CalledProcessError as e:
output = ustr(e.output,
encoding='utf-8',
errors="backslashreplace")
if chk_err:
msg = u"Command: [{0}], " \
u"return code: [{1}], " \
u"result: [{2}]".format(cmd, e.returncode, output)
if e.returncode in expected_errors:
logger.info(msg)
else:
logger.error(msg)
return e.returncode, output
except Exception as e:
if chk_err:
logger.error(u"Command [{0}] raised unexpected exception: [{1}]"
.format(cmd, ustr(e)))
return -1, ustr(e)
return 0, output
def _download(self):
for uri in self.pkg.uris:
if not HostPluginProtocol.is_default_channel() and self._fetch(uri.uri):
break
elif self.host is not None and self.host.ensure_initialized():
if not HostPluginProtocol.is_default_channel():
logger.warn("Download unsuccessful, falling back to host plugin")
else:
logger.verbose("Using host plugin as default channel")
uri, headers = self.host.get_artifact_request(uri.uri, self.host.manifest_uri)
if self._fetch(uri, headers=headers):
if not HostPluginProtocol.is_default_channel():
logger.verbose("Setting host plugin as default channel")
HostPluginProtocol.set_default_channel(True)
break
else:
logger.warn("Host plugin download unsuccessful")
else:
logger.error("No download channels available")
if not os.path.isfile(self.get_agent_pkg_path()):
msg = u"Unable to download Agent {0} from any URI".format(self.name)
add_event(
AGENT_NAME,
op=WALAEventOperation.Download,
version=CURRENT_VERSION,
is_success=False,
message=msg)
raise UpdateError(msg)
return
def get_primary_interface(self):
"""
Get the name of the primary interface, which is the one with the
default route attached to it; if there are multiple default routes,
the primary has the lowest Metric.
:return: the interface which has the default route
"""
# from linux/route.h
RTF_GATEWAY = 0x02
DEFAULT_DEST = "00000000"
hdr_iface = "Iface"
hdr_dest = "Destination"
hdr_flags = "Flags"
hdr_metric = "Metric"
idx_iface = -1
idx_dest = -1
idx_flags = -1
idx_metric = -1
primary = None
primary_metric = None
if not self.disable_route_warning:
logger.info("examine /proc/net/route for primary interface")
with open('/proc/net/route') as routing_table:
idx = 0
for header in filter(lambda h: len(h) > 0, routing_table.readline().strip(" \n").split("\t")):
if header == hdr_iface:
idx_iface = idx
elif header == hdr_dest:
idx_dest = idx
elif header == hdr_flags:
idx_flags = idx
elif header == hdr_metric:
idx_metric = idx
idx = idx + 1
for entry in routing_table.readlines():
route = entry.strip(" \n").split("\t")
if route[idx_dest] == DEFAULT_DEST and int(route[idx_flags]) & RTF_GATEWAY == RTF_GATEWAY:
metric = int(route[idx_metric])
iface = route[idx_iface]
if primary is None or metric < primary_metric:
primary = iface
primary_metric = metric
if primary is None:
primary = ''
if not self.disable_route_warning:
with open('/proc/net/route') as routing_table_fh:
routing_table_text = routing_table_fh.read()
logger.error('could not determine primary interface, '
'please ensure /proc/net/route is correct:\n'
'{0}'.format(routing_table_text))
self.disable_route_warning = True
else:
logger.info('primary interface is [{0}]'.format(primary))
self.disable_route_warning = False
return primary
def _log_event(name, op, message, duration, is_success=True):
global _EVENT_MSG
message = _encode_message(op, message)
if not is_success:
logger.error(_EVENT_MSG, name, op, message, duration)
else:
logger.info(_EVENT_MSG, name, op, message, duration)
def save_protocol(self, protocol_name):
"""
Save protocol endpoint
"""
protocol_file_path = os.path.join(conf.get_lib_dir(), PROTOCOL_FILE_NAME)
try:
fileutil.write_file(protocol_file_path, protocol_name)
except IOError as e:
logger.error("Failed to save protocol endpoint: {0}", e)
def report_ready(self, thumbprint=None):
status = ProvisionStatus(status="Ready")
status.properties.certificateThumbprint = thumbprint
try:
protocol = self.protocol_util.get_protocol()
protocol.report_provision_status(status)
except ProtocolError as e:
logger.error("Reporting Ready failed: {0}", e)
self.report_event(ustr(e))
def _mkdir(self, timestamp):
d = self.history_dir(timestamp)
try:
fileutil.mkdir(d, mode=0o700)
return True
except IOError as e:
logger.error("{0} : {1}".format(d, e.strerror))
return False
def report_not_ready(self, sub_status, description):
status = ProvisionStatus(status="NotReady", subStatus=sub_status,
description=description)
try:
protocol = self.protocol_util.get_protocol()
protocol.report_provision_status(status)
except ProtocolError as e:
logger.error("Reporting NotReady failed: {0}", e)
self.report_event(ustr(e))
def conf_sshd(self, disable_password):
logger.error('"conf_sshd" not supported.')
def process_events(self):
"""
Returns a list of events that need to be sent to the telemetry pipeline and deletes the corresponding files
from the events directory.
"""
event_directory_full_path = os.path.join(conf.get_lib_dir(),
EVENTS_DIRECTORY)
event_files = os.listdir(event_directory_full_path)
debug_info = CollectOrReportEventDebugInfo(
operation=CollectOrReportEventDebugInfo.OP_COLLECT)
for event_file in event_files:
try:
match = EVENT_FILE_REGEX.search(event_file)
if match is None:
continue
event_file_path = os.path.join(event_directory_full_path,
event_file)
try:
logger.verbose("Processing event file: {0}",
event_file_path)
with open(event_file_path, "rb") as event_fd:
event_data = event_fd.read().decode("utf-8")
event = parse_event(event_data)
# "legacy" events are events produced by previous versions of the agent (<= 2.2.46) and extensions;
# they do not include all the telemetry fields, so we add them here
is_legacy_event = match.group('agent_event') is None
if is_legacy_event:
# We'll use the file creation time for the event's timestamp
event_file_creation_time_epoch = os.path.getmtime(
event_file_path)
event_file_creation_time = datetime.datetime.fromtimestamp(
event_file_creation_time_epoch)
if event.is_extension_event():
_CollectAndEnqueueEvents._trim_legacy_extension_event_parameters(
event)
CollectTelemetryEventsHandler.add_common_params_to_telemetry_event(
event, event_file_creation_time)
else:
_CollectAndEnqueueEvents._update_legacy_agent_event(
event, event_file_creation_time)
self._send_telemetry_events_handler.enqueue_event(event)
finally:
# Todo: We should delete files after ensuring that we sent the data to Wireserver successfully
# from our end rather than deleting first and sending later. This is to ensure the data reliability
# of the agent telemetry pipeline.
os.remove(event_file_path)
except ServiceStoppedError as stopped_error:
logger.error(
"Unable to enqueue events as service stopped: {0}, skipping events collection"
.format(ustr(stopped_error)))
except UnicodeError as uni_err:
debug_info.update_unicode_error(uni_err)
except Exception as error:
debug_info.update_op_error(error)
debug_info.report_debug_info()
def mount_resource_disk(self, mount_point):
device = self.osutil.device_for_ide_port(1)
if device is None:
raise ResourceDiskError("unable to detect disk topology")
logger.info('Resource disk device {0} found.', device)
# 2. Get partition
device = "/dev/{0}".format(device)
partition = device + "1"
logger.info('Resource disk partition {0} found.', partition)
# 3. Mount partition
mount_list = shellutil.run_get_output("mount")[1]
existing = self.osutil.get_mount_point(mount_list, device)
if existing:
logger.info("Resource disk [{0}] is already mounted [{1}]",
partition, existing)
return existing
try:
fileutil.mkdir(mount_point, mode=0o755)
except OSError as ose:
msg = "Failed to create mount point " \
"directory [{0}]: {1}".format(mount_point, ose)
logger.error(msg)
raise ResourceDiskError(msg=msg, inner=ose)
force_option = 'F'
if self.fs == 'xfs':
force_option = 'f'
mkfs_string = "mkfs.{0} -{2} {1}".format(self.fs, partition,
force_option)
# Compare to the Default mount_resource_disk, we don't check for GPT that is not supported on OpenWRT
ret = self.change_partition_type(suppress_message=True,
option_str="{0} 1 -n".format(device))
ptype = ret[1].strip()
if ptype == "7" and self.fs != "ntfs":
logger.info("The partition is formatted with ntfs, updating "
"partition type to 83")
self.change_partition_type(suppress_message=False,
option_str="{0} 1 83".format(device))
self.reread_partition_table(device)
logger.info("Format partition [{0}]", mkfs_string)
shellutil.run(mkfs_string)
else:
logger.info("The partition type is {0}", ptype)
mount_options = conf.get_resourcedisk_mountoptions()
mount_string = self.get_mount_string(mount_options, partition,
mount_point)
attempts = 5
while not os.path.exists(partition) and attempts > 0:
logger.info("Waiting for partition [{0}], {1} attempts remaining",
partition, attempts)
sleep(5)
attempts -= 1
if not os.path.exists(partition):
raise ResourceDiskError(
"Partition was not created [{0}]".format(partition))
if os.path.ismount(mount_point):
logger.warn("Disk is already mounted on {0}", mount_point)
else:
# Some kernels seem to issue an async partition re-read after a
# command invocation. This causes mount to fail if the
# partition re-read is not complete by the time mount is
# attempted. Seen in CentOS 7.2. Force a sequential re-read of
# the partition and try mounting.
logger.info("Mounting after re-reading partition info.")
self.reread_partition_table(device)
logger.info("Mount resource disk [{0}]", mount_string)
ret, output = shellutil.run_get_output(mount_string)
if ret:
logger.warn(
"Failed to mount resource disk. "
"Attempting to format and retry mount. [{0}]", output)
shellutil.run(mkfs_string)
ret, output = shellutil.run_get_output(mount_string)
if ret:
raise ResourceDiskError("Could not mount {0} "
"after syncing partition table: "
"[{1}] {2}".format(
partition, ret, output))
logger.info("Resource disk {0} is mounted at {1} with {2}", device,
mount_point, self.fs)
return mount_point
def chowner(path, owner):
if not os.path.exists(path):
logger.error("Path does not exist: {0}".format(path))
else:
owner_info = pwd.getpwnam(owner)
os.chown(path, owner_info[2], owner_info[3])
def mount_resource_disk(self, mount_point):
device = self.osutil.device_for_ide_port(1)
if device is None:
raise ResourceDiskError("unable to detect disk topology")
device = "/dev/{0}".format(device)
partition = device + "1"
mount_list = shellutil.run_get_output("mount")[1]
existing = self.osutil.get_mount_point(mount_list, device)
if existing:
logger.info("Resource disk [{0}] is already mounted [{1}]",
partition,
existing)
return existing
try:
fileutil.mkdir(mount_point, mode=0o755)
except OSError as ose:
msg = "Failed to create mount point " \
"directory [{0}]: {1}".format(mount_point, ose)
logger.error(msg)
raise ResourceDiskError(msg=msg, inner=ose)
logger.info("Examining partition table")
ret = shellutil.run_get_output("parted {0} print".format(device))
if ret[0]:
raise ResourceDiskError("Could not determine partition info for "
"{0}: {1}".format(device, ret[1]))
force_option = 'F'
if self.fs == 'xfs':
force_option = 'f'
mkfs_string = "mkfs.{0} -{2} {1}".format(self.fs, partition, force_option)
if "gpt" in ret[1]:
logger.info("GPT detected, finding partitions")
parts = [x for x in ret[1].split("\n") if
re.match("^\s*[0-9]+", x)]
logger.info("Found {0} GPT partition(s).", len(parts))
if len(parts) > 1:
logger.info("Removing old GPT partitions")
for i in range(1, len(parts) + 1):
logger.info("Remove partition {0}", i)
shellutil.run("parted {0} rm {1}".format(device, i))
logger.info("Creating new GPT partition")
shellutil.run("parted {0} mkpart primary 0% 100%".format(device))
logger.info("Format partition [{0}]", mkfs_string)
shellutil.run(mkfs_string)
else:
logger.info("GPT not detected, determining filesystem")
ret = self.change_partition_type(suppress_message=True, option_str="{0} 1 -n".format(device))
ptype = ret[1].strip()
if ptype == "7" and self.fs != "ntfs":
logger.info("The partition is formatted with ntfs, updating "
"partition type to 83")
self.change_partition_type(suppress_message=False, option_str="{0} 1 83".format(device))
self.reread_partition_table(device)
logger.info("Format partition [{0}]", mkfs_string)
shellutil.run(mkfs_string)
else:
logger.info("The partition type is {0}", ptype)
mount_options = conf.get_resourcedisk_mountoptions()
mount_string = self.get_mount_string(mount_options,
partition,
mount_point)
attempts = 5
while not os.path.exists(partition) and attempts > 0:
logger.info("Waiting for partition [{0}], {1} attempts remaining",
partition,
attempts)
sleep(5)
attempts -= 1
if not os.path.exists(partition):
raise ResourceDiskError("Partition was not created [{0}]".format(partition))
logger.info("Mount resource disk [{0}]", mount_string)
ret, output = shellutil.run_get_output(mount_string, chk_err=False)
# if the exit code is 32, then the resource disk can be already mounted
if ret == 32 and output.find("is already mounted") != -1:
logger.warn("Could not mount resource disk: {0}", output)
elif ret != 0:
# Some kernels seem to issue an async partition re-read after a
# 'parted' command invocation. This causes mount to fail if the
# partition re-read is not complete by the time mount is
# attempted. Seen in CentOS 7.2. Force a sequential re-read of
# the partition and try mounting.
logger.warn("Failed to mount resource disk. "
"Retry mounting after re-reading partition info.")
self.reread_partition_table(device)
ret, output = shellutil.run_get_output(mount_string)
if ret:
logger.warn("Failed to mount resource disk. "
"Attempting to format and retry mount. [{0}]",
output)
shellutil.run(mkfs_string)
ret, output = shellutil.run_get_output(mount_string)
if ret:
raise ResourceDiskError("Could not mount {0} "
"after syncing partition table: "
"[{1}] {2}".format(partition,
ret,
output))
logger.info("Resource disk {0} is mounted at {1} with {2}",
device,
mount_point,
self.fs)
return mount_point
def useradd(self, username, expiration=None):
logger.error('"useradd" not supported.')
def del_account(self, username):
if self.is_sys_user(username):
logger.error("{0} is a system user. Will not delete it.", username)
self._run_command_without_raising(["touch", "/var/run/utmp"])
self._run_command_without_raising(["userdel", "-r", username])
self.conf_sudoer(username, remove=True)
def get_endpoint_from_leases_path(pathglob):
"""
Try to discover and decode the wireserver endpoint in the
specified dhcp leases path.
:param pathglob: The path containing dhcp lease files
:return: The endpoint if available, otherwise None
"""
endpoint = None
HEADER_LEASE = "lease"
HEADER_OPTION = "option unknown-245"
HEADER_DNS = "option domain-name-servers"
HEADER_EXPIRE = "expire"
FOOTER_LEASE = "}"
FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S"
logger.info("looking for leases in path [{0}]".format(pathglob))
for lease_file in glob.glob(pathglob):
leases = open(lease_file).read()
if HEADER_OPTION in leases:
cached_endpoint = None
has_option_245 = False
expired = True # assume expired
for line in leases.splitlines():
if line.startswith(HEADER_LEASE):
cached_endpoint = None
has_option_245 = False
expired = True
elif HEADER_DNS in line:
cached_endpoint = line.replace(HEADER_DNS,
'').strip(" ;")
elif HEADER_OPTION in line:
has_option_245 = True
elif HEADER_EXPIRE in line:
if "never" in line:
expired = False
else:
try:
expire_string = line.split(" ",
4)[-1].strip(";")
expire_date = datetime.datetime.strptime(
expire_string, FORMAT_DATETIME)
if expire_date > datetime.datetime.utcnow():
expired = False
except:
logger.error(
"could not parse expiry token '{0}'".
format(line))
elif FOOTER_LEASE in line:
logger.info(
"dhcp entry:{0}, 245:{1}, expired:{2}".format(
cached_endpoint, has_option_245, expired))
if not expired and cached_endpoint is not None and has_option_245:
endpoint = cached_endpoint
logger.info(
"found endpoint [{0}]".format(endpoint))
# we want to return the last valid entry, so
# keep searching
if endpoint is not None:
logger.info("cached endpoint found [{0}]".format(endpoint))
else:
logger.info("cached endpoint not found")
return endpoint
def del_account(self, username):
if self.is_sys_user(username):
logger.error("{0} is a system user. Will not delete it.", username)
shellutil.run("> /var/run/utmp")
shellutil.run("userdel -f -r " + username)
self.conf_sudoer(username, remove=True)
def install_driver(self):
#Install the appropriate driver package for the RDMA firmware
nd_version = self.get_rdma_version()
if not nd_version:
logger.error(
"RDMA: Could not determine firmware version. No driver will be installed"
)
return
#replace . with _, we are looking for number like 144_0
nd_version = re.sub('\.', '_', nd_version)
#Check to see if we need to reconfigure driver
status, module_name = shellutil.run_get_output(
'modprobe -R hv_network_direct', chk_err=False)
if status != 0:
logger.info(
"RDMA: modprobe -R hv_network_direct failed. Use module name hv_network_direct"
)
module_name = "hv_network_direct"
else:
module_name = module_name.strip()
logger.info("RDMA: current RDMA driver %s nd_version %s" %
(module_name, nd_version))
if module_name == 'hv_network_direct_%s' % nd_version:
logger.info(
"RDMA: driver is installed and ND version matched. Skip reconfiguring driver"
)
return
#Reconfigure driver if one is available
status, output = shellutil.run_get_output(
'modinfo hv_network_direct_%s' % nd_version)
if status == 0:
logger.info(
"RDMA: driver with ND version is installed. Link to module name"
)
self.update_modprobed_conf(nd_version)
return
#Driver not found. We need to check to see if we need to update kernel
if not conf.enable_rdma_update():
logger.info("RDMA: driver update is disabled. Skip kernel update")
return
status, output = shellutil.run_get_output('uname -r')
if status != 0:
return
if not re.search('-azure$', output):
logger.error("RDMA: skip driver update on non-Azure kernel")
return
kernel_version = re.sub('-azure$', '', output)
kernel_version = re.sub('-', '.', kernel_version)
#Find the new kernel package version
status, output = shellutil.run_get_output('apt-get update')
if status != 0:
return
status, output = shellutil.run_get_output(
'apt-cache show --no-all-versions linux-azure')
if status != 0:
return
r = re.search('Version: (\S+)', output)
if not r:
logger.error("RDMA: version not found in package linux-azure.")
return
package_version = r.groups()[0]
#Remove the ending .<upload number> after <ABI number>
package_version = re.sub("\.\d+$", "", package_version)
logger.info('RDMA: kernel_version=%s package_version=%s' %
(kernel_version, package_version))
kernel_version_array = [int(x) for x in kernel_version.split('.')]
package_version_array = [int(x) for x in package_version.split('.')]
if kernel_version_array < package_version_array:
logger.info(
"RDMA: newer version available, update kernel and reboot")
status, output = shellutil.run_get_output(
'apt-get -y install linux-azure')
if status:
logger.error("RDMA: kernel update failed")
return
self.reboot_system()
else:
logger.error(
"RDMA: no kernel update is avaiable for ND version %s" %
nd_version)
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing
file.
To allow efficient execution, fallocate will be tried first. This
includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate``
command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform
single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes < 0:
raise ValueError(nbytes)
if os.path.isfile(filename):
os.remove(filename)
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
with open(filename, 'w') as f:
try:
os.posix_fallocate(f.fileno(), 0, nbytes)
return 0
except:
# Not confident with this thing, just keep trying...
pass
# fallocate command
fn_sh = shellutil.quote((filename, ))
ret = shellutil.run(u"umask 0077 && fallocate -l {0} {1}".format(
nbytes, fn_sh))
if ret == 0:
return ret
logger.info("fallocate unsuccessful, falling back to dd")
# dd fallback
dd_maxbs = 64 * 1024**2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} " \
"conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))
if ret == 0:
logger.info("dd successful")
else:
logger.error("dd unsuccessful")
return ret
def mkfile(self, filename, nbytes):
"""
Create a non-sparse file of that size. Deletes and replaces existing
file.
To allow efficient execution, fallocate will be tried first. This
includes
``os.posix_fallocate`` on Python 3.3+ (unix) and the ``fallocate``
command
in the popular ``util-linux{,-ng}`` package.
A dd fallback will be tried too. When size < 64M, perform
single-pass dd.
Otherwise do two-pass dd.
"""
if not isinstance(nbytes, int):
nbytes = int(nbytes)
if nbytes <= 0:
raise ResourceDiskError("Invalid swap size [{0}]".format(nbytes))
if os.path.isfile(filename):
os.remove(filename)
# If file system is xfs, use dd right away as we have been reported that
# swap enabling fails in xfs fs when disk space is allocated with
# fallocate
ret = 0
fn_sh = shellutil.quote((filename,))
if self.fs != 'xfs':
# os.posix_fallocate
if sys.version_info >= (3, 3):
# Probable errors:
# - OSError: Seen on Cygwin, libc notimpl?
# - AttributeError: What if someone runs this under...
fd = None
try:
fd = os.open(
filename,
os.O_CREAT | os.O_WRONLY | os.O_EXCL,
stat.S_IRUSR | stat.S_IWUSR)
os.posix_fallocate(fd, 0, nbytes)
return 0
except BaseException:
# Not confident with this thing, just keep trying...
pass
finally:
if fd is not None:
os.close(fd)
# fallocate command
ret = shellutil.run(
u"umask 0077 && fallocate -l {0} {1}".format(nbytes, fn_sh))
if ret == 0:
return ret
logger.info("fallocate unsuccessful, falling back to dd")
# dd fallback
dd_maxbs = 64 * 1024 ** 2
dd_cmd = "umask 0077 && dd if=/dev/zero bs={0} count={1} " \
"conv=notrunc of={2}"
blocks = int(nbytes / dd_maxbs)
if blocks > 0:
ret = shellutil.run(dd_cmd.format(dd_maxbs, blocks, fn_sh)) << 8
remains = int(nbytes % dd_maxbs)
if remains > 0:
ret += shellutil.run(dd_cmd.format(remains, 1, fn_sh))
if ret == 0:
logger.info("dd successful")
else:
logger.error("dd unsuccessful")
return ret
def conf_sudoer(self, username, nopasswd=False, remove=False):
logger.error('"conf_sudoer" not supported.')
def chpasswd(self, username, password, crypt_id=6, salt_len=10):
logger.error('"chpasswd" not supported.')
def get_dhcp_lease_endpoint(self):
"""
OpenBSD has a sligthly different lease file format.
"""
endpoint = None
pathglob = '/var/db/dhclient.leases.{}'.format(self.get_first_if()[0])
HEADER_LEASE = "lease"
HEADER_OPTION = "option option-245"
HEADER_EXPIRE = "expire"
FOOTER_LEASE = "}"
FORMAT_DATETIME = "%Y/%m/%d %H:%M:%S %Z"
logger.info("looking for leases in path [{0}]".format(pathglob))
for lease_file in glob.glob(pathglob):
leases = open(lease_file).read()
if HEADER_OPTION in leases:
cached_endpoint = None
has_option_245 = False
expired = True # assume expired
for line in leases.splitlines():
if line.startswith(HEADER_LEASE):
cached_endpoint = None
has_option_245 = False
expired = True
elif HEADER_OPTION in line:
try:
ipaddr = line.split(" ")[-1].strip(";").split(":")
cached_endpoint = \
".".join(str(int(d, 16)) for d in ipaddr)
has_option_245 = True
except ValueError:
logger.error("could not parse '{0}'".format(line))
elif HEADER_EXPIRE in line:
if "never" in line:
expired = False
else:
try:
expire_string = line.split(" ",
4)[-1].strip(";")
expire_date = datetime.datetime.strptime(
expire_string, FORMAT_DATETIME)
if expire_date > datetime.datetime.utcnow():
expired = False
except ValueError:
logger.error("could not parse expiry token "
"'{0}'".format(line))
elif FOOTER_LEASE in line:
logger.info(
"dhcp entry:{0}, 245:{1}, expired: {2}".format(
cached_endpoint, has_option_245, expired))
if not expired and cached_endpoint is not None and has_option_245:
endpoint = cached_endpoint
logger.info(
"found endpoint [{0}]".format(endpoint))
# we want to return the last valid entry, so
# keep searching
if endpoint is not None:
logger.info("cached endpoint found [{0}]".format(endpoint))
else:
logger.info("cached endpoint not found")
return endpoint
def reboot_system(self):
logger.info('Rebooting system')
ret = shellutil.run('reboot')
if ret != 0:
logger.error('Failed to reboot the system')
def _save_sys_config(self):
cmd = "/usr/bin/tmsh save sys config"
rc = shellutil.run(cmd)
if rc != 0:
logger.error("WARNING: Cannot save sys config on 1st boot.")
return rc
def chmod(path, mode):
if not os.path.exists(path):
logger.error("Path does not exist: {0}".format(path))
else:
os.chmod(path, mode)
def install_driver(self):
"""Install the driver. This is distribution specific and must
be overwritten in the child implementation."""
logger.error('RDMAHandler.install_driver not implemented')
def del_account(self, username):
if self.is_sys_user(username):
logger.error("{0} is a system user. Will not delete it.", username)
self._run_command_without_raising(['touch', '/var/run/utx.active'])
self._run_command_without_raising(['rmuser', '-y', username])
self.conf_sudoer(username, remove=True)
def install_driver(self):
"""Install the appropriate driver package for the RDMA firmware"""
fw_version = self.get_rdma_version()
if not fw_version:
error_msg = 'RDMA: Could not determine firmware version. '
error_msg += 'Therefore, no driver will be installed.'
logger.error(error_msg)
return
zypper_install = 'zypper -n in %s'
zypper_install_noref = 'zypper -n --no-refresh in %s'
zypper_lock = 'zypper addlock %s'
zypper_remove = 'zypper -n rm %s'
zypper_search = 'zypper -n se -s %s'
zypper_unlock = 'zypper removelock %s'
package_name = 'dummy'
# Figure out the kernel that is running to find the proper kmp
cmd = 'uname -r'
status, kernel_release = shellutil.run_get_output(cmd)
if 'default' in kernel_release:
package_name = 'msft-rdma-kmp-default'
info_msg = 'RDMA: Detected kernel-default'
logger.info(info_msg)
elif 'azure' in kernel_release:
package_name = 'msft-rdma-kmp-azure'
info_msg = 'RDMA: Detected kernel-azure'
logger.info(info_msg)
else:
error_msg = 'RDMA: Could not detect kernel build, unable to '
error_msg += 'load kernel module. Kernel release: "%s"'
logger.error(error_msg % kernel_release)
return
cmd = zypper_search % package_name
status, repo_package_info = shellutil.run_get_output(cmd)
driver_package_versions = []
driver_package_installed = False
for entry in repo_package_info.split('\n'):
if package_name in entry:
sections = entry.split('|')
if len(sections) < 4:
error_msg = 'RDMA: Unexpected output from"%s": "%s"'
logger.error(error_msg % (cmd, entry))
continue
installed = sections[0].strip()
version = sections[3].strip()
driver_package_versions.append(version)
if fw_version in version and installed.startswith('i'):
info_msg = 'RDMA: Matching driver package "%s-%s" '
info_msg += 'is already installed, nothing to do.'
logger.info(info_msg % (package_name, version))
return True
if installed.startswith('i'):
# A driver with a different version is installed
driver_package_installed = True
cmd = zypper_unlock % package_name
result = shellutil.run(cmd)
info_msg = 'Driver with different version installed '
info_msg += 'unlocked package "%s".'
logger.info(info_msg % (package_name))
# If we get here the driver package is installed but the
# version doesn't match or no package is installed
requires_reboot = False
if driver_package_installed:
# Unloading the particular driver with rmmod does not work
# We have to reboot after the new driver is installed
if self.is_driver_loaded():
info_msg = 'RDMA: Currently loaded driver does not match the '
info_msg += 'firmware implementation, reboot will be required.'
logger.info(info_msg)
requires_reboot = True
logger.info("RDMA: removing package %s" % package_name)
cmd = zypper_remove % package_name
shellutil.run(cmd)
logger.info("RDMA: removed package %s" % package_name)
logger.info("RDMA: looking for fw version %s in packages" % fw_version)
for entry in driver_package_versions:
if fw_version not in entry:
logger.info("Package '%s' is not a match." % entry)
else:
logger.info("Package '%s' is a match. Installing." % entry)
complete_name = '%s-%s' % (package_name, entry)
cmd = zypper_install % complete_name
result = shellutil.run(cmd)
if result:
error_msg = 'RDMA: Failed install of package "%s" '
error_msg += 'from available repositories.'
logger.error(error_msg % complete_name)
msg = 'RDMA: Successfully installed "%s" from '
msg += 'configured repositories'
logger.info(msg % complete_name)
# Lock the package so it does not accidentally get updated
cmd = zypper_lock % package_name
result = shellutil.run(cmd)
info_msg = 'Applied lock to "%s"' % package_name
logger.info(info_msg)
if not self.load_driver_module() or requires_reboot:
self.reboot_system()
return True
else:
logger.info("RDMA: No suitable match in repos. Trying local.")
local_packages = glob.glob('/opt/microsoft/rdma/*.rpm')
for local_package in local_packages:
logger.info("Examining: %s" % local_package)
if local_package.endswith('.src.rpm'):
continue
if (package_name in local_package
and fw_version in local_package):
logger.info("RDMA: Installing: %s" % local_package)
cmd = zypper_install_noref % local_package
result = shellutil.run(cmd)
if result and result != 106:
error_msg = 'RDMA: Failed install of package "%s" '
error_msg += 'from local package cache'
logger.error(error_msg % local_package)
break
msg = 'RDMA: Successfully installed "%s" from '
msg += 'local package cache'
logger.info(msg % (local_package))
# Lock the package so it does not accidentally get updated
cmd = zypper_lock % package_name
result = shellutil.run(cmd)
info_msg = 'Applied lock to "%s"' % package_name
logger.info(info_msg)
if not self.load_driver_module() or requires_reboot:
self.reboot_system()
return True
else:
error_msg = 'Unable to find driver package that matches '
error_msg += 'RDMA firmware version "%s"' % fw_version
logger.error(error_msg)
return
def stop_mgmt_service(self):
logger.error('"stop_mgmt_service" not supported.')
def del_root_password(self):
logger.error('"del_root_password" not supported.')
def install_driver(self):
"""Install the appropriate driver package for the RDMA firmware"""
fw_version = RDMAHandler.get_rdma_version()
if not fw_version:
error_msg = 'RDMA: Could not determine firmware version. '
error_msg += 'Therefore, no driver will be installed.'
logger.error(error_msg)
return
zypper_install = 'zypper -n in %s'
zypper_remove = 'zypper -n rm %s'
zypper_search = 'zypper se -s %s'
package_name = 'msft-rdma-kmp-default'
cmd = zypper_search % package_name
status, repo_package_info = shellutil.run_get_output(cmd)
driver_package_versions = []
driver_package_installed = False
for entry in repo_package_info.split('\n'):
if package_name in entry:
sections = entry.split('|')
if len(sections) < 4:
error_msg = 'RDMA: Unexpected output from"%s": "%s"'
logger.error(error_msg % (cmd, entry))
continue
installed = sections[0].strip()
version = sections[3].strip()
driver_package_versions.append(version)
if fw_version in version and installed == 'i':
info_msg = 'RDMA: Matching driver package "%s-%s" '
info_msg += 'is already installed, nothing to do.'
logger.info(info_msg % (package_name, version))
return True
if installed == 'i':
driver_package_installed = True
# If we get here the driver package is installed but the
# version doesn't match or no package is installed
requires_reboot = False
if driver_package_installed:
# Unloading the particular driver with rmmod does not work
# We have to reboot after the new driver is installed
if self.is_driver_loaded():
info_msg = 'RDMA: Currently loaded driver does not match the '
info_msg += 'firmware implementation, reboot will be required.'
logger.info(info_msg)
requires_reboot = True
logger.info("RDMA: removing package %s" % package_name)
cmd = zypper_remove % package_name
shellutil.run(cmd)
logger.info("RDMA: removed package %s" % package_name)
logger.info("RDMA: looking for fw version %s in packages" % fw_version)
for entry in driver_package_versions:
if not fw_version in version:
logger.info("Package '%s' is not a match." % entry)
else:
logger.info("Package '%s' is a match. Installing." % entry)
complete_name = '%s-%s' % (package_name, version)
cmd = zypper_install % complete_name
result = shellutil.run(cmd)
if result:
error_msg = 'RDMA: Failed install of package "%s" '
error_msg += 'from available repositories.'
logger.error(error_msg % complete_name)
msg = 'RDMA: Successfully installed "%s" from '
msg += 'configured repositories'
logger.info(msg % complete_name)
self.load_driver_module()
if requires_reboot:
self.reboot_system()
return True
else:
logger.info("RDMA: No suitable match in repos. Trying local.")
local_packages = glob.glob('/opt/microsoft/rdma/*.rpm')
for local_package in local_packages:
logger.info("Examining: %s" % local_package)
if local_package.endswith('.src.rpm'):
continue
if (package_name in local_package
and fw_version in local_package):
logger.info("RDMA: Installing: %s" % local_package)
cmd = zypper_install % local_package
result = shellutil.run(cmd)
if result:
error_msg = 'RDMA: Failed install of package "%s" '
error_msg += 'from local package cache'
logger.error(error_msg % local_package)
break
msg = 'RDMA: Successfully installed "%s" from '
msg += 'local package cache'
logger.info(msg % (local_package))
self.load_driver_module()
if requires_reboot:
self.reboot_system()
return True
else:
error_msg = 'Unable to find driver package that matches '
error_msg += 'RDMA firmware version "%s"' % fw_version
logger.error(error_msg)
return
def read_route_table():
"""
Return a list of strings comprising the route table as in the Linux /proc/net/route format. The input taken is from FreeBSDs
`netstat -rn -f inet` command. Here is what the function does in detail:
1. Runs `netstat -rn -f inet` which outputs a column formatted list of ipv4 routes in priority order like so:
> Routing tables
>
> Internet:
> Destination Gateway Flags Refs Use Netif Expire
> default 61.221.xx.yy UGS 0 247 em1
> 10 10.10.110.5 UGS 0 50 em0
> 10.10.110/26 link#1 UC 0 0 em0
> 10.10.110.5 00:1b:0d:e6:58:40 UHLW 2 0 em0 1145
> 61.221.xx.yy/29 link#2 UC 0 0 em1
> 61.221.xx.yy 00:1b:0d:e6:57:c0 UHLW 2 0 em1 1055
> 61.221.xx/24 link#2 UC 0 0 em1
> 127.0.0.1 127.0.0.1 UH 0 0 lo0
2. Convert it to an array of lines that resemble an equivalent /proc/net/route content on a Linux system like so:
> Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT
> gre828 00000000 00000000 0001 0 0 0 000000F8 0 0 0
> ens160 00000000 FE04700A 0003 0 0 100 00000000 0 0 0
> gre828 00000008 00000000 0001 0 0 0 000000FE 0 0 0
> ens160 0004700A 00000000 0001 0 0 100 00FFFFFF 0 0 0
> gre828 2504700A 00000000 0005 0 0 0 FFFFFFFF 0 0 0
> gre828 3704700A 00000000 0005 0 0 0 FFFFFFFF 0 0 0
> gre828 4104700A 00000000 0005 0 0 0 FFFFFFFF 0 0 0
:return: Entries in the ipv4 route priority list from `netstat -rn -f inet` in the linux `/proc/net/route` style
:rtype: list(str)
"""
def _get_netstat_rn_ipv4_routes():
"""
Runs `netstat -rn -f inet` and parses its output and returns a list of routes where the key is the column name
and the value is the value in the column, stripped of leading and trailing whitespace.
:return: List of dictionaries representing routes in the ipv4 route priority list from `netstat -rn -f inet`
:rtype: list(dict)
"""
cmd = ["netstat", "-rn", "-f", "inet"]
output = shellutil.run_command(cmd, log_error=True)
output_lines = output.split("\n")
if len(output_lines) < 3:
raise OSUtilError(
"`netstat -rn -f inet` output seems to be empty")
output_lines = [line.strip() for line in output_lines if line]
if "Internet:" not in output_lines:
raise OSUtilError(
"`netstat -rn -f inet` output seems to contain no ipv4 routes"
)
route_header_line = output_lines.index("Internet:") + 1
# Parse the file structure and left justify the routes
route_start_line = route_header_line + 1
route_line_length = max(
[len(line) for line in output_lines[route_header_line:]])
netstat_route_list = [
line.ljust(route_line_length)
for line in output_lines[route_start_line:]
]
# Parse the headers
_route_headers = output_lines[route_header_line].split()
n_route_headers = len(_route_headers)
route_columns = {}
for i in range(0, n_route_headers - 1):
route_columns[_route_headers[i]] = (
output_lines[route_header_line].index(_route_headers[i]),
(output_lines[route_header_line].index(
_route_headers[i + 1]) - 1))
route_columns[_route_headers[n_route_headers - 1]] = (
output_lines[route_header_line].index(
_route_headers[n_route_headers - 1]), None)
# Parse the routes
netstat_routes = []
n_netstat_routes = len(netstat_route_list)
for i in range(0, n_netstat_routes):
netstat_route = {}
for column in route_columns:
netstat_route[column] = netstat_route_list[
i][route_columns[column][0]:route_columns[column]
[1]].strip()
netstat_route["Metric"] = n_netstat_routes - i
netstat_routes.append(netstat_route)
# Return the Sections
return netstat_routes
def _ipv4_ascii_address_to_hex(ipv4_ascii_address):
"""
Converts an IPv4 32bit address from its ASCII notation (ie. 127.0.0.1) to an 8 digit padded hex notation
(ie. "0100007F") string.
:return: 8 character long hex string representation of the IP
:rtype: string
"""
# Raises socket.error if the IP is not a valid IPv4
return "%08X" % int(
binascii.hexlify(
struct.pack(
"!I",
struct.unpack(
"=I",
socket.inet_pton(socket.AF_INET,
ipv4_ascii_address))[0])), 16)
def _ipv4_cidr_mask_to_hex(ipv4_cidr_mask):
"""
Converts an subnet mask from its CIDR integer notation (ie. 32) to an 8 digit padded hex notation
(ie. "FFFFFFFF") string representing its bitmask form.
:return: 8 character long hex string representation of the IP
:rtype: string
"""
return "{0:08x}".format(
struct.unpack(
"=I",
struct.pack("!I", (0xffffffff << (32 - ipv4_cidr_mask))
& 0xffffffff))[0]).upper()
def _ipv4_cidr_destination_to_hex(destination):
"""
Converts an destination address from its CIDR notation (ie. 127.0.0.1/32 or default or localhost) to an 8
digit padded hex notation (ie. "0100007F" or "00000000" or "0100007F") string and its subnet bitmask
also in hex (FFFFFFFF).
:return: tuple of 8 character long hex string representation of the IP and 8 character long hex string representation of the subnet mask
:rtype: tuple(string, int)
"""
destination_ip = "0.0.0.0"
destination_subnetmask = 32
if destination != "default":
if destination == "localhost":
destination_ip = "127.0.0.1"
else:
destination_ip = destination.split("/")
if len(destination_ip) > 1:
destination_subnetmask = int(destination_ip[1])
destination_ip = destination_ip[0]
hex_destination_ip = _ipv4_ascii_address_to_hex(destination_ip)
hex_destination_subnetmask = _ipv4_cidr_mask_to_hex(
destination_subnetmask)
return hex_destination_ip, hex_destination_subnetmask
def _try_ipv4_gateway_to_hex(gateway):
"""
If the gateway is an IPv4 address, return its IP in hex, else, return "00000000"
:return: 8 character long hex string representation of the IP of the gateway
:rtype: string
"""
try:
return _ipv4_ascii_address_to_hex(gateway)
except socket.error:
return "00000000"
def _ascii_route_flags_to_bitmask(ascii_route_flags):
"""
Converts route flags to a bitmask of their equivalent linux/route.h values.
:return: integer representation of a 16 bit mask
:rtype: int
"""
bitmask_flags = 0
RTF_UP = 0x0001
RTF_GATEWAY = 0x0002
RTF_HOST = 0x0004
RTF_DYNAMIC = 0x0010
if "U" in ascii_route_flags:
bitmask_flags |= RTF_UP
if "G" in ascii_route_flags:
bitmask_flags |= RTF_GATEWAY
if "H" in ascii_route_flags:
bitmask_flags |= RTF_HOST
if "S" not in ascii_route_flags:
bitmask_flags |= RTF_DYNAMIC
return bitmask_flags
def _freebsd_netstat_rn_route_to_linux_proc_net_route(netstat_route):
"""
Converts a single FreeBSD `netstat -rn -f inet` route to its equivalent /proc/net/route line. ie:
> default 0.0.0.0 UGS 0 247 em1
to
> em1 00000000 00000000 0003 0 0 0 FFFFFFFF 0 0 0
:return: string representation of the equivalent /proc/net/route line
:rtype: string
"""
network_interface = netstat_route["Netif"]
hex_destination_ip, hex_destination_subnetmask = _ipv4_cidr_destination_to_hex(
netstat_route["Destination"])
hex_gateway = _try_ipv4_gateway_to_hex(netstat_route["Gateway"])
bitmask_flags = _ascii_route_flags_to_bitmask(
netstat_route["Flags"])
dummy_refcount = 0
dummy_use = 0
route_metric = netstat_route["Metric"]
dummy_mtu = 0
dummy_window = 0
dummy_irtt = 0
return "{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}\t{9}\t{10}".format(
network_interface, hex_destination_ip, hex_gateway,
bitmask_flags, dummy_refcount, dummy_use, route_metric,
hex_destination_subnetmask, dummy_mtu, dummy_window,
dummy_irtt)
linux_style_route_file = [
"Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT"
]
try:
netstat_routes = _get_netstat_rn_ipv4_routes()
# Make sure the `netstat -rn -f inet` contains columns for Netif, Destination, Gateway and Flags which are needed to convert
# to the Linux Format
if len(netstat_routes) > 0:
missing_headers = []
if "Netif" not in netstat_routes[0]:
missing_headers.append("Netif")
if "Destination" not in netstat_routes[0]:
missing_headers.append("Destination")
if "Gateway" not in netstat_routes[0]:
missing_headers.append("Gateway")
if "Flags" not in netstat_routes[0]:
missing_headers.append("Flags")
if missing_headers:
raise KeyError(
"`netstat -rn -f inet` output is missing columns required to convert to the Linux /proc/net/route format; columns are [{0}]"
.format(missing_headers))
# Parse the Netstat IPv4 Routes
for netstat_route in netstat_routes:
try:
linux_style_route = _freebsd_netstat_rn_route_to_linux_proc_net_route(
netstat_route)
linux_style_route_file.append(linux_style_route)
except Exception:
# Skip the route
continue
except Exception as e:
logger.error("Cannot read route table [{0}]", ustr(e))
return linux_style_route_file
def del_account(self, username):
if self.is_sys_user(username):
logger.error("{0} is a system user. Will not delete it.", username)
shellutil.run('> /var/run/utx.active')
shellutil.run('rmuser -y ' + username)
self.conf_sudoer(username, remove=True)
def http_request(method,
url,
data,
headers=None,
max_retry=3,
chk_proxy=False):
"""
Sending http request to server
On error, sleep 10 and retry max_retry times.
"""
host, port, secure, rel_uri = _parse_url(url)
# Check proxy
proxy_host, proxy_port = (None, None)
if chk_proxy:
proxy_host, proxy_port = get_http_proxy()
# If httplib module is not built with ssl support. Fallback to http
if secure and not hasattr(httpclient, "HTTPSConnection"):
logger.warn("httplib is not built with ssl support")
secure = False
# If httplib module doesn't support https tunnelling. Fallback to http
if secure and proxy_host is not None and proxy_port is not None \
and not hasattr(httpclient.HTTPSConnection, "set_tunnel"):
logger.warn("httplib does not support https tunnelling "
"(new in python 2.7)")
secure = False
logger.verbose("HTTP method: [{0}]", method)
logger.verbose("HTTP host: [{0}]", host)
logger.verbose("HTTP uri: [{0}]", rel_uri)
logger.verbose("HTTP port: [{0}]", port)
logger.verbose("HTTP data: [{0}]", data)
logger.verbose("HTTP secure: [{0}]", secure)
logger.verbose("HTTP headers: [{0}]", headers)
logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)
for retry in range(0, max_retry):
try:
resp = _http_request(method,
host,
rel_uri,
port=port,
data=data,
secure=secure,
headers=headers,
proxy_host=proxy_host,
proxy_port=proxy_port)
logger.verbose("HTTP response status: [{0}]", resp.status)
return resp
except httpclient.HTTPException as e:
logger.warn('HTTPException: [{0}]', e)
except IOError as e:
logger.warn('IOError: [{0}]', e)
if retry < max_retry - 1:
logger.info("Retry {0}", retry)
time.sleep(RETRY_WAITING_INTERVAL)
else:
logger.error("All retries failed")
if url is not None and len(url) > 100:
url_log = url[0:100] # In case the url is too long
else:
url_log = url
raise HttpError("HTTPError: {0} {1}".format(method, url_log))
