def run(self):
#If provision is enabled, run default provision handler
if conf.get_provision_enabled():
super(UbuntuProvisionHandler, self).run()
return
logger.info("run Ubuntu provision handler")
provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
if os.path.isfile(provisioned):
return
logger.info("Waiting cloud-init to copy ovf-env.xml.")
self.wait_for_ovfenv()
protocol = self.protocol_util.get_protocol()
self.report_not_ready("Provisioning", "Starting")
logger.info("Sleep 15 seconds to prevent throttling")
time.sleep(15) #Sleep to prevent throttling
try:
logger.info("Wait for ssh host key to be generated.")
thumbprint = self.wait_for_ssh_host_key()
fileutil.write_file(provisioned, "")
logger.info("Finished provisioning")
except ProvisionError as e:
logger.error("Provision failed: {0}", e)
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(ustr(e))
return
self.report_ready(thumbprint)
self.report_event("Provision succeed", is_success=True)
def run(self):
# if provisioning is already done, return
provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
if os.path.isfile(provisioned):
logger.info("Provisioning already completed, skipping.")
return
thumbprint = None
# If provision is not enabled, report ready and then return
if not conf.get_provision_enabled():
logger.info("Provisioning is disabled, skipping.")
else:
logger.info("Running default provisioning handler")
try:
if not self.validate_cloud_init(is_expected=False):
raise ProvisionError("cloud-init appears to be running, "
"this is not expected, cannot continue")
logger.info("Copying ovf-env.xml")
ovf_env = self.protocol_util.copy_ovf_env()
self.protocol_util.get_protocol_by_file()
self.report_not_ready("Provisioning", "Starting")
logger.info("Starting provisioning")
self.provision(ovf_env)
thumbprint = self.reg_ssh_host_key()
self.osutil.restart_ssh_service()
self.report_event("Provision succeed", is_success=True)
except (ProtocolError, ProvisionError) as e:
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(ustr(e))
logger.error("Provisioning failed: {0}", ustr(e))
return
# write out provisioned file and report Ready
fileutil.write_file(provisioned, "")
self.report_ready(thumbprint)
logger.info("Provisioning complete")
def conf_sudoer(self, username, nopasswd=False, remove=False):
sudoers_dir = conf.get_sudoers_dir()
sudoers_wagent = os.path.join(sudoers_dir, 'waagent')
if not remove:
# for older distros create sudoers.d
if not os.path.isdir(sudoers_dir):
sudoers_file = os.path.join(sudoers_dir, '../sudoers')
# create the sudoers.d directory
os.mkdir(sudoers_dir)
# add the include of sudoers.d to the /etc/sudoers
sudoers = '\n#includedir ' + sudoers_dir + '\n'
fileutil.append_file(sudoers_file, sudoers)
sudoer = None
if nopasswd:
sudoer = "{0} ALL=(ALL) NOPASSWD: ALL\n".format(username)
else:
sudoer = "{0} ALL=(ALL) ALL\n".format(username)
fileutil.append_file(sudoers_wagent, sudoer)
fileutil.chmod(sudoers_wagent, 0o440)
else:
#Remove user from sudoers
if os.path.isfile(sudoers_wagent):
try:
content = fileutil.read_file(sudoers_wagent)
sudoers = content.split("\n")
sudoers = [x for x in sudoers if username not in x]
fileutil.write_file(sudoers_wagent, "\n".join(sudoers))
except IOError as e:
raise OSUtilError("Failed to remove sudoer: {0}".format(e))
def set_handler_state(self, handler_state):
state_dir = self.get_conf_dir()
try:
state_file = os.path.join(state_dir, "HandlerState")
fileutil.write_file(state_file, handler_state)
except IOError as e:
self.logger.error("Failed to set state: {0}", e)
def run(self):
# If provision is enabled, run default provision handler
if conf.get_provision_enabled():
logger.warn("Provisioning flag is enabled, this is not typical"
"in Ubuntu, please ensure your config is correct.")
super(UbuntuProvisionHandler, self).run()
return
provisioned = os.path.join(conf.get_lib_dir(), "provisioned")
if os.path.isfile(provisioned):
logger.info("Provisioning already completed, skipping.")
return
logger.info("Running Ubuntu provisioning handler")
self.wait_for_ovfenv()
self.protocol_util.get_protocol()
self.report_not_ready("Provisioning", "Starting")
try:
thumbprint = self.wait_for_ssh_host_key()
fileutil.write_file(provisioned, "")
logger.info("Finished provisioning")
except ProvisionError as e:
logger.error("Provisioning failed: {0}", ustr(e))
self.report_not_ready("ProvisioningFailed", ustr(e))
self.report_event(ustr(e))
return
self.report_ready(thumbprint)
self.report_event("Provision succeed", is_success=True)
def deploy_ssh_pubkey(self, username, pubkey):
"""
Deploy authorized_key
"""
path, thumbprint, value = pubkey
if path is None:
raise OSUtilError("Public key path is None")
crytputil = CryptUtil(conf.get_openssl_cmd())
path = self._norm_path(path)
dir_path = os.path.dirname(path)
fileutil.mkdir(dir_path, mode=0o700, owner=username)
if value is not None:
if not value.startswith("ssh-"):
raise OSUtilError("Bad public key: {0}".format(value))
fileutil.write_file(path, value)
elif thumbprint is not None:
lib_dir = conf.get_lib_dir()
crt_path = os.path.join(lib_dir, thumbprint + '.crt')
if not os.path.isfile(crt_path):
raise OSUtilError("Can't find {0}.crt".format(thumbprint))
pub_path = os.path.join(lib_dir, thumbprint + '.pub')
pub = crytputil.get_pubkey_from_crt(crt_path)
fileutil.write_file(pub_path, pub)
self.set_selinux_context(pub_path,
'unconfined_u:object_r:ssh_home_t:s0')
self.openssl_to_openssh(pub_path, path)
fileutil.chmod(pub_path, 0o600)
else:
raise OSUtilError("SSH public key Fingerprint and Value are None")
self.set_selinux_context(path, 'unconfined_u:object_r:ssh_home_t:s0')
fileutil.chowner(path, username)
fileutil.chmod(path, 0o644)
def _write_pid_file(self):
pid_files = self._get_pid_files()
pid_dir, pid_name, pid_re = self._get_pid_parts()
previous_pid_file = None \
if len(pid_files) <= 0 \
else pid_files[-1]
pid_index = -1 \
if previous_pid_file is None \
else int(pid_re.match(os.path.basename(previous_pid_file)).group(1))
pid_file = os.path.join(pid_dir, "{0}_{1}".format(pid_index+1, pid_name))
try:
fileutil.write_file(pid_file, ustr(os.getpid()))
logger.info(u"{0} running as process {1}", CURRENT_AGENT, ustr(os.getpid()))
except Exception as e:
pid_file = None
logger.warn(
u"Expection writing goal state agent {0} pid to {1}: {2}",
CURRENT_AGENT,
pid_file,
ustr(e))
return pid_files, pid_file
def _fetch(self, uri, headers=None, use_proxy=True):
package = None
try:
is_healthy = True
error_response = ''
resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers)
if restutil.request_succeeded(resp):
package = resp.read()
fileutil.write_file(self.get_agent_pkg_path(),
bytearray(package),
asbin=True)
logger.verbose(u"Agent {0} downloaded from {1}", self.name, uri)
else:
error_response = restutil.read_response_error(resp)
logger.verbose("Fetch was unsuccessful [{0}]", error_response)
is_healthy = not restutil.request_failed_at_hostplugin(resp)
if self.host is not None:
self.host.report_fetch_health(uri, is_healthy, source='GuestAgent', response=error_response)
except restutil.HttpError as http_error:
if isinstance(http_error, ResourceGoneError):
raise
logger.verbose(u"Agent {0} download from {1} failed [{2}]",
self.name,
uri,
http_error)
return package is not None
def test_get_pid_files_returns_previous(self):
for n in range(1250):
fileutil.write_file(os.path.join(self.tmp_dir, str(n)+"_waagent.pid"), ustr(n+1))
previous_pid_file, pid_file, = self.update_handler._get_pid_files()
self.assertEqual("1249_waagent.pid", os.path.basename(previous_pid_file))
self.assertEqual("1250_waagent.pid", os.path.basename(pid_file))
return
def get_gafamily_manifest(self, vmagent_manifest, goal_state):
local_file = MANIFEST_FILE_NAME.format(vmagent_manifest.family,
goal_state.incarnation)
local_file = os.path.join(conf.get_lib_dir(), local_file)
xml_text = self.fetch_manifest(vmagent_manifest.versionsManifestUris)
fileutil.write_file(local_file, xml_text)
return ExtensionManifest(xml_text)
def copy_ovf_env(self):
"""
Copy ovf env file from dvd to hard disk.
Remove password before save it to the disk
"""
dvd_mount_point = conf.get_dvd_mount_point()
ovf_file_path_on_dvd = os.path.join(dvd_mount_point, OVF_FILE_NAME)
tag_file_path_on_dvd = os.path.join(dvd_mount_point, TAG_FILE_NAME)
try:
self.osutil.mount_dvd()
ovfxml = fileutil.read_file(ovf_file_path_on_dvd, remove_bom=True)
ovfenv = OvfEnv(ovfxml)
ovfxml = re.sub("<UserPassword>.*?<", "<UserPassword>*<", ovfxml)
ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME)
fileutil.write_file(ovf_file_path, ovfxml)
if os.path.isfile(tag_file_path_on_dvd):
logger.info("Found {0} in provisioning ISO", TAG_FILE_NAME)
tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME)
shutil.copyfile(tag_file_path_on_dvd, tag_file_path)
except (OSUtilError, IOError) as e:
raise ProtocolError(ustr(e))
try:
self.osutil.umount_dvd()
self.osutil.eject_dvd()
except OSUtilError as e:
logger.warn(ustr(e))
return ovfenv
def conf_sudoer(self, username, nopasswd=False, remove=False):
doas_conf = "/etc/doas.conf"
doas = None
if not remove:
if not os.path.isfile(doas_conf):
# always allow root to become root
doas = "permit keepenv nopass root\n"
fileutil.append_file(doas_conf, doas)
if nopasswd:
doas = "permit keepenv nopass {0}\n".format(username)
else:
doas = "permit keepenv persist {0}\n".format(username)
fileutil.append_file(doas_conf, doas)
fileutil.chmod(doas_conf, 0o644)
else:
# Remove user from doas.conf
if os.path.isfile(doas_conf):
try:
content = fileutil.read_file(doas_conf)
doas = content.split("\n")
doas = [x for x in doas if username not in x]
fileutil.write_file(doas_conf, "\n".join(doas))
except IOError as err:
raise OSUtilError("Failed to remove sudoer: "
"{0}".format(err))
def test_rw_utf8_file(self):
test_file=os.path.join(self.tmp_dir, self.test_file)
content = u"\u6211"
fileutil.write_file(test_file, content, encoding="utf-8")
content_read = fileutil.read_file(test_file)
self.assertEquals(content, content_read)
os.remove(test_file)
def openssl_to_openssh(self, input_file, output_file):
pubkey = fileutil.read_file(input_file)
try:
cryptutil = CryptUtil(conf.get_openssl_cmd())
ssh_rsa_pubkey = cryptutil.asn1_to_ssh(pubkey)
except CryptError as e:
raise OSUtilError(ustr(e))
fileutil.write_file(output_file, ssh_rsa_pubkey)
def set_memory_limit(self, limit=None, unit='megabytes'):
if 'memory' in self.cgroups:
value = self._format_memory_value(unit, limit)
memory_limit_file = self._get_cgroup_file('memory', 'memory.limit_in_bytes')
logger.verbose("writing {0} to {1}".format(value, memory_limit_file))
fileutil.write_file(memory_limit_file, '{0}\n'.format(value))
else:
raise CGroupsException("Memory hierarchy not available in this cgroup")
def test_read_write_file(self):
test_file=os.path.join(self.tmp_dir, self.test_file)
content = ustr(uuid.uuid4())
fileutil.write_file(test_file, content)
content_read = fileutil.read_file(test_file)
self.assertEquals(content, content_read)
os.remove(test_file)
def update_settings_file(self, settings_file, settings):
settings_file = os.path.join(self.get_conf_dir(), settings_file)
try:
fileutil.write_file(settings_file, settings)
except IOError as e:
fileutil.clean_ioerror(e,
paths=[settings_file])
raise ExtensionError(u"Failed to update settings file", e)
def download(self):
self.logger.info("Download extension package")
self.set_operation(WALAEventOperation.Download)
if self.pkg is None:
raise ExtensionError("No package uri found")
package = None
for uri in self.pkg.uris:
try:
package = self.protocol.download_ext_handler_pkg(uri.uri)
if package is not None:
break
except Exception as e:
logger.warn("Error while downloading extension: {0}", e)
if package is None:
raise ExtensionError("Failed to download extension")
self.logger.info("Unpack extension package")
pkg_file = os.path.join(conf.get_lib_dir(),
os.path.basename(uri.uri) + ".zip")
try:
fileutil.write_file(pkg_file, bytearray(package), asbin=True)
zipfile.ZipFile(pkg_file).extractall(self.get_base_dir())
except IOError as e:
raise ExtensionError(u"Failed to write and unzip plugin", e)
#Add user execute permission to all files under the base dir
for file in fileutil.get_all_files(self.get_base_dir()):
fileutil.chmod(file, os.stat(file).st_mode | stat.S_IXUSR)
self.report_event(message="Download succeeded")
self.logger.info("Initialize extension directory")
#Save HandlerManifest.json
man_file = fileutil.search_file(self.get_base_dir(),
'HandlerManifest.json')
if man_file is None:
raise ExtensionError("HandlerManifest.json not found")
try:
man = fileutil.read_file(man_file, remove_bom=True)
fileutil.write_file(self.get_manifest_file(), man)
except IOError as e:
raise ExtensionError(u"Failed to save HandlerManifest.json", e)
#Create status and config dir
try:
status_dir = self.get_status_dir()
fileutil.mkdir(status_dir, mode=0o700)
conf_dir = self.get_conf_dir()
fileutil.mkdir(conf_dir, mode=0o700)
except IOError as e:
raise ExtensionError(u"Failed to create status or config dir", e)
#Save HandlerEnvironment.json
self.create_handler_env()
def _save_protocol(self, protocol_name):
"""
Save protocol endpoint
"""
protocol_file_path = self._get_protocol_file_path()
try:
fileutil.write_file(protocol_file_path, protocol_name)
except IOError as e:
logger.error("Failed to save protocol endpoint: {0}", e)
def test_write_pid_file(self):
for n in range(1112):
fileutil.write_file(os.path.join(self.tmp_dir, str(n)+"_waagent.pid"), ustr(n+1))
with patch('os.getpid', return_value=1112):
previous_pid_file, pid_file = self.update_handler._write_pid_file()
self.assertEqual("1111_waagent.pid", os.path.basename(previous_pid_file))
self.assertEqual("1112_waagent.pid", os.path.basename(pid_file))
self.assertEqual(fileutil.read_file(pid_file), ustr(1112))
return
def set_block_device_timeout(self, dev, timeout):
if dev is not None and timeout is not None:
file_path = "/sys/block/{0}/device/timeout".format(dev)
content = fileutil.read_file(file_path)
original = content.splitlines()[0].rstrip()
if original != timeout:
fileutil.write_file(file_path, timeout)
logger.info("Set block dev timeout: {0} with timeout: {1}",
dev, timeout)
def save_protocol(self, protocol_name):
"""
Save protocol endpoint
"""
protocol_file_path = os.path.join(conf.get_lib_dir(), PROTOCOL_FILE_NAME)
try:
fileutil.write_file(protocol_file_path, protocol_name)
except IOError as e:
logger.error("Failed to save protocol endpoint: {0}", e)
def _set_sentinal(self, agent=CURRENT_AGENT):
try:
fileutil.write_file(self._sentinal_file_path(), agent)
except Exception as e:
logger.warn(
u"Exception writing sentinal file {0}: {1}",
self._sentinal_file_path(),
str(e))
return
def del_root_password(self):
try:
passwd_file_path = conf.get_passwd_file_path()
passwd_content = fileutil.read_file(passwd_file_path)
passwd = passwd_content.split('\n')
new_passwd = [x for x in passwd if not x.startswith("root:")]
new_passwd.insert(0, "root:*LOCK*:14600::::::")
fileutil.write_file(passwd_file_path, "\n".join(new_passwd))
except IOError as e:
raise OSUtilError("Failed to delete root password:{0}".format(e))
def set_handler_state(self, handler_state):
state_dir = self.get_conf_dir()
state_file = os.path.join(state_dir, "HandlerState")
try:
if not os.path.exists(state_dir):
fileutil.mkdir(state_dir, mode=0o700)
fileutil.write_file(state_file, handler_state)
except IOError as e:
fileutil.clean_ioerror(e, paths=[state_file])
self.logger.error("Failed to set state: {0}", e)
def download_ext_handler_pkg(self, uri, destination, headers=None, use_proxy=True):
success = False
try:
resp = restutil.http_get(uri, headers=headers, use_proxy=use_proxy)
if restutil.request_succeeded(resp):
fileutil.write_file(destination, bytearray(resp.read()), asbin=True)
success = True
except Exception as e:
logger.warn("Failed to download from: {0}".format(uri), e)
return success
def test_ensure_no_orphans_kills_after_interval(self):
fileutil.write_file(os.path.join(self.tmp_dir, "0_waagent.pid"), ustr(41))
with patch('os.kill') as mock_kill:
calls, sleeps = self._test_ensure_no_orphans(
invocations=4,
interval=3*GOAL_STATE_INTERVAL)
self.assertEqual(3, calls)
self.assertEqual(2, sleeps)
self.assertEqual(1, mock_kill.call_count)
return
def test_clean_ioerror_removes_files(self):
fd, f = tempfile.mkstemp()
os.close(fd)
fileutil.write_file(f, 'Not empty')
e = IOError()
e.errno = errno.ENOSPC
fileutil.clean_ioerror(e, paths=[f])
self.assertFalse(os.path.isdir(f))
self.assertFalse(os.path.isfile(f))
def test_del_lib_dir_files(self,
distro_name,
distro_version,
distro_full_name,
mock_conf):
dirs = [
'WALinuxAgent-2.2.26/config',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/status'
]
files = [
'HostingEnvironmentConfig.xml',
'Incarnation',
'Protocol',
'SharedConfig.xml',
'WireServerEndpoint',
'Extensions.1.xml',
'ExtensionsConfig.1.xml',
'GoalState.1.xml',
'Extensions.2.xml',
'ExtensionsConfig.2.xml',
'GoalState.2.xml',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config/42.settings',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config/HandlerStatus',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config/HandlerState',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/status/12.notstatus',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/mrseq',
'WALinuxAgent-2.2.26/config/0.settings'
]
tmp = tempfile.mkdtemp()
mock_conf.return_value = tmp
for d in dirs:
fileutil.mkdir(os.path.join(tmp, d))
for f in files:
fileutil.write_file(os.path.join(tmp, f), "Value")
deprovision_handler = get_deprovision_handler(distro_name,
distro_version,
distro_full_name)
warnings = []
actions = []
deprovision_handler.del_lib_dir_files(warnings, actions)
deprovision_handler.del_ext_handler_files(warnings, actions)
self.assertTrue(len(warnings) == 0)
self.assertTrue(len(actions) == 2)
self.assertEqual(fileutil.rm_files, actions[0].func)
self.assertEqual(fileutil.rm_files, actions[1].func)
self.assertEqual(11, len(actions[0].args))
self.assertEqual(3, len(actions[1].args))
for f in actions[0].args:
self.assertTrue(os.path.basename(f) in files)
for f in actions[1].args:
self.assertTrue(f[len(tmp)+1:] in files)
def _set_sentinel(self, agent=CURRENT_AGENT, msg="Unknown cause"):
try:
fileutil.write_file(
self._sentinel_file_path(),
"[{0}] [{1}]".format(agent, msg))
except Exception as e:
logger.warn(
u"Exception writing sentinel file {0}: {1}",
self._sentinel_file_path(),
str(e))
return
def _write_pid_file(self):
previous_pid_file, pid_file = self._get_pid_files()
try:
fileutil.write_file(pid_file, ustr(os.getpid()))
logger.info(u"{0} running as process {1}", CURRENT_AGENT,
ustr(os.getpid()))
except Exception as e:
pid_file = None
logger.warn(
u"Expection writing goal state agent {0} pid to {1}: {2}",
CURRENT_AGENT, pid_file, ustr(e))
return previous_pid_file, pid_file
def conf_sshd(self, disable_password):
option = "no" if disable_password else "yes"
conf_file_path = conf.get_sshd_conf_file_path()
conf_file = fileutil.read_file(conf_file_path).split("\n")
textutil.set_ssh_config(conf_file, "PasswordAuthentication", option)
textutil.set_ssh_config(conf_file, "ChallengeResponseAuthentication",
option)
textutil.set_ssh_config(conf_file, "ClientAliveInterval", "180")
fileutil.write_file(conf_file_path, "\n".join(conf_file))
logger.info("{0} SSH password-based authentication methods.".format(
"Disabled" if disable_password else "Enabled"))
logger.info("Configured SSH client probing to keep connections alive.")
def test_agent_logs_if_extension_log_directory_is_a_file(self, mock_dir, mock_log):
ext_log_dir = os.path.join(self.tmp_dir, "FauxLogDir")
mock_dir.return_value = ext_log_dir
fileutil.write_file(ext_log_dir, "Foo")
self.assertTrue(os.path.isfile(ext_log_dir))
self.assertFalse(os.path.isdir(ext_log_dir))
agent = Agent(False, # pylint: disable=unused-variable
conf_file_path=os.path.join(data_dir, "test_waagent.conf"))
self.assertTrue(os.path.isfile(ext_log_dir))
self.assertFalse(os.path.isdir(ext_log_dir))
self.assertEqual(1, mock_log.call_count)
def check_pid(self):
"""Check whether daemon is already running"""
pid = None
pid_file = conf.get_agent_pid_file_path()
if os.path.isfile(pid_file):
pid = fileutil.read_file(pid_file)
if self.osutil.check_pid_alive(pid):
logger.info("Daemon is already running: {0}", pid)
sys.exit(0)
fileutil.write_file(pid_file, ustr(os.getpid()))
def copy_ovf_env(self):
"""
Copy ovf env file from dvd to hard disk.
Remove password before save it to the disk
"""
dvd_mount_point = conf.get_dvd_mount_point()
ovf_file_path_on_dvd = os.path.join(dvd_mount_point, OVF_FILE_NAME)
tag_file_path_on_dvd = os.path.join(dvd_mount_point, TAG_FILE_NAME)
ovf_file_path = os.path.join(conf.get_lib_dir(), OVF_FILE_NAME)
tag_file_path = os.path.join(conf.get_lib_dir(), TAG_FILE_NAME)
try:
self.osutil.mount_dvd()
except OSUtilError as e:
raise ProtocolError("[CopyOvfEnv] Error mounting dvd: "
"{0}".format(ustr(e)))
try:
ovfxml = fileutil.read_file(ovf_file_path_on_dvd, remove_bom=True)
ovfenv = OvfEnv(ovfxml)
except IOError as e:
raise ProtocolError("[CopyOvfEnv] Error reading file "
"{0}: {1}".format(ovf_file_path_on_dvd,
ustr(e)))
try:
ovfxml = re.sub(PASSWORD_PATTERN,
PASSWORD_REPLACEMENT,
ovfxml)
fileutil.write_file(ovf_file_path, ovfxml)
except IOError as e:
raise ProtocolError("[CopyOvfEnv] Error writing file "
"{0}: {1}".format(ovf_file_path,
ustr(e)))
try:
if os.path.isfile(tag_file_path_on_dvd):
logger.info("Found {0} in provisioning ISO", TAG_FILE_NAME)
shutil.copyfile(tag_file_path_on_dvd, tag_file_path)
except IOError as e:
raise ProtocolError("[CopyOvfEnv] Error copying file "
"{0} to {1}: {2}".format(tag_file_path,
tag_file_path,
ustr(e)))
try:
self.osutil.umount_dvd()
self.osutil.eject_dvd()
except OSUtilError as e:
logger.warn(ustr(e))
return ovfenv
def _ensure_partition_assigned(self):
"""
Assign the VM to a partition (0 - 99). Downloaded updates may be configured
to run on only some VMs; the assigned partition determines eligibility.
"""
if not os.path.exists(self._partition_file):
partition = ustr(int(datetime.utcnow().microsecond / 10000))
fileutil.write_file(self._partition_file, partition)
add_event(AGENT_NAME,
version=CURRENT_VERSION,
op=WALAEventOperation.Partition,
is_success=True,
message=partition)
def _prepare_handler_state(self):
handler_state_path = os.path.join(
self.tmp_dir,
"handler_state",
self.ext_handler_i.get_full_name())
os.makedirs(handler_state_path)
fileutil.write_file(
os.path.join(handler_state_path, "state"),
self.handler_state)
fileutil.write_file(
os.path.join(handler_state_path, "status"),
json.dumps(get_properties(self.handler_status)))
return
def test_clean_ioerror_removes_directories(self):
d1 = tempfile.mkdtemp() # pylint: disable=invalid-name
d2 = tempfile.mkdtemp() # pylint: disable=invalid-name
for n in ['foo', 'bar']: # pylint: disable=invalid-name
fileutil.write_file(os.path.join(d2, n), 'Not empty')
e = IOError() # pylint: disable=invalid-name
e.errno = errno.ENOSPC
fileutil.clean_ioerror(e, paths=[d1, d2])
self.assertFalse(os.path.isdir(d1))
self.assertFalse(os.path.isfile(d1))
self.assertFalse(os.path.isdir(d2))
self.assertFalse(os.path.isfile(d2))
def _fetch(self, uri, headers=None):
package = None
try:
resp = restutil.http_get(uri, chk_proxy=True, headers=headers)
if resp.status == restutil.httpclient.OK:
package = resp.read()
fileutil.write_file(self.get_agent_pkg_path(),
bytearray(package),
asbin=True)
logger.info(u"Agent {0} downloaded from {1}", self.name, uri)
except restutil.HttpError as http_error:
logger.verbose(u"Agent {0} download from {1} failed [{2}]",
self.name, uri, http_error)
return package is not None
def set_handler_status(self, status="NotReady", message="", code=0):
state_dir = self.get_conf_dir()
handler_status = ExtHandlerStatus()
handler_status.name = self.ext_handler.name
handler_status.version = self.ext_handler.properties.version
handler_status.message = message
handler_status.code = code
handler_status.status = status
status_file = os.path.join(state_dir, "HandlerStatus")
try:
fileutil.write_file(status_file, json.dumps(get_properties(handler_status)))
except (IOError, ValueError, ProtocolError) as e:
self.logger.error("Failed to save handler status: {0}", e)
def test_cleanup_legacy_cgroups_should_remove_legacy_cgroups(self):
# Set up a mock /var/run/waagent.pid file
daemon_pid_file = os.path.join(self.tmp_dir, "waagent.pid")
fileutil.write_file(daemon_pid_file, "42\n")
# Set up old controller cgroups, but do not add the daemon's PID to them
legacy_cpu_cgroup = CGroupsTools.create_legacy_agent_cgroup(self.cgroups_file_system_root, "cpu", '')
legacy_memory_cgroup = CGroupsTools.create_legacy_agent_cgroup(self.cgroups_file_system_root, "memory", '')
with patch("azurelinuxagent.common.cgroupapi.get_agent_pid_file_path", return_value=daemon_pid_file):
legacy_cgroups = SystemdCgroupsApi().cleanup_legacy_cgroups()
self.assertEqual(legacy_cgroups, 2, "cleanup_legacy_cgroups() did not find all the expected cgroups")
self.assertFalse(os.path.exists(legacy_cpu_cgroup), "cleanup_legacy_cgroups() did not remove the CPU legacy cgroup")
self.assertFalse(os.path.exists(legacy_memory_cgroup), "cleanup_legacy_cgroups() did not remove the memory legacy cgroup")
def create_handler_env(self):
env = [{
"name": self.ext_handler.name,
"version": HANDLER_ENVIRONMENT_VERSION,
"handlerEnvironment": {
"logFolder": self.get_log_dir(),
"configFolder": self.get_conf_dir(),
"statusFolder": self.get_status_dir(),
"heartbeatFile": self.get_heartbeat_file()
}
}]
try:
fileutil.write_file(self.get_env_file(), json.dumps(env))
except IOError as e:
raise ExtensionError(u"Failed to save handler environment", e)
def test_provision(self, mock_util, distro_name, distro_version, distro_full_name):
provision_handler = get_provision_handler(distro_name, distro_version,
distro_full_name)
mock_osutil = MagicMock()
mock_osutil.decode_customdata = Mock(return_value="")
provision_handler.osutil = mock_osutil
provision_handler.protocol_util.osutil = mock_osutil
provision_handler.protocol_util.get_protocol = MagicMock()
conf.get_dvd_mount_point = Mock(return_value=self.tmp_dir)
ovfenv_file = os.path.join(self.tmp_dir, OVF_FILE_NAME)
ovfenv_data = load_data("ovf-env.xml")
fileutil.write_file(ovfenv_file, ovfenv_data)
provision_handler.run()
def activate_resource_disk(self):
logger.info("Activate resource disk")
try:
mount_point = conf.get_resourcedisk_mountpoint()
mount_point = self.mount_resource_disk(mount_point)
warning_file = os.path.join(mount_point,
DATALOSS_WARNING_FILE_NAME)
try:
fileutil.write_file(warning_file, DATA_LOSS_WARNING)
except IOError as e:
logger.warn("Failed to write data loss warning:{0}", e)
return mount_point
except ResourceDiskError as e:
logger.error("Failed to mount resource disk {0}", e)
add_event(name=AGENT_NAME, is_success=False, message=ustr(e),
op=WALAEventOperation.ActivateResourceDisk)
def download_ext_handler_pkg(self,
uri,
destination,
headers=None,
use_proxy=True):
success = False
try:
resp = restutil.http_get(uri, headers=headers, use_proxy=use_proxy)
if restutil.request_succeeded(resp):
fileutil.write_file(destination,
bytearray(resp.read()),
asbin=True)
success = True
except Exception as e:
logger.warn("Failed to download from: {0}".format(uri), e)
return success
def save_customdata(self, ovfenv):
customdata = ovfenv.customdata
if customdata is None:
return
logger.info("Save custom data")
lib_dir = conf.get_lib_dir()
if conf.get_decode_customdata():
customdata = self.osutil.decode_customdata(customdata)
customdata_file = os.path.join(lib_dir, CUSTOM_DATA_FILE)
fileutil.write_file(customdata_file, customdata)
if conf.get_execute_customdata():
logger.info("Execute custom data")
os.chmod(customdata_file, 0o700)
shellutil.run(customdata_file)
def test_del_lib_dir_files(self, distro_name, distro_version,
distro_full_name, mock_conf):
dirs = [
'WALinuxAgent-2.2.26/config',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/status'
]
files = [
'HostingEnvironmentConfig.xml', 'Incarnation', 'Protocol',
'SharedConfig.xml', 'WireServerEndpoint', 'Extensions.1.xml',
'ExtensionsConfig.1.xml', 'GoalState.1.xml', 'Extensions.2.xml',
'ExtensionsConfig.2.xml', 'GoalState.2.xml',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config/42.settings',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config/HandlerStatus',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/config/HandlerState',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/status/12.notstatus',
'Microsoft.Azure.Extensions.CustomScript-2.0.6/mrseq',
'WALinuxAgent-2.2.26/config/0.settings'
]
tmp = tempfile.mkdtemp()
mock_conf.return_value = tmp
for d in dirs: # pylint: disable=invalid-name
fileutil.mkdir(os.path.join(tmp, d))
for f in files: # pylint: disable=invalid-name
fileutil.write_file(os.path.join(tmp, f), "Value")
deprovision_handler = get_deprovision_handler(distro_name,
distro_version,
distro_full_name)
warnings = []
actions = []
deprovision_handler.del_lib_dir_files(warnings, actions)
deprovision_handler.del_ext_handler_files(warnings, actions)
self.assertTrue(len(warnings) == 0)
self.assertTrue(len(actions) == 2)
self.assertEqual(fileutil.rm_files, actions[0].func)
self.assertEqual(fileutil.rm_files, actions[1].func)
self.assertEqual(11, len(actions[0].args))
self.assertEqual(3, len(actions[1].args))
for f in actions[0].args: # pylint: disable=invalid-name
self.assertTrue(os.path.basename(f) in files)
for f in actions[1].args: # pylint: disable=invalid-name
self.assertTrue(f[len(tmp) + 1:] in files)
def del_root_password(self):
try:
passwd_file_path = conf.get_passwd_file_path()
try:
passwd_content = fileutil.read_file(passwd_file_path)
if not passwd_content:
# Empty file is no better than no file
raise FileNotFoundError
except FileNotFoundError:
new_passwd = ["root:*LOCK*:14600::::::"]
else:
passwd = passwd_content.split('\n')
new_passwd = [x for x in passwd if not x.startswith("root:")]
new_passwd.insert(0, "root:*LOCK*:14600::::::")
fileutil.write_file(passwd_file_path, "\n".join(new_passwd))
except IOError as e:
raise OSUtilError("Failed to delete root password:{0}".format(e))
pass
def _fetch(self, uri, headers=None, chk_proxy=True):
package = None
try:
resp = restutil.http_get(uri, chk_proxy=chk_proxy, headers=headers)
if resp.status == restutil.httpclient.OK:
package = resp.read()
fileutil.write_file(self.get_agent_pkg_path(),
bytearray(package),
asbin=True)
logger.verbose(u"Agent {0} downloaded from {1}", self.name,
uri)
else:
logger.verbose("Fetch was unsuccessful [{0}]",
HostPluginProtocol.read_response_error(resp))
except restutil.HttpError as http_error:
logger.verbose(u"Agent {0} download from {1} failed [{2}]",
self.name, uri, http_error)
return package is not None
def test_log_collector_parses_commands_in_manifest(self):
# Ensure familiar commands are parsed and unknowns are ignored (like diskinfo and malformed entries)
file_to_collect = os.path.join(self.root_collect_dir, "waagent.log")
folder_to_list = self.root_collect_dir
manifest_content = """
echo,### Test header ###
unknown command
ll,{0}
copy,{1}
diskinfo,""".format(folder_to_list, file_to_collect)
manifest_file_path = os.path.join(self.tmp_dir, "manifest")
write_file(manifest_file_path, manifest_content)
lc = LogCollector(manifest_file_path)
archive = lc.collect_logs()
with open(self.output_results_file_path, "r") as fh:
results = fh.readlines()
# Assert echo was parsed
self.assertTrue(
any([line.endswith("### Test header ###\n") for line in results]))
# Assert unknown command was reported
self.assertTrue(
any([
line.endswith("ERROR Couldn\'t parse \"unknown command\"\n")
for line in results
]))
# Assert ll was parsed
self.assertTrue(
any([
"ls -alF {0}".format(folder_to_list) in line
for line in results
]))
# Assert copy was parsed
self._assert_archive_created(archive)
self._assert_files_are_in_archive(expected_files=[file_to_collect])
no_files = self._get_number_of_files_in_archive()
self.assertEquals(
1, no_files,
"Expected 1 file in archive, found {0}!".format(no_files))
def test_remove_dirs(self):
dirs = []
for n in range(0,5):
dirs.append(tempfile.mkdtemp())
for d in dirs:
for n in range(0, random.choice(range(0,10))):
fileutil.write_file(os.path.join(d, "test"+str(n)), "content")
for n in range(0, random.choice(range(0,10))):
dd = os.path.join(d, "testd"+str(n))
os.mkdir(dd)
for nn in range(0, random.choice(range(0,10))):
os.symlink(dd, os.path.join(dd, "sym"+str(nn)))
for n in range(0, random.choice(range(0,10))):
os.symlink(d, os.path.join(d, "sym"+str(n)))
fileutil.rm_dirs(*dirs)
for d in dirs:
self.assertEqual(len(os.listdir(d)), 0)
def openssl_to_openssh(self, input_file, output_file):
cryptutil = CryptUtil(conf.get_openssl_cmd())
ret, out = shellutil.run_get_output(conf.get_openssl_cmd() +
" rsa -pubin -noout -text -in '" +
input_file + "'")
if ret != 0:
raise OSUtilError('openssl failed with {0}'.format(ret))
modulus = []
exponent = []
buf = None
for line in out.split('\n'):
if line.startswith('Modulus:'):
buf = modulus
buf.append(line)
continue
if line.startswith('Exponent:'):
buf = exponent
buf.append(line)
continue
if buf and line:
buf.append(line.strip().replace(':', ''))
def text_to_num(buf):
if len(buf) == 1:
return int(buf[0].split()[1])
return long(''.join(buf[1:]), 16)
n = text_to_num(modulus)
e = text_to_num(exponent)
keydata = bytearray()
keydata.extend(struct.pack('>I', len('ssh-rsa')))
keydata.extend(b'ssh-rsa')
keydata.extend(struct.pack('>I', len(cryptutil.num_to_bytes(e))))
keydata.extend(cryptutil.num_to_bytes(e))
keydata.extend(struct.pack('>I', len(cryptutil.num_to_bytes(n)) + 1))
keydata.extend(b'\0')
keydata.extend(cryptutil.num_to_bytes(n))
keydata_base64 = base64.b64encode(bytebuffer(keydata))
fileutil.write_file(
output_file,
ustr(b'ssh-rsa ' + keydata_base64 + b'\n', encoding='utf-8'))
def del_root_password(self):
try:
passwd_file_path = conf.get_passwd_file_path()
try:
passwd_content = fileutil.read_file(passwd_file_path)
if not passwd_content:
# Empty file is no better than no file
raise IOError(errno.ENOENT, "Empty File", passwd_file_path)
except (IOError, OSError) as file_read_err:
if file_read_err.errno != errno.ENOENT:
raise
new_passwd = ["root:*LOCK*:14600::::::"]
else:
passwd = passwd_content.split('\n')
new_passwd = [x for x in passwd if not x.startswith("root:")]
new_passwd.insert(0, "root:*LOCK*:14600::::::")
fileutil.write_file(passwd_file_path, "\n".join(new_passwd))
except IOError as e: # pylint: disable=C0103
raise OSUtilError("Failed to delete root password:{0}".format(e))
pass # pylint: disable=W0107
def test_cleanup_legacy_cgroups_should_disable_cgroups_when_the_daemon_was_added_to_the_legacy_cgroup_on_systemd(
self, _):
# Set up a mock /var/run/waagent.pid file
daemon_pid = "42"
daemon_pid_file = os.path.join(self.tmp_dir, "waagent.pid")
fileutil.write_file(daemon_pid_file, daemon_pid + "\n")
# Set up old controller cgroups and add the daemon PID to them
CGroupsTools.create_legacy_agent_cgroup(self.cgroups_file_system_root,
"cpu", daemon_pid)
CGroupsTools.create_legacy_agent_cgroup(self.cgroups_file_system_root,
"memory", daemon_pid)
# Start tracking a couple of dummy cgroups
CGroupsTelemetry.track_cgroup(
CGroup("dummy", "/sys/fs/cgroup/memory/system.slice/dummy.service",
"cpu"))
CGroupsTelemetry.track_cgroup(
CGroup("dummy", "/sys/fs/cgroup/memory/system.slice/dummy.service",
"memory"))
cgroup_configurator = CGroupConfigurator.get_instance()
with patch("azurelinuxagent.common.cgroupconfigurator.add_event"
) as mock_add_event:
with patch(
"azurelinuxagent.common.cgroupapi.get_agent_pid_file_path",
return_value=daemon_pid_file):
cgroup_configurator.cleanup_legacy_cgroups()
self.assertEquals(len(mock_add_event.call_args_list), 1)
_, kwargs = mock_add_event.call_args_list[0]
self.assertEquals(kwargs['op'], 'CGroupsCleanUp')
self.assertFalse(kwargs['is_success'])
self.assertEquals(
kwargs['message'],
"Failed to process legacy cgroups. Collection of resource usage data will be disabled. [CGroupsException] The daemon's PID ({0}) was already added to the legacy cgroup; this invalidates resource usage data."
.format(daemon_pid))
self.assertFalse(cgroup_configurator.enabled())
self.assertEquals(len(CGroupsTelemetry._tracked), 0)
def update_dat_conf(paths, ipv4_addr):
"""
Looks at paths for dat.conf file and updates the ip address for the
infiniband interface.
"""
logger.info("Updating DAPL configuration file")
for f in paths:
logger.info("RDMA: trying {0}".format(f))
if not os.path.isfile(f):
logger.info(
"RDMA: DAPL config not found at {0}".format(f))
continue
logger.info("RDMA: DAPL config is at: {0}".format(f))
cfg = fileutil.read_file(f)
new_cfg = RDMADeviceHandler.replace_dat_conf_contents(
cfg, ipv4_addr)
fileutil.write_file(f, new_cfg)
logger.info("RDMA: DAPL configuration is updated")
return
raise Exception("RDMA: DAPL configuration file not found at predefined paths")
def deploy_ssh_keypair(self, username, keypair):
"""
Deploy id_rsa and id_rsa.pub
"""
path, thumbprint = keypair
path = self._norm_path(path)
dir_path = os.path.dirname(path)
fileutil.mkdir(dir_path, mode=0o700, owner=username)
lib_dir = conf.get_lib_dir()
prv_path = os.path.join(lib_dir, thumbprint + '.prv')
if not os.path.isfile(prv_path):
raise OSUtilError("Can't find {0}.prv".format(thumbprint))
shutil.copyfile(prv_path, path)
pub_path = path + '.pub'
crytputil = CryptUtil(conf.get_openssl_cmd())
pub = crytputil.get_pubkey_from_prv(prv_path)
fileutil.write_file(pub_path, pub)
self.set_selinux_context(pub_path, 'unconfined_u:object_r:ssh_home_t:s0')
self.set_selinux_context(path, 'unconfined_u:object_r:ssh_home_t:s0')
os.chmod(path, 0o644)
os.chmod(pub_path, 0o600)
def test_scvmm_detection_with_file(self):
# setup
conf.get_dvd_mount_point = Mock(return_value=self.tmp_dir)
conf.get_detect_scvmm_env = Mock(return_value=True)
scvmm_file = os.path.join(self.tmp_dir, scvmm.VMM_CONF_FILE_NAME)
fileutil.write_file(scvmm_file, "")
with patch.object(scvmm.ScvmmHandler, 'start_scvmm_agent') as po:
with patch('os.listdir', return_value=["sr0", "sr1", "sr2"]):
with patch('time.sleep', return_value=0):
# execute
failed = False
try:
scvmm.get_scvmm_handler().run()
except: # pylint: disable=bare-except
failed = True
# assert
self.assertTrue(failed)
self.assertTrue(po.call_count == 1)
# cleanup
os.remove(scvmm_file)
def set_hostname(self, hostname):
#
# In order for the identity-node service to properly detect the
# hostname from the contents of /etc/nodename, the file needs to
# contain a newline after the hostname. Otherwise, the service
# will simply assign "unknown" as the hostname for the system.
#
fileutil.write_file('/etc/nodename', '{0}\n'.format(hostname))
# Make it happen NOW.
ret = shellutil.run('uname -S {0}'.format(hostname))
if ret:
raise OSUtilError(
'Unable to set hostname to {0}.'.format(hostname))
#
# Unfortunately, there isn't a way to cause the service refresh
# executed above a synchronous operation. Thus, without this
# loop, it would be possible for this function to return without
# having the hostname having been updated yet.
#
# Setting the hostname on the other platforms is a synchronous
# operation, so we've opted to enforce this fuction as being
# synchronus as well.
#
actual = None
for i in range(0, 10):
ret = shellutil.run_get_output('hostname')
if ret[0] == 0:
actual = ret[1].strip()
else:
raise OSUtilError('Unable to retrieve hostname')
if hostname == actual:
break
else:
time.sleep(1)
if actual != hostname:
raise OSUtilError('Unable to modify hostname to the desired value')
def _fetch(self, uri, headers=None, use_proxy=True):
package = None
try:
resp = restutil.http_get(uri, use_proxy=use_proxy, headers=headers)
if restutil.request_succeeded(resp):
package = resp.read()
fileutil.write_file(self.get_agent_pkg_path(),
bytearray(package),
asbin=True)
logger.verbose(u"Agent {0} downloaded from {1}", self.name,
uri)
else:
logger.verbose("Fetch was unsuccessful [{0}]",
restutil.read_response_error(resp))
except restutil.HttpError as http_error:
if isinstance(http_error, ResourceGoneError):
raise
logger.verbose(u"Agent {0} download from {1} failed [{2}]",
self.name, uri, http_error)
return package is not None
def test_provision_telemetry_fail(
self,
mock_util, # pylint: disable=unused-argument
distro_name,
distro_version,
distro_full_name,
_):
"""
Assert that the agent issues one telemetry message as part of a
failed provisioning.
1. Provision
"""
ph = get_provision_handler(
distro_name,
distro_version, # pylint: disable=invalid-name
distro_full_name)
ph.report_event = MagicMock()
ph.reg_ssh_host_key = MagicMock(
side_effect=ProvisionError("--unit-test--"))
mock_osutil = MagicMock()
mock_osutil.decode_customdata = Mock(return_value="")
ph.osutil = mock_osutil
ph.protocol_util.osutil = mock_osutil
ph.protocol_util.get_protocol = MagicMock()
conf.get_dvd_mount_point = Mock(return_value=self.tmp_dir)
ovfenv_file = os.path.join(self.tmp_dir, OVF_FILE_NAME)
ovfenv_data = load_data("ovf-env.xml")
fileutil.write_file(ovfenv_file, ovfenv_data)
ph.run()
positional_args, kw_args = ph.report_event.call_args_list[0] # pylint: disable=unused-variable
self.assertTrue(
re.match(
r'Provisioning failed: \[ProvisionError\] --unit-test-- \(\d+\.\d+s\)',
positional_args[0]) is not None)
