def apply(self, action, auth_config=None):
"""
Install a set of packages via APT, returning the packages actually installed or updated.
Arguments:
action -- a dict of package name to version; version can be empty, a single string or a list of strings
Exceptions:
ToolError -- on expected failures (such as a non-zero exit code)
"""
pkgs_changed = []
if not action:
log.debug("No packages specified for APT")
return pkgs_changed
cache_result = ProcessHelper(['apt-cache', '-q', 'gencaches']).call()
if cache_result.returncode:
log.error("APT gencache failed. Output: %s", cache_result.stdout)
raise ToolError("Could not create apt cache", cache_result.returncode)
pkg_specs = []
for pkg_name in action:
if action[pkg_name]:
if isinstance(action[pkg_name], basestring):
pkg_keys = ['%s=%s' % (pkg_name, action[pkg_name])]
else:
pkg_keys = ['%s=%s' % (pkg_name, ver) if ver else pkg_name for ver in action[pkg_name]]
else:
pkg_keys = [pkg_name]
pkgs_filtered = [pkg_key for pkg_key in pkg_keys if self._pkg_filter(pkg_key, pkg_name)]
if pkgs_filtered:
pkg_specs.extend(pkgs_filtered)
pkgs_changed.append(pkg_name)
if not pkg_specs:
log.info("All APT packages were already installed")
return []
log.info("Attempting to install %s via APT", pkg_specs)
env = dict(os.environ)
env['DEBIAN_FRONTEND'] = 'noninteractive'
result = ProcessHelper(['apt-get', '-q', '-y', 'install'] + pkg_specs, env=env).call()
if result.returncode:
log.error("apt-get failed. Output: %s", result.stdout)
raise ToolError("Could not successfully install APT packages", result.returncode)
log.info("APT installed %s", pkgs_changed)
log.debug("APT output: %s", result.stdout)
return pkgs_changed
def _install_gem(self, pkg, ver=None):
"""Install a gem if the version is not already installed; return True if installed, False if skipped."""
if self._gem_is_installed(pkg, ver):
log.info("%s-%s is already installed, skipping.", pkg, ver)
return False
else:
log.info("Installing %s version %s via gem", pkg, ver)
install_command = [
'gem', 'install', '-b', '--no-ri', '--no-rdoc', pkg
]
if ver:
install_command.extend(['-v', '= %s' % ver])
result = ProcessHelper(install_command).call()
if result.returncode:
log.error("Gem failed. Output: %s", result.stdout)
raise ToolError("Failed to install gem: %s-%s" % (pkg, ver),
result.returncode)
else:
log.info("Gem installed: %s-%s", pkg, ver)
log.debug("Gem output: %s", result.stdout)
return True
def get_package_version(cls, pkg, is_file=True):
"""
Given the name of an installed package or package location, return a tuple of (name, version-release)
of either the installed package or the specified package location
Parameters:
- pkg: the package name/location
- is_file : if True, pkg refers to a package location; if False, the name of an installed package
"""
query_mode = '-qp' if is_file else '-qa'
log.debug("Querying for version of package %s", pkg)
query_result = ProcessHelper([
'rpm', query_mode, '--queryformat',
'%{NAME}|%{VERSION}-%{RELEASE}', '--nosignature', pkg
],
stderr=subprocess.PIPE).call()
log.debug("RPM stdout: %s", query_result.stdout)
log.debug("RPM stderr: %s", query_result.stderr)
if query_result.returncode:
log.error("Could not determine package contained by rpm at %s",
pkg)
return (None, None)
# The output from the command is just name|version-release
name, sep, version = query_result.stdout.strip().partition('|')
return (name, version)
def create_group(group_name, gid=None):
"""Create a group in the OS, returning True if one is created"""
try:
group_record = grp.getgrnam(group_name)
if gid and str(group_record[2]) != gid:
raise ToolError("Group %s exists with gid %s, but gid %s was requested" % (group_name, group_record[2], gid))
log.debug("Group %s already exists", group_name)
return False
except KeyError:
pass
cmd = ['/usr/sbin/groupadd', '-r']
if gid:
cmd.extend(['-g', gid])
cmd.append(group_name)
result = ProcessHelper(cmd).call()
if result.returncode:
log.error("Failed to create group %s", group_name)
log.debug("Groupadd output: %s", result.stdout)
raise ToolError("Failed to create group %s" % group_name)
else:
log.info("Created group %s successfully", group_name)
return True
def _pkg_available(self, pkg):
# --showduplicates seems to be required to see downgradable versions when running yum non-interactively
# but not when running interactively -- but we rarely run interactively
result = ProcessHelper(
['yum', '-C', '-y', '--showduplicates', 'list', 'available',
pkg]).call()
return result.returncode == 0
def _run_hook(self, hook, cmd_msg):
if hook.singleton:
log.info("Hook %s is configured to run as a singleton", hook.name)
leader = self.cfn_client.elect_command_leader(self.stack_name,
cmd_msg['CommandName'],
cmd_msg['InvocationId'],
self.listener_id)
if leader == self.listener_id:
log.info("This listener is the leader. Continuing with action")
else:
log.info("This listener is not the leader; %s is the leader.", leader)
return True
action_env = self._get_environment(cmd_msg)
result_queue = cmd_msg['ResultQueue']
log.info("Running action for %s", hook.name)
log.debug("Action environment: %s", action_env)
action = hook.action
if hook.runas:
action = ['su', hook.runas, '-c', action]
result = ProcessHelper(action, stderr=subprocess.PIPE, env=action_env).call()
log.debug("Action for %s output: %s", hook.name, result.stdout if result.stdout else '<None>')
if not hook.send_result:
return True
result_msg = { 'DispatcherId' : cmd_msg['DispatcherId'],
'InvocationId' : cmd_msg['InvocationId'],
'CommandName' : cmd_msg['CommandName'],
'Status' : "FAILURE" if result.returncode else "SUCCESS",
'ListenerId' : self.listener_id }
if result.returncode:
result_stderr = result.stderr.rstrip()
log.warn("Action for %s exited with %s, returning FAILURE", hook.name, result.returncode)
result_msg['Message'] = result_stderr if len(result_stderr) <= 1024 else result_stderr[0:500] + '...' + result_stderr[-500:]
else:
result_stdout = result.stdout.rstrip()
if len(result_stdout) > 1024:
log.error("stdout for %s was greater than 1024 in length, which is not allowed", hook.name)
result_msg['Status'] = 'FAILURE'
result_msg['Message'] = 'Result data was longer than 1024 bytes. Started with: ' + result_stdout[0:100]
else:
log.info("Action for %s succeeded, returning SUCCESS", hook.name)
result_msg['Data'] = result_stdout
try:
self.sqs_client.send_message(result_queue, json.dumps(result_msg), request_credentials=self._creds_provider.credentials)
except Exception:
log.exception('Error sending result; will leave message in queue')
return False
return True
def _is_service_running(self, service):
cmd = self._get_service_executable(service)
cmd.append("status")
result = ProcessHelper(cmd).call()
if result.returncode:
return False
return True
def _stop_service(self, service):
cmd = self._get_service_executable(service)
cmd.append("stop")
result = ProcessHelper(cmd).call()
if result.returncode:
log.error("Could not stop service %s; return code was %s", service,
result.returncode)
log.debug("Service output: %s", result.stdout)
raise ToolError("Could not stop %s" % service)
else:
log.info("Stopped %s successfully", service)
def _process_hook(self, hook, metadata):
try:
new_data = self._retrieve_path_data(hook.path)
except InFlightStatusError:
return
old_data = self._jsonConverter.deserialize(
metadata.get(hook.name + "|" + hook.path, None))
if 'post.add' in hook.triggers and not old_data and new_data:
log.info("Previous state not found; action for %s will be run",
hook.name)
elif 'post.remove' in hook.triggers and old_data and not new_data:
log.info('Path %s was removed; action for %s will be run',
hook.path, hook.name)
elif 'post.update' in hook.triggers and old_data and new_data and old_data != new_data:
log.info(
"Data has changed from previous state; action for %s will be run",
hook.name)
else:
log.debug("No change in path %s for hook %s", hook.path, hook.name)
metadata[hook.name + "|" +
hook.path] = self._jsonConverter.serialize(new_data)
JsonFileManager.write(self.dir, 'metadata_db.json', metadata)
return
log.info("Running action for %s", hook.name)
action_env = dict(os.environ)
env_key = self._retrieve_env_key(hook.path)
if old_data:
action_env['CFN_OLD_%s' % env_key] = self._as_string(old_data)
if new_data:
action_env['CFN_NEW_%s' % env_key] = self._as_string(new_data)
action = hook.action
if hook.runas:
action = ['su', hook.runas, '-c', action]
result = ProcessHelper(action, env=action_env).call()
if result.returncode:
log.warn(
"Action for %s exited with %s; will retry on next iteration",
hook.name, result.returncode)
else:
metadata[hook.name + '|' +
hook.path] = self._jsonConverter.serialize(new_data)
JsonFileManager.write(self.dir, 'metadata_db.json', metadata)
log.debug("Action for %s output: %s", hook.name,
result.stdout if result.stdout else '<None>')
def _is_installed(self, pkg):
pkg_with_version = RpmTool.get_package_version(pkg)
if not pkg_with_version or not pkg_with_version[0]:
# If there's an error retrieving the version, assume we have to install it (a failure there will be terminal)
return True
pkg_spec = '-'.join(
pkg_with_version) if pkg_with_version[1] else pkg_with_version[0]
# rpm -q will try to find the specific RPM in the local system
# --quiet will reduce this command to just an exit code
test_result = ProcessHelper(['rpm', '-q', '--quiet', pkg_spec]).call()
# if rpm -q returns 0, that means the package exists
return test_result.returncode == 0
def apply(self, action, auth_config=None):
"""
Install a set of packages via RPM, returning the packages actually installed or updated.
Arguments:
action -- a dict of package name to version; version can be empty, a single string or a list of strings
Exceptions:
ToolError -- on expected failures (such as a non-zero exit code)
"""
pkgs_changed = []
if not action.keys():
log.debug("No packages installed for RPM")
return pkgs_changed
pkgs = []
for pkg_name, loc in action.iteritems():
pkgs_to_process = ([loc] if isinstance(loc, basestring) else loc)
pkgs_filtered = [
pkg_key for pkg_key in pkgs_to_process
if self._package_filter(pkg_key)
]
if pkgs_filtered:
pkgs.extend(pkgs_filtered)
pkgs_changed.append(pkg_name)
if not pkgs:
log.info("All RPMs were already installed")
return []
log.debug("Installing %s via RPM", pkgs)
result = ProcessHelper(
['rpm', '-U', '--quiet', '--nosignature', '--replacepkgs'] +
pkgs).call()
if result.returncode:
log.error("RPM failed. Output: %s", result.stdout)
raise ToolError("Could not successfully install rpm packages",
result.returncode)
else:
log.debug("RPM output: %s", result.stdout)
return pkgs_changed
def set_service_enabled(self, service, enabled=True):
if not os.path.exists(self._executable):
raise ToolError("Cannot find chkconfig")
result = ProcessHelper(
[self._executable, service, 'on' if enabled else 'off']).call()
if result.returncode:
log.error("chkconfig failed with error %s. Output: %s",
result.returncode, result.stdout)
raise ToolError(
"Could not %s service %s" %
("enable" if enabled else "disable", service),
result.returncode)
else:
log.info("%s service %s", "enabled" if enabled else "disabled",
service)
def _gem_is_installed(self, pkg, ver=None):
""""
Check to see if a package at version ver is installed.
If ver is not specified, just check for the package.
"""
log.debug("Checking to see if %s-%s is already installed", pkg, ver)
queryCmd = ['gem', 'query', '-i', '-n', '^%s$' % pkg]
if ver:
queryCmd.extend(['-v', '%s' % ver])
result = ProcessHelper(queryCmd).call()
if result.returncode:
return False
else:
return True
def _pkg_installed(self, pkg, pkg_name):
"""
Test if a package is installed (exact version match if version is specified), returning a boolean.
Arguments:
pkg -- the full package specification (including version if specified) in pkg=version format
pkg_name -- the name of the package
"""
result = ProcessHelper(['dpkg-query', '-f', '${Status}|${Package}=${Version}', '-W', pkg_name], stderr=subprocess.PIPE).call()
if result.returncode or not result.stdout:
return False
status,divider,spec = result.stdout.strip().partition('|')
if status.rpartition(" ")[2] != 'installed':
return False
return spec.startswith(pkg)
def apply(self, action, auth_config=None):
"""
Install a set of packages via easy_install, returning the packages actually installed or updated.
Arguments:
action -- a dict of package name to version; version can be empty, a single string or a list of strings
Exceptions:
ToolError -- on expected failures (such as a non-zero exit code)
"""
pkgs_changed = []
if not action.keys():
log.debug("No packages specified for python")
return pkgs_changed
pkgs = []
for pkg in action:
if not action[pkg] or isinstance(action[pkg], basestring):
pkgs.append(PythonTool._pkg_spec(pkg, action[pkg]))
else:
pkgs.extend(
PythonTool._pkg_spec(pkg, ver) for ver in action[pkg])
pkgs_changed.append(pkg)
log.info("Attempting to install %s via easy_install", pkgs)
result = ProcessHelper(['easy_install'] + pkgs).call()
if result.returncode:
log.error("easy_install failed. Output: %s", result.stdout)
raise ToolError("Could not successfully install python packages",
result.returncode)
else:
log.info("easy_install installed %s", pkgs)
log.debug("easy_install output: %s", result.stdout)
return pkgs_changed
def _modify_user(user_name, groups=[], homedir=None):
""" Modify a user and return True, else return False """
if not homedir and not groups:
log.info("No homedir or groups specified; not modifying %s", user_name)
return False
cmd = ['/usr/sbin/usermod']
if groups:
gids = _get_gids(groups)
current_gids = _gids_for_user(user_name)
if frozenset(gids) ^ frozenset(current_gids):
cmd.extend(['-G', ','.join(gids)])
else:
log.debug("Groups have not changed for %s", user_name)
if homedir:
if homedir != _get_user_homedir(user_name):
cmd.extend(['-d', homedir])
else:
log.debug("Homedir has not changed for %s", user_name)
if len(cmd) == 1:
log.debug("User %s does not need modification", user_name)
return False
cmd.append(user_name)
result = ProcessHelper(cmd).call()
if result.returncode:
log.error("Failed to modify user %s", user_name)
log.debug("Usermod output: %s", result.stdout)
raise ToolError("Failed to modify user %s" % user_name)
else:
log.info("Modified user %s successfully", user_name)
return True
def _create_user(user_name, groups=[], homedir=None, uid=None):
gids = _get_gids(groups)
cmd = ['/usr/sbin/useradd', '-M', '-r', '--shell', '/sbin/nologin']
if homedir:
cmd.extend(['-d', homedir])
if uid:
cmd.extend(['-u', uid])
if gids:
cmd.extend(['-G', ','.join(gids)])
cmd.append(user_name)
result = ProcessHelper(cmd).call()
if result.returncode:
log.error("Failed to add user %s", user_name)
log.debug("Useradd output: %s", result.stdout)
raise ToolError("Failed to add user %s" % user_name)
else:
log.info("Added user %s successfully", user_name)
def _pkg_available(self, pkg):
result = ProcessHelper(['apt-cache', '-q', 'show', pkg]).call()
return result.returncode == 0
def _pkg_installed(self, pkg):
result = ProcessHelper(['yum', '-C', '-y', 'list', 'installed',
pkg]).call()
return result.returncode == 0
def apply(self, action):
"""
Execute a set of commands, returning a list of commands that were executed.
Arguments:
action -- a dict of command to attributes, where attributes has keys of:
command: the command to run (a string or list)
cwd: working directory (a string)
env: a dictionary of environment variables
test: a commmand to run; if it returns zero, the command will run
ignoreErrors: if true, ignore errors
waitAfterCompletion: # of seconds to wait after completion (or "forever")
defaults: a command to run; the stdout will be used to provide defaults
Exceptions:
ToolError -- on expected failures
"""
commands_run = []
if not action:
log.debug("No commands specified")
return commands_run
for name in sorted(action.keys()):
log.debug(u"Running command %s", name)
attributes = action[name]
if "defaults" in attributes:
log.debug(u"Generating defaults for command %s", name)
defaultsResult = ProcessHelper(attributes['defaults'], stderr=subprocess.PIPE).call()
log.debug(u"Defaults script for %s output: %s", name, defaultsResult.stdout.decode('utf-8'))
if defaultsResult.returncode:
log.error(u"Defaults script failed for %s: %s", name, defaultsResult.stderr.decode('utf-8'))
raise ToolError(u"Defaults script for command %s failed" % name)
default_attrs = attributes
default_env = default_attrs.get("env",{})
attributes = json.loads(defaultsResult.stdout)
user_env = attributes.get("env",{})
user_env.update(default_env)
attributes.update(default_attrs)
attributes["env"] = user_env
if not "command" in attributes:
log.error(u"No command specified for %s", name)
raise ToolError(u"%s does not specify the 'command' attribute, which is required" % name)
cwd = os.path.expanduser(attributes["cwd"]) if "cwd" in attributes else None
env = attributes.get("env", None)
if "test" in attributes:
log.debug(u"Running test for command %s", name)
test = attributes["test"]
testResult = LoggingProcessHelper(test, name=u'Test for Command %s' % name, env=env, cwd=cwd).call()
log.debug(u"Test command output: %s", testResult.stdout.decode('utf-8'))
if testResult.returncode:
log.info(u"Test failed with code %s", testResult.returncode)
continue
else:
log.debug(u"Test for command %s passed", name)
else:
log.debug(u"No test for command %s", name)
cmd_to_run = attributes["command"]
if "runas" in attributes:
if os.name == 'nt':
raise ToolError(u'Command %s specified "runas", which is not supported on Windows' % name)
if isinstance(cmd_to_run, basestring):
cmd_to_run = u'su %s -c %s' % (attributes['runas'], cmd_to_run)
else:
cmd_to_run = ['su', attributes['runas'], '-c'] + cmd_to_run
commandResult = LoggingProcessHelper(cmd_to_run, name=u'Command %s' % name, env=env, cwd=cwd).call()
if commandResult.returncode:
log.error(u"Command %s (%s) failed", name, attributes["command"])
log.debug(u"Command %s output: %s", name, commandResult.stdout.decode('utf-8'))
if interpret_boolean(attributes.get("ignoreErrors")):
log.info("ignoreErrors set to true, continuing build")
commands_run.append(name)
else:
raise ToolError(u"Command %s failed" % name)
else:
log.info(u"Command %s succeeded", name)
log.debug(u"Command %s output: %s", name, commandResult.stdout.decode('utf-8'))
commands_run.append(name)
return commands_run
def apply(self, action, auth_config=None):
"""
Install a set of packages via yum, returning the packages actually installed or updated.
Arguments:
action -- a dict of package name to version; version can be empty, a single string or a list of strings
Exceptions:
ToolError -- on expected failures (such as a non-zero exit code)
"""
pkgs_changed = []
if not action.keys():
log.debug("No packages specified for yum")
return pkgs_changed
cache_result = ProcessHelper(['yum', '-y', 'makecache']).call()
if cache_result.returncode:
log.error("Yum makecache failed. Output: %s", cache_result.stdout)
raise ToolError("Could not create yum cache",
cache_result.returncode)
pkg_specs_to_upgrade = []
pkg_specs_to_downgrade = []
for pkg_name in action:
if action[pkg_name]:
if isinstance(action[pkg_name], basestring):
pkg_ver = action[pkg_name]
else:
# Yum only cares about one version anyway... so take the max specified version in the list
pkg_ver = RpmTool.max_version(action[pkg_name])
else:
pkg_ver = None
pkg_spec = '%s-%s' % (pkg_name, pkg_ver) if pkg_ver else pkg_name
if self._pkg_installed(pkg_spec):
# If the EXACT requested spec is installed, don't do anything
log.debug("%s will not be installed as it is already present",
pkg_spec)
elif not self._pkg_available(pkg_spec):
# If the requested spec is not available, blow up
log.error("%s is not available to be installed", pkg_spec)
raise ToolError(
"Yum does not have %s available for installation" %
pkg_spec)
elif not pkg_ver:
# If they didn't request a specific version, always upgrade
pkg_specs_to_upgrade.append(pkg_spec)
pkgs_changed.append(pkg_name)
else:
# They've requested a specific version that's available but not installed.
# Figure out if it's an upgrade or a downgrade
installed_version = RpmTool.get_package_version(
pkg_name, False)[1]
if self._should_upgrade(pkg_ver, installed_version):
pkg_specs_to_upgrade.append(pkg_spec)
pkgs_changed.append(pkg_name)
else:
log.debug("Downgrading to %s from installed version %s",
pkg_spec, installed_version)
pkg_specs_to_downgrade.append(pkg_spec)
pkgs_changed.append(pkg_name)
if not pkgs_changed:
log.debug("All yum packages were already installed")
return []
if pkg_specs_to_upgrade:
log.debug("Installing/updating %s via yum", pkg_specs_to_upgrade)
result = ProcessHelper(['yum', '-y', 'install'] +
pkg_specs_to_upgrade).call()
if result.returncode:
log.error("Yum failed. Output: %s", result.stdout)
raise ToolError(
"Could not successfully install/update yum packages",
result.returncode)
if pkg_specs_to_downgrade:
log.debug("Downgrading %s via yum", pkg_specs_to_downgrade)
result = ProcessHelper(['yum', '-y', 'downgrade'] +
pkg_specs_to_downgrade).call()
if result.returncode:
log.error("Yum failed. Output: %s", result.stdout)
raise ToolError(
"Could not successfully downgrade yum packages",
result.returncode)
log.info("Yum installed %s", pkgs_changed)
return pkgs_changed