def _get_script_arguments(self, named_args=None, positional_args=None):
"""
Build a string of named and positional arguments which are passed to the
script.
:param named_args: Dictionary with named arguments.
:type named_args: ``dict``.
:param positional_args: List with positional arguments.
:type positional_args: ``dict``.
:rtype: ``str``
"""
command_parts = []
# add all named_args in the format <kwarg_op>name=value (e.g. --name=value)
if named_args is not None:
for (arg, value) in six.iteritems(named_args):
if value is None or (isinstance(value, (str, six.text_type)) and len(value) < 1):
LOG.debug('Ignoring arg %s as its value is %s.', arg, value)
continue
if isinstance(value, bool):
if value is True:
command_parts.append(arg)
else:
values = (quote_unix(arg), quote_unix(six.text_type(value)))
command_parts.append(six.text_type('%s=%s' % values))
# add the positional args
if positional_args:
quoted_pos_args = [quote_unix(pos_arg) for pos_arg in positional_args]
pos_args_string = ' '.join(quoted_pos_args)
command_parts.append(pos_args_string)
return ' '.join(command_parts)
def get_pack_base_path(pack_name):
"""
Return full absolute base path to the content pack directory.
Note: This function looks for a pack in all the load paths and return path to the first pack
which matched the provided name.
If a pack is not found, we return a pack which points to the first packs directory (this is
here for backward compatibility reasons).
:param pack_name: Content pack name.
:type pack_name: ``str``
:rtype: ``str``
"""
if not pack_name:
return None
packs_base_paths = get_packs_base_paths()
for packs_base_path in packs_base_paths:
pack_base_path = os.path.join(packs_base_path, quote_unix(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
if os.path.isdir(pack_base_path):
return pack_base_path
# Path with the provided name not found
pack_base_path = os.path.join(packs_base_paths[0], quote_unix(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
return pack_base_path
def get_full_command_string(self):
# Note: We pass -E to sudo because we want to preserve user provided environment variables
env_str = self._get_env_vars_export_string()
cwd = self.get_cwd()
if self.sudo:
if env_str:
command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command))
else:
command = quote_unix('cd %s && %s' % (cwd, self.command))
sudo_arguments = ' '.join(self._get_common_sudo_arguments())
command = 'sudo %s -- bash -c %s' % (sudo_arguments, command)
if self.sudo_password:
command = ('set +o history ; echo -e %s | %s' %
(quote_unix('%s\n' % (self.sudo_password)), command))
else:
if env_str:
command = '%s && cd %s && %s' % (env_str, cwd,
self.command)
else:
command = 'cd %s && %s' % (cwd, self.command)
LOG.debug('Command to run on remote host will be: %s', command)
return command
def put(self, local_path, remote_path, mode=None, mirror_local_mode=False):
"""
Upload a file to the remote node.
:type local_path: ``st``
:param local_path: File path on the local node.
:type remote_path: ``str``
:param remote_path: File path on the remote node.
:type mode: ``int``
:param mode: Permissions mode for the file. E.g. 0744.
:type mirror_local_mode: ``int``
:param mirror_local_mode: Should remote file mirror local mode.
:return: Attributes of the remote file.
:rtype: :class:`posix.stat_result` or ``None``
"""
if not local_path or not remote_path:
raise Exception("Need both local_path and remote_path. local: %s, remote: %s" % local_path, remote_path)
local_path = quote_unix(local_path)
remote_path = quote_unix(remote_path)
extra = {
"_local_path": local_path,
"_remote_path": remote_path,
"_mode": mode,
"_mirror_local_mode": mirror_local_mode,
}
self.logger.debug("Uploading file", extra=extra)
if not os.path.exists(local_path):
raise Exception("Path %s does not exist locally." % local_path)
rattrs = self.sftp.put(local_path, remote_path)
if mode or mirror_local_mode:
local_mode = mode
if not mode or mirror_local_mode:
local_mode = os.stat(local_path).st_mode
# Cast to octal integer in case of string
if isinstance(local_mode, basestring):
local_mode = int(local_mode, 8)
local_mode = local_mode & 07777
remote_mode = rattrs.st_mode
# Only bitshift if we actually got an remote_mode
if remote_mode is not None:
remote_mode = remote_mode & 07777
if local_mode != remote_mode:
self.sftp.chmod(remote_path, local_mode)
return rattrs
def _get_env_vars_export_string(self):
if self.env_vars:
# Envrionment variables could contain spaces and open us to shell
# injection attacks. Always quote the key and the value.
exports = ' '.join(
'%s=%s' % (quote_unix(k), quote_unix(v))
for k, v in self.env_vars.iteritems()
)
shell_env_str = '%s %s' % (ShellCommandAction.EXPORT_CMD, exports)
else:
shell_env_str = ''
return shell_env_str
def get_pack_base_path(pack_name, include_trailing_slash=False, use_pack_cache=False):
"""
Return full absolute base path to the content pack directory.
Note: This function looks for a pack in all the load paths and return path to the first pack
which matched the provided name.
If a pack is not found, we return a pack which points to the first packs directory (this is
here for backward compatibility reasons).
:param pack_name: Content pack name.
:type pack_name: ``str``
:param include_trailing_slash: True to include trailing slash.
:type include_trailing_slash: ``bool``
:param use_pack_cache: True to cache base paths on per-pack basis. This help in situations
where this method is called multiple times with the same pack name.
:type use_pack_cache`` ``bool``
:rtype: ``str``
"""
if not pack_name:
return None
if use_pack_cache and pack_name in PACK_NAME_TO_BASE_PATH_CACHE:
return PACK_NAME_TO_BASE_PATH_CACHE[pack_name]
packs_base_paths = get_packs_base_paths()
for packs_base_path in packs_base_paths:
pack_base_path = os.path.join(packs_base_path, quote_unix(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
if os.path.isdir(pack_base_path):
if include_trailing_slash and not pack_base_path.endswith(os.path.sep):
pack_base_path += os.path.sep
PACK_NAME_TO_BASE_PATH_CACHE[pack_name] = pack_base_path
return pack_base_path
# Path with the provided name not found
pack_base_path = os.path.join(packs_base_paths[0], quote_unix(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
if include_trailing_slash and not pack_base_path.endswith(os.path.sep):
pack_base_path += os.path.sep
PACK_NAME_TO_BASE_PATH_CACHE[pack_name] = pack_base_path
return pack_base_path
def get_rpm_package_list(name_startswith):
cmd = 'rpm -qa | grep %s' % (quote_unix(name_startswith))
exit_code, stdout, _ = run_command(cmd=cmd, shell=True)
lines = stdout.split('\n')
packages = []
for line in lines:
line = line.strip()
if not line:
continue
split = line.rsplit('.', 1)
split = split[0].split('-', 1)
name = split[0]
version = split[1]
if not name.startswith(name_startswith):
continue
item = {
'name': name,
'version': version
}
packages.append(item)
return packages
def get_deb_package_list(name_startswith):
cmd = 'dpkg -l | grep %s' % (quote_unix(name_startswith))
exit_code, stdout, _ = run_command(cmd=cmd, shell=True)
lines = stdout.split('\n')
packages = []
for line in lines:
line = line.strip()
if not line:
continue
split = re.split(r'\s+', line)
name = split[1]
version = split[2]
if not name.startswith(name_startswith):
continue
item = {
'name': name,
'version': version
}
packages.append(item)
return packages
def _get_script_arguments(self, named_args=None, positional_args=None):
"""
Build a string of named and positional arguments which are passed to the
script.
:param named_args: Dictionary with named arguments.
:type named_args: ``dict``.
:param positional_args: List with positional arguments.
:type positional_args: ``dict``.
:rtype: ``str``
"""
command_parts = []
# add all named_args in the format <kwarg_op>name=value (e.g. --name=value)
if named_args is not None:
for (arg, value) in six.iteritems(named_args):
if value is None or (isinstance(value, (str, unicode)) and len(value) < 1):
LOG.debug("Ignoring arg %s as its value is %s.", arg, value)
continue
if isinstance(value, bool):
if value is True:
command_parts.append(arg)
else:
command_parts.append("%s=%s" % (arg, quote_unix(str(value))))
# add the positional args
if positional_args:
command_parts.append(positional_args)
return " ".join(command_parts)
def delete_dir(self, path, force=False, timeout=None):
"""
Delete a dir on remote box.
:param path: Path to remote dir to be deleted.
:type path: ``str``
:param force: Optional Forcefully remove dir.
:type force: ``bool``
:param timeout: Optional Time to wait for dir to be deleted. Only relevant for force.
:type timeout: ``int``
:return: True if the file has been successfully deleted, False
otherwise.
:rtype: ``bool``
"""
path = quote_unix(path)
extra = {"_path": path}
if force:
command = "rm -rf %s" % path
extra["_command"] = command
extra["_force"] = force
self.logger.debug("Deleting dir", extra=extra)
return self.run(command, timeout=timeout)
self.logger.debug("Deleting dir", extra=extra)
return self.sftp.rmdir(path)
def get_full_command_string(self):
# Note: We pass -E to sudo because we want to preserve user provided
# environment variables
if self.sudo:
command = quote_unix(self.command)
command = 'sudo -E -- bash -c %s' % (command)
else:
if self.user and self.user != LOGGED_USER_USERNAME:
# Need to use sudo to run as a different user
user = quote_unix(self.user)
command = quote_unix(self.command)
command = 'sudo -E -u %s -- bash -c %s' % (user, command)
else:
command = self.command
return command
def _setup_pack_virtualenv(self, pack_name, update=False):
"""
Setup virtual environment for the provided pack.
:param pack_name: Pack name.
:type pack_name: ``str``
"""
# Prevent directory traversal by whitelisting allowed characters in the
# pack name
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
raise Exception(msg)
if not os.path.exists(self._base_virtualenvs_path):
os.makedirs(self._base_virtualenvs_path)
# If we don't want to update, or if the virtualenv doesn't exist, let's create it.
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
self._remove_virtualenv(virtualenv_path=virtualenv_path)
# 1. Create virtual environment
self.logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
self._create_virtualenv(virtualenv_path=virtualenv_path)
# 2. Install base requirements which are common to all the packs
self.logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
self._install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
self.logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
self._install_requirements(virtualenv_path, requirements_file_path)
else:
self.logger.debug('No pack specific requirements found')
self.logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name,
'updated' if update else 'created',
virtualenv_path))
def get_full_command_string(self):
# Note: We pass -E to sudo because we want to preserve user provided environment variables
if self.sudo:
command = quote_unix(self.command)
sudo_arguments = ' '.join(self._get_common_sudo_arguments())
command = 'sudo %s -- bash -c %s' % (sudo_arguments, command)
else:
if self.user and self.user != LOGGED_USER_USERNAME:
# Need to use sudo to run as a different (requested) user
user = quote_unix(self.user)
sudo_arguments = ' '.join(self._get_user_sudo_arguments(user=user))
command = quote_unix(self.command)
command = 'sudo %s -- bash -c %s' % (sudo_arguments, command)
else:
command = self.command
return command
def _format_command(self):
script_arguments = self._get_script_arguments(named_args=self.named_args,
positional_args=self.positional_args)
if self.sudo:
if script_arguments:
command = quote_unix('%s %s' % (self.remote_script, script_arguments))
else:
command = quote_unix(self.remote_script)
command = 'sudo -E -- bash -c %s' % (command)
else:
script_path = quote_unix(self.remote_script)
if script_arguments:
command = '%s %s' % (script_path, script_arguments)
else:
command = script_path
return command
def get_full_command_string(self):
script_arguments = self._get_script_arguments(named_args=self.named_args,
positional_args=self.positional_args)
if self.sudo:
if script_arguments:
command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments))
else:
command = quote_unix(self.script_local_path_abs)
command = 'sudo -E -- bash -c %s' % (command)
else:
if self.user and self.user != LOGGED_USER_USERNAME:
# Need to use sudo to run as a different user
user = quote_unix(self.user)
if script_arguments:
command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments))
else:
command = quote_unix(self.script_local_path_abs)
command = 'sudo -E -u %s -- bash -c %s' % (user, command)
else:
script_path = quote_unix(self.script_local_path_abs)
if script_arguments:
command = '%s %s' % (script_path, script_arguments)
else:
command = script_path
return command
def _format_command(self):
script_arguments = self._get_script_arguments(named_args=self.named_args,
positional_args=self.positional_args)
if self.sudo:
if script_arguments:
command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments))
else:
command = quote_unix(self.script_local_path_abs)
sudo_arguments = ' '.join(self._get_common_sudo_arguments())
command = 'sudo %s -- bash -c %s' % (sudo_arguments, command)
else:
if self.user and self.user != LOGGED_USER_USERNAME:
# Need to use sudo to run as a different user
user = quote_unix(self.user)
if script_arguments:
command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments))
else:
command = quote_unix(self.script_local_path_abs)
sudo_arguments = ' '.join(self._get_user_sudo_arguments(user=user))
command = 'sudo %s -- bash -c %s' % (sudo_arguments, command)
else:
script_path = quote_unix(self.script_local_path_abs)
if script_arguments:
command = '%s %s' % (script_path, script_arguments)
else:
command = script_path
return command
def get_full_command_string(self):
# Note: We pass -E to sudo because we want to preserve user provided
# environment variables
env_str = self._get_env_vars_export_string()
cwd = self.get_cwd()
if self.sudo:
if env_str:
command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command))
else:
command = quote_unix('cd %s && %s' % (cwd, self.command))
command = 'sudo -E -- bash -c %s' % (command)
else:
if env_str:
command = '%s && cd %s && %s' % (env_str, cwd,
self.command)
else:
command = 'cd %s && %s' % (cwd, self.command)
LOG.debug('Command to run on remote host will be: %s', command)
return command
def _get_env_vars_export_string(self):
if self.env_vars:
env_vars = copy.copy(self.env_vars)
# If sudo_password is provided, explicitly disable bash history to make sure password
# is not logged, because password is provided via command line
if self.sudo and self.sudo_password:
env_vars['HISTFILE'] = '/dev/null'
env_vars['HISTSIZE'] = '0'
# Sort the dict to guarantee consistent order
env_vars = collections.OrderedDict(sorted(env_vars.items()))
# Environment variables could contain spaces and open us to shell
# injection attacks. Always quote the key and the value.
exports = ' '.join(
'%s=%s' % (quote_unix(k), quote_unix(v))
for k, v in six.iteritems(env_vars)
)
shell_env_str = '%s %s' % (ShellCommandAction.EXPORT_CMD, exports)
else:
shell_env_str = ''
return shell_env_str
def _get_command_string(self, cmd, args):
"""
Escape the command arguments and form a command string.
:type cmd: ``str``
:type args: ``list``
:rtype: ``str``
"""
assert isinstance(args, (list, tuple))
args = [quote_unix(arg) for arg in args]
args = ' '.join(args)
result = '%s %s' % (cmd, args)
return result
def mkdir(self, dir_path):
"""
Create a directory on remote box.
:param dir_path: Path to remote directory to be created.
:type dir_path: ``str``
:return: Returns nothing if successful else raises IOError exception.
:rtype: ``None``
"""
dir_path = quote_unix(dir_path)
extra = {"_dir_path": dir_path}
self.logger.debug("mkdir", extra=extra)
return self.sftp.mkdir(dir_path)
def _prepare_command(self, has_inputs, inputs_file_path):
LOG.debug('CloudSlang home: %s', self._cloudslang_home)
cloudslang_binary = os.path.join(self._cloudslang_home, 'bin/cslang')
LOG.debug('Using CloudSlang binary: %s', cloudslang_binary)
command_args = ['--f', self._flow_path,
'--cp', self._cloudslang_home]
if has_inputs:
command_args += ['--if', inputs_file_path]
command = cloudslang_binary + " run " + " ".join([quote_unix(arg) for arg in command_args])
LOG.info('Executing action via CloudSlangRunner: %s', self.runner_id)
LOG.debug('Command is: %s', command)
return command
def delete_file(self, path):
"""
Delete a file on remote box.
:param path: Path to remote file to be deleted.
:type path: ``str``
:return: True if the file has been successfully deleted, False
otherwise.
:rtype: ``bool``
"""
path = quote_unix(path)
extra = {"_path": path}
self.logger.debug("Deleting file", extra=extra)
self.sftp.unlink(path)
return True
def __init__(
self,
name,
action_exec_id,
script_local_path_abs,
script_local_libs_path_abs,
named_args=None,
positional_args=None,
env_vars=None,
on_behalf_user=None,
user=None,
password=None,
private_key=None,
remote_dir=None,
hosts=None,
parallel=True,
sudo=False,
timeout=None,
cwd=None,
):
super(RemoteScriptAction, self).__init__(
name=name,
action_exec_id=action_exec_id,
script_local_path_abs=script_local_path_abs,
user=user,
named_args=named_args,
positional_args=positional_args,
env_vars=env_vars,
sudo=sudo,
timeout=timeout,
cwd=cwd,
)
self.script_local_libs_path_abs = script_local_libs_path_abs
self.script_local_dir, self.script_name = os.path.split(self.script_local_path_abs)
self.remote_dir = remote_dir if remote_dir is not None else "/tmp"
self.remote_libs_path_abs = os.path.join(self.remote_dir, ACTION_LIBS_DIR)
self.on_behalf_user = on_behalf_user
self.password = password
self.private_key = private_key
self.remote_script = os.path.join(self.remote_dir, quote_unix(self.script_name))
self.hosts = hosts
self.parallel = parallel
self.command = self._format_command()
LOG.debug("RemoteScriptAction: command to run on remote box: %s", self.command)
def run(self, packs, abs_repo_base):
intersection = BLOCKED_PACKS & frozenset(packs)
if len(intersection) > 0:
names = ", ".join(list(intersection))
raise ValueError("Uninstall includes an uninstallable pack - %s." % (names))
# 1. Delete pack content
for fp in os.listdir(abs_repo_base):
abs_fp = os.path.join(abs_repo_base, fp)
if fp in packs and os.path.isdir(abs_fp):
self.logger.debug('Deleting pack directory "%s"' % (abs_fp))
shutil.rmtree(abs_fp)
# 2. Delete pack virtual environment
for pack_name in packs:
pack_name = quote_unix(pack_name)
virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name)
if os.path.isdir(virtualenv_path):
self.logger.debug('Deleting virtualenv "%s" for pack "%s"' % (virtualenv_path, pack_name))
shutil.rmtree(virtualenv_path)
def _format_command(self):
script_arguments = self._get_script_arguments(named_args=self.named_args,
positional_args=self.positional_args)
env_str = self._get_env_vars_export_string()
cwd = quote_unix(self.get_cwd())
script_path = quote_unix(self.remote_script)
if self.sudo:
if script_arguments:
if env_str:
command = quote_unix('%s && cd %s && %s %s' % (
env_str, cwd, script_path, script_arguments))
else:
command = quote_unix('cd %s && %s %s' % (
cwd, script_path, script_arguments))
else:
if env_str:
command = quote_unix('%s && cd %s && %s' % (
env_str, cwd, script_path))
else:
command = quote_unix('cd %s && %s' % (cwd, script_path))
sudo_arguments = ' '.join(self._get_common_sudo_arguments())
command = 'sudo %s -- bash -c %s' % (sudo_arguments, command)
if self.sudo_password:
command = ('set +o history ; echo -e %s | %s' %
(quote_unix('%s\n' % (self.sudo_password)), command))
else:
if script_arguments:
if env_str:
command = '%s && cd %s && %s %s' % (env_str, cwd,
script_path, script_arguments)
else:
command = 'cd %s && %s %s' % (cwd, script_path, script_arguments)
else:
if env_str:
command = '%s && cd %s && %s' % (env_str, cwd, script_path)
else:
command = 'cd %s && %s' % (cwd, script_path)
return command
def test_quote_unix(self):
arguments = [
'foo',
'foo bar',
'foo1 bar1',
'"foo"',
'"foo" "bar"',
"'foo bar'"
]
expected_values = [
"""
foo
""",
"""
'foo bar'
""",
"""
'foo1 bar1'
""",
"""
'"foo"'
""",
"""
'"foo" "bar"'
""",
"""
''"'"'foo bar'"'"''
"""
]
for argument, expected_value in zip(arguments, expected_values):
actual_value = quote_unix(value=argument)
expected_value = expected_value.lstrip()
self.assertEqual(actual_value, expected_value.strip())
def get_pack_directory(pack_name):
"""
Retrieve a directory for the provided pack.
If a directory for the provided pack doesn't exist in any of the search paths, None
is returned instead.
Note: If same pack exists in multiple search path, path to the first one is returned.
:param pack_name: Pack name.
:type pack_name: ``str``
:return: Pack to the pack directory.
:rtype: ``str`` or ``None``
"""
packs_base_paths = get_packs_base_paths()
for packs_base_path in packs_base_paths:
pack_base_path = os.path.join(packs_base_path, quote_unix(pack_name))
pack_base_path = os.path.abspath(pack_base_path)
if os.path.isdir(pack_base_path):
return pack_base_path
return None
def _format_command(self):
script_arguments = self._get_script_arguments(named_args=self.named_args,
positional_args=self.positional_args)
env_str = self._get_env_vars_export_string()
cwd = quote_unix(self.get_cwd())
script_path = quote_unix(self.remote_script)
if self.sudo:
if script_arguments:
if env_str:
command = quote_unix('%s && cd %s && %s %s' % (
env_str, cwd, script_path, script_arguments))
else:
command = quote_unix('cd %s && %s %s' % (
cwd, script_path, script_arguments))
else:
if env_str:
command = quote_unix('%s && cd %s && %s' % (
env_str, cwd, script_path))
else:
command = quote_unix('cd %s && %s' % (cwd, script_path))
command = 'sudo -E -- bash -c %s' % (command)
else:
if script_arguments:
if env_str:
command = '%s && cd %s && %s %s' % (env_str, cwd,
script_path, script_arguments)
else:
command = 'cd %s && %s %s' % (cwd, script_path, script_arguments)
else:
if env_str:
command = '%s && cd %s && %s' % (env_str, cwd, script_path)
else:
command = 'cd %s && %s' % (cwd, script_path)
return command
def setup_pack_virtualenv(pack_name,
update=False,
logger=None,
include_pip=True,
include_setuptools=True,
include_wheel=True,
proxy_config=None,
no_download=True,
force_owner_group=True):
"""
Setup virtual environment for the provided pack.
:param pack_name: Name of the pack to setup the virtualenv for.
:type pack_name: ``str``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
:param logger: Optional logger instance to use. If not provided it defaults to the module
level logger.
:param no_download: Do not download and install latest version of pre-installed packages such
as pip and distutils.
:type no_download: ``bool``
"""
logger = logger or LOG
if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path,
'virtualenvs/')
virtualenv_path = os.path.join(base_virtualenvs_path,
quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
logger.debug('Setting up virtualenv for pack "%s" (%s)' %
(pack_name, pack_path))
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name,
search_paths)
raise Exception(msg)
# 1. Create virtualenv if it doesn't exist
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 1. Create virtual environment
logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
create_virtualenv(virtualenv_path=virtualenv_path,
logger=logger,
include_pip=include_pip,
include_setuptools=include_setuptools,
include_wheel=include_wheel,
no_download=no_download)
# 2. Install base requirements which are common to all the packs
logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement,
proxy_config=proxy_config,
logger=logger)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
install_requirements(virtualenv_path=virtualenv_path,
requirements_file_path=requirements_file_path,
proxy_config=proxy_config,
logger=logger)
else:
logger.debug('No pack specific requirements found')
# 4. Set the owner group
if force_owner_group:
apply_pack_owner_group(pack_path=virtualenv_path)
action = 'updated' if update else 'created'
logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, action, virtualenv_path))
def run(self,
cmd,
timeout=None,
quote=False,
call_line_handler_func=False):
"""
Note: This function is based on paramiko's exec_command()
method.
:param timeout: How long to wait (in seconds) for the command to finish (optional).
:type timeout: ``float``
:param call_line_handler_func: True to call handle_stdout_line_func function for each line
of received stdout and handle_stderr_line_func for each
line of stderr.
:type call_line_handler_func: ``bool``
"""
if quote:
cmd = quote_unix(cmd)
extra = {'_cmd': cmd}
self.logger.info('Executing command', extra=extra)
# Use the system default buffer size
bufsize = -1
transport = self.client.get_transport()
chan = transport.open_session()
start_time = time.time()
if cmd.startswith('sudo'):
# Note that fabric does this as well. If you set pty, stdout and stderr
# streams will be combined into one.
chan.get_pty()
chan.exec_command(cmd)
stdout = StringIO()
stderr = StringIO()
# Create a stdin file and immediately close it to prevent any
# interactive script from hanging the process.
stdin = chan.makefile('wb', bufsize)
stdin.close()
# Receive all the output
# Note #1: This is used instead of chan.makefile approach to prevent
# buffering issues and hanging if the executed command produces a lot
# of output.
#
# Note #2: If you are going to remove "ready" checks inside the loop
# you are going to have a bad time. Trying to consume from a channel
# which is not ready will block for indefinitely.
exit_status_ready = chan.exit_status_ready()
if exit_status_ready:
stdout_data = self._consume_stdout(
chan=chan, call_line_handler_func=call_line_handler_func)
stdout_data = stdout_data.getvalue()
stderr_data = self._consume_stderr(
chan=chan, call_line_handler_func=call_line_handler_func)
stderr_data = stderr_data.getvalue()
stdout.write(stdout_data)
stderr.write(stderr_data)
while not exit_status_ready:
current_time = time.time()
elapsed_time = (current_time - start_time)
if timeout and (elapsed_time > timeout):
# TODO: Is this the right way to clean up?
chan.close()
stdout = strip_shell_chars(stdout.getvalue())
stderr = strip_shell_chars(stderr.getvalue())
raise SSHCommandTimeoutError(cmd=cmd,
timeout=timeout,
stdout=stdout,
stderr=stderr)
stdout_data = self._consume_stdout(
chan=chan, call_line_handler_func=call_line_handler_func)
stdout_data = stdout_data.getvalue()
stderr_data = self._consume_stderr(
chan=chan, call_line_handler_func=call_line_handler_func)
stderr_data = stderr_data.getvalue()
stdout.write(stdout_data)
stderr.write(stderr_data)
# We need to check the exit status here, because the command could
# print some output and exit during this sleep below.
exit_status_ready = chan.exit_status_ready()
if exit_status_ready:
break
# Short sleep to prevent busy waiting
eventlet.sleep(self.SLEEP_DELAY)
# print('Wait over. Channel must be ready for host: %s' % self.hostname)
# Receive the exit status code of the command we ran.
status = chan.recv_exit_status()
stdout = strip_shell_chars(stdout.getvalue())
stderr = strip_shell_chars(stderr.getvalue())
extra = {'_status': status, '_stdout': stdout, '_stderr': stderr}
self.logger.debug('Command finished', extra=extra)
return [stdout, stderr, status]
def setup_pack_virtualenv(pack_name, update=False, logger=None):
"""
Setup virtual environment for the provided pack.
:param pack_name: Name of the pack to setup the virtualenv for.
:type pack_name: ``str``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
:param logger: Optional logger instance to use. If not provided it defaults to the module
level logger.
"""
logger = logger or LOG
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/')
logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
raise Exception(msg)
# 1. Create virtualenv if it doesn't exist
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 1. Create virtual environment
logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
create_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 2. Install base requirements which are common to all the packs
logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path, requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
install_requirements(virtualenv_path=virtualenv_path,
requirements_file_path=requirements_file_path)
else:
logger.debug('No pack specific requirements found')
action = 'updated' if update else 'created'
logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, action, virtualenv_path))
def run(self, cmd, timeout=None, quote=False):
"""
Note: This function is based on paramiko's exec_command()
method.
:param timeout: How long to wait (in seconds) for the command to
finish (optional).
:type timeout: ``float``
"""
if quote:
cmd = quote_unix(cmd)
extra = {"_cmd": cmd}
self.logger.info("Executing command", extra=extra)
# Use the system default buffer size
bufsize = -1
transport = self.client.get_transport()
chan = transport.open_session()
start_time = time.time()
if cmd.startswith("sudo"):
# Note that fabric does this as well. If you set pty, stdout and stderr
# streams will be combined into one.
chan.get_pty()
chan.exec_command(cmd)
stdout = StringIO()
stderr = StringIO()
# Create a stdin file and immediately close it to prevent any
# interactive script from hanging the process.
stdin = chan.makefile("wb", bufsize)
stdin.close()
# Receive all the output
# Note #1: This is used instead of chan.makefile approach to prevent
# buffering issues and hanging if the executed command produces a lot
# of output.
#
# Note #2: If you are going to remove "ready" checks inside the loop
# you are going to have a bad time. Trying to consume from a channel
# which is not ready will block for indefinitely.
exit_status_ready = chan.exit_status_ready()
if exit_status_ready:
stdout.write(self._consume_stdout(chan).getvalue())
stderr.write(self._consume_stderr(chan).getvalue())
while not exit_status_ready:
current_time = time.time()
elapsed_time = current_time - start_time
if timeout and (elapsed_time > timeout):
# TODO: Is this the right way to clean up?
chan.close()
stdout = strip_shell_chars(stdout.getvalue())
stderr = strip_shell_chars(stderr.getvalue())
raise SSHCommandTimeoutError(cmd=cmd, timeout=timeout, stdout=stdout, stderr=stderr)
stdout.write(self._consume_stdout(chan).getvalue())
stderr.write(self._consume_stderr(chan).getvalue())
# We need to check the exist status here, because the command could
# print some output and exit during this sleep bellow.
exit_status_ready = chan.exit_status_ready()
if exit_status_ready:
break
# Short sleep to prevent busy waiting
eventlet.sleep(self.SLEEP_DELAY)
# print('Wait over. Channel must be ready for host: %s' % self.hostname)
# Receive the exit status code of the command we ran.
status = chan.recv_exit_status()
stdout = strip_shell_chars(stdout.getvalue())
stderr = strip_shell_chars(stderr.getvalue())
extra = {"_status": status, "_stdout": stdout, "_stderr": stderr}
self.logger.debug("Command finished", extra=extra)
return [stdout, stderr, status]
check=True,
stdout=subprocess.PIPE)
distro = result.stdout.decode("utf-8").strip()
if not distro:
raise ValueError("Fail to detect distribution we are running on")
return distro
if len(sys.argv) < 3:
raise ValueError("Usage: service.py <action> <service>")
distro = get_linux_distribution()
args = {"act": quote_unix(sys.argv[1]), "service": quote_unix(sys.argv[2])}
print("Detected distro: %s" % (distro))
if re.search(distro, "Ubuntu"):
if os.path.isfile("/etc/init/%s.conf" % args["service"]):
cmd_args = ["service", args["service"], args["act"]]
elif os.path.isfile("/etc/init.d/%s" % args["service"]):
cmd_args = ["/etc/init.d/%s" % (args["service"]), args["act"]]
else:
print("Unknown service")
sys.exit(2)
elif (re.search(distro, "Redhat") or re.search(distro, "Fedora")
or re.search(distro, "CentOS") or re.search(distro, "Rocky Linux")):
cmd_args = ["systemctl", args["act"], args["service"]]
