def _get_script_arguments(self, named_args=None, positional_args=None): """ Build a string of named and positional arguments which are passed to the script. :param named_args: Dictionary with named arguments. :type named_args: ``dict``. :param positional_args: List with positional arguments. :type positional_args: ``dict``. :rtype: ``str`` """ command_parts = [] # add all named_args in the format <kwarg_op>name=value (e.g. --name=value) if named_args is not None: for (arg, value) in six.iteritems(named_args): if value is None or (isinstance(value, (str, six.text_type)) and len(value) < 1): LOG.debug('Ignoring arg %s as its value is %s.', arg, value) continue if isinstance(value, bool): if value is True: command_parts.append(arg) else: values = (quote_unix(arg), quote_unix(six.text_type(value))) command_parts.append(six.text_type('%s=%s' % values)) # add the positional args if positional_args: quoted_pos_args = [quote_unix(pos_arg) for pos_arg in positional_args] pos_args_string = ' '.join(quoted_pos_args) command_parts.append(pos_args_string) return ' '.join(command_parts)
def get_pack_base_path(pack_name): """ Return full absolute base path to the content pack directory. Note: This function looks for a pack in all the load paths and return path to the first pack which matched the provided name. If a pack is not found, we return a pack which points to the first packs directory (this is here for backward compatibility reasons). :param pack_name: Content pack name. :type pack_name: ``str`` :rtype: ``str`` """ if not pack_name: return None packs_base_paths = get_packs_base_paths() for packs_base_path in packs_base_paths: pack_base_path = os.path.join(packs_base_path, quote_unix(pack_name)) pack_base_path = os.path.abspath(pack_base_path) if os.path.isdir(pack_base_path): return pack_base_path # Path with the provided name not found pack_base_path = os.path.join(packs_base_paths[0], quote_unix(pack_name)) pack_base_path = os.path.abspath(pack_base_path) return pack_base_path
def get_full_command_string(self): # Note: We pass -E to sudo because we want to preserve user provided environment variables env_str = self._get_env_vars_export_string() cwd = self.get_cwd() if self.sudo: if env_str: command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command)) else: command = quote_unix('cd %s && %s' % (cwd, self.command)) sudo_arguments = ' '.join(self._get_common_sudo_arguments()) command = 'sudo %s -- bash -c %s' % (sudo_arguments, command) if self.sudo_password: command = ('set +o history ; echo -e %s | %s' % (quote_unix('%s\n' % (self.sudo_password)), command)) else: if env_str: command = '%s && cd %s && %s' % (env_str, cwd, self.command) else: command = 'cd %s && %s' % (cwd, self.command) LOG.debug('Command to run on remote host will be: %s', command) return command
def put(self, local_path, remote_path, mode=None, mirror_local_mode=False): """ Upload a file to the remote node. :type local_path: ``st`` :param local_path: File path on the local node. :type remote_path: ``str`` :param remote_path: File path on the remote node. :type mode: ``int`` :param mode: Permissions mode for the file. E.g. 0744. :type mirror_local_mode: ``int`` :param mirror_local_mode: Should remote file mirror local mode. :return: Attributes of the remote file. :rtype: :class:`posix.stat_result` or ``None`` """ if not local_path or not remote_path: raise Exception("Need both local_path and remote_path. local: %s, remote: %s" % local_path, remote_path) local_path = quote_unix(local_path) remote_path = quote_unix(remote_path) extra = { "_local_path": local_path, "_remote_path": remote_path, "_mode": mode, "_mirror_local_mode": mirror_local_mode, } self.logger.debug("Uploading file", extra=extra) if not os.path.exists(local_path): raise Exception("Path %s does not exist locally." % local_path) rattrs = self.sftp.put(local_path, remote_path) if mode or mirror_local_mode: local_mode = mode if not mode or mirror_local_mode: local_mode = os.stat(local_path).st_mode # Cast to octal integer in case of string if isinstance(local_mode, basestring): local_mode = int(local_mode, 8) local_mode = local_mode & 07777 remote_mode = rattrs.st_mode # Only bitshift if we actually got an remote_mode if remote_mode is not None: remote_mode = remote_mode & 07777 if local_mode != remote_mode: self.sftp.chmod(remote_path, local_mode) return rattrs
def _get_env_vars_export_string(self): if self.env_vars: # Envrionment variables could contain spaces and open us to shell # injection attacks. Always quote the key and the value. exports = ' '.join( '%s=%s' % (quote_unix(k), quote_unix(v)) for k, v in self.env_vars.iteritems() ) shell_env_str = '%s %s' % (ShellCommandAction.EXPORT_CMD, exports) else: shell_env_str = '' return shell_env_str
def get_pack_base_path(pack_name, include_trailing_slash=False, use_pack_cache=False): """ Return full absolute base path to the content pack directory. Note: This function looks for a pack in all the load paths and return path to the first pack which matched the provided name. If a pack is not found, we return a pack which points to the first packs directory (this is here for backward compatibility reasons). :param pack_name: Content pack name. :type pack_name: ``str`` :param include_trailing_slash: True to include trailing slash. :type include_trailing_slash: ``bool`` :param use_pack_cache: True to cache base paths on per-pack basis. This help in situations where this method is called multiple times with the same pack name. :type use_pack_cache`` ``bool`` :rtype: ``str`` """ if not pack_name: return None if use_pack_cache and pack_name in PACK_NAME_TO_BASE_PATH_CACHE: return PACK_NAME_TO_BASE_PATH_CACHE[pack_name] packs_base_paths = get_packs_base_paths() for packs_base_path in packs_base_paths: pack_base_path = os.path.join(packs_base_path, quote_unix(pack_name)) pack_base_path = os.path.abspath(pack_base_path) if os.path.isdir(pack_base_path): if include_trailing_slash and not pack_base_path.endswith(os.path.sep): pack_base_path += os.path.sep PACK_NAME_TO_BASE_PATH_CACHE[pack_name] = pack_base_path return pack_base_path # Path with the provided name not found pack_base_path = os.path.join(packs_base_paths[0], quote_unix(pack_name)) pack_base_path = os.path.abspath(pack_base_path) if include_trailing_slash and not pack_base_path.endswith(os.path.sep): pack_base_path += os.path.sep PACK_NAME_TO_BASE_PATH_CACHE[pack_name] = pack_base_path return pack_base_path
def get_rpm_package_list(name_startswith): cmd = 'rpm -qa | grep %s' % (quote_unix(name_startswith)) exit_code, stdout, _ = run_command(cmd=cmd, shell=True) lines = stdout.split('\n') packages = [] for line in lines: line = line.strip() if not line: continue split = line.rsplit('.', 1) split = split[0].split('-', 1) name = split[0] version = split[1] if not name.startswith(name_startswith): continue item = { 'name': name, 'version': version } packages.append(item) return packages
def get_deb_package_list(name_startswith): cmd = 'dpkg -l | grep %s' % (quote_unix(name_startswith)) exit_code, stdout, _ = run_command(cmd=cmd, shell=True) lines = stdout.split('\n') packages = [] for line in lines: line = line.strip() if not line: continue split = re.split(r'\s+', line) name = split[1] version = split[2] if not name.startswith(name_startswith): continue item = { 'name': name, 'version': version } packages.append(item) return packages
def _get_script_arguments(self, named_args=None, positional_args=None): """ Build a string of named and positional arguments which are passed to the script. :param named_args: Dictionary with named arguments. :type named_args: ``dict``. :param positional_args: List with positional arguments. :type positional_args: ``dict``. :rtype: ``str`` """ command_parts = [] # add all named_args in the format <kwarg_op>name=value (e.g. --name=value) if named_args is not None: for (arg, value) in six.iteritems(named_args): if value is None or (isinstance(value, (str, unicode)) and len(value) < 1): LOG.debug("Ignoring arg %s as its value is %s.", arg, value) continue if isinstance(value, bool): if value is True: command_parts.append(arg) else: command_parts.append("%s=%s" % (arg, quote_unix(str(value)))) # add the positional args if positional_args: command_parts.append(positional_args) return " ".join(command_parts)
def delete_dir(self, path, force=False, timeout=None): """ Delete a dir on remote box. :param path: Path to remote dir to be deleted. :type path: ``str`` :param force: Optional Forcefully remove dir. :type force: ``bool`` :param timeout: Optional Time to wait for dir to be deleted. Only relevant for force. :type timeout: ``int`` :return: True if the file has been successfully deleted, False otherwise. :rtype: ``bool`` """ path = quote_unix(path) extra = {"_path": path} if force: command = "rm -rf %s" % path extra["_command"] = command extra["_force"] = force self.logger.debug("Deleting dir", extra=extra) return self.run(command, timeout=timeout) self.logger.debug("Deleting dir", extra=extra) return self.sftp.rmdir(path)
def get_full_command_string(self): # Note: We pass -E to sudo because we want to preserve user provided # environment variables if self.sudo: command = quote_unix(self.command) command = 'sudo -E -- bash -c %s' % (command) else: if self.user and self.user != LOGGED_USER_USERNAME: # Need to use sudo to run as a different user user = quote_unix(self.user) command = quote_unix(self.command) command = 'sudo -E -u %s -- bash -c %s' % (user, command) else: command = self.command return command
def _setup_pack_virtualenv(self, pack_name, update=False): """ Setup virtual environment for the provided pack. :param pack_name: Pack name. :type pack_name: ``str`` """ # Prevent directory traversal by whitelisting allowed characters in the # pack name if not re.match(PACK_NAME_WHITELIST, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name)) virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) if not os.path.exists(self._base_virtualenvs_path): os.makedirs(self._base_virtualenvs_path) # If we don't want to update, or if the virtualenv doesn't exist, let's create it. if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists self._remove_virtualenv(virtualenv_path=virtualenv_path) # 1. Create virtual environment self.logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) self._create_virtualenv(virtualenv_path=virtualenv_path) # 2. Install base requirements which are common to all the packs self.logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: self._install_requirement(virtualenv_path=virtualenv_path, requirement=requirement) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: self.logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) self._install_requirements(virtualenv_path, requirements_file_path) else: self.logger.debug('No pack specific requirements found') self.logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, 'updated' if update else 'created', virtualenv_path))
def get_full_command_string(self): # Note: We pass -E to sudo because we want to preserve user provided environment variables if self.sudo: command = quote_unix(self.command) sudo_arguments = ' '.join(self._get_common_sudo_arguments()) command = 'sudo %s -- bash -c %s' % (sudo_arguments, command) else: if self.user and self.user != LOGGED_USER_USERNAME: # Need to use sudo to run as a different (requested) user user = quote_unix(self.user) sudo_arguments = ' '.join(self._get_user_sudo_arguments(user=user)) command = quote_unix(self.command) command = 'sudo %s -- bash -c %s' % (sudo_arguments, command) else: command = self.command return command
def _format_command(self): script_arguments = self._get_script_arguments(named_args=self.named_args, positional_args=self.positional_args) if self.sudo: if script_arguments: command = quote_unix('%s %s' % (self.remote_script, script_arguments)) else: command = quote_unix(self.remote_script) command = 'sudo -E -- bash -c %s' % (command) else: script_path = quote_unix(self.remote_script) if script_arguments: command = '%s %s' % (script_path, script_arguments) else: command = script_path return command
def get_full_command_string(self): script_arguments = self._get_script_arguments(named_args=self.named_args, positional_args=self.positional_args) if self.sudo: if script_arguments: command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments)) else: command = quote_unix(self.script_local_path_abs) command = 'sudo -E -- bash -c %s' % (command) else: if self.user and self.user != LOGGED_USER_USERNAME: # Need to use sudo to run as a different user user = quote_unix(self.user) if script_arguments: command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments)) else: command = quote_unix(self.script_local_path_abs) command = 'sudo -E -u %s -- bash -c %s' % (user, command) else: script_path = quote_unix(self.script_local_path_abs) if script_arguments: command = '%s %s' % (script_path, script_arguments) else: command = script_path return command
def _format_command(self): script_arguments = self._get_script_arguments(named_args=self.named_args, positional_args=self.positional_args) if self.sudo: if script_arguments: command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments)) else: command = quote_unix(self.script_local_path_abs) sudo_arguments = ' '.join(self._get_common_sudo_arguments()) command = 'sudo %s -- bash -c %s' % (sudo_arguments, command) else: if self.user and self.user != LOGGED_USER_USERNAME: # Need to use sudo to run as a different user user = quote_unix(self.user) if script_arguments: command = quote_unix('%s %s' % (self.script_local_path_abs, script_arguments)) else: command = quote_unix(self.script_local_path_abs) sudo_arguments = ' '.join(self._get_user_sudo_arguments(user=user)) command = 'sudo %s -- bash -c %s' % (sudo_arguments, command) else: script_path = quote_unix(self.script_local_path_abs) if script_arguments: command = '%s %s' % (script_path, script_arguments) else: command = script_path return command
def get_full_command_string(self): # Note: We pass -E to sudo because we want to preserve user provided # environment variables env_str = self._get_env_vars_export_string() cwd = self.get_cwd() if self.sudo: if env_str: command = quote_unix('%s && cd %s && %s' % (env_str, cwd, self.command)) else: command = quote_unix('cd %s && %s' % (cwd, self.command)) command = 'sudo -E -- bash -c %s' % (command) else: if env_str: command = '%s && cd %s && %s' % (env_str, cwd, self.command) else: command = 'cd %s && %s' % (cwd, self.command) LOG.debug('Command to run on remote host will be: %s', command) return command
def _get_env_vars_export_string(self): if self.env_vars: env_vars = copy.copy(self.env_vars) # If sudo_password is provided, explicitly disable bash history to make sure password # is not logged, because password is provided via command line if self.sudo and self.sudo_password: env_vars['HISTFILE'] = '/dev/null' env_vars['HISTSIZE'] = '0' # Sort the dict to guarantee consistent order env_vars = collections.OrderedDict(sorted(env_vars.items())) # Environment variables could contain spaces and open us to shell # injection attacks. Always quote the key and the value. exports = ' '.join( '%s=%s' % (quote_unix(k), quote_unix(v)) for k, v in six.iteritems(env_vars) ) shell_env_str = '%s %s' % (ShellCommandAction.EXPORT_CMD, exports) else: shell_env_str = '' return shell_env_str
def _get_command_string(self, cmd, args): """ Escape the command arguments and form a command string. :type cmd: ``str`` :type args: ``list`` :rtype: ``str`` """ assert isinstance(args, (list, tuple)) args = [quote_unix(arg) for arg in args] args = ' '.join(args) result = '%s %s' % (cmd, args) return result
def mkdir(self, dir_path): """ Create a directory on remote box. :param dir_path: Path to remote directory to be created. :type dir_path: ``str`` :return: Returns nothing if successful else raises IOError exception. :rtype: ``None`` """ dir_path = quote_unix(dir_path) extra = {"_dir_path": dir_path} self.logger.debug("mkdir", extra=extra) return self.sftp.mkdir(dir_path)
def _prepare_command(self, has_inputs, inputs_file_path): LOG.debug('CloudSlang home: %s', self._cloudslang_home) cloudslang_binary = os.path.join(self._cloudslang_home, 'bin/cslang') LOG.debug('Using CloudSlang binary: %s', cloudslang_binary) command_args = ['--f', self._flow_path, '--cp', self._cloudslang_home] if has_inputs: command_args += ['--if', inputs_file_path] command = cloudslang_binary + " run " + " ".join([quote_unix(arg) for arg in command_args]) LOG.info('Executing action via CloudSlangRunner: %s', self.runner_id) LOG.debug('Command is: %s', command) return command
def delete_file(self, path): """ Delete a file on remote box. :param path: Path to remote file to be deleted. :type path: ``str`` :return: True if the file has been successfully deleted, False otherwise. :rtype: ``bool`` """ path = quote_unix(path) extra = {"_path": path} self.logger.debug("Deleting file", extra=extra) self.sftp.unlink(path) return True
def __init__( self, name, action_exec_id, script_local_path_abs, script_local_libs_path_abs, named_args=None, positional_args=None, env_vars=None, on_behalf_user=None, user=None, password=None, private_key=None, remote_dir=None, hosts=None, parallel=True, sudo=False, timeout=None, cwd=None, ): super(RemoteScriptAction, self).__init__( name=name, action_exec_id=action_exec_id, script_local_path_abs=script_local_path_abs, user=user, named_args=named_args, positional_args=positional_args, env_vars=env_vars, sudo=sudo, timeout=timeout, cwd=cwd, ) self.script_local_libs_path_abs = script_local_libs_path_abs self.script_local_dir, self.script_name = os.path.split(self.script_local_path_abs) self.remote_dir = remote_dir if remote_dir is not None else "/tmp" self.remote_libs_path_abs = os.path.join(self.remote_dir, ACTION_LIBS_DIR) self.on_behalf_user = on_behalf_user self.password = password self.private_key = private_key self.remote_script = os.path.join(self.remote_dir, quote_unix(self.script_name)) self.hosts = hosts self.parallel = parallel self.command = self._format_command() LOG.debug("RemoteScriptAction: command to run on remote box: %s", self.command)
def run(self, packs, abs_repo_base): intersection = BLOCKED_PACKS & frozenset(packs) if len(intersection) > 0: names = ", ".join(list(intersection)) raise ValueError("Uninstall includes an uninstallable pack - %s." % (names)) # 1. Delete pack content for fp in os.listdir(abs_repo_base): abs_fp = os.path.join(abs_repo_base, fp) if fp in packs and os.path.isdir(abs_fp): self.logger.debug('Deleting pack directory "%s"' % (abs_fp)) shutil.rmtree(abs_fp) # 2. Delete pack virtual environment for pack_name in packs: pack_name = quote_unix(pack_name) virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name) if os.path.isdir(virtualenv_path): self.logger.debug('Deleting virtualenv "%s" for pack "%s"' % (virtualenv_path, pack_name)) shutil.rmtree(virtualenv_path)
def _format_command(self): script_arguments = self._get_script_arguments(named_args=self.named_args, positional_args=self.positional_args) env_str = self._get_env_vars_export_string() cwd = quote_unix(self.get_cwd()) script_path = quote_unix(self.remote_script) if self.sudo: if script_arguments: if env_str: command = quote_unix('%s && cd %s && %s %s' % ( env_str, cwd, script_path, script_arguments)) else: command = quote_unix('cd %s && %s %s' % ( cwd, script_path, script_arguments)) else: if env_str: command = quote_unix('%s && cd %s && %s' % ( env_str, cwd, script_path)) else: command = quote_unix('cd %s && %s' % (cwd, script_path)) sudo_arguments = ' '.join(self._get_common_sudo_arguments()) command = 'sudo %s -- bash -c %s' % (sudo_arguments, command) if self.sudo_password: command = ('set +o history ; echo -e %s | %s' % (quote_unix('%s\n' % (self.sudo_password)), command)) else: if script_arguments: if env_str: command = '%s && cd %s && %s %s' % (env_str, cwd, script_path, script_arguments) else: command = 'cd %s && %s %s' % (cwd, script_path, script_arguments) else: if env_str: command = '%s && cd %s && %s' % (env_str, cwd, script_path) else: command = 'cd %s && %s' % (cwd, script_path) return command
def test_quote_unix(self): arguments = [ 'foo', 'foo bar', 'foo1 bar1', '"foo"', '"foo" "bar"', "'foo bar'" ] expected_values = [ """ foo """, """ 'foo bar' """, """ 'foo1 bar1' """, """ '"foo"' """, """ '"foo" "bar"' """, """ ''"'"'foo bar'"'"'' """ ] for argument, expected_value in zip(arguments, expected_values): actual_value = quote_unix(value=argument) expected_value = expected_value.lstrip() self.assertEqual(actual_value, expected_value.strip())
def get_pack_directory(pack_name): """ Retrieve a directory for the provided pack. If a directory for the provided pack doesn't exist in any of the search paths, None is returned instead. Note: If same pack exists in multiple search path, path to the first one is returned. :param pack_name: Pack name. :type pack_name: ``str`` :return: Pack to the pack directory. :rtype: ``str`` or ``None`` """ packs_base_paths = get_packs_base_paths() for packs_base_path in packs_base_paths: pack_base_path = os.path.join(packs_base_path, quote_unix(pack_name)) pack_base_path = os.path.abspath(pack_base_path) if os.path.isdir(pack_base_path): return pack_base_path return None
def _format_command(self): script_arguments = self._get_script_arguments(named_args=self.named_args, positional_args=self.positional_args) env_str = self._get_env_vars_export_string() cwd = quote_unix(self.get_cwd()) script_path = quote_unix(self.remote_script) if self.sudo: if script_arguments: if env_str: command = quote_unix('%s && cd %s && %s %s' % ( env_str, cwd, script_path, script_arguments)) else: command = quote_unix('cd %s && %s %s' % ( cwd, script_path, script_arguments)) else: if env_str: command = quote_unix('%s && cd %s && %s' % ( env_str, cwd, script_path)) else: command = quote_unix('cd %s && %s' % (cwd, script_path)) command = 'sudo -E -- bash -c %s' % (command) else: if script_arguments: if env_str: command = '%s && cd %s && %s %s' % (env_str, cwd, script_path, script_arguments) else: command = 'cd %s && %s %s' % (cwd, script_path, script_arguments) else: if env_str: command = '%s && cd %s && %s' % (env_str, cwd, script_path) else: command = 'cd %s && %s' % (cwd, script_path) return command
def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, include_setuptools=True, include_wheel=True, proxy_config=None, no_download=True, force_owner_group=True): """ Setup virtual environment for the provided pack. :param pack_name: Name of the pack to setup the virtualenv for. :type pack_name: ``str`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. :param no_download: Do not download and install latest version of pre-installed packages such as pip and distutils. :type no_download: ``bool`` """ logger = logger or LOG if not re.match(PACK_REF_WHITELIST_REGEX, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) logger.debug('Setting up virtualenv for pack "%s" (%s)' % (pack_name, pack_path)) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) # 1. Create virtualenv if it doesn't exist if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 1. Create virtual environment logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip, include_setuptools=include_setuptools, include_wheel=include_wheel, no_download=no_download) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: install_requirement(virtualenv_path=virtualenv_path, requirement=requirement, proxy_config=proxy_config, logger=logger) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) install_requirements(virtualenv_path=virtualenv_path, requirements_file_path=requirements_file_path, proxy_config=proxy_config, logger=logger) else: logger.debug('No pack specific requirements found') # 4. Set the owner group if force_owner_group: apply_pack_owner_group(pack_path=virtualenv_path) action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path))
def run(self, cmd, timeout=None, quote=False, call_line_handler_func=False): """ Note: This function is based on paramiko's exec_command() method. :param timeout: How long to wait (in seconds) for the command to finish (optional). :type timeout: ``float`` :param call_line_handler_func: True to call handle_stdout_line_func function for each line of received stdout and handle_stderr_line_func for each line of stderr. :type call_line_handler_func: ``bool`` """ if quote: cmd = quote_unix(cmd) extra = {'_cmd': cmd} self.logger.info('Executing command', extra=extra) # Use the system default buffer size bufsize = -1 transport = self.client.get_transport() chan = transport.open_session() start_time = time.time() if cmd.startswith('sudo'): # Note that fabric does this as well. If you set pty, stdout and stderr # streams will be combined into one. chan.get_pty() chan.exec_command(cmd) stdout = StringIO() stderr = StringIO() # Create a stdin file and immediately close it to prevent any # interactive script from hanging the process. stdin = chan.makefile('wb', bufsize) stdin.close() # Receive all the output # Note #1: This is used instead of chan.makefile approach to prevent # buffering issues and hanging if the executed command produces a lot # of output. # # Note #2: If you are going to remove "ready" checks inside the loop # you are going to have a bad time. Trying to consume from a channel # which is not ready will block for indefinitely. exit_status_ready = chan.exit_status_ready() if exit_status_ready: stdout_data = self._consume_stdout( chan=chan, call_line_handler_func=call_line_handler_func) stdout_data = stdout_data.getvalue() stderr_data = self._consume_stderr( chan=chan, call_line_handler_func=call_line_handler_func) stderr_data = stderr_data.getvalue() stdout.write(stdout_data) stderr.write(stderr_data) while not exit_status_ready: current_time = time.time() elapsed_time = (current_time - start_time) if timeout and (elapsed_time > timeout): # TODO: Is this the right way to clean up? chan.close() stdout = strip_shell_chars(stdout.getvalue()) stderr = strip_shell_chars(stderr.getvalue()) raise SSHCommandTimeoutError(cmd=cmd, timeout=timeout, stdout=stdout, stderr=stderr) stdout_data = self._consume_stdout( chan=chan, call_line_handler_func=call_line_handler_func) stdout_data = stdout_data.getvalue() stderr_data = self._consume_stderr( chan=chan, call_line_handler_func=call_line_handler_func) stderr_data = stderr_data.getvalue() stdout.write(stdout_data) stderr.write(stderr_data) # We need to check the exit status here, because the command could # print some output and exit during this sleep below. exit_status_ready = chan.exit_status_ready() if exit_status_ready: break # Short sleep to prevent busy waiting eventlet.sleep(self.SLEEP_DELAY) # print('Wait over. Channel must be ready for host: %s' % self.hostname) # Receive the exit status code of the command we ran. status = chan.recv_exit_status() stdout = strip_shell_chars(stdout.getvalue()) stderr = strip_shell_chars(stderr.getvalue()) extra = {'_status': status, '_stdout': stdout, '_stderr': stderr} self.logger.debug('Command finished', extra=extra) return [stdout, stderr, status]
def setup_pack_virtualenv(pack_name, update=False, logger=None): """ Setup virtual environment for the provided pack. :param pack_name: Name of the pack to setup the virtualenv for. :type pack_name: ``str`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. """ logger = logger or LOG if not re.match(PACK_NAME_WHITELIST, pack_name): raise ValueError('Invalid pack name "%s"' % (pack_name)) base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') logger.debug('Setting up virtualenv for pack "%s"' % (pack_name)) virtualenv_path = os.path.join(base_virtualenvs_path, quote_unix(pack_name)) # Ensure pack directory exists in one of the search paths pack_path = get_pack_directory(pack_name=pack_name) if not pack_path: packs_base_paths = get_packs_base_paths() search_paths = ', '.join(packs_base_paths) msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) raise Exception(msg) # 1. Create virtualenv if it doesn't exist if not update or not os.path.exists(virtualenv_path): # 0. Delete virtual environment if it exists remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 1. Create virtual environment logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') for requirement in BASE_PACK_REQUIREMENTS: install_requirement(virtualenv_path=virtualenv_path, requirement=requirement) # 3. Install pack-specific requirements requirements_file_path = os.path.join(pack_path, 'requirements.txt') has_requirements = os.path.isfile(requirements_file_path) if has_requirements: logger.debug('Installing pack specific requirements from "%s"' % (requirements_file_path)) install_requirements(virtualenv_path=virtualenv_path, requirements_file_path=requirements_file_path) else: logger.debug('No pack specific requirements found') action = 'updated' if update else 'created' logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' % (pack_name, action, virtualenv_path))
def run(self, cmd, timeout=None, quote=False): """ Note: This function is based on paramiko's exec_command() method. :param timeout: How long to wait (in seconds) for the command to finish (optional). :type timeout: ``float`` """ if quote: cmd = quote_unix(cmd) extra = {"_cmd": cmd} self.logger.info("Executing command", extra=extra) # Use the system default buffer size bufsize = -1 transport = self.client.get_transport() chan = transport.open_session() start_time = time.time() if cmd.startswith("sudo"): # Note that fabric does this as well. If you set pty, stdout and stderr # streams will be combined into one. chan.get_pty() chan.exec_command(cmd) stdout = StringIO() stderr = StringIO() # Create a stdin file and immediately close it to prevent any # interactive script from hanging the process. stdin = chan.makefile("wb", bufsize) stdin.close() # Receive all the output # Note #1: This is used instead of chan.makefile approach to prevent # buffering issues and hanging if the executed command produces a lot # of output. # # Note #2: If you are going to remove "ready" checks inside the loop # you are going to have a bad time. Trying to consume from a channel # which is not ready will block for indefinitely. exit_status_ready = chan.exit_status_ready() if exit_status_ready: stdout.write(self._consume_stdout(chan).getvalue()) stderr.write(self._consume_stderr(chan).getvalue()) while not exit_status_ready: current_time = time.time() elapsed_time = current_time - start_time if timeout and (elapsed_time > timeout): # TODO: Is this the right way to clean up? chan.close() stdout = strip_shell_chars(stdout.getvalue()) stderr = strip_shell_chars(stderr.getvalue()) raise SSHCommandTimeoutError(cmd=cmd, timeout=timeout, stdout=stdout, stderr=stderr) stdout.write(self._consume_stdout(chan).getvalue()) stderr.write(self._consume_stderr(chan).getvalue()) # We need to check the exist status here, because the command could # print some output and exit during this sleep bellow. exit_status_ready = chan.exit_status_ready() if exit_status_ready: break # Short sleep to prevent busy waiting eventlet.sleep(self.SLEEP_DELAY) # print('Wait over. Channel must be ready for host: %s' % self.hostname) # Receive the exit status code of the command we ran. status = chan.recv_exit_status() stdout = strip_shell_chars(stdout.getvalue()) stderr = strip_shell_chars(stderr.getvalue()) extra = {"_status": status, "_stdout": stdout, "_stderr": stderr} self.logger.debug("Command finished", extra=extra) return [stdout, stderr, status]
check=True, stdout=subprocess.PIPE) distro = result.stdout.decode("utf-8").strip() if not distro: raise ValueError("Fail to detect distribution we are running on") return distro if len(sys.argv) < 3: raise ValueError("Usage: service.py <action> <service>") distro = get_linux_distribution() args = {"act": quote_unix(sys.argv[1]), "service": quote_unix(sys.argv[2])} print("Detected distro: %s" % (distro)) if re.search(distro, "Ubuntu"): if os.path.isfile("/etc/init/%s.conf" % args["service"]): cmd_args = ["service", args["service"], args["act"]] elif os.path.isfile("/etc/init.d/%s" % args["service"]): cmd_args = ["/etc/init.d/%s" % (args["service"]), args["act"]] else: print("Unknown service") sys.exit(2) elif (re.search(distro, "Redhat") or re.search(distro, "Fedora") or re.search(distro, "CentOS") or re.search(distro, "Rocky Linux")): cmd_args = ["systemctl", args["act"], args["service"]]