def setUpClass(cls):
        super(SensorContainerTestCase, cls).setUpClass()

        st2tests.config.parse_args()

        username = cfg.CONF.database.username if hasattr(
            cfg.CONF.database, 'username') else None
        password = cfg.CONF.database.password if hasattr(
            cfg.CONF.database, 'password') else None
        cls.db_connection = db_setup(cfg.CONF.database.db_name,
                                     cfg.CONF.database.host,
                                     cfg.CONF.database.port,
                                     username=username,
                                     password=password,
                                     ensure_indexes=False)

        # Register sensors
        register_sensors(packs_base_paths=[PACKS_BASE_PATH],
                         use_pack_cache=False)

        # Create virtualenv for examples pack
        virtualenv_path = '/tmp/virtualenvs/examples'

        run_command(cmd=['rm', '-rf', virtualenv_path])

        cmd = [
            'virtualenv', '--system-site-packages', '--python', PYTHON_BINARY,
            virtualenv_path
        ]
        run_command(cmd=cmd)
Exemple #2
0
    def test_run_command_success(self):
        # 0 exit code
        exit_code, stdout, stderr, timed_out = run_command(
            cmd='echo "test stdout" ; >&2 echo "test stderr"', shell=True)
        self.assertEqual(exit_code, 0)
        self.assertEqual(stdout.strip(), b"test stdout")
        self.assertEqual(stderr.strip(), b"test stderr")
        self.assertFalse(timed_out)

        # non-zero exit code
        exit_code, stdout, stderr, timed_out = run_command(
            cmd='echo "test stdout" ; >&2 echo "test stderr" ; exit 5',
            shell=True)
        self.assertEqual(exit_code, 5)
        self.assertEqual(stdout.strip(), b"test stdout")
        self.assertEqual(stderr.strip(), b"test stderr")
        self.assertFalse(timed_out)

        # implicit non zero code (invalid command)
        exit_code, stdout, stderr, timed_out = run_command(
            cmd="foobarbarbazrbar", shell=True)
        self.assertEqual(exit_code, 127)
        self.assertEqual(stdout.strip(), b"")
        self.assertTrue(b"foobarbarbazrbar: not found" in stderr.strip())
        self.assertFalse(timed_out)
    def setUpClass(cls):
        super(SensorContainerTestCase, cls).setUpClass()

        st2tests.config.parse_args()

        username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
        password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
        cls.db_connection = db_setup(
            cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
            username=username, password=password, ensure_indexes=False)

        # NOTE: We need to perform this patching because test fixtures are located outside of the
        # packs base paths directory. This will never happen outside the context of test fixtures.
        cfg.CONF.content.packs_base_paths = PACKS_BASE_PATH

        # Register sensors
        register_sensors(packs_base_paths=[PACKS_BASE_PATH], use_pack_cache=False)

        # Create virtualenv for examples pack
        virtualenv_path = '/tmp/virtualenvs/examples'

        run_command(cmd=['rm', '-rf', virtualenv_path])

        cmd = ['virtualenv', '--system-site-packages', '--python', PYTHON_BINARY, virtualenv_path]
        run_command(cmd=cmd)
    def setUpClass(cls):
        super(SensorContainerTestCase, cls).setUpClass()

        st2tests.config.parse_args()

        username = cfg.CONF.database.username if hasattr(
            cfg.CONF.database, 'username') else None
        password = cfg.CONF.database.password if hasattr(
            cfg.CONF.database, 'password') else None
        cls.db_connection = db_setup(cfg.CONF.database.db_name,
                                     cfg.CONF.database.host,
                                     cfg.CONF.database.port,
                                     username=username,
                                     password=password,
                                     ensure_indexes=False)

        # NOTE: We need to perform this patching because test fixtures are located outside of the
        # packs base paths directory. This will never happen outside the context of test fixtures.
        cfg.CONF.content.packs_base_paths = PACKS_BASE_PATH

        # Register sensors
        register_sensors(packs_base_paths=[PACKS_BASE_PATH],
                         use_pack_cache=False)

        # Create virtualenv for examples pack
        virtualenv_path = '/tmp/virtualenvs/examples'

        run_command(cmd=['rm', '-rf', virtualenv_path])

        cmd = [
            'virtualenv', '--system-site-packages', '--python', PYTHON_BINARY,
            virtualenv_path
        ]
        run_command(cmd=cmd)
    def setUpClass(cls):
        super(SensorContainerTestCase, cls).setUpClass()
        return

        st2tests.config.parse_args()

        username = cfg.CONF.database.username if hasattr(
            cfg.CONF.database, 'username') else None
        password = cfg.CONF.database.password if hasattr(
            cfg.CONF.database, 'password') else None
        cls.db_connection = db_setup(cfg.CONF.database.db_name,
                                     cfg.CONF.database.host,
                                     cfg.CONF.database.port,
                                     username=username,
                                     password=password,
                                     ensure_indexes=False)

        # Register sensors
        register_sensors(packs_base_paths=['/opt/stackstorm/packs'],
                         use_pack_cache=False)

        # Create virtualenv for examples pack
        virtualenv_path = '/opt/stackstorm/virtualenvs/examples'
        cmd = ['virtualenv', '--system-site-packages', virtualenv_path]
        run_command(cmd=cmd)
Exemple #6
0
    def _upload_file(self, local_path, base_path):
        """
        Upload provided file to the remote server in a temporary directory.

        :param local_path: Local path to the file to upload.
        :type local_path: ``str``

        :param base_path: Absolute base path for the share.
        :type base_path: ``str``
        """
        file_name = os.path.basename(local_path)

        temporary_directory_name = str(uuid.uuid4())
        command = 'mkdir %s' % (quote_windows(temporary_directory_name))

        # 1. Create a temporary dir for out scripts (ignore errors if it already exists)
        # Note: We don't necessary have access to $TEMP so we create a temporary directory for our
        # us in the root of the share we are using and have access to
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        LOG.debug('Creating temp directory "%s"' % (temporary_directory_name))

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           timeout=CREATE_DIRECTORY_TIMEOUT)

        extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
        LOG.debug('Directory created', extra=extra)

        # 2. Upload file to temporary directory
        remote_path = PATH_SEPARATOR.join([temporary_directory_name, file_name])

        values = {
            'local_path': quote_windows(local_path),
            'remote_path': quote_windows(remote_path)
        }
        command = 'put %(local_path)s %(remote_path)s' % values
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        extra = {'local_path': local_path, 'remote_path': remote_path}
        LOG.debug('Uploading file to "%s"' % (remote_path))

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           timeout=UPLOAD_FILE_TIMEOUT)

        extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
        LOG.debug('File uploaded to "%s"' % (remote_path), extra=extra)

        full_remote_file_path = base_path + '\\' + remote_path
        full_temporary_directory_path = base_path + '\\' + temporary_directory_name

        return full_remote_file_path, full_temporary_directory_path
    def _upload_file(self, local_path, base_path):
        """
        Upload provided file to the remote server in a temporary directory.

        :param local_path: Local path to the file to upload.
        :type local_path: ``str``

        :param base_path: Absolute base path for the share.
        :type base_path: ``str``
        """
        file_name = os.path.basename(local_path)

        temporary_directory_name = str(uuid.uuid4())
        command = 'mkdir %s' % (quote_windows(temporary_directory_name))

        # 1. Create a temporary dir for out scripts (ignore errors if it already exists)
        # Note: We don't necessary have access to $TEMP so we create a temporary directory for our
        # us in the root of the share we are using and have access to
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        LOG.debug('Creating temp directory "%s"' % (temporary_directory_name))

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           timeout=CREATE_DIRECTORY_TIMEOUT)

        extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
        LOG.debug('Directory created', extra=extra)

        # 2. Upload file to temporary directory
        remote_path = PATH_SEPARATOR.join([temporary_directory_name, file_name])

        values = {
            'local_path': quote_windows(local_path),
            'remote_path': quote_windows(remote_path)
        }
        command = 'put %(local_path)s %(remote_path)s' % values
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        extra = {'local_path': local_path, 'remote_path': remote_path}
        LOG.debug('Uploading file to "%s"' % (remote_path))

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           timeout=UPLOAD_FILE_TIMEOUT)

        extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
        LOG.debug('File uploaded to "%s"' % (remote_path), extra=extra)

        full_remote_file_path = base_path + '\\' + remote_path
        full_temporary_directory_path = base_path + '\\' + temporary_directory_name

        return full_remote_file_path, full_temporary_directory_path
Exemple #8
0
    def _get_share_absolute_path(self, share):
        """
        Retrieve full absolute path for a share with the provided name.

        :param share: Share name.
        :type share: ``str``
        """
        command = 'net share %s' % (quote_windows(share))
        args = self._get_winexe_command_args(host=self._host, username=self._username,
                                             password=self._password,
                                             command=command)

        LOG.debug('Retrieving full absolute path for share "%s"' % (share))
        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           timeout=self._timeout)

        if exit_code != 0:
            msg = 'Failed to retrieve absolute path for share "%s"' % (share)
            raise Exception(msg)

        share_info = self._parse_share_information(stdout=stdout)
        share_path = share_info.get('path', None)

        if not share_path:
            msg = 'Failed to retrieve absolute path for share "%s"' % (share)
            raise Exception(msg)

        return share_path
Exemple #9
0
    def _run_cli_command(self, command):
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=command,
            stdin=None,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True,
            timeout=self._timeout,
            kill_func=kill_process)

        error = None
        if timed_out:
            error = 'Action failed to complete in %s seconds' % self._timeout
            exit_code = -9

        succeeded = (exit_code == 0)
        result = {
            'failed': not succeeded,
            'succeeded': succeeded,
            'return_code': exit_code,
            'stdout': stdout,
            'stderr': stderr
        }
        if error:
            result['error'] = error

        status = LIVEACTION_STATUS_SUCCEEDED if succeeded else LIVEACTION_STATUS_FAILED
        return result, status
Exemple #10
0
    def _get_share_absolute_path(self, share):
        """
        Retrieve full absolute path for a share with the provided name.

        :param share: Share name.
        :type share: ``str``
        """
        command = 'net share %s' % (quote_windows(share))
        args = self._get_winexe_command_args(host=self._host,
                                             username=self._username,
                                             password=self._password,
                                             command=command)

        LOG.debug('Retrieving full absolute path for share "%s"' % (share))
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=args,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            timeout=self._timeout)

        if exit_code != 0:
            msg = 'Failed to retrieve absolute path for share "%s"' % (share)
            raise Exception(msg)

        share_info = self._parse_share_information(stdout=stdout)
        share_path = share_info.get('path', None)

        if not share_path:
            msg = 'Failed to retrieve absolute path for share "%s"' % (share)
            raise Exception(msg)

        return share_path
Exemple #11
0
    def _run_script(self, script_path, arguments=None):
        """
        :param script_path: Full path to the script on the remote server.
        :type script_path: ``str``

        :param arguments: The arguments to pass to the script.
        :type arguments: ``str``
        """
        if arguments is not None:
            command = '%s %s %s' % (POWERSHELL_COMMAND,
                                    quote_windows(script_path), arguments)
        else:
            command = '%s %s' % (POWERSHELL_COMMAND,
                                 quote_windows(script_path))
        args = self._get_winexe_command_args(host=self._host,
                                             username=self._username,
                                             password=self._password,
                                             command=command)

        LOG.debug('Running script "%s"' % (script_path))

        # Note: We don't send anything over stdin, we just create an unused pipe
        # to avoid some obscure failures
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=args,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            timeout=self._timeout)

        extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
        LOG.debug('Command returned', extra=extra)

        return exit_code, stdout, stderr, timed_out
Exemple #12
0
    def _run_script(self, script_path, arguments=None):
        """
        :param script_path: Full path to the script on the remote server.
        :type script_path: ``str``

        :param arguments: The arguments to pass to the script.
        :type arguments: ``str``
        """
        if arguments is not None:
            command = '%s %s %s' % (POWERSHELL_COMMAND, quote_windows(script_path), arguments)
        else:
            command = '%s %s' % (POWERSHELL_COMMAND, quote_windows(script_path))
        args = self._get_winexe_command_args(host=self._host, username=self._username,
                                             password=self._password,
                                             command=command)

        LOG.debug('Running script "%s"' % (script_path))

        # Note: We don't send anything over stdin, we just create an unused pipe
        # to avoid some obscure failures
        exit_code, stdout, stderr, timed_out = run_command(cmd=args,
                                                           stdin=subprocess.PIPE,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           shell=False,
                                                           timeout=self._timeout)

        extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
        LOG.debug('Command returned', extra=extra)

        return exit_code, stdout, stderr, timed_out
Exemple #13
0
def apply_pack_owner_group(pack_path):
    """
    Switch owner group of the pack / virtualenv directory to the configured
    group.

    NOTE: This requires sudo access.
    """
    pack_group = utils.get_pack_group()

    if pack_group:
        LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group))

        if SUDO_BINARY:
            args = ['sudo', 'chgrp', '-R', pack_group, pack_path]
        else:
            # Environments where sudo is not available (e.g. docker)
            args = ['chgrp', '-R', pack_group, pack_path]

        exit_code, _, stderr, _ = shell.run_command(args)

        if exit_code != 0:
            # Non fatal, but we still log it
            LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' %
                      (pack_path, pack_group, stderr))

    return True
Exemple #14
0
    def _run_cli_command(self, command):
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=command, stdin=None,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            shell=True, timeout=self._timeout, kill_func=kill_process)

        error = None
        if timed_out:
            error = 'Action failed to complete in %s seconds' % self._timeout
            exit_code = -9

        succeeded = (exit_code == 0)
        result = {
            'failed': not succeeded,
            'succeeded': succeeded,
            'return_code': exit_code,
            'stdout': stdout,
            'stderr': stderr
        }
        if error:
            result['error'] = error

        status = LIVEACTION_STATUS_SUCCEEDED if succeeded else LIVEACTION_STATUS_FAILED
        self._log_action_completion(logger=LOG, result=result, status=status, exit_code=exit_code)
        return result, status
Exemple #15
0
    def test_run_command_timeout_no_shell_no_custom_kill_func(self):
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=["sleep", "1599"], preexec_func=os.setsid, timeout=1)
        self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
        self.assertEqual(stdout.strip(), b"")
        self.assertEqual(stderr.strip(), b"")
        self.assertTrue(timed_out)

        # Verify there is no zombie process left laying around
        self.assertNoStrayProcessesLeft("sleep 1599")
    def setUpClass(cls):
        super(SensorContainerTestCase, cls).setUpClass()
        return

        st2tests.config.parse_args()

        username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
        password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
        cls.db_connection = db_setup(
            cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
            username=username, password=password, ensure_indexes=False)

        # Register sensors
        register_sensors(packs_base_paths=['/opt/stackstorm/packs'], use_pack_cache=False)

        # Create virtualenv for examples pack
        virtualenv_path = '/opt/stackstorm/virtualenvs/examples'
        cmd = ['virtualenv', '--system-site-packages', virtualenv_path]
        run_command(cmd=cmd)
    def _delete_file(self, file_path):
        command = 'rm %(file_path)s' % {'file_path': quote_windows(file_path)}
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE, shell=False,
                                         timeout=DELETE_FILE_TIMEOUT)

        return exit_code == 0
Exemple #18
0
    def _apply_pack_permissions(self, pack_path):
        """
        Will recursively apply permission 770 to pack and its contents.
        """
        # 1. switch owner group to configured group
        pack_group = utils.get_pack_group()
        if pack_group:
            shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        # 2. Setup the right permissions and group ownership
        # These mask is same as mode = 775
        mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        os.chmod(pack_path, mode)

        # Yuck! Since os.chmod does not support chmod -R walk manually.
        for root, dirs, files in os.walk(pack_path):
            for d in dirs:
                os.chmod(os.path.join(root, d), mode)
            for f in files:
                os.chmod(os.path.join(root, f), mode)
Exemple #19
0
    def _apply_pack_permissions(self, pack_path):
        """
        Will recursively apply permission 770 to pack and its contents.
        """
        # 1. switch owner group to configured group
        pack_group = utils.get_pack_group()
        if pack_group:
            shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        # 2. Setup the right permissions and group ownership
        # These mask is same as mode = 775
        mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        os.chmod(pack_path, mode)

        # Yuck! Since os.chmod does not support chmod -R walk manually.
        for root, dirs, files in os.walk(pack_path):
            for d in dirs:
                os.chmod(os.path.join(root, d), mode)
            for f in files:
                os.chmod(os.path.join(root, f), mode)
Exemple #20
0
    def _delete_file(self, file_path):
        command = 'rm %(file_path)s' % {'file_path': quote_windows(file_path)}
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE, shell=False,
                                         timeout=DELETE_FILE_TIMEOUT)

        return exit_code == 0
Exemple #21
0
    def _delete_directory(self, directory_path):
        command = 'rmdir %(directory_path)s' % {'directory_path': quote_windows(directory_path)}
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        LOG.debug('Removing directory "%s"' % (directory_path))
        exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE, shell=False,
                                         timeout=DELETE_DIRECTORY_TIMEOUT)

        return exit_code == 0
    def _delete_directory(self, directory_path):
        command = 'rmdir %(directory_path)s' % {'directory_path': quote_windows(directory_path)}
        args = self._get_smbclient_command_args(host=self._host, username=self._username,
                                                password=self._password, command=command,
                                                share=self._share)

        LOG.debug('Removing directory "%s"' % (directory_path))
        exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
                                         stderr=subprocess.PIPE, shell=False,
                                         timeout=DELETE_DIRECTORY_TIMEOUT)

        return exit_code == 0
Exemple #23
0
    def setUpClass(cls):
        super(SensorContainerTestCase, cls).setUpClass()

        st2tests.config.parse_args()

        username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
        password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
        cls.db_connection = db_setup(
            cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
            username=username, password=password, ensure_indexes=False)

        # Register sensors
        register_sensors(packs_base_paths=[PACKS_BASE_PATH], use_pack_cache=False)

        # Create virtualenv for examples pack
        virtualenv_path = '/tmp/virtualenvs/examples'

        run_command(cmd=['rm', '-rf', virtualenv_path])

        cmd = ['virtualenv', '--system-site-packages', '--python', PYTHON_BINARY, virtualenv_path]
        run_command(cmd=cmd)
Exemple #24
0
    def run(self, action_parameters):
        pack = self.get_pack_name()
        user = self.get_user()
        serialized_parameters = json.dumps(
            action_parameters) if action_parameters else ''
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        python_path = get_sandbox_python_binary_path(pack=pack)

        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            raise Exception(msg)

        if not self.entry_point:
            raise Exception('Action "%s" is missing entry_point attribute' %
                            (self.action.name))

        args = [
            python_path, WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--user=%s' % (user),
            '--parent-args=%s' % (json.dumps(sys.argv[1:]))
        ]

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
        env['PYTHONPATH'] = get_sandbox_python_path(
            inherit_from_parent=True, inherit_parent_virtualenv=True)

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        exit_code, stdout, stderr, timed_out = run_command(
            cmd=args,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            env=env,
            timeout=self._timeout)

        return self._get_output_values(exit_code, stdout, stderr, timed_out)
Exemple #25
0
    def assertNoStrayProcessesLeft(self, grep_string: str) -> None:
        """
        Assert that there are no stray / zombie processes left with the provided command line
        string.
        """
        exit_code, stdout, stderr, timed_out = run_command(
            cmd="ps aux | grep %s | grep -v grep" % (quote_unix(grep_string)),
            shell=True,
        )

        if stdout.strip() != b"" and stderr.strip() != b"":
            raise AssertionError(
                "Expected no stray processes, but found Some. stdout: %s, stderr: %s"
                % (stdout, stderr))
Exemple #26
0
    def run(self, action_parameters):
        pack = self.get_pack_name()
        user = self.get_user()
        serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        python_path = get_sandbox_python_binary_path(pack=pack)

        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            raise Exception(msg)

        if not self.entry_point:
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        args = [
            python_path,
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--user=%s' % (user),
            '--parent-args=%s' % (json.dumps(sys.argv[1:]))
        ]

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
        env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
                                                    inherit_parent_virtualenv=True)

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           env=env, timeout=self._timeout)

        return self._get_output_values(exit_code, stdout, stderr, timed_out)
    def run(self, action_parameters):
        # Make sure the dependencies are available
        self._verify_winexe_exists()

        args = self._get_winexe_command_args(host=self._host,
                                             username=self._username,
                                             password=self._password,
                                             command=self._command)

        # Note: We don't send anything over stdin, we just create an unused pipe
        # to avoid some obscure failures
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=args,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            timeout=self._timeout)

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
        else:
            error = None

        if exit_code != 0:
            error = self._parse_winexe_error(stdout=stdout, stderr=stderr)

        result = stdout

        output = {
            'stdout': stdout,
            'stderr': stderr,
            'exit_code': exit_code,
            'result': result
        }

        if error:
            output['error'] = error

        status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
        self._log_action_completion(logger=LOG,
                                    result=output,
                                    status=status,
                                    exit_code=exit_code)
        return (status, output, None)
Exemple #28
0
    def test_run_command_timeout_shell_and_custom_kill_func(self):
        # This test represents our local runner setup where we use a preexec_func + custom kill_func
        # NOTE: When using shell=True. we should alaways use custom kill_func to ensure child shell
        # processses are in fact killed as well.
        exit_code, stdout, stderr, timed_out = run_command(
            cmd='echo "pre sleep";  sleep 1589; echo "post sleep"',
            preexec_func=os.setsid,
            timeout=1,
            kill_func=kill_process,
            shell=True,
        )
        self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
        self.assertEqual(stdout.strip(), b"pre sleep")
        self.assertEqual(stderr.strip(), b"")
        self.assertTrue(timed_out)

        # Verify there is no zombie process left laying around
        self.assertNoStrayProcessesLeft("sleep 1589")
Exemple #29
0
def apply_pack_owner_group(pack_path):
    """
    Switch owner group of the pack / virtualenv directory to the configured
    group.

    NOTE: This requires sudo access.
    """
    pack_group = utils.get_pack_group()

    if pack_group:
        LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group))
        exit_code, _, stderr, _ = shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        if exit_code != 0:
            # Non fatal, but we still log it
            LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' %
                      (pack_path, pack_group, stderr))

    return True
Exemple #30
0
def apply_pack_owner_group(pack_path):
    """
    Switch owner group of the pack / virtualenv directory to the configured
    group.

    NOTE: This requires sudo access.
    """
    pack_group = utils.get_pack_group()

    if pack_group:
        LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group))
        exit_code, _, stderr, _ = shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])

        if exit_code != 0:
            # Non fatal, but we still log it
            LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' %
                      (pack_path, pack_group, stderr))

    return True
Exemple #31
0
    def test_run_command_timeout_no_shell_no_custom_kill_func_and_read_funcs(
            self):
        def mock_read_stdout(process_stdout, stdout_buffer):
            try:
                stdout_buffer.write(process_stdout.readline())
            except greenlet.GreenletExit:
                pass

        def mock_read_stderr(process_stderr, stderr_buffer):
            try:
                stderr_buffer.write(process_stderr.readline())
            except greenlet.GreenletExit:
                pass

        read_stdout_buffer = BytesIO()
        read_stderr_buffer = BytesIO()

        script_path = os.path.abspath(
            os.path.join(BASE_DIR,
                         "../fixtures/print_to_stdout_stderr_sleep.sh"))

        exit_code, stdout, stderr, timed_out = run_command(
            cmd=[script_path],
            preexec_func=os.setsid,
            timeout=3,
            read_stdout_func=mock_read_stdout,
            read_stderr_func=mock_read_stderr,
            read_stdout_buffer=read_stdout_buffer,
            read_stderr_buffer=read_stderr_buffer,
        )
        self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
        self.assertEqual(stdout.strip(), b"pre sleep")
        self.assertEqual(stderr.strip(), b"pre sleep stderr")
        self.assertTrue(timed_out)

        # Only initial produced line should be read
        self.assertEqual(read_stdout_buffer.getvalue().strip(), b"pre sleep")
        self.assertEqual(read_stderr_buffer.getvalue().strip(),
                         b"pre sleep stderr")

        # Verify there is no zombie process left laying around
        self.assertNoStrayProcessesLeft("print_to_stdout_stderr_sleep")
    def run(self, action_parameters):
        # Make sure the dependencies are available
        self._verify_winexe_exists()

        args = self._get_winexe_command_args(host=self._host, username=self._username,
                                             password=self._password,
                                             command=self._command)

        # Note: We don't send anything over stdin, we just create an unused pipe
        # to avoid some obscure failures
        exit_code, stdout, stderr, timed_out = run_command(cmd=args,
                                                           stdin=subprocess.PIPE,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           shell=False,
                                                           timeout=self._timeout)

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
        else:
            error = None

        if exit_code != 0:
            error = self._parse_winexe_error(stdout=stdout, stderr=stderr)

        result = stdout

        output = {
            'stdout': stdout,
            'stderr': stderr,
            'exit_code': exit_code,
            'result': result
        }

        if error:
            output['error'] = error

        status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
        self._log_action_completion(logger=LOG, result=output, status=status, exit_code=exit_code)
        return (status, output, None)
Exemple #33
0
    def test_run_command_timeout_shell_and_custom_kill_func_and_read_funcs(
            self):
        # This test represents our local runner setup where we use a preexec_func + custom kill_func
        # NOTE: When using shell=True. we should alaways use custom kill_func to ensure child shell
        # processses are in fact killed as well.
        def mock_read_stdout(process_stdout, stdout_buffer):
            stdout_buffer.write(process_stdout.read())

        def mock_read_stderr(process_stderr, stderr_buffer):
            stderr_buffer.write(process_stderr.read())

        read_stdout_buffer = BytesIO()
        read_stderr_buffer = BytesIO()

        exit_code, stdout, stderr, timed_out = run_command(
            cmd=
            'echo "pre sleep"; >&2 echo "pre sleep stderr" ; sleep 1589; echo "post sleep"',
            preexec_func=os.setsid,
            timeout=1,
            kill_func=kill_process,
            shell=True,
            read_stdout_func=mock_read_stdout,
            read_stderr_func=mock_read_stderr,
            read_stdout_buffer=read_stdout_buffer,
            read_stderr_buffer=read_stderr_buffer,
        )
        self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
        self.assertEqual(stdout.strip(), b"pre sleep")
        self.assertEqual(stderr.strip(), b"pre sleep stderr")
        self.assertTrue(timed_out)

        # Only initial produced line should be read
        self.assertEqual(read_stdout_buffer.getvalue().strip(), b"pre sleep")
        self.assertEqual(read_stderr_buffer.getvalue().strip(),
                         b"pre sleep stderr")

        # Verify there is no zombie process left laying around
        self.assertNoStrayProcessesLeft("sleep 1589")
Exemple #34
0
    def run(self, action_parameters):
        LOG.debug('Running pythonrunner.')
        LOG.debug('Getting pack name.')
        pack = self.get_pack_ref()
        LOG.debug('Getting user.')
        user = self.get_user()
        LOG.debug('Serializing parameters.')
        serialized_parameters = json.dumps(action_parameters if action_parameters else {})
        LOG.debug('Getting virtualenv_path.')
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        LOG.debug('Getting python path.')
        if self._sandbox:
            python_path = get_sandbox_python_binary_path(pack=pack)
        else:
            python_path = sys.executable

        LOG.debug('Checking virtualenv path.')
        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            LOG.error('virtualenv_path set but not a directory: %s', msg)
            raise Exception(msg)

        LOG.debug('Checking entry_point.')
        if not self.entry_point:
            LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        # Note: We pass config as command line args so the actual wrapper process is standalone
        # and doesn't need access to db
        LOG.debug('Setting args.')

        if self._use_parent_args:
            parent_args = json.dumps(sys.argv[1:])
        else:
            parent_args = json.dumps([])

        args = [
            python_path,
            '-u',  # unbuffered mode so streaming mode works as expected
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--user=%s' % (user),
            '--parent-args=%s' % (parent_args),
        ]

        # If parameter size is larger than the maximum allowed by Linux kernel
        # we need to swap to stdin to communicate parameters. This avoids a
        # failure to fork the wrapper process when using large parameters.
        stdin = None
        stdin_params = None
        if len(serialized_parameters) >= MAX_PARAM_LENGTH:
            stdin = subprocess.PIPE
            LOG.debug('Parameters are too big...changing to stdin')
            stdin_params = '{"parameters": %s}\n' % (serialized_parameters)
            args.append('--stdin-parameters')
        else:
            LOG.debug('Parameters are just right...adding them to arguments')
            args.append('--parameters=%s' % (serialized_parameters))

        if self._config:
            args.append('--config=%s' % (json.dumps(self._config)))

        if self._log_level != PYTHON_RUNNER_DEFAULT_LOG_LEVEL:
            # We only pass --log-level parameter if non default log level value is specified
            args.append('--log-level=%s' % (self._log_level))

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        LOG.debug('Setting env.')
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)

        sandbox_python_path = get_sandbox_python_path(inherit_from_parent=True,
                                                      inherit_parent_virtualenv=True)

        if self._enable_common_pack_libs:
            try:
                pack_common_libs_path = self._get_pack_common_libs_path(pack_ref=pack)
            except Exception as e:
                LOG.debug('Failed to retrieve pack common lib path: %s' % (str(e)))
                print(e)
                # There is no MongoDB connection available in Lambda and pack common lib
                # functionality is not also mandatory for Lambda so we simply ignore those errors.
                # Note: We should eventually refactor this code to make runner standalone and not
                # depend on a db connection (as it was in the past) - this param should be passed
                # to the runner by the action runner container
                pack_common_libs_path = None
        else:
            pack_common_libs_path = None

        # Remove leading : (if any)
        if sandbox_python_path.startswith(':'):
            sandbox_python_path = sandbox_python_path[1:]

        if self._enable_common_pack_libs and pack_common_libs_path:
            env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path
        else:
            env['PYTHONPATH'] = sandbox_python_path

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(store_execution_output_data,
                                                        output_type='stdout')
        store_execution_stderr_line = functools.partial(store_execution_output_data,
                                                        output_type='stderr')

        read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stdout_line)
        read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stderr_line)

        command_string = list2cmdline(args)
        if stdin_params:
            command_string = 'echo %s | %s' % (quote_unix(stdin_params), command_string)

        LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
                                                                 command_string))
        exit_code, stdout, stderr, timed_out = run_command(cmd=args,
                                                           stdin=stdin,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           shell=False,
                                                           env=env,
                                                           timeout=self._timeout,
                                                           read_stdout_func=read_and_store_stdout,
                                                           read_stderr_func=read_and_store_stderr,
                                                           read_stdout_buffer=stdout,
                                                           read_stderr_buffer=stderr,
                                                           stdin_value=stdin_params)
        LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out)
        LOG.debug('Returning.')
        return self._get_output_values(exit_code, stdout, stderr, timed_out)
Exemple #35
0
    def _run(self, action):
        env_vars = self._env

        if not self.entry_point:
            script_action = False
        else:
            script_action = True

        args = action.get_full_command_string()
        sanitized_args = action.get_sanitized_full_command_string()

        # For consistency with the old Fabric based runner, make sure the file is executable
        if script_action:
            script_local_path_abs = self.entry_point
            args = 'chmod +x %s ; %s' % (script_local_path_abs, args)
            sanitized_args = 'chmod +x %s ; %s' % (script_local_path_abs, sanitized_args)

        env = os.environ.copy()

        # Include user provided env vars (if any)
        env.update(env_vars)

        # Include common st2 env vars
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)

        LOG.info('Executing action via LocalRunner: %s', self.runner_id)
        LOG.info('[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
                 (action.name, action.action_exec_id, sanitized_args, action.user, action.sudo))

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(store_execution_output_data,
                                                        output_type='stdout')
        store_execution_stderr_line = functools.partial(store_execution_output_data,
                                                        output_type='stderr')

        read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stdout_line)
        read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stderr_line)

        # If sudo password is provided, pass it to the subprocess via stdin>
        # Note: We don't need to explicitly escape the argument because we pass command as a list
        # to subprocess.Popen and all the arguments are escaped by the function.
        if self._sudo_password:
            LOG.debug('Supplying sudo password via stdin')
            echo_process = subprocess.Popen(['echo', self._sudo_password + '\n'],
                                            stdout=subprocess.PIPE)
            stdin = echo_process.stdout
        else:
            stdin = None

        # Make sure os.setsid is called on each spawned process so that all processes
        # are in the same group.

        # Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
        # bash means that multiple independent processes are spawned without them being
        # children of the process we have access to and this requires use of pkill.
        # Ideally os.killpg should have done the trick but for some reason that failed.
        # Note: pkill will set the returncode to 143 so we don't need to explicitly set
        # it to some non-zero value.
        exit_code, stdout, stderr, timed_out = shell.run_command(cmd=args,
                                                                 stdin=stdin,
                                                                 stdout=subprocess.PIPE,
                                                                 stderr=subprocess.PIPE,
                                                                 shell=True,
                                                                 cwd=self._cwd,
                                                                 env=env,
                                                                 timeout=self._timeout,
                                                                 preexec_func=os.setsid,
                                                                 kill_func=kill_process,
                                                           read_stdout_func=read_and_store_stdout,
                                                           read_stderr_func=read_and_store_stderr,
                                                           read_stdout_buffer=stdout,
                                                           read_stderr_buffer=stderr)

        error = None

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
            exit_code = -1 * exit_code_constants.SIGKILL_EXIT_CODE

        # Detect if user provided an invalid sudo password or sudo is not configured for that user
        if self._sudo_password:
            if re.search(r'sudo: \d+ incorrect password attempts', stderr):
                match = re.search(r'\[sudo\] password for (.+?)\:', stderr)

                if match:
                    username = match.groups()[0]
                else:
                    username = '******'

                error = ('Invalid sudo password provided or sudo is not configured for this user '
                        '(%s)' % (username))
                exit_code = -1

        succeeded = (exit_code == exit_code_constants.SUCCESS_EXIT_CODE)

        result = {
            'failed': not succeeded,
            'succeeded': succeeded,
            'return_code': exit_code,
            'stdout': strip_shell_chars(stdout),
            'stderr': strip_shell_chars(stderr)
        }

        if error:
            result['error'] = error

        status = PROC_EXIT_CODE_TO_LIVEACTION_STATUS_MAP.get(
            str(exit_code),
            action_constants.LIVEACTION_STATUS_FAILED
        )

        return (status, jsonify.json_loads(result, BaseLocalShellRunner.KEYS_TO_TRANSFORM), None)
Exemple #36
0
    def run(self, action_parameters):
        LOG.debug('Running pythonrunner.')
        LOG.debug('Getting pack name.')
        pack = self.get_pack_ref()
        LOG.debug('Getting user.')
        user = self.get_user()
        LOG.debug('Serializing parameters.')
        serialized_parameters = json.dumps(
            action_parameters) if action_parameters else ''
        LOG.debug('Getting virtualenv_path.')
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        LOG.debug('Getting python path.')
        if self._sandbox:
            python_path = get_sandbox_python_binary_path(pack=pack)
        else:
            python_path = sys.executable

        LOG.debug('Checking virtualenv path.')
        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            LOG.error('virtualenv_path set but not a directory: %s', msg)
            raise Exception(msg)

        LOG.debug('Checking entry_point.')
        if not self.entry_point:
            LOG.error('Action "%s" is missing entry_point attribute' %
                      (self.action.name))
            raise Exception('Action "%s" is missing entry_point attribute' %
                            (self.action.name))

        # Note: We pass config as command line args so the actual wrapper process is standalone
        # and doesn't need access to db
        LOG.debug('Setting args.')
        args = [
            python_path,
            '-u',  # unbuffered mode so streaming mode works as expected
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--user=%s' % (user),
            '--parent-args=%s' % (json.dumps(sys.argv[1:])),
        ]

        if self._config:
            args.append('--config=%s' % (json.dumps(self._config)))

        if self._log_level != 'debug':
            # We only pass --log-level parameter if non default log level value is specified
            args.append('--log-level=%s' % (self._log_level))

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        LOG.debug('Setting env.')
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)

        sandbox_python_path = get_sandbox_python_path(
            inherit_from_parent=True, inherit_parent_virtualenv=True)

        if self._enable_common_pack_libs:
            try:
                pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(
                    pack_ref=pack)
            except Exception:
                # There is no MongoDB connection available in Lambda and pack common lib
                # functionality is not also mandatory for Lambda so we simply ignore those errors.
                # Note: We should eventually refactor this code to make runner standalone and not
                # depend on a db connection (as it was in the past) - this param should be passed
                # to the runner by the action runner container
                pack_common_libs_path = None
        else:
            pack_common_libs_path = None

        if self._enable_common_pack_libs and pack_common_libs_path:
            env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path
        else:
            env['PYTHONPATH'] = sandbox_python_path

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(
            store_execution_output_data, output_type='stdout')
        store_execution_stderr_line = functools.partial(
            store_execution_output_data, output_type='stderr')

        read_and_store_stdout = make_read_and_store_stream_func(
            execution_db=self.execution,
            action_db=self.action,
            store_data_func=store_execution_stdout_line)
        read_and_store_stderr = make_read_and_store_stream_func(
            execution_db=self.execution,
            action_db=self.action,
            store_data_func=store_execution_stderr_line)

        command_string = list2cmdline(args)
        LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' %
                  (env['PATH'], env['PYTHONPATH'], command_string))
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=args,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=False,
            env=env,
            timeout=self._timeout,
            read_stdout_func=read_and_store_stdout,
            read_stderr_func=read_and_store_stderr,
            read_stdout_buffer=stdout,
            read_stderr_buffer=stderr)
        LOG.debug('Returning values: %s, %s, %s, %s' %
                  (exit_code, stdout, stderr, timed_out))
        LOG.debug('Returning.')
        return self._get_output_values(exit_code, stdout, stderr, timed_out)
Exemple #37
0
    def _run(self, action):
        env_vars = self._env

        if not self.entry_point:
            script_action = False
        else:
            script_action = True

        args = action.get_full_command_string()
        sanitized_args = action.get_sanitized_full_command_string()

        # For consistency with the old Fabric based runner, make sure the file is executable
        if script_action:
            script_local_path_abs = self.entry_point
            args = "chmod +x %s ; %s" % (script_local_path_abs, args)
            sanitized_args = "chmod +x %s ; %s" % (
                script_local_path_abs,
                sanitized_args,
            )

        env = os.environ.copy()

        # Include user provided env vars (if any)
        env.update(env_vars)

        # Include common st2 env vars
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)

        LOG.info("Executing action via LocalRunner: %s", self.runner_id)
        LOG.info(
            "[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s"
            % (
                action.name,
                action.action_exec_id,
                sanitized_args,
                action.user,
                action.sudo,
            )
        )

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(
            store_execution_output_data, output_type="stdout"
        )
        store_execution_stderr_line = functools.partial(
            store_execution_output_data, output_type="stderr"
        )

        read_and_store_stdout = make_read_and_store_stream_func(
            execution_db=self.execution,
            action_db=self.action,
            store_data_func=store_execution_stdout_line,
        )
        read_and_store_stderr = make_read_and_store_stream_func(
            execution_db=self.execution,
            action_db=self.action,
            store_data_func=store_execution_stderr_line,
        )

        subprocess = concurrency.get_subprocess_module()

        # If sudo password is provided, pass it to the subprocess via stdin>
        # Note: We don't need to explicitly escape the argument because we pass command as a list
        # to subprocess.Popen and all the arguments are escaped by the function.
        if self._sudo_password:
            LOG.debug("Supplying sudo password via stdin")
            echo_process = concurrency.subprocess_popen(
                ["echo", self._sudo_password + "\n"], stdout=subprocess.PIPE
            )
            stdin = echo_process.stdout
        else:
            stdin = None

        # Make sure os.setsid is called on each spawned process so that all processes
        # are in the same group.

        # Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
        # bash means that multiple independent processes are spawned without them being
        # children of the process we have access to and this requires use of pkill.
        # Ideally os.killpg should have done the trick but for some reason that failed.
        # Note: pkill will set the returncode to 143 so we don't need to explicitly set
        # it to some non-zero value.
        exit_code, stdout, stderr, timed_out = shell.run_command(
            cmd=args,
            stdin=stdin,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True,
            cwd=self._cwd,
            env=env,
            timeout=self._timeout,
            preexec_func=os.setsid,
            kill_func=kill_process,
            read_stdout_func=read_and_store_stdout,
            read_stderr_func=read_and_store_stderr,
            read_stdout_buffer=stdout,
            read_stderr_buffer=stderr,
        )

        error = None

        if timed_out:
            error = "Action failed to complete in %s seconds" % (self._timeout)
            exit_code = -1 * exit_code_constants.SIGKILL_EXIT_CODE

        # Detect if user provided an invalid sudo password or sudo is not configured for that user
        if self._sudo_password:
            if re.search(r"sudo: \d+ incorrect password attempts", stderr):
                match = re.search(r"\[sudo\] password for (.+?)\:", stderr)

                if match:
                    username = match.groups()[0]
                else:
                    username = "******"

                error = (
                    "Invalid sudo password provided or sudo is not configured for this user "
                    "(%s)" % (username)
                )
                exit_code = -1

        succeeded = exit_code == exit_code_constants.SUCCESS_EXIT_CODE

        result = {
            "failed": not succeeded,
            "succeeded": succeeded,
            "return_code": exit_code,
            "stdout": strip_shell_chars(stdout),
            "stderr": strip_shell_chars(stderr),
        }

        if error:
            result["error"] = error

        status = PROC_EXIT_CODE_TO_LIVEACTION_STATUS_MAP.get(
            str(exit_code), action_constants.LIVEACTION_STATUS_FAILED
        )

        return (
            status,
            jsonify.json_loads(result, BaseLocalShellRunner.KEYS_TO_TRANSFORM),
            None,
        )
Exemple #38
0
    def run(self, action_parameters):
        pack = self.get_pack_name()
        serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        python_path = get_sandbox_python_binary_path(pack=pack)

        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            raise Exception(msg)

        if not self.entry_point:
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        args = [
            python_path,
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--parent-args=%s' % (json.dumps(sys.argv[1:]))
        ]

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
        env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
                                                    inherit_parent_virtualenv=True)

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           env=env, timeout=self._timeout)

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
        else:
            error = None

        if ACTION_OUTPUT_RESULT_DELIMITER in stdout:
            split = stdout.split(ACTION_OUTPUT_RESULT_DELIMITER)
            assert len(split) == 3
            result = split[1].strip()
            stdout = split[0] + split[2]
        else:
            result = None

        try:
            result = json.loads(result)
        except:
            pass

        output = {
            'stdout': stdout,
            'stderr': stderr,
            'exit_code': exit_code,
            'result': result
        }

        if error:
            output['error'] = error

        if exit_code == 0:
            status = LIVEACTION_STATUS_SUCCEEDED
        elif timed_out:
            status = LIVEACTION_STATUS_TIMED_OUT
        else:
            status = LIVEACTION_STATUS_FAILED

        return (status, output, None)
Exemple #39
0
    def create_git_worktree(self, content_version):
        """
        Create a git worktree for the provided git content version.

        :return: Path to the created git worktree directory.
        :rtype: ``str``
        """
        pack_name = self.get_pack_name()
        pack_directory = get_pack_directory(pack_name=pack_name)
        worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX)

        # Set class variables
        self.git_worktree_revision = content_version
        self.git_worktree_path = worktree_path

        extra = {
            'pack_name': pack_name,
            'pack_directory': pack_directory,
            'content_version': content_version,
            'worktree_path': worktree_path
        }

        if not os.path.isdir(pack_directory):
            msg = (
                'Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t '
                'exist.' % (pack_name, pack_directory))
            raise ValueError(msg)

        args = [
            'git', '-C', pack_directory, 'worktree', 'add', worktree_path,
            content_version
        ]
        cmd = list2cmdline(args)

        LOG.debug(
            'Creating git worktree for pack "%s", content version "%s" and execution '
            'id "%s" in "%s"' %
            (pack_name, content_version, self.execution_id, worktree_path),
            extra=extra)
        LOG.debug('Command: %s' % (cmd))
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=cmd,
            cwd=pack_directory,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True)

        if exit_code != 0:
            self._handle_git_worktree_error(pack_name=pack_name,
                                            pack_directory=pack_directory,
                                            content_version=content_version,
                                            exit_code=exit_code,
                                            stdout=stdout,
                                            stderr=stderr)
        else:
            LOG.debug('Git worktree created in "%s"' % (worktree_path),
                      extra=extra)

        # Make sure system / action runner user can access that directory
        args = ['chmod', '777', worktree_path]
        cmd = list2cmdline(args)
        run_command(cmd=cmd, shell=True)

        return worktree_path
Exemple #40
0
    def run(self, action_parameters):
        pack = self.action.pack if self.action else DEFAULT_PACK_NAME
        serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        python_path = get_sandbox_python_binary_path(pack=pack)

        if virtualenv_path and not os.path.isdir(virtualenv_path):
            msg = PACK_VIRTUALENV_DOESNT_EXIST % (pack, pack)
            raise Exception(msg)

        if not self.entry_point:
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        args = [
            python_path,
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--parent-args=%s' % (json.dumps(sys.argv[1:]))
        ]

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        env = os.environ.copy()
        env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
                                                    inherit_parent_virtualenv=True)

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           env=env, timeout=self._timeout)

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
        else:
            error = None

        if ACTION_OUTPUT_RESULT_DELIMITER in stdout:
            split = stdout.split(ACTION_OUTPUT_RESULT_DELIMITER)
            assert len(split) == 3
            result = split[1].strip()
            stdout = split[0] + split[2]
        else:
            result = None

        try:
            result = json.loads(result)
        except:
            pass

        output = {
            'stdout': stdout,
            'stderr': stderr,
            'exit_code': exit_code,
            'result': result
        }

        if error:
            output['error'] = error

        status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
        self._log_action_completion(logger=LOG, result=output, status=status, exit_code=exit_code)
        return (status, output, None)
Exemple #41
0
    def run(self, action_parameters):
        env_vars = self._env

        if not self.entry_point:
            script_action = False
            command = self.runner_parameters.get(RUNNER_COMMAND, None)
            action = ShellCommandAction(name=self.action_name,
                                        action_exec_id=str(self.liveaction_id),
                                        command=command,
                                        user=self._user,
                                        env_vars=env_vars,
                                        sudo=self._sudo,
                                        timeout=self._timeout,
                                        sudo_password=self._sudo_password)
        else:
            script_action = True
            script_local_path_abs = self.entry_point
            positional_args, named_args = self._get_script_args(action_parameters)
            named_args = self._transform_named_args(named_args)

            action = ShellScriptAction(name=self.action_name,
                                       action_exec_id=str(self.liveaction_id),
                                       script_local_path_abs=script_local_path_abs,
                                       named_args=named_args,
                                       positional_args=positional_args,
                                       user=self._user,
                                       env_vars=env_vars,
                                       sudo=self._sudo,
                                       timeout=self._timeout,
                                       cwd=self._cwd,
                                       sudo_password=self._sudo_password)

        args = action.get_full_command_string()
        sanitized_args = action.get_sanitized_full_command_string()

        # For consistency with the old Fabric based runner, make sure the file is executable
        if script_action:
            args = 'chmod +x %s ; %s' % (script_local_path_abs, args)
            sanitized_args = 'chmod +x %s ; %s' % (script_local_path_abs, sanitized_args)

        env = os.environ.copy()

        # Include user provided env vars (if any)
        env.update(env_vars)

        # Include common st2 env vars
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)

        LOG.info('Executing action via LocalRunner: %s', self.runner_id)
        LOG.info('[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
                 (action.name, action.action_exec_id, sanitized_args, action.user, action.sudo))

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(store_execution_output_data,
                                                        output_type='stdout')
        store_execution_stderr_line = functools.partial(store_execution_output_data,
                                                        output_type='stderr')

        read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stdout_line)
        read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stderr_line)

        # If sudo password is provided, pass it to the subprocess via stdin>
        # Note: We don't need to explicitly escape the argument because we pass command as a list
        # to subprocess.Popen and all the arguments are escaped by the function.
        if self._sudo_password:
            LOG.debug('Supplying sudo password via stdin')
            echo_process = subprocess.Popen(['echo', self._sudo_password + '\n'],
                                            stdout=subprocess.PIPE)
            stdin = echo_process.stdout
        else:
            stdin = None

        # Make sure os.setsid is called on each spawned process so that all processes
        # are in the same group.

        # Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
        # bash means that multiple independent processes are spawned without them being
        # children of the process we have access to and this requires use of pkill.
        # Ideally os.killpg should have done the trick but for some reason that failed.
        # Note: pkill will set the returncode to 143 so we don't need to explicitly set
        # it to some non-zero value.
        exit_code, stdout, stderr, timed_out = shell.run_command(cmd=args,
                                                                 stdin=stdin,
                                                                 stdout=subprocess.PIPE,
                                                                 stderr=subprocess.PIPE,
                                                                 shell=True,
                                                                 cwd=self._cwd,
                                                                 env=env,
                                                                 timeout=self._timeout,
                                                                 preexec_func=os.setsid,
                                                                 kill_func=kill_process,
                                                           read_stdout_func=read_and_store_stdout,
                                                           read_stderr_func=read_and_store_stderr,
                                                           read_stdout_buffer=stdout,
                                                           read_stderr_buffer=stderr)

        error = None

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
            exit_code = -1 * exit_code_constants.SIGKILL_EXIT_CODE

        # Detect if user provided an invalid sudo password or sudo is not configured for that user
        if self._sudo_password:
            if re.search('sudo: \d+ incorrect password attempts', stderr):
                match = re.search('\[sudo\] password for (.+?)\:', stderr)

                if match:
                    username = match.groups()[0]
                else:
                    username = '******'

                error = ('Invalid sudo password provided or sudo is not configured for this user '
                        '(%s)' % (username))
                exit_code = -1

        succeeded = (exit_code == exit_code_constants.SUCCESS_EXIT_CODE)

        result = {
            'failed': not succeeded,
            'succeeded': succeeded,
            'return_code': exit_code,
            'stdout': strip_shell_chars(stdout),
            'stderr': strip_shell_chars(stderr)
        }

        if error:
            result['error'] = error

        status = PROC_EXIT_CODE_TO_LIVEACTION_STATUS_MAP.get(
            str(exit_code),
            action_constants.LIVEACTION_STATUS_FAILED
        )

        return (status, jsonify.json_loads(result, LocalShellRunner.KEYS_TO_TRANSFORM), None)
Exemple #42
0
    def run(self, action_parameters):
        LOG.debug('Running pythonrunner.')
        LOG.debug('Getting pack name.')
        pack = self.get_pack_name()
        LOG.debug('Getting user.')
        user = self.get_user()
        LOG.debug('Serializing parameters.')
        serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
        LOG.debug('Getting virtualenv_path.')
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        LOG.debug('Getting python path.')
        python_path = get_sandbox_python_binary_path(pack=pack)

        LOG.debug('Checking virtualenv path.')
        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            LOG.error('virtualenv_path set but not a directory: %s', msg)
            raise Exception(msg)

        LOG.debug('Checking entry_point.')
        if not self.entry_point:
            LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        LOG.debug('Setting args.')
        args = [
            python_path,
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--user=%s' % (user),
            '--parent-args=%s' % (json.dumps(sys.argv[1:]))
        ]

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        LOG.debug('Setting env.')
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
        env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
                                                    inherit_parent_virtualenv=True)

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        command_string = list2cmdline(args)
        LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
                                                                 command_string))
        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           env=env, timeout=self._timeout)
        LOG.debug('Returning values: %s, %s, %s, %s' % (exit_code, stdout, stderr, timed_out))
        LOG.debug('Returning.')
        return self._get_output_values(exit_code, stdout, stderr, timed_out)
Exemple #43
0
    def create_git_worktree(self, content_version):
        """
        Create a git worktree for the provided git content version.

        :return: Path to the created git worktree directory.
        :rtype: ``str``
        """
        pack_name = self.get_pack_name()
        pack_directory = get_pack_directory(pack_name=pack_name)
        worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX)

        # Set class variables
        self.git_worktree_revision = content_version
        self.git_worktree_path = worktree_path

        extra = {
            'pack_name': pack_name,
            'pack_directory': pack_directory,
            'content_version': content_version,
            'worktree_path': worktree_path
        }

        if not os.path.isdir(pack_directory):
            msg = ('Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t '
                   'exist.' % (pack_name, pack_directory))
            raise ValueError(msg)

        args = [
            'git',
            '-C',
            pack_directory,
            'worktree',
            'add',
            worktree_path,
            content_version
        ]
        cmd = list2cmdline(args)

        LOG.debug('Creating git worktree for pack "%s", content version "%s" and execution '
                  'id "%s" in "%s"' % (pack_name, content_version, self.execution_id,
                                       worktree_path), extra=extra)
        LOG.debug('Command: %s' % (cmd))
        exit_code, stdout, stderr, timed_out = run_command(cmd=cmd,
                                                           cwd=pack_directory,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           shell=True)

        if exit_code != 0:
            self._handle_git_worktree_error(pack_name=pack_name, pack_directory=pack_directory,
                                            content_version=content_version,
                                            exit_code=exit_code, stdout=stdout, stderr=stderr)
        else:
            LOG.debug('Git worktree created in "%s"' % (worktree_path), extra=extra)

        # Make sure system / action runner user can access that directory
        args = [
            'chmod',
            '777',
            worktree_path
        ]
        cmd = list2cmdline(args)
        run_command(cmd=cmd, shell=True)

        return worktree_path
    def run(self, action_parameters):
        env_vars = self._env

        if not self.entry_point:
            script_action = False
            command = self.runner_parameters.get(RUNNER_COMMAND, None)
            action = ShellCommandAction(name=self.action_name,
                                        action_exec_id=str(self.liveaction_id),
                                        command=command,
                                        user=self._user,
                                        env_vars=env_vars,
                                        sudo=self._sudo,
                                        timeout=self._timeout)
        else:
            script_action = True
            script_local_path_abs = self.entry_point
            positional_args, named_args = self._get_script_args(
                action_parameters)
            named_args = self._transform_named_args(named_args)

            action = ShellScriptAction(
                name=self.action_name,
                action_exec_id=str(self.liveaction_id),
                script_local_path_abs=script_local_path_abs,
                named_args=named_args,
                positional_args=positional_args,
                user=self._user,
                env_vars=env_vars,
                sudo=self._sudo,
                timeout=self._timeout,
                cwd=self._cwd)

        args = action.get_full_command_string()

        # For consistency with the old Fabric based runner, make sure the file is executable
        if script_action:
            args = 'chmod +x %s ; %s' % (script_local_path_abs, args)

        env = os.environ.copy()

        # Include user provided env vars (if any)
        env.update(env_vars)

        # Include common st2 env vars
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)

        LOG.info('Executing action via LocalRunner: %s', self.runner_id)
        LOG.info(
            '[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
            (action.name, action.action_exec_id, args, action.user,
             action.sudo))

        # Make sure os.setsid is called on each spawned process so that all processes
        # are in the same group.

        # Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
        # bash means that multiple independent processes are spawned without them being
        # children of the process we have access to and this requires use of pkill.
        # Ideally os.killpg should have done the trick but for some reason that failed.
        # Note: pkill will set the returncode to 143 so we don't need to explicitly set
        # it to some non-zero value.
        exit_code, stdout, stderr, timed_out = run_command(
            cmd=args,
            stdin=None,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True,
            cwd=self._cwd,
            env=env,
            timeout=self._timeout,
            preexec_func=os.setsid,
            kill_func=kill_process)

        error = None

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
            exit_code = -9

        succeeded = (exit_code == 0)

        result = {
            'failed': not succeeded,
            'succeeded': succeeded,
            'return_code': exit_code,
            'stdout': strip_shell_chars(stdout),
            'stderr': strip_shell_chars(stderr)
        }

        if error:
            result['error'] = error

        if exit_code == 0:
            status = LIVEACTION_STATUS_SUCCEEDED
        elif timed_out:
            status = LIVEACTION_STATUS_TIMED_OUT
        else:
            status = LIVEACTION_STATUS_FAILED

        return (status,
                jsonify.json_loads(result,
                                   LocalShellRunner.KEYS_TO_TRANSFORM), None)
Exemple #45
0
    def run(self, action_parameters):
        env_vars = self._env

        if not self.entry_point:
            script_action = False
            command = self.runner_parameters.get(RUNNER_COMMAND, None)
            action = ShellCommandAction(name=self.action_name,
                                        action_exec_id=str(self.liveaction_id),
                                        command=command,
                                        user=self._user,
                                        env_vars=env_vars,
                                        sudo=self._sudo,
                                        timeout=self._timeout)
        else:
            script_action = True
            script_local_path_abs = self.entry_point
            positional_args, named_args = self._get_script_args(action_parameters)
            named_args = self._transform_named_args(named_args)

            action = ShellScriptAction(name=self.action_name,
                                       action_exec_id=str(self.liveaction_id),
                                       script_local_path_abs=script_local_path_abs,
                                       named_args=named_args,
                                       positional_args=positional_args,
                                       user=self._user,
                                       env_vars=env_vars,
                                       sudo=self._sudo,
                                       timeout=self._timeout,
                                       cwd=self._cwd)

        args = action.get_full_command_string()

        # For consistency with the old Fabric based runner, make sure the file is executable
        if script_action:
            args = 'chmod +x %s ; %s' % (script_local_path_abs, args)

        env = os.environ.copy()

        # Include user provided env vars (if any)
        env.update(env_vars)

        # Include common st2 env vars
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)

        LOG.info('Executing action via LocalRunner: %s', self.runner_id)
        LOG.info('[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
                 (action.name, action.action_exec_id, args, action.user, action.sudo))

        # Make sure os.setsid is called on each spawned process so that all processes
        # are in the same group.

        # Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
        # bash means that multiple independent processes are spawned without them being
        # children of the process we have access to and this requires use of pkill.
        # Ideally os.killpg should have done the trick but for some reason that failed.
        # Note: pkill will set the returncode to 143 so we don't need to explicitly set
        # it to some non-zero value.
        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdin=None,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           shell=True,
                                                           cwd=self._cwd,
                                                           env=env,
                                                           timeout=self._timeout,
                                                           preexec_func=os.setsid,
                                                           kill_func=kill_process)

        error = None

        if timed_out:
            error = 'Action failed to complete in %s seconds' % (self._timeout)
            exit_code = -9

        succeeded = (exit_code == 0)

        result = {
            'failed': not succeeded,
            'succeeded': succeeded,
            'return_code': exit_code,
            'stdout': strip_last_newline_char(stdout),
            'stderr': strip_last_newline_char(stderr)
        }

        if error:
            result['error'] = error

        status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
        return (status, jsonify.json_loads(result, LocalShellRunner.KEYS_TO_TRANSFORM), None)
Exemple #46
0
    def run(self, action_parameters):
        LOG.debug('Running pythonrunner.')
        LOG.debug('Getting pack name.')
        pack = self.get_pack_ref()
        LOG.debug('Getting user.')
        user = self.get_user()
        LOG.debug('Serializing parameters.')
        serialized_parameters = json.dumps(action_parameters if action_parameters else {})
        LOG.debug('Getting virtualenv_path.')
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        LOG.debug('Getting python path.')
        if self._sandbox:
            python_path = get_sandbox_python_binary_path(pack=pack)
        else:
            python_path = sys.executable

        LOG.debug('Checking virtualenv path.')
        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            LOG.error('virtualenv_path set but not a directory: %s', msg)
            raise Exception(msg)

        LOG.debug('Checking entry_point.')
        if not self.entry_point:
            LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        # Note: We pass config as command line args so the actual wrapper process is standalone
        # and doesn't need access to db
        LOG.debug('Setting args.')

        if self._use_parent_args:
            parent_args = json.dumps(sys.argv[1:])
        else:
            parent_args = json.dumps([])

        args = [
            python_path,
            '-u',  # unbuffered mode so streaming mode works as expected
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--user=%s' % (user),
            '--parent-args=%s' % (parent_args),
        ]

        # If parameter size is larger than the maximum allowed by Linux kernel
        # we need to swap to stdin to communicate parameters. This avoids a
        # failure to fork the wrapper process when using large parameters.
        stdin = None
        stdin_params = None
        if len(serialized_parameters) >= MAX_PARAM_LENGTH:
            stdin = subprocess.PIPE
            LOG.debug('Parameters are too big...changing to stdin')
            stdin_params = '{"parameters": %s}\n' % (serialized_parameters)
            args.append('--stdin-parameters')
        else:
            LOG.debug('Parameters are just right...adding them to arguments')
            args.append('--parameters=%s' % (serialized_parameters))

        if self._config:
            args.append('--config=%s' % (json.dumps(self._config)))

        if self._log_level != PYTHON_RUNNER_DEFAULT_LOG_LEVEL:
            # We only pass --log-level parameter if non default log level value is specified
            args.append('--log-level=%s' % (self._log_level))

        # We need to ensure all the st2 dependencies are also available to the subprocess
        LOG.debug('Setting env.')
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)

        sandbox_python_path = get_sandbox_python_path_for_python_action(
            pack=pack,
            inherit_from_parent=True,
            inherit_parent_virtualenv=True)

        if self._enable_common_pack_libs:
            try:
                pack_common_libs_path = self._get_pack_common_libs_path(pack_ref=pack)
            except Exception as e:
                LOG.debug('Failed to retrieve pack common lib path: %s' % (str(e)))
                # There is no MongoDB connection available in Lambda and pack common lib
                # functionality is not also mandatory for Lambda so we simply ignore those errors.
                # Note: We should eventually refactor this code to make runner standalone and not
                # depend on a db connection (as it was in the past) - this param should be passed
                # to the runner by the action runner container
                pack_common_libs_path = None
        else:
            pack_common_libs_path = None

        # Remove leading : (if any)
        if sandbox_python_path.startswith(':'):
            sandbox_python_path = sandbox_python_path[1:]

        if self._enable_common_pack_libs and pack_common_libs_path:
            sandbox_python_path = pack_common_libs_path + ':' + sandbox_python_path

        env['PYTHONPATH'] = sandbox_python_path

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(store_execution_output_data,
                                                        output_type='stdout')
        store_execution_stderr_line = functools.partial(store_execution_output_data,
                                                        output_type='stderr')

        read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stdout_line)
        read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stderr_line)

        command_string = list2cmdline(args)
        if stdin_params:
            command_string = 'echo %s | %s' % (quote_unix(stdin_params), command_string)

        LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
                                                                 command_string))
        exit_code, stdout, stderr, timed_out = run_command(cmd=args,
                                                           stdin=stdin,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           shell=False,
                                                           env=env,
                                                           timeout=self._timeout,
                                                           read_stdout_func=read_and_store_stdout,
                                                           read_stderr_func=read_and_store_stderr,
                                                           read_stdout_buffer=stdout,
                                                           read_stderr_buffer=stderr,
                                                           stdin_value=stdin_params)
        LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out)
        LOG.debug('Returning.')
        return self._get_output_values(exit_code, stdout, stderr, timed_out)
Exemple #47
0
    def run(self, action_parameters):
        LOG.debug('Running pythonrunner.')
        LOG.debug('Getting pack name.')
        pack = self.get_pack_name()
        LOG.debug('Getting user.')
        user = self.get_user()
        LOG.debug('Serializing parameters.')
        serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
        LOG.debug('Getting virtualenv_path.')
        virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
        LOG.debug('Getting python path.')
        python_path = get_sandbox_python_binary_path(pack=pack)

        LOG.debug('Checking virtualenv path.')
        if virtualenv_path and not os.path.isdir(virtualenv_path):
            format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
            msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
            LOG.error('virtualenv_path set but not a directory: %s', msg)
            raise Exception(msg)

        LOG.debug('Checking entry_point.')
        if not self.entry_point:
            LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
            raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))

        LOG.debug('Setting args.')
        args = [
            python_path,
            '-u',
            WRAPPER_SCRIPT_PATH,
            '--pack=%s' % (pack),
            '--file-path=%s' % (self.entry_point),
            '--parameters=%s' % (serialized_parameters),
            '--user=%s' % (user),
            '--parent-args=%s' % (json.dumps(sys.argv[1:]))
        ]

        # We need to ensure all the st2 dependencies are also available to the
        # subprocess
        LOG.debug('Setting env.')
        env = os.environ.copy()
        env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
        env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
                                                    inherit_parent_virtualenv=True)

        # Include user provided environment variables (if any)
        user_env_vars = self._get_env_vars()
        env.update(user_env_vars)

        # Include common st2 environment variables
        st2_env_vars = self._get_common_action_env_variables()
        env.update(st2_env_vars)
        datastore_env_vars = self._get_datastore_access_env_vars()
        env.update(datastore_env_vars)

        stdout = StringIO()
        stderr = StringIO()

        store_execution_stdout_line = functools.partial(store_execution_output_data,
                                                        output_type='stdout')
        store_execution_stderr_line = functools.partial(store_execution_output_data,
                                                        output_type='stderr')

        read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stdout_line)
        read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
            action_db=self.action, store_data_func=store_execution_stderr_line)

        command_string = list2cmdline(args)
        LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
                                                                 command_string))
        exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE, shell=False,
                                                           env=env,
                                                           timeout=self._timeout,
                                                           read_stdout_func=read_and_store_stdout,
                                                           read_stderr_func=read_and_store_stderr,
                                                           read_stdout_buffer=stdout,
                                                           read_stderr_buffer=stderr)
        LOG.debug('Returning values: %s, %s, %s, %s' % (exit_code, stdout, stderr, timed_out))
        LOG.debug('Returning.')
        return self._get_output_values(exit_code, stdout, stderr, timed_out)