Beispiel #1
0
    def run_with_result_store(self, manager_interrupt, result_store):
        """Run with a given ConnectResultStore."""
        serializer = SourceSerializer(self.scan_task.source)
        source = serializer.data

        if self.scan_job.options is not None:
            forks = self.scan_job.options.max_concurrency
        else:
            forks = ScanOptions.get_default_forks()

        if self.scan_task.source.options is not None:
            use_paramiko = self.scan_task.source.options.use_paramiko
        else:
            use_paramiko = False

        connection_port = source['port']
        credentials = source['credentials']

        remaining_hosts = result_store.remaining_hosts()

        for cred_id in credentials:
            check_manager_interrupt(manager_interrupt.value)
            credential = Credential.objects.get(pk=cred_id)
            if not remaining_hosts:
                message = 'Skipping credential %s.  No remaining hosts.' % \
                    credential.name
                self.scan_task.log_message(message)
                break

            message = 'Attempting credential %s.' % credential.name
            self.scan_task.log_message(message)

            try:
                scan_message, scan_result = _connect(manager_interrupt,
                                                     self.scan_task,
                                                     remaining_hosts,
                                                     result_store, credential,
                                                     connection_port, forks,
                                                     use_paramiko)
                if scan_result != ScanTask.COMPLETED:
                    return scan_message, scan_result
            except AnsibleRunnerException as ansible_error:
                remaining_hosts_str = ', '.join(result_store.remaining_hosts())
                error_message = 'Connect scan task failed with credential %s.'\
                    ' Error: %s Hosts: %s' %\
                    (credential.name, ansible_error, remaining_hosts_str)
                return error_message, ScanTask.FAILED

            remaining_hosts = result_store.remaining_hosts()

            logger.debug('Failed systems: %s', remaining_hosts)

        for host in remaining_hosts:
            # We haven't connected to these hosts with any
            # credentials, so they have failed.
            result_store.record_result(host, self.scan_task.source, None,
                                       SystemConnectionResult.FAILED)

        return None, ScanTask.COMPLETED
Beispiel #2
0
    def run_with_result_store(self, result_store):
        """Run with a given ConnectResultStore."""
        serializer = SourceSerializer(self.scan_task.source)
        source = serializer.data

        if self.scan_job.options is None:
            forks = ScanOptions.get_default_forks()
        else:
            forks = self.scan_job.options.max_concurrency
        connection_port = source['port']
        credentials = source['credentials']

        remaining_hosts = result_store.remaining_hosts()

        for cred_id in credentials:
            credential = Credential.objects.get(pk=cred_id)
            if not remaining_hosts:
                message = 'Skipping credential %s.  No remaining hosts.' % \
                    credential.name
                self.scan_task.log_message(message)
                break

            message = 'Attempting credential %s.' % credential.name
            self.scan_task.log_message(message)

            cred_data = CredentialSerializer(credential).data
            callback = ConnectResultCallback(result_store, credential,
                                             self.scan_task.source)
            try:
                connect(remaining_hosts,
                        callback,
                        cred_data,
                        connection_port,
                        forks=forks)
            except AnsibleError as ansible_error:
                remaining_hosts_str = ', '.join(result_store.remaining_hosts())
                error_message = 'Connect scan task failed with credential %s.'\
                    ' Error: %s Hosts: %s' %\
                    (credential.name, ansible_error, remaining_hosts_str)
                return error_message, ScanTask.FAILED

            remaining_hosts = result_store.remaining_hosts()

            logger.debug('Failed systems: %s', remaining_hosts)

        for host in remaining_hosts:
            # We haven't connected to these hosts with any
            # credentials, so they have failed.
            result_store.record_result(host, self.scan_task.source, None,
                                       SystemConnectionResult.FAILED)

        return None, ScanTask.COMPLETED
Beispiel #3
0
    def __init__(self, scan_job, scan_task):
        """Set context for interface."""
        self.scan_job = scan_job
        if scan_job.options is None:
            self.max_concurrency = ScanOptions.get_default_forks()
        else:
            self.max_concurrency = scan_job.options.max_concurrency

        if scan_task.scan_type == ScanTask.SCAN_TYPE_CONNECT:
            self.connect_scan_task = scan_task
            self.inspect_scan_task = None
        else:
            self.connect_scan_task = scan_task.prerequisites.first()
            self.inspect_scan_task = scan_task
        self.source = scan_task.source
Beispiel #4
0
    def setUp(self):
        """Create test setup."""
        management.call_command('flush', '--no-input')
        self.cred = Credential.objects.create(name='cred1',
                                              username='******',
                                              password='******',
                                              become_password=None,
                                              ssh_keyfile=None)
        self.cred_for_upload = self.cred.id

        self.source = Source(name='source1', source_type='network', port=22)
        self.source.save()

        self.source2 = Source(name='source2', source_type='network', port=22)
        self.source2.save()
        self.source2.credentials.add(self.cred)
        self.concurrency = ScanOptions.get_default_forks()
Beispiel #5
0
class ScanOptionsSerializer(NotEmptySerializer):
    """Serializer for the ScanOptions model."""

    max_concurrency = IntegerField(required=False,
                                   min_value=1,
                                   max_value=200,
                                   default=ScanOptions.get_default_forks())
    disabled_optional_products = DisableOptionalProductsOptionsSerializer(
        required=False)
    enabled_extended_product_search = ExtendedProductSearchOptionsSerializer(
        required=False)

    class Meta:
        """Metadata for serializer."""

        model = ScanOptions
        fields = [
            'max_concurrency', 'disabled_optional_products',
            'enabled_extended_product_search'
        ]
Beispiel #6
0
    def _inspect_scan(self,
                      manager_interrupt,
                      connected,
                      roles=DEFAULT_ROLES,
                      base_ssh_executable=None,
                      ssh_timeout=None):
        """Execute the host scan with the initialized source.

        :param manager_interrupt: Signal used to communicate termination
            of scan
        :param connected: list of (host, credential) pairs to inspect
        :param roles: list of roles to execute
        :param base_ssh_executable: ssh executable, or None for
            'ssh'. Will be wrapped with a timeout before being passed
            to Ansible.
        :param ssh_timeout: string in the format of the 'timeout'
            command. Timeout for individual tasks.
        :returns: An array of dictionaries of facts

        """
        connection_port = self.scan_task.source.port

        if self.scan_task.source.options is not None:
            use_paramiko = self.scan_task.source.options.use_paramiko
        else:
            use_paramiko = None

        if self.scan_job.options is not None:
            forks = self.scan_job.options.max_concurrency
            extra_vars = self.scan_job.options.get_extra_vars()
        else:
            forks = ScanOptions.get_default_forks()
            extra_vars = ScanOptions.get_default_extra_vars()

        if extra_vars.get(ScanOptions.EXT_PRODUCT_SEARCH_DIRS) is None:
            extra_vars[ScanOptions.EXT_PRODUCT_SEARCH_DIRS] = \
                ' '.join(DEFAULT_SCAN_DIRS)

        ssh_executable = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         '../../../bin/timeout_ssh'))

        base_ssh_executable = base_ssh_executable or 'ssh'
        ssh_timeout = ssh_timeout or settings.QPC_SSH_INSPECT_TIMEOUT

        # pylint: disable=line-too-long
        # the ssh arg is required for become-pass because
        # ansible checks for an exact string match of ssh
        # anywhere in the command array
        # See https://github.com/ansible/ansible/blob/stable-2.3/lib/ansible/plugins/connection/ssh.py#L490-L500 # noqa
        # timeout_ssh will remove the ssh argument before running the command
        ssh_args = [
            '--executable=' + base_ssh_executable, '--timeout=' + ssh_timeout,
            'ssh'
        ]

        group_names, inventory = _construct_scan_inventory(
            connected,
            connection_port,
            forks,
            ssh_executable=ssh_executable,
            ssh_args=ssh_args)
        inventory_file = write_inventory(inventory)

        error_msg = ''
        log_message = 'START INSPECT PROCESSING GROUPS'\
            ' with use_paramiko: %s, '\
            '%d forks and extra_vars=%s' % (use_paramiko,
                                            forks,
                                            extra_vars)
        self.scan_task.log_message(log_message)
        scan_result = ScanTask.COMPLETED
        scan_message = 'success'
        for idx, group_name in enumerate(group_names):
            if manager_interrupt.value == ScanJob.JOB_TERMINATE_CANCEL:
                raise NetworkCancelException()

            if manager_interrupt.value == ScanJob.JOB_TERMINATE_PAUSE:
                raise NetworkPauseException()

            log_message = 'START INSPECT PROCESSING GROUP %d of %d' % (
                (idx + 1), len(group_names))
            self.scan_task.log_message(log_message)
            callback =\
                InspectResultCallback(
                    scan_task=self.scan_task)
            playbook = {
                'name': 'scan systems for product fingerprint facts',
                'hosts': group_name,
                'gather_facts': False,
                'roles': roles
            }
            result = run_playbook(inventory_file,
                                  callback,
                                  playbook,
                                  extra_vars,
                                  use_paramiko,
                                  forks=forks)

            if result != TaskQueueManager.RUN_OK:
                new_error_msg = _construct_error_msg(result)
                callback.finalize_failed_hosts()
                if result not in [
                        TaskQueueManager.RUN_UNREACHABLE_HOSTS,
                        TaskQueueManager.RUN_FAILED_HOSTS
                ]:
                    error_msg += '{}\n'.format(new_error_msg)

        if error_msg != '':
            raise AnsibleError(error_msg)

        return scan_message, scan_result
Beispiel #7
0
    def _inspect_scan(self, connected, roles=DEFAULT_ROLES,
                      base_ssh_executable=None,
                      ssh_timeout=None):
        """Execute the host scan with the initialized source.

        :param connected: list of (host, credential) pairs to inspect
        :param roles: list of roles to execute
        :param base_ssh_executable: ssh executable, or None for
            'ssh'. Will be wrapped with a timeout before being passed
            to Ansible.
        :param ssh_timeout: string in the format of the 'timeout'
            command. Timeout for individual tasks.
        :returns: An array of dictionaries of facts

        """
        playbook = {'name': 'scan systems for product fingerprint facts',
                    'hosts': 'all',
                    'gather_facts': False,
                    'strategy': 'free',
                    'roles': roles}
        connection_port = self.scan_task.source.port

        if self.scan_job.options is not None:
            forks = self.scan_job.options.max_concurrency
            extra_vars = self.scan_job.options.get_extra_vars()
        else:
            forks = ScanOptions.get_default_forks()
            extra_vars = ScanOptions.get_default_extra_vars()

        if extra_vars.get(ScanOptions.EXT_PRODUCT_SEARCH_DIRS) is None:
            extra_vars[ScanOptions.EXT_PRODUCT_SEARCH_DIRS] = \
                ' '.join(DEFAULT_SCAN_DIRS)

        ssh_executable = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         '../../../bin/timeout_ssh'))

        base_ssh_executable = base_ssh_executable or 'ssh'
        ssh_timeout = ssh_timeout or DEFAULT_TIMEOUT
        ssh_args = ['--executable=' + base_ssh_executable,
                    '--timeout=' + ssh_timeout]

        group_names, inventory = _construct_scan_inventory(
            connected, connection_port, forks,
            ssh_executable=ssh_executable,
            ssh_args=ssh_args)
        inventory_file = write_inventory(inventory)

        error_msg = ''
        log_message = 'START PROCESSING GROUPS with %d forks' \
            ' and extra_vars=%s' % (forks, extra_vars)
        self.scan_task.log_message(log_message)
        scan_result = ScanTask.COMPLETED
        scan_message = 'success'
        for idx, group_name in enumerate(group_names):
            log_message = 'START PROCESSING GROUP %d of %d' % (
                (idx + 1), len(group_names))
            self.scan_task.log_message(log_message)
            callback =\
                InspectResultCallback(
                    scan_task=self.scan_task)
            playbook = {'name': 'scan systems for product fingerprint facts',
                        'hosts': group_name,
                        'gather_facts': False,
                        'roles': roles}
            result = run_playbook(
                inventory_file, callback, playbook,
                extra_vars, forks=forks)

            if result != TaskQueueManager.RUN_OK:
                new_error_msg = _construct_error_msg(result)
                callback.finalize_failed_hosts()
                if result != TaskQueueManager.RUN_UNREACHABLE_HOSTS and \
                        result != TaskQueueManager.RUN_FAILED_HOSTS:
                    error_msg += '{}\n'.format(new_error_msg)

        if error_msg != '':
            raise AnsibleError(error_msg)

        return scan_message, scan_result
Beispiel #8
0
    def _inspect_scan(self,
                      manager_interrupt,
                      connected,
                      base_ssh_executable=None,
                      ssh_timeout=None):
        """Execute the host scan with the initialized source.

        :param manager_interrupt: Signal used to communicate termination
            of scan
        :param connected: list of (host, credential) pairs to inspect
        :param roles: list of roles to execute
        :param base_ssh_executable: ssh executable, or None for
            'ssh'. Will be wrapped with a timeout before being passed
            to Ansible.
        :param ssh_timeout: string in the format of the 'timeout'
            command. Timeout for individual tasks.
        :returns: An array of dictionaries of facts

        Note: base_ssh_executable & ssh_timeout are parameters that
        are only used for testing.
        """
        # pylint: disable=too-many-locals,too-many-arguments
        # pylint: disable=too-many-branches,too-many-statements
        connection_port = self.scan_task.source.port

        if self.scan_task.source.options is not None:
            use_paramiko = self.scan_task.source.options.use_paramiko
        else:
            use_paramiko = False

        if self.scan_job.options is not None:
            forks = self.scan_job.options.max_concurrency
            extra_vars = self.scan_job.options.get_extra_vars()
        else:
            forks = ScanOptions.get_default_forks()
            extra_vars = ScanOptions.get_default_extra_vars()

        if extra_vars.get(ScanOptions.EXT_PRODUCT_SEARCH_DIRS) is None:
            extra_vars[ScanOptions.EXT_PRODUCT_SEARCH_DIRS] = \
                ' '.join(DEFAULT_SCAN_DIRS)

        ssh_executable = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         '../../../bin/timeout_ssh'))

        base_ssh_executable = base_ssh_executable or 'ssh'
        ssh_timeout = ssh_timeout or settings.QPC_SSH_INSPECT_TIMEOUT
        # pylint: disable=line-too-long
        # the ssh arg is required for become-pass because
        # ansible checks for an exact string match of ssh
        # anywhere in the command array
        # See https://github.com/ansible/ansible/blob/stable-2.3/lib/ansible/plugins/connection/ssh.py#L490-L500 # noqa
        # timeout_ssh will remove the ssh argument before running the command
        ssh_args = [
            '--executable=' + base_ssh_executable, '--timeout=' + ssh_timeout,
            'ssh'
        ]

        group_names, inventory = _construct_scan_inventory(
            connected,
            connection_port,
            forks,
            ssh_executable=ssh_executable,
            ssh_args=ssh_args)
        inventory_file = write_to_yaml(inventory)

        error_msg = None
        log_message = 'START INSPECT PROCESSING GROUPS'\
            ' with use_paramiko: %s, '\
            '%d forks and extra_vars=%s' % (use_paramiko,
                                            forks,
                                            extra_vars)
        self.scan_task.log_message(log_message)
        scan_result = ScanTask.COMPLETED

        # Build Ansible Runner Dependencies
        for idx, group_name in enumerate(group_names):
            check_manager_interrupt(manager_interrupt.value)
            log_message = 'START INSPECT PROCESSING GROUP %d of %d' % (
                (idx + 1), len(group_names))
            self.scan_task.log_message(log_message)
            call = InspectResultCallback(self.scan_task, manager_interrupt)

            # Build Ansible Runner Parameters
            runner_settings = {
                'idle_timeout': int(settings.NETWORK_INSPECT_JOB_TIMEOUT),
                'job_timeout': int(settings.NETWORK_INSPECT_JOB_TIMEOUT),
                'pexpect_timeout': 5
            }
            playbook_path = os.path.join(settings.BASE_DIR,
                                         'scanner/network/runner/inspect.yml')
            extra_vars['variable_host'] = group_name
            cmdline_list = []
            vault_file_path = '--vault-password-file=%s' % (
                settings.DJANGO_SECRET_PATH)
            cmdline_list.append(vault_file_path)
            forks_cmd = '--forks=%s' % (forks)
            cmdline_list.append(forks_cmd)
            if use_paramiko:
                cmdline_list.append('--connection=paramiko')
            all_commands = ' '.join(cmdline_list)

            if int(settings.ANSIBLE_LOG_LEVEL) == 0:
                quiet_bool = True
                verbosity_lvl = 0
            else:
                quiet_bool = False
                verbosity_lvl = int(settings.ANSIBLE_LOG_LEVEL)

            try:
                runner_obj = ansible_runner.run(
                    quiet=quiet_bool,
                    settings=runner_settings,
                    inventory=inventory_file,
                    extravars=extra_vars,
                    event_handler=call.event_callback,
                    cancel_callback=call.cancel_callback,
                    playbook=playbook_path,
                    cmdline=all_commands,
                    verbosity=verbosity_lvl)
            except Exception as error:
                error_msg = error
                raise AnsibleRunnerException(error_msg)

            final_status = runner_obj.status
            if final_status != 'successful':
                if final_status == 'canceled':
                    interrupt = manager_interrupt.value
                    if interrupt == ScanJob.JOB_TERMINATE_CANCEL:
                        msg = log_messages.NETWORK_PLAYBOOK_STOPPED % (
                            'INSPECT', 'canceled')
                    else:
                        msg = log_messages.NETWORK_PLAYBOOK_STOPPED % (
                            'INSPECT', 'paused')
                    self.scan_task.log_message(msg)
                    check_manager_interrupt(interrupt)
                if final_status not in ['unreachable', 'failed']:
                    if final_status == 'timeout':
                        error_msg = log_messages.NETWORK_TIMEOUT_ERR
                    else:
                        error_msg = log_messages.NETWORK_UNKNOWN_ERR
                    scan_result = ScanTask.FAILED
                call.finalize_failed_hosts()
        return error_msg, scan_result
    def setUp(self):
        """Create test case setup."""
        self.cred = Credential(
            name='cred1',
            username='******',
            password='******',
            ssh_keyfile='keyfile',
            become_method='sudo',
            become_user='******',
            become_password='******')
        self.cred.save()

        # Source with excluded hosts
        self.source = Source(
            name='source1',
            hosts='["1.2.3.4", "1.2.3.5"]',
            exclude_hosts='["1.2.3.5", "1.2.3.6"]',
            source_type='network',
            port=22)
        self.source.save()
        self.source.credentials.add(self.cred)
        self.source.save()

        self.scan_job, self.scan_task = create_scan_job(
            self.source, ScanTask.SCAN_TYPE_CONNECT)

        self.scan_task.update_stats('TEST NETWORK CONNECT.', sys_failed=0)

        # Source without excluded hosts
        self.source2 = Source(
            name='source2',
            hosts='["1.2.3.4"]',
            source_type='network',
            port=22)
        self.source2.save()
        self.source2.credentials.add(self.cred)
        self.source2.save()

        self.scan_job2, self.scan_task2 = create_scan_job(
            self.source2, ScanTask.SCAN_TYPE_CONNECT, 'source2',)

        self.scan_task2.update_stats('TEST NETWORK CONNECT.', sys_failed=0)

        # Scans with options & no excluded hosts
        source_options = SourceOptions(use_paramiko=True)
        source_options.save()
        self.source3 = Source(
            name='source3',
            hosts='["1.2.3.4","1.2.3.5","1.2.3.6"]',
            source_type='network',
            port=22,
            options=source_options)
        self.source3.save()
        self.source3.credentials.add(self.cred)
        self.source3.save()

        scan_options = ScanOptions(max_concurrency=2)
        scan_options.save()

        self.scan_job3, self.scan_task3 = create_scan_job(
            self.source3, ScanTask.SCAN_TYPE_CONNECT, 'source3', scan_options)
        self.scan_task3.update_stats('TEST NETWORK CONNECT.', sys_failed=0)
        self.concurrency = ScanOptions.get_default_forks()