Пример #1
0
    def _result_stdout_raw_limited(self,
                                   start_line=0,
                                   end_line=None,
                                   redact_sensitive=True,
                                   escape_ascii=False):
        return_buffer = u""
        if end_line is not None:
            end_line = int(end_line)
        stdout_lines = self.result_stdout_raw_handle().readlines()
        absolute_end = len(stdout_lines)
        for line in stdout_lines[int(start_line):end_line]:
            return_buffer += line
        if int(start_line) < 0:
            start_actual = len(stdout_lines) + int(start_line)
            end_actual = len(stdout_lines)
        else:
            start_actual = int(start_line)
            if end_line is not None:
                end_actual = min(int(end_line), len(stdout_lines))
            else:
                end_actual = len(stdout_lines)

        if redact_sensitive:
            return_buffer = UriCleaner.remove_sensitive(return_buffer)
        if escape_ascii:
            return_buffer = self._escape_ascii(return_buffer)

        return return_buffer, start_actual, end_actual, absolute_end
Пример #2
0
def test_uri_scm_cleartext_redact_and_replace(test_data):
    uri = test_data['uri']
    redacted_str = UriCleaner.remove_sensitive(test_data['text'])
    assert uri.username not in redacted_str
    assert uri.password not in redacted_str
    # Ensure the host didn't get redacted
    assert redacted_str.count(uri.host) == test_data['host_occurrences']
Пример #3
0
 def _result_stdout_raw(self, redact_sensitive=False, escape_ascii=False):
     content = self.result_stdout_raw_handle().read()
     if redact_sensitive:
         content = UriCleaner.remove_sensitive(content)
     if escape_ascii:
         content = self._escape_ascii(content)
     return content
Пример #4
0
def test_uri_scm_simple_redacted():
    for uri in TEST_URIS:
        redacted_str = UriCleaner.remove_sensitive(str(uri))
        if uri.username:
            assert uri.username not in redacted_str
        if uri.password:
            assert uri.username not in redacted_str
Пример #5
0
def test_multiple_non_uri_redact():
    non_uri = 'https://www.famfamfam.com](http://www.famfamfam.com/fijdlfd hi '
    non_uri += 'https://www.famfamfam.com](http://www.famfamfam.com/fijdlfd world '
    non_uri += 'https://www.famfamfam.com](http://www.famfamfam.com/fijdlfd foo '
    non_uri += 'https://*****:*****@giggity.com bar'
    redacted_str = UriCleaner.remove_sensitive(non_uri)
    assert redacted_str == '$encrypted$ hi $encrypted$ world $encrypted$ foo https://$encrypted$:[email protected] bar'
Пример #6
0
def test_non_uri_redact(username, password, not_uri, expected):
    redacted_str = UriCleaner.remove_sensitive(not_uri)
    if username:
        assert username not in redacted_str
    if password:
        assert password not in redacted_str

    assert redacted_str == expected
Пример #7
0
def test_uri_scm_multiple(uri):
    cleartext = ''
    cleartext += str(uri) + ' '
    cleartext += str(uri) + '\n'

    redacted_str = UriCleaner.remove_sensitive(str(uri))
    if uri.username:
        assert uri.username not in redacted_str
    if uri.password:
        assert uri.password not in redacted_str
Пример #8
0
def test_uri_scm_multiple_replaced(uri):
    cleartext = ''
    find_count = 0

    cleartext += str(uri) + ' '
    find_count += uri.get_secret_count()

    cleartext += str(uri) + '\n'
    find_count += uri.get_secret_count()

    redacted_str = UriCleaner.remove_sensitive(cleartext)
    assert redacted_str.count(UriCleaner.REPLACE_STR) == find_count
Пример #9
0
    def event_handler(self, event_data):
        #
        # ⚠️  D-D-D-DANGER ZONE ⚠️
        # This method is called once for *every event* emitted by Ansible
        # Runner as a playbook runs.  That means that changes to the code in
        # this method are _very_ likely to introduce performance regressions.
        #
        # Even if this function is made on average .05s slower, it can have
        # devastating performance implications for playbooks that emit
        # tens or hundreds of thousands of events.
        #
        # Proceed with caution!
        #
        """
        Ansible runner puts a parent_uuid on each event, no matter what the type.
        AWX only saves the parent_uuid if the event is for a Job.
        """
        # cache end_line locally for RunInventoryUpdate tasks
        # which generate job events from two 'streams':
        # ansible-inventory and the awx.main.commands.inventory_import
        # logger

        if event_data.get(self.event_data_key, None):
            if self.event_data_key != 'job_id':
                event_data.pop('parent_uuid', None)
        if self.parent_workflow_job_id:
            event_data['workflow_job_id'] = self.parent_workflow_job_id
        event_data['job_created'] = self.job_created
        if self.host_map:
            host = event_data.get('event_data', {}).get('host', '').strip()
            if host:
                event_data['host_name'] = host
                if host in self.host_map:
                    event_data['host_id'] = self.host_map[host]
            else:
                event_data['host_name'] = ''
                event_data['host_id'] = ''
            if event_data.get('event') == 'playbook_on_stats':
                event_data['host_map'] = self.host_map

        if isinstance(self, RunnerCallbackForProjectUpdate):
            # need a better way to have this check.
            # it's common for Ansible's SCM modules to print
            # error messages on failure that contain the plaintext
            # basic auth credentials (username + password)
            # it's also common for the nested event data itself (['res']['...'])
            # to contain unredacted text on failure
            # this is a _little_ expensive to filter
            # with regex, but project updates don't have many events,
            # so it *should* have a negligible performance impact
            task = event_data.get('event_data', {}).get('task_action')
            try:
                if task in ('git', 'svn'):
                    event_data_json = json.dumps(event_data)
                    event_data_json = UriCleaner.remove_sensitive(
                        event_data_json)
                    event_data = json.loads(event_data_json)
            except json.JSONDecodeError:
                pass

        if 'event_data' in event_data:
            event_data['event_data']['guid'] = self.guid

        # To prevent overwhelming the broadcast queue, skip some websocket messages
        if self.recent_event_timings:
            cpu_time = time.time()
            first_window_time = self.recent_event_timings[0]
            last_window_time = self.recent_event_timings[-1]

            if event_data.get('event') in MINIMAL_EVENTS:
                should_emit = True  # always send some types like playbook_on_stats
            elif event_data.get('stdout') == '' and event_data[
                    'start_line'] == event_data['end_line']:
                should_emit = False  # exclude events with no output
            else:
                should_emit = any([
                    # if 30the most recent websocket message was sent over 1 second ago
                    cpu_time - first_window_time > 1.0,
                    # if the very last websocket message came in over 1/30 seconds ago
                    self.recent_event_timings.maxlen *
                    (cpu_time - last_window_time) > 1.0,
                    # if the queue is not yet full
                    len(self.recent_event_timings
                        ) != self.recent_event_timings.maxlen,
                ])

            if should_emit:
                self.recent_event_timings.append(cpu_time)
            else:
                event_data.setdefault('event_data', {})
                event_data['skip_websocket_message'] = True

        elif self.recent_event_timings.maxlen:
            self.recent_event_timings.append(time.time())

        event_data.setdefault(self.event_data_key, self.instance.id)
        self.dispatcher.dispatch(event_data)
        self.event_ct += 1
        '''
        Handle artifacts
        '''
        if event_data.get('event_data', {}).get('artifact_data', {}):
            self.instance.artifacts = event_data['event_data']['artifact_data']
            self.instance.save(update_fields=['artifacts'])

        return False
Пример #10
0
def test_uri_scm_simple_replaced():
    for uri in TEST_URIS:
        redacted_str = UriCleaner.remove_sensitive(str(uri))
        assert redacted_str.count(
            UriCleaner.REPLACE_STR) == uri.get_secret_count()
Пример #11
0
def test_large_string_performance():
    length = 100000
    redacted = UriCleaner.remove_sensitive('x' * length)
    assert len(redacted) == length