def _result_stdout_raw_limited(self,
start_line=0,
end_line=None,
redact_sensitive=True,
escape_ascii=False):
return_buffer = u""
if end_line is not None:
end_line = int(end_line)
stdout_lines = self.result_stdout_raw_handle().readlines()
absolute_end = len(stdout_lines)
for line in stdout_lines[int(start_line):end_line]:
return_buffer += line
if int(start_line) < 0:
start_actual = len(stdout_lines) + int(start_line)
end_actual = len(stdout_lines)
else:
start_actual = int(start_line)
if end_line is not None:
end_actual = min(int(end_line), len(stdout_lines))
else:
end_actual = len(stdout_lines)
if redact_sensitive:
return_buffer = UriCleaner.remove_sensitive(return_buffer)
if escape_ascii:
return_buffer = self._escape_ascii(return_buffer)
return return_buffer, start_actual, end_actual, absolute_end
def test_uri_scm_cleartext_redact_and_replace(test_data):
uri = test_data['uri']
redacted_str = UriCleaner.remove_sensitive(test_data['text'])
assert uri.username not in redacted_str
assert uri.password not in redacted_str
# Ensure the host didn't get redacted
assert redacted_str.count(uri.host) == test_data['host_occurrences']
def _result_stdout_raw(self, redact_sensitive=False, escape_ascii=False):
content = self.result_stdout_raw_handle().read()
if redact_sensitive:
content = UriCleaner.remove_sensitive(content)
if escape_ascii:
content = self._escape_ascii(content)
return content
def test_uri_scm_simple_redacted():
for uri in TEST_URIS:
redacted_str = UriCleaner.remove_sensitive(str(uri))
if uri.username:
assert uri.username not in redacted_str
if uri.password:
assert uri.username not in redacted_str
def test_multiple_non_uri_redact():
non_uri = 'https://www.famfamfam.com](http://www.famfamfam.com/fijdlfd hi '
non_uri += 'https://www.famfamfam.com](http://www.famfamfam.com/fijdlfd world '
non_uri += 'https://www.famfamfam.com](http://www.famfamfam.com/fijdlfd foo '
non_uri += 'https://*****:*****@giggity.com bar'
redacted_str = UriCleaner.remove_sensitive(non_uri)
assert redacted_str == '$encrypted$ hi $encrypted$ world $encrypted$ foo https://$encrypted$:[email protected] bar'
def test_non_uri_redact(username, password, not_uri, expected):
redacted_str = UriCleaner.remove_sensitive(not_uri)
if username:
assert username not in redacted_str
if password:
assert password not in redacted_str
assert redacted_str == expected
def test_uri_scm_multiple(uri):
cleartext = ''
cleartext += str(uri) + ' '
cleartext += str(uri) + '\n'
redacted_str = UriCleaner.remove_sensitive(str(uri))
if uri.username:
assert uri.username not in redacted_str
if uri.password:
assert uri.password not in redacted_str
def test_uri_scm_multiple_replaced(uri):
cleartext = ''
find_count = 0
cleartext += str(uri) + ' '
find_count += uri.get_secret_count()
cleartext += str(uri) + '\n'
find_count += uri.get_secret_count()
redacted_str = UriCleaner.remove_sensitive(cleartext)
assert redacted_str.count(UriCleaner.REPLACE_STR) == find_count
def event_handler(self, event_data):
#
# ⚠️ D-D-D-DANGER ZONE ⚠️
# This method is called once for *every event* emitted by Ansible
# Runner as a playbook runs. That means that changes to the code in
# this method are _very_ likely to introduce performance regressions.
#
# Even if this function is made on average .05s slower, it can have
# devastating performance implications for playbooks that emit
# tens or hundreds of thousands of events.
#
# Proceed with caution!
#
"""
Ansible runner puts a parent_uuid on each event, no matter what the type.
AWX only saves the parent_uuid if the event is for a Job.
"""
# cache end_line locally for RunInventoryUpdate tasks
# which generate job events from two 'streams':
# ansible-inventory and the awx.main.commands.inventory_import
# logger
if event_data.get(self.event_data_key, None):
if self.event_data_key != 'job_id':
event_data.pop('parent_uuid', None)
if self.parent_workflow_job_id:
event_data['workflow_job_id'] = self.parent_workflow_job_id
event_data['job_created'] = self.job_created
if self.host_map:
host = event_data.get('event_data', {}).get('host', '').strip()
if host:
event_data['host_name'] = host
if host in self.host_map:
event_data['host_id'] = self.host_map[host]
else:
event_data['host_name'] = ''
event_data['host_id'] = ''
if event_data.get('event') == 'playbook_on_stats':
event_data['host_map'] = self.host_map
if isinstance(self, RunnerCallbackForProjectUpdate):
# need a better way to have this check.
# it's common for Ansible's SCM modules to print
# error messages on failure that contain the plaintext
# basic auth credentials (username + password)
# it's also common for the nested event data itself (['res']['...'])
# to contain unredacted text on failure
# this is a _little_ expensive to filter
# with regex, but project updates don't have many events,
# so it *should* have a negligible performance impact
task = event_data.get('event_data', {}).get('task_action')
try:
if task in ('git', 'svn'):
event_data_json = json.dumps(event_data)
event_data_json = UriCleaner.remove_sensitive(
event_data_json)
event_data = json.loads(event_data_json)
except json.JSONDecodeError:
pass
if 'event_data' in event_data:
event_data['event_data']['guid'] = self.guid
# To prevent overwhelming the broadcast queue, skip some websocket messages
if self.recent_event_timings:
cpu_time = time.time()
first_window_time = self.recent_event_timings[0]
last_window_time = self.recent_event_timings[-1]
if event_data.get('event') in MINIMAL_EVENTS:
should_emit = True # always send some types like playbook_on_stats
elif event_data.get('stdout') == '' and event_data[
'start_line'] == event_data['end_line']:
should_emit = False # exclude events with no output
else:
should_emit = any([
# if 30the most recent websocket message was sent over 1 second ago
cpu_time - first_window_time > 1.0,
# if the very last websocket message came in over 1/30 seconds ago
self.recent_event_timings.maxlen *
(cpu_time - last_window_time) > 1.0,
# if the queue is not yet full
len(self.recent_event_timings
) != self.recent_event_timings.maxlen,
])
if should_emit:
self.recent_event_timings.append(cpu_time)
else:
event_data.setdefault('event_data', {})
event_data['skip_websocket_message'] = True
elif self.recent_event_timings.maxlen:
self.recent_event_timings.append(time.time())
event_data.setdefault(self.event_data_key, self.instance.id)
self.dispatcher.dispatch(event_data)
self.event_ct += 1
'''
Handle artifacts
'''
if event_data.get('event_data', {}).get('artifact_data', {}):
self.instance.artifacts = event_data['event_data']['artifact_data']
self.instance.save(update_fields=['artifacts'])
return False
def test_uri_scm_simple_replaced():
for uri in TEST_URIS:
redacted_str = UriCleaner.remove_sensitive(str(uri))
assert redacted_str.count(
UriCleaner.REPLACE_STR) == uri.get_secret_count()
def test_large_string_performance():
length = 100000
redacted = UriCleaner.remove_sensitive('x' * length)
assert len(redacted) == length