def test_create(self):
name = 's'
request = HttpRequest()
request.user = self.user
serializer = SourceSerializer(data={'name': name},
context={'request': request})
self.assertTrue(serializer.is_valid())
source = serializer.save()
# Ensure the database matches what we saved
self.assertEqual(Source.objects.get(pk=source.pk), source)
self.assertEqual(Source.objects.get(pk=source.pk).name, name)
def test_format_source(self):
"""Test the format source method."""
start = datetime.now()
source = Source(
name='source1',
hosts=json.dumps(['1.2.3.4']),
source_type='network',
port=22)
source.save()
end = datetime.now()
scan_job, scan_task = create_scan_job(source)
scan_task.update_stats('', sys_count=10, sys_scanned=9, sys_failed=1)
scan_job.start_time = start
scan_job.end_time = end
scan_job.status = ScanTask.COMPLETED
scan_job.save()
serializer = SourceSerializer(source)
json_source = serializer.data
out = format_source(json_source)
expected = {'id': 1,
'name': 'source1',
'source_type': 'network',
'port': 22,
'hosts': ['1.2.3.4'],
'connection': {'id': 1, 'start_time': start,
'end_time': end, 'status': 'completed',
'systems_count': 10,
'systems_scanned': 9,
'systems_failed': 1}}
self.assertEqual(out, expected)
def test_connect_inventory(self):
"""Test construct ansible inventory dictionary."""
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
hc_serializer = CredentialSerializer(self.cred)
cred = hc_serializer.data
inventory_dict = construct_connect_inventory(hosts, cred,
connection_port)
expected = {
'all': {
'hosts': {
'1.2.3.4': None
},
'vars': {
'ansible_become_pass': '******',
'ansible_port': 22,
'ansible_ssh_pass': '******',
'ansible_ssh_private_key_file': 'keyfile',
'ansible_user': '******',
'ansible_become_method': 'sudo',
'ansible_become_user': '******'
}
}
}
self.assertEqual(inventory_dict, expected)
def test_format_source(self):
"""Test the format source method."""
start = datetime.now()
source = Source(
name='source1',
hosts=json.dumps(['1.2.3.4']),
source_type='network',
port=22)
source.save()
end = datetime.now()
scan_job, scan_task = create_scan_job(source)
scan_task.update_stats(
'', sys_count=10, sys_scanned=9, sys_failed=1, sys_unreachable=0)
scan_job.start_time = start
scan_job.end_time = end
scan_job.status = ScanTask.COMPLETED
scan_job.save()
source.most_recent_connect_scan = scan_job
source.save()
serializer = SourceSerializer(source)
json_source = serializer.data
out = format_source(json_source)
# pylint: disable=line-too-long
expected = {'id': 1, 'name': 'source1', 'source_type': 'network', 'port': 22, 'hosts': ['1.2.3.4'], 'connection': {'id': 1, 'start_time': start, 'end_time': end, 'systems_count': 10, 'systems_scanned': 9, 'systems_failed': 1, 'systems_unreachable': 0, 'status_details': {'job_status_message': 'Job is pending.'}, 'status': 'completed', 'source_systems_count': 10, 'source_systems_scanned': 9, 'source_systems_failed': 1, 'source_systems_unreachable': 0}} # noqa
self.assertEqual(out, expected)
def test_scan_inventory(self):
"""Test construct ansible inventory dictionary."""
serializer = SourceSerializer(self.source)
source = serializer.data
connection_port = source['port']
hc_serializer = CredentialSerializer(self.cred)
cred = hc_serializer.data
inventory_dict = construct_scan_inventory([('1.2.3.4', cred)],
connection_port, 50)
expected = {
'all': {
'children': {
'group_0': {
'hosts': {
'1.2.3.4': {
'ansible_user': '******',
'ansible_ssh_pass': '******',
'ansible_host': '1.2.3.4'
}
}
}
},
'vars': {
'ansible_port': 22
}
}
}
self.assertEqual(inventory_dict[1], expected)
def run_with_result_store(self, manager_interrupt, result_store):
"""Run with a given ConnectResultStore."""
serializer = SourceSerializer(self.scan_task.source)
source = serializer.data
if self.scan_job.options is not None:
forks = self.scan_job.options.max_concurrency
else:
forks = ScanOptions.get_default_forks()
if self.scan_task.source.options is not None:
use_paramiko = self.scan_task.source.options.use_paramiko
else:
use_paramiko = False
connection_port = source['port']
credentials = source['credentials']
remaining_hosts = result_store.remaining_hosts()
for cred_id in credentials:
check_manager_interrupt(manager_interrupt.value)
credential = Credential.objects.get(pk=cred_id)
if not remaining_hosts:
message = 'Skipping credential %s. No remaining hosts.' % \
credential.name
self.scan_task.log_message(message)
break
message = 'Attempting credential %s.' % credential.name
self.scan_task.log_message(message)
try:
scan_message, scan_result = _connect(manager_interrupt,
self.scan_task,
remaining_hosts,
result_store, credential,
connection_port, forks,
use_paramiko)
if scan_result != ScanTask.COMPLETED:
return scan_message, scan_result
except AnsibleRunnerException as ansible_error:
remaining_hosts_str = ', '.join(result_store.remaining_hosts())
error_message = 'Connect scan task failed with credential %s.'\
' Error: %s Hosts: %s' %\
(credential.name, ansible_error, remaining_hosts_str)
return error_message, ScanTask.FAILED
remaining_hosts = result_store.remaining_hosts()
logger.debug('Failed systems: %s', remaining_hosts)
for host in remaining_hosts:
# We haven't connected to these hosts with any
# credentials, so they have failed.
result_store.record_result(host, self.scan_task.source, None,
SystemConnectionResult.FAILED)
return None, ScanTask.COMPLETED
def test_connect_src2(self, mock_run):
"""Test connect flow with mocked manager."""
serializer = SourceSerializer(self.source2)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
_connect(Value('i', ScanJob.JOB_RUN), self.scan_task, hosts, Mock(),
self.cred, connection_port)
mock_run.assert_called_with(ANY)
def retrieve(self, request, pk=None): # pylint: disable=unused-argument
"""Get a source."""
source = get_object_or_404(self.queryset, pk=pk)
serializer = SourceSerializer(source)
json_source = serializer.data
# Create expanded host cred JSON
expand_credential(source, json_source)
return Response(json_source)
def test_connect(self, mock_run):
"""Test connect flow with mocked manager."""
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
hc_serializer = CredentialSerializer(self.cred)
cred = hc_serializer.data
connect(hosts, Mock(), cred, connection_port)
mock_run.assert_called_with(ANY)
def test_connect_paramiko(self, mock_run):
"""Test connect with paramiko."""
mock_run.return_value.status = 'successful'
serializer = SourceSerializer(self.source3)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
_connect(Value('i', ScanJob.JOB_RUN),
self.scan_task, hosts, Mock(), self.cred,
connection_port, self.concurrency)
mock_run.assert_called()
def test_connect_failure_src2(self, mock_run, mock_ssh_pass):
"""Test connect flow with mocked manager and failure."""
serializer = SourceSerializer(self.source2)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
with self.assertRaises(AnsibleError):
_connect(Value('i', ScanJob.JOB_RUN), self.scan_task, hosts,
Mock(), self.cred, connection_port)
mock_run.assert_called()
mock_ssh_pass.assert_called()
def test_connect_runner_error(self, mock_run):
"""Test connect flow with mocked manager."""
mock_run.side_effect = AnsibleRunnerException('Fail')
serializer = SourceSerializer(self.source2)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
with self.assertRaises(AnsibleRunnerException):
_connect(Value('i', ScanJob.JOB_RUN), self.scan_task,
hosts, Mock(), self.cred, connection_port,
self.concurrency)
mock_run.assert_called()
def test_connect_failure(self, mock_run, mock_ssh_pass):
"""Test connect flow with mocked manager and failure."""
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
hc_serializer = CredentialSerializer(self.cred)
cred = hc_serializer.data
with self.assertRaises(AnsibleError):
connect(hosts, Mock(), cred, connection_port)
mock_run.assert_called()
mock_ssh_pass.assert_called()
def test_secret_file_fail(self, mock_run):
"""Test modifying the log level."""
mock_run.side_effect = AnsibleRunnerException()
serializer = SourceSerializer(self.source2)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
with self.assertRaises(AnsibleRunnerException):
_connect(Value('i', ScanJob.JOB_RUN),
self.scan_task, hosts, Mock(), self.cred,
connection_port, self.concurrency)
mock_run.assert_called()
def run_with_result_store(self, result_store):
"""Run with a given ConnectResultStore."""
serializer = SourceSerializer(self.scan_task.source)
source = serializer.data
if self.scan_job.options is None:
forks = ScanOptions.get_default_forks()
else:
forks = self.scan_job.options.max_concurrency
connection_port = source['port']
credentials = source['credentials']
remaining_hosts = result_store.remaining_hosts()
for cred_id in credentials:
credential = Credential.objects.get(pk=cred_id)
if not remaining_hosts:
message = 'Skipping credential %s. No remaining hosts.' % \
credential.name
self.scan_task.log_message(message)
break
message = 'Attempting credential %s.' % credential.name
self.scan_task.log_message(message)
cred_data = CredentialSerializer(credential).data
callback = ConnectResultCallback(result_store, credential,
self.scan_task.source)
try:
connect(remaining_hosts,
callback,
cred_data,
connection_port,
forks=forks)
except AnsibleError as ansible_error:
remaining_hosts_str = ', '.join(result_store.remaining_hosts())
error_message = 'Connect scan task failed with credential %s.'\
' Error: %s Hosts: %s' %\
(credential.name, ansible_error, remaining_hosts_str)
return error_message, ScanTask.FAILED
remaining_hosts = result_store.remaining_hosts()
logger.debug('Failed systems: %s', remaining_hosts)
for host in remaining_hosts:
# We haven't connected to these hosts with any
# credentials, so they have failed.
result_store.record_result(host, self.scan_task.source, None,
SystemConnectionResult.FAILED)
return None, ScanTask.COMPLETED
def test_connect(self, mock_run):
"""Test connect flow with mocked manager."""
mock_run.return_value.status = 'successful'
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
exclude_hosts = source['exclude_hosts']
connection_port = source['port']
_connect(Value('i', ScanJob.JOB_RUN),
self.scan_task, hosts, Mock(), self.cred,
connection_port, self.concurrency,
exclude_hosts)
mock_run.assert_called()
def test_modifying_log_level(self, mock_run):
"""Test modifying the log level."""
mock_run.return_value.status = 'successful'
serializer = SourceSerializer(self.source2)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
_connect(Value('i', ScanJob.JOB_RUN),
self.scan_task, hosts, Mock(), self.cred,
connection_port, self.concurrency)
mock_run.assert_called()
calls = mock_run.mock_calls
# Check to see if the parameter was passed into the runner.run()
self.assertIn('verbosity=1', str(calls[0]))
def retrieve(self, request, pk=None): # pylint: disable=unused-argument
"""Get a source."""
if not pk or (pk and not is_int(pk)):
error = {'id': [_(messages.COMMON_ID_INV)]}
raise ValidationError(error)
source = get_object_or_404(self.queryset, pk=pk)
serializer = SourceSerializer(source)
json_source = serializer.data
# Create expanded host cred JSON
format_source(json_source)
return Response(json_source)
def test_connect_inventory(self):
"""Test construct ansible inventory dictionary."""
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
exclude_hosts = source['exclude_hosts']
connection_port = source['port']
hc_serializer = CredentialSerializer(self.cred)
cred = hc_serializer.data
path = '/path/to/executable.py'
ssh_timeout = '0.1s'
ssh_args = ['--executable=' + path, '--timeout=' + ssh_timeout, 'ssh']
_, inventory_dict = _construct_connect_inventory(hosts,
cred,
connection_port,
1,
exclude_hosts,
ssh_executable=path,
ssh_args=ssh_args)
# pylint: disable=line-too-long
expected = {
'all': {
'children': {
'group_0': {
'hosts': {
'1.2.3.4': {
'ansible_host':
'1.2.3.4',
'ansible_ssh_executable':
'/path/to/executable.py',
'ansible_ssh_common_args':
'--executable=/path/to/executable.py --timeout=0.1s ssh'
}
}
}
}, # noqa
'vars': {
'ansible_port': 22,
'ansible_user': '******',
'ansible_ssh_pass': '******',
'ansible_ssh_private_key_file': 'keyfile',
'ansible_become_pass': '******',
'ansible_become_method': 'sudo',
'ansible_become_user': '******'
}
}
}
self.assertEqual(inventory_dict, expected)
def list(self, request): # pylint: disable=unused-argument
"""List the sources."""
# List objects
queryset = self.filter_queryset(self.get_queryset())
# For each source, expand creds
result = []
for source in queryset:
serializer = SourceSerializer(source)
json_source = serializer.data
# Create expanded host cred JSON
expand_credential(source, json_source)
result.append(json_source)
return Response(result)
def test_connect_ssh_crash(self, mock_run):
"""Simulate an ssh crash."""
mock_run.return_value.status = 'successful'
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
exclude_hosts = source['exclude_hosts']
connection_port = source['port']
path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
'test_util/crash.py'))
_connect(Value('i', ScanJob.JOB_RUN), self.scan_task,
hosts, Mock(), self.cred, connection_port,
self.concurrency, exclude_hosts,
base_ssh_executable=path)
mock_run.assert_called()
def test_pause_connect(self, mock_run):
"""Test pause of connect."""
# Test cancel at _connect level
serializer = SourceSerializer(self.source3)
source = serializer.data
hosts = source['hosts']
connection_port = source['port']
with self.assertRaises(NetworkPauseException):
_connect(Value('i', ScanJob.JOB_TERMINATE_PAUSE),
self.scan_task, hosts, Mock(), self.cred,
connection_port, self.concurrency)
# Test cancel at run() level
mock_run.side_effect = NetworkPauseException()
scanner = ConnectTaskRunner(self.scan_job3, self.scan_task3)
_, scan_result = scanner.run(Value('i', ScanJob.JOB_RUN))
self.assertEqual(scan_result, ScanTask.PAUSED)
def test_connect_ssh_hang(self, mock_run):
"""Simulate an ssh hang."""
serializer = SourceSerializer(self.source)
source = serializer.data
hosts = source['hosts']
exclude_hosts = source['exclude_hosts']
connection_port = source['port']
path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
'../../../test_util/hang.py'))
_connect(Value('i', ScanJob.JOB_RUN),
self.scan_task,
hosts,
Mock(),
self.cred,
connection_port,
exclude_hosts,
base_ssh_executable=path,
ssh_timeout='0.1s')
mock_run.assert_called_with(ANY)
def discovery(self):
"""Execute the discovery scan with the initialized source.
:returns: list of connected hosts credential tuples and
list of host that failed connection
"""
connected = []
serializer = SourceSerializer(self.scan_task.source)
source = serializer.data
remaining = source['hosts']
credentials = source['credentials']
connection_port = source['port']
forks = self.scan_job.options.max_concurrency
for cred_id in credentials:
cred_obj = Credential.objects.get(pk=cred_id)
hc_serializer = CredentialSerializer(cred_obj)
cred = hc_serializer.data
connected, remaining = connect(remaining,
cred,
connection_port,
forks=forks)
# Update the scan counts
if self.scan_task.systems_count is None:
self.scan_task.systems_count = len(connected) + len(remaining)
self.scan_task.systems_scanned = 0
self.scan_task.systems_failed = 0
self.scan_task.systems_scanned += len(connected)
self.scan_task.save()
if remaining == []:
break
logger.info('Connect scan completed for %s.', self.scan_task)
logger.info('Successfully connected to %d systems.', len(connected))
if bool(remaining):
logger.warning('Failed to connect to %d systems.', len(remaining))
logger.debug('Failed systems: %s', remaining)
return connected, remaining
def expand_inspect_results(job_inspect_result, json_job_inspect_result):
"""Expand the inspection results.
Take collection of json_job_inspect_result ids and pull objects from db.
create slim dictionary version of rows of key/value to return to user.
"""
if json_job_inspect_result is not None and\
json_job_inspect_result.get(RESULTS_KEY):
json_job_inspect_result_list = []
for result in job_inspect_result.results.all():
source_serializer = SourceSerializer(result.source)
json_source = source_serializer.data
json_source.pop('credentials', None)
json_source.pop('hosts', None)
json_source.pop('port', None)
systems = expand_sys_inspect_results(result)
json_job_inspect_result_out = {
'source': json_source,
'systems': systems
}
json_job_inspect_result_list.append(json_job_inspect_result_out)
json_job_inspect_result[RESULTS_KEY] = json_job_inspect_result_list
def list(self, request): # pylint: disable=unused-argument
"""List the sources."""
# List objects
result = []
queryset = self.filter_queryset(self.get_queryset())
page = self.paginate_queryset(queryset)
if page is not None:
serializer = self.get_serializer(page, many=True)
for source in serializer.data:
# Create expanded host cred JSON
format_source(source)
result.append(source)
return self.get_paginated_response(serializer.data)
for source in queryset:
serializer = SourceSerializer(source)
json_source = serializer.data
# Create expanded host cred JSON
format_source(json_source)
result.append(json_source)
return Response(result)
def get(self, request, pk, format=None):
#print(pk)
source = self.get_object(pk + " ")
ser = SourceSerializer(source)
return Response(ser.data)
