def test_node_comparison_eq(self):
node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.node.name = 'test'
node.name = 'test'
self.assertEqual(self.node, node)
async def test_get_nodes_for_scanning(self, cfg):
cfg._cfg = self.cfg
cfg['portdetection.test_name.networks.include'] = [
'127.0.0.2/31', '127.0.0.4/32'
]
node_1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
node_2 = Node(ip=ipaddress.ip_address('127.0.0.2'), node_id=2)
node_3 = Node(ip=ipaddress.ip_address('127.0.0.3'), node_id=3)
node_4 = Node(ip=ipaddress.ip_address('127.0.0.4'), node_id=4)
nodes = {node_1, node_2, node_3}
future = Future()
future.set_result(nodes)
self.aucote.topdis.get_all_nodes.return_value = future
self.aucote.topdis.get_snmp_nodes.return_value = Future()
self.aucote.topdis.get_snmp_nodes.return_value.set_result({node_4})
self.thread.storage.get_nodes = MagicMock(return_value=[node_2])
scan = Scan()
result = await self.thread._get_nodes_for_scanning(scan)
expected = [node_4, node_3]
self.assertListEqual(result, expected)
def setUp(self):
"""
Prepare some internal variables:
exploit, port, exploits, script, vulnerability, scan_task
"""
super(NmapPortScanTaskTest, self).setUp()
self.aucote = MagicMock()
self.exploit = Exploit(exploit_id=1, app='nmap', name='test')
self.exploit_vuln_non_exist = Exploit(exploit_id=2,
app='nmap',
name='test2')
self.exploits = Exploits()
self.exploits.add(self.exploit)
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=None)
self.node_ipv6 = Node(ip=ipaddress.ip_address('::1'), node_id=None)
self.port = Port(number=22,
node=self.node,
transport_protocol=TransportProtocol.TCP)
self.port.service_name = 'ssh'
self.port.scan = Scan()
self.port_ipv6 = Port(number=22,
node=self.node_ipv6,
transport_protocol=TransportProtocol.TCP)
self.script = NmapScript(port=self.port,
parser=NmapParser(),
exploit=self.exploit,
name='test',
args='test_args')
self.script.get_result = MagicMock(return_value='test')
self.script2 = NmapScript(port=self.port,
parser=NmapInfoParser(),
exploit=self.exploit_vuln_non_exist,
name='test2')
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.scan_task = NmapPortScanTask(
context=self.context,
port=self.port,
script_classes=[self.script, self.script2],
rate=1337)
self.scan_task.store_scan_end = MagicMock()
future = Future()
future.set_result(ElementTree.fromstring(self.XML))
self.scan_task.command.async_call = MagicMock(return_value=future)
self.cfg = {'tools': {'nmap': {'scripts_dir': '', 'timeout': 0}}}
def setUp(self):
self.node = Node(node_id=13, ip=ipaddress.ip_address('127.0.0.5'))
self.scan_1 = Scan(start=150421)
self.scan_2 = Scan(start=159985)
self.port_1 = Port(transport_protocol=TransportProtocol.TCP,
number=80,
node=self.node)
self.port_1.scan = self.scan_1
self.port_scan_1 = PortScan(port=self.port_1, scan=self.scan_1)
self.port_2 = Port(transport_protocol=TransportProtocol.UDP,
number=19,
node=self.node)
self.port_2.scan = self.scan_2
self.port_scan_2 = PortScan(port=self.port_2, scan=self.scan_2)
self.type = VulnerabilityChangeType.PORTDETECTION
self.change_1 = PortDetectionChange(change_time=159986,
current_finding=self.port_scan_1,
previous_finding=None)
self.change_2 = PortDetectionChange(change_time=159911,
current_finding=None,
previous_finding=self.port_scan_2)
async def test_nodes(self, mock_task):
self.executor._ports = []
node_1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
node_2 = Node(ip=ipaddress.ip_address('127.0.0.2'), node_id=2)
node_3 = Node(ip=ipaddress.ip_address('127.0.0.3'), node_id=3)
node_4 = Node(ip=ipaddress.ip_address('127.0.0.3'), node_id=3)
self.executor.nodes = [node_1, node_2, node_3, node_4]
mock_task.return_value.assign_tasks_for_node = MagicMock(
return_value=Future())
mock_task.return_value.assign_tasks_for_node.return_value.set_result(
True)
self.executor.scan_only = False
await self.executor.run()
mock_task.return_value.assign_tasks_for_node.assert_has_calls(
(call(node_1), call(node_2), call(node_3)), any_order=True)
def read_input(filename):
"""
Reads the input file and parse it to internal structs
:param filename: Name of the input file
:return: Nodes for the page rank algorithm
"""
first_line = True
num_of_nodes = 0
counter = 0
nodes = []
with open(filename) as file:
for line in file:
if first_line:
num_of_nodes = int(line)
for i in range(num_of_nodes):
nodes.append(Node(str(i)))
first_line = False
continue
elif num_of_nodes == counter:
break
else:
neighbors = map(lambda x: int(x) - 1, line.split())
nodes[counter].add_neighbors(neighbors)
counter += 1
for node in nodes:
print(f"{node.name}, neig: {node.neighbors}\n")
return nodes
def pagerank(nodes, iter, dp):
"""
Execute Page Rank algorithm on a collection of nodes
:param nodes: list of interconnected nodes
:param iter: number of iterations
:param dp: damping factor
:return:
"""
n = nodes
curr = iter
r = 1.0 - dp
neighbors = n.map(lambda node: (str(node.name), node.neighbors))
while curr > 0:
print('ITERATION: ' + str(iter - curr))
contribs = n.flatMap(lambda node: list(
map(lambda ne:
(str(ne), node.rank / len(node.neighbors)), node.neighbors)))
new_ranks = contribs.reduceByKey(lambda a, b: a + b).mapValues(
lambda acc: r + dp * acc)
n = new_ranks.fullOuterJoin(neighbors).map(lambda node: Node(
int(node[0]), node[1][1], node[1][0] if node[1][0] else r))
curr -= 1
n = n.collect()
n.sort(key=lambda node: node.name)
return n
def test_hash(self):
ip = MagicMock()
id = -1
expected = hash((4294967295, ip))
result = hash(Node(ip=ip, node_id=id))
self.assertEqual(result, expected)
async def test_scan_ports_with_stop_task_exception(self):
nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))]
self.task.prepare_args = MagicMock()
self.task.prepare_args.side_effect = StopCommandException
result = await self.task.scan_ports(nodes=nodes)
expected = []
self.assertEqual(result, expected)
def test_ports_comparison_none(self):
node1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
port1 = Port(node=node1,
number=1,
transport_protocol=TransportProtocol.UDP)
self.assertNotEqual(port1, None)
self.assertFalse(port1 is None)
async def _get_nodes(self, url: str) -> set:
"""
Get nodes from Topdis
Returns:
set of unique nodes (Node object)
"""
url = '{0}/{1}'.format(self.api, url)
resource = await HTTPClient.instance().get(url)
nodes_cfg = ujson.loads(resource.body)
timestamp = parse_time_to_timestamp(nodes_cfg['meta']['requestTime'])
ip_node = {}
for node_struct in nodes_cfg['nodes']:
for node_ip in node_struct['ips']:
node = Node(ip=ipaddress.ip_address(node_ip),
node_id=node_struct['id'])
node.name = node_struct['displayName']
node.scan = Scan(start=timestamp)
software = node_struct.get('software', {})
os = software.get('os', {})
if os.get('discoveryType') in (
TopisOSDiscoveryType.DIRECT.value, ):
node.os.name, node.os.version = os.get('name'), os.get(
'version')
try:
node.os.cpe = Service.build_cpe(
product=node.os.name,
version=node.os.version,
part=CPEType.OS)
except:
node.os.cpe = None
ip_node[node.ip] = node
nodes = set(ip_node.values())
log.debug('Got %i nodes from topdis', len(nodes))
return nodes
def setUp(self):
super(NmapToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1,
name='test_name',
risk_level=RiskLevel.NONE)
self.exploit2 = Exploit(exploit_id=2,
name='test_name',
risk_level=RiskLevel.HIGH)
self.exploit_conf_args = Exploit(exploit_id=3)
self.exploit_conf_args.name = 'test_name2'
self.exploit_conf_args.risk_level = RiskLevel.HIGH
self.config = {
'scripts': {
'test_name': {
'args': 'test_args'
},
'test_name2': {
'args': MagicMock()
}
}
}
self.cfg = {
'tools': {
'nmap': {
'disable_scripts': [],
},
'common': {
'rate': 1337,
'http': {
'useragent': 'test_useragent'
}
}
},
'config_filename': ''
}
self.exploits = [self.exploit, self.exploit2]
self.port = Port(number=13,
transport_protocol=TransportProtocol.TCP,
node=Node(node_id=1,
ip=ipaddress.ip_address('127.0.0.1')))
self.port.scan = Scan(start=14, end=13)
self.port.protocol = 'test_service'
self.aucote = MagicMock(storage=Storage(":memory:"))
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.nmap_tool = NmapTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
def test_get_url_http_proxy(self):
node1 = Node(ip=ipaddress.ip_address('::1'), node_id=1)
port1 = Port(node=node1,
number=1,
transport_protocol=TransportProtocol.TCP)
port1.protocol = 'http-proxy'
expected = "http://[::1]:1"
self.assertEqual(port1.url, expected)
def test_get_url(self):
node1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
port1 = Port(node=node1,
number=1,
transport_protocol=TransportProtocol.TCP)
port1.protocol = 'http'
expected = "http://127.0.0.1:1"
self.assertEqual(port1.url, expected)
def test_scan_ports_with_exception(self):
nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))]
self.task.prepare_args = MagicMock(return_value=Future())
self.task.prepare_args.return_value.set_result(MagicMock())
self.task.command.async_call.side_effect = NonXMLOutputException()
result = yield self.task.scan_ports(nodes)
expected = []
self.assertEqual(result, expected)
def test_diff_two_last_scans(self, serializer):
current_scan = Scan()
self.thread.scan = current_scan
previous_scan = Scan()
node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.aucote.storage.get_nodes_by_scan.return_value = [node]
port_added = Port(node,
transport_protocol=TransportProtocol.TCP,
number=88)
port_added.row_id = 17
port_removed = Port(node,
transport_protocol=TransportProtocol.TCP,
number=80)
port_removed.row_id = 18
port_unchanged = Port(node,
transport_protocol=TransportProtocol.TCP,
number=22)
port_unchanged.row_id = 19
port_scans_current = [
PortScan(port=port_unchanged, scan=current_scan),
PortScan(port=port_added, scan=current_scan)
]
port_scans_previous = [
PortScan(port=port_unchanged, scan=previous_scan),
PortScan(port=port_removed, scan=previous_scan)
]
self.aucote.storage.get_scans_by_node.return_value = [
current_scan, previous_scan
]
self.aucote.storage.get_ports_by_scan_and_node.side_effect = (
port_scans_current, port_scans_previous)
expected = [
PortDetectionChange(current_finding=port_scans_current[1],
previous_finding=None),
PortDetectionChange(current_finding=None,
previous_finding=port_scans_previous[1])
]
self.thread.diff_with_last_scan()
self.aucote.storage.get_nodes_by_scan.assert_called_once_with(
scan=current_scan)
self.assertEqual(len(self.aucote.storage.save_changes.call_args_list),
1)
result = self.aucote.storage.save_changes.call_args[0][0]
self.assertCountEqual(result, expected)
self.assertCountEqual([
serializer.call_args_list[0][0][0],
serializer.call_args_list[1][0][0]
], expected)
def test_ports_comparison_eq(self):
node1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
port1 = Port(node=node1,
number=1,
transport_protocol=TransportProtocol.TCP)
port2 = Port(node=node1,
number=1,
transport_protocol=TransportProtocol.TCP)
self.assertEqual(port1, port2)
def test_equality_different_types(self):
node1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
port1 = Port(node=node1,
number=1,
transport_protocol=TransportProtocol.TCP)
port2 = MagicMock(node=node1,
number=1,
transport_protocol=TransportProtocol.TCP)
self.assertNotEqual(port1, port2)
def test_prepare_args_ipv6(self):
node = Node(12, ipaddress.ip_address("::1"))
self.port.node = node
result = self.hydra_script_task.prepare_args()
expected = [
'-L', 'test_logins', '-6', '-P', 'test_passwords', '-s', '22',
'::1', 'ssh'
]
self.assertEqual(result, expected)
def setUp(self):
exploit = Exploit(exploit_id=3)
port = Port(node=Node(node_id=2, ip=ipaddress.ip_address('127.0.0.1')),
transport_protocol=TransportProtocol.TCP,
number=16)
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = SSLScriptTask(port=port,
exploits=[exploit],
context=self.context)
def get_app(self):
self.aucote = MagicMock()
self.scanner = TCPScanner(aucote=self.aucote, host='localhost', port=1339)
self.scanner.NAME = 'test_name'
self.scanner.scan.start = 1290
self.scanner.nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))]
self.aucote.scanners = [self.scanner, ToolsScanner(name='tools', aucote=self.aucote)]
self.app = Application([
(r"/api/v1/scanner/([\w_]+)", ScannersHandler, {'aucote': self.aucote}),
(r"/api/v1/scanners", ScannersHandler, {'aucote': self.aucote})])
return self.app
def test_get_ports_for_scanning(self):
node_1 = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
node_2 = Node(ip=ipaddress.ip_address('127.0.0.2'), node_id=2)
node_3 = Node(ip=ipaddress.ip_address('127.0.0.3'), node_id=3)
port_1 = Port(node=node_1,
number=80,
transport_protocol=TransportProtocol.TCP)
port_2 = Port(node=node_2,
number=80,
transport_protocol=TransportProtocol.TCP)
port_3 = Port(node=node_3,
number=80,
transport_protocol=TransportProtocol.TCP)
ports = [port_1, port_2]
result = self.executor._get_ports_for_scanning(ports, [port_2, port_3])
expected = [port_1]
self.assertListEqual(result, expected)
def setUp(self):
super(NmapPortInfoTaskTest, self).setUp()
self.aucote = MagicMock()
self.aucote.task_mapper.assign_tasks = MagicMock(return_value=Future())
self.aucote.task_mapper.assign_tasks.return_value.set_result(
MagicMock())
self.exploit = Exploit(exploit_id=4)
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.node.os = Service(name='os test name',
version='os test version',
cpe='cpe:2.3:o:vendor:product:-:*:*:*:*:*:*:*')
self.node_ipv6 = Node(ip=ipaddress.ip_address('::1'), node_id=None)
self.port = Port(number=22,
transport_protocol=TransportProtocol.TCP,
node=self.node)
self.port.scan = Scan()
self.port_ipv6 = Port(number=22,
node=self.node_ipv6,
transport_protocol=TransportProtocol.TCP)
self.scanner = Scanner(aucote=self.aucote)
self.scanner.NAME = 'tools'
self.context = ScanContext(aucote=self.aucote, scanner=self.scanner)
self.port_info = NmapPortInfoTask(context=self.context, port=self.port)
self.cfg = {
'portdetection': {
'tools': {
'scan_rate': 1337
},
'expiration_period': '7d'
},
'tools': {
'nmap': {
'scripts_dir': 'test'
}
},
}
def test_scan_ports(self, mock_parser):
expected = MagicMock()
mock_parser.return_value = expected
nodes = [Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))]
self.task.prepare_args = MagicMock(return_value=Future())
self.task.prepare_args.return_value.set_result(MagicMock())
future_1 = Future()
future_1.set_result(MagicMock())
self.task.command.async_call.return_value = future_1
result = yield self.task.scan_ports(nodes)
self.assertEqual(result, expected)
def setUp(self):
super(PortScanTest, self).setUp()
cfg = {
'portdetection': {
'tcp': {
'ports': {
'include': ['55'],
'exclude': [],
},
'scan_rate': 1030,
'host_timeout': '600'
},
'udp': {
'ports': {
'include': [],
'exclude': []
},
'scan_rate': 30,
'defeat_icmp_ratelimit': False,
'max_retries': 77
},
'_internal': {
'udp_retries': 2
}
},
'tools': {
'nmap': {
'cmd': 'nmap',
'scripts_dir': '',
}
}
}
self.cfg = cfg
self.kudu_queue = MagicMock()
self.scanner = PortsScan(ipv6=True, tcp=True, udp=False)
node = Node(ip=ipaddress.ip_address('192.168.1.5'), node_id=None)
node.scan = Scan()
self.nodes = [node]
def setUp(self):
super(WhatWebTaskTest, self).setUp()
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.port = Port(node=self.node,
number=19,
transport_protocol=TransportProtocol.UDP)
self.port.protocol = 'http'
self.aucote = MagicMock()
self.exploit = Exploit(app='whatweb', name='whatweb', exploit_id=1)
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = WhatWebTask(port=self.port,
context=self.context,
exploits=[self.exploit])
def setUp(self):
self.serializer = Serializer()
self.vuln = Vulnerability(subid=15)
node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
node.os = MagicMock()
node.os.name_with_version = 'test_name_and_version'
self.context = ScanContext(aucote=None, scanner=TCPScanner(MagicMock(), MagicMock(), MagicMock(), MagicMock()))
self.vuln.context = self.context
self.port = Port(node=node, number=22, transport_protocol=TransportProtocol.TCP)
self.port.protocol = 'ssh'
self.port.scan = Scan()
self.port.scan.start = datetime.datetime(2016, 8, 16, 15, 23, 10, 183095, tzinfo=utc).timestamp()
self.vuln.port = self.port
self.vuln.output = 'Test'
self.vuln.scan = Scan(start=datetime.datetime(2016, 8, 16, 15, 23, 10, 183095, tzinfo=utc).timestamp(),
scanner='tcp')
self.exploit = Exploit(exploit_id=1)
self.exploit.app = 'test_app'
self.exploit.name = 'test_name'
self.exploit.title = 'test_title'
self.exploit.description = 'test_description'
self.exploit.risk_level = RiskLevel.from_name('High')
self.exploit.cve = 'CVE-2018-0001'
self.exploit.cvss = 9.8
self.exploit.metric = ExploitMetric.VNC_INFO
self.exploit.category = ExploitCategory.VULN
self.exploit.tags = {ExploitTag.HTTP, ExploitTag.SSL, ExploitTag.HTTPS}
self.vuln.exploit = self.exploit
self.vuln.time = datetime.datetime(2016, 8, 16, 15, 23, 10, 183095, tzinfo=utc).timestamp()
def setUp(self):
super(WhatWebToolTest, self).setUp()
self.aucote = MagicMock()
self.exploit = Exploit(exploit_id=1)
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.port = Port(node=self.node,
transport_protocol=TransportProtocol.UDP,
number=87)
self.config = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.tool = WhatWebTool(context=self.context,
exploits=[self.exploit],
port=self.port,
config=self.config)
async def test_assign_task_for_node_disabled(self, cfg):
self.cfg['tools']['test']['enable'] = False
cfg._cfg = self.cfg
self.task_mapper._filter_exploits = MagicMock()
class_mock = self.EXECUTOR_CONFIG['apps']['test']['class']
self.task_mapper._aucote.exploits.find_by_apps = MagicMock()
mock_apps = self.task_mapper._aucote.exploits.find_by_apps
mock_apps.return_value = {'test': [MagicMock()]}
node = Node(1, ipaddress.ip_address('127.0.0.1'))
await self.task_mapper.assign_tasks_for_node(node)
self.assertFalse(class_mock.called)
def setUp(self):
super(HydraToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1,
name='hydra',
app='hydra',
risk_level=RiskLevel.NONE)
self.exploit2 = Exploit(exploit_id=2,
name='hydra',
app='hydra',
risk_level=RiskLevel.HIGH)
self.config = {
'services': ['ssh', 'vnc'],
'mapper': {
'test': 'ssh'
},
'without-login': '******'
}
self.exploits = [self.exploit, self.exploit2]
self.node = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))
self.port = Port(number=12,
transport_protocol=TransportProtocol.TCP,
node=self.node)
self.port.protocol = 'test'
self.port.scan = Scan(start=14)
self.port_no_login = Port(number=12,
transport_protocol=TransportProtocol.TCP,
node=self.node)
self.port_no_login.protocol = 'vnc'
self.port_no_login.scan = Scan(start=14)
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.hydra_tool = HydraTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
self.hydra_tool_without_login = HydraTool(context=self.context,
exploits=self.exploits,
port=self.port_no_login,
config=self.config)