class ScanContextTest(AsyncTestCase):
def setUp(self):
super(ScanContextTest, self).setUp()
self.scan = MagicMock()
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote, scanner=self.scan)
def test_add_task(self):
task = MagicMock()
self.context.add_task(task)
self.assertIn(task, self.context.tasks)
self.aucote.add_async_task.assert_called_once_with(
task, manager=TaskManagerType.REGULAR)
def test_non_end_scan(self):
self.context.tasks = [
MagicMock(has_finished=MagicMock(return_value=False))
]
result = self.context.is_scan_end()
self.assertFalse(result)
def test_scan_end(self):
self.context.tasks = [
MagicMock(has_finished=MagicMock(return_value=True))
]
self.context.end = 0
result = self.context.is_scan_end()
self.assertTrue(result)
def setUp(self):
super(AucoteHttpHeadersTaskTest, self).setUp()
HTTPClient._instance = MagicMock()
self.port = Port(node=MagicMock(),
transport_protocol=None,
number=None)
self.port.scan = Scan()
self.aucote = MagicMock()
self.exploit = MagicMock()
self.exploit.name = "test"
self.config = {
'headers': {
'test': HeaderDefinition(pattern='test_nie', obligatory=True)
}
}
self.custom_headers = {
'Accept-Encoding': 'gzip, deflate',
'User-Agent': 'test'
}
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = AucoteHttpHeadersTask(port=self.port,
context=self.context,
exploits=[self.exploit],
config=self.config)
def setUp(self):
super(TaskMapperTest, self).setUp()
self.executor = Mock()
self.exploits = OrderedDict({
'test': [
Exploit(exploit_id=1, name='test_1'),
Exploit(exploit_id=2, name='test_2')
]
})
self.exploits.update(test2=[Exploit(exploit_id=3)])
self.executor.exploits.find_all_matching.return_value = self.exploits
self.context = ScanContext(aucote=self.executor,
scanner=MagicMock(scan=Scan()))
self.task_mapper = TaskMapper(context=self.context)
self.cfg = {
'portdetection': {
'_internal': {
'categories': ['other', 'brute']
}
},
'tools': {
'test': {
'enable': True
},
'test2': {
'enable': True
}
}
}
def setUp(self):
super(NmapToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1,
name='test_name',
risk_level=RiskLevel.NONE)
self.exploit2 = Exploit(exploit_id=2,
name='test_name',
risk_level=RiskLevel.HIGH)
self.exploit_conf_args = Exploit(exploit_id=3)
self.exploit_conf_args.name = 'test_name2'
self.exploit_conf_args.risk_level = RiskLevel.HIGH
self.config = {
'scripts': {
'test_name': {
'args': 'test_args'
},
'test_name2': {
'args': MagicMock()
}
}
}
self.cfg = {
'tools': {
'nmap': {
'disable_scripts': [],
},
'common': {
'rate': 1337,
'http': {
'useragent': 'test_useragent'
}
}
},
'config_filename': ''
}
self.exploits = [self.exploit, self.exploit2]
self.port = Port(number=13,
transport_protocol=TransportProtocol.TCP,
node=Node(node_id=1,
ip=ipaddress.ip_address('127.0.0.1')))
self.port.scan = Scan(start=14, end=13)
self.port.protocol = 'test_service'
self.aucote = MagicMock(storage=Storage(":memory:"))
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.nmap_tool = NmapTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
def setUp(self):
exploit = Exploit(exploit_id=3)
port = Port(node=Node(node_id=2, ip=ipaddress.ip_address('127.0.0.1')),
transport_protocol=TransportProtocol.TCP,
number=16)
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = SSLScriptTask(port=port,
exploits=[exploit],
context=self.context)
def setUp(self):
self.aucote = MagicMock()
self.port = Port(node=MagicMock(),
transport_protocol=None,
number=MagicMock())
self.exploit = MagicMock()
self.scan = Scan()
self.context = ScanContext(aucote=self.aucote, scanner=None)
self.task = PortTask(context=self.context,
port=self.port,
exploits=[self.exploit])
def setUp(self):
"""
Set up init variables
"""
super().setUp()
self.executor = MagicMock()
self.executor.kudu_queue = MagicMock()
self.executor.exploits = MagicMock()
self.context = ScanContext(aucote=self.executor,
scanner=MagicMock(scan=Scan()))
self.task = Task(context=self.context)
def setUp(self):
"""
Prepare some internal variables:
exploit, port, exploits, script, vulnerability, scan_task
"""
super(NmapPortScanTaskTest, self).setUp()
self.aucote = MagicMock()
self.exploit = Exploit(exploit_id=1, app='nmap', name='test')
self.exploit_vuln_non_exist = Exploit(exploit_id=2,
app='nmap',
name='test2')
self.exploits = Exploits()
self.exploits.add(self.exploit)
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=None)
self.node_ipv6 = Node(ip=ipaddress.ip_address('::1'), node_id=None)
self.port = Port(number=22,
node=self.node,
transport_protocol=TransportProtocol.TCP)
self.port.service_name = 'ssh'
self.port.scan = Scan()
self.port_ipv6 = Port(number=22,
node=self.node_ipv6,
transport_protocol=TransportProtocol.TCP)
self.script = NmapScript(port=self.port,
parser=NmapParser(),
exploit=self.exploit,
name='test',
args='test_args')
self.script.get_result = MagicMock(return_value='test')
self.script2 = NmapScript(port=self.port,
parser=NmapInfoParser(),
exploit=self.exploit_vuln_non_exist,
name='test2')
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.scan_task = NmapPortScanTask(
context=self.context,
port=self.port,
script_classes=[self.script, self.script2],
rate=1337)
self.scan_task.store_scan_end = MagicMock()
future = Future()
future.set_result(ElementTree.fromstring(self.XML))
self.scan_task.command.async_call = MagicMock(return_value=future)
self.cfg = {'tools': {'nmap': {'scripts_dir': '', 'timeout': 0}}}
def setUp(self):
super(AucoteHttpHeadersToolTest, self).setUp()
self.aucote = MagicMock()
self.exploits = MagicMock()
self.port = MagicMock()
self.config = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.tool = AucoteHttpHeadersTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
def setUp(self):
super(WhatWebTaskTest, self).setUp()
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.port = Port(node=self.node,
number=19,
transport_protocol=TransportProtocol.UDP)
self.port.protocol = 'http'
self.aucote = MagicMock()
self.exploit = Exploit(app='whatweb', name='whatweb', exploit_id=1)
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = WhatWebTask(port=self.port,
context=self.context,
exploits=[self.exploit])
def setUp(self):
super().setUp()
self.aucote = MagicMock()
self.aucote.storage.filename = ":memory:"
self.exploits = MagicMock()
self.config = MagicMock()
self.port = MagicMock()
self.scan = Scan()
self.context = ScanContext(aucote=self.aucote, scanner=None)
self.tool = Tool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
def setUp(self):
super(WhatWebToolTest, self).setUp()
self.aucote = MagicMock()
self.exploit = Exploit(exploit_id=1)
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.port = Port(node=self.node,
transport_protocol=TransportProtocol.UDP,
number=87)
self.config = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.tool = WhatWebTool(context=self.context,
exploits=[self.exploit],
port=self.port,
config=self.config)
def setUp(self):
super(CVESearchToolTest, self).setUp()
self.aucote = MagicMock()
self.exploits = MagicMock()
self.port = MagicMock()
self.config = MagicMock()
self.node = MagicMock()
self.scan = Scan()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.tool = CVESearchTool(context=self.context,
exploits=self.exploits,
port=self.port,
node=self.node,
config=self.config)
def setUp(self, cfg):
super(ExecutorTest, self).setUp()
cfg._cfg = {
'portdetection': {
'_internal': {
'port_period': None,
'broadcast': True
}
}
}
self.cfg = cfg
self.aucote = MagicMock()
self.aucote.storage = MagicMock()
self.context = ScanContext(aucote=self.aucote, scanner=MagicMock())
self.context.scanner.scan = Scan()
self.executor = Executor(context=self.context)
def setUp(self):
super(HydraToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1,
name='hydra',
app='hydra',
risk_level=RiskLevel.NONE)
self.exploit2 = Exploit(exploit_id=2,
name='hydra',
app='hydra',
risk_level=RiskLevel.HIGH)
self.config = {
'services': ['ssh', 'vnc'],
'mapper': {
'test': 'ssh'
},
'without-login': '******'
}
self.exploits = [self.exploit, self.exploit2]
self.node = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))
self.port = Port(number=12,
transport_protocol=TransportProtocol.TCP,
node=self.node)
self.port.protocol = 'test'
self.port.scan = Scan(start=14)
self.port_no_login = Port(number=12,
transport_protocol=TransportProtocol.TCP,
node=self.node)
self.port_no_login.protocol = 'vnc'
self.port_no_login.scan = Scan(start=14)
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.hydra_tool = HydraTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
self.hydra_tool_without_login = HydraTool(context=self.context,
exploits=self.exploits,
port=self.port_no_login,
config=self.config)
def setUp(self):
super(SkipfishToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1)
self.exploit.name = 'skipfish'
self.exploit.risk_level = RiskLevel.NONE
self.config = {}
self.exploits = [self.exploit]
self.port = Port(node=Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1')), number=3,
transport_protocol=TransportProtocol.TCP)
self.port.scan = Scan(start=13, end=45)
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote, scanner=MagicMock(scan=Scan()))
self.skipfish_tool = SkipfishTool(context=self.context, exploits=self.exploits, port=self.port,
config=self.config)
def setUp(self):
super(SietTaskTest, self).setUp()
self.context = ScanContext(aucote=MagicMock(),
scanner=TCPScanner(MagicMock(), MagicMock(),
MagicMock(),
MagicMock()))
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=14)
self.scan = Scan()
self.port = Port(node=self.node,
number=46,
transport_protocol=TransportProtocol.TCP,
scan=self.scan)
self.exploit = Exploit(exploit_id=15,
app='aucote-scripts',
name='siet')
self.task = SietTask(context=self.context,
port=self.port,
exploits=[self.exploit])
self.cfg = {'portdetection': {'expiration_period': '7d'}}
def setUp(self):
super(HydraScriptTaskTest, self).setUp()
self.aucote = MagicMock()
self.port = Port(node=Node(ip='127.0.0.1', node_id=None),
transport_protocol=TransportProtocol.TCP,
number=22)
self.port.service_name = 'ssh'
self.port.scan = Scan()
self.exploit = Exploit(exploit_id=1)
self.scan = Scan()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.hydra_script_task = HydraScriptTask(
exploits=[self.exploit],
context=self.context,
port=self.port,
service=self.port.service_name)
self.hydra_script_task.store_scan_end = MagicMock()
self.hydra_script_task.aucote.exploits.find.return_value = self.exploit
def setUp(self):
super(CommandTaskTest, self).setUp()
self.aucote = MagicMock()
self.port = Port(node=MagicMock(),
transport_protocol=None,
number=None)
self.port.scan = Scan()
self.command = MagicMock(NAME='test_name')
future = Future()
self.future_return = MagicMock()
future.set_result(self.future_return)
self.command.async_call = MagicMock(return_value=future)
self.exploit = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = CommandTask(context=self.context,
port=self.port,
command=self.command,
exploits=[self.exploit])
self.cfg = {'tools': {'test_name': {'timeout': 0}}}
def setUp(self):
super(NmapPortInfoTaskTest, self).setUp()
self.aucote = MagicMock()
self.aucote.task_mapper.assign_tasks = MagicMock(return_value=Future())
self.aucote.task_mapper.assign_tasks.return_value.set_result(
MagicMock())
self.exploit = Exploit(exploit_id=4)
self.node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
self.node.os = Service(name='os test name',
version='os test version',
cpe='cpe:2.3:o:vendor:product:-:*:*:*:*:*:*:*')
self.node_ipv6 = Node(ip=ipaddress.ip_address('::1'), node_id=None)
self.port = Port(number=22,
transport_protocol=TransportProtocol.TCP,
node=self.node)
self.port.scan = Scan()
self.port_ipv6 = Port(number=22,
node=self.node_ipv6,
transport_protocol=TransportProtocol.TCP)
self.scanner = Scanner(aucote=self.aucote)
self.scanner.NAME = 'tools'
self.context = ScanContext(aucote=self.aucote, scanner=self.scanner)
self.port_info = NmapPortInfoTask(context=self.context, port=self.port)
self.cfg = {
'portdetection': {
'tools': {
'scan_rate': 1337
},
'expiration_period': '7d'
},
'tools': {
'nmap': {
'scripts_dir': 'test'
}
},
}
def setUp(self):
self.serializer = Serializer()
self.vuln = Vulnerability(subid=15)
node = Node(ip=ipaddress.ip_address('127.0.0.1'), node_id=1)
node.os = MagicMock()
node.os.name_with_version = 'test_name_and_version'
self.context = ScanContext(aucote=None, scanner=TCPScanner(MagicMock(), MagicMock(), MagicMock(), MagicMock()))
self.vuln.context = self.context
self.port = Port(node=node, number=22, transport_protocol=TransportProtocol.TCP)
self.port.protocol = 'ssh'
self.port.scan = Scan()
self.port.scan.start = datetime.datetime(2016, 8, 16, 15, 23, 10, 183095, tzinfo=utc).timestamp()
self.vuln.port = self.port
self.vuln.output = 'Test'
self.vuln.scan = Scan(start=datetime.datetime(2016, 8, 16, 15, 23, 10, 183095, tzinfo=utc).timestamp(),
scanner='tcp')
self.exploit = Exploit(exploit_id=1)
self.exploit.app = 'test_app'
self.exploit.name = 'test_name'
self.exploit.title = 'test_title'
self.exploit.description = 'test_description'
self.exploit.risk_level = RiskLevel.from_name('High')
self.exploit.cve = 'CVE-2018-0001'
self.exploit.cvss = 9.8
self.exploit.metric = ExploitMetric.VNC_INFO
self.exploit.category = ExploitCategory.VULN
self.exploit.tags = {ExploitTag.HTTP, ExploitTag.SSL, ExploitTag.HTTPS}
self.vuln.exploit = self.exploit
self.vuln.time = datetime.datetime(2016, 8, 16, 15, 23, 10, 183095, tzinfo=utc).timestamp()
def setUp(self, cfg):
super(CVESearchServiceTaskTest, self).setUp()
cfg._cfg = {'tools': {'cve-search': {'api': 'localhost:200'}}}
self.example_output = ''
with open(
path.join(path.dirname(path.abspath(__file__)),
'example_output.json'), 'rb') as f:
self.example_output = f.read()
self.node = Node(ip='127.0.0.1', node_id=None)
self.port = Port(node=self.node,
transport_protocol=TransportProtocol.TCP,
number=22)
self.port.service_name = 'ssh'
self.port.scan = Scan()
self.port.service = Service()
self.app = Service()
self.app_2 = Service()
self.app.cpe = 'cpe:/a:microsoft:iexplorer:8.0.6001:beta'
self.app_2.cpe = 'cpe:/a:microsoft:aexplorer:8.0.6001:beta'
self.cpe_txt = 'cpe:/a:microsoft:internet_explorer:8.0.6001:beta'
self.os_cpe_txt = 'cpe:/o:a:b:4'
self.cpe_without_version = 'cpe:/o:cisco:ios'
self.node.os.cpe = self.os_cpe_txt
self.port.service.cpe = self.cpe_txt
self.exploit = Exploit(exploit_id=1337,
name='cve-search',
app='cve-search')
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.task = CVESearchServiceTask(context=self.context,
port=self.port,
exploits=[self.exploit])
self.vuln_1 = Vulnerability(port=self.port,
exploit=self.exploit,
cve='CVE-2016-8612',
cvss=3.3,
output='test summary 1',
context=self.context,
subid=0)
self.vuln_2 = Vulnerability(port=self.port,
exploit=self.exploit,
cve='CVE-2017-9798',
cvss=5.0,
output='test summary 2',
context=self.context,
subid=1)
self.vuln_3 = Vulnerability(port=self.port,
exploit=self.exploit,
cve='CVE-2017-9788',
cvss=6.4,
output='test summary 3',
context=self.context,
subid=2)
class ScanAsyncTask(object):
"""
Parent class for all scanning tasks
"""
LIVE_SCAN_CRON = '* * * * *'
PROTOCOL = None
NAME = None
TOPDIS_MIN_TIME = 5
TOPDIS_MAX_TIME = 30
TOPDIS_RETRIES = 5
def __init__(self, aucote):
self._current_scan = []
self._aucote = aucote
self.context = None
self.scan = Scan(protocol=self.PROTOCOL, scanner=self.NAME,
init=False) # ToDo: move it inside
self._shutdown_condition = Event()
self.status = ScanStatus.IDLE
self.run_now = False
@property
def aucote(self):
return self._aucote
def _init(self):
if self.context is not None:
raise Exception("Scan context already exists")
self.context = ScanContext(aucote=self.aucote, scanner=self)
async def __call__(self, resume=False):
try:
self._init()
if not cfg['portdetection.{name}.scan_enabled'.format(
name=self.NAME)]:
log.info("Scanner %s is disabled", self.NAME)
return
log.info("Starting %s scanner", self.NAME)
if resume:
self.scan.resume = True
else:
self.scan.resume = False
result = await self.run(resume=resume)
run_after = cfg['portdetection.{name}.run_after'.format(
name=self.NAME)]
for scan_name in run_after:
scan_task = self.aucote.async_task_managers[
TaskManagerType.SCANNER].cron_task(scan_name)
if scan_task is not None:
self.aucote.ioloop.add_callback(
partial(scan_task, skip_cron=True))
return result
finally:
self.context.end = time.time()
self.context = None
self.expire_vulnerabilities()
async def run(self, **kwargs):
raise NotImplementedError()
@property
def shutdown_condition(self):
"""
Event which is set when no scan in progress
Returns:
Event
"""
return self._shutdown_condition
async def _get_nodes_for_scanning(self,
timestamp=None,
filter_out_storage=True,
scan=None):
"""
Get nodes for scan since timestamp.
- If timestamp is None, it is equal: current timestamp - node scan period
- Restrict nodes to allowed networks
Args:
timestamp (float):
Returns:
list
"""
if scan is not None:
nodes = {
'snmp': set(self.storage.get_non_finished_nodes(scan)),
'hosts': set()
}
for node in nodes['snmp']:
node.scan = self.scan
else:
nodes = {'snmp': await self.topdis.get_snmp_nodes()}
nodes['hosts'] = await self.topdis.get_all_nodes() - nodes['snmp']
if filter_out_storage:
storage_nodes = set(
self.storage.get_nodes(pasttime=self._scan_interval(),
timestamp=timestamp,
scan=self.scan))
nodes['hosts'] = nodes['hosts'] - storage_nodes
nodes['snmp'] = nodes['snmp'] - storage_nodes
include_networks = self._get_networks_list()
exclude_networks = self._get_excluded_networks_list()
return_value = []
if cfg['portdetection.{name}.scan_devices.snmp'.format(
name=self.NAME)]:
return_value.extend(node for node in list(nodes['snmp'])
if node.ip.exploded in include_networks
and node.ip.exploded not in exclude_networks)
if cfg['portdetection.{name}.scan_devices.host'.format(
name=self.NAME)]:
return_value.extend(node for node in list(nodes['hosts'])
if node.ip.exploded in include_networks
and node.ip.exploded not in exclude_networks)
return return_value
def _get_networks_list(self):
"""
Returns list of networks from configuration file
Returns:
IPSet: set of networks
"""
try:
return IPSet(cfg['portdetection.{name}.networks.include'.format(
name=self.NAME)])
except KeyError:
log.error(
"Please set portdetection.%s.networks.include in configuration file!",
self.NAME)
exit()
def _get_excluded_networks_list(self):
"""
List of excluded networks from configuration file
Returns:
IPSet: set of networks
"""
try:
return IPSet(cfg['portdetection.{name}.networks.exclude'.format(
name=self.NAME)])
except KeyError:
return []
@property
def storage(self):
"""
Handler to application storage
Returns:
Storage
"""
return self.aucote.storage
@property
def current_scan(self):
"""
List of currently scan nodes
Returns:
list
"""
return self._current_scan[:]
@current_scan.setter
def current_scan(self, val):
self._current_scan = val
@property
def previous_scan(self):
"""
Returns previous scan timestamp
Returns:
float
"""
return int(croniter(self._scan_cron(), time.time()).get_prev())
@property
def next_scan(self):
"""
Time of next regular scan
Returns:
float
"""
return int(croniter(self._scan_cron(), time.time()).get_next())
def _scan_interval(self):
"""
Get interval between particular node scan
Returns:
int
"""
if cfg['portdetection.{name}.scan_type'.format(
name=self.NAME)] == ScanType.PERIODIC.value:
return 0
return parse_period(
cfg['portdetection.{name}.live_scan.min_time_gap'.format(
name=self.NAME)])
def _scan_cron(self):
"""
Get scan cron
Returns:
str
"""
if cfg['portdetection.{name}.scan_type'.format(
name=self.NAME)] == ScanType.LIVE.value:
return self.LIVE_SCAN_CRON
return cfg['portdetection.{name}.periodic_scan.cron'.format(
name=self.NAME)]
def is_exploit_allowed(self, exploit):
"""
Check if exploit can be executed by scanner
Args:
exploit:
Returns:
bool
"""
return exploit.id in map(
int, cfg['portdetection.{0}.scripts'.format(self.NAME)])
async def _clean_scan(self):
"""
Clean scan and update scan status
Returns:
None
"""
await self.update_scan_status(ScanStatus.IDLE)
self.scan.rowid = None # ToDo: Do it more pythonic
self._shutdown_condition.set()
async def update_scan_status(self, status=None):
"""
Update scan status base on status value
Args:
status (ScanStatus):
Returns:
None
"""
self.status = status
if not cfg.toucan or cfg['portdetection.{name}.scan_type'.format(
name=self.NAME)] == ScanType.LIVE.value:
return
current_status = cfg.get('portdetection.{0}.status.*'.format(
self.NAME),
cache=False)
data = {'portdetection': {self.NAME: {'status': {}}}}
log.debug("Current status for %s is %s", self.NAME, current_status.cfg)
next_scan = round(current_status['next_scan_start'])
if next_scan != self.next_scan:
data['portdetection'][
self.NAME]['status']['next_scan_start'] = self.next_scan
if self.scan.start:
previous_scan_start = current_status['scan_start']
if previous_scan_start != self.scan.start:
data['portdetection'][self.NAME]['status'][
'previous_scan_start'] = previous_scan_start
data['portdetection'][
self.NAME]['status']['scan_start'] = self.scan.start
if status is not None:
current_status_code = current_status['code']
if current_status_code != status.value:
data['portdetection'][
self.NAME]['status']['code'] = status.value
if status is ScanStatus.IDLE and self.scan.start is not None:
data['portdetection'][self.NAME]['status'][
'previous_scan_duration'] = int(time.time() - self.scan.start)
if data['portdetection'][self.NAME]['status']:
log.debug("Update toucan by %s with %s", self.NAME, data)
await cfg.toucan.async_push_config(data,
overwrite=True,
keep_history=False)
@property
def topdis(self):
"""
Topdis API object
Returns:
Topdis
"""
return self.aucote.topdis
async def stop(self):
"""
Stops scan by stopping/cancelling all its related tasks
"""
log.info('Stopping scan %s', self.NAME)
if self.context is None:
log.warning("There is no %s scan in progress", self.NAME)
return
self.context.cancel()
if not self.context.is_scan_end():
tasks = self.context.unfinished_tasks()
log.warning('Cancelling %s tasks for scan %s', len(tasks),
self.NAME)
for task in tasks:
task.cancel()
await self.context.wait_on_scan_end()
log.info('Scan %s cancelled successfully', self.NAME)
def prepare_vulnerability_for_kudu(self, vuln: 'Vulnerability'):
"""
Update vulnerability to meet all fields required by kudu serializer
"""
data = self.storage.portdetection_vulns(vuln)
os_service = Service(name=data['os_name'],
version=data['os_version'],
cpe=data['os_cpe'])
vuln.port.node.os = os_service
vuln.port.protocol = data['protocol']
vuln.port.banner = data['banner']
vuln.port.service.name = data['name']
vuln.port.service.version = data['version']
vuln.port.service.cpe = data['cpe']
return vuln
def expire_vulnerabilities(self):
"""
Update validation time of vulnerabilites
"""
# Do not update already deprecated vulnerabilities, so get all later than timestamp - expiration_period
timestamp = time.time()
expiration_period = parse_period(
cfg['portdetection.expiration_period'])
vulns = self.storage.expire_vulnerabilities(timestamp=timestamp -
expiration_period)
for vuln in vulns:
# There is some mismatch between kudu and local storage
if vuln.exploit.id == 0 and vuln.subid > 0:
continue
self.prepare_vulnerability_for_kudu(vuln)
self.store_vulnerability(vuln)
def store_vulnerability(self, vuln):
"""
Saves vulnerability into database: kudu and local storage
"""
expiration_period = parse_period(
cfg['portdetection.expiration_period'])
log.debug('Found vulnerability %s for %s',
vuln.exploit.id if vuln.exploit is not None else None,
vuln.port)
try:
# Do not save vulnerability which is already saved: FixMe: better save and update vulns
if vuln.expiration_time is None:
self.aucote.storage.save_vulnerabilities(
vulnerabilities=[vuln], scan=self.scan)
except Exception:
log.warning(
'Error during saving vulnerability (%s, %s) to the storage',
vuln.exploit.id if vuln.exploit is not None else None,
vuln.subid)
# FixMe: A little bit hacking here: Serializer doesn't have access to Toucan,
# so I have to set expiration time in vuln. Future solution: Incorporate serializer into Aucote instance
if vuln.expiration_time is None:
vuln.expiration_time = vuln.time + expiration_period
msg = Serializer.serialize_vulnerability(vuln)
self.aucote.kudu_queue.send_msg(msg)
def __str__(self):
return self.__class__.__name__
def get_last_scan(self, resume: Optional[bool] = None) -> Optional['Scan']:
"""
Get last scan from database
"""
scans = self.storage.get_scans(self.PROTOCOL,
self.NAME,
amount=2,
resume=resume)
if not scans:
return None
if scans[0].rowid == self.scan.rowid:
if len(scans) == 1:
return None
return scans[1]
return scans[0]
def get_previous_non_resumed_scan(self):
"""
Returns scan before last scan (which is not current scan)
Returns:
"""
scans = self.storage.get_scans(self.PROTOCOL,
self.NAME,
amount=3,
resume=False)
if not scans:
return None
# We should have at least 2 scans
if len(scans) == 1:
return None
if len(scans) == 2:
# If last scan is current scan, the second one is truly last scan
if scans[0].rowid == self.scan.rowid:
return None
return scans[1]
return scans[2]
def _init(self):
if self.context is not None:
raise Exception("Scan context already exists")
self.context = ScanContext(aucote=self.aucote, scanner=self)
def setUp(self):
super(ScanContextTest, self).setUp()
self.scan = MagicMock()
self.aucote = MagicMock()
self.context = ScanContext(aucote=self.aucote, scanner=self.scan)
def get_app(self):
self.aucote = MagicMock(unfinished_tasks=4)
self.context = ScanContext(aucote=self.aucote, scanner=None)
self.tasks = AsyncTaskManager()
self.tasks._is_running = False
tasks = [
{
'port': 45,
'id': 1,
'ip': '127.0.0.1'
},
{
'port': 56,
'id': 2,
'ip': '127.0.0.2'
},
{
'port': 67,
'id': 3,
'ip': '127.0.0.3'
},
]
worker_tasks = [
{
'port': 78,
'id': 4,
'ip': '127.0.0.4'
},
{
'port': 89,
'id': 5,
'ip': '127.0.0.5'
},
]
for task in tasks:
self.tasks.add_task(
PortTask(context=self.context,
port=Port(
node=Node(node_id=task['id'],
ip=ipaddress.ip_address(task['ip'])),
number=task['port'],
transport_protocol=TransportProtocol.TCP,
),
exploits=[]))
self.tasks._task_workers = {
number:
PortTask(context=self.context,
port=Port(
node=Node(node_id=task['id'],
ip=ipaddress.ip_address(task['ip'])),
number=task['port'],
transport_protocol=TransportProtocol.TCP,
),
exploits=[])
for number, task in enumerate(worker_tasks)
}
class test_function:
def __str__(self):
return 'test_function'
def __call__(self, *args, **kwargs):
pass
self.tasks.add_crontab_task(test_function(), '0 0 0 0 0')
self.tasks._task_workers.update({2: None, 3: None, 4: None})
self.aucote.async_task_managers = {TaskManagerType.SCANNER: self.tasks}
self.app = Application([(r"/api/v1/tasks", TasksHandler, {
'aucote': self.aucote
})])
return self.app
