def test_pipeline_hostlist_enqueue(runner, queue_factory, job_completed_factory): # pylint: disable=unused-argument """test hostlist enqueue""" next_queue = queue_factory.create( name='test queue ack scan', config=yaml_dump({'module': 'nmap', 'args': 'arg1'}), ) queue = queue_factory.create( name='test queue ipv6 dns discover', config=yaml_dump({'module': 'six_dns_discover', 'delay': '1'}), ) job_completed_factory.create( queue=queue, make_output=Path('tests/server/data/parser-six_dns_discover-job.zip').read_bytes() ) current_app.config['SNER_PLANNER']['pipelines'] = [ { 'type': 'queue', 'steps': [ {'step': 'load_job', 'queue': queue.name}, {'step': 'project_hostlist'}, {'step': 'filter_netranges', 'netranges': ['::1/128']}, {'step': 'enqueue', 'queue': next_queue.name}, {'step': 'archive_job'} ] } ] Planner(oneshot=True).run() assert Target.query.count() == 1
def test_scanning_pipeline(runner, queue_factory, job_completed_factory): # pylint: disable=unused-argument """test scanning pipeline""" queue1 = queue_factory.create(name='queue1', config=yaml_dump({'module': 'nmap', 'args': 'arg1'})) job_completed_factory.create( queue=queue1, make_output=Path('tests/server/data/parser-nmap-job.zip').read_bytes() ) queue_factory.create(name='queue2', config=yaml_dump({'module': 'nmap', 'args': 'arg2'})) queue_factory.create(name='queue3', config=yaml_dump({'module': 'nmap', 'args': 'arg3'})) current_app.config['SNER_PLANNER'] = yaml.safe_load(""" step_groups: service_scanning: - step: enqueue queue: 'queue2' - step: enqueue queue: 'queue3' pipelines: - type: queue steps: - step: load_job queue: 'queue1' - step: filter_tarpits - step: project_servicelist - step: run_group name: service_scanning - step: archive_job """) Planner(oneshot=True).run() assert Target.query.count() == 10
def test_pipeline_servicelist_enqueue(runner, queue_factory, job_completed_factory): # pylint: disable=unused-argument """test servicelist enqueue""" next_queue = queue_factory.create( name='test queue version scan', config=yaml_dump({'module': 'manymap', 'args': 'arg1'}), ) queue = queue_factory.create( name='test queue disco', config=yaml_dump({'module': 'nmap', 'args': 'arg1'}), ) job_completed_factory.create( queue=queue, make_output=Path('tests/server/data/parser-nmap-job.zip').read_bytes() ) current_app.config['SNER_PLANNER']['pipelines'] = [ { 'type': 'queue', 'steps': [ {'step': 'load_job', 'queue': queue.name}, {'step': 'project_servicelist'}, {'step': 'enqueue', 'queue': next_queue.name}, {'step': 'archive_job'} ] } ] Planner(oneshot=True).run() assert Target.query.count() == 5
def test_discovery_pipeline(runner, queue_factory): # pylint: disable=unused-argument """test discovery pipeline""" queue_factory.create(name='queue1', config=yaml_dump({'module': 'nmap', 'args': 'arg1'})) queue_factory.create(name='queue2', config=yaml_dump({'module': 'nmap', 'args': 'arg2'})) current_app.config['SNER_PLANNER'] = yaml.safe_load(""" common: home_netranges_ipv4: &home_netranges_ipv4 ['127.0.0.0/24'] step_groups: service_discovery: - step: enqueue queue: 'queue1' - step: enqueue queue: 'queue2' pipelines: - type: interval name: discover_ipv4 interval: 120days steps: - step: enumerate_ipv4 netranges: *home_netranges_ipv4 - step: run_group name: service_discovery """) Planner(oneshot=True).run() assert Target.query.count() == 2*256
def test_load_import_archive(app, queue_factory, job_completed_factory): # pylint: disable=unused-argument """test load, import, archive steps""" queue = queue_factory.create( name='test queue', config=yaml_dump({ 'module': 'nmap', 'args': 'arg1' }), ) job_completed_factory.create( queue=queue, make_output=Path('tests/server/data/parser-nmap-job.zip').read_bytes()) ctx = Context() ctx = load_job(ctx, queue.name) assert ctx.job assert len(ctx.data.hosts) == 1 assert len(ctx.data.services) == 5 ctx = import_job(ctx) assert len(Host.query.all()) == 1 assert len(Service.query.all()) == 5 ctx = archive_job(ctx) archive_path = Path( current_app.config['SNER_VAR']) / 'planner_archive' / f'{ctx.job.id}' assert archive_path.exists() # trigger stop pipeline when no job finished with pytest.raises(StopPipeline): ctx = load_job(ctx, queue.name)
def test_pipeline_rescan_services(runner, host_factory, service_factory, queue_factory): # pylint: disable=unused-argument """test rescan_services pipeline""" service_factory.create(host=host_factory.create(address='127.0.0.1')) service_factory.create(host=host_factory.create(address='::1')) queue_factory.create( name='test vscan', config=yaml_dump({'module': 'nmap', 'args': 'arg1'}), ) current_app.config['SNER_PLANNER']['pipelines'] = [ { 'type': 'generic', 'steps': [ { 'step': 'rescan_services', 'interval': '0s', 'queue': 'test vscan', } ] } ] Planner(oneshot=True).run() assert Target.query.count() == 2
def test_pipeline_import_job(runner, queue_factory, job_completed_factory): # pylint: disable=unused-argument """test import typed pipeline""" queue = queue_factory.create( name='test queue disco', config=yaml_dump({'module': 'nmap', 'args': 'arg1'}), ) job_completed_factory.create( queue=queue, make_output=Path('tests/server/data/parser-nmap-job.zip').read_bytes() ) current_app.config['SNER_PLANNER']['pipelines'] = [ { 'type': 'queue', 'steps': [ {'step': 'load_job', 'queue': queue.name}, {'step': 'import_job'}, {'step': 'archive_job'} ] } ] Planner(oneshot=True).run() assert len(Host.query.all()) == 1 assert len(Service.query.all()) == 5
def longrun_target(longrun_a, queue_factory, target_factory): # pylint: disable=redefined-outer-name """queue target fixture""" queue = queue_factory.create(name='testqueue', config=yaml_dump(longrun_a['config'])) target = target_factory.create(queue=queue, target=longrun_a['targets'][0]) yield target
class QueueFactory(BaseModelFactory): # pylint: disable=too-few-public-methods """test queue model factory""" class Meta: # pylint: disable=too-few-public-methods """test queue model factory""" model = Queue name = 'testqueue' config = yaml_dump({'module': 'dummy', 'args': '--arg1 abc --arg2'}) group_size = 1 priority = 10 active = True
def dummy_target(queue_factory, target_factory): # pylint: disable=redefined-outer-name """dummy target fixture""" config = { 'module': 'dummy', 'args': '--arg1', } queue = queue_factory.create(name='testqueue', config=yaml_dump(config)) target = target_factory.create(queue=queue, target='target1') yield target
def test_rescan_hosts(app, host_factory, queue_factory): # pylint: disable=unused-argument """test rescan_hosts""" host_factory.create(address='127.0.0.1') host_factory.create(address='::1') queue_factory.create( name='test vscan', config=yaml_dump({ 'module': 'nmap', 'args': 'arg1' }), ) ctx = rescan_hosts(Context(), '0s') assert len(ctx.data) == 2
def test_pipeline_rescan_services_largedataset(runner, queue_factory, host_factory): # pylint: disable=unused-argument """test rescan_services pipeline testing with large dataset""" logger = logging.getLogger(__name__) logger.info('lot_of_targets prepare start') queue = queue_factory.create( name='test vscan', config=yaml_dump({'module': 'nmap', 'args': 'arg1'}), ) existing_targets_count = 10**6 # bypass all db layers for performance query = 'INSERT INTO target (queue_id, target) VALUES ' + ','.join([str((queue.id, str(idx))) for idx in range(existing_targets_count)]) db.session.execute(query) logger.info('lot_of_targets prepare end') logger.info('lot_of_services prepare start') for addr in range(10): host = host_factory.create(address=str(ip_address(addr))) # bypass all db layers for performance query = 'INSERT INTO service (host_id, proto, port, tags) VALUES ' + ','.join([str((host.id, 'tcp', str(idx), '{}')) for idx in range(64000)]) db.session.execute(query) logging.info('prepared %s', host) logger.info('lot_of_services prepare end') db.session.expire_all() current_app.config['SNER_PLANNER']['pipelines'] = [ { 'type': 'generic', 'steps': [ { 'step': 'rescan_services', 'interval': '0s', 'queue': 'test vscan', } ] } ] Planner(oneshot=True).run() assert Target.query.count() == existing_targets_count + Service.query.count()
def test_import_pipeline(runner, queue_factory, job_completed_factory): # pylint: disable=unused-argument """test import pipeline""" queue = queue_factory.create(name='queue1', config=yaml_dump({'module': 'nmap', 'args': 'arg1'})) job_completed_factory.create(queue=queue, make_output=Path('tests/server/data/parser-nmap-job.zip').read_bytes()) current_app.config['SNER_PLANNER'] = yaml.safe_load(""" pipelines: - type: queue steps: - step: load_job queue: 'queue1' - step: import_job - step: archive_job """) Planner(oneshot=True).run() assert len(Host.query.all()) == 1 assert len(Service.query.all()) == 5
def initdata(): # pylint: disable=too-many-statements """put initial data to database""" # auth test data db.session.add( User(username='******', active=True, roles=['user', 'operator', 'admin'])) # scheduler test data db.session.add( Excl(family=ExclFamily.network, value='127.66.66.0/26', comment='blacklist 1')) db.session.add( Excl(family=ExclFamily.regex, value=r'^tcp://.*:22$', comment='avoid ssh')) queue = Queue(name='dev dummy', config=yaml_dump({ 'module': 'dummy', 'args': '--dummyparam 1' }), group_size=2, priority=10, active=True) db.session.add(queue) for target in range(3): db.session.add(Target(target=target, queue=queue)) db.session.add( Queue( name='pentest full syn scan', config=yaml_dump({ 'module': 'nmap', 'args': '-sS -A -p1-65535 -Pn --max-retries 3 --script-timeout 10m --min-hostgroup 20 --min-rate 900 --max-rate 1500' }), group_size=20, priority=10, )) db.session.add( Queue( name='disco syn scan top10000', config=yaml_dump({ 'module': 'nmap', 'args': '-sS --top-ports 10000 -Pn', 'timing_perhost': 4 }), group_size=1000, priority=10, )) db.session.add( Queue( name='disco ipv6 dns discover', config=yaml_dump({ 'module': 'six_dns_discover', 'delay': 1 }), group_size=1000, priority=10, )) db.session.add( Queue( name='disco ipv6 enum discover', config=yaml_dump({ 'module': 'six_enum_discover', 'rate': 100 }), group_size=5, priority=10, )) db.session.add( Queue( name='data version scan basic', config=yaml_dump({ 'module': 'manymap', 'args': '-sV --version-intensity 4 -Pn', 'delay': 5 }), group_size=50, priority=15, )) db.session.add( Queue( name='data jarm scan', config=yaml_dump({ 'module': 'jarm', 'delay': 5 }), group_size=50, priority=15, )) db.session.add( Queue( name='data script scan basic', config=yaml_dump({ 'module': 'manymap', 'args': '-sS --script default,http-headers,ldap-rootdse,ssl-cert,ssl-enum-ciphers,ssh-auth-methods --script-timeout 10m -Pn', 'delay': 5 }), group_size=50, priority=15, )) # storage test data host1 aggregable_vuln = { 'name': 'aggregable vuln', 'xtype': 'x.agg', 'severity': SeverityEnum.medium } host = Host(address='127.4.4.4', hostname='testhost.testdomain.test<script>alert(1);</script>', os='Test Linux 1', comment='a some unknown service server') db.session.add(host) db.session.add( Service(host=host, proto='tcp', port=12345, state='open:testreason', name='svcx', info='testservice banner', comment='manual testservice comment')) db.session.add(Vuln(host=host, **aggregable_vuln)) # storage test data host2 host = Host(address='127.3.3.3', hostname='testhost1.testdomain.test', os='Test Linux 2', comment='another server') db.session.add(host) db.session.add( Service(host=host, proto='tcp', port=12345, state='closed:testreason', name='svcx')) db.session.add( Vuln(host=host, name='test vulnerability', xtype='testxtype.123', severity=SeverityEnum.critical, comment='a test vulnerability comment', refs=['ref1', 'ref2'], tags=['tag1', 'tag2'])) db.session.add( Vuln(host=host, name='another test vulnerability', xtype='testxtype.124', severity=SeverityEnum.high, comment='another vulnerability comment', tags=None)) db.session.add( Vuln(host=host, name='vulnerability1', xtype='testxtype.124', severity=SeverityEnum.medium, tags=['info'])) db.session.add( Vuln(host=host, name='vulnerability2', xtype='testxtype.124', severity=SeverityEnum.low, tags=['report'])) db.session.add( Vuln(host=host, name='vulnerability2', xtype='testxtype.124', severity=SeverityEnum.info, tags=['info'])) db.session.add( Vuln(host=host, service=Service.query.first(), name='vulnerability3', xtype='testxtype.124', severity=SeverityEnum.unknown, tags=['report'])) db.session.add(Vuln(host=host, **aggregable_vuln)) db.session.add( Note(host=host, xtype='sner.testnote', data='testnote data', comment='test note comment')) db.session.commit()