def test_run_scan(self):
"""
Test running a scan.
"""
# create organisations
self.magenta_org = Organization(name="Magenta", pk=1)
self.magenta_org.save()
self.theydontwantyouto_org = Organization(name="TheyDontWantYouTo", pk=2)
self.theydontwantyouto_org.save()
# create scanners
self.magenta_scanner = WebScanner(
name="Magenta",
url="http://magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
)
self.theydontwantyouto_scanner = WebScanner(
name="TheyDontWantYouTo",
url="http://theydontwantyou.to",
organization=self.theydontwantyouto_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
)
self.magenta_scanner.save()
self.theydontwantyouto_scanner.save()
# create Rules and rulesets
self.reg1 = RegexPattern(pattern_string='fællesskaber', pk=1)
self.reg2 = RegexPattern(pattern_string='Ombudsmand', pk=2)
self.reg3 = RegexPattern(pattern_string='projektnetwerk', pk=3)
self.reg4 = RegexPattern(pattern_string='secure', pk=4)
self.reg5 = RegexPattern(pattern_string='control', pk=5)
self.reg6 = RegexPattern(pattern_string='breathe', pk=6)
self.reg1.save()
self.reg2.save()
self.reg3.save()
self.reg4.save()
self.reg5.save()
self.reg6.save()
# Create rule sets
self.tr_set1 = RegexRule(name='MagentaTestRule1', sensitivity=Sensitivity.OK, organization=self.magenta_org,
pk=1)
self.tr_set2 = RegexRule(name='TheyDontWantYouToKnow', sensitivity=Sensitivity.OK,
organization=self.theydontwantyouto_org, pk=2)
self.tr_set1.save()
self.tr_set2.save()
self.tr_set1.patterns.add(self.reg1)
self.tr_set1.patterns.add(self.reg2)
self.tr_set1.patterns.add(self.reg3)
self.tr_set2.patterns.add(self.reg4)
self.tr_set1.save()
self.tr_set2.save()
self.magenta_scanner.rules.add(self.tr_set1)
self.magenta_scanner.save()
self.magenta_scanner.run("test", blocking=True)
def test_validate_domain(self):
"""Test validating domains."""
# Make sure example.com does not validate in any of the possible
# methods
all_methods = [WebScanner.WEBSCANFILE, WebScanner.METAFIELD]
for validation_method in all_methods:
domain = WebScanner(url="http://www.example.com/",
validation_method=validation_method,
organization=self.example,
pk=2)
domain.save()
self.assertFalse(validate_domain(domain))
def test_run_scan(self):
"""Test running a scan."""
scanner = WebScanner(url="http://www.magenta.dk/",
organization=self.magenta,
validation_method=WebScanner.ROBOTSTXT,
validation_status=1, schedule="")
scanner.save()
scan = scanner.run('kaflaflibob')
self.assertIsInstance(scan, Scan)
# we have no scanner manager
self.assertEqual(scan.status, scan.NEW)
self.assertFalse(scanner.is_running)
def test_validate_domain(self):
"""Test validating domains."""
# Make sure example.com does not validate in any of the possible
# methods
all_methods = [WebScanner.WEBSCANFILE, WebScanner.METAFIELD]
# Make sure Magenta's website validates using all possible methods
# Magenta's website is under re-construction.
"""for validation_method in [WebDomain.WEBSCANFILE, WebDomain.METAFIELD]:
domain = WebDomain(url="http://www.magenta.dk/",
validation_method=validation_method,
organization=self.magenta,
pk=1)
domain.save()
print("VALIDATING", validation_method)
self.assertTrue(validate_domain(domain))"""
for validation_method in all_methods:
domain = WebScanner(url="http://www.example.com/",
validation_method=validation_method,
organization=self.example,
pk=2)
domain.save()
self.assertFalse(validate_domain(domain))
class test():
def setUp(self):
# create organisations
self.magenta_org = Organization(name="Magenta", pk=ORG_PK)
self.magenta_org.save()
# create scanners
self.magenta_scanner = WebScanner(
name="Magenta",
url="https://www.magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
do_last_modified_check=False,
do_last_modified_check_head_request=False,
pk=SCANNER_PK,
)
self.magenta_scanner_no_recur = WebScanner(
name="Magenta_no_recurrence",
url="https://www.magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
do_last_modified_check_head_request=False,
pk=SCANNER_PK + 1,
)
self.magenta_scanner_tomorrow = WebScanner(
name="Magenta_tomorrow",
url="https://www.magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
do_last_modified_check_head_request=False,
pk=SCANNER_PK + 2,
)
self.magenta_scanner.save()
self.magenta_scanner_no_recur.save()
self.magenta_scanner_tomorrow.save()
# create regex cule
self.reg1 = RegexPattern(pattern_string="fællesskaber", pk=PATTERN_PK)
self.reg1.save()
self.tr_set1 = RegexRule(
name="MagentaTestRule1",
sensitivity=Sensitivity.OK,
organization=self.magenta_org,
pk=RULE_PK,
)
self.tr_set1.patterns.add(self.reg1)
self.tr_set1.save()
self.magenta_scanner.rules.add(self.tr_set1)
self.magenta_scanner_no_recur.rules.add(self.tr_set1)
self.magenta_scanner_tomorrow.rules.add(self.tr_set1)
self.magenta_scanner.save()
self.magenta_scanner_no_recur.save()
self.magenta_scanner_tomorrow.save()
# create recurrence. Multiple recurernces can be given as a list
tomorrow = (time_now().weekday() + 1) % 7
rrule_daily = recurrence.Rule(recurrence.DAILY)
rrule_tomorrow = recurrence.Rule(recurrence.WEEKLY, byday=tomorrow)
self.magenta_scanner.schedule.rrules = [rrule_daily]
self.magenta_scanner_tomorrow.schedule.rrules = [rrule_tomorrow]
self.magenta_scanner.save()
self.magenta_scanner_tomorrow.save()
# open queue
self.runner = PipelineTestRunner(read="os2ds_scan_specs",
write=set(),
heartbeat=6000)
def tearDown(self):
self.reg1.delete()
self.tr_set1.delete()
self.magenta_scanner.delete()
self.magenta_scanner_no_recur.delete()
self.magenta_scanner_tomorrow.delete()
self.magenta_org.delete()
self.runner.clear()
def get_models(self):
# run this to get models already existing in DB
self.magenta_scanner = WebScanner.objects.filter(pk=SCANNER_PK).get()
self.magenta_scanner_no_recur = WebScanner.objects.filter(
pk=SCANNER_PK + 1).get()
self.magenta_scanner_tomorrow = WebScanner.objects.filter(
pk=SCANNER_PK + 2).get()
self.tr_set1 = RegexRule.objects.filter(name="MagentaTestRule1").get()
self.reg1 = \
RegexPattern.objects.filter(pattern_string="fællesskaber").get()
self.magenta_org = Organization.objects.filter(pk=ORG_PK).get()
# open queue
self.runner = PipelineTestRunner(read="os2ds_scan_specs",
write=set(),
heartbeat=6000)
def test_cron(self):
# capture stdout from cron.py
# Another way is to redirect sys.stdout=buf=StringIO()
# https://stackoverflow.com/a/1218951
with StringIO() as buf, redirect_stdout(buf):
call_command("cron", **{"now": True}, stdout=buf)
self.output = buf.getvalue().rstrip()
print(f"!!!! {self.output}")
self.runner.channel.basic_consume("os2ds_scan_specs", result_received)
try:
self.runner.run_consumer()
except StopHandling as e:
print(
messages[0][0],
messages[1][0],
"Wrong scan_spec from scanner started by cron job",
)
# these checks are maybe a bit redundant...
print(self.output, "Running scanner Magenta",
"Wrong scanner were started by cron")
print(len(self.output.split('\n')), 1,
"Wrong number of scanners were started")
def setUp(self):
# create organisations
self.magenta_org = Organization(name="Magenta", pk=ORG_PK)
self.magenta_org.save()
# create scanners
self.magenta_scanner = WebScanner(
name="Magenta",
url="https://www.magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
do_last_modified_check=False,
do_last_modified_check_head_request=False,
pk=SCANNER_PK,
)
self.magenta_scanner_no_recur = WebScanner(
name="Magenta_no_recurrence",
url="https://www.magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
do_last_modified_check_head_request=False,
pk=SCANNER_PK + 1,
)
self.magenta_scanner_tomorrow = WebScanner(
name="Magenta_tomorrow",
url="https://www.magenta.dk",
organization=self.magenta_org,
validation_status=WebScanner.VALID,
download_sitemap=False,
do_last_modified_check_head_request=False,
pk=SCANNER_PK + 2,
)
self.magenta_scanner.save()
self.magenta_scanner_no_recur.save()
self.magenta_scanner_tomorrow.save()
# create regex cule
self.reg1 = RegexPattern(pattern_string="fællesskaber", pk=PATTERN_PK)
self.reg1.save()
self.tr_set1 = RegexRule(
name="MagentaTestRule1",
sensitivity=Sensitivity.OK,
organization=self.magenta_org,
pk=RULE_PK,
)
self.tr_set1.patterns.add(self.reg1)
self.tr_set1.save()
self.magenta_scanner.rules.add(self.tr_set1)
self.magenta_scanner_no_recur.rules.add(self.tr_set1)
self.magenta_scanner_tomorrow.rules.add(self.tr_set1)
self.magenta_scanner.save()
self.magenta_scanner_no_recur.save()
self.magenta_scanner_tomorrow.save()
# create recurrence. Multiple recurernces can be given as a list
tomorrow = (time_now().weekday() + 1) % 7
rrule_daily = recurrence.Rule(recurrence.DAILY)
rrule_tomorrow = recurrence.Rule(recurrence.WEEKLY, byday=tomorrow)
self.magenta_scanner.schedule.rrules = [rrule_daily]
self.magenta_scanner_tomorrow.schedule.rrules = [rrule_tomorrow]
self.magenta_scanner.save()
self.magenta_scanner_tomorrow.save()
# open queue
self.runner = PipelineTestRunner(read="os2ds_scan_specs",
write=set(),
heartbeat=6000)