def test_simple_regex_match(self):
print(Source.from_url(data_url).to_json_object())
obj = {
"scan_tag": {
"scanner": {
"name": "integration_test",
"pk": 0
},
"time": "2020-01-01T00:00:00+00:00"
},
"source": Source.from_url(data_url).to_json_object(),
"rule": rule.to_json_object()
}
self.messages.append((
obj,
"os2ds_scan_specs",
))
self.run_pipeline()
self.assertEqual(len(self.unhandled), 2)
results = {body["origin"]: body for body, _ in self.unhandled}
self.assertTrue(results["os2ds_matches"]["matched"],
"RegexRule match failed")
self.assertEqual(results["os2ds_matches"]["matches"], expected_matches,
"RegexRule match did not produce expected result")
def test_simple_regex_match(self):
print(Source.from_url(data_url).to_json_object())
obj = {
"scan_tag": "integration_test",
"source": Source.from_url(data_url).to_json_object(),
"rule": rule.to_json_object()
}
self.channel.basic_publish(exchange='',
routing_key="os2ds_scan_specs",
body=dumps(obj).encode())
messages = {}
def result_received(a, b, c, d):
body = loads(d.decode("utf-8"))
messages[body["origin"]] = body
if len(messages) == 2:
raise StopHandling()
self.channel.basic_consume("os2ds_results", result_received)
try:
self.channel.start_consuming()
except StopHandling as e:
self.assertTrue(messages["os2ds_matches"]["matched"],
"RegexRule match failed")
self.assertEqual(
messages["os2ds_matches"]["matches"], expected_matches,
"RegexRule match did not produce expected result")
def test_simple_regex_match(self):
print(Source.from_url(data_url).to_json_object())
obj = {
"scan_tag": {
"scanner": {
"name": "integration_test",
"pk": 0
},
"time": "2020-01-01T00:00:00+00:00"
},
"source": Source.from_url(data_url).to_json_object(),
"rule": rule.to_json_object()
}
self.runner.channel.basic_publish(exchange='',
routing_key="os2ds_scan_specs",
body=dumps(obj).encode())
try:
self.runner.run_consumer()
except StopHandling as e:
self.assertTrue(self.runner.messages["os2ds_matches"]["matched"],
"RegexRule match failed")
self.assertEqual(
self.runner.messages["os2ds_matches"]["matches"],
expected_matches,
"RegexRule match did not produce expected result")
def process(self, source, sm, depth=0):
if depth == 0:
self.assertIsNone(source.handle,
"{0}: unexpected backing handle".format(source))
for handle in source.handles(sm):
print("{0}{1}".format(" " * depth, handle))
guessed = Source.from_handle(handle)
computed = Source.from_handle(handle, sm)
if computed or guessed:
self.process(computed or guessed, sm, depth + 1)
elif handle.name == "url":
with handle.follow(sm).make_stream() as fp:
url = fp.read().decode("utf-8")
self.process(Source.from_url(url), sm, depth + 1)
elif handle.name == "test-vector" or isinstance(
source, DataSource):
r = handle.follow(sm)
self.assertTrue(r.check(), "check() method failed")
reported_size = r.get_size()
last_modified = r.get_last_modified()
with r.make_stream() as fp:
stream_raw = fp.read()
stream_size = len(stream_raw)
stream_content = stream_raw.decode("utf-8")
with r.make_path() as p:
with open(p, "rb") as fp:
file_raw = fp.read()
file_size = len(file_raw)
file_content = file_raw.decode("utf-8")
self.assertIsInstance(last_modified, SingleResult,
("{0}: last modification date is not a"
" SingleResult").format(handle))
self.assertIsInstance(
last_modified.value, datetime,
("{0}: last modification date value is not a"
"datetime.datetime").format(handle))
self.assertIsInstance(reported_size, SingleResult,
("{0}: resource length is not a"
" SingleResult").format(handle))
self.assertEqual(
stream_size, reported_size.value,
"{0}: model stream length invalid".format(handle))
self.assertEqual(
file_size, reported_size.value,
"{0}: model stream length invalid".format(handle))
self.assertEqual(
file_raw, stream_raw,
"{0}: model file and stream not equal".format(handle))
self.assertEqual(stream_content, self.correct_content,
"{0}: model stream invalid".format(handle))
self.assertEqual(file_content, self.correct_content,
"{0}: model file invalid".format(handle))
def test_smbc_url(self):
with SourceManager() as sm:
source = Source.from_url(
"smbc://*****:*****@samba/general")
try:
with contextlib.closing(source.handles(sm)) as c:
next(c)
except Exception:
self.skipTest("test Samba server not up (not running in CI?)")
self.process(source, sm)
def handle(self, **kwargs):
urls = kwargs['urls']
guess, summarise = kwargs['guess'], kwargs['summarise']
with SourceManager() as sm:
for i in urls:
try:
s = Source.from_url(i)
url_explorer.print_source(sm,
s,
guess=guess,
summarise=summarise)
except UnknownSchemeError:
pass
def test_simple_regex_match(self):
print(Source.from_url(data_url).to_json_object())
obj = {
"scan_tag": {
"scanner": {
"name": "integration_test",
"pk": 0
},
"time": "2020-01-01T00:00:00+00:00"
},
"source": Source.from_url(data_url).to_json_object(),
"rule": rule.to_json_object()
}
self.runner.channel.basic_publish(exchange='',
routing_key="os2ds_scan_specs",
body=dumps(obj).encode())
messages = {}
def result_received(channel, method, properties, body):
channel.basic_ack(method.delivery_tag)
body = loads(body.decode("utf-8"))
messages[body["origin"]] = body
if len(messages) == 2:
raise StopHandling()
self.runner.channel.basic_consume("os2ds_results", result_received)
try:
self.runner.run_consumer()
except StopHandling as e:
self.assertTrue(messages["os2ds_matches"]["matched"],
"RegexRule match failed")
self.assertEqual(
messages["os2ds_matches"]["matches"], expected_matches,
"RegexRule match did not produce expected result")
def test_handles_failure(self):
with self.assertRaises(Exception):
with SourceManager() as sm:
source = Source.from_url("http://example.invalid./")
with contextlib.closing(source.handles(sm)) as handles:
next(handles)
def test_invalid_url(self):
with self.assertRaises(UnknownSchemeError):
Source.from_url("Well, this just isn't a URL at all!")
def test_invalid_scheme(self):
with self.assertRaises(UnknownSchemeError):
Source.from_url("xxx-invalid://data/20")
def test_local_url(self):
with SourceManager() as sm:
self.process(Source.from_url("file://" + test_data_path), sm)