def test_ip_obj() -> None:
"""Test ip handling"""
api = act.api.Act("", None, "error")
assert act.api.helpers.ip_obj("2001:67c:21e0::16") == (
"ipv6",
"2001:067c:21e0:0000:0000:0000:0000:0016",
)
assert act.api.helpers.ip_obj("::1") == (
"ipv6",
"0000:0000:0000:0000:0000:0000:0000:0001",
)
assert act.api.helpers.ip_obj("127.0.0.1") == ("ipv4", "127.0.0.1")
assert act.api.helpers.ip_obj("127.000.00.01") == ("ipv4", "127.0.0.1")
with pytest.raises(ValueError):
assert act.api.helpers.ip_obj("x.y.z") == ("ipv4", "x.y.x")
with pytest.raises(ValueError):
assert act.api.helpers.ip_obj("300.300.300.300") == ("ipv4", "x.y.x")
assert api.fact("resolvesTo").source("fqdn", "localhost").destination(
"ipv4", "127.0.0.1") == api.fact("resolvesTo").source(
"fqdn",
"localhost").destination(*act.api.helpers.ip_obj("127.0.0.1"))
def process(api: act.api.Act,
pdns_baseurl: str,
apikey: str,
timeout: int = 299,
proxy_string: Optional[Text] = None,
output_format: Text = "json",
batch_size: int = 1000,
limit: int = 0) -> None:
"""Read queries from stdin, resolve each one through passivedns
printing generic_uploader data to stdout"""
for query in sys.stdin:
query = query.strip()
if not query:
continue
i = 0
for row in pdns_query(pdns_baseurl,
apikey,
timeout=timeout,
query=query,
proxy_string=proxy_string,
batch_size=batch_size,
limit=limit):
rrtype = row["rrtype"]
i += 1
if limit == i:
if kind(query) in (IPv4, IPv6):
act.api.helpers.handle_fact(api.fact(
"excessive", "resolvesTo").source(
*act.api.helpers.ip_obj(row["answer"])),
output_format=output_format)
else:
act.api.helpers.handle_fact(api.fact(
"excessive", "resolvesTo").source("fqdn", query),
output_format=output_format)
if rrtype in ("a", "aaaa"):
act.api.helpers.handle_fact(api.fact("resolvesTo").source(
"fqdn", row["query"]).destination(
*act.api.helpers.ip_obj(row["answer"])),
output_format=output_format)
elif rrtype == "cname":
act.api.helpers.handle_fact(api.fact("resolvesTo").source(
"fqdn", row["query"]).destination("fqdn", row["answer"]),
output_format=output_format)
elif rrtype == "ptr":
pass # We do not insert ptr to act
else:
warning("Unsupported rrtype: %s: %s" % (rrtype, row))
def test_add_uri_ipv6() -> None: # type: ignore
""" Test for extraction of facts from uri with ipv4 """
api = act.api.Act("", None, "error")
uri = "http://[2001:67c:21e0::16]"
facts = act.api.helpers.uri_facts(api, uri)
assert len(facts) == 2
assert api.fact("scheme", "http").source("uri", uri) in facts
assert api.fact("componentOf").source("ipv6", "2001:067c:21e0:0000:0000:0000:0000:0016").destination("uri", uri) \
in facts
def test_add_uri_fqdn() -> None: # type: ignore
""" Test for extraction of facts from uri with fqdn """
api = act.api.Act("", None, "error")
uri = "http://www.mnemonic.no/home"
facts = act.api.helpers.uri_facts(api, uri)
assert len(facts) == 4
assert api.fact("componentOf").source("fqdn", "www.mnemonic.no").destination("uri", uri) \
in facts
assert api.fact("componentOf").source("path", "/home").destination("uri", uri) in facts
assert api.fact("scheme", "http").source("uri", uri) in facts
assert api.fact("basename", "home").source("path", "/home") in facts
def test_add_uri_ipv4() -> None: # type: ignore
""" Test for extraction of facts from uri with ipv4 """
api = act.api.Act("", None, "error")
uri = "http://127.0.0.1:8080/home"
facts = act.api.helpers.uri_facts(api, uri)
assert len(facts) == 5
assert api.fact("componentOf").source("ipv4", "127.0.0.1").destination("uri", uri) in facts
assert api.fact("componentOf").source("path", "/home").destination("uri", uri) in facts
assert api.fact("scheme", "http").source("uri", uri) in facts
assert api.fact("basename", "home").source("path", "/home") in facts
assert api.fact("port", "8080").source("uri", uri) in facts
def process(api: act.api.Act,
shorteners: List[Text],
user_agent: Text,
proxies: Dict[Text, Text],
output_format: Text = "json") -> None:
"""Read queries from stdin, resolve each one through passivedns printing
generic_uploader data to stdout"""
for query in sys.stdin:
query = query.strip()
if not query:
continue
n = 0
while True:
redirect = check_redirect(query, shorteners, user_agent, proxies)
if redirect == query or n > MAX_RECURSIVE:
break
n += 1
act.api.helpers.handle_uri(api, query, output_format=output_format)
act.api.helpers.handle_uri(api,
redirect,
output_format=output_format)
act.api.helpers.handle_fact(api.fact("redirectsTo").source(
"uri", query).destination("uri", redirect),
output_format=output_format)
query = redirect
def process(
api: act.api.Act,
pdns_baseurl: str,
apikey: str,
timeout: int = 299,
proxy_string: Optional[Text] = None,
output_format: Text = "json") -> None:
"""Read queries from stdin, resolve each one through passivedns
printing generic_uploader data to stdout"""
for query in sys.stdin:
query = query.strip()
if not query:
continue
for row in pdns_query(pdns_baseurl, apikey, timeout=timeout, query=query, proxy_string=proxy_string):
rrtype = row["rrtype"]
if rrtype in RRTYPE_M:
act.api.helpers.handle_fact(
api.fact(RRTYPE_M[rrtype]["fact_t"],
RRTYPE_M[rrtype]["fact_v"])
.source(RRTYPE_M[rrtype]["source_t"], row["query"])
.destination(RRTYPE_M[rrtype]["dest_t"], row["answer"]), output_format=output_format)
elif rrtype == "ptr":
pass # We do not insert ptr to act
else:
warning("Unsupported rrtype: %s: %s" % (rrtype, row))
def process(api: act.api.Act,
proxy=None,
output_format: Text = "json") -> None:
"""Read queries from stdin"""
for query in sys.stdin:
query = query.strip()
if not query:
continue
for asn, network in lookup_ip(query, proxy):
act.api.helpers.handle_fact(api.fact('memberOf').source(
'ipv4', query).destination('ipv4Network', network),
output_format=output_format)
act.api.helpers.handle_fact(api.fact('memberOf').source(
'ipv4Network', network).destination('asn', asn),
output_format=output_format)
def test_validator_no_validator(caplog) -> None:
api = act.api.Act("", None)
act.api.helpers.handle_fact.cache_clear()
# Should return None if fact does not validate
fact = handle_fact(
api.fact("mentions").source("report",
"xyz").destination("uri",
"X7f://cve-2014-0224"))
# No validator is specified so the above should return a fact
assert fact is not None
# Should not log errors
assert caplog.text == ""
def handle_alias(api: act.api.Act,
tool1: Text,
tool2: Text,
submit: bool,
output_format: Text = "json"):
try:
fact = api.fact("alias") \
.bidirectional("tool", tool1, "tool", tool2)
if submit:
handle_fact(fact)
elif output_format == "json":
print(fact.json())
else:
print(fact)
except act.api.base.ResponseError as e:
error("Unable to create linked fact: %s" % e)
def test_validate_same_object() -> None:
api = act.api.Act(
"",
None,
strict_validator=True,
object_formatter=object_format,
object_validator=object_validates,
)
act.api.helpers.handle_fact.cache_clear()
with pytest.raises(ValidationError,
match=r"Source object can not be equal to.*"):
handle_fact(
api.fact("mentions").source("report",
"xyz").destination("report", "xyz"))
def test_format() -> None:
api = act.api.Act(
"",
None,
object_formatter=object_format,
object_validator=object_validates,
)
act.api.helpers.handle_fact.cache_clear()
ta_alias = handle_fact(
api.fact("alias").source("threatActor",
"APT29").destination("threatActor",
"Cozy Bear"))
assert ta_alias.source_object.value == "apt29"
assert ta_alias.destination_object.value == "cozy bear"
def test_validator_strict() -> None:
api = act.api.Act(
"",
None,
strict_validator=True,
object_formatter=object_format,
object_validator=object_validates,
)
act.api.helpers.handle_fact.cache_clear()
with pytest.raises(ValidationError,
match=r"Destination object does not validate.*"):
handle_fact(
api.fact("mentions").source("report", "xyz").destination(
"uri", ".X7f://cve-2014-0224"))
def test_validator_no_strict(caplog) -> None:
api = act.api.Act(
"",
None,
object_formatter=object_format,
object_validator=object_validates,
)
# Should return None if fact does not validate
fact = handle_fact(
api.fact("mentions").source("report",
"xyz").destination("uri",
"X7f://cve-2014-0224"))
assert fact is None
assert "Destination object does not validate:" in caplog.text
def test_add_uri_ipv6_with_port_path_query() -> None: # type: ignore
""" Test for extraction of facts from uri with ipv6, path and query """
api = act.api.Act("", None, "error")
uri = "http://[2001:67c:21e0::16]:8080/path?q=a"
facts = act.api.helpers.uri_facts(api, uri)
assert len(facts) == 6
assert api.fact("scheme", "http").source("uri", uri) in facts
assert api.fact("componentOf").source("ipv6", "2001:067c:21e0:0000:0000:0000:0000:0016").destination("uri", uri) \
in facts
assert api.fact("port", "8080").source("uri", uri) in facts
assert api.fact("componentOf").source("path", "/path").destination("uri", uri) in facts
assert api.fact("basename", "path").source("path", "/path") in facts
assert api.fact("componentOf").source("query", "q=a").destination("uri", uri) in facts
def process(
api: act.api.Act,
pdns_baseurl: str,
apikey: str,
timeout: int = 299,
proxy_string: Optional[Text] = None,
output_format: Text = "json",
batch_size: int = 1000,
limit: int = 0,
no_tlp_access_mode: bool = False,
) -> None:
"""Read queries from stdin, resolve each one through passivedns
printing generic_uploader data to stdout"""
for query in sys.stdin:
query = query.strip()
if not query:
continue
i = 0
for row in pdns_query(
pdns_baseurl,
apikey,
timeout=timeout,
query=query,
proxy_string=proxy_string,
batch_size=batch_size,
limit=limit,
):
rrtype = row["rrtype"]
if no_tlp_access_mode:
# Use Default Access Mode
access_mode = api.config.access_mode
else:
# Set Access Mode based on TLP
access_mode = ("Public" if row.get("tlp") in ("green", "white")
else "RoleBased")
i += 1
if limit == i:
if kind(query) in (IPv4, IPv6):
act.api.helpers.handle_fact(
api.fact("excessive",
"resolvesTo",
access_mode=access_mode).source(
*act.api.helpers.ip_obj(row["answer"])),
output_format=output_format,
)
else:
act.api.helpers.handle_fact(
api.fact("excessive",
"resolvesTo",
access_mode=access_mode).source(
"fqdn", query),
output_format=output_format,
)
if row["query"] == row["answer"]:
warning(f'{row["query"]} resolves to itself, skipping: {row}')
continue
if rrtype in ("a", "aaaa"):
act.api.helpers.handle_fact(
api.fact("resolvesTo", access_mode=access_mode).source(
"fqdn", row["query"]).destination(
*act.api.helpers.ip_obj(row["answer"])),
output_format=output_format,
)
elif rrtype == "cname":
act.api.helpers.handle_fact(
api.fact("resolvesTo", access_mode=access_mode).source(
"fqdn",
row["query"]).destination("fqdn", row["answer"]),
output_format=output_format,
)
elif rrtype == "ptr":
pass # We do not insert ptr to act
else:
warning("Unsupported rrtype: %s: %s" % (rrtype, row))
def test_scio2_facts(capsys) -> None: # type: ignore
"""Test for scio2 facts, by comparing to captue of stdout"""
with open("test/scio-doc.json") as scio_doc:
doc = json.loads(scio_doc.read())
api = act.api.Act(
"",
None,
"error",
strict_validator=True,
object_formatter=object_format,
object_validator=object_validates,
)
act.api.helpers.handle_fact.cache_clear()
scio.add_to_act(api, doc, output_format="str")
captured = capsys.readouterr()
facts = set(captured.out.split("\n"))
report_id = doc["hexdigest"]
sha256 = doc["indicators"]["sha256"][0]
uri = doc["indicators"]["uri"][0] # "http://www.us-cert.gov/tlp."
fact_assertions = [
api.fact("name", "TA18-149A.stix.xml").source("report", report_id),
api.fact("mentions")
.source("report", report_id)
.destination("ipv4", "187.127.112.60"),
api.fact("mentions")
.source("report", report_id)
.destination("ipv6", "0000:0000:0000:0000:0000:0000:0000:0001"),
api.fact("mentions")
.source("report", report_id)
.destination("hash", "4613f51087f01715bf9132c704aea2c2"),
api.fact("mentions").source("report", report_id).destination("hash", sha256),
api.fact("mentions")
.source("report", report_id)
.destination("country", "Colombia"),
api.fact("mentions").source("report", report_id).destination("uri", uri),
api.fact("represents")
.source("report", report_id)
.destination("content", report_id),
api.fact("at").source("content", report_id).destination("uri", doc["uri"]),
api.fact("componentOf")
.source("fqdn", "www.us-cert.gov")
.destination("uri", uri),
api.fact("componentOf").source("path", "/tlp.").destination("uri", uri),
api.fact("scheme", "http").source("uri", uri),
api.fact("mentions").source("report", report_id).destination("tool", "cobra"),
api.fact("mentions")
.source("report", report_id)
.destination("threatActor", "hidden cobra"),
api.fact("mentions")
.source("report", report_id)
.destination("sector", "finance"),
api.fact("mentions")
.source("report", report_id)
.destination("uri", "email://[email protected]"),
api.fact("mentions")
.source("report", report_id)
.destination("ipv4Network", "192.168.0.0/16"),
api.fact("represents").source("hash", sha256).destination("content", sha256),
api.fact("mentions")
.source("report", report_id)
.destination("vulnerability", "cve-2019-222"),
api.fact("mentions")
.source("report", report_id)
.destination("vulnerability", "ms16-034"),
api.fact("mentions")
.source("report", report_id)
.destination("technique", "T1055"),
api.fact("mentions")
.source("report", report_id)
.destination("technique", "T1547.001"),
api.fact("mentions")
.source("report", report_id)
.destination("tactic", "TA0003"),
]
for fact_assertion in fact_assertions:
if not str(fact_assertion) in facts:
print(f"{fact_assertion} is missing")
assert str(fact_assertion) in facts
def test_scio_facts(capsys) -> None: # type: ignore
""" Test for scio facts, by comparing to captue of stdout """
with open("test/scio-doc.json") as scio_doc:
doc = json.loads(scio_doc.read())
api = act.api.Act("", None, "error")
scio.add_to_act(api, doc, output_format="str")
captured = capsys.readouterr()
facts = set(captured.out.split("\n"))
report_id = doc["hexdigest"]
sha256 = doc["indicators"]["sha256"][0]
uri = doc["indicators"]["uri"][0] # "http://www.us-cert.gov/tlp."
fact_assertions = [
api.fact("name", "TA18-149A.stix.xml").source("report", report_id),
api.fact("mentions",
"ipv4").source("report",
report_id).destination("ipv4",
"187.127.112.60"),
api.fact("mentions", "hash").source("report", report_id).destination(
"hash", "4613f51087f01715bf9132c704aea2c2"),
api.fact("mentions",
"hash").source("report",
report_id).destination("hash", sha256),
api.fact("mentions", "country").source("report",
report_id).destination(
"country", "Colombia"),
api.fact("mentions", "uri").source("report",
report_id).destination("uri", uri),
api.fact("componentOf").source("fqdn", "www.us-cert.gov").destination(
"uri", uri),
api.fact("componentOf").source("path",
"/tlp.").destination("uri", uri),
api.fact("scheme", "http").source("uri", uri),
api.fact("mentions",
"tool").source("report",
report_id).destination("tool", "kore"),
api.fact("mentions", "email").source("report", report_id).destination(
"email", "*****@*****.**"),
api.fact("mentions",
"ipv4Network").source("report", report_id).destination(
"ipv4Network", "192.168.0.0/16"),
api.fact("represents").source("hash",
sha256).destination("content", sha256)
]
for fact_assertion in fact_assertions:
assert str(fact_assertion) in facts
def test_argus_case_facts(capsys, caplog) -> None: # type: ignore
""" Test for argus case facts, by comparing to captue of stdout """
with open("test/data/argus-event.json") as argus_event:
event = json.loads(argus_event.read())
api = act.api.Act("", None, "error")
argus.handle_argus_event(api,
event,
content_props=["file.sha256", "process.sha256"],
hash_props=[
"file.md5", "process.md5", "file.sha1",
"process.sha1", "file.sha512",
"process.sha512"
],
output_format="str")
captured = capsys.readouterr()
facts = set(captured.out.split("\n"))
logs = [rec.message for rec in caplog.records]
print(captured.out)
prop = event["properties"]
uri = event["uri"]
incident_id = "ARGUS-{}".format(event["associatedCase"]["id"])
event_id = "ARGUS-{}".format(event["id"])
signature = event["attackInfo"]["signature"]
# Fact chain from md5 hash through content to alert
md5_chain = act.api.fact.fact_chain(
api.fact("represents").source("hash", prop["file.md5"]).destination(
"content", "*"),
api.fact("observedIn",
"event").source("content",
"*").destination("event", event_id))
sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
fact_assertions = [
api.fact("attributedTo",
"incident").source("event", event_id).destination(
"incident", incident_id),
api.fact("observedIn",
"event").source("content",
sha256).destination("event", event_id),
api.fact("detects",
"event").source("signature",
signature).destination("event", event_id),
api.fact("name", "Infected host").source("incident", incident_id),
api.fact("observedIn",
"event").source("uri", uri).destination("event", event_id),
api.fact("componentOf").source("fqdn", "test-domain.com").destination(
"uri", uri),
api.fact("componentOf").source("path",
"/path.cgi").destination("uri", uri),
api.fact("scheme", "http").source("uri", uri),
api.fact("observedIn", "event").source("uri",
"tcp://1.2.3.4").destination(
"event", event_id),
]
fact_negative_assertions = [
# This fact should not exist, since we only add IPs with public addresses
api.fact("observedIn",
"event").source("uri", "tcp://192.168.1.1").destination(
"event", event_id),
# We have URI, so this should not be constructed from the fqdn
api.fact("observedIn",
"event").source("uri", "tcp://test-domain.com").destination(
"event", event_id),
# Not valid content hash (sha256)
api.fact("observedIn",
"event").source("content",
"bogus").destination("event", event_id),
]
assert 'Illegal sha256: "bogus" in property "file.sha256"' in logs
for fact_assertion in fact_assertions:
assert str(fact_assertion) in facts
for fact_assertion in fact_negative_assertions:
assert str(fact_assertion) not in facts
for fact_assertion in md5_chain:
assert str(fact_assertion) in facts
def test_argus_case_facts(capsys, caplog) -> None: # type: ignore
"""Test for argus case facts, by comparing to captue of stdout"""
with open("test/data/argus-event.json") as argus_event:
event = json.loads(argus_event.read())
api = act.api.Act(
"",
None,
"error",
strict_validator=True,
object_formatter=object_format,
object_validator=object_validates,
)
act.api.helpers.handle_fact.cache_clear()
argus.handle_argus_event(
api,
event,
content_props=["file.sha256", "process.sha256"],
hash_props=[
"file.md5",
"process.md5",
"file.sha1",
"process.sha1",
"file.sha512",
"process.sha512",
],
output_format="str",
)
captured = capsys.readouterr()
facts = set(captured.out.split("\n"))
logs = [rec.message for rec in caplog.records]
print(captured.out)
prop = event["properties"]
uri1 = event["uri"]
uri2 = "http://test-domain2.com/path.cgi"
uri3 = "http://test-domain3.com/abc"
case_id = "ARGUS-{}".format(event["associatedCase"]["id"])
observationTime = event["startTimestamp"]
signature = event["attackInfo"]["signature"]
# Fact chain from md5 hash through content to incident
md5_chain = act.api.fact.fact_chain(
api.fact("represents")
.source("hash", prop["file.md5"])
.destination("content", "*"),
api.fact("observedIn").source("content", "*").destination("incident", case_id),
)
# Fact chain from event through technique to tactic
tactic_chain = act.api.fact.fact_chain(
api.fact("observedIn")
.source("technique", "*")
.destination("incident", case_id),
api.fact("implements")
.source("technique", "*")
.destination("tactic", "TA0007"),
)
sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
fact_assertions = (
api.fact("observedIn")
.source("content", sha256)
.destination("incident", case_id),
api.fact("name", "Infected host").source("incident", case_id),
api.fact("observedIn").source("uri", uri1).destination("incident", case_id),
api.fact("observedIn").source("uri", uri2).destination("incident", case_id),
api.fact("observedIn").source("uri", uri3).destination("incident", case_id),
api.fact("componentOf")
.source("fqdn", "test-domain.com")
.destination("uri", uri1),
api.fact("componentOf").source("path", "/path.cgi").destination("uri", uri1),
api.fact("scheme", "http").source("uri", uri1),
api.fact("observedIn")
.source("uri", "tcp://1.2.3.4")
.destination("incident", case_id),
)
# All facts should have a corresponding meta fact observationTime
meta_fact_assertions = [
fact.meta("observationTime", str(observationTime))
for fact in fact_assertions + tactic_chain + md5_chain
]
fact_negative_assertions = [
# signature is removed from the data model in 2.0
api.fact("detects")
.source("signature", signature)
.destination("incident", case_id),
# This fact should not exist, since we only add IPs with public addresses
api.fact("observedIn")
.source("uri", "tcp://192.168.1.1")
.destination("incident", case_id),
# This fact should not exist, since it does not have scheme
api.fact("observedIn")
.source("uri", "illegal-url.com")
.destination("incident", case_id),
# We have URI, so this should not be constructed from the fqdn
api.fact("observedIn")
.source("uri", "tcp://test-domain.com")
.destination("incident", case_id),
# Not valid content hash (sha256)
api.fact("observedIn")
.source("content", "bogus")
.destination("incident", case_id),
]
assert 'Illegal sha256: "bogus" in property "file.sha256"' in logs
for fact_assertion in fact_assertions:
assert str(fact_assertion) in facts
for fact_assertion in fact_negative_assertions:
assert str(fact_assertion) not in facts
for fact_assertion in md5_chain:
assert str(fact_assertion) in facts
for fact_assertion in tactic_chain:
assert str(fact_assertion) in facts
for fact_assertion in meta_fact_assertions:
assert str(fact_assertion) in facts
