def test_ip_obj() -> None: """Test ip handling""" api = act.api.Act("", None, "error") assert act.api.helpers.ip_obj("2001:67c:21e0::16") == ( "ipv6", "2001:067c:21e0:0000:0000:0000:0000:0016", ) assert act.api.helpers.ip_obj("::1") == ( "ipv6", "0000:0000:0000:0000:0000:0000:0000:0001", ) assert act.api.helpers.ip_obj("127.0.0.1") == ("ipv4", "127.0.0.1") assert act.api.helpers.ip_obj("127.000.00.01") == ("ipv4", "127.0.0.1") with pytest.raises(ValueError): assert act.api.helpers.ip_obj("x.y.z") == ("ipv4", "x.y.x") with pytest.raises(ValueError): assert act.api.helpers.ip_obj("300.300.300.300") == ("ipv4", "x.y.x") assert api.fact("resolvesTo").source("fqdn", "localhost").destination( "ipv4", "127.0.0.1") == api.fact("resolvesTo").source( "fqdn", "localhost").destination(*act.api.helpers.ip_obj("127.0.0.1"))
def process(api: act.api.Act, pdns_baseurl: str, apikey: str, timeout: int = 299, proxy_string: Optional[Text] = None, output_format: Text = "json", batch_size: int = 1000, limit: int = 0) -> None: """Read queries from stdin, resolve each one through passivedns printing generic_uploader data to stdout""" for query in sys.stdin: query = query.strip() if not query: continue i = 0 for row in pdns_query(pdns_baseurl, apikey, timeout=timeout, query=query, proxy_string=proxy_string, batch_size=batch_size, limit=limit): rrtype = row["rrtype"] i += 1 if limit == i: if kind(query) in (IPv4, IPv6): act.api.helpers.handle_fact(api.fact( "excessive", "resolvesTo").source( *act.api.helpers.ip_obj(row["answer"])), output_format=output_format) else: act.api.helpers.handle_fact(api.fact( "excessive", "resolvesTo").source("fqdn", query), output_format=output_format) if rrtype in ("a", "aaaa"): act.api.helpers.handle_fact(api.fact("resolvesTo").source( "fqdn", row["query"]).destination( *act.api.helpers.ip_obj(row["answer"])), output_format=output_format) elif rrtype == "cname": act.api.helpers.handle_fact(api.fact("resolvesTo").source( "fqdn", row["query"]).destination("fqdn", row["answer"]), output_format=output_format) elif rrtype == "ptr": pass # We do not insert ptr to act else: warning("Unsupported rrtype: %s: %s" % (rrtype, row))
def test_add_uri_ipv6() -> None: # type: ignore """ Test for extraction of facts from uri with ipv4 """ api = act.api.Act("", None, "error") uri = "http://[2001:67c:21e0::16]" facts = act.api.helpers.uri_facts(api, uri) assert len(facts) == 2 assert api.fact("scheme", "http").source("uri", uri) in facts assert api.fact("componentOf").source("ipv6", "2001:067c:21e0:0000:0000:0000:0000:0016").destination("uri", uri) \ in facts
def test_add_uri_fqdn() -> None: # type: ignore """ Test for extraction of facts from uri with fqdn """ api = act.api.Act("", None, "error") uri = "http://www.mnemonic.no/home" facts = act.api.helpers.uri_facts(api, uri) assert len(facts) == 4 assert api.fact("componentOf").source("fqdn", "www.mnemonic.no").destination("uri", uri) \ in facts assert api.fact("componentOf").source("path", "/home").destination("uri", uri) in facts assert api.fact("scheme", "http").source("uri", uri) in facts assert api.fact("basename", "home").source("path", "/home") in facts
def test_add_uri_ipv4() -> None: # type: ignore """ Test for extraction of facts from uri with ipv4 """ api = act.api.Act("", None, "error") uri = "http://127.0.0.1:8080/home" facts = act.api.helpers.uri_facts(api, uri) assert len(facts) == 5 assert api.fact("componentOf").source("ipv4", "127.0.0.1").destination("uri", uri) in facts assert api.fact("componentOf").source("path", "/home").destination("uri", uri) in facts assert api.fact("scheme", "http").source("uri", uri) in facts assert api.fact("basename", "home").source("path", "/home") in facts assert api.fact("port", "8080").source("uri", uri) in facts
def process(api: act.api.Act, shorteners: List[Text], user_agent: Text, proxies: Dict[Text, Text], output_format: Text = "json") -> None: """Read queries from stdin, resolve each one through passivedns printing generic_uploader data to stdout""" for query in sys.stdin: query = query.strip() if not query: continue n = 0 while True: redirect = check_redirect(query, shorteners, user_agent, proxies) if redirect == query or n > MAX_RECURSIVE: break n += 1 act.api.helpers.handle_uri(api, query, output_format=output_format) act.api.helpers.handle_uri(api, redirect, output_format=output_format) act.api.helpers.handle_fact(api.fact("redirectsTo").source( "uri", query).destination("uri", redirect), output_format=output_format) query = redirect
def process( api: act.api.Act, pdns_baseurl: str, apikey: str, timeout: int = 299, proxy_string: Optional[Text] = None, output_format: Text = "json") -> None: """Read queries from stdin, resolve each one through passivedns printing generic_uploader data to stdout""" for query in sys.stdin: query = query.strip() if not query: continue for row in pdns_query(pdns_baseurl, apikey, timeout=timeout, query=query, proxy_string=proxy_string): rrtype = row["rrtype"] if rrtype in RRTYPE_M: act.api.helpers.handle_fact( api.fact(RRTYPE_M[rrtype]["fact_t"], RRTYPE_M[rrtype]["fact_v"]) .source(RRTYPE_M[rrtype]["source_t"], row["query"]) .destination(RRTYPE_M[rrtype]["dest_t"], row["answer"]), output_format=output_format) elif rrtype == "ptr": pass # We do not insert ptr to act else: warning("Unsupported rrtype: %s: %s" % (rrtype, row))
def process(api: act.api.Act, proxy=None, output_format: Text = "json") -> None: """Read queries from stdin""" for query in sys.stdin: query = query.strip() if not query: continue for asn, network in lookup_ip(query, proxy): act.api.helpers.handle_fact(api.fact('memberOf').source( 'ipv4', query).destination('ipv4Network', network), output_format=output_format) act.api.helpers.handle_fact(api.fact('memberOf').source( 'ipv4Network', network).destination('asn', asn), output_format=output_format)
def test_validator_no_validator(caplog) -> None: api = act.api.Act("", None) act.api.helpers.handle_fact.cache_clear() # Should return None if fact does not validate fact = handle_fact( api.fact("mentions").source("report", "xyz").destination("uri", "X7f://cve-2014-0224")) # No validator is specified so the above should return a fact assert fact is not None # Should not log errors assert caplog.text == ""
def handle_alias(api: act.api.Act, tool1: Text, tool2: Text, submit: bool, output_format: Text = "json"): try: fact = api.fact("alias") \ .bidirectional("tool", tool1, "tool", tool2) if submit: handle_fact(fact) elif output_format == "json": print(fact.json()) else: print(fact) except act.api.base.ResponseError as e: error("Unable to create linked fact: %s" % e)
def test_validate_same_object() -> None: api = act.api.Act( "", None, strict_validator=True, object_formatter=object_format, object_validator=object_validates, ) act.api.helpers.handle_fact.cache_clear() with pytest.raises(ValidationError, match=r"Source object can not be equal to.*"): handle_fact( api.fact("mentions").source("report", "xyz").destination("report", "xyz"))
def test_format() -> None: api = act.api.Act( "", None, object_formatter=object_format, object_validator=object_validates, ) act.api.helpers.handle_fact.cache_clear() ta_alias = handle_fact( api.fact("alias").source("threatActor", "APT29").destination("threatActor", "Cozy Bear")) assert ta_alias.source_object.value == "apt29" assert ta_alias.destination_object.value == "cozy bear"
def test_validator_strict() -> None: api = act.api.Act( "", None, strict_validator=True, object_formatter=object_format, object_validator=object_validates, ) act.api.helpers.handle_fact.cache_clear() with pytest.raises(ValidationError, match=r"Destination object does not validate.*"): handle_fact( api.fact("mentions").source("report", "xyz").destination( "uri", ".X7f://cve-2014-0224"))
def test_validator_no_strict(caplog) -> None: api = act.api.Act( "", None, object_formatter=object_format, object_validator=object_validates, ) # Should return None if fact does not validate fact = handle_fact( api.fact("mentions").source("report", "xyz").destination("uri", "X7f://cve-2014-0224")) assert fact is None assert "Destination object does not validate:" in caplog.text
def test_add_uri_ipv6_with_port_path_query() -> None: # type: ignore """ Test for extraction of facts from uri with ipv6, path and query """ api = act.api.Act("", None, "error") uri = "http://[2001:67c:21e0::16]:8080/path?q=a" facts = act.api.helpers.uri_facts(api, uri) assert len(facts) == 6 assert api.fact("scheme", "http").source("uri", uri) in facts assert api.fact("componentOf").source("ipv6", "2001:067c:21e0:0000:0000:0000:0000:0016").destination("uri", uri) \ in facts assert api.fact("port", "8080").source("uri", uri) in facts assert api.fact("componentOf").source("path", "/path").destination("uri", uri) in facts assert api.fact("basename", "path").source("path", "/path") in facts assert api.fact("componentOf").source("query", "q=a").destination("uri", uri) in facts
def process( api: act.api.Act, pdns_baseurl: str, apikey: str, timeout: int = 299, proxy_string: Optional[Text] = None, output_format: Text = "json", batch_size: int = 1000, limit: int = 0, no_tlp_access_mode: bool = False, ) -> None: """Read queries from stdin, resolve each one through passivedns printing generic_uploader data to stdout""" for query in sys.stdin: query = query.strip() if not query: continue i = 0 for row in pdns_query( pdns_baseurl, apikey, timeout=timeout, query=query, proxy_string=proxy_string, batch_size=batch_size, limit=limit, ): rrtype = row["rrtype"] if no_tlp_access_mode: # Use Default Access Mode access_mode = api.config.access_mode else: # Set Access Mode based on TLP access_mode = ("Public" if row.get("tlp") in ("green", "white") else "RoleBased") i += 1 if limit == i: if kind(query) in (IPv4, IPv6): act.api.helpers.handle_fact( api.fact("excessive", "resolvesTo", access_mode=access_mode).source( *act.api.helpers.ip_obj(row["answer"])), output_format=output_format, ) else: act.api.helpers.handle_fact( api.fact("excessive", "resolvesTo", access_mode=access_mode).source( "fqdn", query), output_format=output_format, ) if row["query"] == row["answer"]: warning(f'{row["query"]} resolves to itself, skipping: {row}') continue if rrtype in ("a", "aaaa"): act.api.helpers.handle_fact( api.fact("resolvesTo", access_mode=access_mode).source( "fqdn", row["query"]).destination( *act.api.helpers.ip_obj(row["answer"])), output_format=output_format, ) elif rrtype == "cname": act.api.helpers.handle_fact( api.fact("resolvesTo", access_mode=access_mode).source( "fqdn", row["query"]).destination("fqdn", row["answer"]), output_format=output_format, ) elif rrtype == "ptr": pass # We do not insert ptr to act else: warning("Unsupported rrtype: %s: %s" % (rrtype, row))
def test_scio2_facts(capsys) -> None: # type: ignore """Test for scio2 facts, by comparing to captue of stdout""" with open("test/scio-doc.json") as scio_doc: doc = json.loads(scio_doc.read()) api = act.api.Act( "", None, "error", strict_validator=True, object_formatter=object_format, object_validator=object_validates, ) act.api.helpers.handle_fact.cache_clear() scio.add_to_act(api, doc, output_format="str") captured = capsys.readouterr() facts = set(captured.out.split("\n")) report_id = doc["hexdigest"] sha256 = doc["indicators"]["sha256"][0] uri = doc["indicators"]["uri"][0] # "http://www.us-cert.gov/tlp." fact_assertions = [ api.fact("name", "TA18-149A.stix.xml").source("report", report_id), api.fact("mentions") .source("report", report_id) .destination("ipv4", "187.127.112.60"), api.fact("mentions") .source("report", report_id) .destination("ipv6", "0000:0000:0000:0000:0000:0000:0000:0001"), api.fact("mentions") .source("report", report_id) .destination("hash", "4613f51087f01715bf9132c704aea2c2"), api.fact("mentions").source("report", report_id).destination("hash", sha256), api.fact("mentions") .source("report", report_id) .destination("country", "Colombia"), api.fact("mentions").source("report", report_id).destination("uri", uri), api.fact("represents") .source("report", report_id) .destination("content", report_id), api.fact("at").source("content", report_id).destination("uri", doc["uri"]), api.fact("componentOf") .source("fqdn", "www.us-cert.gov") .destination("uri", uri), api.fact("componentOf").source("path", "/tlp.").destination("uri", uri), api.fact("scheme", "http").source("uri", uri), api.fact("mentions").source("report", report_id).destination("tool", "cobra"), api.fact("mentions") .source("report", report_id) .destination("threatActor", "hidden cobra"), api.fact("mentions") .source("report", report_id) .destination("sector", "finance"), api.fact("mentions") .source("report", report_id) .destination("uri", "email://[email protected]"), api.fact("mentions") .source("report", report_id) .destination("ipv4Network", "192.168.0.0/16"), api.fact("represents").source("hash", sha256).destination("content", sha256), api.fact("mentions") .source("report", report_id) .destination("vulnerability", "cve-2019-222"), api.fact("mentions") .source("report", report_id) .destination("vulnerability", "ms16-034"), api.fact("mentions") .source("report", report_id) .destination("technique", "T1055"), api.fact("mentions") .source("report", report_id) .destination("technique", "T1547.001"), api.fact("mentions") .source("report", report_id) .destination("tactic", "TA0003"), ] for fact_assertion in fact_assertions: if not str(fact_assertion) in facts: print(f"{fact_assertion} is missing") assert str(fact_assertion) in facts
def test_scio_facts(capsys) -> None: # type: ignore """ Test for scio facts, by comparing to captue of stdout """ with open("test/scio-doc.json") as scio_doc: doc = json.loads(scio_doc.read()) api = act.api.Act("", None, "error") scio.add_to_act(api, doc, output_format="str") captured = capsys.readouterr() facts = set(captured.out.split("\n")) report_id = doc["hexdigest"] sha256 = doc["indicators"]["sha256"][0] uri = doc["indicators"]["uri"][0] # "http://www.us-cert.gov/tlp." fact_assertions = [ api.fact("name", "TA18-149A.stix.xml").source("report", report_id), api.fact("mentions", "ipv4").source("report", report_id).destination("ipv4", "187.127.112.60"), api.fact("mentions", "hash").source("report", report_id).destination( "hash", "4613f51087f01715bf9132c704aea2c2"), api.fact("mentions", "hash").source("report", report_id).destination("hash", sha256), api.fact("mentions", "country").source("report", report_id).destination( "country", "Colombia"), api.fact("mentions", "uri").source("report", report_id).destination("uri", uri), api.fact("componentOf").source("fqdn", "www.us-cert.gov").destination( "uri", uri), api.fact("componentOf").source("path", "/tlp.").destination("uri", uri), api.fact("scheme", "http").source("uri", uri), api.fact("mentions", "tool").source("report", report_id).destination("tool", "kore"), api.fact("mentions", "email").source("report", report_id).destination( "email", "*****@*****.**"), api.fact("mentions", "ipv4Network").source("report", report_id).destination( "ipv4Network", "192.168.0.0/16"), api.fact("represents").source("hash", sha256).destination("content", sha256) ] for fact_assertion in fact_assertions: assert str(fact_assertion) in facts
def test_argus_case_facts(capsys, caplog) -> None: # type: ignore """ Test for argus case facts, by comparing to captue of stdout """ with open("test/data/argus-event.json") as argus_event: event = json.loads(argus_event.read()) api = act.api.Act("", None, "error") argus.handle_argus_event(api, event, content_props=["file.sha256", "process.sha256"], hash_props=[ "file.md5", "process.md5", "file.sha1", "process.sha1", "file.sha512", "process.sha512" ], output_format="str") captured = capsys.readouterr() facts = set(captured.out.split("\n")) logs = [rec.message for rec in caplog.records] print(captured.out) prop = event["properties"] uri = event["uri"] incident_id = "ARGUS-{}".format(event["associatedCase"]["id"]) event_id = "ARGUS-{}".format(event["id"]) signature = event["attackInfo"]["signature"] # Fact chain from md5 hash through content to alert md5_chain = act.api.fact.fact_chain( api.fact("represents").source("hash", prop["file.md5"]).destination( "content", "*"), api.fact("observedIn", "event").source("content", "*").destination("event", event_id)) sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" fact_assertions = [ api.fact("attributedTo", "incident").source("event", event_id).destination( "incident", incident_id), api.fact("observedIn", "event").source("content", sha256).destination("event", event_id), api.fact("detects", "event").source("signature", signature).destination("event", event_id), api.fact("name", "Infected host").source("incident", incident_id), api.fact("observedIn", "event").source("uri", uri).destination("event", event_id), api.fact("componentOf").source("fqdn", "test-domain.com").destination( "uri", uri), api.fact("componentOf").source("path", "/path.cgi").destination("uri", uri), api.fact("scheme", "http").source("uri", uri), api.fact("observedIn", "event").source("uri", "tcp://1.2.3.4").destination( "event", event_id), ] fact_negative_assertions = [ # This fact should not exist, since we only add IPs with public addresses api.fact("observedIn", "event").source("uri", "tcp://192.168.1.1").destination( "event", event_id), # We have URI, so this should not be constructed from the fqdn api.fact("observedIn", "event").source("uri", "tcp://test-domain.com").destination( "event", event_id), # Not valid content hash (sha256) api.fact("observedIn", "event").source("content", "bogus").destination("event", event_id), ] assert 'Illegal sha256: "bogus" in property "file.sha256"' in logs for fact_assertion in fact_assertions: assert str(fact_assertion) in facts for fact_assertion in fact_negative_assertions: assert str(fact_assertion) not in facts for fact_assertion in md5_chain: assert str(fact_assertion) in facts
def test_argus_case_facts(capsys, caplog) -> None: # type: ignore """Test for argus case facts, by comparing to captue of stdout""" with open("test/data/argus-event.json") as argus_event: event = json.loads(argus_event.read()) api = act.api.Act( "", None, "error", strict_validator=True, object_formatter=object_format, object_validator=object_validates, ) act.api.helpers.handle_fact.cache_clear() argus.handle_argus_event( api, event, content_props=["file.sha256", "process.sha256"], hash_props=[ "file.md5", "process.md5", "file.sha1", "process.sha1", "file.sha512", "process.sha512", ], output_format="str", ) captured = capsys.readouterr() facts = set(captured.out.split("\n")) logs = [rec.message for rec in caplog.records] print(captured.out) prop = event["properties"] uri1 = event["uri"] uri2 = "http://test-domain2.com/path.cgi" uri3 = "http://test-domain3.com/abc" case_id = "ARGUS-{}".format(event["associatedCase"]["id"]) observationTime = event["startTimestamp"] signature = event["attackInfo"]["signature"] # Fact chain from md5 hash through content to incident md5_chain = act.api.fact.fact_chain( api.fact("represents") .source("hash", prop["file.md5"]) .destination("content", "*"), api.fact("observedIn").source("content", "*").destination("incident", case_id), ) # Fact chain from event through technique to tactic tactic_chain = act.api.fact.fact_chain( api.fact("observedIn") .source("technique", "*") .destination("incident", case_id), api.fact("implements") .source("technique", "*") .destination("tactic", "TA0007"), ) sha256 = "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b" fact_assertions = ( api.fact("observedIn") .source("content", sha256) .destination("incident", case_id), api.fact("name", "Infected host").source("incident", case_id), api.fact("observedIn").source("uri", uri1).destination("incident", case_id), api.fact("observedIn").source("uri", uri2).destination("incident", case_id), api.fact("observedIn").source("uri", uri3).destination("incident", case_id), api.fact("componentOf") .source("fqdn", "test-domain.com") .destination("uri", uri1), api.fact("componentOf").source("path", "/path.cgi").destination("uri", uri1), api.fact("scheme", "http").source("uri", uri1), api.fact("observedIn") .source("uri", "tcp://1.2.3.4") .destination("incident", case_id), ) # All facts should have a corresponding meta fact observationTime meta_fact_assertions = [ fact.meta("observationTime", str(observationTime)) for fact in fact_assertions + tactic_chain + md5_chain ] fact_negative_assertions = [ # signature is removed from the data model in 2.0 api.fact("detects") .source("signature", signature) .destination("incident", case_id), # This fact should not exist, since we only add IPs with public addresses api.fact("observedIn") .source("uri", "tcp://192.168.1.1") .destination("incident", case_id), # This fact should not exist, since it does not have scheme api.fact("observedIn") .source("uri", "illegal-url.com") .destination("incident", case_id), # We have URI, so this should not be constructed from the fqdn api.fact("observedIn") .source("uri", "tcp://test-domain.com") .destination("incident", case_id), # Not valid content hash (sha256) api.fact("observedIn") .source("content", "bogus") .destination("incident", case_id), ] assert 'Illegal sha256: "bogus" in property "file.sha256"' in logs for fact_assertion in fact_assertions: assert str(fact_assertion) in facts for fact_assertion in fact_negative_assertions: assert str(fact_assertion) not in facts for fact_assertion in md5_chain: assert str(fact_assertion) in facts for fact_assertion in tactic_chain: assert str(fact_assertion) in facts for fact_assertion in meta_fact_assertions: assert str(fact_assertion) in facts