def setUp(self): self.maxDiff = 10000 heartbleed = SubRecord( { # with explicit proto field indices "heartbeat_support": Boolean(pr_index=11), "heartbleed_vulnerable": Boolean(category="Vulnerabilities", pr_ignore=True), "timestamp": DateTime(pr_index=10) }, pr_index=77) self.host = Record({ "ipstr": IPv4Address(required=True, examples=["8.8.8.8"], pr_index=1), "ip": Unsigned32BitInteger(doc="The IP Address of the host", pr_index=2), Port(443): SubRecord({ "tls": String(pr_index=1), "heartbleed": heartbleed }, category="heartbleed", pr_index=3), "tags": ListOf(String(), pr_index=47) })
def setUp(self): self.host = Record({ "ipstr": IPv4Address(required=True), "ip": Unsigned32BitInteger(), }) self.domain = Record({ "domain": String(required=True), })
def test_extends(self): host = Record({ "host": IPv4Address(required=True), "time": DateTime(required=True), "data": SubRecord({}), "error": String() }) banner_grab = Record({"data": SubRecord({"banner": String()})}, extends=host) tls_banner_grab = Record({"data": SubRecord({"tls": SubRecord({})})}, extends=banner_grab) smtp_starttls = Record({"data": SubRecord({"ehlo": String()})}, extends=tls_banner_grab) valid = Record({ "host": IPv4Address(required=True), "time": DateTime(required=True), "data": SubRecord({ "banner": String(), "tls": SubRecord({}), "ehlo": String() }), "error": String() }) self.assertEqual(smtp_starttls.to_dict(), valid.to_dict())
def setUp(self): self.maxDiff = 10000 Child = SubRecordType( { "foo": Boolean(), "bar": Boolean(validation_policy="error"), }, validation_policy="error") self.record = Record({ "a": Child(validation_policy="error"), "b": Child(validation_policy="warn"), "c": Child(validation_policy="ignore"), "d": Child(validation_policy="inherit"), })
def test_parses_ipv4_records(self): ipv4_host_ssh = Record({ Port(22): SubRecord({ "ssh": SubRecord({ "banner": SubRecord({ "comment": String(), "timestamp": DateTime() }) }) }) }) ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
def test_subrecord_type_override(self): SSH = SubRecordType( { "banner": SubRecord({ "comment": String(), "timestamp": DateTime() }) }, doc="class doc", required=False) self.assertEqual(SSH.DOC, "class doc") self.assertEqual(SSH.REQUIRED, False) ssh = SSH(doc="instance doc", required=True) ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})}) self.assertEqual(ssh.doc, "instance doc") self.assertEqual(ssh.required, True) ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record')) # class unchanged self.assertEqual(SSH.DOC, "class doc") self.assertEqual(SSH.REQUIRED, False)
def test_es_dynamic_record(self): ipv4_host_with_dynamic_strict = Record() es = ipv4_host_with_dynamic_strict.to_es("strict-record") self.assertFalse("dynamic" in es["strict-record"]) ipv4_host_with_dynamic_strict = Record(es_dynamic_policy="strict") es = ipv4_host_with_dynamic_strict.to_es("strict-record") self.assertEqual("strict", es["strict-record"]["dynamic"])
def test_child_subtree_overrides_and_inherits(self): schema = Record( { Port(445): SubRecord({ "smb": SubRecord({"banner": SubRecord({"smb_v1": Boolean()})}, validation_policy="error") }) }, validation_policy="warn") doc = { "445": { "smb": { "banner": { "smb_v1": True, "metadata": {}, } } } } self.assertRaises(DataValidationException, lambda: schema.validate(doc))
class PathLogUnitTests(unittest.TestCase): sub_type = SubRecord( { "sub1": String(), "sub2": SubRecord({ "sub2sub1": Unsigned8BitInteger(), "sub2sub2": NestedListOf(String(), "sub2sub2.subrecord_name"), }), "sub3": Enum(values=["a", "b", "c"]) }, validation_policy="error") SCHEMA = Record( { "a": SubRecord({ "a1": String(), "a2": ListOf(sub_type), "a3": Unsigned8BitInteger(), }), "b": String(), }, validation_policy="error") def test_good(self): good = { "a": { "a1": "{a.a1}:good", "a2": [ { "sub1": "{a.a2[0].sub1}:good", "sub2": { "sub2sub1": 1, "sub2sub2": [ "{a.a2[0].sub2.sub2sub2[0]}:good", "{a.a2[0].sub2.sub2sub2[1]}:good", ], }, }, { "sub1": "{a.a2[1].sub1}:good", "sub2": { "sub2sub1": 1, "sub2sub2": [], }, }, ], "a3": 1, }, "b": "{b}:good", } self.SCHEMA.validate(good, policy="error") def test_bad_root(self): bad1 = { "does_not_exist": "{does_not_exist}:bad1", "a": { "a1": "{a.a1}:bad1", "a2": [ { "sub1": "{a.a2[0].sub1}:bad1", "sub2": { "sub2sub1": 1, "sub2sub2": [ "{a.a2[0].sub2.sub2sub2[0]}:bad1", "{a.a2[0].sub2.sub2sub2[1]}:bad1", ], }, }, { "sub1": "{a.a2[1].sub1}:bad1", "sub2": { "sub2sub1": 1, "sub2sub2": [], }, }, ], "a3": 1, }, "b": "{b}:bad1", } try: self.SCHEMA.validate(bad1, policy="error") self.assertTrue(False, "bad1 failed to fail") except DataValidationException as e: self.assertTrue(not e.path) del (bad1["does_not_exist"]) bad1["b"] = 23 try: self.SCHEMA.validate(bad1, policy="error") self.assertTrue(False, "bad1 failed to fail") except DataValidationException as e: self.assertEquals(e.path, ["b"]) def test_bad_a_key(self): bad = { "a": { "does_not_exist": 23, "a1": "{a.a1}:bad1", "a2": [ { "sub1": "{a.a2[0].sub1}:bad1", "sub2": { "sub2sub1": 1, "sub2sub2": [ "{a.a2[0].sub2.sub2sub2[0]}:bad1", "{a.a2[0].sub2.sub2sub2[1]}:bad1", ], }, }, { "sub1": "{a.a2[1].sub1}:bad1", "sub2": { "sub2sub1": 1, "sub2sub2": [], }, }, ], "a3": 1, }, "b": "{b}:bad1", } try: self.SCHEMA.validate(bad, policy="error") self.assertTrue(False, "bad failed to fail") except DataValidationException as e: self.assertEqual(e.path, ["a"]) del (bad["a"]["does_not_exist"]) bad["a"]["a3"] = "not an int" try: ret = self.SCHEMA.validate(bad, policy="error") self.assertTrue(False, "bad failed to fail") except DataValidationException as e: self.assertEqual(e.path, ["a", "a3"]) def test_bad_deep_key(self): bad = { "a": { "a1": "{a.a1}:bad", "a2": [ { "sub1": "{a.a2[0].sub1}:bad", "sub2": { "sub2sub1": 1, "sub2sub2": [ "{a.a2[0].sub2.sub2sub2[0]}:bad", "{a.a2[0].sub2.sub2sub2[1]}:bad", ], "does_not_exist": "fake", }, }, { "sub1": "{a.a2[1].sub1}:bad1", "sub2": { "sub2sub1": 1, "sub2sub2": [], }, }, ], "a3": 1, }, "b": "{b}:bad1", } try: self.SCHEMA.validate(bad, policy="error") self.assertTrue(False, "failed to fail") except DataValidationException as e: self.assertEqual(e.path, [ "a", "a2", 0, "sub2", ]) del (bad["a"]["a2"][0]["sub2"]["does_not_exist"]) bad["a"]["a2"][0]["sub2"]["sub2sub2"][1] = {"wrong type": "bad type"} try: self.SCHEMA.validate(bad, policy="error") self.assertTrue(False, "bad failed to fail") except DataValidationException as e: self.assertEqual(e.path, ["a", "a2", 0, "sub2", "sub2sub2", 1])
class ValidationPolicies(unittest.TestCase): def setUp(self): self.maxDiff = 10000 Child = SubRecordType( { "foo": Boolean(), "bar": Boolean(validation_policy="error"), }, validation_policy="error") self.record = Record({ "a": Child(validation_policy="error"), "b": Child(validation_policy="warn"), "c": Child(validation_policy="ignore"), "d": Child(validation_policy="inherit"), }) def test_policy_setting_warn(self): self.record.validate({"b": {"foo": "string value"}}) def test_policy_setting_ignore(self): self.record.validate({"c": {"foo": "string value"}}) def test_policy_setting_error(self): self.assertRaises( DataValidationException, lambda: self.record.validate({"c": { "bar": "string value" }})) def test_policy_setting_inherit(self): self.assertRaises( DataValidationException, lambda: self.record.validate({"a": { "foo": "string value" }})) def test_policy_setting_multi_level_inherit(self): self.assertRaises( DataValidationException, lambda: self.record.validate({"a": { "bar": "string value" }})) def test_explicit_policy(self): self.record.validate({"a": {"foo": "string value"}}, policy="ignore") def test_child_subtree_overrides_and_inherits(self): schema = Record( { Port(445): SubRecord({ "smb": SubRecord({"banner": SubRecord({"smb_v1": Boolean()})}, validation_policy="error") }) }, validation_policy="warn") doc = { "445": { "smb": { "banner": { "smb_v1": True, "metadata": {}, } } } } self.assertRaises(DataValidationException, lambda: schema.validate(doc))
class CompileAndValidationTests(unittest.TestCase): def assertBigQuerySchemaEqual(self, a, b): """ Assert that the given BigQuery schemas are equivalent. BigQuery schemas consist of lists whose order doesn't matter, dicts, and privimites. """ # allow python to have the first pass at deciding whether two objects # are equal. If they aren't, apply less strict logic (e.g., allow lists # of differing orders to be equal). if a == b: return else: self.assertEquals(type(a), type(b)) if isinstance(a, collections.Sized) \ and isinstance(b, collections.Sized): self.assertEquals(len(a), len(b)) if isinstance(a, list) and isinstance(b, list): name_ordered_a = sorted(a, key=lambda x: x['name']) name_ordered_b = sorted(b, key=lambda x: x['name']) for x, y in zip(name_ordered_a, name_ordered_b): self.assertBigQuerySchemaEqual(x, y) elif isinstance(a, dict): for k in a: self.assertIn(k, b) self.assertBigQuerySchemaEqual(a[k], b[k]) else: self.assertEquals(a, b) def setUp(self): self.maxDiff = 10000 heartbleed = SubRecord( { # with explicit proto field indices "heartbeat_support": Boolean(pr_index=11), "heartbleed_vulnerable": Boolean(category="Vulnerabilities", pr_ignore=True), "timestamp": DateTime(pr_index=10) }, pr_index=77) self.host = Record({ "ipstr": IPv4Address(required=True, examples=["8.8.8.8"], pr_index=1), "ip": Unsigned32BitInteger(doc="The IP Address of the host", pr_index=2), Port(443): SubRecord({ "tls": String(pr_index=1), "heartbleed": heartbleed }, category="heartbleed", pr_index=3), "tags": ListOf(String(), pr_index=47) }) def test_bigquery(self): global VALID_BIG_QUERY r = self.host.to_bigquery() self.assertBigQuerySchemaEqual(r, VALID_BIG_QUERY) def test_proto(self): global VALID_PROTO r = self.host.to_proto("host") self.assertEqual(r, VALID_PROTO) def test_elasticsearch(self): global VALID_ELASTIC_SEARCH r = self.host.to_es("host") self.assertEqual(r, VALID_ELASTIC_SEARCH) def test_docs_output(self): global VALID_DOCS_OUTPUT_FOR_ES_FIELDS r = self.host.docs_es("host") self.assertEqual(r, VALID_DOCS_OUTPUT_FOR_ES_FIELDS) global VALID_DOCS_OUTPUT_FOR_BIG_QUERY_FIELDS r = self.host.docs_bq("host") self.assertEqual(r, VALID_DOCS_OUTPUT_FOR_BIG_QUERY_FIELDS) def test_validation_known_good(self): test = { "ipstr": "141.212.120.1", "ip": 2379511809, "443": { "tls": "test" } } self.host.validate(test) def test_validation_bad_key(self): test = { "keydne": "141.212.120.1asdf", "ip": 2379511809, "443": { "tls": "test" } } try: self.host.validate(test) raise Exception("validation did not fail") except DataValidationException: pass def test_validation_bad_value(self): test = {"ipstr": 10, "ip": 2379511809, "443": {"tls": "test"}} try: self.host.validate(test) raise Exception("validation did not fail") except DataValidationException: pass def test_merge_no_conflict(self): a = SubRecord({"a": String(), "b": SubRecord({"c": String()})}) b = SubRecord({ "d": String(), }) valid = SubRecord({ "a": String(), "b": SubRecord({"c": String()}), "d": String(), }) self.assertEqual(a.merge(b).to_dict(), valid.to_dict()) def test_merge_different_types(self): a = SubRecord({ "a": String(), }) b = SubRecord({"a": SubRecord({})}) try: a.merge(b) raise Exception("validation did not fail") except MergeConflictException: pass def test_merge_unmergable_types(self): a = SubRecord({ "a": String(), }) b = SubRecord({ "a": String(), }) try: a.merge(b) raise Exception("validation did not fail") except MergeConflictException: pass def test_merge_recursive(self): a = SubRecord({"m": SubRecord({"a": String()})}) b = SubRecord({"a": String(), "m": SubRecord({"b": String()})}) c = SubRecord({ "a": String(), "m": SubRecord({ "a": String(), "b": String() }) }) self.assertEquals(a.merge(b).to_dict(), c.to_dict()) def test_extends(self): host = Record({ "host": IPv4Address(required=True), "time": DateTime(required=True), "data": SubRecord({}), "error": String() }) banner_grab = Record({"data": SubRecord({"banner": String()})}, extends=host) tls_banner_grab = Record({"data": SubRecord({"tls": SubRecord({})})}, extends=banner_grab) smtp_starttls = Record({"data": SubRecord({"ehlo": String()})}, extends=tls_banner_grab) valid = Record({ "host": IPv4Address(required=True), "time": DateTime(required=True), "data": SubRecord({ "banner": String(), "tls": SubRecord({}), "ehlo": String() }), "error": String() }) self.assertEqual(smtp_starttls.to_dict(), valid.to_dict()) def test_null_required(self): test = { "ipstr": None, } try: self.host.validate(test) raise Exception("validation did not fail") except DataValidationException: pass #def test_missing_required(self): # # ipstr is not set # test = { # "443":{ # "tls":"string", # } # } # try: # self.host.validate(test) # self.fail("ipstr is missing") # except DataValidationException: # pass def test_null_subkey(self): test = { "ipstr": "1.2.3.4", "443": { "heartbleed": None, } } try: self.host.validate(test) self.fail("heartbleed is null") except DataValidationException: pass def test_null_port(self): test = { "ipstr": "1.2.3.4", "443": None, } try: self.host.validate(test) self.fail("443 is null") except DataValidationException: pass def test_null_notrequired(self): test = {"ip": None, "443": {"tls": "None"}} self.host.validate(test) def test_parses_ipv4_records(self): ipv4_host_ssh = Record({ Port(22): SubRecord({ "ssh": SubRecord({ "banner": SubRecord({ "comment": String(), "timestamp": DateTime() }) }) }) }) ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record')) def test_es_dynamic_record(self): ipv4_host_with_dynamic_strict = Record() es = ipv4_host_with_dynamic_strict.to_es("strict-record") self.assertFalse("dynamic" in es["strict-record"]) ipv4_host_with_dynamic_strict = Record(es_dynamic_policy="strict") es = ipv4_host_with_dynamic_strict.to_es("strict-record") self.assertEqual("strict", es["strict-record"]["dynamic"]) def test_subrecord_types(self): SSH = SubRecordType( { "banner": SubRecord({ "comment": String(), "timestamp": DateTime() }) }, doc="class doc", required=False) self.assertEqual(SSH.DOC, "class doc") self.assertEqual(SSH.REQUIRED, False) ssh = SSH(doc="instance doc") ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})}) self.assertEqual(ssh.doc, "instance doc") self.assertEqual(ssh.required, False) ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record')) # class unchanged self.assertEqual(SSH.DOC, "class doc") self.assertEqual(SSH.REQUIRED, False) def test_subrecord_type_override(self): SSH = SubRecordType( { "banner": SubRecord({ "comment": String(), "timestamp": DateTime() }) }, doc="class doc", required=False) self.assertEqual(SSH.DOC, "class doc") self.assertEqual(SSH.REQUIRED, False) ssh = SSH(doc="instance doc", required=True) ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})}) self.assertEqual(ssh.doc, "instance doc") self.assertEqual(ssh.required, True) ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record')) # class unchanged self.assertEqual(SSH.DOC, "class doc") self.assertEqual(SSH.REQUIRED, False)
from zschema.keys import Port from zschema.compounds import ListOf, Record, SubRecord from zschema.leaves import Boolean, DateTime, IPv4Address, String, Unsigned32BitInteger heartbleed = SubRecord({ "heartbeat_support": Boolean(), "heartbleed_vulnerable": Boolean(), "timestamp": DateTime() }) host = Record({ "ipstr": IPv4Address(required=True), "ip": Unsigned32BitInteger(), Port(443): SubRecord({ "tls": String(), "heartbleed": heartbleed }), "tags": ListOf(String()) })