def setUp(self):
self.maxDiff = 10000
heartbleed = SubRecord(
{ # with explicit proto field indices
"heartbeat_support":
Boolean(pr_index=11),
"heartbleed_vulnerable":
Boolean(category="Vulnerabilities", pr_ignore=True),
"timestamp":
DateTime(pr_index=10)
},
pr_index=77)
self.host = Record({
"ipstr":
IPv4Address(required=True, examples=["8.8.8.8"], pr_index=1),
"ip":
Unsigned32BitInteger(doc="The IP Address of the host", pr_index=2),
Port(443):
SubRecord({
"tls": String(pr_index=1),
"heartbleed": heartbleed
},
category="heartbleed",
pr_index=3),
"tags":
ListOf(String(), pr_index=47)
})
def setUp(self):
self.host = Record({
"ipstr": IPv4Address(required=True),
"ip": Unsigned32BitInteger(),
})
self.domain = Record({
"domain": String(required=True),
})
def test_extends(self):
host = Record({
"host": IPv4Address(required=True),
"time": DateTime(required=True),
"data": SubRecord({}),
"error": String()
})
banner_grab = Record({"data": SubRecord({"banner": String()})},
extends=host)
tls_banner_grab = Record({"data": SubRecord({"tls": SubRecord({})})},
extends=banner_grab)
smtp_starttls = Record({"data": SubRecord({"ehlo": String()})},
extends=tls_banner_grab)
valid = Record({
"host":
IPv4Address(required=True),
"time":
DateTime(required=True),
"data":
SubRecord({
"banner": String(),
"tls": SubRecord({}),
"ehlo": String()
}),
"error":
String()
})
self.assertEqual(smtp_starttls.to_dict(), valid.to_dict())
def setUp(self):
self.maxDiff = 10000
Child = SubRecordType(
{
"foo": Boolean(),
"bar": Boolean(validation_policy="error"),
},
validation_policy="error")
self.record = Record({
"a": Child(validation_policy="error"),
"b": Child(validation_policy="warn"),
"c": Child(validation_policy="ignore"),
"d": Child(validation_policy="inherit"),
})
def test_parses_ipv4_records(self):
ipv4_host_ssh = Record({
Port(22):
SubRecord({
"ssh":
SubRecord({
"banner":
SubRecord({
"comment": String(),
"timestamp": DateTime()
})
})
})
})
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
def test_subrecord_type_override(self):
SSH = SubRecordType(
{
"banner": SubRecord({
"comment": String(),
"timestamp": DateTime()
})
},
doc="class doc",
required=False)
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
ssh = SSH(doc="instance doc", required=True)
ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})})
self.assertEqual(ssh.doc, "instance doc")
self.assertEqual(ssh.required, True)
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
# class unchanged
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
def test_es_dynamic_record(self):
ipv4_host_with_dynamic_strict = Record()
es = ipv4_host_with_dynamic_strict.to_es("strict-record")
self.assertFalse("dynamic" in es["strict-record"])
ipv4_host_with_dynamic_strict = Record(es_dynamic_policy="strict")
es = ipv4_host_with_dynamic_strict.to_es("strict-record")
self.assertEqual("strict", es["strict-record"]["dynamic"])
def test_child_subtree_overrides_and_inherits(self):
schema = Record(
{
Port(445):
SubRecord({
"smb":
SubRecord({"banner": SubRecord({"smb_v1": Boolean()})},
validation_policy="error")
})
},
validation_policy="warn")
doc = {
"445": {
"smb": {
"banner": {
"smb_v1": True,
"metadata": {},
}
}
}
}
self.assertRaises(DataValidationException,
lambda: schema.validate(doc))
class PathLogUnitTests(unittest.TestCase):
sub_type = SubRecord(
{
"sub1":
String(),
"sub2":
SubRecord({
"sub2sub1":
Unsigned8BitInteger(),
"sub2sub2":
NestedListOf(String(), "sub2sub2.subrecord_name"),
}),
"sub3":
Enum(values=["a", "b", "c"])
},
validation_policy="error")
SCHEMA = Record(
{
"a":
SubRecord({
"a1": String(),
"a2": ListOf(sub_type),
"a3": Unsigned8BitInteger(),
}),
"b":
String(),
},
validation_policy="error")
def test_good(self):
good = {
"a": {
"a1":
"{a.a1}:good",
"a2": [
{
"sub1": "{a.a2[0].sub1}:good",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:good",
"{a.a2[0].sub2.sub2sub2[1]}:good",
],
},
},
{
"sub1": "{a.a2[1].sub1}:good",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:good",
}
self.SCHEMA.validate(good, policy="error")
def test_bad_root(self):
bad1 = {
"does_not_exist": "{does_not_exist}:bad1",
"a": {
"a1":
"{a.a1}:bad1",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad1",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad1",
"{a.a2[0].sub2.sub2sub2[1]}:bad1",
],
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad1, policy="error")
self.assertTrue(False, "bad1 failed to fail")
except DataValidationException as e:
self.assertTrue(not e.path)
del (bad1["does_not_exist"])
bad1["b"] = 23
try:
self.SCHEMA.validate(bad1, policy="error")
self.assertTrue(False, "bad1 failed to fail")
except DataValidationException as e:
self.assertEquals(e.path, ["b"])
def test_bad_a_key(self):
bad = {
"a": {
"does_not_exist":
23,
"a1":
"{a.a1}:bad1",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad1",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad1",
"{a.a2[0].sub2.sub2sub2[1]}:bad1",
],
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a"])
del (bad["a"]["does_not_exist"])
bad["a"]["a3"] = "not an int"
try:
ret = self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a", "a3"])
def test_bad_deep_key(self):
bad = {
"a": {
"a1":
"{a.a1}:bad",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad",
"{a.a2[0].sub2.sub2sub2[1]}:bad",
],
"does_not_exist":
"fake",
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, [
"a",
"a2",
0,
"sub2",
])
del (bad["a"]["a2"][0]["sub2"]["does_not_exist"])
bad["a"]["a2"][0]["sub2"]["sub2sub2"][1] = {"wrong type": "bad type"}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a", "a2", 0, "sub2", "sub2sub2", 1])
class ValidationPolicies(unittest.TestCase):
def setUp(self):
self.maxDiff = 10000
Child = SubRecordType(
{
"foo": Boolean(),
"bar": Boolean(validation_policy="error"),
},
validation_policy="error")
self.record = Record({
"a": Child(validation_policy="error"),
"b": Child(validation_policy="warn"),
"c": Child(validation_policy="ignore"),
"d": Child(validation_policy="inherit"),
})
def test_policy_setting_warn(self):
self.record.validate({"b": {"foo": "string value"}})
def test_policy_setting_ignore(self):
self.record.validate({"c": {"foo": "string value"}})
def test_policy_setting_error(self):
self.assertRaises(
DataValidationException,
lambda: self.record.validate({"c": {
"bar": "string value"
}}))
def test_policy_setting_inherit(self):
self.assertRaises(
DataValidationException,
lambda: self.record.validate({"a": {
"foo": "string value"
}}))
def test_policy_setting_multi_level_inherit(self):
self.assertRaises(
DataValidationException,
lambda: self.record.validate({"a": {
"bar": "string value"
}}))
def test_explicit_policy(self):
self.record.validate({"a": {"foo": "string value"}}, policy="ignore")
def test_child_subtree_overrides_and_inherits(self):
schema = Record(
{
Port(445):
SubRecord({
"smb":
SubRecord({"banner": SubRecord({"smb_v1": Boolean()})},
validation_policy="error")
})
},
validation_policy="warn")
doc = {
"445": {
"smb": {
"banner": {
"smb_v1": True,
"metadata": {},
}
}
}
}
self.assertRaises(DataValidationException,
lambda: schema.validate(doc))
class CompileAndValidationTests(unittest.TestCase):
def assertBigQuerySchemaEqual(self, a, b):
"""
Assert that the given BigQuery schemas are equivalent.
BigQuery schemas consist of lists whose order doesn't matter, dicts,
and privimites.
"""
# allow python to have the first pass at deciding whether two objects
# are equal. If they aren't, apply less strict logic (e.g., allow lists
# of differing orders to be equal).
if a == b:
return
else:
self.assertEquals(type(a), type(b))
if isinstance(a, collections.Sized) \
and isinstance(b, collections.Sized):
self.assertEquals(len(a), len(b))
if isinstance(a, list) and isinstance(b, list):
name_ordered_a = sorted(a, key=lambda x: x['name'])
name_ordered_b = sorted(b, key=lambda x: x['name'])
for x, y in zip(name_ordered_a, name_ordered_b):
self.assertBigQuerySchemaEqual(x, y)
elif isinstance(a, dict):
for k in a:
self.assertIn(k, b)
self.assertBigQuerySchemaEqual(a[k], b[k])
else:
self.assertEquals(a, b)
def setUp(self):
self.maxDiff = 10000
heartbleed = SubRecord(
{ # with explicit proto field indices
"heartbeat_support":
Boolean(pr_index=11),
"heartbleed_vulnerable":
Boolean(category="Vulnerabilities", pr_ignore=True),
"timestamp":
DateTime(pr_index=10)
},
pr_index=77)
self.host = Record({
"ipstr":
IPv4Address(required=True, examples=["8.8.8.8"], pr_index=1),
"ip":
Unsigned32BitInteger(doc="The IP Address of the host", pr_index=2),
Port(443):
SubRecord({
"tls": String(pr_index=1),
"heartbleed": heartbleed
},
category="heartbleed",
pr_index=3),
"tags":
ListOf(String(), pr_index=47)
})
def test_bigquery(self):
global VALID_BIG_QUERY
r = self.host.to_bigquery()
self.assertBigQuerySchemaEqual(r, VALID_BIG_QUERY)
def test_proto(self):
global VALID_PROTO
r = self.host.to_proto("host")
self.assertEqual(r, VALID_PROTO)
def test_elasticsearch(self):
global VALID_ELASTIC_SEARCH
r = self.host.to_es("host")
self.assertEqual(r, VALID_ELASTIC_SEARCH)
def test_docs_output(self):
global VALID_DOCS_OUTPUT_FOR_ES_FIELDS
r = self.host.docs_es("host")
self.assertEqual(r, VALID_DOCS_OUTPUT_FOR_ES_FIELDS)
global VALID_DOCS_OUTPUT_FOR_BIG_QUERY_FIELDS
r = self.host.docs_bq("host")
self.assertEqual(r, VALID_DOCS_OUTPUT_FOR_BIG_QUERY_FIELDS)
def test_validation_known_good(self):
test = {
"ipstr": "141.212.120.1",
"ip": 2379511809,
"443": {
"tls": "test"
}
}
self.host.validate(test)
def test_validation_bad_key(self):
test = {
"keydne": "141.212.120.1asdf",
"ip": 2379511809,
"443": {
"tls": "test"
}
}
try:
self.host.validate(test)
raise Exception("validation did not fail")
except DataValidationException:
pass
def test_validation_bad_value(self):
test = {"ipstr": 10, "ip": 2379511809, "443": {"tls": "test"}}
try:
self.host.validate(test)
raise Exception("validation did not fail")
except DataValidationException:
pass
def test_merge_no_conflict(self):
a = SubRecord({"a": String(), "b": SubRecord({"c": String()})})
b = SubRecord({
"d": String(),
})
valid = SubRecord({
"a": String(),
"b": SubRecord({"c": String()}),
"d": String(),
})
self.assertEqual(a.merge(b).to_dict(), valid.to_dict())
def test_merge_different_types(self):
a = SubRecord({
"a": String(),
})
b = SubRecord({"a": SubRecord({})})
try:
a.merge(b)
raise Exception("validation did not fail")
except MergeConflictException:
pass
def test_merge_unmergable_types(self):
a = SubRecord({
"a": String(),
})
b = SubRecord({
"a": String(),
})
try:
a.merge(b)
raise Exception("validation did not fail")
except MergeConflictException:
pass
def test_merge_recursive(self):
a = SubRecord({"m": SubRecord({"a": String()})})
b = SubRecord({"a": String(), "m": SubRecord({"b": String()})})
c = SubRecord({
"a": String(),
"m": SubRecord({
"a": String(),
"b": String()
})
})
self.assertEquals(a.merge(b).to_dict(), c.to_dict())
def test_extends(self):
host = Record({
"host": IPv4Address(required=True),
"time": DateTime(required=True),
"data": SubRecord({}),
"error": String()
})
banner_grab = Record({"data": SubRecord({"banner": String()})},
extends=host)
tls_banner_grab = Record({"data": SubRecord({"tls": SubRecord({})})},
extends=banner_grab)
smtp_starttls = Record({"data": SubRecord({"ehlo": String()})},
extends=tls_banner_grab)
valid = Record({
"host":
IPv4Address(required=True),
"time":
DateTime(required=True),
"data":
SubRecord({
"banner": String(),
"tls": SubRecord({}),
"ehlo": String()
}),
"error":
String()
})
self.assertEqual(smtp_starttls.to_dict(), valid.to_dict())
def test_null_required(self):
test = {
"ipstr": None,
}
try:
self.host.validate(test)
raise Exception("validation did not fail")
except DataValidationException:
pass
#def test_missing_required(self):
# # ipstr is not set
# test = {
# "443":{
# "tls":"string",
# }
# }
# try:
# self.host.validate(test)
# self.fail("ipstr is missing")
# except DataValidationException:
# pass
def test_null_subkey(self):
test = {
"ipstr": "1.2.3.4",
"443": {
"heartbleed": None,
}
}
try:
self.host.validate(test)
self.fail("heartbleed is null")
except DataValidationException:
pass
def test_null_port(self):
test = {
"ipstr": "1.2.3.4",
"443": None,
}
try:
self.host.validate(test)
self.fail("443 is null")
except DataValidationException:
pass
def test_null_notrequired(self):
test = {"ip": None, "443": {"tls": "None"}}
self.host.validate(test)
def test_parses_ipv4_records(self):
ipv4_host_ssh = Record({
Port(22):
SubRecord({
"ssh":
SubRecord({
"banner":
SubRecord({
"comment": String(),
"timestamp": DateTime()
})
})
})
})
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
def test_es_dynamic_record(self):
ipv4_host_with_dynamic_strict = Record()
es = ipv4_host_with_dynamic_strict.to_es("strict-record")
self.assertFalse("dynamic" in es["strict-record"])
ipv4_host_with_dynamic_strict = Record(es_dynamic_policy="strict")
es = ipv4_host_with_dynamic_strict.to_es("strict-record")
self.assertEqual("strict", es["strict-record"]["dynamic"])
def test_subrecord_types(self):
SSH = SubRecordType(
{
"banner": SubRecord({
"comment": String(),
"timestamp": DateTime()
})
},
doc="class doc",
required=False)
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
ssh = SSH(doc="instance doc")
ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})})
self.assertEqual(ssh.doc, "instance doc")
self.assertEqual(ssh.required, False)
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
# class unchanged
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
def test_subrecord_type_override(self):
SSH = SubRecordType(
{
"banner": SubRecord({
"comment": String(),
"timestamp": DateTime()
})
},
doc="class doc",
required=False)
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
ssh = SSH(doc="instance doc", required=True)
ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})})
self.assertEqual(ssh.doc, "instance doc")
self.assertEqual(ssh.required, True)
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
# class unchanged
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
from zschema.keys import Port
from zschema.compounds import ListOf, Record, SubRecord
from zschema.leaves import Boolean, DateTime, IPv4Address, String, Unsigned32BitInteger
heartbleed = SubRecord({
"heartbeat_support": Boolean(),
"heartbleed_vulnerable": Boolean(),
"timestamp": DateTime()
})
host = Record({
"ipstr": IPv4Address(required=True),
"ip": Unsigned32BitInteger(),
Port(443): SubRecord({
"tls": String(),
"heartbleed": heartbleed
}),
"tags": ListOf(String())
})