def setUp(self):
self.maxDiff = 10000
heartbleed = SubRecord(
{ # with explicit proto field indices
"heartbeat_support":
Boolean(pr_index=11),
"heartbleed_vulnerable":
Boolean(category="Vulnerabilities", pr_ignore=True),
"timestamp":
DateTime(pr_index=10)
},
pr_index=77)
self.host = Record({
"ipstr":
IPv4Address(required=True, examples=["8.8.8.8"], pr_index=1),
"ip":
Unsigned32BitInteger(doc="The IP Address of the host", pr_index=2),
Port(443):
SubRecord({
"tls": String(pr_index=1),
"heartbleed": heartbleed
},
category="heartbleed",
pr_index=3),
"tags":
ListOf(String(), pr_index=47)
})
def test_merge_recursive(self):
a = SubRecord({"m": SubRecord({"a": String()})})
b = SubRecord({"a": String(), "m": SubRecord({"b": String()})})
c = SubRecord({
"a": String(),
"m": SubRecord({
"a": String(),
"b": String()
})
})
self.assertEquals(a.merge(b).to_dict(), c.to_dict())
def test_merge_unmergable_types(self):
a = SubRecord({
"a": String(),
})
b = SubRecord({
"a": String(),
})
try:
a.merge(b)
raise Exception("validation did not fail")
except MergeConflictException:
pass
def test_with_args(self):
Certificate = SubRecord.with_args({}, doc="A parsed certificate.")
CertificateChain = ListOf.with_args(Certificate())
AlgorithmType = String.with_args(doc="An algorithm identifier",
examples=["a", "b", "c"])
OtherType = SubRecord({
"ca":
Certificate(doc="The CA certificate."),
"host":
Certificate(doc="The host certificate."),
"chain":
CertificateChain(doc="The certificate chain."),
"host_alg":
AlgorithmType(doc="The host algorithm", examples=["x", "y"]),
"client_alg":
AlgorithmType(doc="The client algorithm"),
"sig_alg":
AlgorithmType(examples=["p", "q"]),
})
# Check default
self.assertEqual("A parsed certificate.", Certificate().doc)
# Check overridden
self.assertEqual("The CA certificate.", OtherType.definition["ca"].doc)
self.assertEqual("The host certificate.",
OtherType.definition["host"].doc)
# Check ListOf
self.assertEqual("The certificate chain.",
OtherType.definition["chain"].doc)
# Check that instance default is used in child
self.assertEqual("A parsed certificate.",
OtherType.definition["chain"].object_.doc)
# Check Leaf type doc overrides
self.assertEqual("The host algorithm",
OtherType.definition["host_alg"].doc)
self.assertEqual("The client algorithm",
OtherType.definition["client_alg"].doc)
self.assertEqual("An algorithm identifier",
OtherType.definition["sig_alg"].doc)
# Check that examples are inherited
self.assertEqual(["a", "b", "c"],
OtherType.definition["client_alg"].examples)
# Check that examples are overridden
self.assertEqual(["x", "y"], OtherType.definition["host_alg"].examples)
self.assertEqual(["p", "q"], OtherType.definition["sig_alg"].examples)
def test_merge_no_conflict(self):
a = SubRecord({"a": String(), "b": SubRecord({"c": String()})})
b = SubRecord({
"d": String(),
})
valid = SubRecord({
"a": String(),
"b": SubRecord({"c": String()}),
"d": String(),
})
self.assertEqual(a.merge(b).to_dict(), valid.to_dict())
def test_parses_ipv4_records(self):
ipv4_host_ssh = Record({
Port(22):
SubRecord({
"ssh":
SubRecord({
"banner":
SubRecord({
"comment": String(),
"timestamp": DateTime()
})
})
})
})
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
def test_extends(self):
host = Record({
"host": IPv4Address(required=True),
"time": DateTime(required=True),
"data": SubRecord({}),
"error": String()
})
banner_grab = Record({"data": SubRecord({"banner": String()})},
extends=host)
tls_banner_grab = Record({"data": SubRecord({"tls": SubRecord({})})},
extends=banner_grab)
smtp_starttls = Record({"data": SubRecord({"ehlo": String()})},
extends=tls_banner_grab)
valid = Record({
"host":
IPv4Address(required=True),
"time":
DateTime(required=True),
"data":
SubRecord({
"banner": String(),
"tls": SubRecord({}),
"ehlo": String()
}),
"error":
String()
})
self.assertEqual(smtp_starttls.to_dict(), valid.to_dict())
def test_subrecord_type_override(self):
SSH = SubRecordType(
{
"banner": SubRecord({
"comment": String(),
"timestamp": DateTime()
})
},
doc="class doc",
required=False)
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
ssh = SSH(doc="instance doc", required=True)
ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})})
self.assertEqual(ssh.doc, "instance doc")
self.assertEqual(ssh.required, True)
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
# class unchanged
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
def test_subrecord_child_types_can_override_parent_attributes(self):
Certificate = SubRecordType({}, doc="A parsed certificate.")
c = Certificate(doc="The CA certificate.")
OtherType = SubRecord(
{
"ca": c,
"host": Certificate(doc="The host certificate."),
},
doc="hello")
self.assertEqual("A parsed certificate.", Certificate().doc)
self.assertEqual("The CA certificate.", OtherType.definition["ca"].doc)
self.assertEqual("The host certificate.",
OtherType.definition["host"].doc)
def test_subrecord_type_extends(self):
S = SubRecordType({
"provided": Boolean(),
})
extended_type = SubRecord(
{
"property": String(),
"record": SubRecord({
"another": String(),
}),
},
extends=S())
base = S()
extends = extended_type
self.assertNotIsInstance(extends, S)
self.assertFalse(base.definition['provided'].exclude)
self.assertFalse(extended_type.definition['provided'].exclude)
base.definition['provided'].exclude = ['bigquery']
self.assertEqual(['bigquery'], base.definition['provided'].exclude)
self.assertFalse(extended_type.definition['provided'].exclude)
def test_child_subtree_overrides_and_inherits(self):
schema = Record(
{
Port(445):
SubRecord({
"smb":
SubRecord({"banner": SubRecord({"smb_v1": Boolean()})},
validation_policy="error")
})
},
validation_policy="warn")
doc = {
"445": {
"smb": {
"banner": {
"smb_v1": True,
"metadata": {},
}
}
}
}
self.assertRaises(DataValidationException,
lambda: schema.validate(doc))
class PathLogUnitTests(unittest.TestCase):
sub_type = SubRecord(
{
"sub1":
String(),
"sub2":
SubRecord({
"sub2sub1":
Unsigned8BitInteger(),
"sub2sub2":
NestedListOf(String(), "sub2sub2.subrecord_name"),
}),
"sub3":
Enum(values=["a", "b", "c"])
},
validation_policy="error")
SCHEMA = Record(
{
"a":
SubRecord({
"a1": String(),
"a2": ListOf(sub_type),
"a3": Unsigned8BitInteger(),
}),
"b":
String(),
},
validation_policy="error")
def test_good(self):
good = {
"a": {
"a1":
"{a.a1}:good",
"a2": [
{
"sub1": "{a.a2[0].sub1}:good",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:good",
"{a.a2[0].sub2.sub2sub2[1]}:good",
],
},
},
{
"sub1": "{a.a2[1].sub1}:good",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:good",
}
self.SCHEMA.validate(good, policy="error")
def test_bad_root(self):
bad1 = {
"does_not_exist": "{does_not_exist}:bad1",
"a": {
"a1":
"{a.a1}:bad1",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad1",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad1",
"{a.a2[0].sub2.sub2sub2[1]}:bad1",
],
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad1, policy="error")
self.assertTrue(False, "bad1 failed to fail")
except DataValidationException as e:
self.assertTrue(not e.path)
del (bad1["does_not_exist"])
bad1["b"] = 23
try:
self.SCHEMA.validate(bad1, policy="error")
self.assertTrue(False, "bad1 failed to fail")
except DataValidationException as e:
self.assertEquals(e.path, ["b"])
def test_bad_a_key(self):
bad = {
"a": {
"does_not_exist":
23,
"a1":
"{a.a1}:bad1",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad1",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad1",
"{a.a2[0].sub2.sub2sub2[1]}:bad1",
],
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a"])
del (bad["a"]["does_not_exist"])
bad["a"]["a3"] = "not an int"
try:
ret = self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a", "a3"])
def test_bad_deep_key(self):
bad = {
"a": {
"a1":
"{a.a1}:bad",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad",
"{a.a2[0].sub2.sub2sub2[1]}:bad",
],
"does_not_exist":
"fake",
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, [
"a",
"a2",
0,
"sub2",
])
del (bad["a"]["a2"][0]["sub2"]["does_not_exist"])
bad["a"]["a2"][0]["sub2"]["sub2sub2"][1] = {"wrong type": "bad type"}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a", "a2", 0, "sub2", "sub2sub2", 1])
from zschema.keys import Port
from zschema.compounds import ListOf, Record, SubRecord
from zschema.leaves import Boolean, DateTime, IPv4Address, String, Unsigned32BitInteger
heartbleed = SubRecord({
"heartbeat_support": Boolean(),
"heartbleed_vulnerable": Boolean(),
"timestamp": DateTime()
})
host = Record({
"ipstr": IPv4Address(required=True),
"ip": Unsigned32BitInteger(),
Port(443): SubRecord({
"tls": String(),
"heartbleed": heartbleed
}),
"tags": ListOf(String())
})