def test_with_args_positional_override(self):
class Positional(Keyable):
def __init__(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
self.doc = kwargs.get("doc")
# Leave the positional argument (list type) blank, but specify a category.
GenericCategorizedList = ListOf.with_args(category="my category")
# Get lists with different types (but the same category).
categorized_string_list = GenericCategorizedList(String())
categorized_cert_list = GenericCategorizedList(Positional())
self.assertEqual("my category", categorized_string_list.category)
self.assertEqual("my category", categorized_cert_list.category)
# ListOf(...) takes only one positional arg, so StringList(Positional()) should raise.
StringList = ListOf.with_args(String())
self.assertRaises(Exception, lambda: StringList(Positional()))
# ListOf(...) needs exactly one positional arg, so GenericCategorizedList() should also raise.
self.assertRaises(Exception, lambda: CategorizedCertificateList())
# Confirm that positional args are pulled from the proper location
MyPositional = Positional.with_args("a", doc="default docs")
p0 = MyPositional()
p0doc = MyPositional(doc="some docs")
# Passing positional args to the factory constructor and the contructor is not allowed
self.assertRaises(Exception, lambda: MyPositional("b"))
self.assertRaises(Exception, lambda: MyPositional("x, y"))
self.assertEqual(("a", ), p0.args)
self.assertEqual("default docs", p0.doc)
self.assertEqual(("a", ), p0doc.args)
self.assertEqual("some docs", p0doc.doc)
def setUp(self):
self.maxDiff = 10000
heartbleed = SubRecord(
{ # with explicit proto field indices
"heartbeat_support":
Boolean(pr_index=11),
"heartbleed_vulnerable":
Boolean(category="Vulnerabilities", pr_ignore=True),
"timestamp":
DateTime(pr_index=10)
},
pr_index=77)
self.host = Record({
"ipstr":
IPv4Address(required=True, examples=["8.8.8.8"], pr_index=1),
"ip":
Unsigned32BitInteger(doc="The IP Address of the host", pr_index=2),
Port(443):
SubRecord({
"tls": String(pr_index=1),
"heartbleed": heartbleed
},
category="heartbleed",
pr_index=3),
"tags":
ListOf(String(), pr_index=47)
})
def test_merge_unmergable_types(self):
a = SubRecord({
"a": String(),
})
b = SubRecord({
"a": String(),
})
try:
a.merge(b)
raise Exception("validation did not fail")
except MergeConflictException:
pass
def test_ListOf_exclude(self):
a = ListOf(String())
self.assertFalse(a.exclude_bigquery)
self.assertFalse(a.exclude_elasticsearch)
b = ListOf(String(exclude=["bigquery"]))
self.assertTrue(b.exclude_bigquery)
self.assertFalse(b.exclude_elasticsearch)
c = ListOf(String(), exclude=["elasticsearch"])
self.assertFalse(c.exclude_bigquery)
self.assertTrue(c.exclude_elasticsearch)
d = ListOf(String(exclude=["bigquery"]), exclude=["elasticsearch"])
self.assertTrue(d.exclude_bigquery)
self.assertTrue(d.exclude_elasticsearch)
def setUp(self):
self.host = Record({
"ipstr": IPv4Address(required=True),
"ip": Unsigned32BitInteger(),
})
self.domain = Record({
"domain": String(required=True),
})
def test_extends(self):
host = Record({
"host": IPv4Address(required=True),
"time": DateTime(required=True),
"data": SubRecord({}),
"error": String()
})
banner_grab = Record({"data": SubRecord({"banner": String()})},
extends=host)
tls_banner_grab = Record({"data": SubRecord({"tls": SubRecord({})})},
extends=banner_grab)
smtp_starttls = Record({"data": SubRecord({"ehlo": String()})},
extends=tls_banner_grab)
valid = Record({
"host":
IPv4Address(required=True),
"time":
DateTime(required=True),
"data":
SubRecord({
"banner": String(),
"tls": SubRecord({}),
"ehlo": String()
}),
"error":
String()
})
self.assertEqual(smtp_starttls.to_dict(), valid.to_dict())
def test_subrecord_type_extends(self):
S = SubRecordType({
"provided": Boolean(),
})
extended_type = SubRecord(
{
"property": String(),
"record": SubRecord({
"another": String(),
}),
},
extends=S())
base = S()
extends = extended_type
self.assertNotIsInstance(extends, S)
self.assertFalse(base.definition['provided'].exclude)
self.assertFalse(extended_type.definition['provided'].exclude)
base.definition['provided'].exclude = ['bigquery']
self.assertEqual(['bigquery'], base.definition['provided'].exclude)
self.assertFalse(extended_type.definition['provided'].exclude)
def test_with_args(self):
Certificate = SubRecord.with_args({}, doc="A parsed certificate.")
CertificateChain = ListOf.with_args(Certificate())
AlgorithmType = String.with_args(doc="An algorithm identifier",
examples=["a", "b", "c"])
OtherType = SubRecord({
"ca":
Certificate(doc="The CA certificate."),
"host":
Certificate(doc="The host certificate."),
"chain":
CertificateChain(doc="The certificate chain."),
"host_alg":
AlgorithmType(doc="The host algorithm", examples=["x", "y"]),
"client_alg":
AlgorithmType(doc="The client algorithm"),
"sig_alg":
AlgorithmType(examples=["p", "q"]),
})
# Check default
self.assertEqual("A parsed certificate.", Certificate().doc)
# Check overridden
self.assertEqual("The CA certificate.", OtherType.definition["ca"].doc)
self.assertEqual("The host certificate.",
OtherType.definition["host"].doc)
# Check ListOf
self.assertEqual("The certificate chain.",
OtherType.definition["chain"].doc)
# Check that instance default is used in child
self.assertEqual("A parsed certificate.",
OtherType.definition["chain"].object_.doc)
# Check Leaf type doc overrides
self.assertEqual("The host algorithm",
OtherType.definition["host_alg"].doc)
self.assertEqual("The client algorithm",
OtherType.definition["client_alg"].doc)
self.assertEqual("An algorithm identifier",
OtherType.definition["sig_alg"].doc)
# Check that examples are inherited
self.assertEqual(["a", "b", "c"],
OtherType.definition["client_alg"].examples)
# Check that examples are overridden
self.assertEqual(["x", "y"], OtherType.definition["host_alg"].examples)
self.assertEqual(["p", "q"], OtherType.definition["sig_alg"].examples)
def test_multiple_subrecord_types(self):
A = SubRecordType({
"first": String(),
}, type_name="A")
B = SubRecordType({
"second": Boolean(),
}, type_name="B")
a = A()
self.assertIn("first", a.definition)
b = B()
self.assertIn("second", b.definition)
a = A()
self.assertIn("first", a.definition)
def test_parses_ipv4_records(self):
ipv4_host_ssh = Record({
Port(22):
SubRecord({
"ssh":
SubRecord({
"banner":
SubRecord({
"comment": String(),
"timestamp": DateTime()
})
})
})
})
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
def test_merge_recursive(self):
a = SubRecord({"m": SubRecord({"a": String()})})
b = SubRecord({"a": String(), "m": SubRecord({"b": String()})})
c = SubRecord({
"a": String(),
"m": SubRecord({
"a": String(),
"b": String()
})
})
self.assertEquals(a.merge(b).to_dict(), c.to_dict())
def test_merge_no_conflict(self):
a = SubRecord({"a": String(), "b": SubRecord({"c": String()})})
b = SubRecord({
"d": String(),
})
valid = SubRecord({
"a": String(),
"b": SubRecord({"c": String()}),
"d": String(),
})
self.assertEqual(a.merge(b).to_dict(), valid.to_dict())
def test_subrecord_type_override(self):
SSH = SubRecordType(
{
"banner": SubRecord({
"comment": String(),
"timestamp": DateTime()
})
},
doc="class doc",
required=False)
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
ssh = SSH(doc="instance doc", required=True)
ipv4_host_ssh = Record({Port(22): SubRecord({"ssh": ssh})})
self.assertEqual(ssh.doc, "instance doc")
self.assertEqual(ssh.required, True)
ipv4_host_ssh.validate(json_fixture('ipv4-ssh-record'))
# class unchanged
self.assertEqual(SSH.DOC, "class doc")
self.assertEqual(SSH.REQUIRED, False)
def test_subrecord_type(self):
A = SubRecordType({
"string": String(),
"boolean": Boolean(),
})
first = A()
second = A()
# The class returned by SubRecordType() should be constructable into
# unique objects
self.assertIsNot(first, second)
self.assertTrue(issubclass(A, SubRecord))
self.assertIsInstance(first, A)
self.assertIsInstance(second, A)
# Check the properties aren't shared
self.assertIsNone(first.definition['string'].doc)
self.assertIsNone(second.definition['string'].doc)
first.definition['string'].doc = "hello"
self.assertIsNone(second.definition['string'].doc)
class PathLogUnitTests(unittest.TestCase):
sub_type = SubRecord(
{
"sub1":
String(),
"sub2":
SubRecord({
"sub2sub1":
Unsigned8BitInteger(),
"sub2sub2":
NestedListOf(String(), "sub2sub2.subrecord_name"),
}),
"sub3":
Enum(values=["a", "b", "c"])
},
validation_policy="error")
SCHEMA = Record(
{
"a":
SubRecord({
"a1": String(),
"a2": ListOf(sub_type),
"a3": Unsigned8BitInteger(),
}),
"b":
String(),
},
validation_policy="error")
def test_good(self):
good = {
"a": {
"a1":
"{a.a1}:good",
"a2": [
{
"sub1": "{a.a2[0].sub1}:good",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:good",
"{a.a2[0].sub2.sub2sub2[1]}:good",
],
},
},
{
"sub1": "{a.a2[1].sub1}:good",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:good",
}
self.SCHEMA.validate(good, policy="error")
def test_bad_root(self):
bad1 = {
"does_not_exist": "{does_not_exist}:bad1",
"a": {
"a1":
"{a.a1}:bad1",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad1",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad1",
"{a.a2[0].sub2.sub2sub2[1]}:bad1",
],
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad1, policy="error")
self.assertTrue(False, "bad1 failed to fail")
except DataValidationException as e:
self.assertTrue(not e.path)
del (bad1["does_not_exist"])
bad1["b"] = 23
try:
self.SCHEMA.validate(bad1, policy="error")
self.assertTrue(False, "bad1 failed to fail")
except DataValidationException as e:
self.assertEquals(e.path, ["b"])
def test_bad_a_key(self):
bad = {
"a": {
"does_not_exist":
23,
"a1":
"{a.a1}:bad1",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad1",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad1",
"{a.a2[0].sub2.sub2sub2[1]}:bad1",
],
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a"])
del (bad["a"]["does_not_exist"])
bad["a"]["a3"] = "not an int"
try:
ret = self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a", "a3"])
def test_bad_deep_key(self):
bad = {
"a": {
"a1":
"{a.a1}:bad",
"a2": [
{
"sub1": "{a.a2[0].sub1}:bad",
"sub2": {
"sub2sub1":
1,
"sub2sub2": [
"{a.a2[0].sub2.sub2sub2[0]}:bad",
"{a.a2[0].sub2.sub2sub2[1]}:bad",
],
"does_not_exist":
"fake",
},
},
{
"sub1": "{a.a2[1].sub1}:bad1",
"sub2": {
"sub2sub1": 1,
"sub2sub2": [],
},
},
],
"a3":
1,
},
"b": "{b}:bad1",
}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, [
"a",
"a2",
0,
"sub2",
])
del (bad["a"]["a2"][0]["sub2"]["does_not_exist"])
bad["a"]["a2"][0]["sub2"]["sub2sub2"][1] = {"wrong type": "bad type"}
try:
self.SCHEMA.validate(bad, policy="error")
self.assertTrue(False, "bad failed to fail")
except DataValidationException as e:
self.assertEqual(e.path, ["a", "a2", 0, "sub2", "sub2sub2", 1])
from zschema.keys import Port
from zschema.compounds import ListOf, Record, SubRecord
from zschema.leaves import Boolean, DateTime, IPv4Address, String, Unsigned32BitInteger
heartbleed = SubRecord({
"heartbeat_support": Boolean(),
"heartbleed_vulnerable": Boolean(),
"timestamp": DateTime()
})
host = Record({
"ipstr": IPv4Address(required=True),
"ip": Unsigned32BitInteger(),
Port(443): SubRecord({
"tls": String(),
"heartbleed": heartbleed
}),
"tags": ListOf(String())
})