def test_keyword_of_fields(keyword, field_cls):
s_f = StringField()
n_f = NumberField()
i_f = IntField()
field = field_cls([n_f, Var({'role_1': s_f}), Var({'role_2': i_f})])
assert s(field.get_schema()) == {keyword: [n_f.get_schema()]}
assert s(field.get_schema(role='role_1')) == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert s(field.get_schema(role='role_2')) == {
keyword: [n_f.get_schema(), i_f.get_schema()]
}
field = field_cls(
Var(
{
'role_1': [n_f,
Var({'role_1': s_f}),
Var({'role_2': i_f})],
'role_2': [Var({'role_2': i_f})],
},
propagate='role_1'))
assert s(field.get_schema(role='role_1')) == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
with pytest.raises(SchemaGenerationException):
field.get_schema(role='role_2')
def test_array_field():
s_f = StringField()
n_f = NumberField()
field = ArrayField(Var({
'role_1': s_f,
'role_2': n_f,
}))
schema = field.get_schema(role='role_1')
assert schema['items'] == s_f.get_schema()
schema = field.get_schema(role='role_2')
assert schema['items'] == n_f.get_schema()
schema = field.get_schema()
assert 'items' not in schema
_ = lambda value: Var({'role_1': value})
field = ArrayField(s_f,
min_items=_(1),
max_items=_(2),
unique_items=_(True),
additional_items=_(True))
assert field.get_schema() == {
'type': 'array',
'items': s_f.get_schema(),
}
assert field.get_schema(role='role_1') == {
'type': 'array',
'items': s_f.get_schema(),
'minItems': 1,
'maxItems': 2,
'uniqueItems': True,
'additionalItems': True,
}
class Normalization(BaseNormalization):
"""Normalization schema for a specific domain."""
domain_name = domain_schema['name']
domain = StringField(enum=[domain_name], required=True)
class Fields(BaseNormalization.Fields):
mapping = StrictDict(
{k: StringField()
for k in domain_schema['fields']})
fields = DocumentField(Fields())
events = DictField({
event_name: StrictDict({
'enum':
StrictDict({
enum_name: StrictDict({
enum_option: StringField()
for enum_option in enum_options
})
for enum_name, enum_options in event_info.get('enum',
{}).items()
}),
'mapping':
StrictDict(
{k: StringField()
for k in event_info.get('fields', [])}),
'filter':
StringField(required=True)
})
for event_name, event_info in domain_schema['events'].items()
})
def test_keyword_of_fields(keyword, field_cls):
s_f = StringField()
n_f = NumberField()
i_f = IntField()
field = field_cls([n_f, Var({'role_1': s_f}), Var({'role_2': i_f})])
assert field.get_schema() == {keyword: [n_f.get_schema()]}
assert field.get_schema(role='role_1') == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert field.get_schema(role='role_2') == {
keyword: [n_f.get_schema(), i_f.get_schema()]
}
field = field_cls(
Var(
{
'role_1': [n_f,
Var({'role_1': s_f}),
Var({'role_2': i_f})],
'role_2': [Var({'role_2': i_f})],
},
propagate='role_1'))
assert field.get_schema() == {keyword: []}
assert field.get_schema(role='role_1') == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert field.get_schema(role='role_2') == {keyword: []}
class ValueLabelPair(DocumentWithoutAddProp):
value = OneOfField([
NumberField(),
StringField(max_length=250),
BooleanField()
])
label = StringField(required=True, max_length=100)
class BaseNormalization(Document):
"""Base class for a normalization configuration."""
domain_name = None
class TimestampInfo(Document):
field = StringField(required=True)
format = StringField(required=True)
class Fields(Document):
scope = StringField()
mapping = DictField()
name = StringField(required=True)
filter_query = BooleanField(default=False)
os_types = ArrayField(StringField(), required=False)
domain = StringField(required=True)
strict = BooleanField(required=True)
timestamp = DocumentField(TimestampInfo(), required=True)
fields = DocumentField(Fields(), required=True)
events = DictField(additional_properties=DictField(
{
'enum':
DictField(additional_properties=DictField(
additional_properties=StringField())),
'mapping':
DictField(),
'filter':
StringField(required=True)
},
required=True))
def test_keyword_of_fields(keyword, field_cls):
s_f = StringField()
n_f = NumberField()
i_f = IntField()
field = field_cls([n_f, Var(role_1=s_f), Var(role_2=i_f)])
assert field.get_schema() == {
keyword: [n_f.get_schema()]
}
assert field.get_schema(role='role_1') == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert field.get_schema(role='role_2') == {
keyword: [n_f.get_schema(), i_f.get_schema()]
}
field = field_cls(Var(
role_1=[n_f, Var(role_1=s_f), Var(role_2=i_f)],
role_2=[Var(role_2=i_f)],
roles_to_pass_down=['role_1']
))
assert field.get_schema() == {keyword: []}
assert field.get_schema(role='role_1') == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert field.get_schema(role='role_2') == {keyword: []}
def test_dict_field():
s_f = StringField()
_ = lambda value: Var({'role_1': value})
field = DictField(properties=Var(
role_1={'name': Var(role_1=s_f)},
role_2={'name': Var(role_2=s_f)},
roles_to_pass_down=['role_1']
), pattern_properties=Var(
role_1={'.*': Var(role_1=s_f)},
role_2={'.*': Var(role_2=s_f)},
roles_to_pass_down=['role_1']
), additional_properties=_(s_f), min_properties=_(1), max_properties=_(2))
assert field.get_schema() == {
'type': 'object'
}
assert field.get_schema(role='role_1') == {
'type': 'object',
'properties': {
'name': s_f.get_schema(),
},
'patternProperties': {
'.*': s_f.get_schema(),
},
'additionalProperties': s_f.get_schema(),
'minProperties': 1,
'maxProperties': 2,
}
assert field.get_schema(role='role_2') == {
'type': 'object',
'properties': {},
'patternProperties': {},
}
def test_array_field():
s_f = StringField()
n_f = NumberField()
field = ArrayField(Var({
'role_1': s_f,
'role_2': n_f,
}))
schema = field.get_schema(role='role_1')
assert schema['items'] == s_f.get_schema()
schema = field.get_schema(role='role_2')
assert schema['items'] == n_f.get_schema()
schema = field.get_schema()
assert 'items' not in schema
_ = lambda value: Var({'role_1': value})
field = ArrayField(s_f, min_items=_(1), max_items=_(2), unique_items=_(True), additional_items=_(True))
assert field.get_schema() == {
'type': 'array',
'items': s_f.get_schema(),
}
assert field.get_schema(role='role_1') == {
'type': 'array',
'items': s_f.get_schema(),
'minItems': 1,
'maxItems': 2,
'uniqueItems': True,
'additionalItems': True,
}
class Message(Document):
with Scope(DB_ROLE) as db:
db.uuid = StringField(required=True)
created_at = IntField(
required=when_not(PARTIAL_RESPONSE_ROLE, REQUEST_ROLE))
text = StringField(required=when_not(PARTIAL_RESPONSE_ROLE))
field_that_is_never_present = Var(
{'NEVER': StringField(required=True)})
class TabContent(DocumentWithoutAddProp):
entity = ArrayField(DocumentField(ConfigurationEntity, as_ref=True), required=True)
name = StringField(required=True, pattern="^[\/\w]+$", max_length=250)
title = StringField(required=True, max_length=50)
options = DocumentField(Hooks, as_ref=True)
table = DocumentField(ConfigurationTable, as_ref=True)
conf = StringField(required=False, max_length=100)
restHandlerName = StringField(required=False, max_length=100)
# Provisioning tab level hook on configuration page
hook = DocumentField(Hooks, as_ref=True)
class Domain(Document):
"""Meta schema for defining a query domain."""
class EventInfo(Document):
enum = DictField(
additional_properties=ArrayField(StringField(eql_name)))
fields = ArrayField(StringField(eql_name))
name = StringField(required=True)
fields = ArrayField(StringField(), required=True)
events = DictField(additional_properties=DocumentField(EventInfo()))
class InputsPage(DocumentWithoutAddProp):
title = StringField(required=True, max_length=60)
description = StringField(max_length=200)
table = DocumentField(InputsTable, as_ref=True, required=True)
services = ArrayField(DictField(
properties={
"name": StringField(required=True, pattern="^[0-9a-zA-Z][0-9a-zA-Z_-]*$", max_length=50),
"title": StringField(required=True, max_length=100),
"entity": ArrayField(DocumentField(InputsEntity, as_ref=True), required=True),
"options": DocumentField(Hooks, as_ref=True),
"groups": ArrayField(DictField(
properties={
"options": DictField(
properties={
"isExpandable": BooleanField(),
"expand": BooleanField()
}
),
"label": StringField(required=True, max_length=100),
"field": ArrayField(StringField(required=True, pattern="^\w+$"))
}
), required=False),
"style": StringField(required=False, enum=["page", "dialog"]),
"hook": DictField(required=False),
"conf": StringField(required=False, max_length=100),
"restHandlerName": StringField(required=False, max_length=100)
}
), required=True)
menu = DictField(required=False)
class User(Document):
class Options(object):
roles_to_propagate = not_(PARTIAL_RESPONSE_ROLE)
with Scope(DB_ROLE) as db:
db._id = StringField(required=True)
db.version = StringField(required=True)
with Scope(lambda r: r.startswith(RESPONSE_ROLE) or r == REQUEST_ROLE) as response:
response.id = StringField(required=when_not(PARTIAL_RESPONSE_ROLE))
with Scope(not_(REQUEST_ROLE)) as not_request:
not_request.messages = ArrayField(DocumentField(Message), required=when_not(PARTIAL_RESPONSE_ROLE))
class Task(Document):
class Options(object):
title = 'Task'
description = 'A task.'
definition_id = 'task'
id = IntField(required=Var({'response': True}))
name = StringField(required=True, min_length=5)
type = StringField(required=True, enum=['TYPE_1', 'TYPE_2'])
created_at = DateTimeField(required=True)
author = Var({'response': DocumentField(User)})
class Response(Document):
data = DictField(
{
'name': StringField(min_length=2, required=True),
'url': UriField(),
'main': BooleanField(default=False),
},
required=True)
data_id = IntField()
system = StringField(default='dev')
class Alerts(DocumentWithoutAddProp):
name = StringField(required=True,
pattern="^[a-zA-Z0-9_]+$",
max_length=100)
label = StringField(required=True, max_length=100)
description = StringField(required=True)
activeResponse = DictField(properties={
"task":
ArrayField(StringField(required=True), required=True, min_items=1),
"supportsAdhoc":
BooleanField(required=True),
"subject":
ArrayField(StringField(required=True), required=True, min_items=1),
"category":
ArrayField(StringField(required=True), required=True, min_items=1),
"technology":
ArrayField(DocumentField(Technology, as_ref=True),
required=True,
min_items=1),
"drilldownUri":
StringField(required=False),
"sourcetype":
StringField(required=False, pattern="^[a-zA-Z0-9:-_]+$", max_length=50)
},
required=False)
entity = ArrayField(DocumentField(AlertEntity, as_ref=True))
def test_dict_field():
s_f = StringField()
_ = lambda value: Var({'role_1': value})
field = DictField(properties=Var(
{
'role_1': {
'name': Var({'role_1': s_f})
},
'role_2': {
'name': Var({'role_2': s_f})
},
},
propagate='role_1'),
pattern_properties=Var(
{
'role_1': {
'.*': Var({'role_1': s_f})
},
'role_2': {
'.*': Var({'role_2': s_f})
},
},
propagate='role_1'),
additional_properties=_(s_f),
min_properties=_(1),
max_properties=_(2))
assert s(field.get_schema()) == s({'type': 'object'})
assert s(field.get_schema(role='role_1')) == s({
'type':
'object',
'properties': {
'name': s_f.get_schema(),
},
'patternProperties': {
'.*': s_f.get_schema(),
},
'additionalProperties':
s_f.get_schema(),
'minProperties':
1,
'maxProperties':
2,
})
assert s(field.get_schema(role='role_2')) == s({
'type': 'object',
'properties': {},
'patternProperties': {},
})
def test_string_field():
_ = lambda value: Var({'role_1': value})
field = StringField(format=_('date-time'),
min_length=_(1),
max_length=_(2))
assert field.get_schema() == {'type': 'string'}
assert field.get_schema(role='role_1') == {
'type': 'string',
'format': 'date-time',
'minLength': 1,
'maxLength': 2,
}
with pytest.raises(ValueError) as e:
StringField(pattern=_('('))
assert str(e.value) == 'Invalid regular expression: unbalanced parenthesis'
class Child(Base):
class Options(object):
definition_id = 'child'
title = 'Child'
b = StringField()
c = DocumentField(RECURSIVE_REFERENCE_CONSTANT)
class Base(Document):
class Options(object):
inheritance_mode = ALL_OF
definition_id = 'base'
title = 'Base'
a = StringField()
class Analytic(Document):
"""Schema for an individual analytic."""
metadata = DocumentField(AnalyticMetadata,
required=True,
additional_properties=True)
query = StringField(required=True)
def test_string_field():
_ = lambda value: Var({'role_1': value})
field = StringField(format=_('date-time'), min_length=_(1), max_length=_(2))
assert field.get_schema() == {
'type': 'string'
}
assert field.get_schema(role='role_1') == {
'type': 'string',
'format': 'date-time',
'minLength': 1,
'maxLength': 2,
}
with pytest.raises(ValueError) as e:
StringField(pattern=_('('))
assert str(e.value) == 'Invalid regular expression: unbalanced parenthesis'
class InputsEntity(Entity):
# Prevnet Splunk reserved inputs field keys being used in the user customized inputs
# https://jira.splunk.com/browse/ADDON-13014#comment-1493170
field = StringField(
required=True,
pattern="(?!^(?:persistentQueueSize|queueSize|start_by_shell|output_mode|output_field|owner|app|sharing)$)(?:^\w+$)"
)
class Table(DocumentWithoutAddProp):
moreInfo = ArrayField(DictField(
properties={
"field": StringField(required=True, pattern="^\w+$"),
"label": StringField(required=True, max_length=30),
"mapping": DictField(required=False)
}
))
# Header field names needs to be display on UI
header = ArrayField(DictField(
properties={
"field": StringField(required=True, pattern="^\w+$"),
"label": StringField(required=True, max_length=30),
"mapping": DictField(required=False),
"customCell": DictField(required=False)
}
), required=True)
# custom Row implementation if required for special cases
customRow = DictField(required=False)
def test_scope():
scope = Scope(DB_ROLE)
f = StringField()
scope.login = f
assert scope.login == f
assert scope.__fields__ == {
'login': f,
}
with pytest.raises(AttributeError):
scope.gsomgsom
class Meta(DocumentWithoutAddProp):
displayName = StringField(required=True, max_length=200)
name = StringField(required=True, pattern="^[^<>\:\"\/\\\|\?\*]+$")
restRoot = StringField(required=True, pattern="^\w+$")
apiVersion = StringField(required=False, pattern="^(?:\d{1,3}\.){2}\d{1,3}$")
version = StringField(required=True)
schemaVersion = StringField( pattern="^(?:\d{1,3}\.){2}\d{1,3}$")
def test_keyword_of_fields(keyword, field_cls):
s_f = StringField()
n_f = NumberField()
i_f = IntField()
field = field_cls([n_f, Var({'role_1': s_f}), Var({'role_2': i_f})])
assert field.get_schema() == {
keyword: [n_f.get_schema()]
}
assert field.get_schema(role='role_1') == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert field.get_schema(role='role_2') == {
keyword: [n_f.get_schema(), i_f.get_schema()]
}
field = field_cls(Var({
'role_1': [n_f, Var({'role_1': s_f}), Var({'role_2': i_f})],
'role_2': [Var({'role_2': i_f})],
}, propagate='role_1'))
assert field.get_schema() == {keyword: []}
assert field.get_schema(role='role_1') == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert field.get_schema(role='role_2') == {keyword: []}
def test_keyword_of_fields(keyword, field_cls):
s_f = StringField()
n_f = NumberField()
i_f = IntField()
field = field_cls([n_f, Var({'role_1': s_f}), Var({'role_2': i_f})])
assert s(field.get_schema()) == {
keyword: [n_f.get_schema()]
}
assert s(field.get_schema(role='role_1')) == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
assert s(field.get_schema(role='role_2')) == {
keyword: [n_f.get_schema(), i_f.get_schema()]
}
field = field_cls(Var({
'role_1': [n_f, Var({'role_1': s_f}), Var({'role_2': i_f})],
'role_2': [Var({'role_2': i_f})],
}, propagate='role_1'))
assert s(field.get_schema(role='role_1')) == {
keyword: [n_f.get_schema(), s_f.get_schema()]
}
with pytest.raises(SchemaGenerationException):
field.get_schema(role='role_2')
def test_dict_field():
s_f = StringField()
_ = lambda value: Var({'role_1': value})
field = DictField(properties=Var(
{
'role_1': {'name': Var({'role_1': s_f})},
'role_2': {'name': Var({'role_2': s_f})},
},
propagate='role_1'
), pattern_properties=Var(
{
'role_1': {'.*': Var({'role_1': s_f})},
'role_2': {'.*': Var({'role_2': s_f})},
},
propagate='role_1'
), additional_properties=_(s_f), min_properties=_(1), max_properties=_(2))
assert s(field.get_schema()) == s({
'type': 'object'
})
assert s(field.get_schema(role='role_1')) == s({
'type': 'object',
'properties': {
'name': s_f.get_schema(),
},
'patternProperties': {
'.*': s_f.get_schema(),
},
'additionalProperties': s_f.get_schema(),
'minProperties': 1,
'maxProperties': 2,
})
assert s(field.get_schema(role='role_2')) == s({
'type': 'object',
'properties': {},
'patternProperties': {},
})
class AlertEntity(DocumentWithoutAddProp):
field = StringField(required=True, pattern="^\w+$")
label = StringField(required=True, max_length=30)
type = StringField(required=True,
enum=["text", "singleSelect", "checkbox", "radio", "singleSelectSplunkSearch"])
help = StringField(max_length=200)
defaultValue = OneOfField([
NumberField(),
StringField(max_length=250),
BooleanField()
])
required = BooleanField()
search = StringField(max_length=200)
valueField = StringField(max_length=200)
labelField = StringField(max_length=200)
options = DictField(
properties={
"items": ArrayField(DocumentField(ValueLabelPair, as_ref=True))
}
)
from jsl import (StringField, ArrayField, Var, DictField, BaseOfField,
NotField, Document, DocumentField)
a = StringField()
b = StringField()
c = StringField()
d = StringField()
e = StringField()
f = StringField()
g = StringField()
h = StringField()
j = StringField()
def test_array_field():
field = ArrayField(Var({
'role_1': a,
'role_2': b,
}),
additional_items=Var({
'role_3': c,
'role_4': d,
}))
assert set(field.iter_all_fields()) == set([a, b, c, d])
field = ArrayField(Var({
'role_1': (a, b),
'role_2': c
}),
additional_items=d)
assert set(field.iter_all_fields()) == set([a, b, c, d])
class Fields(Document):
scope = StringField()
mapping = DictField()
class TimestampInfo(Document):
field = StringField(required=True)
format = StringField(required=True)
class EventInfo(Document):
enum = DictField(
additional_properties=ArrayField(StringField(eql_name)))
fields = ArrayField(StringField(eql_name))
def test_not_field():
s_f = StringField()
field = NotField(Var({'role_1': s_f}))
assert field.get_schema() == {'not': {}}
assert field.get_schema(role='role_1') == {'not': s_f.get_schema()}
class AnalyticMetadata(Document):
"""Base class for all analytics. Can be extended for cloud."""
id = StringField(pattern=UUID_PATTERN, required=True)
categories = ArrayField(StringField(enum=['detect', 'hunt']),
required=True)
contributors = ArrayField(StringField(), required=True)
confidence = StringField(enum=['low', 'medium', 'high'], required=True)
created_date = StringField(required=True)
description = StringField(required=True)
name = StringField(required=True)
notes = StringField(required=False)
os = StringField(required=False)
references = ArrayField(StringField(), required=False)
tactics = ArrayField(StringField(enum=TACTICS), required=False)
tags = ArrayField(StringField(), required=False)
techniques = ArrayField(StringField(), required=False)
updated_date = StringField(required=True)
class Fields(BaseNormalization.Fields):
mapping = StrictDict(
{k: StringField()
for k in domain_schema['fields']})
