def test_schema_registry_with_anyof_type():
schema_registry.add('soi_id',
{'id': {
'anyof_type': ['string', 'integer']
}})
schema = {'soi': {'schema': 'soi_id'}}
assert_success({'soi': {'id': 'hello'}}, schema)
def load_schemas():
# type: () -> dict
""" Parse all schemas defined in current directory and return it as a map of validators
:return:
"""
root = dirname(__file__)
files = [f for f in listdir(root) if f.endswith('.yaml')]
versions = 'versions.yaml'
if versions not in files:
raise RuntimeError('missing required config versions.yaml')
files.remove(versions)
# sort versions in asc order
files = list(sorted(files))
for file in files:
with open(join(root, file)) as f:
conf = yaml.load(f, Loader=yaml.SafeLoader)
conf_name, _ = splitext(basename(file))
for schema_name, desc in conf.get('schemas', {}).items():
schema_registry.add(f'{conf_name}.{schema_name}', desc)
for rule_name, desc in conf.get('rules', {}).items():
rules_set_registry.add(f'{conf_name}.{rule_name}', desc)
with open(join(root, versions)) as f:
wconf = yaml.load(f, Loader=yaml.SafeLoader)
if '_all' not in wconf:
raise RuntimeError('invalid versions.yaml: missing _all section')
if 'versions' not in wconf:
raise RuntimeError('invalid versions.yaml: missing versions section')
res = {
'default': OptionalValidator(wconf['_all'], allow_unknown=True)
}
for ver, desc in wconf['versions'].items():
desc.update(wconf['_all'])
res[str(ver)] = OptionalValidator(desc)
return res
def test_schema_normalization_does_not_abort(rule):
schema_registry.clear()
schema_registry.add(
"schema_ref",
{},
)
validator = Validator(
schema={
"field": {
rule: {"type": "string"},
"schema": "schema_ref",
},
}
)
assert validator.schema["field"][rule]["type"] == ("string",)
def register_schemas(schema_names, schemas, verbose=True):
"""
Register a list of schemas, with corresponding names.
"""
# Register the schemas
list(map(lambda n, s: schema_registry.add(n, s), schema_names, schemas))
if verbose:
# Print
print("Registered schemas in Cerberus: ")
list(map(lambda n: print(f'- {n}'), schema_names))
def test_schema_registry_simple():
schema_registry.add('foo', {'bar': {'type': 'string'}})
schema = {'a': {'schema': 'foo'}, 'b': {'schema': 'foo'}}
document = {'a': {'bar': 'a'}, 'b': {'bar': 'b'}}
assert_success(document, schema)
def test_top_level_reference():
schema_registry.add('peng', {'foo': {'type': 'integer'}})
document = {'foo': 42}
assert_success(document, 'peng')
def test_schema_registry_simple():
schema_registry.add('foo', {'bar': {'type': 'string'}})
schema = {'a': {'schema': 'foo'},
'b': {'schema': 'foo'}}
document = {'a': {'bar': 'a'}, 'b': {'bar': 'b'}}
assert_success(document, schema)
def test_schema_registry_with_anyof_type():
schema_registry.add('soi_id', {'id': {'anyof_type': ['string', 'integer']}})
schema = {'soi': {'schema': 'soi_id'}}
assert_success({'soi': {'id': 'hello'}}, schema)
def test_top_level_reference():
schema_registry.add('peng', {'foo': {'type': 'integer'}})
document = {'foo': 42}
assert_success(document, 'peng')
from cerberus import rules_set_registry, schema_registry, TypeDefinition, Validator
from cerberus.benchmarks import DOCUMENTS_PATH
rules_set_registry.add("path_rules", {"coerce": Path, "type": "path"})
schema_registry.add(
"field_3_schema",
{
# an outer rule requires all fields' values to be a list
"field_31": {"contains": 0, "empty": False},
"field_32": {
"default": [None, None, None],
"items": [
{"type": "integer"},
{"type": "string"},
{"type": ["integer", "string"]},
],
"itemsrules": {"nullable": True},
},
},
)
def schema_1_field_3_allow_unknown_check_with(field, value, error):
if len(value) > 9:
error(field, "Requires a smaller list.")
schema_1 = {
# -*- coding: utf-8 -*-
from cerberus import Validator, schema_registry
from sentry_sdk import configure_scope
import io, struct
schema_registry.add(
'staff', {
'staff_type': {
'type': 'string',
'allowed': ['directors', 'writers', 'producers', 'roles']
},
'name': {
'type': 'string'
},
'photo': {
'type': 'string'
},
'role': {
'type': 'string'
},
})
movie_schema = {
'id': {
'type': 'string'
},
'meta_ids': {
'type': 'dict'
},
'title': {
'type': 'string'
def _set_metadata_registry():
schema_registry.add(
"TranslatedText", {
"lang": {
"type": "string"
},
"_content": {
"type": "string",
"required": True
}
})
schema_registry.add(
"Alias", {
"Context": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
}
})
schema_registry.add(
"Description", {
"TranslatedText": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("TranslatedText")
}
}
})
schema_registry.add("title",
{"_content": {
"type": "string",
"required": True
}})
schema_registry.add(
"leaf", {
"ID": {
"type": "string",
"required": True
},
"href": {
"type": "string",
"required": True
},
"title": {
"type": "dict",
"schema": schema_registry.get("title")
}
})
schema_registry.add(
"WhereClauseRef",
{"WhereClauseOID": {
"type": "string",
"required": True
}})
schema_registry.add(
"ValueListRef",
{"ValueListOID": {
"type": "string",
"required": True
}})
schema_registry.add(
"PDFPageRef", {
"Type": {
"type": "string",
"required": True,
"allowed": ["PhysicalRef", "NamedDestination"]
},
"PageRefs": {
"type": "string"
},
"FirstPage": {
"type": "integer"
},
"LastPage": {
"type": "integer"
},
"Title": {
"type": "string"
}
})
schema_registry.add(
"DocumentRef", {
"leafID": {
"type": "string",
"required": True
},
"PDFPageRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("PDFPageRef")
}
}
})
schema_registry.add(
"ItemRef", {
"ItemOID": {
"type": "string",
"required": True
},
"OrderNumber": {
"type": "integer",
"required": False
},
"Mandatory": {
"type": "string",
"required": False,
"allowed": ["Yes", "No"]
},
"KeySequence": {
"type": "integer",
"required": False
},
"MethodOID": {
"type": "string",
"required": False
},
"Role": {
"type": "string",
"required": False
},
"RoleCodeListOID": {
"type": "string",
"required": False
},
"IsNonStandard": {
"type": "string",
"allowed": ["Yes"]
},
"HasNoData": {
"type": "string",
"allowed": ["Yes"]
},
"WhereClauseRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("WhereClauseRef")
}
}
})
schema_registry.add("SubClass", {
"Name": {
"type": "string"
},
"ParentClass": {
"type": "string"
}
})
schema_registry.add(
"Class", {
"Name": {
"type": "string"
},
"SubClass": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("SubClass")
}
}
})
schema_registry.add(
"ItemGroupDef", {
"OID": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
},
"Repeating": {
"type": "string",
"allowed": ["Yes", "No"]
},
"IsReferenceData": {
"type": "string",
"allowed": ["Yes", "No"]
},
"SASDatasetName": {
"type": "string"
},
"Domain": {
"type": "string"
},
"Origin": {
"type": "string"
},
"Purpose": {
"type": "string"
},
"Structure": {
"type": "string",
"required": True
},
"ArchiveLocationID": {
"type": "string"
},
"CommentOID": {
"type": "string"
},
"IsNonStandard": {
"type": "string",
"allowed": ["Yes"]
},
"StandardOID": {
"type": "string"
},
"HasNoData": {
"type": "string",
"allowed": ["Yes"]
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"ItemRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("ItemRef")
}
},
"Alias": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Alias")
}
},
"Class": {
"type": "dict",
"schema": schema_registry.get("Class")
},
"leaf": {
"type": "dict",
"schema": schema_registry.get("leaf")
}
})
schema_registry.add(
"FormalExpression", {
"Context": {
"type": "string",
"required": True
},
"_content": {
"type": "string",
"required": True
}
})
schema_registry.add(
"RangeCheck", {
"Comparator": {
"type": "string",
"allowed":
["LT", "LE", "GT", "GE", "EQ", "NE", "IN", "NOTIN"]
},
"SoftHard": {
"type": "string",
"allowed": ["Soft", "Hard"]
},
"ItemOID": {
"type": "string",
"required": True
},
"CheckValue": {
"type": "list",
"schema": {
"type": "dict",
"schema": {
"_content": {
"type": "string"
}
}
}
}
})
schema_registry.add(
"Origin", {
"Type": {
"type":
"string",
"required":
True,
"allowed": [
"Collected", "Derived", "Assigned", "Protocol",
"Predecessor", "Not Available"
]
},
"Source": {
"type": "string",
"allowed":
["Subject", "Investigator", "Vendor", "Sponsor"]
},
"DocumentRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("DocumentRef")
}
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
}
})
schema_registry.add("CodeListRef", {"CodeListOID": {"type": "string"}})
schema_registry.add(
"ItemDef", {
"OID": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
},
"DataType": {
"type":
"string",
"allowed": [
"text", "integer", "float", "date", "time", "datetime",
"string", "boolean", "double", "hexBinary",
"base64Binary", "hexFloat", "base64Float",
"partialDate", "partialTime", "partialDatetime",
"durationDatetime", "intervalDatetime",
"incompleteDatetime", "incompleteDate",
"incompleteTime", "URI"
]
},
"Length": {
"type": "integer"
},
"SignificantDigits": {
"type": "integer"
},
"SASFieldName": {
"type": "string"
},
"DisplayFormat": {
"type": "string"
},
"CommentOID": {
"type": "string"
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"CodeListRef": {
"type": "dict",
"schema": schema_registry.get("CodeListRef")
},
"Origin": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Origin")
}
},
"ValueListRef": {
"type": "dict",
"schema": schema_registry.get("ValueListRef")
},
"Alias": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Alias")
}
}
})
schema_registry.add(
"CodeListItem", {
"CodedValue": {
"type": "string",
"required": True
},
"Rank": {
"type": "float"
},
"OrderNumber": {
"type": "integer"
},
"ExtendedValue": {
"type": "string",
"allowed": ["Yes"]
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"Decode": {
"type": "dict",
"schema": {
"TranslatedText": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("TranslatedText")
}
}
}
},
"Alias": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Alias")
}
}
})
schema_registry.add(
"EnumeratedItem", {
"CodedValue": {
"type": "string",
"required": True
},
"Rank": {
"type": "float"
},
"OrderNumber": {
"type": "integer"
},
"ExtendedValue": {
"type": "string",
"allowed": ["Yes"]
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"Alias": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Alias")
}
}
})
schema_registry.add(
"ExternalCodeList", {
"Dictionary": {
"type": "string"
},
"Version": {
"type": "string"
},
"ref": {
"type": "string"
},
"href": {
"type": "string"
}
})
schema_registry.add(
"CodeList", {
"OID": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
},
"DataType": {
"type": "string",
"allowed": ["text", "integer", "float", "string"]
},
"IsNonStandard": {
"type": "string",
"allowed": ["Yes"]
},
"StandardOID": {
"type": "string"
},
"CommentOID": {
"type": "string"
},
"SASFormatName": {
"type": "string"
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"CodeListItem": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("CodeListItem")
}
},
"EnumeratedItem": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("EnumeratedItem")
}
},
"ExternalCodeList": {
"type": "dict",
"schema": schema_registry.get("ExternalCodeList")
},
"Alias": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Alias")
}
}
})
schema_registry.add(
"AnnotatedCRF", {
"DocumentRef": {
"type": "dict",
"schema": schema_registry.get("DocumentRef")
}
})
schema_registry.add(
"SupplementalDoc", {
"DocumentRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("DocumentRef")
}
}
})
schema_registry.add(
"WhereClauseDef", {
"OID": {
"type": "string",
"required": True
},
"CommentOID": {
"type": "string"
},
"RangeCheck": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("RangeCheck")
}
}
})
schema_registry.add(
"ValueListDef", {
"OID": {
"type": "string",
"required": True
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"ItemRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("ItemRef")
}
}
})
schema_registry.add(
"CommentDef", {
"OID": {
"type": "string",
"required": True
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"DocumentRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("DocumentRef")
}
}
})
schema_registry.add(
"MethodDef", {
"OID": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
},
"Type": {
"type": "string",
"required": True,
"allowed":
["Computation", "Imputation", "Transpose", "Other"]
},
"Description": {
"type": "dict",
"schema": schema_registry.get("Description")
},
"FormalExpression": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("FormalExpression")
}
},
"Alias": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Alias")
}
},
"DocumentRef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("DocumentRef")
}
}
})
schema_registry.add(
"Standard", {
"OID": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
},
"Type": {
"type": "string",
"required": True
},
"PublishingSet": {
"type": "string"
},
"Version": {
"type": "string",
"required": True
},
"Status": {
"type": "string"
},
"CommentOID": {
"type": "string"
}
})
schema_registry.add(
"Standards", {
"Standard": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("Standard")
}
}
})
schema_registry.add(
"MetaDataVersion", {
"OID": {
"type": "string",
"required": True
},
"Name": {
"type": "string",
"required": True
},
"Description": {
"type": "string"
},
"DefineVersion": {
"type": "string",
"required": True
},
"CommentOID": {
"type": "string"
},
"Standards": {
"type": "dict",
"schema": schema_registry.get("Standards")
},
"AnnotatedCRF": {
"type": "dict",
"schema": schema_registry.get("AnnotatedCRF")
},
"SupplementalDoc": {
"type": "dict",
"schema": schema_registry.get("SupplementalDoc")
},
"ValueListDef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("ValueListDef")
}
},
"WhereClauseDef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("WhereClauseDef")
}
},
"ItemGroupDef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("ItemGroupDef")
}
},
"ItemDef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("ItemDef")
}
},
"CodeList": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("CodeList")
}
},
"MethodDef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("MethodDef")
}
},
"CommentDef": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("CommentDef")
}
},
"leaf": {
"type": "list",
"schema": {
"type": "dict",
"schema": schema_registry.get("leaf")
}
}
})
schema_registry.add("StudyName",
{"_content": {
"type": "string",
"required": True
}})
schema_registry.add("StudyDescription",
{"_content": {
"type": "string",
"required": True
}})
schema_registry.add("ProtocolName",
{"_content": {
"type": "string",
"required": True
}})
schema_registry.add(
"GlobalVariables", {
"StudyName": {
"type": "dict",
"schema": schema_registry.get("StudyName")
},
"StudyDescription": {
"type": "dict",
"schema": schema_registry.get("StudyDescription")
},
"ProtocolName": {
"type": "dict",
"schema": schema_registry.get("ProtocolName")
}
})
schema_registry.add(
"Study", {
"OID": {
"type": "string",
"required": True
},
"GlobalVariables": {
"type": "dict",
"schema": schema_registry.get("GlobalVariables")
},
"MetaDataVersion": {
"type": "dict",
"schema": schema_registry.get("MetaDataVersion")
}
})
def _set_metadata_registry():
""" a cerberus json schema has been generated from the odm_1_3_2 model """
schema_registry.add("TranslatedText", {"lang": {"type": "string"},
"_content": {"type": "string", "required": True}})
schema_registry.add("Alias", {
"Context": {"type": "string", "required": True},
"Name": {"type": "string", "required": True}
})
schema_registry.add("Description", {"TranslatedText": {"type": "list",
"schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}})
schema_registry.add("StudyEventRef", {
"StudyEventOID": {"type": "string", "required": True},
"OrderNumber": {"type": "integer", "required": False},
"Mandatory": {"type": "string", "allowed": ["Yes", "No"]},
"CollectionExceptionOID": {"type": "string", "required": False}
})
schema_registry.add("Protocol", {
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"StudyEventRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("StudyEventRef")}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("FormRef", {
"FormOID": {"type": "string", "required": True},
"OrderNumber": {"type": "integer", "required": False},
"Mandatory": {"type": "string", "allowed": ["Yes", "No"]},
"CollectionExceptionOID": {"type": "string", "required": False}
})
schema_registry.add("StudyEventDef", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Repeating": {"type": "string", "allowed": ["Yes", "No"]},
"Type": {"type": "string", "allowed": ["Scheduled", "Unscheduled", "Common"]},
"Category": {"type": "string"},
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"FormRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormRef")}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("ItemGroupRef", {
"ItemGroupOID": {"type": "string", "required": True},
"OrderNumber": {"type": "integer", "required": False},
"Mandatory": {"type": "string", "allowed": ["Yes", "No"]},
"CollectionExceptionOID": {"type": "string", "required": False}
})
schema_registry.add("ArchiveLayout", {
"OID": {"type": "string", "required": True},
"PdfFileName": {"type": "string", "required": True},
})
schema_registry.add("FormDef", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Repeating": {"type": "string", "allowed": ["Yes", "No"]},
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"ItemGroupRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemGroupRef")}},
"ArchiveLayout": {"type": "list","schema": {"type": "dict", "schema": schema_registry.get("ArchiveLayout")}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("ItemRef", {
"ItemOID": {"type": "string", "required": True},
"OrderNumber": { "type": "integer", "required": False},
"Mandatory": { "type": "string", "required": False, "allowed": ["Yes", "No"]},
"KeySequence": { "type": "integer", "required": False},
"MethodOID": {"type": "string", "required": False},
"Role": {"type": "string", "required": False},
"RoleCodeListOID": {"type": "string", "required": False},
"CollectionExceptionOID": {"type": "string", "required": False},
})
schema_registry.add("ItemGroupDef", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Repeating": {"type": "string", "allowed": ["Yes", "No"]},
"IsReferenceData": {"type": "string", "allowed": ["Yes", "No"]},
"SASDatasetName": {"type": "string"},
"Domain": {"type": "string"},
"Origin": {"type": "string"},
"Purpose": {"type": "string"},
"Comment": {"type": "string"},
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"ItemRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemRef")}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("FormalExpression", {
"Context": {"type": "string", "required": True},
"_content": {"type": "string", "required": True}
})
schema_registry.add("MeasurementUnitRef", {
"MeasurementUnitOID": {"type": "string", "required": True}
})
schema_registry.add("RangeCheck", {
"Comparator": {"type": "string", "allowed": ["LT", "LE", "GT", "GE", "EQ", "NE", "IN", "NOTIN"]},
"SoftHard": {"type": "string", "allowed": ["Soft", "Hard"]},
"CheckValue": {"type": "list", "schema": {"type": "dict", "schema": {"_content": {"type": "string"}}}},
"FormalExpression": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormalExpression")}},
"MeasurementUnitRef": {"type": "dict", "schema": schema_registry.get("MeasurementUnitRef")},
"ErrorMessage": {"type": "dict", "schema": {"TranslatedText": {"type": "list",
"schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}}
})
schema_registry.add("ExternalQuestion", {
"Dictionary": {"type": "string", "required": False},
"Version": {"type": "string", "required": False},
"Code": {"type": "string", "required": False},
})
schema_registry.add("CodeListRef", {
"CodeListOID": {"type": "string", "required": True}
})
schema_registry.add("ItemDef", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"DataType": {"type": "string", "allowed": ["text", "integer", "float", "date", "time", "datetime", "string",
"boolean", "double", "hexBinary", "base64Binary", "hexFloat",
"base64Float", "partialDate", "partialTime", "partialDatetime",
"durationDatetime", "intervalDatetime", "incompleteDatetime",
"incompleteDate", "incompleteTime", "URI"]},
"Length": {"type": "integer"},
"SignificantDigits": {"type": "integer"},
"SASFieldName": {"type": "string"},
"SDSVarName": {"type": "string"},
"Origin": {"type": "string"},
"Comment": {"type": "string"},
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"Question": {"type": "dict", "schema": {"TranslatedText": {"type": "list",
"schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}},
"ExternalQuestion": {"type": "dict", "schema": schema_registry.get("ExternalQuestion")},
"MeasurementUnitRef": {"type": "list", "schema": { "type": "dict", "schema": schema_registry.get("MeasurementUnitRef")}},
"CodeListRef": {"type": "dict", "schema": schema_registry.get("CodeListRef")},
"ErrorMessage": {"type": "dict", "schema": {"TranslatedText": {"type": "list",
"schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}},
"RangeCheck": {"type": "dict", "schema": schema_registry.get("RangeCheck")},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("CodeListItem", {
"CodedValue": {"type": "string", "required": True},
"Rank": {"type": "float"},
"OrderNumber": {"type": "integer"},
"Decode": {"type": "dict", "schema": {"TranslatedText": {"type": "list",
"schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("EnumeratedItem", {
"CodedValue": {"type": "string", "required": True},
"Rank": {"type": "float"},
"OrderNumber": {"type": "integer"},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("ExternalCodeList", {
"Dictionary": {"type": "string"},
"Version": {"type": "string"},
"ref": {"type": "string"},
"href": {"type": "string"}
})
schema_registry.add("CodeList", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"DataType": {"type": "string", "allowed": ["text", "integer", "float", "string"]},
"SASFormatName": {"type": "string"},
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"CodeListItem": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("CodeListItem")}},
"EnumeratedItem": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("EnumeratedItem")}},
"ExternalCodeList": {"type": "dict", "schema": schema_registry.get("ExternalCodeList")},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("ConditionDef", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Description": {"type": "dict", "schema": schema_registry.get("Description")},
"FormalExpression": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormalExpression")}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("MethodDef", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Type": {"type": "string", "required": True, "allowed": ["Computation", "Imputation", "Transpose", "Other"]},
"Description": {"type": "dict", "required": True, "schema": schema_registry.get("Description")},
"FormalExpression": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormalExpression")}},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("Include", {
"StudyOID": {"type": "string", "required": True},
"MetaDataVersionOID": {"type": "string", "required": True}
})
schema_registry.add("MetaDataVersion", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Description": {"type": "string"},
"Include": {"type": "dict", "schema": schema_registry.get("Include")},
"Protocol": {"type": "dict", "schema": schema_registry.get("Protocol")},
"StudyEventDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("StudyEventDef")}},
"FormDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormDef")}},
"ItemGroupDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemGroupDef")}},
"ItemDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemDef")}},
"CodeList": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("CodeList")}},
"ConditionDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ConditionDef")}},
"MethodDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("MethodDef")}}
})
schema_registry.add("MeasurementUnit", {
"OID": {"type": "string", "required": True},
"Name": {"type": "string", "required": True},
"Symbol": {"type": "dict", "schema" : {
"TranslatedText": {"type": "list",
"schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}
},
"Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}}
})
schema_registry.add("BasicDefinitions", {
"MeasurementUnit": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("MeasurementUnit")}}
})
schema_registry.add("Study", {
"OID": {"type": "string", "required": True},
"GlobalVariables": {"type": "dict", "required": True, "schema": {
"StudyName": {"schema": {"_content": {"type": "string", "required": True}}},
"StudyDescription": {"schema": {"_content": {"type": "string", "required": True}}},
"ProtocolName": {"schema": {"_content": {"type": "string", "required": True}}}
}
},
"BasicDefinitions": {"type": "dict", "schema": schema_registry.get("BasicDefinitions")},
"MetaDataVersion": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("MetaDataVersion")}}
})
schema_registry.add("ODM", {
"Description": {"type": "string"},
"FileType": {"type": "string", "required": True, "allowed": ["Snapshot", "Transactional"]},
"Granularity": {"type": "string", "allowed": ["All", "Metadata", "AdminData", "ReferenceData",
"AllClinicalData", "SingleSite", "SingleSubject"]},
"Archival": {"type": "string", "allowed": ["Yes", "No"]},
"FileOID": {"type": "string", "required": True},
"CreationDateTime": {"type": "string", "required": True},
"PriorFileOID": {"type": "string"},
"AsOfDateTime": {"type": "string"},
"FileType": {"type": "string", "required": True, "allowed": ["Snapshot", "Transactional"]},
"ODMVersion": {"type": "string", "allowed": ["1.3.2", "1.3"]},
"Originator": {"type": "string"},
"SourceSystem": {"type": "string"},
"SourceSystemVersion": {"type": "string"},
"ID": {"type": "string"},
"schemaLocation": {"type": "string"},
"Study": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Study")}}
})
from cerberus import schema_registry
__author__ = 'Björn Schrader <*****@*****.**>'
__license__ = 'GNU General Public License http://www.gnu.org/licenses/gpl.html'
__copyright__ = 'Copyright (C) 2017 by WLANThermo Project - Released under terms of the GPLv3 License'
# Fields in all sensor definitions
schema_registry.add(
'sensor', {
'name': {
'required': True,
'type': 'string'
},
'unit': {
'required': True,
'type': 'string'
},
'type': {
'required': True,
'type': 'string'
},
})
# Fields in sensor definition for type 'voltage'
schema_registry.add('sensor_voltage', {})
# Fields in sensor definition for type 'voltage'
schema_registry.add('sensor_resistance', {})
# Fields in sensor definition for type 'ntc'
def file_validator(log):
schema_registry.add('organizational_unit',
yaml.load(ORGANIZATIONAL_UNIT_SCHEMA))
schema_registry.add('sc_policy', yaml.load(SC_POLICY_SCHEMA))
schema_registry.add('team', yaml.load(TEAM_SCHEMA))
schema_registry.add('account', yaml.load(ACCOUNT_SCHEMA))
schema_registry.add('user', yaml.load(USER_SCHEMA))
schema_registry.add('group', yaml.load(GROUP_SCHEMA))
schema_registry.add('local_user', yaml.load(LOCAL_USER_SCHEMA))
schema_registry.add('delegation', yaml.load(DELEGATION_SCHEMA))
schema_registry.add('custom_policy', yaml.load(CUSTOM_POLICY_SCHEMA))
log.debug("adding subschema to schema_registry: {}".format(
schema_registry.all().keys()))
vfile = Validator(yaml.load(SPEC_FILE_SCHEMA))
log.debug("file_validator_schema: {}".format(vfile.schema))
return vfile
class Config:
"""
This class is unique in that no instances of it should be created. It is
used as a wrapper around a Dictionary object named config that is contains
important values used throughout DuoLogSync. The _config class variable
should only be accessed through getter and setter methods and should only
be set once. There are useful methods defined in this class for generating
a config Dictionary from a YAML file, validating the config against a
Schema and setting defaults for a config Dictionary when optional fields
are not given values.
"""
# Format type constants
CEF = 'CEF'
JSON = 'JSON'
# Log type constants
ADMIN = 'adminaction'
AUTH = 'auth'
TELEPHONY = 'telephony'
DIRECTORY_DEFAULT = '/tmp'
LOG_FILEPATH_DEFAULT = DIRECTORY_DEFAULT + '/' + 'duologsync.log'
LOG_FORMAT_DEFAULT = 'JSON'
API_OFFSET_DEFAULT = 180
API_TIMEOUT_DEFAULT = 120
CHECKPOINTING_ENABLED_DEFAULT = False
CHECKPOINTING_DIRECTORY_DEFAULT = DIRECTORY_DEFAULT
PROXY_SERVER_DEFAULT = ''
PROXY_PORT_DEFAULT = 0
SYSLOG_ENABLED_DEFAULT = False
SYSLOG_FORMAT_DEFAULT = 'RFC5424'
# To understand these schema definitions better, compare side-by-side to
# the template_config.yml file
# Version of the config file
VERSION = {'type': 'string', 'empty': False, 'required': True}
# Fields for changing the functionality of DuoLogSync
DLS_SETTINGS = {
'type': 'dict',
'default': {},
'schema': {
'log_filepath': {
'type': 'string',
'empty': False,
'default': LOG_FILEPATH_DEFAULT
},
'log_format': {
'type': 'string',
'empty': False,
'allowed': [CEF, JSON],
'default': LOG_FORMAT_DEFAULT
},
'api': {
'type': 'dict',
'default': {},
'schema': {
'offset': {
'type': 'number',
'min': 0,
'max': 180,
'default': API_OFFSET_DEFAULT
},
'timeout': {
'type': 'number',
'default': API_TIMEOUT_DEFAULT
}
}
},
'checkpointing': {
'type': 'dict',
'default': {},
'schema': {
'enabled': {
'type': 'boolean',
'default': CHECKPOINTING_ENABLED_DEFAULT
},
'directory': {
'type': 'string',
'empty': False,
'default': CHECKPOINTING_DIRECTORY_DEFAULT
}
}
},
'proxy': {
'type': 'dict',
'default': {},
'schema': {
'proxy_server': {
'type': 'string',
'default': PROXY_SERVER_DEFAULT
},
'proxy_port': {
'type': 'number',
'default': PROXY_PORT_DEFAULT
}
}
},
'syslog': {
'type': 'dict',
'default': {},
'schema': {
'enabled': {
'type': 'boolean',
'default': SYSLOG_ENABLED_DEFAULT
},
'format': {
'type': 'string',
'empty': False,
'default': SYSLOG_FORMAT_DEFAULT
}
}
}
}
}
# Schema for a server inside of servers list
schema_registry.add(
'server', {
'id': {
'type': 'string',
'required': True,
'empty': False
},
'hostname': {
'type': 'string',
'required': True,
'empty': False
},
'port': {
'type': 'integer',
'required': True,
'min': 0,
'max': 65535
},
'protocol': {
'type':
'string',
'required':
True,
'oneof': [{
'allowed': ['TCPSSL'],
'dependencies': ['cert_filepath']
}, {
'allowed': ['TCP', 'UDP']
}]
},
'cert_filepath': {
'type': 'string',
'empty': False
}
})
# List of servers and how DLS will communicate with them
SERVERS = {
'type': 'list',
'required': True,
'minlength': 1,
'schema': {
'type': 'dict',
'schema': 'server'
}
}
# Describe which servers the logs of certain endpoints should be sent to
schema_registry.add(
'endpoint_server_mapping', {
'server': {
'type': 'string',
'empty': False,
'required': True
},
'endpoints': {
'type': 'list',
'empty': False,
'required': True,
'allowed': [ADMIN, AUTH, TELEPHONY]
}
})
# Account definition, which is used to access Duo logs and tell DLS which
# logs to fetch and to which servers those logs should be sent
ACCOUNT = {
'type': 'dict',
'required': True,
'schema': {
'skey': {
'type': 'string',
'required': True,
'empty': False
},
'ikey': {
'type': 'string',
'required': True,
'empty': False
},
'hostname': {
'type': 'string',
'required': True,
'empty': False
},
'endpoint_server_mappings': {
'type': 'list',
'empty': False,
'required': True,
'schema': {
'type': 'dict',
'schema': 'endpoint_server_mapping'
}
},
'is_msp': {
'type': 'boolean',
'default': False
},
'block_list': {
'type': 'list',
'default': []
}
}
}
# Schema for validating the structure of a config dictionary generated from
# a user-provided YAML file
SCHEMA = {
'version': VERSION,
'dls_settings': DLS_SETTINGS,
'servers': SERVERS,
'account': ACCOUNT
}
# Generate a Validator object with the given schema
SCHEMA_VALIDATOR = Validator(SCHEMA)
# Private class variable, should not be accessed directly, only through
# getter and setter methods
_config = None
# Used to ensure that the _config variable is set once and only once
_config_is_set = False
@classmethod
def _check_config_is_set(cls):
"""
Used to check that this Config object is set before trying to access
or set values
"""
if cls._config_is_set:
return
raise RuntimeError('Cannot access values of config before setting it')
@classmethod
def set_config(cls, config):
"""
Function used to set the config of a Config object once and only once.
@param config Dictionary used to set a Config object's 'config'
instance variable
"""
if cls._config_is_set:
raise RuntimeError('Config object already set. Cannot set Config '
'object more than once')
cls._config = config
cls._config_is_set = True
@classmethod
def get_value(cls, keys):
"""
Getter for a Config object's 'config' instance variable
"""
cls._check_config_is_set()
curr_value = cls._config
for key in keys:
curr_value = curr_value.get(key)
if curr_value is None:
raise ValueError(f"{key} is an invalid key for this Config")
return curr_value
@classmethod
def get_log_filepath(cls):
"""@return the filepath where DLS program messages should be saved"""
return cls.get_value(['dls_settings', 'log_filepath'])
@classmethod
def get_log_format(cls):
"""@return how Duo logs should be formatted"""
return cls.get_value(['dls_settings', 'log_format'])
@classmethod
def get_api_offset(cls):
"""@return the timestamp from which record retrieval should begin"""
return cls.get_value(['dls_settings', 'api', 'offset'])
@classmethod
def get_api_timeout(cls):
"""@return the seconds to wait between API calls"""
return cls.get_value(['dls_settings', 'api', 'timeout'])
@classmethod
def get_checkpointing_enabled(cls):
"""@return whether checkpoint files should be used to recover offsets"""
return cls.get_value(['dls_settings', 'checkpointing', 'enabled'])
@classmethod
def get_checkpoint_dir(cls):
"""@return the directory where checkpoint files should be stored"""
return cls.get_value(['dls_settings', 'checkpointing', 'directory'])
@classmethod
def get_servers(cls):
"""@return the list of servers to which Duo logs will be written"""
return cls.get_value(['servers'])
@classmethod
def get_account_ikey(cls):
"""@return the ikey of the account in config"""
return cls.get_value(['account', 'ikey'])
@classmethod
def get_account_skey(cls):
"""@return the skey of the account in config"""
return cls.get_value(['account', 'skey'])
@classmethod
def get_account_hostname(cls):
"""@return the hostname of the account in config"""
return cls.get_value(['account', 'hostname'])
@classmethod
def get_account_endpoint_server_mappings(cls):
"""@return the endpoint_server_mappings of the account in config"""
return cls.get_value(['account', 'endpoint_server_mappings'])
@classmethod
def get_account_block_list(cls):
"""@return the block_list of the account in config"""
return cls.get_value(['account', 'block_list'])
@classmethod
def account_is_msp(cls):
"""@return whether the account in config is an MSP account"""
return cls.get_value(['account', 'is_msp'])
@classmethod
def get_proxy_server(cls):
"""@return the proxy_server in config"""
return cls.get_value(['dls_settings', 'proxy', 'proxy_server'])
@classmethod
def get_proxy_port(cls):
"""@return the proxy_port in config"""
return cls.get_value(['dls_settings', 'proxy', 'proxy_port'])
@classmethod
def get_syslog_enabled(cls):
"""@return whether syslog headers should be added to JSON logs"""
return cls.get_value(['dls_settings', 'syslog', 'enabled'])
@classmethod
def get_syslog_format(cls):
"""@return the format of syslog header should be used"""
return cls.get_value(['dls_settings', 'syslog', 'format'])
@classmethod
def create_config(cls, config_filepath):
"""
Attemp to read the file at config_filepath and generate a config
Dictionary object based on a defined JSON schema
@param config_filepath File from which to generate a config object
"""
shutdown_reason = None
try:
with open(config_filepath) as config_file:
# PyYAML gives better error messages for streams than for files
config_file_data = config_file.read()
config = yaml.full_load(config_file_data)
# Check config against a schema to ensure all the needed fields
# and values are defined
config = cls._validate_and_normalize_config(config)
if config.get('dls_settings').get('api').get(
'timeout') < cls.API_TIMEOUT_DEFAULT:
config['dls_settings']['api'][
'timeout'] = cls.API_TIMEOUT_DEFAULT
Program.log(
'DuoLogSync: Setting default api timeout to 120 seconds.'
)
# Will occur when given a bad filepath or a bad file
except OSError as os_error:
shutdown_reason = f"{os_error}"
Program.log('DuoLogSync: Failed to open the config file. Check '
'that the filename is correct')
# Will occur if the config file does not contain valid YAML
except YAMLError as yaml_error:
shutdown_reason = f"{yaml_error}"
Program.log('DuoLogSync: Failed to parse the config file. Check '
'that the config file has valid YAML.')
# Validation of the config against a schema failed
except ValueError:
shutdown_reason = f"{cls.SCHEMA_VALIDATOR.errors}"
Program.log('DuoLogSync: Validation of the config file failed. '
'Check that required fields have proper values.')
# No exception raised during the try block, return config
else:
# Calculate offset as a timestamp and rewrite its value in config
offset = config.get('dls_settings').get('api').get('offset')
offset = datetime.utcnow() - timedelta(days=offset)
config['dls_settings']['api']['offset'] = int(offset.timestamp())
return config
# At this point, it is guaranteed that an exception was raised, which
# means that it is shutdown time
Program.initiate_shutdown(shutdown_reason)
return None
@classmethod
def _validate_and_normalize_config(cls, config):
"""
Use a schema and the cerberus library to validate that the given config
dictionary has a valid structure
@param config Dictionary for which to validate the structure
"""
# Config is not a valid structure
if not cls.SCHEMA_VALIDATOR.validate(config):
raise ValueError
config = cls.SCHEMA_VALIDATOR.normalized(config)
return config
@staticmethod
def get_value_from_keys(dictionary, keys):
"""
Drill down into dictionary to retrieve a value given a list of keys
@param dictionary dict to retrieve a value from
@param fields List of fields to follow to retrieve a value
@return value from the log found after following the list of keys given
"""
value = dictionary
for key in keys:
value = value.get(key)
if value is None:
break
return value
def register_schema(fpath):
schemas_dict = yaml.load(open(fpath, 'r'))
for schema in schemas_dict:
schema_registry.add(schema, schemas_dict[schema])
'type': 'string',
'regex': REX_UUID,
})
rules_set_registry.add('UUIDNullable', {
'type': 'string',
'regex': REX_UUID,
'nullable': True,
})
schema_registry.add(
'JSUser', {
'nickname': {
'type': 'string',
},
'discriminator': {
'type': 'string',
'regex': r'^[0-9a-f]{4}$'
}
}
)
schema_registry.add('JSChatEventMessagePayload', {
'user': {
'schema': 'JSUser',
},
'message': {
'type': 'string',
}
})
