def test_schema_registry_with_anyof_type(): schema_registry.add('soi_id', {'id': { 'anyof_type': ['string', 'integer'] }}) schema = {'soi': {'schema': 'soi_id'}} assert_success({'soi': {'id': 'hello'}}, schema)
def load_schemas(): # type: () -> dict """ Parse all schemas defined in current directory and return it as a map of validators :return: """ root = dirname(__file__) files = [f for f in listdir(root) if f.endswith('.yaml')] versions = 'versions.yaml' if versions not in files: raise RuntimeError('missing required config versions.yaml') files.remove(versions) # sort versions in asc order files = list(sorted(files)) for file in files: with open(join(root, file)) as f: conf = yaml.load(f, Loader=yaml.SafeLoader) conf_name, _ = splitext(basename(file)) for schema_name, desc in conf.get('schemas', {}).items(): schema_registry.add(f'{conf_name}.{schema_name}', desc) for rule_name, desc in conf.get('rules', {}).items(): rules_set_registry.add(f'{conf_name}.{rule_name}', desc) with open(join(root, versions)) as f: wconf = yaml.load(f, Loader=yaml.SafeLoader) if '_all' not in wconf: raise RuntimeError('invalid versions.yaml: missing _all section') if 'versions' not in wconf: raise RuntimeError('invalid versions.yaml: missing versions section') res = { 'default': OptionalValidator(wconf['_all'], allow_unknown=True) } for ver, desc in wconf['versions'].items(): desc.update(wconf['_all']) res[str(ver)] = OptionalValidator(desc) return res
def test_schema_normalization_does_not_abort(rule): schema_registry.clear() schema_registry.add( "schema_ref", {}, ) validator = Validator( schema={ "field": { rule: {"type": "string"}, "schema": "schema_ref", }, } ) assert validator.schema["field"][rule]["type"] == ("string",)
def register_schemas(schema_names, schemas, verbose=True): """ Register a list of schemas, with corresponding names. """ # Register the schemas list(map(lambda n, s: schema_registry.add(n, s), schema_names, schemas)) if verbose: # Print print("Registered schemas in Cerberus: ") list(map(lambda n: print(f'- {n}'), schema_names))
def test_schema_registry_simple(): schema_registry.add('foo', {'bar': {'type': 'string'}}) schema = {'a': {'schema': 'foo'}, 'b': {'schema': 'foo'}} document = {'a': {'bar': 'a'}, 'b': {'bar': 'b'}} assert_success(document, schema)
def test_top_level_reference(): schema_registry.add('peng', {'foo': {'type': 'integer'}}) document = {'foo': 42} assert_success(document, 'peng')
def test_schema_registry_with_anyof_type(): schema_registry.add('soi_id', {'id': {'anyof_type': ['string', 'integer']}}) schema = {'soi': {'schema': 'soi_id'}} assert_success({'soi': {'id': 'hello'}}, schema)
from cerberus import rules_set_registry, schema_registry, TypeDefinition, Validator from cerberus.benchmarks import DOCUMENTS_PATH rules_set_registry.add("path_rules", {"coerce": Path, "type": "path"}) schema_registry.add( "field_3_schema", { # an outer rule requires all fields' values to be a list "field_31": {"contains": 0, "empty": False}, "field_32": { "default": [None, None, None], "items": [ {"type": "integer"}, {"type": "string"}, {"type": ["integer", "string"]}, ], "itemsrules": {"nullable": True}, }, }, ) def schema_1_field_3_allow_unknown_check_with(field, value, error): if len(value) > 9: error(field, "Requires a smaller list.") schema_1 = {
# -*- coding: utf-8 -*- from cerberus import Validator, schema_registry from sentry_sdk import configure_scope import io, struct schema_registry.add( 'staff', { 'staff_type': { 'type': 'string', 'allowed': ['directors', 'writers', 'producers', 'roles'] }, 'name': { 'type': 'string' }, 'photo': { 'type': 'string' }, 'role': { 'type': 'string' }, }) movie_schema = { 'id': { 'type': 'string' }, 'meta_ids': { 'type': 'dict' }, 'title': { 'type': 'string'
def _set_metadata_registry(): schema_registry.add( "TranslatedText", { "lang": { "type": "string" }, "_content": { "type": "string", "required": True } }) schema_registry.add( "Alias", { "Context": { "type": "string", "required": True }, "Name": { "type": "string", "required": True } }) schema_registry.add( "Description", { "TranslatedText": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("TranslatedText") } } }) schema_registry.add("title", {"_content": { "type": "string", "required": True }}) schema_registry.add( "leaf", { "ID": { "type": "string", "required": True }, "href": { "type": "string", "required": True }, "title": { "type": "dict", "schema": schema_registry.get("title") } }) schema_registry.add( "WhereClauseRef", {"WhereClauseOID": { "type": "string", "required": True }}) schema_registry.add( "ValueListRef", {"ValueListOID": { "type": "string", "required": True }}) schema_registry.add( "PDFPageRef", { "Type": { "type": "string", "required": True, "allowed": ["PhysicalRef", "NamedDestination"] }, "PageRefs": { "type": "string" }, "FirstPage": { "type": "integer" }, "LastPage": { "type": "integer" }, "Title": { "type": "string" } }) schema_registry.add( "DocumentRef", { "leafID": { "type": "string", "required": True }, "PDFPageRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("PDFPageRef") } } }) schema_registry.add( "ItemRef", { "ItemOID": { "type": "string", "required": True }, "OrderNumber": { "type": "integer", "required": False }, "Mandatory": { "type": "string", "required": False, "allowed": ["Yes", "No"] }, "KeySequence": { "type": "integer", "required": False }, "MethodOID": { "type": "string", "required": False }, "Role": { "type": "string", "required": False }, "RoleCodeListOID": { "type": "string", "required": False }, "IsNonStandard": { "type": "string", "allowed": ["Yes"] }, "HasNoData": { "type": "string", "allowed": ["Yes"] }, "WhereClauseRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("WhereClauseRef") } } }) schema_registry.add("SubClass", { "Name": { "type": "string" }, "ParentClass": { "type": "string" } }) schema_registry.add( "Class", { "Name": { "type": "string" }, "SubClass": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("SubClass") } } }) schema_registry.add( "ItemGroupDef", { "OID": { "type": "string", "required": True }, "Name": { "type": "string", "required": True }, "Repeating": { "type": "string", "allowed": ["Yes", "No"] }, "IsReferenceData": { "type": "string", "allowed": ["Yes", "No"] }, "SASDatasetName": { "type": "string" }, "Domain": { "type": "string" }, "Origin": { "type": "string" }, "Purpose": { "type": "string" }, "Structure": { "type": "string", "required": True }, "ArchiveLocationID": { "type": "string" }, "CommentOID": { "type": "string" }, "IsNonStandard": { "type": "string", "allowed": ["Yes"] }, "StandardOID": { "type": "string" }, "HasNoData": { "type": "string", "allowed": ["Yes"] }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "ItemRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("ItemRef") } }, "Alias": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Alias") } }, "Class": { "type": "dict", "schema": schema_registry.get("Class") }, "leaf": { "type": "dict", "schema": schema_registry.get("leaf") } }) schema_registry.add( "FormalExpression", { "Context": { "type": "string", "required": True }, "_content": { "type": "string", "required": True } }) schema_registry.add( "RangeCheck", { "Comparator": { "type": "string", "allowed": ["LT", "LE", "GT", "GE", "EQ", "NE", "IN", "NOTIN"] }, "SoftHard": { "type": "string", "allowed": ["Soft", "Hard"] }, "ItemOID": { "type": "string", "required": True }, "CheckValue": { "type": "list", "schema": { "type": "dict", "schema": { "_content": { "type": "string" } } } } }) schema_registry.add( "Origin", { "Type": { "type": "string", "required": True, "allowed": [ "Collected", "Derived", "Assigned", "Protocol", "Predecessor", "Not Available" ] }, "Source": { "type": "string", "allowed": ["Subject", "Investigator", "Vendor", "Sponsor"] }, "DocumentRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("DocumentRef") } }, "Description": { "type": "dict", "schema": schema_registry.get("Description") } }) schema_registry.add("CodeListRef", {"CodeListOID": {"type": "string"}}) schema_registry.add( "ItemDef", { "OID": { "type": "string", "required": True }, "Name": { "type": "string", "required": True }, "DataType": { "type": "string", "allowed": [ "text", "integer", "float", "date", "time", "datetime", "string", "boolean", "double", "hexBinary", "base64Binary", "hexFloat", "base64Float", "partialDate", "partialTime", "partialDatetime", "durationDatetime", "intervalDatetime", "incompleteDatetime", "incompleteDate", "incompleteTime", "URI" ] }, "Length": { "type": "integer" }, "SignificantDigits": { "type": "integer" }, "SASFieldName": { "type": "string" }, "DisplayFormat": { "type": "string" }, "CommentOID": { "type": "string" }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "CodeListRef": { "type": "dict", "schema": schema_registry.get("CodeListRef") }, "Origin": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Origin") } }, "ValueListRef": { "type": "dict", "schema": schema_registry.get("ValueListRef") }, "Alias": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Alias") } } }) schema_registry.add( "CodeListItem", { "CodedValue": { "type": "string", "required": True }, "Rank": { "type": "float" }, "OrderNumber": { "type": "integer" }, "ExtendedValue": { "type": "string", "allowed": ["Yes"] }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "Decode": { "type": "dict", "schema": { "TranslatedText": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("TranslatedText") } } } }, "Alias": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Alias") } } }) schema_registry.add( "EnumeratedItem", { "CodedValue": { "type": "string", "required": True }, "Rank": { "type": "float" }, "OrderNumber": { "type": "integer" }, "ExtendedValue": { "type": "string", "allowed": ["Yes"] }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "Alias": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Alias") } } }) schema_registry.add( "ExternalCodeList", { "Dictionary": { "type": "string" }, "Version": { "type": "string" }, "ref": { "type": "string" }, "href": { "type": "string" } }) schema_registry.add( "CodeList", { "OID": { "type": "string", "required": True }, "Name": { "type": "string", "required": True }, "DataType": { "type": "string", "allowed": ["text", "integer", "float", "string"] }, "IsNonStandard": { "type": "string", "allowed": ["Yes"] }, "StandardOID": { "type": "string" }, "CommentOID": { "type": "string" }, "SASFormatName": { "type": "string" }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "CodeListItem": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("CodeListItem") } }, "EnumeratedItem": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("EnumeratedItem") } }, "ExternalCodeList": { "type": "dict", "schema": schema_registry.get("ExternalCodeList") }, "Alias": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Alias") } } }) schema_registry.add( "AnnotatedCRF", { "DocumentRef": { "type": "dict", "schema": schema_registry.get("DocumentRef") } }) schema_registry.add( "SupplementalDoc", { "DocumentRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("DocumentRef") } } }) schema_registry.add( "WhereClauseDef", { "OID": { "type": "string", "required": True }, "CommentOID": { "type": "string" }, "RangeCheck": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("RangeCheck") } } }) schema_registry.add( "ValueListDef", { "OID": { "type": "string", "required": True }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "ItemRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("ItemRef") } } }) schema_registry.add( "CommentDef", { "OID": { "type": "string", "required": True }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "DocumentRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("DocumentRef") } } }) schema_registry.add( "MethodDef", { "OID": { "type": "string", "required": True }, "Name": { "type": "string", "required": True }, "Type": { "type": "string", "required": True, "allowed": ["Computation", "Imputation", "Transpose", "Other"] }, "Description": { "type": "dict", "schema": schema_registry.get("Description") }, "FormalExpression": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("FormalExpression") } }, "Alias": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Alias") } }, "DocumentRef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("DocumentRef") } } }) schema_registry.add( "Standard", { "OID": { "type": "string", "required": True }, "Name": { "type": "string", "required": True }, "Type": { "type": "string", "required": True }, "PublishingSet": { "type": "string" }, "Version": { "type": "string", "required": True }, "Status": { "type": "string" }, "CommentOID": { "type": "string" } }) schema_registry.add( "Standards", { "Standard": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("Standard") } } }) schema_registry.add( "MetaDataVersion", { "OID": { "type": "string", "required": True }, "Name": { "type": "string", "required": True }, "Description": { "type": "string" }, "DefineVersion": { "type": "string", "required": True }, "CommentOID": { "type": "string" }, "Standards": { "type": "dict", "schema": schema_registry.get("Standards") }, "AnnotatedCRF": { "type": "dict", "schema": schema_registry.get("AnnotatedCRF") }, "SupplementalDoc": { "type": "dict", "schema": schema_registry.get("SupplementalDoc") }, "ValueListDef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("ValueListDef") } }, "WhereClauseDef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("WhereClauseDef") } }, "ItemGroupDef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("ItemGroupDef") } }, "ItemDef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("ItemDef") } }, "CodeList": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("CodeList") } }, "MethodDef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("MethodDef") } }, "CommentDef": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("CommentDef") } }, "leaf": { "type": "list", "schema": { "type": "dict", "schema": schema_registry.get("leaf") } } }) schema_registry.add("StudyName", {"_content": { "type": "string", "required": True }}) schema_registry.add("StudyDescription", {"_content": { "type": "string", "required": True }}) schema_registry.add("ProtocolName", {"_content": { "type": "string", "required": True }}) schema_registry.add( "GlobalVariables", { "StudyName": { "type": "dict", "schema": schema_registry.get("StudyName") }, "StudyDescription": { "type": "dict", "schema": schema_registry.get("StudyDescription") }, "ProtocolName": { "type": "dict", "schema": schema_registry.get("ProtocolName") } }) schema_registry.add( "Study", { "OID": { "type": "string", "required": True }, "GlobalVariables": { "type": "dict", "schema": schema_registry.get("GlobalVariables") }, "MetaDataVersion": { "type": "dict", "schema": schema_registry.get("MetaDataVersion") } })
def _set_metadata_registry(): """ a cerberus json schema has been generated from the odm_1_3_2 model """ schema_registry.add("TranslatedText", {"lang": {"type": "string"}, "_content": {"type": "string", "required": True}}) schema_registry.add("Alias", { "Context": {"type": "string", "required": True}, "Name": {"type": "string", "required": True} }) schema_registry.add("Description", {"TranslatedText": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}) schema_registry.add("StudyEventRef", { "StudyEventOID": {"type": "string", "required": True}, "OrderNumber": {"type": "integer", "required": False}, "Mandatory": {"type": "string", "allowed": ["Yes", "No"]}, "CollectionExceptionOID": {"type": "string", "required": False} }) schema_registry.add("Protocol", { "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "StudyEventRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("StudyEventRef")}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("FormRef", { "FormOID": {"type": "string", "required": True}, "OrderNumber": {"type": "integer", "required": False}, "Mandatory": {"type": "string", "allowed": ["Yes", "No"]}, "CollectionExceptionOID": {"type": "string", "required": False} }) schema_registry.add("StudyEventDef", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Repeating": {"type": "string", "allowed": ["Yes", "No"]}, "Type": {"type": "string", "allowed": ["Scheduled", "Unscheduled", "Common"]}, "Category": {"type": "string"}, "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "FormRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormRef")}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("ItemGroupRef", { "ItemGroupOID": {"type": "string", "required": True}, "OrderNumber": {"type": "integer", "required": False}, "Mandatory": {"type": "string", "allowed": ["Yes", "No"]}, "CollectionExceptionOID": {"type": "string", "required": False} }) schema_registry.add("ArchiveLayout", { "OID": {"type": "string", "required": True}, "PdfFileName": {"type": "string", "required": True}, }) schema_registry.add("FormDef", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Repeating": {"type": "string", "allowed": ["Yes", "No"]}, "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "ItemGroupRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemGroupRef")}}, "ArchiveLayout": {"type": "list","schema": {"type": "dict", "schema": schema_registry.get("ArchiveLayout")}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("ItemRef", { "ItemOID": {"type": "string", "required": True}, "OrderNumber": { "type": "integer", "required": False}, "Mandatory": { "type": "string", "required": False, "allowed": ["Yes", "No"]}, "KeySequence": { "type": "integer", "required": False}, "MethodOID": {"type": "string", "required": False}, "Role": {"type": "string", "required": False}, "RoleCodeListOID": {"type": "string", "required": False}, "CollectionExceptionOID": {"type": "string", "required": False}, }) schema_registry.add("ItemGroupDef", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Repeating": {"type": "string", "allowed": ["Yes", "No"]}, "IsReferenceData": {"type": "string", "allowed": ["Yes", "No"]}, "SASDatasetName": {"type": "string"}, "Domain": {"type": "string"}, "Origin": {"type": "string"}, "Purpose": {"type": "string"}, "Comment": {"type": "string"}, "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "ItemRef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemRef")}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("FormalExpression", { "Context": {"type": "string", "required": True}, "_content": {"type": "string", "required": True} }) schema_registry.add("MeasurementUnitRef", { "MeasurementUnitOID": {"type": "string", "required": True} }) schema_registry.add("RangeCheck", { "Comparator": {"type": "string", "allowed": ["LT", "LE", "GT", "GE", "EQ", "NE", "IN", "NOTIN"]}, "SoftHard": {"type": "string", "allowed": ["Soft", "Hard"]}, "CheckValue": {"type": "list", "schema": {"type": "dict", "schema": {"_content": {"type": "string"}}}}, "FormalExpression": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormalExpression")}}, "MeasurementUnitRef": {"type": "dict", "schema": schema_registry.get("MeasurementUnitRef")}, "ErrorMessage": {"type": "dict", "schema": {"TranslatedText": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}} }) schema_registry.add("ExternalQuestion", { "Dictionary": {"type": "string", "required": False}, "Version": {"type": "string", "required": False}, "Code": {"type": "string", "required": False}, }) schema_registry.add("CodeListRef", { "CodeListOID": {"type": "string", "required": True} }) schema_registry.add("ItemDef", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "DataType": {"type": "string", "allowed": ["text", "integer", "float", "date", "time", "datetime", "string", "boolean", "double", "hexBinary", "base64Binary", "hexFloat", "base64Float", "partialDate", "partialTime", "partialDatetime", "durationDatetime", "intervalDatetime", "incompleteDatetime", "incompleteDate", "incompleteTime", "URI"]}, "Length": {"type": "integer"}, "SignificantDigits": {"type": "integer"}, "SASFieldName": {"type": "string"}, "SDSVarName": {"type": "string"}, "Origin": {"type": "string"}, "Comment": {"type": "string"}, "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "Question": {"type": "dict", "schema": {"TranslatedText": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}}, "ExternalQuestion": {"type": "dict", "schema": schema_registry.get("ExternalQuestion")}, "MeasurementUnitRef": {"type": "list", "schema": { "type": "dict", "schema": schema_registry.get("MeasurementUnitRef")}}, "CodeListRef": {"type": "dict", "schema": schema_registry.get("CodeListRef")}, "ErrorMessage": {"type": "dict", "schema": {"TranslatedText": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}}, "RangeCheck": {"type": "dict", "schema": schema_registry.get("RangeCheck")}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("CodeListItem", { "CodedValue": {"type": "string", "required": True}, "Rank": {"type": "float"}, "OrderNumber": {"type": "integer"}, "Decode": {"type": "dict", "schema": {"TranslatedText": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("EnumeratedItem", { "CodedValue": {"type": "string", "required": True}, "Rank": {"type": "float"}, "OrderNumber": {"type": "integer"}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("ExternalCodeList", { "Dictionary": {"type": "string"}, "Version": {"type": "string"}, "ref": {"type": "string"}, "href": {"type": "string"} }) schema_registry.add("CodeList", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "DataType": {"type": "string", "allowed": ["text", "integer", "float", "string"]}, "SASFormatName": {"type": "string"}, "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "CodeListItem": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("CodeListItem")}}, "EnumeratedItem": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("EnumeratedItem")}}, "ExternalCodeList": {"type": "dict", "schema": schema_registry.get("ExternalCodeList")}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("ConditionDef", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Description": {"type": "dict", "schema": schema_registry.get("Description")}, "FormalExpression": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormalExpression")}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("MethodDef", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Type": {"type": "string", "required": True, "allowed": ["Computation", "Imputation", "Transpose", "Other"]}, "Description": {"type": "dict", "required": True, "schema": schema_registry.get("Description")}, "FormalExpression": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormalExpression")}}, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("Include", { "StudyOID": {"type": "string", "required": True}, "MetaDataVersionOID": {"type": "string", "required": True} }) schema_registry.add("MetaDataVersion", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Description": {"type": "string"}, "Include": {"type": "dict", "schema": schema_registry.get("Include")}, "Protocol": {"type": "dict", "schema": schema_registry.get("Protocol")}, "StudyEventDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("StudyEventDef")}}, "FormDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("FormDef")}}, "ItemGroupDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemGroupDef")}}, "ItemDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ItemDef")}}, "CodeList": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("CodeList")}}, "ConditionDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("ConditionDef")}}, "MethodDef": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("MethodDef")}} }) schema_registry.add("MeasurementUnit", { "OID": {"type": "string", "required": True}, "Name": {"type": "string", "required": True}, "Symbol": {"type": "dict", "schema" : { "TranslatedText": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("TranslatedText")}}} }, "Alias": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Alias")}} }) schema_registry.add("BasicDefinitions", { "MeasurementUnit": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("MeasurementUnit")}} }) schema_registry.add("Study", { "OID": {"type": "string", "required": True}, "GlobalVariables": {"type": "dict", "required": True, "schema": { "StudyName": {"schema": {"_content": {"type": "string", "required": True}}}, "StudyDescription": {"schema": {"_content": {"type": "string", "required": True}}}, "ProtocolName": {"schema": {"_content": {"type": "string", "required": True}}} } }, "BasicDefinitions": {"type": "dict", "schema": schema_registry.get("BasicDefinitions")}, "MetaDataVersion": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("MetaDataVersion")}} }) schema_registry.add("ODM", { "Description": {"type": "string"}, "FileType": {"type": "string", "required": True, "allowed": ["Snapshot", "Transactional"]}, "Granularity": {"type": "string", "allowed": ["All", "Metadata", "AdminData", "ReferenceData", "AllClinicalData", "SingleSite", "SingleSubject"]}, "Archival": {"type": "string", "allowed": ["Yes", "No"]}, "FileOID": {"type": "string", "required": True}, "CreationDateTime": {"type": "string", "required": True}, "PriorFileOID": {"type": "string"}, "AsOfDateTime": {"type": "string"}, "FileType": {"type": "string", "required": True, "allowed": ["Snapshot", "Transactional"]}, "ODMVersion": {"type": "string", "allowed": ["1.3.2", "1.3"]}, "Originator": {"type": "string"}, "SourceSystem": {"type": "string"}, "SourceSystemVersion": {"type": "string"}, "ID": {"type": "string"}, "schemaLocation": {"type": "string"}, "Study": {"type": "list", "schema": {"type": "dict", "schema": schema_registry.get("Study")}} })
from cerberus import schema_registry __author__ = 'Björn Schrader <*****@*****.**>' __license__ = 'GNU General Public License http://www.gnu.org/licenses/gpl.html' __copyright__ = 'Copyright (C) 2017 by WLANThermo Project - Released under terms of the GPLv3 License' # Fields in all sensor definitions schema_registry.add( 'sensor', { 'name': { 'required': True, 'type': 'string' }, 'unit': { 'required': True, 'type': 'string' }, 'type': { 'required': True, 'type': 'string' }, }) # Fields in sensor definition for type 'voltage' schema_registry.add('sensor_voltage', {}) # Fields in sensor definition for type 'voltage' schema_registry.add('sensor_resistance', {}) # Fields in sensor definition for type 'ntc'
def file_validator(log): schema_registry.add('organizational_unit', yaml.load(ORGANIZATIONAL_UNIT_SCHEMA)) schema_registry.add('sc_policy', yaml.load(SC_POLICY_SCHEMA)) schema_registry.add('team', yaml.load(TEAM_SCHEMA)) schema_registry.add('account', yaml.load(ACCOUNT_SCHEMA)) schema_registry.add('user', yaml.load(USER_SCHEMA)) schema_registry.add('group', yaml.load(GROUP_SCHEMA)) schema_registry.add('local_user', yaml.load(LOCAL_USER_SCHEMA)) schema_registry.add('delegation', yaml.load(DELEGATION_SCHEMA)) schema_registry.add('custom_policy', yaml.load(CUSTOM_POLICY_SCHEMA)) log.debug("adding subschema to schema_registry: {}".format( schema_registry.all().keys())) vfile = Validator(yaml.load(SPEC_FILE_SCHEMA)) log.debug("file_validator_schema: {}".format(vfile.schema)) return vfile
class Config: """ This class is unique in that no instances of it should be created. It is used as a wrapper around a Dictionary object named config that is contains important values used throughout DuoLogSync. The _config class variable should only be accessed through getter and setter methods and should only be set once. There are useful methods defined in this class for generating a config Dictionary from a YAML file, validating the config against a Schema and setting defaults for a config Dictionary when optional fields are not given values. """ # Format type constants CEF = 'CEF' JSON = 'JSON' # Log type constants ADMIN = 'adminaction' AUTH = 'auth' TELEPHONY = 'telephony' DIRECTORY_DEFAULT = '/tmp' LOG_FILEPATH_DEFAULT = DIRECTORY_DEFAULT + '/' + 'duologsync.log' LOG_FORMAT_DEFAULT = 'JSON' API_OFFSET_DEFAULT = 180 API_TIMEOUT_DEFAULT = 120 CHECKPOINTING_ENABLED_DEFAULT = False CHECKPOINTING_DIRECTORY_DEFAULT = DIRECTORY_DEFAULT PROXY_SERVER_DEFAULT = '' PROXY_PORT_DEFAULT = 0 SYSLOG_ENABLED_DEFAULT = False SYSLOG_FORMAT_DEFAULT = 'RFC5424' # To understand these schema definitions better, compare side-by-side to # the template_config.yml file # Version of the config file VERSION = {'type': 'string', 'empty': False, 'required': True} # Fields for changing the functionality of DuoLogSync DLS_SETTINGS = { 'type': 'dict', 'default': {}, 'schema': { 'log_filepath': { 'type': 'string', 'empty': False, 'default': LOG_FILEPATH_DEFAULT }, 'log_format': { 'type': 'string', 'empty': False, 'allowed': [CEF, JSON], 'default': LOG_FORMAT_DEFAULT }, 'api': { 'type': 'dict', 'default': {}, 'schema': { 'offset': { 'type': 'number', 'min': 0, 'max': 180, 'default': API_OFFSET_DEFAULT }, 'timeout': { 'type': 'number', 'default': API_TIMEOUT_DEFAULT } } }, 'checkpointing': { 'type': 'dict', 'default': {}, 'schema': { 'enabled': { 'type': 'boolean', 'default': CHECKPOINTING_ENABLED_DEFAULT }, 'directory': { 'type': 'string', 'empty': False, 'default': CHECKPOINTING_DIRECTORY_DEFAULT } } }, 'proxy': { 'type': 'dict', 'default': {}, 'schema': { 'proxy_server': { 'type': 'string', 'default': PROXY_SERVER_DEFAULT }, 'proxy_port': { 'type': 'number', 'default': PROXY_PORT_DEFAULT } } }, 'syslog': { 'type': 'dict', 'default': {}, 'schema': { 'enabled': { 'type': 'boolean', 'default': SYSLOG_ENABLED_DEFAULT }, 'format': { 'type': 'string', 'empty': False, 'default': SYSLOG_FORMAT_DEFAULT } } } } } # Schema for a server inside of servers list schema_registry.add( 'server', { 'id': { 'type': 'string', 'required': True, 'empty': False }, 'hostname': { 'type': 'string', 'required': True, 'empty': False }, 'port': { 'type': 'integer', 'required': True, 'min': 0, 'max': 65535 }, 'protocol': { 'type': 'string', 'required': True, 'oneof': [{ 'allowed': ['TCPSSL'], 'dependencies': ['cert_filepath'] }, { 'allowed': ['TCP', 'UDP'] }] }, 'cert_filepath': { 'type': 'string', 'empty': False } }) # List of servers and how DLS will communicate with them SERVERS = { 'type': 'list', 'required': True, 'minlength': 1, 'schema': { 'type': 'dict', 'schema': 'server' } } # Describe which servers the logs of certain endpoints should be sent to schema_registry.add( 'endpoint_server_mapping', { 'server': { 'type': 'string', 'empty': False, 'required': True }, 'endpoints': { 'type': 'list', 'empty': False, 'required': True, 'allowed': [ADMIN, AUTH, TELEPHONY] } }) # Account definition, which is used to access Duo logs and tell DLS which # logs to fetch and to which servers those logs should be sent ACCOUNT = { 'type': 'dict', 'required': True, 'schema': { 'skey': { 'type': 'string', 'required': True, 'empty': False }, 'ikey': { 'type': 'string', 'required': True, 'empty': False }, 'hostname': { 'type': 'string', 'required': True, 'empty': False }, 'endpoint_server_mappings': { 'type': 'list', 'empty': False, 'required': True, 'schema': { 'type': 'dict', 'schema': 'endpoint_server_mapping' } }, 'is_msp': { 'type': 'boolean', 'default': False }, 'block_list': { 'type': 'list', 'default': [] } } } # Schema for validating the structure of a config dictionary generated from # a user-provided YAML file SCHEMA = { 'version': VERSION, 'dls_settings': DLS_SETTINGS, 'servers': SERVERS, 'account': ACCOUNT } # Generate a Validator object with the given schema SCHEMA_VALIDATOR = Validator(SCHEMA) # Private class variable, should not be accessed directly, only through # getter and setter methods _config = None # Used to ensure that the _config variable is set once and only once _config_is_set = False @classmethod def _check_config_is_set(cls): """ Used to check that this Config object is set before trying to access or set values """ if cls._config_is_set: return raise RuntimeError('Cannot access values of config before setting it') @classmethod def set_config(cls, config): """ Function used to set the config of a Config object once and only once. @param config Dictionary used to set a Config object's 'config' instance variable """ if cls._config_is_set: raise RuntimeError('Config object already set. Cannot set Config ' 'object more than once') cls._config = config cls._config_is_set = True @classmethod def get_value(cls, keys): """ Getter for a Config object's 'config' instance variable """ cls._check_config_is_set() curr_value = cls._config for key in keys: curr_value = curr_value.get(key) if curr_value is None: raise ValueError(f"{key} is an invalid key for this Config") return curr_value @classmethod def get_log_filepath(cls): """@return the filepath where DLS program messages should be saved""" return cls.get_value(['dls_settings', 'log_filepath']) @classmethod def get_log_format(cls): """@return how Duo logs should be formatted""" return cls.get_value(['dls_settings', 'log_format']) @classmethod def get_api_offset(cls): """@return the timestamp from which record retrieval should begin""" return cls.get_value(['dls_settings', 'api', 'offset']) @classmethod def get_api_timeout(cls): """@return the seconds to wait between API calls""" return cls.get_value(['dls_settings', 'api', 'timeout']) @classmethod def get_checkpointing_enabled(cls): """@return whether checkpoint files should be used to recover offsets""" return cls.get_value(['dls_settings', 'checkpointing', 'enabled']) @classmethod def get_checkpoint_dir(cls): """@return the directory where checkpoint files should be stored""" return cls.get_value(['dls_settings', 'checkpointing', 'directory']) @classmethod def get_servers(cls): """@return the list of servers to which Duo logs will be written""" return cls.get_value(['servers']) @classmethod def get_account_ikey(cls): """@return the ikey of the account in config""" return cls.get_value(['account', 'ikey']) @classmethod def get_account_skey(cls): """@return the skey of the account in config""" return cls.get_value(['account', 'skey']) @classmethod def get_account_hostname(cls): """@return the hostname of the account in config""" return cls.get_value(['account', 'hostname']) @classmethod def get_account_endpoint_server_mappings(cls): """@return the endpoint_server_mappings of the account in config""" return cls.get_value(['account', 'endpoint_server_mappings']) @classmethod def get_account_block_list(cls): """@return the block_list of the account in config""" return cls.get_value(['account', 'block_list']) @classmethod def account_is_msp(cls): """@return whether the account in config is an MSP account""" return cls.get_value(['account', 'is_msp']) @classmethod def get_proxy_server(cls): """@return the proxy_server in config""" return cls.get_value(['dls_settings', 'proxy', 'proxy_server']) @classmethod def get_proxy_port(cls): """@return the proxy_port in config""" return cls.get_value(['dls_settings', 'proxy', 'proxy_port']) @classmethod def get_syslog_enabled(cls): """@return whether syslog headers should be added to JSON logs""" return cls.get_value(['dls_settings', 'syslog', 'enabled']) @classmethod def get_syslog_format(cls): """@return the format of syslog header should be used""" return cls.get_value(['dls_settings', 'syslog', 'format']) @classmethod def create_config(cls, config_filepath): """ Attemp to read the file at config_filepath and generate a config Dictionary object based on a defined JSON schema @param config_filepath File from which to generate a config object """ shutdown_reason = None try: with open(config_filepath) as config_file: # PyYAML gives better error messages for streams than for files config_file_data = config_file.read() config = yaml.full_load(config_file_data) # Check config against a schema to ensure all the needed fields # and values are defined config = cls._validate_and_normalize_config(config) if config.get('dls_settings').get('api').get( 'timeout') < cls.API_TIMEOUT_DEFAULT: config['dls_settings']['api'][ 'timeout'] = cls.API_TIMEOUT_DEFAULT Program.log( 'DuoLogSync: Setting default api timeout to 120 seconds.' ) # Will occur when given a bad filepath or a bad file except OSError as os_error: shutdown_reason = f"{os_error}" Program.log('DuoLogSync: Failed to open the config file. Check ' 'that the filename is correct') # Will occur if the config file does not contain valid YAML except YAMLError as yaml_error: shutdown_reason = f"{yaml_error}" Program.log('DuoLogSync: Failed to parse the config file. Check ' 'that the config file has valid YAML.') # Validation of the config against a schema failed except ValueError: shutdown_reason = f"{cls.SCHEMA_VALIDATOR.errors}" Program.log('DuoLogSync: Validation of the config file failed. ' 'Check that required fields have proper values.') # No exception raised during the try block, return config else: # Calculate offset as a timestamp and rewrite its value in config offset = config.get('dls_settings').get('api').get('offset') offset = datetime.utcnow() - timedelta(days=offset) config['dls_settings']['api']['offset'] = int(offset.timestamp()) return config # At this point, it is guaranteed that an exception was raised, which # means that it is shutdown time Program.initiate_shutdown(shutdown_reason) return None @classmethod def _validate_and_normalize_config(cls, config): """ Use a schema and the cerberus library to validate that the given config dictionary has a valid structure @param config Dictionary for which to validate the structure """ # Config is not a valid structure if not cls.SCHEMA_VALIDATOR.validate(config): raise ValueError config = cls.SCHEMA_VALIDATOR.normalized(config) return config @staticmethod def get_value_from_keys(dictionary, keys): """ Drill down into dictionary to retrieve a value given a list of keys @param dictionary dict to retrieve a value from @param fields List of fields to follow to retrieve a value @return value from the log found after following the list of keys given """ value = dictionary for key in keys: value = value.get(key) if value is None: break return value
def register_schema(fpath): schemas_dict = yaml.load(open(fpath, 'r')) for schema in schemas_dict: schema_registry.add(schema, schemas_dict[schema])
'type': 'string', 'regex': REX_UUID, }) rules_set_registry.add('UUIDNullable', { 'type': 'string', 'regex': REX_UUID, 'nullable': True, }) schema_registry.add( 'JSUser', { 'nickname': { 'type': 'string', }, 'discriminator': { 'type': 'string', 'regex': r'^[0-9a-f]{4}$' } } ) schema_registry.add('JSChatEventMessagePayload', { 'user': { 'schema': 'JSUser', }, 'message': { 'type': 'string', } })