def test_fixed_named_type(): """https://github.com/fastavro/fastavro/issues/450""" schema = { "type": "record", "name": "test_fixed_named_type", "fields": [ { "name": "test1", "type": { "type": "fixed", "name": "my_fixed", "size": 4, }, }, { "name": "test2", "type": "my_fixed", }, ], } record = {"test1": b"1234", "test2": b"4321"} parsed_schema = parse_schema(schema) validate(record, parsed_schema)
def test_validate_should_not_parse_schema_if_it_was_parsed_already(): named_schemas = {} parse_schema( { "name": "B", "type": "record", "fields": [{ "name": "bar", "type": "string" }] }, named_schemas, ) a_schema = parse_schema( { "name": "A", "type": "record", "fields": [{ "name": "b", "type": "B" }] }, named_schemas, ) records = [{"b": {"bar": "bar"}}] validate_many(records, a_schema) validate(records[0], a_schema)
def test_validator_numeric(): for datum, schema in [ (1, 'int'), (1, 'long'), (1.0, 'float'), (1.0, 'double'), (1, 'float'), (1, 'double'), ]: validate(datum, schema) for datum, schema in [ (1.0, 'int'), (1.0, 'long'), ("1.0", 'float'), ("1.0", 'double'), ("1", 'float'), ("1", 'double'), (True, 'int'), (True, 'long'), (True, 'float'), (True, 'double'), (False, 'int'), (False, 'long'), (False, 'float'), (False, 'double'), ]: with pytest.raises(ValidationError): validate(datum, schema) pytest.fail("{} should not validate as {}".format(datum, schema))
def test_enum_named_type(): """https://github.com/fastavro/fastavro/issues/450""" schema = { "type": "record", "name": "test_enum_named_type", "fields": [ { "name": "test1", "type": { "type": "enum", "name": "my_enum", "symbols": ["FOO", "BAR"], }, }, { "name": "test2", "type": "my_enum", }, ], } record = {"test1": "FOO", "test2": "BAR"} parsed_schema = parse_schema(schema) validate(record, parsed_schema)
def test_record_named_type(): """https://github.com/fastavro/fastavro/issues/450""" schema = { "type": "record", "name": "test_record_named_type", "fields": [{ "name": "test1", "type": { "type": "record", "name": "my_record", "fields": [{ "name": "field1", "type": "string", }] }, }, { "name": "test2", "type": "my_record", }] } record = {"test1": {"field1": "foo"}, "test2": {"field1": "bar"}} parsed_schema = parse_schema(schema) validate(record, parsed_schema)
def test_validate_array(): my_schema = { "fields": [ { "name": "array", "type": { "type": "array", "items": "string", }, }, ], "namespace": "namespace", "name": "test_validate_array", "type": "record" } datum = {"array": [1]} with pytest.raises(ValidationError) as exc: validate(datum, my_schema) for error in exc.value.errors: assert error.field == 'namespace.test_validate_array.array'
def test_validate_map(): my_schema = { "fields": [ { "name": "map", "type": { "type": "map", "values": "string", }, }, ], "namespace": "namespace", "name": "test_validate_map", "type": "record" } datum = {"map": {"key": 1}} with pytest.raises(ValidationError) as exc: validate(datum, my_schema) for error in exc.value.errors: assert error.field == 'namespace.test_validate_map.map'
def test_validate_with_unparsed_schema(): """https://github.com/fastavro/fastavro/issues/416""" schema = { "type": "record", "name": "my_schema_v1", "fields": [{ "name": "field1", "type": { "type": "record", "name": "my_field_type", "fields": [{ "name": "sub_field", "type": "string" }] } }, { "name": "field2", "type": "my_field_type" }] } datum = {"field1": {"sub_field": "foo"}, "field2": {"sub_field": "bar"}} validate(datum, schema)
def test_validator_numeric(): for datum, schema in [ (1, "int"), (1, "long"), (1.0, "float"), (1.0, "double"), (1, "float"), (1, "double"), ]: validate(datum, schema) for datum, schema in [ (1.0, "int"), (1.0, "long"), ("1.0", "float"), ("1.0", "double"), ("1", "float"), ("1", "double"), (True, "int"), (True, "long"), (True, "float"), (True, "double"), (False, "int"), (False, "long"), (False, "float"), (False, "double"), (datetime(2020, 1, 1), "int"), (datetime(2020, 1, 1), "long"), ]: with pytest.raises(ValidationError): validate(datum, schema) pytest.fail(f"{datum} should not validate as {schema}")
def test_validator_numeric(): for datum, schema in [ (1, 'int'), (1, 'long'), (1.0, 'float'), (1.0, 'double'), (1, 'float'), (1, 'double'), ]: validate(datum, schema) for datum, schema in [ (1.0, 'int'), (1.0, 'long'), ("1.0", 'float'), ("1.0", 'double'), ("1", 'float'), ("1", 'double'), (True, 'int'), (True, 'long'), (True, 'float'), (True, 'double'), (False, 'int'), (False, 'long'), (False, 'float'), (False, 'double'), ]: with pytest.raises(ValidationError): validate(datum, schema) pytest.fail("{} should not validate as {}".format(datum, schema))
def test_validator_logical(): """https://github.com/fastavro/fastavro/issues/365""" for datum, schema in [ (1, {"type": "long", "logicalType": "timestamp-micros"}), ]: validate(datum, schema) for datum, schema in [ ("foo", {"type": "long", "logicalType": "timestamp-micros"}), ]: with pytest.raises(ValidationError): validate(datum, schema) pytest.fail(f"{datum} should not validate as {schema}")
def test_validate_clobbering_SCHEMA_DEFS(): """https://github.com/fastavro/fastavro/issues/416""" schema_1 = { "type": "record", "name": "my_schema_v1", "fields": [ { "name": "field1", "type": { "type": "record", "name": "my_field_type", "fields": [{ "name": "sub_field", "type": "string" }], }, }, { "name": "field2", "type": "my_field_type" }, ], } parsed_schema_1 = parse_schema(schema_1) datum_1 = {"field1": {"sub_field": "foo"}, "field2": {"sub_field": "bar"}} validate(datum_1, parsed_schema_1) schema_2 = { "type": "record", "name": "my_schema_v2", "fields": [{ "name": "field1", "type": { "type": "record", "name": "my_field_type", "fields": [{ "name": "sub_field", "type": "int" }], }, }], } parse_schema(schema_2) validate(datum_1, parsed_schema_1)
def validate(self, msg, schema_name): logger.debug( f'Validating {msg} using the class {self.__class__.__name__} and schema {schema_name}' ) schema = self.schema_retriever.get_schema(schema_name=schema_name) if schema is None: return False else: return validation.validate(msg, schema, raise_errors=True)
def test_validator_numeric(): for datum, schema in [ (1, 'int'), (1, 'long'), (1.0, 'float'), (1.0, 'double'), (1, 'float'), (1, 'double'), ]: validate(datum, schema) for datum, schema in [ (1.0, 'int'), (1.0, 'long'), ("1.0", 'float'), ("1.0", 'double'), ("1", 'float'), ("1", 'double'), ]: with pytest.raises(ValidationError): validate(datum, schema)
def test_validate_map(): my_schema = { "fields": [ { "name": "map", "type": { "type": "map", "values": "string", }, }, ], "namespace": "namespace", "name": "test_validate_map", "type": "record" } datum = {"map": {"key": 1}} with pytest.raises(ValidationError) as exc: validate(datum, my_schema) for error in exc.value.errors: assert error.field == 'namespace.test_validate_map.map'
def test_validate_array(): my_schema = { "fields": [ { "name": "array", "type": { "type": "array", "items": "string", }, }, ], "namespace": "namespace", "name": "test_validate_array", "type": "record" } datum = {"array": [1]} with pytest.raises(ValidationError) as exc: validate(datum, my_schema) for error in exc.value.errors: assert error.field == 'namespace.test_validate_array.array'
def test_validator_numeric_numpy(): np_ints = [ np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, ] np_floats = [ np.float_, np.float16, np.float32, np.float64, ] schema_ints = ['int', 'long'] schema_floats = ['float', 'double'] # all these should work for nptype, schema in zip(np_ints, schema_ints): validate(nptype(1), schema) for nptype, schema in zip(np_ints, schema_floats): validate(nptype(1), schema) for nptype, schema in zip(np_floats, schema_floats): validate(nptype(1), schema) # these shouldn't work for nptype, schema in zip(np_floats, schema_ints): with pytest.raises(ValidationError): validate(nptype(1), schema)
def test_validator_numeric_numpy(): np_ints = [ np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, ] np_floats = [ np.float_, np.float16, np.float32, np.float64, ] schema_ints = ['int', 'long'] schema_floats = ['float', 'double'] # all these should work for nptype, schema in zip(np_ints, schema_ints): validate(nptype(1), schema) for nptype, schema in zip(np_ints, schema_floats): validate(nptype(1), schema) for nptype, schema in zip(np_floats, schema_floats): validate(nptype(1), schema) # these shouldn't work for nptype, schema in zip(np_floats, schema_ints): with pytest.raises(ValidationError): validate(nptype(1), schema)
def test_schema(self): self.assertTrue(validate(self.converted[0], self.SCHEMA))
"ontology_reference": "NCIT", "values": { "concept_code": "C164337" } }], "links": [{ "name": "member_of", "dst": "cohort", "multiplicity": "MANY_TO_ONE" }] } # Metadata assert validate( { "name": "Metadata", "misc": {}, "nodes": [icdc_case_meta, icdc_cohort_meta] }, pfb_schema) # data for PFB message: cohort_data = { "id": "n201", "cohort_description": "arm1", "cohort_dose": "10mg/kg" } assert validate(("icdc.cohort", cohort_data), pfb_schema) case_data = { "id": "n101",
from fastavro.validation import validate import json with open('schema.avsc', 'r') as schema_file: schema = json.loads(schema_file.read()) message = { 'age': 28.0, # 'myname': 'test', # 'asd': 'test2' } try: validate(message, schema) print('Message is matching schema') except Exception as ex: print(ex)
def test_schema(self): self.assertTrue(validate(self.type_converted, self.SCHEMA))
outdir = "avro-data" if not os.path.exists(outdir): os.makedirs(outdir) with open(f"avro/{document}.schema.json", "r") as f: schema_data = f.read() schema = avro.schema.Parse(schema_data) outfile = open(f"{outdir}/{document}.avro", "wb") writer = avro.datafile.DataFileWriter(outfile, avro.io.DatumWriter(), schema) with open(f"data/{document}.ndjson", "r") as f: data = f.readlines() try: orig = None for line in data: orig = json.loads(line) out = convert(orig, schema) writer.append(out) except: with open("test.json", "w") as f: json.dump(orig, f) with open("test-schema.json", "w") as f: json.dump(schema.to_json(), f, indent=2) validation.validate(out, parse_schema(schema.to_json())) writer.close()