def test_fixed_named_type():
"""https://github.com/fastavro/fastavro/issues/450"""
schema = {
"type":
"record",
"name":
"test_fixed_named_type",
"fields": [
{
"name": "test1",
"type": {
"type": "fixed",
"name": "my_fixed",
"size": 4,
},
},
{
"name": "test2",
"type": "my_fixed",
},
],
}
record = {"test1": b"1234", "test2": b"4321"}
parsed_schema = parse_schema(schema)
validate(record, parsed_schema)
def test_validate_should_not_parse_schema_if_it_was_parsed_already():
named_schemas = {}
parse_schema(
{
"name": "B",
"type": "record",
"fields": [{
"name": "bar",
"type": "string"
}]
},
named_schemas,
)
a_schema = parse_schema(
{
"name": "A",
"type": "record",
"fields": [{
"name": "b",
"type": "B"
}]
},
named_schemas,
)
records = [{"b": {"bar": "bar"}}]
validate_many(records, a_schema)
validate(records[0], a_schema)
def test_validator_numeric():
for datum, schema in [
(1, 'int'),
(1, 'long'),
(1.0, 'float'),
(1.0, 'double'),
(1, 'float'),
(1, 'double'),
]:
validate(datum, schema)
for datum, schema in [
(1.0, 'int'),
(1.0, 'long'),
("1.0", 'float'),
("1.0", 'double'),
("1", 'float'),
("1", 'double'),
(True, 'int'),
(True, 'long'),
(True, 'float'),
(True, 'double'),
(False, 'int'),
(False, 'long'),
(False, 'float'),
(False, 'double'),
]:
with pytest.raises(ValidationError):
validate(datum, schema)
pytest.fail("{} should not validate as {}".format(datum, schema))
def test_enum_named_type():
"""https://github.com/fastavro/fastavro/issues/450"""
schema = {
"type":
"record",
"name":
"test_enum_named_type",
"fields": [
{
"name": "test1",
"type": {
"type": "enum",
"name": "my_enum",
"symbols": ["FOO", "BAR"],
},
},
{
"name": "test2",
"type": "my_enum",
},
],
}
record = {"test1": "FOO", "test2": "BAR"}
parsed_schema = parse_schema(schema)
validate(record, parsed_schema)
def test_record_named_type():
"""https://github.com/fastavro/fastavro/issues/450"""
schema = {
"type":
"record",
"name":
"test_record_named_type",
"fields": [{
"name": "test1",
"type": {
"type": "record",
"name": "my_record",
"fields": [{
"name": "field1",
"type": "string",
}]
},
}, {
"name": "test2",
"type": "my_record",
}]
}
record = {"test1": {"field1": "foo"}, "test2": {"field1": "bar"}}
parsed_schema = parse_schema(schema)
validate(record, parsed_schema)
def test_validate_array():
my_schema = {
"fields": [
{
"name": "array",
"type": {
"type": "array",
"items": "string",
},
},
],
"namespace":
"namespace",
"name":
"test_validate_array",
"type":
"record"
}
datum = {"array": [1]}
with pytest.raises(ValidationError) as exc:
validate(datum, my_schema)
for error in exc.value.errors:
assert error.field == 'namespace.test_validate_array.array'
def test_validate_map():
my_schema = {
"fields": [
{
"name": "map",
"type": {
"type": "map",
"values": "string",
},
},
],
"namespace":
"namespace",
"name":
"test_validate_map",
"type":
"record"
}
datum = {"map": {"key": 1}}
with pytest.raises(ValidationError) as exc:
validate(datum, my_schema)
for error in exc.value.errors:
assert error.field == 'namespace.test_validate_map.map'
def test_validate_with_unparsed_schema():
"""https://github.com/fastavro/fastavro/issues/416"""
schema = {
"type":
"record",
"name":
"my_schema_v1",
"fields": [{
"name": "field1",
"type": {
"type": "record",
"name": "my_field_type",
"fields": [{
"name": "sub_field",
"type": "string"
}]
}
}, {
"name": "field2",
"type": "my_field_type"
}]
}
datum = {"field1": {"sub_field": "foo"}, "field2": {"sub_field": "bar"}}
validate(datum, schema)
def test_validator_numeric():
for datum, schema in [
(1, "int"),
(1, "long"),
(1.0, "float"),
(1.0, "double"),
(1, "float"),
(1, "double"),
]:
validate(datum, schema)
for datum, schema in [
(1.0, "int"),
(1.0, "long"),
("1.0", "float"),
("1.0", "double"),
("1", "float"),
("1", "double"),
(True, "int"),
(True, "long"),
(True, "float"),
(True, "double"),
(False, "int"),
(False, "long"),
(False, "float"),
(False, "double"),
(datetime(2020, 1, 1), "int"),
(datetime(2020, 1, 1), "long"),
]:
with pytest.raises(ValidationError):
validate(datum, schema)
pytest.fail(f"{datum} should not validate as {schema}")
def test_validator_numeric():
for datum, schema in [
(1, 'int'),
(1, 'long'),
(1.0, 'float'),
(1.0, 'double'),
(1, 'float'),
(1, 'double'),
]:
validate(datum, schema)
for datum, schema in [
(1.0, 'int'),
(1.0, 'long'),
("1.0", 'float'),
("1.0", 'double'),
("1", 'float'),
("1", 'double'),
(True, 'int'),
(True, 'long'),
(True, 'float'),
(True, 'double'),
(False, 'int'),
(False, 'long'),
(False, 'float'),
(False, 'double'),
]:
with pytest.raises(ValidationError):
validate(datum, schema)
pytest.fail("{} should not validate as {}".format(datum, schema))
def test_validator_logical():
"""https://github.com/fastavro/fastavro/issues/365"""
for datum, schema in [
(1, {"type": "long", "logicalType": "timestamp-micros"}),
]:
validate(datum, schema)
for datum, schema in [
("foo", {"type": "long", "logicalType": "timestamp-micros"}),
]:
with pytest.raises(ValidationError):
validate(datum, schema)
pytest.fail(f"{datum} should not validate as {schema}")
def test_validate_clobbering_SCHEMA_DEFS():
"""https://github.com/fastavro/fastavro/issues/416"""
schema_1 = {
"type":
"record",
"name":
"my_schema_v1",
"fields": [
{
"name": "field1",
"type": {
"type": "record",
"name": "my_field_type",
"fields": [{
"name": "sub_field",
"type": "string"
}],
},
},
{
"name": "field2",
"type": "my_field_type"
},
],
}
parsed_schema_1 = parse_schema(schema_1)
datum_1 = {"field1": {"sub_field": "foo"}, "field2": {"sub_field": "bar"}}
validate(datum_1, parsed_schema_1)
schema_2 = {
"type":
"record",
"name":
"my_schema_v2",
"fields": [{
"name": "field1",
"type": {
"type": "record",
"name": "my_field_type",
"fields": [{
"name": "sub_field",
"type": "int"
}],
},
}],
}
parse_schema(schema_2)
validate(datum_1, parsed_schema_1)
def validate(self, msg, schema_name):
logger.debug(
f'Validating {msg} using the class {self.__class__.__name__} and schema {schema_name}'
)
schema = self.schema_retriever.get_schema(schema_name=schema_name)
if schema is None:
return False
else:
return validation.validate(msg, schema, raise_errors=True)
def test_validator_numeric():
for datum, schema in [
(1, 'int'),
(1, 'long'),
(1.0, 'float'),
(1.0, 'double'),
(1, 'float'),
(1, 'double'),
]:
validate(datum, schema)
for datum, schema in [
(1.0, 'int'),
(1.0, 'long'),
("1.0", 'float'),
("1.0", 'double'),
("1", 'float'),
("1", 'double'),
]:
with pytest.raises(ValidationError):
validate(datum, schema)
def test_validate_map():
my_schema = {
"fields": [
{
"name": "map",
"type": {
"type": "map",
"values": "string",
},
},
],
"namespace": "namespace",
"name": "test_validate_map",
"type": "record"
}
datum = {"map": {"key": 1}}
with pytest.raises(ValidationError) as exc:
validate(datum, my_schema)
for error in exc.value.errors:
assert error.field == 'namespace.test_validate_map.map'
def test_validate_array():
my_schema = {
"fields": [
{
"name": "array",
"type": {
"type": "array",
"items": "string",
},
},
],
"namespace": "namespace",
"name": "test_validate_array",
"type": "record"
}
datum = {"array": [1]}
with pytest.raises(ValidationError) as exc:
validate(datum, my_schema)
for error in exc.value.errors:
assert error.field == 'namespace.test_validate_array.array'
def test_validator_numeric_numpy():
np_ints = [
np.int_,
np.intc,
np.intp,
np.int8,
np.int16,
np.int32,
np.int64,
np.uint8,
np.uint16,
np.uint32,
np.uint64,
]
np_floats = [
np.float_,
np.float16,
np.float32,
np.float64,
]
schema_ints = ['int', 'long']
schema_floats = ['float', 'double']
# all these should work
for nptype, schema in zip(np_ints, schema_ints):
validate(nptype(1), schema)
for nptype, schema in zip(np_ints, schema_floats):
validate(nptype(1), schema)
for nptype, schema in zip(np_floats, schema_floats):
validate(nptype(1), schema)
# these shouldn't work
for nptype, schema in zip(np_floats, schema_ints):
with pytest.raises(ValidationError):
validate(nptype(1), schema)
def test_validator_numeric_numpy():
np_ints = [
np.int_,
np.intc,
np.intp,
np.int8,
np.int16,
np.int32,
np.int64,
np.uint8,
np.uint16,
np.uint32,
np.uint64,
]
np_floats = [
np.float_,
np.float16,
np.float32,
np.float64,
]
schema_ints = ['int', 'long']
schema_floats = ['float', 'double']
# all these should work
for nptype, schema in zip(np_ints, schema_ints):
validate(nptype(1), schema)
for nptype, schema in zip(np_ints, schema_floats):
validate(nptype(1), schema)
for nptype, schema in zip(np_floats, schema_floats):
validate(nptype(1), schema)
# these shouldn't work
for nptype, schema in zip(np_floats, schema_ints):
with pytest.raises(ValidationError):
validate(nptype(1), schema)
def test_schema(self):
self.assertTrue(validate(self.converted[0], self.SCHEMA))
"ontology_reference": "NCIT",
"values": {
"concept_code": "C164337"
}
}],
"links": [{
"name": "member_of",
"dst": "cohort",
"multiplicity": "MANY_TO_ONE"
}]
}
# Metadata
assert validate(
{
"name": "Metadata",
"misc": {},
"nodes": [icdc_case_meta, icdc_cohort_meta]
}, pfb_schema)
# data for PFB message:
cohort_data = {
"id": "n201",
"cohort_description": "arm1",
"cohort_dose": "10mg/kg"
}
assert validate(("icdc.cohort", cohort_data), pfb_schema)
case_data = {
"id": "n101",
from fastavro.validation import validate
import json
with open('schema.avsc', 'r') as schema_file:
schema = json.loads(schema_file.read())
message = {
'age': 28.0,
# 'myname': 'test',
# 'asd': 'test2'
}
try:
validate(message, schema)
print('Message is matching schema')
except Exception as ex:
print(ex)
def test_schema(self):
self.assertTrue(validate(self.type_converted, self.SCHEMA))
outdir = "avro-data"
if not os.path.exists(outdir):
os.makedirs(outdir)
with open(f"avro/{document}.schema.json", "r") as f:
schema_data = f.read()
schema = avro.schema.Parse(schema_data)
outfile = open(f"{outdir}/{document}.avro", "wb")
writer = avro.datafile.DataFileWriter(outfile, avro.io.DatumWriter(), schema)
with open(f"data/{document}.ndjson", "r") as f:
data = f.readlines()
try:
orig = None
for line in data:
orig = json.loads(line)
out = convert(orig, schema)
writer.append(out)
except:
with open("test.json", "w") as f:
json.dump(orig, f)
with open("test-schema.json", "w") as f:
json.dump(schema.to_json(), f, indent=2)
validation.validate(out, parse_schema(schema.to_json()))
writer.close()
