Beispiel #1
0
    def testValidateUnion(self):
        example_schema = """\
    ["int", "null"]
    """
        datum = None
        result = avro_io.Validate(schema.Parse(example_schema), datum)

        self.assertTrue(result)
 def testValidate(self):
   passed = 0
   for example_schema, datum in SCHEMAS_TO_VALIDATE:
     logging.debug('Schema: %r', example_schema)
     logging.debug('Datum: %r', datum)
     validated = avro_io.Validate(schema.Parse(example_schema), datum)
     logging.debug('Valid: %s', validated)
     if validated: passed += 1
   self.assertEqual(passed, len(SCHEMAS_TO_VALIDATE))
Beispiel #3
0
    def testValidateUnionError(self):
        example_schema = """\
    ["int", "null"]
    """
        datum = "there should not be a string here"
        expected_regex = "datum should be one of following: \['int', 'null']"

        with self.assertRaisesRegex(avro_io.AvroTypeException, expected_regex):
            avro_io.Validate(schema.Parse(example_schema), datum)
Beispiel #4
0
    def testValidateShouldRaiseFormattedError(self):
        example_schema = '{"type": "int"}'
        datum = "aaa"

        expected_regex = "datum should be int type," \
                         " but as value we got 'aaa'"

        with self.assertRaisesRegex(avro_io.AvroPrimitiveTypeException,
                                    expected_regex):
            avro_io.Validate(schema.Parse(example_schema), datum)
Beispiel #5
0
    def testValidateNestedShouldRaiseFormattedError(self):
        example_schema = """\
    {"type": "record",
     "name": "a",
     "fields": [
      {"name": "field1", "type": {
        "type": "array", "items": "int"}
        }
     ]
    }
    """
        datum = {"field1": {"field2": "11", "field3": []}}

        expected_regex = 'Field "field1"' \
                         ' datum should be list type'

        with self.assertRaisesRegex(avro_io.AvroTypeException, expected_regex):
            avro_io.Validate(schema.Parse(example_schema), datum)
Beispiel #6
0
    def testValidateNested(self):
        example_schema = """\
    {"type": "record",
     "name": "a",
     "fields": [
      {"name": "field1", "type": {
        "type": "record", "name": "b", "fields": [
          {"name": "field2", "type": "long"},
          {"name": "field3", "type": "int"}
        ]}
        }
     ]
    }
    """
        datum = {"field1": {"field2": 11, "field3": 11}}

        result = avro_io.Validate(schema.Parse(example_schema), datum)

        self.assertTrue(result)
Beispiel #7
0
                          coords={
                              'time':
                              pd.to_datetime(['2019-01-01', '2019-01-02']),
                              'asset': np.array(['ASSET1', '2ASSET'])
                          })

    print("check conversion to avro record")

    av_record = convert_output_to_avro_record(output)
    import json

    schema = avs.Parse(json.dumps(output_schema))
    reader = aio.DatumReader(schema, schema)
    parsed = reader.read(aio.BinaryDecoder(BytesIO(av_record)))

    assert aio.Validate(schema, parsed)
    assert parsed['coords']['time'] == [
        str(i)[0:10] for i in output.coords[ds.TIME].values
    ]
    assert parsed['coords']['asset'] == output.coords[ds.ASSET].values.tolist()
    assert parsed['values'] == output.values.reshape([4]).tolist()

    print("check conversion from avro record")

    output2 = convert_avro_record_to_output(av_record)

    assert (output.identical(output2))

    print("check conversion to avro file")

    avr_file = output_to_avro_file(output)