"fields": [
{"name": "name", "type": "string"},
{"name": "age", "type": "int"}
]
}
"""
def __init__(self):
super(AvroTestCoder, self).__init__(self.SCHEMA)
class AvroTestRecord(AvroRecord):
pass
coders_registry.register_coder(AvroTestRecord, AvroTestCoder)
class AvroCoderTest(unittest.TestCase):
def test_avro_record_coder(self):
real_coder = coders_registry.get_coder(AvroTestRecord)
expected_coder = AvroTestCoder()
self.assertEqual(
real_coder.encode(
AvroTestRecord({
"name": "Daenerys targaryen",
"age": 23
})),
expected_coder.encode(
AvroTestRecord({
"name": "Daenerys targaryen",
import pandas as pd
from parameterized import parameterized
from past.builtins import unicode
import apache_beam as beam
from apache_beam.coders import RowCoder
from apache_beam.coders.typecoders import registry as coders_registry
from apache_beam.dataframe import schemas
from apache_beam.dataframe import transforms
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
Simple = typing.NamedTuple('Simple', [('name', unicode), ('id', int),
('height', float)])
coders_registry.register_coder(Simple, RowCoder)
Animal = typing.NamedTuple('Animal', [('animal', unicode),
('max_speed', typing.Optional[float])])
coders_registry.register_coder(Animal, RowCoder)
def matches_df(expected):
def check_df_pcoll_equal(actual):
actual = pd.concat(actual)
sorted_actual = actual.sort_values(
by=list(actual.columns)).reset_index(drop=True)
sorted_expected = expected.sort_values(
by=list(expected.columns)).reset_index(drop=True)
if not sorted_actual.equals(sorted_expected):
raise AssertionError(
'Dataframes not equal: \n\nActual:\n%s\n\nExpected:\n%s' %
Person = typing.NamedTuple(
"Person",
[
("name", unicode),
("age", np.int32),
("address", typing.Optional[unicode]),
("aliases", typing.List[unicode]),
("knows_javascript", bool),
# TODO(BEAM-7372): Use bytes instead of ByteString
("payload", typing.Optional[typing.ByteString]),
("custom_metadata", typing.Mapping[unicode, int]),
("favorite_time", Timestamp),
])
coders_registry.register_coder(Person, RowCoder)
class RowCoderTest(unittest.TestCase):
JON_SNOW = Person(
name="Jon Snow",
age=np.int32(23),
address=None,
aliases=["crow", "wildling"],
knows_javascript=False,
payload=None,
custom_metadata={},
favorite_time=Timestamp.from_rfc3339('2016-03-18T23:22:59.123456Z'),
)
PEOPLE = [
JON_SNOW,
