"fields": [ {"name": "name", "type": "string"}, {"name": "age", "type": "int"} ] } """ def __init__(self): super(AvroTestCoder, self).__init__(self.SCHEMA) class AvroTestRecord(AvroRecord): pass coders_registry.register_coder(AvroTestRecord, AvroTestCoder) class AvroCoderTest(unittest.TestCase): def test_avro_record_coder(self): real_coder = coders_registry.get_coder(AvroTestRecord) expected_coder = AvroTestCoder() self.assertEqual( real_coder.encode( AvroTestRecord({ "name": "Daenerys targaryen", "age": 23 })), expected_coder.encode( AvroTestRecord({ "name": "Daenerys targaryen",
import pandas as pd from parameterized import parameterized from past.builtins import unicode import apache_beam as beam from apache_beam.coders import RowCoder from apache_beam.coders.typecoders import registry as coders_registry from apache_beam.dataframe import schemas from apache_beam.dataframe import transforms from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to Simple = typing.NamedTuple('Simple', [('name', unicode), ('id', int), ('height', float)]) coders_registry.register_coder(Simple, RowCoder) Animal = typing.NamedTuple('Animal', [('animal', unicode), ('max_speed', typing.Optional[float])]) coders_registry.register_coder(Animal, RowCoder) def matches_df(expected): def check_df_pcoll_equal(actual): actual = pd.concat(actual) sorted_actual = actual.sort_values( by=list(actual.columns)).reset_index(drop=True) sorted_expected = expected.sort_values( by=list(expected.columns)).reset_index(drop=True) if not sorted_actual.equals(sorted_expected): raise AssertionError( 'Dataframes not equal: \n\nActual:\n%s\n\nExpected:\n%s' %
Person = typing.NamedTuple( "Person", [ ("name", unicode), ("age", np.int32), ("address", typing.Optional[unicode]), ("aliases", typing.List[unicode]), ("knows_javascript", bool), # TODO(BEAM-7372): Use bytes instead of ByteString ("payload", typing.Optional[typing.ByteString]), ("custom_metadata", typing.Mapping[unicode, int]), ("favorite_time", Timestamp), ]) coders_registry.register_coder(Person, RowCoder) class RowCoderTest(unittest.TestCase): JON_SNOW = Person( name="Jon Snow", age=np.int32(23), address=None, aliases=["crow", "wildling"], knows_javascript=False, payload=None, custom_metadata={}, favorite_time=Timestamp.from_rfc3339('2016-03-18T23:22:59.123456Z'), ) PEOPLE = [ JON_SNOW,