def read_call_response(self, message_name, decoder):
"""
The format of a call response is:
* response metadata, a map with values of type bytes
* a one-byte error flag boolean, followed by either:
o if the error flag is false,
the message response, serialized per the message's response schema.
o if the error flag is true,
the error, serialized per the message's error union schema.
"""
# response metadata
response_metadata = META_READER.read(decoder)
# remote response schema
remote_message_schema = self.remote_protocol.messages.get(message_name)
if remote_message_schema is None:
raise schema.AvroException('Unknown remote message: %s' %
message_name)
# local response schema
local_message_schema = self.local_protocol.messages.get(message_name)
if local_message_schema is None:
raise schema.AvroException('Unknown local message: %s' %
message_name)
# error flag
if not decoder.read_boolean():
writers_schema = remote_message_schema.response
readers_schema = local_message_schema.response
return self.read_response(writers_schema, readers_schema, decoder)
else:
writers_schema = remote_message_schema.errors
readers_schema = local_message_schema.errors
raise self.read_error(writers_schema, readers_schema, decoder)
def write_data(self, writers_schema, datum, encoder):
# function dispatch to write datum
if writers_schema.type == 'null':
encoder.write_null(datum)
elif writers_schema.type == 'boolean':
encoder.write_boolean(datum)
elif writers_schema.type == 'string':
encoder.write_utf8(datum)
elif writers_schema.type == 'int':
encoder.write_int(datum)
elif writers_schema.type == 'long':
encoder.write_long(datum)
elif writers_schema.type == 'float':
encoder.write_float(datum)
elif writers_schema.type == 'double':
encoder.write_double(datum)
elif writers_schema.type == 'bytes':
encoder.write_bytes(datum)
elif writers_schema.type == 'fixed':
self.write_fixed(writers_schema, datum, encoder)
elif writers_schema.type == 'enum':
self.write_enum(writers_schema, datum, encoder)
elif writers_schema.type == 'array':
self.write_array(writers_schema, datum, encoder)
elif writers_schema.type == 'map':
self.write_map(writers_schema, datum, encoder)
elif writers_schema.type in ['union', 'error_union']:
self.write_union(writers_schema, datum, encoder)
elif writers_schema.type in ['record', 'error', 'request']:
self.write_record(writers_schema, datum, encoder)
else:
fail_msg = 'Unknown type: %s' % writers_schema.type
raise schema.AvroException(fail_msg)
def skip_data(self, writers_schema, decoder):
if writers_schema.type == 'null':
return decoder.skip_null()
elif writers_schema.type == 'boolean':
return decoder.skip_boolean()
elif writers_schema.type == 'string':
return decoder.skip_utf8()
elif writers_schema.type == 'int':
return decoder.skip_int()
elif writers_schema.type == 'long':
return decoder.skip_long()
elif writers_schema.type == 'float':
return decoder.skip_float()
elif writers_schema.type == 'double':
return decoder.skip_double()
elif writers_schema.type == 'bytes':
return decoder.skip_bytes()
elif writers_schema.type == 'fixed':
return self.skip_fixed(writers_schema, decoder)
elif writers_schema.type == 'enum':
return self.skip_enum(writers_schema, decoder)
elif writers_schema.type == 'array':
return self.skip_array(writers_schema, decoder)
elif writers_schema.type == 'map':
return self.skip_map(writers_schema, decoder)
elif writers_schema.type in ['union', 'error_union']:
return self.skip_union(writers_schema, decoder)
elif writers_schema.type in ['record', 'error', 'request']:
return self.skip_record(writers_schema, decoder)
else:
fail_msg = "Unknown schema type: %s" % writers_schema.type
raise schema.AvroException(fail_msg)
def read_data(self, writers_schema, readers_schema, decoder):
# schema matching
if not SlowDatumReader.match_schemas(writers_schema, readers_schema):
fail_msg = 'Schemas do not match.'
raise SchemaResolutionException(fail_msg, writers_schema,
readers_schema)
# schema resolution: reader's schema is a union, writer's schema is not
if (writers_schema.type not in ['union', 'error_union']
and readers_schema.type in ['union', 'error_union']):
for s in readers_schema.schemas:
if SlowDatumReader.match_schemas(writers_schema, s):
return self.read_data(writers_schema, s, decoder)
fail_msg = 'Schemas do not match.'
raise SchemaResolutionException(fail_msg, writers_schema,
readers_schema)
# function dispatch for reading data based on type of writer's schema
if writers_schema.type == 'null':
return decoder.read_null()
elif writers_schema.type == 'boolean':
return decoder.read_boolean()
elif writers_schema.type == 'string':
return decoder.read_utf8()
elif writers_schema.type == 'int':
return decoder.read_int()
elif writers_schema.type == 'long':
return decoder.read_long()
elif writers_schema.type == 'float':
return decoder.read_float()
elif writers_schema.type == 'double':
return decoder.read_double()
elif writers_schema.type == 'bytes':
return decoder.read_bytes()
elif writers_schema.type == 'fixed':
return self.read_fixed(writers_schema, readers_schema, decoder)
elif writers_schema.type == 'enum':
return self.read_enum(writers_schema, readers_schema, decoder)
elif writers_schema.type == 'array':
return self.read_array(writers_schema, readers_schema, decoder)
elif writers_schema.type == 'map':
return self.read_map(writers_schema, readers_schema, decoder)
elif writers_schema.type in ['union', 'error_union']:
return self.read_union(writers_schema, readers_schema, decoder)
elif writers_schema.type in ['record', 'error', 'request']:
return self.read_record(writers_schema, readers_schema, decoder)
else:
fail_msg = "Cannot read unknown schema type: %s" % writers_schema.type
raise schema.AvroException(fail_msg)
def read_handshake_response(self, decoder):
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
match = handshake_response.get('match')
if match == 'BOTH':
self.send_protocol = False
return True
elif match == 'CLIENT':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = False
return True
elif match == 'NONE':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = True
return False
else:
raise schema.AvroException('Unexpected match: %s' % match)
def _read_default_value(self, field_schema, default_value):
"""
Basically a JSON Decoder?
"""
if field_schema.type == 'null':
return None
elif field_schema.type == 'boolean':
return bool(default_value)
elif field_schema.type == 'int':
return int(default_value)
elif field_schema.type == 'long':
return long(default_value)
elif field_schema.type in ['float', 'double']:
return float(default_value)
elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']:
return default_value
elif field_schema.type == 'array':
read_array = []
for json_val in default_value:
item_val = self._read_default_value(field_schema.items,
json_val)
read_array.append(item_val)
return read_array
elif field_schema.type == 'map':
read_map = {}
for key, json_val in default_value.items():
map_val = self._read_default_value(field_schema.values,
json_val)
read_map[key] = map_val
return read_map
elif field_schema.type in ['union', 'error_union']:
return self._read_default_value(field_schema.schemas[0],
default_value)
elif field_schema.type == 'record':
read_record = {}
for field in field_schema.fields:
json_val = default_value.get(field.name)
if json_val is None: json_val = field.default
field_val = self._read_default_value(field.type, json_val)
read_record[field.name] = field_val
return read_record
else:
fail_msg = 'Unknown type: %s' % field_schema.type
raise schema.AvroException(fail_msg)
def _read_header(self):
# seek to the beginning of the file to get magic block
self.reader.seek(0, 0)
# read header into a dict
header = self.datum_reader.read_data(
META_SCHEMA, META_SCHEMA, self.raw_decoder)
# check magic number
if header.get('magic') != MAGIC:
fail_msg = "Not an Avro data file: %s doesn't match %s."\
% (header.get('magic'), MAGIC)
raise schema.AvroException(fail_msg)
# set metadata
self._meta = header['meta']
# set sync marker
self._sync_marker = header['sync']
def write_call_request(self, message_name, request_datum, encoder):
"""
The format of a call request is:
* request metadata, a map with values of type bytes
* the message name, an Avro string, followed by
* the message parameters. Parameters are serialized according to
the message's request declaration.
"""
# request metadata (not yet implemented)
request_metadata = {}
META_WRITER.write(request_metadata, encoder)
# message name
message = self.local_protocol.messages.get(message_name)
if message is None:
raise schema.AvroException('Unknown message: %s' % message_name)
encoder.write_utf8(message.name)
# message parameters
self.write_request(message.request, request_datum, encoder)
def check_crc32(self, bytes):
checksum = STRUCT_CRC32.unpack(self.read(4))[0];
if crc32(bytes) & 0xffffffff != checksum:
raise schema.AvroException("Checksum failure")
def respond(self, call_request):
"""
Called by a server to deserialize a request, compute and serialize
a response or error. Compare to 'handle()' in Thrift.
"""
buffer_reader = StringIO(call_request)
buffer_decoder = io.BinaryDecoder(buffer_reader)
buffer_writer = StringIO()
buffer_encoder = io.BinaryEncoder(buffer_writer)
error = None
response_metadata = {}
try:
remote_protocol = self.process_handshake(buffer_decoder,
buffer_encoder)
# handshake failure
if remote_protocol is None:
return buffer_writer.getvalue()
# read request using remote protocol
request_metadata = META_READER.read(buffer_decoder)
remote_message_name = buffer_decoder.read_utf8()
# get remote and local request schemas so we can do
# schema resolution (one fine day)
remote_message = remote_protocol.messages.get(remote_message_name)
if remote_message is None:
fail_msg = 'Unknown remote message: %s' % remote_message_name
raise schema.AvroException(fail_msg)
local_message = self.local_protocol.messages.get(
remote_message_name)
if local_message is None:
fail_msg = 'Unknown local message: %s' % remote_message_name
raise schema.AvroException(fail_msg)
writers_schema = remote_message.request
readers_schema = local_message.request
request = self.read_request(writers_schema, readers_schema,
buffer_decoder)
# perform server logic
try:
response = self.invoke(local_message, request)
except AvroRemoteException as e:
error = e
except Exception as e:
error = AvroRemoteException(str(e))
# write response using local protocol
META_WRITER.write(response_metadata, buffer_encoder)
buffer_encoder.write_boolean(error is not None)
if error is None:
writers_schema = local_message.response
self.write_response(writers_schema, response, buffer_encoder)
else:
writers_schema = local_message.errors
self.write_error(writers_schema, error, buffer_encoder)
except schema.AvroException as e:
error = AvroRemoteException(str(e))
buffer_encoder = io.BinaryEncoder(StringIO())
META_WRITER.write(response_metadata, buffer_encoder)
buffer_encoder.write_boolean(True)
self.write_error(SYSTEM_ERROR_SCHEMA, error, buffer_encoder)
return buffer_writer.getvalue()
