def _generic_to_json(self, data_obj, writers_schema):
if self.use_logical_types and writers_schema.props.get('logicalType'):
lt = self.logical_types.get(writers_schema.props.get(
'logicalType')) # type: logical.LogicalTypeProcessor
if lt.can_convert(writers_schema):
if lt.validate(writers_schema, data_obj):
data_obj = lt.convert(writers_schema, data_obj)
else:
raise schema.AvroException(
'Wrong object for %s logical type' %
writers_schema.props.get('logicalType'))
if writers_schema.type in _PRIMITIVE_TYPES:
result = self._primitive_to_json(data_obj, writers_schema)
elif writers_schema.type == 'fixed':
result = self._fixed_to_json(data_obj, writers_schema)
elif writers_schema.type == 'enum':
result = self._enum_to_json(data_obj, writers_schema)
elif writers_schema.type == 'array':
result = self._array_to_json(data_obj, writers_schema)
elif writers_schema.type == 'map':
result = self._map_to_json(data_obj, writers_schema)
elif writers_schema.type in ['record', 'error', 'request']:
result = self._record_to_json(data_obj, writers_schema)
elif writers_schema.type in ['union', 'error_union']:
result = self._union_to_json(data_obj, writers_schema)
else:
raise schema.AvroException('Invalid schema type: %s' %
writers_schema.type)
return result
def respond(self, call_request):
buffer_reader = io.BytesIO(call_request)
buffer_decoder = BinaryDecoder(buffer_reader)
buffer_writer = io.BytesIO()
buffer_encoder = BinaryEncoder(buffer_writer)
error = None
response_metadata = {}
try:
remote_protocol = self.process_handshake(buffer_decoder,
buffer_encoder)
if remote_protocol is None or self.local_protocol is None:
return buffer_writer.getvalue()
DatumReader(schema.parse(
'{"type": "map", "values": "bytes"}')).read(buffer_decoder)
remote_message_name = buffer_decoder.read_utf8()
remote_message = remote_protocol.messages.get(remote_message_name)
if remote_message is None:
fail_msg = 'Unknown remote message: %s' % remote_message_name
raise schema.AvroException(fail_msg)
local_message = self.local_protocol.messages.get(
remote_message_name)
if local_message is None:
fail_msg = 'Unknown local message: %s' % remote_message_name
raise schema.AvroException(fail_msg)
writers_schema = remote_message.request
readers_schema = local_message.request
request = self.read_request(writers_schema, readers_schema,
buffer_decoder)
response = None
try:
response = self.invoke(self.local_protocol, local_message,
request)
except AvroRemoteException as e:
error = e
except Exception as e:
error = AvroRemoteException(str(e))
DatumWriter(
schema.parse('{"type": "map", "values": "bytes"}')).write(
response_metadata, buffer_encoder)
buffer_encoder.write_boolean(error is not None)
if error is None:
writers_schema = local_message.response
self.write_response(writers_schema, response, buffer_encoder)
else:
writers_schema = local_message.errors
self.write_error(writers_schema, error, buffer_encoder)
except schema.AvroException as e:
error = AvroRemoteException(str(e))
buffer_encoder = BinaryEncoder(io.BytesIO())
DatumWriter(
schema.parse('{"type": "map", "values": "bytes"}')).write(
response_metadata, buffer_encoder)
buffer_encoder.write_boolean(True)
self.write_error(schema.parse('["string"]'), error, buffer_encoder)
return buffer_encoder.writer.getvalue()
return buffer_writer.getvalue()
def read_call_response(self, message_name, decoder):
"""
The format of a call response is:
* response metadata, a map with values of type bytes
* a one-byte error flag boolean, followed by either:
o if the error flag is false,
the message response, serialized per the message's response schema.
o if the error flag is true,
the error, serialized per the message's error union schema.
"""
# response metadata
response_metadata = META_READER.read(decoder)
# remote response schema
remote_message_schema = self.remote_protocol.messages.get(message_name)
if remote_message_schema is None:
raise schema.AvroException('Unknown remote message: %s' %
message_name)
# local response schema
local_message_schema = self.local_protocol.messages.get(message_name)
if local_message_schema is None:
raise schema.AvroException('Unknown local message: %s' %
message_name)
# error flag
if not decoder.read_boolean():
writers_schema = remote_message_schema.response
readers_schema = local_message_schema.response
return self.read_response(writers_schema, readers_schema, decoder)
else:
writers_schema = remote_message_schema.errors
readers_schema = local_message_schema.errors
raise self.read_error(writers_schema, readers_schema, decoder)
def __init__(self, reader, dreader):
self.__reader = reader
self.__decoder = Decoder(reader)
mag = struct.unpack(
len(_MAGIC).__str__() + 's', self.__reader.read(len(_MAGIC)))[0]
if mag != _MAGIC:
raise schema.AvroException("Not an avro data file")
#find the length
self.__reader.seek(0, 2)
self.__length = self.__reader.tell()
self.__reader.seek(-4, 2)
footersize = (int(ord(self.__reader.read(1)) << 24) +
int(ord(self.__reader.read(1)) << 16) +
int(ord(self.__reader.read(1)) << 8) +
int(ord(self.__reader.read(1))))
seekpos = self.__reader.seek(self.__length - footersize)
metalength = self.__decoder.readlong()
if metalength < 0:
metalength = -metalength
self.__decoder.readlong(
) #ignore byteCount if this is a blocking map
self.__meta = dict()
for i in range(0, metalength):
key = self.__decoder.readutf8()
self.__meta[key] = self.__decoder.readbytes()
self.__sync = self.__meta.get("sync")
self.__count = int(self.__meta.get("count"))
self.__codec = self.__meta.get("codec")
if (self.__codec != None) and (self.__codec != "null"):
raise schema.AvroException("Unknown codec: " + self.__codec)
self.__schema = schema.parse(self.__meta.get("schema").encode("utf-8"))
self.__blockcount = 0
self.__dreader = dreader
self.__dreader.setschema(self.__schema)
self.__reader.seek(len(_MAGIC))
def request(self, msgname, req):
"""Writes a request message and reads a response or error message."""
processed = False
while not processed:
buf = cStringIO.StringIO()
encoder = io.Encoder(buf)
self.__writehandshake(encoder)
requestmeta = dict()
_META_WRITER.write(requestmeta, encoder)
m = self.__localproto.getmessages().get(msgname)
if m is None:
raise schema.AvroException("Not a local message: " +
msgname.__str__())
encoder.writeutf8(m.getname())
self.writerequest(m.getrequest(), req, encoder)
response = self.__transceiver.transceive(buf.getvalue())
decoder = io.Decoder(cStringIO.StringIO(response))
processed = self.__readhandshake(decoder)
responsemeta = _META_READER.read(decoder)
m = self.getremote().getmessages().get(msgname)
if m is None:
raise schema.AvroException("Not a remote message: " +
msgname.__str__())
if not decoder.readboolean():
return self.readresponse(m.getresponse(), decoder)
else:
raise self.readerror(m.geterrors(), decoder)
def __init__(self, handler, func_names):
self.handler = handler
self.processors = {}
for func_name in func_names:
if not hasattr(handler, func_name):
raise schema.AvroException(
'no func. could not create proxy for func: %s', func_name)
else:
func = getattr(handler, func_name)
if not hasattr(func, '__call__'):
raise schema.AvroException(
'not func. could not create proxy for func: %s',
func_name)
self.processors[func_name] = func
def _ReadHandshakeResponse(self, decoder):
"""Reads and processes the handshake response message.
Args:
decoder: Decoder to read messages from.
Returns:
call-response exists (boolean) ???
Raises:
schema.AvroException on ???
"""
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
logger.info('Processing handshake response: %s', handshake_response)
match = handshake_response['match']
if match == 'BOTH':
# Both client and server protocol hashes match:
self._send_protocol = False
return True
elif match == 'CLIENT':
# Client's side hash mismatch:
self._remote_protocol = \
protocol.Parse(handshake_response['serverProtocol'])
self._remote_hash = handshake_response['serverHash']
self._send_protocol = False
return True
elif match == 'NONE':
# Neither client nor server match:
self._remote_protocol = \
protocol.Parse(handshake_response['serverProtocol'])
self._remote_hash = handshake_response['serverHash']
self._send_protocol = True
return False
else:
raise schema.AvroException('handshake_response.match=%r' % match)
def invoke(self, msg, req):
if msg.name == 'send':
message = req['message']
return ("Sent message to " + message['to'] + " from " +
message['from'] + " with body " + message['body'])
else:
raise schema.AvroException("unexpected message:", msg.getname())
def _union_from_json(self, json_obj, writers_schema, readers_schema):
if json_obj is None:
return None
value_type = None
value = None
if not self.fastavro and isinstance(json_obj, collections.Mapping):
items = list(six.iteritems(json_obj))
if not items:
return None
value_type = items[0][0]
value = items[0][1]
if self.fastavro and (isinstance(json_obj, list)
or isinstance(json_obj, tuple)):
if len(json_obj) == 2:
value_type = json_obj[0]
value = json_obj[1]
if value_type is not None:
for s in writers_schema.schemas:
name = self._fullname(s)
if name == value_type:
return self._generic_from_json(value, s, readers_schema)
for s in writers_schema.schemas:
if self.validate(s, json_obj, skip_logical_types=True):
return self._generic_from_json(json_obj, s, readers_schema)
raise schema.AvroException('Datum union type not in schema: %s',
value_type)
def _read_handshake_response(self, decoder):
"""Reads and processes the handshake response message.
Args:
decoder: Decoder to read messages from.
Returns:
call-response exists (boolean) ???
Raises:
schema.AvroException on ???
"""
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
logging.debug("Processing handshake response: %s", handshake_response)
match = handshake_response["match"]
if match == "BOTH":
# Both client and server protocol hashes match:
self._send_protocol = False
return True
elif match == "CLIENT":
# Client side hash mismatch:
self._remote_protocol = protocol.parse(
handshake_response["serverProtocol"])
self._remote_hash = handshake_response["serverHash"]
self._send_protocol = False
return True
elif match == "NONE":
# Neither client nor server match:
self._remote_protocol = protocol.parse(
handshake_response["serverProtocol"])
self._remote_hash = handshake_response["serverHash"]
self._send_protocol = True
return False
else:
raise schema.AvroException("handshake_response.match=%r" % match)
def _write_call_request(self, message_name, request_datum, encoder):
"""Encodes a call request.
The format of a call request is:
- request metadata, as a map with values of type bytes;
- the message name, as an Avro string;
- the message parameters, as specified in the protocol message definition.
Args:
message_name: Name of the protocol message to encode a request for.
request_datum: Request parameters.
encoder: Encoder to encode request with.
"""
# request metadata (not yet implemented)
request_metadata = {}
META_WRITER.write(request_metadata, encoder)
# Identify message to send:
message = self.local_protocol.message_map.get(message_name)
if message is None:
raise schema.AvroException("Unknown message: %s" % message_name)
encoder.write_utf8(message.name)
# message parameters
self._write_request(message.request, request_datum, encoder)
def skip_data(self, writer_schema, decoder):
if writer_schema.type == 'null':
return decoder.skip_null()
elif writer_schema.type == 'boolean':
return decoder.skip_boolean()
elif writer_schema.type == 'string':
return decoder.skip_utf8()
elif writer_schema.type == 'int':
return decoder.skip_int()
elif writer_schema.type == 'long':
return decoder.skip_long()
elif writer_schema.type == 'float':
return decoder.skip_float()
elif writer_schema.type == 'double':
return decoder.skip_double()
elif writer_schema.type == 'bytes':
return decoder.skip_bytes()
elif writer_schema.type == 'fixed':
return self.skip_fixed(writer_schema, decoder)
elif writer_schema.type == 'enum':
return self.skip_enum(writer_schema, decoder)
elif writer_schema.type == 'array':
return self.skip_array(writer_schema, decoder)
elif writer_schema.type == 'map':
return self.skip_map(writer_schema, decoder)
elif writer_schema.type in ['union', 'error_union']:
return self.skip_union(writer_schema, decoder)
elif writer_schema.type in ['record', 'error', 'request']:
return self.skip_record(writer_schema, decoder)
else:
fail_msg = "Unknown schema type: %s" % writer_schema.type
raise schema.AvroException(fail_msg)
def write_data(self, writer_schema, datum, encoder):
# function dispatch to write datum
if writer_schema.type == 'null':
encoder.write_null(datum)
elif writer_schema.type == 'boolean':
encoder.write_boolean(datum)
elif writer_schema.type == 'string':
encoder.write_utf8(datum)
elif writer_schema.type == 'int':
encoder.write_int(datum)
elif writer_schema.type == 'long':
encoder.write_long(datum)
elif writer_schema.type == 'float':
encoder.write_float(datum)
elif writer_schema.type == 'double':
encoder.write_double(datum)
elif writer_schema.type == 'bytes':
encoder.write_bytes(datum)
elif writer_schema.type == 'fixed':
self.write_fixed(writer_schema, datum, encoder)
elif writer_schema.type == 'enum':
self.write_enum(writer_schema, datum, encoder)
elif writer_schema.type == 'array':
self.write_array(writer_schema, datum, encoder)
elif writer_schema.type == 'map':
self.write_map(writer_schema, datum, encoder)
elif writer_schema.type in ['union', 'error_union']:
self.write_union(writer_schema, datum, encoder)
elif writer_schema.type in ['record', 'error', 'request']:
self.write_record(writer_schema, datum, encoder)
else:
fail_msg = 'Unknown type: %s' % writer_schema.type
raise schema.AvroException(fail_msg)
def skipdata(self, schm, decoder):
fn = self.__skipfn.get(schm.gettype())
if fn is not None:
return fn(schm, decoder)
else:
raise schema.AvroException("Unknown type: " +
schema.stringval(schm))
def read_data(self, writers_schema, readers_schema, decoder):
# schema matching
if not DatumReader.match_schemas(writers_schema, readers_schema):
fail_msg = 'Schemas do not match.'
raise SchemaResolutionException(fail_msg, writers_schema,
readers_schema)
# schema resolution: reader's schema is a union, writer's schema is not
if (writers_schema.type not in ['union', 'error_union']
and readers_schema.type in ['union', 'error_union']):
for s in readers_schema.schemas:
if DatumReader.match_schemas(writers_schema, s):
return self.read_data(writers_schema, s, decoder)
fail_msg = 'Schemas do not match.'
raise SchemaResolutionException(fail_msg, writers_schema,
readers_schema)
# function dispatch for reading data based on type of writer's schema
if writers_schema.type == 'null':
return decoder.read_null()
elif writers_schema.type == 'boolean':
return decoder.read_boolean()
elif writers_schema.type == 'string':
return decoder.read_utf8()
elif writers_schema.type == 'int':
return decoder.read_int()
elif writers_schema.type == 'long':
return decoder.read_long()
elif writers_schema.type == 'float':
return decoder.read_float()
elif writers_schema.type == 'double':
return decoder.read_double()
elif writers_schema.type == 'bytes':
if (hasattr(writers_schema, 'logical_type')
and writers_schema.logical_type == 'decimal'):
return decoder.read_decimal_from_bytes(
writers_schema.get_prop('precision'),
writers_schema.get_prop('scale'))
else:
return decoder.read_bytes()
elif writers_schema.type == 'fixed':
if (hasattr(writers_schema, 'logical_type')
and writers_schema.logical_type == 'decimal'):
return decoder.read_decimal_from_fixed(
writers_schema.get_prop('precision'),
writers_schema.get_prop('scale'), writers_schema.size)
return self.read_fixed(writers_schema, readers_schema, decoder)
elif writers_schema.type == 'enum':
return self.read_enum(writers_schema, readers_schema, decoder)
elif writers_schema.type == 'array':
return self.read_array(writers_schema, readers_schema, decoder)
elif writers_schema.type == 'map':
return self.read_map(writers_schema, readers_schema, decoder)
elif writers_schema.type in ['union', 'error_union']:
return self.read_union(writers_schema, readers_schema, decoder)
elif writers_schema.type in ['record', 'error', 'request']:
return self.read_record(writers_schema, readers_schema, decoder)
else:
fail_msg = "Cannot read unknown schema type: %s" % writers_schema.type
raise schema.AvroException(fail_msg)
def respond(self, call_request):
"""
Called by a server to deserialize a request, compute and serialize
a response or error. Compare to 'handle()' in Thrift.
"""
buffer_reader = StringIO(call_request)
buffer_decoder = io.BinaryDecoder(buffer_reader)
buffer_writer = StringIO()
buffer_encoder = io.BinaryEncoder(buffer_writer)
error = None
response_metadata = {}
try:
remote_protocol = self.process_handshake(buffer_decoder,
buffer_encoder)
# handshake failure
if remote_protocol is None:
return buffer_writer.getvalue()
# read request using remote protocol
request_metadata = META_READER.read(buffer_decoder)
remote_message_name = buffer_decoder.read_utf8()
# get remote and local request schemas so we can do
# schema resolution (one fine day)
remote_message = remote_protocol.messages.get(remote_message_name)
if remote_message is None:
fail_msg = 'Unknown remote message: %s' % remote_message_name
raise schema.AvroException(fail_msg)
local_message = self.local_protocol.messages.get(
remote_message_name)
if local_message is None:
fail_msg = 'Unknown local message: %s' % remote_message_name
raise schema.AvroException(fail_msg)
writers_schema = remote_message.request
readers_schema = local_message.request
request = self.read_request(writers_schema, readers_schema,
buffer_decoder)
# perform server logic
try:
response = self.invoke(local_message, request)
except AvroRemoteException, e:
error = e
except Exception, e:
error = AvroRemoteException(str(e))
def _ReadCallResponse(self, message_name, decoder):
"""Reads and processes a method call response.
The format of a call response is:
- response metadata, a map with values of type bytes
- a one-byte error flag boolean, followed by either:
- if the error flag is false,
the message response, serialized per the message's response schema.
- if the error flag is true,
the error, serialized per the message's error union schema.
Args:
message_name:
decoder:
Returns:
???
Raises:
schema.AvroException on ???
"""
# response metadata
response_metadata = META_READER.read(decoder)
# remote response schema
remote_message_schema = self._remote_protocol.message_map.get(
message_name)
if remote_message_schema is None:
raise schema.AvroException('Unknown remote message: %s' %
message_name)
# local response schema
local_message_schema = self._local_protocol.message_map.get(
message_name)
if local_message_schema is None:
raise schema.AvroException('Unknown local message: %s' %
message_name)
# error flag
if not decoder.read_boolean():
writer_schema = remote_message_schema.response
reader_schema = local_message_schema.response
return self._ReadResponse(writer_schema, reader_schema, decoder)
else:
writer_schema = remote_message_schema.errors
reader_schema = local_message_schema.errors
raise self._ReadError(writer_schema, reader_schema, decoder)
def write_data(self, writers_schema, datum, encoder):
# function dispatch to write datum
logical_type = getattr(writers_schema, 'logical_type', None)
if writers_schema.type == 'null':
encoder.write_null(datum)
elif writers_schema.type == 'boolean':
encoder.write_boolean(datum)
elif writers_schema.type == 'string':
encoder.write_utf8(datum)
elif writers_schema.type == 'int':
if logical_type == constants.DATE:
encoder.write_date_int(datum)
elif logical_type == constants.TIME_MILLIS:
encoder.write_time_millis_int(datum)
else:
encoder.write_int(datum)
elif writers_schema.type == 'long':
if logical_type == constants.TIME_MICROS:
encoder.write_time_micros_long(datum)
elif logical_type == constants.TIMESTAMP_MILLIS:
encoder.write_timestamp_millis_long(datum)
elif logical_type == constants.TIMESTAMP_MICROS:
encoder.write_timestamp_micros_long(datum)
else:
encoder.write_long(datum)
elif writers_schema.type == 'float':
encoder.write_float(datum)
elif writers_schema.type == 'double':
encoder.write_double(datum)
elif writers_schema.type == 'bytes':
if logical_type == 'decimal':
encoder.write_decimal_bytes(datum, writers_schema.get_prop('scale'))
else:
encoder.write_bytes(datum)
elif writers_schema.type == 'fixed':
if logical_type == 'decimal':
encoder.write_decimal_fixed(
datum,
writers_schema.get_prop('scale'),
writers_schema.get_prop('size')
)
else:
self.write_fixed(writers_schema, datum, encoder)
elif writers_schema.type == 'enum':
self.write_enum(writers_schema, datum, encoder)
elif writers_schema.type == 'array':
self.write_array(writers_schema, datum, encoder)
elif writers_schema.type == 'map':
self.write_map(writers_schema, datum, encoder)
elif writers_schema.type in ['union', 'error_union']:
self.write_union(writers_schema, datum, encoder)
elif writers_schema.type in ['record', 'error', 'request']:
self.write_record(writers_schema, datum, encoder)
else:
fail_msg = 'Unknown type: %s' % writers_schema.type
raise schema.AvroException(fail_msg)
def __init__(self, local_protocols, proxy_factory):
if proxy_factory is None:
raise schema.AvroException(
'request could not be invoke if proxy_factory is none. ')
self._local_protocol = None
self._proxy_factory = proxy_factory
self._protocol_cache = {}
self._local_protocol_cache = {}
for local_protocol in local_protocols:
self.set_protocol_cache(local_protocol.md5, local_protocol)
self.set_local_protocol_cache(local_protocol.md5, local_protocol)
def readdata(self, actual, expected, decoder):
if actual.gettype() == schema.UNION:
actual = actual.getelementtypes()[int(decoder.readlong())]
if expected.gettype() == schema.UNION:
expected = self._resolve(actual, expected)
if actual.gettype() == schema.NULL:
return None
fn = self.__readfn.get(actual.gettype())
if fn is not None:
return fn(actual, expected, decoder)
else:
raise schema.AvroException("Unknown type: " +
schema.stringval(actual))
def read_data(self, writers_schema, readers_schema, decoder):
## NOTE: no schema reconciliation
method = 'read_%s' % writers_schema.type
if hasattr(decoder, method):
return getattr(decoder, method)()
elif hasattr(self, method):
datum = getattr(self, method)(writers_schema, readers_schema,
decoder)
return types.cast(datum, types.from_schema(writers_schema))
else:
raise _s.AvroException('Unknown type: %r.' % writers_schema.type)
def respond(self, transceiver):
"""Called by a server to deserialize a request, compute and serialize
* a response or error."""
transreq = transceiver.readbuffers()
reader = cStringIO.StringIO(transreq)
decoder = io.Decoder(reader)
buf = cStringIO.StringIO()
encoder = io.Encoder(buf)
error = None
responsemeta = dict()
try:
remoteproto = self.__handshake(transceiver, decoder, encoder)
if remoteproto is None: #handshake failed
return buf.getvalue()
#read request using remote protocol specification
requestmeta = _META_READER.read(decoder)
msgname = decoder.readutf8()
m = remoteproto.getmessages().get(msgname)
if m is None:
raise schema.AvroException("No such remote message: " +
msgname.__str__())
req = self.readrequest(m.getrequest(), decoder)
#read response using local protocol specification
m = self.__localproto.getmessages().get(msgname)
if m is None:
raise schema.AvroException("No such local message: " +
msgname.__str__())
try:
response = self.invoke(m, req)
except AvroRemoteException, e:
error = e
except Exception, e:
error = AvroRemoteException(unicode(e.__str__()))
def write_data(self, writers_schema, datum, encoder):
## NOTE: no schema validation
method = 'write_%s' % writers_schema.type
if hasattr(encoder, method):
getattr(encoder, method)(datum)
elif method == 'write_union':
## Don't getstate() since write_union() needs to know the
## Python type of datum.
self.write_union(writers_schema, datum, encoder)
elif hasattr(self, method):
getattr(self, method)(writers_schema, getstate(datum), encoder)
else:
raise _s.AvroException('Unknown type: %r.' % writers_schema.type)
def read_handshake_response(self, decoder):
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
match = handshake_response.get('match')
if match == 'BOTH':
self.send_protocol = False
return True
elif match == 'CLIENT':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = False
return True
elif match == 'NONE':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = True
return False
else:
raise schema.AvroException('Unexpected match: %s' % match)
def _defaultfieldvalue(self, schm, defaultnode):
if schm.gettype() == schema.RECORD:
record = self.createrecord(schm)
for field in schm.getfields().values():
v = defaultnode.get(field.getname())
if v is None:
v = field.getdefaultvalue()
if v is not None:
record[field.getname()] = self._defaultfieldvalue(
field.getschema(), v)
return record
elif schm.gettype() == schema.ENUM:
return defaultnode
elif schm.gettype() == schema.ARRAY:
array = list()
for node in defaultnode:
array.append(
self._defaultfieldvalue(schm.getelementtype(), node))
return array
elif schm.gettype() == schema.MAP:
map = dict()
for k, v in defaultnode.items():
map[k] = self._defaultfieldvalue(schm.getvaluetype(), v)
return map
elif schm.gettype() == schema.UNION:
return self._defaultfieldvalue(schm.getelementtypes()[0],
defaultnode)
elif schm.gettype() == schema.FIXED:
return defaultnode
elif schm.gettype() == schema.STRING:
return defaultnode
elif schm.gettype() == schema.BYTES:
return defaultnode
elif schm.gettype() == schema.INT:
return int(defaultnode)
elif schm.gettype() == schema.LONG:
return long(defaultnode)
elif schm.gettype() == schema.FLOAT:
return float(defaultnode)
elif schm.gettype() == schema.DOUBLE:
return float(defaultnode)
elif schm.gettype() == schema.BOOLEAN:
return bool(defaultnode)
elif schm.gettype() == schema.NULL:
return None
else:
raise schema.AvroException("Unknown type: " +
schema.stringval(actual))
def _read_default_value(self, field_schema, default_value):
"""
Basically a JSON Decoder?
"""
if field_schema.type == 'null':
return None
elif field_schema.type == 'boolean':
return bool(default_value)
elif field_schema.type == 'int':
return int(default_value)
elif field_schema.type == 'long':
return int(default_value)
elif field_schema.type in ['float', 'double']:
return float(default_value)
elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']:
return default_value
elif field_schema.type == 'array':
read_array = []
for json_val in default_value:
item_val = self._read_default_value(field_schema.items,
json_val)
read_array.append(item_val)
return read_array
elif field_schema.type == 'map':
read_map = {}
for key, json_val in default_value.items():
map_val = self._read_default_value(field_schema.values,
json_val)
read_map[key] = map_val
return read_map
elif field_schema.type in ['union', 'error_union']:
return self._read_default_value(field_schema.schemas[0],
default_value)
elif field_schema.type == 'record':
read_record = {}
read_record['--record'] = field_schema.name
for field in field_schema.fields:
json_val = default_value.get(field.name)
if json_val is None: json_val = field.default
field_val = self._read_default_value(field.type, json_val)
read_record[field.name] = field_val
return read_record
else:
fail_msg = 'Unknown type: %s' % field_schema.type
raise schema.AvroException(fail_msg)
def __readhandshake(self, decoder):
handshake = _HANDSHAKE_REQUESTOR_READER.read(decoder)
print("Handshake.match of protocol:" +
self.__localproto.getname().__str__() + " with:" +
self.__transceiver.getremotename().__str__() + " is " +
handshake.match.__str__())
if handshake.match == _HANDSHAKE_MATCH_BOTH:
return True
elif handshake.match == _HANDSHAKE_MATCH_CLIENT:
self.__setremote(handshake)
return True
elif handshake.match == _HANDSHAKE_MATCH_NONE:
self.__setremote(handshake)
self.__sendlocaltext = True
return False
else:
raise schema.AvroException("Unexpected match: " +
handshake.match.__str__())
def invoke(self, msg, req):
if msg.getname() == 'hello':
print "hello:", req.get("greeting")
return unicode('goodbye')
elif msg.getname() == 'echo':
rec = req.get("record")
print "echo:", rec
return rec
elif msg.getname() == 'echoBytes':
data = req.get("data")
print "echoBytes:", data
return data
elif msg.getname() == 'error':
error = dict()
error["message"] = unicode('an error')
raise ipc.AvroRemoteException(error)
else:
raise schema.AvroException("unexpected message:",msg.getname());
def _read_header(self):
# seek to the beginning of the file to get magic block
self.reader.seek(0, 0)
# read header into a dict
header = self.datum_reader.read_data(META_SCHEMA, META_SCHEMA,
self.raw_decoder)
# check magic number
if header.get('magic') != MAGIC:
fail_msg = "Not an Avro data file: %s doesn't match %s."\
% (header.get('magic'), MAGIC)
raise schema.AvroException(fail_msg)
# set metadata
self._meta = header['meta']
# set sync marker
self._sync_marker = header['sync']
def invoke(self, msg, req):
"""
This functino is invoked by do_POST to handle the request. Invoke handles
the request and get response for the request. This is the key of each node.
All models forwarding and output redirect are done here. Because the invoke
method of initializer only needs to receive the data packet, it does not do
anything in the function and return None.
Args:
msg: Meta data.
req: Contains data packet.
Returns:
None
Raises:
AvroException: if the data does not have correct syntac defined in Schema
"""
if msg.name == 'forward':
init = Initializer.create_init()
try:
init.timer()
if finish:
import json
import threading
global timers
timers['execution_time_classify'] = tt
timers['execution_time'] = tt
with open('tmp-ck-timer.json', 'w') as ftimers:
json.dump(timers, ftimers, indent=2)
x = threading.Thread(target=server.shutdown)
x.daemon = True
x.start()
except Exception as e:
print('Error', e.message)
else:
raise schema.AvroException('unexpected message:', msg.getname())