def _ReadHandshakeResponse(self, decoder):
"""Reads and processes the handshake response message.
Args:
decoder: Decoder to read messages from.
Returns:
call-response exists (boolean) ???
Raises:
schema.AvroException on ???
"""
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
logger.info('Processing handshake response: %s', handshake_response)
match = handshake_response['match']
if match == 'BOTH':
# Both client and server protocol hashes match:
self._send_protocol = False
return True
elif match == 'CLIENT':
# Client's side hash mismatch:
self._remote_protocol = \
protocol.parse(handshake_response['serverProtocol'])
self._remote_hash = handshake_response['serverHash']
self._send_protocol = False
return True
elif match == 'NONE':
# Neither client nor server match:
self._remote_protocol = \
protocol.parse(handshake_response['serverProtocol'])
self._remote_hash = handshake_response['serverHash']
self._send_protocol = True
return False
else:
raise schema.AvroException('handshake_response.match=%r' % match)
def test_valid_cast_to_string_after_parse(self):
"""
Test that the string generated by an Avro Protocol object
is, in fact, a valid Avro protocol.
"""
print('')
print('TEST CAST TO STRING')
print('===================')
print('')
num_correct = 0
for example in VALID_EXAMPLES:
protocol_data = protocol.parse(example.protocol_string)
try:
protocol.parse(str(protocol_data))
except (ValueError, ProtocolParseException):
debug_msg = "%s: STRING CAST FAILURE" % example.name
else:
debug_msg = "%s: STRING CAST SUCCESS" % example.name
num_correct += 1
print(debug_msg)
fail_msg = "Cast to string success on %d out of %d protocols" % \
(num_correct, len(VALID_EXAMPLES))
self.assertEqual(num_correct, len(VALID_EXAMPLES), fail_msg)
def test_equivalence_after_round_trip(self):
"""
1. Given a string, parse it to get Avro protocol "original".
2. Serialize "original" to a string and parse that string
to generate Avro protocol "round trip".
3. Ensure "original" and "round trip" protocols are equivalent.
"""
print('')
print('TEST ROUND TRIP')
print('===============')
print('')
num_correct = 0
for example in VALID_EXAMPLES:
original_protocol = protocol.parse(example.protocol_string)
round_trip_protocol = protocol.parse(str(original_protocol))
if original_protocol == round_trip_protocol:
num_correct += 1
debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
else:
self.fail("Round trip failure: %s %s %s", (example.name, example.protocol_string, str(original_protocol)))
fail_msg = "Round trip success on %d out of %d protocols" % \
(num_correct, len(VALID_EXAMPLES))
self.assertEqual(num_correct, len(VALID_EXAMPLES), fail_msg)
def _read_handshake_response(self, decoder):
"""Reads and processes the handshake response message.
Args:
decoder: Decoder to read messages from.
Returns:
call-response exists (boolean) ???
Raises:
schema.AvroException on ???
"""
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
logging.debug("Processing handshake response: %s", handshake_response)
match = handshake_response["match"]
if match == "BOTH":
# Both client and server protocol hashes match:
self._send_protocol = False
return True
elif match == "CLIENT":
# Client side hash mismatch:
self._remote_protocol = protocol.parse(handshake_response["serverProtocol"])
self._remote_hash = handshake_response["serverHash"]
self._send_protocol = False
return True
elif match == "NONE":
# Neither client nor server match:
self._remote_protocol = protocol.parse(handshake_response["serverProtocol"])
self._remote_hash = handshake_response["serverHash"]
self._send_protocol = True
return False
else:
raise schema.AvroException("handshake_response.match=%r" % match)
def testEquivalenceAfterRoundTrip(self):
"""
1. Given a string, parse it to get Avro protocol "original".
2. Serialize "original" to a string and parse that string
to generate Avro protocol "round trip".
3. Ensure "original" and "round trip" protocols are equivalent.
"""
num_correct = 0
for example in VALID_EXAMPLES:
original_protocol = protocol.parse(example.protocol_string)
round_trip_protocol = protocol.parse(str(original_protocol))
if original_protocol == round_trip_protocol:
num_correct += 1
logging.debug('Successful round-trip for protocol:\n%s',
example.protocol_string)
else:
self.fail(
'Round-trip failure for protocol:\n%s\nOriginal protocol:\n%s'
% (example.protocol_string, str(original_protocol)))
self.assertEqual(
num_correct, len(VALID_EXAMPLES),
'Round trip success on %d out of %d protocols.' %
(num_correct, len(VALID_EXAMPLES)))
def _read_handshake_response(self, decoder):
"""Reads and processes the handshake response message.
Args:
decoder: Decoder to read messages from.
Returns:
call-response exists (boolean) ???
Raises:
schema.AvroException on ???
"""
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
logging.debug("Processing handshake response: %s", handshake_response)
match = handshake_response["match"]
if match == "BOTH":
# Both client and server protocol hashes match:
self._send_protocol = False
return True
elif match == "CLIENT":
# Client side hash mismatch:
self._remote_protocol = protocol.parse(
handshake_response["serverProtocol"])
self._remote_hash = handshake_response["serverHash"]
self._send_protocol = False
return True
elif match == "NONE":
# Neither client nor server match:
self._remote_protocol = protocol.parse(
handshake_response["serverProtocol"])
self._remote_hash = handshake_response["serverHash"]
self._send_protocol = True
return False
else:
raise schema.AvroException("handshake_response.match=%r" % match)
def test_equivalence_after_round_trip(self):
"""
1. Given a string, parse it to get Avro protocol "original".
2. Serialize "original" to a string and parse that string
to generate Avro protocol "round trip".
3. Ensure "original" and "round trip" protocols are equivalent.
"""
print ''
print 'TEST ROUND TRIP'
print '==============='
print ''
num_correct = 0
for example in VALID_EXAMPLES:
original_protocol = protocol.parse(example.protocol_string)
round_trip_protocol = protocol.parse(str(original_protocol))
if original_protocol == round_trip_protocol:
num_correct += 1
debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
else:
self.fail("Round trip failure: %s %s %s", (example.name, example.protocol_string, str(original_protocol)))
fail_msg = "Round trip success on %d out of %d protocols" % \
(num_correct, len(VALID_EXAMPLES))
self.assertEqual(num_correct, len(VALID_EXAMPLES), fail_msg)
def test_valid_cast_to_string_after_parse(self):
"""
Test that the string generated by an Avro Protocol object
is, in fact, a valid Avro protocol.
"""
print ''
print 'TEST CAST TO STRING'
print '==================='
print ''
num_correct = 0
for example in VALID_EXAMPLES:
protocol_data = protocol.parse(example.protocol_string)
try:
try:
protocol.parse(str(protocol_data))
debug_msg = "%s: STRING CAST SUCCESS" % example.name
num_correct += 1
except:
debug_msg = "%s: STRING CAST FAILURE" % example.name
finally:
print debug_msg
fail_msg = "Cast to string success on %d out of %d protocols" % \
(num_correct, len(VALID_EXAMPLES))
self.assertEqual(num_correct, len(VALID_EXAMPLES), fail_msg)
def test_parse(self):
num_correct = 0
for example in EXAMPLES:
try:
protocol.parse(example.protocol_string)
if example.valid:
num_correct += 1
else:
self.fail("Parsed invalid protocol: %s" % (example.name,))
except Exception, e:
if not example.valid:
num_correct += 1
else:
self.fail("Coudl not parse valid protocol: %s" % (example.name,))
def testParse(self):
correct = 0
for iexample, example in enumerate(EXAMPLES):
logging.debug('Parsing protocol #%d:\n%s', iexample, example.protocol_string)
try:
parsed = protocol.parse(example.protocol_string)
if example.valid:
correct += 1
else:
self.fail('Invalid protocol was parsed:\n%s' % example.protocol_string)
except Exception as exn:
if example.valid:
self.fail('Valid protocol failed to parse: %s\n%s'
% (example.protocol_string, traceback.format_exc()))
else:
if logging.getLogger().getEffectiveLevel() <= 5:
logging.debug('Expected error:\n%s', traceback.format_exc())
else:
logging.debug('Expected error: %r', exn)
correct += 1
self.assertEqual(
correct,
len(EXAMPLES),
'Parse behavior correct on %d out of %d protocols.'
% (correct, len(EXAMPLES)))
def process_handshake(self, decoder, encoder):
handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
handshake_response = {}
# determine the remote protocol
client_hash = handshake_request.get('clientHash')
client_protocol = handshake_request.get('clientProtocol')
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
server_hash = handshake_request.get('serverHash')
if self.local_hash == server_hash:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'BOTH'
else:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'CLIENT'
if handshake_response['match'] != 'BOTH':
handshake_response['serverProtocol'] = str(self.local_protocol)
handshake_response['serverHash'] = self.local_hash
HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
return remote_protocol
def process_handshake(self, decoder, encoder):
handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
handshake_response = {}
# determine the remote protocol
client_hash = handshake_request.get('clientHash')
client_protocol = handshake_request.get('clientProtocol')
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
server_hash = handshake_request.get('serverHash')
if self.local_hash == server_hash:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'BOTH'
else:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'CLIENT'
if handshake_response['match'] != 'BOTH':
handshake_response['serverProtocol'] = str(self.local_protocol)
handshake_response['serverHash'] = self.local_hash
HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
return remote_protocol
def process(a_file, output_dir='out'):
avpr_str = open(a_file, 'rb').read()
name_topic_map = get_name_to_topic_map(avpr_str)
x_proto = protocol.parse(avpr_str)
for the_type in x_proto.types:
if the_type.name in name_topic_map:
topic = name_topic_map[the_type.name]
try:
if 0 == len(getattr(the_type, 'fields', [])):
print "%s has no subschema to iterate over" % topic
continue
msg = generate_json_test_message(the_type)
if msg:
print "Generated a test message for %s." % topic
target = join(output_dir, topic.strip('/'))
try:
makedirs(target)
except OSError:
pass
filename = join(target, 'index.json')
open(filename, 'w').write(dumps(msg))
else:
import pdb; pdb.set_trace()
except RecursiveSchemaError, e:
print "Could not generate a test message for %s. (%s)" % (topic, e.message)
def testParse(self):
correct = 0
for iexample, example in enumerate(EXAMPLES):
logging.debug('Parsing protocol #%d:\n%s', iexample,
example.protocol_string)
try:
parsed = protocol.parse(example.protocol_string)
if example.valid:
correct += 1
else:
self.fail('Invalid protocol was parsed:\n%s' %
example.protocol_string)
except Exception as exn:
if example.valid:
self.fail(
'Valid protocol failed to parse: %s\n%s' %
(example.protocol_string, traceback.format_exc()))
else:
if logging.getLogger().getEffectiveLevel() <= 5:
logging.debug('Expected error:\n%s',
traceback.format_exc())
else:
logging.debug('Expected error: %r', exn)
correct += 1
self.assertEqual(
correct, len(EXAMPLES),
'Parse behavior correct on %d out of %d protocols.' %
(correct, len(EXAMPLES)))
def main(avprFile, outputFolder):
if not os.path.exists(outputFolder):
os.makedirs(outputFolder)
schema_formats = avpr.parse(open(avprFile, "r").read())
sorted([(v.type, k) for k, v in schema_formats.types_dict.items()])
for k, v in schema_formats.types_dict.items():
logging.info('Processing key:' + str(k))
checkAndProcessSchema(k, v.to_json(), outputFolder)
def main(avprFile,outputFolder):
if not os.path.exists(outputFolder):
os.makedirs(outputFolder)
schema_formats = avpr.parse(open(avprFile, "r").read())
sorted([(v.type, k) for k, v in schema_formats.types_dict.items()])
for k, v in schema_formats.types_dict.items():
logging.info('Processing key:'+str(k))
checkAndProcessSchema(k,v.to_json(),outputFolder)
def process_handshake(self, decoder, encoder):
handshake_response = {}
try:
handshake_request = DatumReader(
_load_request_schema()).read(decoder)
except SchemaResolutionException:
if self.local_protocol is None:
handshake_response['match'] = 'NONE'
handshake_response['serverProtocol'] = str(NO_FOUND)
handshake_response['serverHash'] = NO_FOUND.md5
DatumWriter(_load_response_schema()).write(
handshake_response, encoder)
raise HandshakeError(encoder.writer.getvalue())
# reset reader
decoder.reader.seek(0, 0)
return self.local_protocol
client_hash = handshake_request.get('clientHash')
client_protocol = handshake_request.get('clientProtocol')
remote_protocol = self.get_protocol_cache(client_hash)
# new handshake
if remote_protocol is None and client_protocol is None:
handshake_response['match'] = 'NONE'
handshake_response['serverProtocol'] = str(NO_FOUND)
handshake_response['serverHash'] = NO_FOUND.md5
DatumWriter(_load_response_schema()).write(handshake_response,
encoder)
return remote_protocol
# client request handshake
if remote_protocol is None and client_protocol is not None:
# compare with client_protocol and cache_protocol
self._local_protocol = self.contains(client_protocol)
if self.local_protocol is None:
handshake_response['match'] = 'NONE'
handshake_response['serverProtocol'] = str(NO_FOUND)
handshake_response['serverHash'] = NO_FOUND.md5
DatumWriter(_load_response_schema()).write(
handshake_response, encoder)
raise HandshakeError(encoder.writer.getvalue())
else:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
handshake_response['match'] = 'CLIENT'
handshake_response['serverProtocol'] = str(self.local_protocol)
handshake_response['serverHash'] = self.local_protocol.md5
DatumWriter(_load_response_schema()).write(
handshake_response, encoder)
return remote_protocol
# success handshake
if remote_protocol is not None:
handshake_response['match'] = 'BOTH'
DatumWriter(_load_response_schema()).write(handshake_response, encoder)
return remote_protocol
def test_inner_namespace_set(self): print '' print 'TEST INNER NAMESPACE' print '===================' print '' proto = protocol.parse(HELLO_WORLD.protocol_string) self.assertEqual(proto.namespace, "com.acme") greeting_type = proto.types_dict['Greeting'] self.assertEqual(greeting_type.namespace, 'com.acme')
def test_parse(self):
num_correct = 0
for example in EXAMPLES:
try:
protocol.parse(example.protocol_string)
if example.valid:
num_correct += 1
else:
self.fail("Parsed invalid protocol: %s" % (example.name,))
except Exception as e:
if not example.valid:
num_correct += 1
else:
self.fail("Coudl not parse valid protocol: %s" % (example.name,))
fail_msg = "Parse behavior correct on %d out of %d protocols." % \
(num_correct, len(EXAMPLES))
self.assertEqual(num_correct, len(EXAMPLES), fail_msg)
def test_inner_namespace_set(self):
print('')
print('TEST INNER NAMESPACE')
print('===================')
print('')
proto = protocol.parse(HELLO_WORLD.protocol_string)
self.assertEqual(proto.namespace, "com.acme")
greeting_type = proto.types_dict['Greeting']
self.assertEqual(greeting_type.namespace, 'com.acme')
def testValidCastToStringAfterParse(self):
"""
Test that the string generated by an Avro Protocol object is,
in fact, a valid Avro protocol.
"""
num_correct = 0
for example in VALID_EXAMPLES:
proto = protocol.parse(example.protocol_string)
try:
protocol.parse(str(proto))
logging.debug('Successfully reparsed protocol:\n%s', example.protocol_string)
num_correct += 1
except:
logging.debug('Failed to reparse protocol:\n%s', example.protocol_string)
fail_msg = (
'Cast to string success on %d out of %d protocols'
% (num_correct, len(VALID_EXAMPLES)))
self.assertEqual(num_correct, len(VALID_EXAMPLES), fail_msg)
def testValidCastToStringAfterParse(self):
"""
Test that the string generated by an Avro Protocol object is,
in fact, a valid Avro protocol.
"""
num_correct = 0
for example in VALID_EXAMPLES:
proto = protocol.parse(example.protocol_string)
try:
protocol.parse(str(proto))
logging.debug('Successfully reparsed protocol:\n%s',
example.protocol_string)
num_correct += 1
except:
logging.debug('Failed to reparse protocol:\n%s',
example.protocol_string)
fail_msg = ('Cast to string success on %d out of %d protocols' %
(num_correct, len(VALID_EXAMPLES)))
self.assertEqual(num_correct, len(VALID_EXAMPLES), fail_msg)
def test_parse(self):
print ''
print 'TEST PARSE'
print '=========='
print ''
num_correct = 0
for example in EXAMPLES:
try:
protocol.parse(example.protocol_string)
if example.valid: num_correct += 1
debug_msg = "%s: PARSE SUCCESS" % example.name
except:
if not example.valid: num_correct += 1
debug_msg = "%s: PARSE FAILURE" % example.name
finally:
print debug_msg
fail_msg = "Parse behavior correct on %d out of %d protocols." % \
(num_correct, len(EXAMPLES))
self.assertEqual(num_correct, len(EXAMPLES), fail_msg)
def __init__(self, sc):
# Set the PySpark context
self._sc = sc
# Load the ADAM avro record schema
self._adam_record_schema = apr.parse(open("adam.avpr", "r").read())
# Set the JavaSparkContext
self._jsc = sc._jsc
# Store the parquet context which as a loader
self._pc = self._sc._jvm.PythonADAMContext(self._sc._jsc.sc())
def get_protocol(self, name, version=None):
"""
Retrieves an avro.protocol object.
First loads the protocol string, then parses it by calling the
avro.protocol parser.
:param name: a service name.
:param version: an optional version.
:return: an avro.protocol.Protocol object.
"""
contents = self.get_protocol_string(name, version=version)
return avro_protocol.parse(contents)
def testEquivalenceAfterRoundTrip(self):
"""
1. Given a string, parse it to get Avro protocol "original".
2. Serialize "original" to a string and parse that string
to generate Avro protocol "round trip".
3. Ensure "original" and "round trip" protocols are equivalent.
"""
num_correct = 0
for example in VALID_EXAMPLES:
original_protocol = protocol.parse(example.protocol_string)
round_trip_protocol = protocol.parse(str(original_protocol))
if original_protocol == round_trip_protocol:
num_correct += 1
logging.debug('Successful round-trip for protocol:\n%s', example.protocol_string)
else:
self.fail('Round-trip failure for protocol:\n%s\nOriginal protocol:\n%s'
% (example.protocol_string, str(original_protocol)))
self.assertEqual(
num_correct,
len(VALID_EXAMPLES),
'Round trip success on %d out of %d protocols.' % (num_correct, len(VALID_EXAMPLES)))
def read_handshake_response(self, decoder):
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
match = handshake_response.get('match')
if match == 'BOTH':
self.send_protocol = False
return True
elif match == 'CLIENT':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = False
return True
elif match == 'NONE':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = True
return False
else:
raise schema.AvroException('Unexpected match: %s' % match)
def read_handshake_response(self, decoder):
handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
match = handshake_response.get('match')
if match == 'BOTH':
self.send_protocol = False
return True
elif match == 'CLIENT':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = False
return True
elif match == 'NONE':
if self.send_protocol:
raise schema.AvroException('Handshake failure.')
self.remote_protocol = protocol.parse(
handshake_response.get('serverProtocol'))
self.remote_hash = handshake_response.get('serverHash')
self.send_protocol = True
return False
else:
raise schema.AvroException('Unexpected match: %s' % match)
def from_filename(endpoint_name, protocol_file_path):
"""
Creates a AvroAPIEndpoint from protocol file identified by the protocol_file_path.
:param endpoint_name: The name of the endpoint.
:param protocol_file_path: The path to the specific protocol file to parse.
:return: The AvroAPIEndpoint
:rtype: user_service.helpers.avro.AvroAPIEndpoint
"""
with open(protocol_file_path) as _file:
protocol = a_protocol.parse(_file.read())
endpoint = FlaskAvroEndpoint(endpoint_name, protocol)
return endpoint
def from_filename(endpoint_name, protocol_file_path):
"""
Creates a AvroAPIEndpoint from protocol file identified by the protocol_file_path.
:param endpoint_name: The name of the endpoint.
:param protocol_file_path: The path to the specific protocol file to parse.
:return: The AvroAPIEndpoint
:rtype: user_service.helpers.avro.AvroAPIEndpoint
"""
with open(protocol_file_path) as _file:
protocol = a_protocol.parse(_file.read())
endpoint = FlaskAvroEndpoint(endpoint_name, protocol)
return endpoint
def _process_handshake(self, decoder, encoder):
"""Processes an RPC handshake.
Args:
decoder: Where to read from.
encoder: Where to write to.
Returns:
The requested Protocol.
"""
handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
logging.debug("Processing handshake request: %s", handshake_request)
# determine the remote protocol
client_hash = handshake_request.get("clientHash")
client_protocol = handshake_request.get("clientProtocol")
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
server_hash = handshake_request.get("serverHash")
if remote_protocol is None:
match = "NONE"
elif self._local_hash == server_hash:
match = "BOTH"
else:
match = "CLIENT"
# HandshakeResponse:
handshake_response = {
"match": match,
"serverProtocol": None,
"serverHash": None,
"meta": None,
}
if match != "BOTH":
handshake_response["serverProtocol"] = str(self.local_protocol)
handshake_response["serverHash"] = self._local_hash
logging.debug("Handshake response: %s", handshake_response)
HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
return remote_protocol
def _process_handshake(self, decoder, encoder):
"""Processes an RPC handshake.
Args:
decoder: Where to read from.
encoder: Where to write to.
Returns:
The requested Protocol.
"""
handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
logging.debug("Processing handshake request: %s", handshake_request)
# determine the remote protocol
client_hash = handshake_request.get("clientHash")
client_protocol = handshake_request.get("clientProtocol")
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
server_hash = handshake_request.get("serverHash")
if remote_protocol is None:
match = "NONE"
elif self._local_hash == server_hash:
match = "BOTH"
else:
match = "CLIENT"
# HandshakeResponse:
handshake_response = {
"match": match,
"serverProtocol": None,
"serverHash": None,
"meta": None,
}
if match != "BOTH":
handshake_response["serverProtocol"] = str(self.local_protocol)
handshake_response["serverHash"] = self._local_hash
logging.debug("Handshake response: %s", handshake_response)
HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
return remote_protocol
def _ProcessHandshake(self, decoder, encoder):
"""Processes an RPC handshake.
Args:
decoder: Where to read from.
encoder: Where to write to.
Returns:
The requested Protocol.
"""
handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
logger.info('Processing handshake request: %s', handshake_request)
# determine the remote protocol
client_hash = handshake_request.get('clientHash')
client_protocol = handshake_request.get('clientProtocol')
remote_protocol = self.get_protocol_cache(client_hash)
if remote_protocol is None and client_protocol is not None:
remote_protocol = protocol.parse(client_protocol)
self.set_protocol_cache(client_hash, remote_protocol)
# evaluate remote's guess of the local protocol
server_hash = handshake_request.get('serverHash')
handshake_response = {}
if self._local_hash == server_hash:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'BOTH'
else:
if remote_protocol is None:
handshake_response['match'] = 'NONE'
else:
handshake_response['match'] = 'CLIENT'
if handshake_response['match'] != 'BOTH':
handshake_response['serverProtocol'] = str(self.local_protocol)
handshake_response['serverHash'] = self._local_hash
logger.info('Handshake response: %s', handshake_response)
HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
return remote_protocol
def test_client_attributes(self):
with open(protocol_file) as _file:
protocol_string = _file.read()
protocol = avro_protocol.parse(protocol_string)
avro_client = avro_ipc.Requestor(protocol, None)
client = core.Client(avro_client)
self.assertTrue(hasattr(client, "foo"))
self.assertIsInstance(client.foo, types.MethodType)
with mock.patch.object(client._client, "request", return_value="bar"):
self.assertRaises(
TypeError,
client.foo,
baz="fizz"
)
response = client.foo()
self.assertEqual("bar", response)
response = client.foo(bar="baz")
self.assertEqual("bar", response)
def __handshake(self, transceiver, decoder, encoder):
request = _HANDSHAKE_RESPONDER_READER.read(decoder)
remoteproto = self.__protocols.get(request.clientHash)
if remoteproto is None and request.clientProtocol is not None:
remoteproto = protocol.parse(request.clientProtocol)
self.__protocols[request.clientHash] = remoteproto
response = _HandshakeResponse()
if self.__localhash == request.serverHash:
if remoteproto is None:
response.match = _HANDSHAKE_MATCH_NONE
else:
response.match = _HANDSHAKE_MATCH_BOTH
else:
if remoteproto is None:
response.match = _HANDSHAKE_MATCH_NONE
else:
response.match = _HANDSHAKE_MATCH_CLIENT
if response.match != _HANDSHAKE_MATCH_BOTH:
response.serverProtocol = unicode(self.__localproto.__str__(),
"utf8")
response.serverHash = self.__localhash
_HANDSHAKE_RESPONDER_WRITER.write(response, encoder)
return remoteproto
def send_message(uri, proto, msg, datum): url_obj = urlparse.urlparse(uri) client = ipc.HTTPTransceiver(url_obj.hostname, url_obj.port) proto_json = file(proto, 'r').read() requestor = ipc.Requestor(protocol.parse(proto_json), client) print requestor.request(msg, datum)
import os
import sys
import avro.ipc as ipc
import avro.protocol as protocol
# TODO(hammer): Figure the canonical place to put this file
PROTO_FILE = os.path.join(os.path.dirname(__file__), 'schema/hbase.avpr')
PROTOCOL = protocol.parse(open(PROTO_FILE).read())
def retry_wrapper(fn):
"""a decorator to add retry symantics to any method that uses hbase"""
def f(self, *args, **kwargs):
try:
return fn(self, *args, **kwargs)
except:
try:
self.close()
except:
pass
self.make_connection()
return fn(self, *args, **kwargs)
return f
class HBaseConnection(object):
"""
Base class for HBase connections. Supplies methods for a few basic
queries and methods for cleanup of thrift results.
"""
def __init__(self, host, port):
self.host = host
inputProtocol = None
outputProtocol = None
TaskType = None
if (inputProtocol is None):
pfile = os.path.split(__file__)[0] + os.sep + "InputProtocol.avpr"
if not (os.path.exists(pfile)):
raise Exception(
"Could not locate the InputProtocol: {0} does not exist".format(
pfile))
with file(pfile, 'r') as hf:
prototxt = hf.read()
inputProtocol = protocol.parse(prototxt)
# use a named tuple to represent the tasktype enumeration
taskschema = inputProtocol.types_dict["TaskType"]
_ttype = collections.namedtuple("_tasktype", taskschema.symbols)
TaskType = _ttype(*taskschema.symbols)
if (outputProtocol is None):
pfile = os.path.split(__file__)[0] + os.sep + "OutputProtocol.avpr"
if not (os.path.exists(pfile)):
raise Exception(
"Could not locate the OutputProtocol: {0} does not exist".format(
pfile))
with file(pfile, 'r') as hf:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import httplib
import avro.ipc as ipc
import avro.protocol as protocol
PROTOCOL = protocol.parse(open("../../serdes/avro/mail.avpr").read())
server_addr = ('127.0.0.1', 9090)
class UsageError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
if __name__ == '__main__':
if len(sys.argv) != 4:
raise UsageError("Usage: <to> <from> <body>")
# client code - attach to the server and send a message
client = ipc.HTTPTransceiver(server_addr[0], server_addr[1])
"""
import time
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from SocketServer import ThreadingMixIn
from multiprocessing import Queue
from threading import Thread
import avro.ipc as ipc
import avro.protocol as protocol
import avro.schema as schema
import numpy as np
import yaml
from termcolor import colored
# data packet format definition
PROTOCOL = protocol.parse(open('resource/image.avpr').read())
class Initializer:
"""
Singleton factory for initializer. The Initializer module has two timers.
The node_timer is for recording statistics for block1 layer model inference
time. The timer is for recording the total inference time from last
fully connected layer.
Attributes:
queue: Queue for storing available block1 models devices.
start: Start time of getting a frame.
count: Total Number of frames gets back.
node_total: Total layer-wise time.
node_count: Total layer-wise frame count.
def send_message(uri, proto, msg, datum):
url_obj = urlparse.urlparse(uri)
client = ipc.HTTPTransceiver(url_obj.hostname, url_obj.port)
proto_json = file(proto, 'r').read()
requestor = ipc.Requestor(protocol.parse(proto_json), client)
print requestor.request(msg, datum)
# -*- coding: utf-8 -*-
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import threading
import avro.ipc as ipc
import avro.protocol as protocol
import avro.schema as schema
import thread
PROTOCOL = protocol.parse(open("../../avro/pingpong.avpr").read())
server_addr = ('localhost', 9090)
class PingpongResponder(ipc.Responder):
def __init__(self):
ipc.Responder.__init__(self, PROTOCOL)
def invoke(self, msg, req):
if msg.name == 'send':
return "pong"
else:
raise schema.AvroException("unexpected message:", msg.getname())
class PingpongHandler(BaseHTTPRequestHandler):
def do_POST(self):
self.responder = PingpongResponder()
call_request_reader = ipc.FramedReader(self.rfile)
call_request = call_request_reader.read_framed_message()
resp_body = self.responder.respond(call_request)
def test_inner_namespace_not_rendered(self):
proto = protocol.parse(HELLO_WORLD.protocol_string)
self.assertEqual('com.acme.Greeting', proto.types[0].fullname)
self.assertEqual('Greeting', proto.types[0].name)
# but there shouldn't be 'namespace' rendered to json on the inner type
self.assertFalse('namespace' in proto.to_json()['types'][0])
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import avro.ipc as ipc
import avro.protocol as protocol
import avro.schema as schema
from movie_service_util import MovieServiceUtil
# PROTOCOL = protocol.parse(open("../avro/mail.avpr").read())
PROTOCOL = protocol.parse(open("../avro/movieservice.avpr").read())
class MovieServiceResponder(ipc.Responder):
def __init__(self):
ipc.Responder.__init__(self, PROTOCOL)
self.moviesData = MovieServiceUtil.create_movies()
def invoke(self, msg, req):
# print("message: " + str(msg))
# print("req:" + str(req))
# print("messsage name :" + str(msg.name))
if msg.name == 'getMovies':
# print("request to be processed")
# print("local data: " + str(self.moviesData))
return self.moviesData
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
import avro.ipc as ipc
import avro.protocol as protocol
import avro.schema as schema
PROTOCOL = protocol.parse(open("../mail.avpr").read())
HOST = ('localhost', 5000)
class MailResponder(ipc.Responder):
def __init__(self):
ipc.Responder.__init__(self, PROTOCOL)
def invoke(self, msg, req):
if msg.name == 'send':
message = req['message']
return ("Sent message to " + message['to']
+ " from " + message['from']
+ " with body " + message['body'])
else:
raise schema.AvroException("unexpected message:", msg.getname())
class MailHandler(BaseHTTPRequestHandler):
def do_POST(self):
self.responder = MailResponder()
call_request_reader = ipc.FramedReader(self.rfile)
call_request = call_request_reader.read_framed_message()
resp_body = self.responder.respond(call_request)
self.send_response(200)
self.send_header('Content-Type', 'avro/binary')
self.end_headers()
}
],
"messages": {
"send": {
"request": [{"name": "message", "type": "Message"}],
"response": "string"
},
"replay": {
"request": [],
"response": "string"
}
}
}
"""
MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
SERVER_HOST = 'localhost'
SERVER_PORT = 9090
class UsageError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def make_requestor(server_host, server_port, protocol):
client = ipc.HTTPTransceiver(SERVER_HOST, SERVER_PORT)
return ipc.Requestor(protocol, client)
if __name__ == '__main__':
if len(sys.argv) not in [4, 5]:
#http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
import unittest, socket, struct
import testio
import avro.ipc as ipc
import avro.genericipc as genericipc
import avro.protocol as protocol
import avro.schema as schema
PROTOCOL = protocol.parse(open("src/test/schemata/simple.avpr").read())
class TestProtocol(unittest.TestCase):
class TestResponder(genericipc.Responder):
def __init__(self):
ipc.ResponderBase.__init__(self, PROTOCOL)
def invoke(self, msg, req):
if msg.getname() == 'hello':
print "hello:", req.get("greeting")
return unicode('goodbye')
elif msg.getname() == 'echo':
rec = req.get("record")
print "echo:", rec
def load_protocol():
avpr = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir), "avro/services.avpr")
return protocol.parse(open(avpr).read())
def __init__(self, proto, msg, datum): proto_json = file(proto, 'r').read() ipc.Responder.__init__(self, protocol.parse(proto_json)) self.msg = msg self.datum = datum
def __init__(self, proto, msg, datum):
proto_json = file(proto, 'r').read()
ipc.Responder.__init__(self, protocol.parse(proto_json))
self.msg = msg
self.datum = datum
import avro.ipc as ipc
import avro.protocol as protocol
avroProtocol = protocol.parse(open("tweetWithAvr.avr").read())
java_rpc_server_address = ("localhost", 9090)
if __name__ == "__main__":
client = ipc.HTTPTransceiver(java_rpc_server_address[0], java_rpc_server_address[1])
requestor = ipc.Requestor(avroProtocol, client)
tweet = {"tweetId": 1, "username": "******", "text": "This is a tweet from python"}
params = {"tweet": tweet}
requestor.request("sendTweet", params)
client.close()
}
],
"messages": {
"send": {
"request": [{"name": "message", "type": "Message"}],
"response": "string"
},
"replay": {
"request": [],
"response": "string"
}
}
}
"""
MAIL_PROTOCOL = protocol.parse(MAIL_PROTOCOL_JSON)
SERVER_HOST = 'localhost'
SERVER_PORT = 9090
def make_requestor(server_host, server_port, protocol):
client = ipc.HTTPTransceiver(SERVER_HOST, SERVER_PORT)
return ipc.Requestor(protocol, client)
if __name__ == '__main__':
if len(sys.argv) not in [4, 5]:
raise avro.errors.UsageError("Usage: <to> <from> <body> [<count>]")
# client code - attach to the server and send a message
# fill in the Message record
def get_client(host='127.0.0.1', port=9170):
schema = os.path.join(root, 'interface/avro', 'cassandra.avpr')
proto = protocol.parse(open(schema).read())
client = ipc.HTTPTransceiver(host, port)
return ipc.Requestor(proto, client)
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import httplib
import avro.ipc as ipc
import avro.protocol as protocol
PROTOCOL = protocol.parse(open("../../avro/example/mail.avpr").read())
server_addr = ('127.0.0.1', 9090)
class UsageError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
if __name__ == '__main__':
if len(sys.argv) != 4:
raise UsageError("Usage: <to> <from> <body>")
# client code - attach to the server and send a message
client = ipc.HTTPTransceiver(server_addr[0], server_addr[1])
import requests
import time
import MySQLdb
import avro.ipc as ipc
import avro.protocol as protocol
from utils.accesskeys import getAccessPropertiesFromConfigService, get_key_from_configservice
from utils.locallogging import debug
splice_path = os.path.join(os.path.dirname(__file__), 'protocols/Splice.avpr')
if not os.path.isfile(splice_path):
exit(1)
PROTOCOL = protocol.parse(open(splice_path).read())
if __name__ == '__main__':
db_host = get_key_from_configservice(key="/helix-aws/spliceui_dbhost")
db_user = get_key_from_configservice(key="/helix-aws/spliceui_dbuser")
db_pass = get_key_from_configservice(key="/helix-aws/spliceui_dbpass")
db = MySQLdb.connect(host=db_host, user=db_user, passwd=db_pass)
users_and_lists = {}
now = time.strftime("%Y-%m-%d %H:%M:%S")
cur = db.cursor()
cur.execute(
"SELECT id, name, primaryEmail AS email, createdate AS create_date, termsacceptancedate AS terms_date, emailverified AS verified FROM InomeSpliceUI.Customers WHERE createdate > DATE_SUB('%s', INTERVAL 24 HOUR) AND createdate <= '%s' ORDER BY createdate DESC"
from threading import Thread
import tensorflow as tf
from keras.layers import Dense, Input
from keras.models import Model
import avro.ipc as ipc
import avro.protocol as protocol
import numpy as np
import time
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
PATH = os.path.abspath(__file__)
DIR_PATH = os.path.dirname(PATH)
# read data packet format.
PROTOCOL = protocol.parse(
open(DIR_PATH + '/resource/message/message.avpr').read())
SIZE = 10
class Responder(ipc.Responder):
""" Responder called by handler when got request. """
def __init__(self):
ipc.Responder.__init__(self, PROTOCOL)
def invoke(self, msg, req):
"""
This function is invoked by do_POST to handle the request. Invoke handles
the request and get response for the request. This is the key of each node.
All models forwarding and output redirect are done here. Because the invoke
method of initializer only needs to receive the data packet, it does not do
anything in the function and return None.
def generate_protocol(protocol_json, use_logical_types=False, custom_imports=None, avro_json_converter=None):
"""
Generate content of the file which will contain concrete classes for RecordSchemas and requests contained
in the avro protocol
:param str protocol_json: JSON containing avro protocol
:param bool use_logical_types: Use logical types extensions if true
:param list[str] custom_imports: Add additional import modules
:param str avro_json_converter: AvroJsonConverter type to use for default values
:return:
"""
if avro_json_converter is None:
avro_json_converter = 'avrojson.AvroJsonConverter'
if '(' not in avro_json_converter:
avro_json_converter += '(use_logical_types=%s, schema_types=__SCHEMA_TYPES)' % use_logical_types
custom_imports = custom_imports or []
if six.PY3:
proto = protocol.Parse(protocol_json)
else:
proto = protocol.parse(protocol_json)
schemas = []
messages = []
schema_names = set()
request_names = set()
known_types = set()
for schema_idx, record_schema in enumerate(proto.types):
if isinstance(record_schema, (schema.RecordSchema, schema.EnumSchema)):
schemas.append((schema_idx, record_schema))
known_types.add(clean_fullname(record_schema.fullname))
for message in (six.itervalues(proto.messages) if six.PY2 else proto.messages):
messages.append((message, message.request, message.response if isinstance(message.response, (
schema.EnumSchema, schema.RecordSchema)) and clean_fullname(message.response.fullname) not in known_types else None))
if isinstance(message.response, (schema.EnumSchema, schema.RecordSchema)):
known_types.add(clean_fullname(message.response.fullname))
namespaces = {}
for schema_idx, record_schema in schemas:
ns, name = ns_.split_fullname(clean_fullname(record_schema.fullname))
if ns not in namespaces:
namespaces[ns] = {'requests': [], 'records': [], 'responses': []}
namespaces[ns]['records'].append((schema_idx, record_schema))
for message, request, response in messages:
fullname = ns_.make_fullname(proto.namespace, clean_fullname(message.name))
ns, name = ns_.split_fullname(fullname)
if ns not in namespaces:
namespaces[ns] = {'requests': [], 'records': [], 'responses': []}
namespaces[ns]['requests'].append(message)
if response:
namespaces[ns]['responses'].append(message)
main_out = StringIO()
writer = TabbedWriter(main_out)
write_preamble(writer, use_logical_types, custom_imports)
write_protocol_preamble(writer, use_logical_types, custom_imports)
write_get_schema(writer)
write_populate_schemas(writer)
writer.write('\n\n\nclass SchemaClasses(object):')
with writer.indent():
writer.write('\n\n')
current_namespace = tuple()
all_ns = sorted(namespaces.keys())
for ns in all_ns:
if not (namespaces[ns]['responses'] or namespaces[ns]['records']):
continue
namespace = ns.split('.')
if namespace != current_namespace:
start_namespace(current_namespace, namespace, writer)
for idx, record in namespaces[ns]['records']:
schema_names.add(clean_fullname(record.fullname))
if isinstance(record, schema.RecordSchema):
write_schema_record(record, writer, use_logical_types)
elif isinstance(record, schema.EnumSchema):
write_enum(record, writer)
for message in namespaces[ns]['responses']:
schema_names.add(clean_fullname(message.response.fullname))
if isinstance(message.response, schema.RecordSchema):
write_schema_record(message.response, writer, use_logical_types)
elif isinstance(message.response, schema.EnumSchema):
write_enum(message.response, writer)
writer.write('\n\npass')
writer.set_tab(0)
writer.write('\n\n\nclass RequestClasses(object):')
with writer.indent() as indent:
writer.write('\n\n')
current_namespace = tuple()
all_ns = sorted(namespaces.keys())
for ns in all_ns:
if not (namespaces[ns]['requests'] or namespaces[ns]['responses']):
continue
namespace = ns.split('.')
if namespace != current_namespace:
start_namespace(current_namespace, namespace, writer)
for message in namespaces[ns]['requests']:
request_names.add(ns_.make_fullname(proto.namespace, clean_fullname(message.name)))
write_protocol_request(message, proto.namespace, writer, use_logical_types)
writer.write('\n\npass')
writer.untab()
writer.set_tab(0)
writer.write('\n__SCHEMA_TYPES = {\n')
writer.tab()
all_ns = sorted(namespaces.keys())
for ns in all_ns:
for idx, record in (namespaces[ns]['records'] or []):
writer.write("'%s': SchemaClasses.%sClass,\n" % (clean_fullname(record.fullname),
clean_fullname(record.fullname)))
for message in (namespaces[ns]['responses'] or []):
writer.write("'%s': SchemaClasses.%sClass,\n" % (clean_fullname(message.response.fullname),
clean_fullname(message.response.fullname)))
for message in (namespaces[ns]['requests'] or []):
name = ns_.make_fullname(proto.namespace, clean_fullname(message.name))
writer.write("'%s': RequestClasses.%sRequestClass, \n" % (name, name))
writer.untab()
writer.write('\n}\n')
writer.write('_json_converter = %s\n\n' % avro_json_converter)
value = main_out.getvalue()
main_out.close()
return value, schema_names, request_names
