def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
if self.bind:
self.listen_to.bind(self.listen_address)
else:
self.listen_to.connect(self.listen_address)
self.logger.info('{} successfully started'.format(self.name))
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(self.name))
message_data = self.broker.recv()
self.logger.debug('{} Got message from broker'.format(self.name))
message_data = self._translate_from_broker(message_data)
message.ParseFromString(message_data)
for scattered in self.scatter(message):
self.listen_to.send(scattered.SerializeToString())
self.logger.debug('{} Sent message'.format(self.name))
if self.reply:
feedback = self.listen_to.recv()
feedback = self._translate_to_broker(feedback)
self.handle_feedback(feedback)
self.broker.send(self.reply_feedback())
return self.name
def set(self, value: bytes, key=None):
"""
Sets a key value pare in the remote database. If the key is not set,
the function returns a new key. Note that the order of the arguments
is reversed from the usual.
.. warning::
If the session attribute is specified, all the keys will be
prepended with the session id.
:param value: Value to be stored
:param key: Key for the k-v storage
:return: New key or the same key
"""
if not type(value) == bytes:
raise TypeError(
'First argument {} must be of type <bytes>'.format(value))
message = PalmMessage()
message.pipeline = str(
uuid4()) # For a set job, the pipeline is not important
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'set'])
message.payload = value
if key and self.session_set:
message.cache = ''.join([self.pipeline, key])
elif key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
self.listen_to.bind(self.listen_address)
self.logger.info('{} successfully started'.format(self.name))
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
message_data = self.broker.recv()
self.logger.debug('{} Got message from broker'.format(self.name))
message_data = self._translate_from_broker(message_data)
message.ParseFromString(message_data)
for scattered in self.scatter(message):
topic, scattered = self.handle_stream(scattered)
self.listen_to.send_multipart(
[topic.encode('utf-8'),
message.SerializeToString()])
self.logger.debug('Component {} Sent message. Topic {}'.format(
self.name, topic))
if self.reply:
feedback = self.listen_to.recv()
feedback = self._translate_to_broker(feedback)
self.handle_feedback(feedback)
self.broker.send(self.reply_feedback())
return self.name
def do_POST(self):
"""
Note that this http server always replies
"""
message = PalmMessage()
self.send_response(200)
self.end_headers()
message_data = self.rfile.read(
int(self.headers.get('Content-Length')))
message.ParseFromString(message_data)
scattered_messages = scatter(message)
if not scattered_messages:
self.wfile.write(b'0')
else:
for scattered in scattered_messages:
scattered = _translate_to_broker(scattered)
if scattered:
broker.send(scattered.SerializeToString())
handle_feedback(broker.recv())
self.wfile.write(reply_feedback())
def dummy_response():
dummy_router = zmq_context.socket(zmq.REP)
dummy_router.bind('inproc://broker')
msg = dummy_router.recv()
message = PalmMessage()
message.ParseFromString(msg)
dummy_router.send(msg)
return message.payload
def fake_terminator():
socket = zmq_context.socket(zmq.REP)
socket.bind('inproc://terminator')
message = PalmMessage()
message.ParseFromString(socket.recv())
print("Got the message at the terminator: ")
socket.send(b'0')
return message
def test_gateway_dealer():
"""
Test function for the complete gateway with a dummy router.
"""
cache = DictDB()
def dummy_response():
dummy_router = zmq_context.socket(zmq.ROUTER)
dummy_router.bind('inproc://broker')
[target, empty, message] = dummy_router.recv_multipart()
dummy_router.send_multipart([target, empty, b'0'])
broker_message = PalmMessage()
broker_message.ParseFromString(message)
dummy_router.send_multipart([b'gateway_dealer', empty, message])
[target, message] = dummy_router.recv_multipart()
def dummy_initiator():
dummy_client = zmq_context.socket(zmq.REQ)
dummy_client.identity = b'0'
dummy_client.connect('inproc://gateway_router')
message = PalmMessage()
message.client = dummy_client.identity
message.pipeline = '0'
message.function = 'f.servername'
message.stage = 1
message.payload = b'This is a message'
dummy_client.send(message.SerializeToString())
return dummy_client.recv()
got = []
dealer = GatewayDealer(cache=cache, logger=logging, messages=1)
router = GatewayRouter(cache=cache, logger=logging, messages=2)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
results = [
executor.submit(dummy_response),
executor.submit(dummy_initiator),
executor.submit(dealer.start),
executor.submit(router.start)
]
for future in concurrent.futures.as_completed(results):
try:
result = future.result()
if result:
got.append(result)
except Exception as exc:
print(exc)
message = PalmMessage()
message.ParseFromString(got[0])
assert message.payload == b'This is a message'
def fake_router():
socket = zmq_context.socket(zmq.REQ)
socket.bind('inproc://broker')
# Give some time for everything to start
time.sleep(1.0)
message = PalmMessage()
message.payload = b'test message'
socket.send(message.SerializeToString())
socket.recv()
def _execution_handler(self):
for i in range(self.messages):
self.logger.debug('Server waiting for messages')
message_data = self.sub_socket.recv_multipart()[1]
self.logger.debug('Got message {}'.format(i + 1))
result = b'0'
self.message = PalmMessage()
try:
self.message.ParseFromString(message_data)
# Handle the fact that the message may be a complete pipeline
try:
if ' ' in self.message.function:
[server, function] = self.message.function.split()[
self.message.stage].split('.')
else:
[server, function] = self.message.function.split('.')
except IndexError:
raise ValueError('Pipeline call not correct. Review the '
'config in your client')
if not self.name == server:
self.logger.error('You called {}, instead of {}'.format(
server, self.name))
else:
try:
user_function = getattr(self, function)
self.logger.debug('Looking for {}'.format(function))
try:
result = user_function(self.message.payload)
except:
self.logger.error('User function gave an error')
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(
exc_type, exc_value, exc_traceback)
for l in lines:
self.logger.exception(l)
except KeyError:
self.logger.error(
'Function {} was not found'.format(function)
)
except DecodeError:
self.logger.error('Message could not be decoded')
# Do nothing if the function returns no value
if result is None:
continue
self.message.payload = result
topic, self.message = self.handle_stream(self.message)
self.pub_socket.send_multipart(
[topic.encode('utf-8'), self.message.SerializeToString()]
)
def dummy_response():
dummy_router = zmq_context.socket(zmq.ROUTER)
dummy_router.bind('inproc://broker')
[target, empty, message] = dummy_router.recv_multipart()
dummy_router.send_multipart([target, empty, b'0'])
broker_message = PalmMessage()
broker_message.ParseFromString(message)
dummy_router.send_multipart([b'gateway_dealer', empty, message])
[target, message] = dummy_router.recv_multipart()
def eval(self,
function,
payload: bytes,
messages: int = 1,
cache: str = ''):
"""
Execute single job.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: Binary message to be sent
:param messages: Number of messages expected to be sent back to the
client
:param cache: Cache data included in the message
:return: If messages=1, the result data. If messages > 1, a list with the results
"""
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
push_socket.send(message.SerializeToString())
result = []
for i in range(messages):
[client, message_data] = sub_socket.recv_multipart()
message.ParseFromString(message_data)
result.append(message.payload)
if messages == 1:
return result[0]
else:
return result
def start(self):
message = PalmMessage
self.logger.info('{} Successfully started'.format(self.name))
initial_message = PalmMessage()
initial_message.pipeline = '0'
initial_message.client = '0'
initial_message.stage = 0
initial_message.function = ''
initial_message.payload = b'0'
self.broker.send(initial_message.SerializeToString())
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
message_data = self.broker.recv()
self.logger.debug('Got message {} from broker'.format(i))
message.ParseFromString(message_data)
for scattered in self.scatter(message):
self.push.send(scattered)
self.handle_feedback(self.pull.recv())
self.broker.send(self.reply_feedback())
return self.name
def _sender(self, socket, function, generator, cache):
for payload in generator:
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
socket.send(message.SerializeToString())
def job(self,
function,
generator,
messages: int = sys.maxsize,
cache: str = ''):
"""
Submit a job with multiple messages to a server.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: A generator that yields a series of binary messages.
:param messages: Number of messages expected to be sent back to the
client. Defaults to infinity (sys.maxsize)
:param cache: Cache data included in the message
:return: an iterator with the messages that are sent back to the client.
"""
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
# Remember that sockets are not thread safe
sender_thread = Thread(target=self._sender,
args=(push_socket, function, generator, cache))
# Sender runs in background.
sender_thread.start()
for i in range(messages):
[client, message_data] = sub_socket.recv_multipart()
if not client.decode('utf-8') == self.uuid:
raise ValueError(
'The client got a message that does not belong')
message = PalmMessage()
message.ParseFromString(message_data)
yield message.payload
def fake_router():
original_message = PalmMessage()
original_message.pipeline = 'pipeline'
original_message.client = 'client'
original_message.stage = 1
original_message.cache = '0'
original_message.function = 'function'
original_message.payload = b'0'
socket = zmq_context.socket(zmq.REQ)
socket.bind(broker_address)
serialized = original_message.SerializeToString()
socket.send(serialized)
socket.recv()
def dummy_initiator():
dummy_client = zmq_context.socket(zmq.REQ)
dummy_client.identity = b'0'
dummy_client.connect('inproc://gateway_router')
message = PalmMessage()
message.client = dummy_client.identity
message.pipeline = '0'
message.function = 'f.servername'
message.stage = 1
message.payload = b'This is a message'
dummy_client.send(message.SerializeToString())
def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
def load_url(url, data):
request = Request(url, data=data)
response = urlopen(request)
return response.read()
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
message_data = self.broker.recv()
self.logger.debug('{} Got message from broker'.format(self.name))
message_data = self._translate_from_broker(message_data)
message.ParseFromString(message_data)
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.max_workers) as executor:
future_to = [
executor.submit(load_url, self.url,
scattered.SerializeToString())
for scattered in self.scatter(message)
]
for future in concurrent.futures.as_completed(future_to):
try:
feedback = future.result()
except Exception as exc:
self.logger.error('HttpConnection generated an error')
lines = traceback.format_exception(*sys.exc_info())
self.logger.exception(lines[0])
feedback = None
if self.reply:
feedback = self._translate_to_broker(feedback)
self.handle_feedback(feedback)
if feedback:
self.broker.send(self.reply_feedback())
else:
self.broker.send(message_data)
def recv(self):
message_data = self.listen_to.recv()
message = PalmMessage()
message.ParseFromString(message_data)
instruction = message.function.split('.')[1]
if instruction == 'set':
if message.cache:
key = message.cache
else:
key = str(uuid4())
self.logger.debug('Cache Service: Set key {}'.format(key))
value = message.payload
self.cache.set(key, value)
return_value = key.encode('utf-8')
elif instruction == 'get':
key = message.payload.decode('utf-8')
self.logger.debug('Cache Service: Get key {}'.format(key))
value = self.cache.get(key)
if not value:
self.logger.error('key {} not present'.format(key))
return_value = b''
else:
return_value = value
elif instruction == 'delete':
key = message.payload.decode('utf-8')
self.logger.debug('Cache Service: Delete key {}'.format(key))
self.cache.delete(key)
return_value = key.encode('utf-8')
else:
self.logger.error('Cache {}:Key not found in the database'.format(
self.name))
return_value = b''
if isinstance(return_value, str):
self.listen_to.send_string(return_value)
else:
self.listen_to.send(return_value)
def __init__(self, name='', db_address='', push_address=None,
pull_address=None, log_level=logging.INFO,
messages=sys.maxsize):
self.uuid = str(uuid4())
# Give a random name if not given
if not name:
self.name = self.uuid
else:
self.name = name
# Configure the log handler
self.logger = logging.getLogger(name=name)
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
)
self.logger.addHandler(handler)
self.logger.setLevel(log_level)
# Configure the connections.
self.push_address = push_address
self.pull_address = pull_address
if not db_address:
raise ValueError('db_address argument is mandatory')
self.db_address = db_address
self.db = zmq_context.socket(zmq.REQ)
self.db.connect(db_address)
self._get_config_from_master()
self.pull = zmq_context.socket(zmq.PULL)
self.pull.connect(self.push_address)
self.push = zmq_context.socket(zmq.PUSH)
self.push.connect(self.pull_address)
self.messages = messages
self.message = PalmMessage()
def change_payload(message: PalmMessage,
new_payload: bytes) -> PalmMessage:
"""
Change the payload of the message
:param message: The binary message to be processed
:param new_payload: The new binary payload
:return: Serialized message with the new payload
"""
message.payload = new_payload
return message
def do_GET(self):
socket = zmq_context.socket(zmq.REQ)
# This is the identity of the socket and the client.
identity = str(uuid4()).encode('utf-8')
socket.identity = identity
socket.connect(self.gateway_router_address)
function = self.path_parser(self.path)
if function:
message = PalmMessage()
message.pipeline = str(uuid4())
message.function = function
content_length = self.headers.get('content-length')
if content_length:
message.payload = self.rfile.read(int(content_length))
else:
message.payload = b'No Payload'
message.stage = 0
# Uses the same identity as the socket to tell the gateway
# router where it has to route to
message.client = identity
socket.send(message.SerializeToString())
message.ParseFromString(socket.recv())
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
else:
self.send_response(404)
self.send_header('Content-type', 'text/plain')
self.end_headers()
message = b'Not found'
self.wfile.write(message.payload)
socket.close()
return
def fake_server(messages=1):
db_socket = zmq_context.socket(zmq.REP)
db_socket.bind('inproc://db')
pull_socket = zmq_context.socket(zmq.PULL)
pull_socket.bind('inproc://pull')
pub_socket = zmq_context.socket(zmq.PUB)
pub_socket.bind('inproc://pub')
# PUB-SUB takes a while
time.sleep(1.0)
for i in range(messages):
message_data = pull_socket.recv()
print(i)
message = PalmMessage()
message.ParseFromString(message_data)
topic = message.client
pub_socket.send_multipart([topic.encode('utf-8'), message_data])
def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
self.listen_to.connect(self.listen_address)
for i in range(self.messages):
self.logger.debug(
'Component {} blocked waiting for broker'.format(self.name))
[me, message_data] = self.broker.recv_multipart()
message.ParseFromString(message_data)
self.logger.debug(
'Component {} Got message from broker'.format(self.name))
target, message = self._translate_from_broker(message)
for scattered in self.scatter(message):
self.listen_to.send_multipart([target.encode('utf-8'), b'',
scattered.SerializeToString()])
self.logger.debug('Component {} sent message'.format(self.name))
self.broker.send(b'')
def handle(self):
if self.request.method == 'POST':
try:
message = PalmMessage()
message.ParseFromString(self.request.data)
# This exports the message information
self.message = message
instruction = message.function.split('.')[1]
result = getattr(self, instruction)(message.payload)
message.payload = result
response_body = message.SerializeToString()
status = '200 OK'
except Exception as exc:
status = '500 Internal Server Error'
response_body = b''
else:
status = '405 Method not allowed'
response_body = b''
return status, response_body
def delete(self, key):
"""
Deletes data in the server's internal cache.
:param key: Key of the data to be deleted
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'delete'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def get(self, key):
"""
Gets a value from server's internal cache
:param key: Key for the data to be selected.
:return: Value
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'get'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv()
def set(self, value, key=None):
"""
Sets a key value pare in the remote database.
:param key:
:param value:
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join(['_', 'set'])
message.payload = value
if key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
self.listen_to.bind(self.listen_address)
self.logger.info('Launch component {}'.format(self.name))
for i in range(self.messages):
self.logger.debug('Component {} blocked waiting messages'.format(self.name))
response = self.listen_to.recv_multipart()
# If the message is from anything but the dealer, send it to the
# router.
if len(response) == 3:
[target, empty, message_data] = response
self.logger.debug('{} Got inbound message'.format(self.name))
try:
message.ParseFromString(message_data)
for scattered in self.scatter(message):
scattered = self._translate_to_broker(scattered)
self.broker.send(scattered.SerializeToString())
self.logger.debug(
'Component {} blocked waiting for broker'.format(
self.name))
self.broker.recv()
except:
self.logger.error('Error in scatter function')
lines = traceback.format_exception(*sys.exc_info())
self.logger.exception(lines[0])
# This is what's different. The response to be sent from the router
# is what it gets from the dealer.
elif len(response) == 4 and response[0] == b'dealer':
self.listen_to.send_multipart(response[1:])
def start(self):
"""
Call this function to start the component
"""
message = PalmMessage()
if self.bind:
self.listen_to.bind(self.listen_address)
else:
self.listen_to.connect(self.listen_address)
self.logger.info('{} successfully started'.format(self.name))
for i in range(self.messages):
self.logger.debug('{} blocked waiting messages'.format(self.name))
message_data = self.listen_to.recv()
self.logger.debug('{} Got inbound message'.format(self.name))
try:
message.ParseFromString(message_data)
for scattered in self.scatter(message):
scattered = self._translate_to_broker(scattered)
self.broker.send(scattered.SerializeToString())
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
self.handle_feedback(self.broker.recv())
if self.reply:
self.listen_to.send(self.reply_feedback())
except:
self.logger.error('Exception in scatter or routing.')
lines = traceback.format_exception(*sys.exc_info())
self.logger.exception(lines[0])
if self.reply:
self.listen_to.send(b'0')
return self.name
def start(self):
self.logger.info('Launch Component {}'.format(self.name))
for i in range(self.messages):
self.logger.debug('Waiting for etcd')
if self.wait_index > 0:
response = self.etcd.wait(self.key, wait_index=self.wait_index)
else:
response = self.etcd.wait(self.key)
self.wait_index = response['node']['modifiedIndex']+1
self.logger.debug('New wait index: {}'.format(self.wait_index))
message = PalmMessage()
message.function = self.function
message.pipeline = ''
message.stage = 0
message.client = 'EtcdPoller'
message.payload = json.dumps(response).encode('utf-8')
# Build the PALM message that tells what to do with the data
self.broker.send(message.SerializeToString())
# Just unblock
self.logger.debug('blocked waiting broker')
self.broker.recv()
self.logger.debug('Got response from broker')