def set(self, value: bytes, key=None):
"""
Sets a key value pare in the remote database. If the key is not set,
the function returns a new key. Note that the order of the arguments
is reversed from the usual.
.. warning::
If the session attribute is specified, all the keys will be
prepended with the session id.
:param value: Value to be stored
:param key: Key for the k-v storage
:return: New key or the same key
"""
if not type(value) == bytes:
raise TypeError(
'First argument {} must be of type <bytes>'.format(value))
message = PalmMessage()
message.pipeline = str(
uuid4()) # For a set job, the pipeline is not important
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'set'])
message.payload = value
if key and self.session_set:
message.cache = ''.join([self.pipeline, key])
elif key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def start(self):
message = PalmMessage
self.logger.info('{} Successfully started'.format(self.name))
initial_message = PalmMessage()
initial_message.pipeline = '0'
initial_message.client = '0'
initial_message.stage = 0
initial_message.function = ''
initial_message.payload = b'0'
self.broker.send(initial_message.SerializeToString())
for i in range(self.messages):
self.logger.debug('{} blocked waiting for broker'.format(
self.name))
message_data = self.broker.recv()
self.logger.debug('Got message {} from broker'.format(i))
message.ParseFromString(message_data)
for scattered in self.scatter(message):
self.push.send(scattered)
self.handle_feedback(self.pull.recv())
self.broker.send(self.reply_feedback())
return self.name
def fake_router():
socket = zmq_context.socket(zmq.REQ)
socket.bind('inproc://broker')
# Give some time for everything to start
time.sleep(1.0)
message = PalmMessage()
message.payload = b'test message'
socket.send(message.SerializeToString())
socket.recv()
def dummy_initiator():
dummy_client = zmq_context.socket(zmq.REQ)
dummy_client.identity = b'0'
dummy_client.connect('inproc://gateway_router')
message = PalmMessage()
message.client = dummy_client.identity
message.pipeline = '0'
message.function = 'f.servername'
message.stage = 1
message.payload = b'This is a message'
dummy_client.send(message.SerializeToString())
def change_payload(message: PalmMessage,
new_payload: bytes) -> PalmMessage:
"""
Change the payload of the message
:param message: The binary message to be processed
:param new_payload: The new binary payload
:return: Serialized message with the new payload
"""
message.payload = new_payload
return message
def do_GET(self):
socket = zmq_context.socket(zmq.REQ)
# This is the identity of the socket and the client.
identity = str(uuid4()).encode('utf-8')
socket.identity = identity
socket.connect(self.gateway_router_address)
function = self.path_parser(self.path)
if function:
message = PalmMessage()
message.pipeline = str(uuid4())
message.function = function
content_length = self.headers.get('content-length')
if content_length:
message.payload = self.rfile.read(int(content_length))
else:
message.payload = b'No Payload'
message.stage = 0
# Uses the same identity as the socket to tell the gateway
# router where it has to route to
message.client = identity
socket.send(message.SerializeToString())
message.ParseFromString(socket.recv())
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
else:
self.send_response(404)
self.send_header('Content-type', 'text/plain')
self.end_headers()
message = b'Not found'
self.wfile.write(message.payload)
socket.close()
return
def eval(self,
function,
payload: bytes,
messages: int = 1,
cache: str = ''):
"""
Execute single job.
:param function: Sting or list of strings following the format
``server.function``.
:param payload: Binary message to be sent
:param messages: Number of messages expected to be sent back to the
client
:param cache: Cache data included in the message
:return: If messages=1, the result data. If messages > 1, a list with the results
"""
push_socket = zmq_context.socket(zmq.PUSH)
push_socket.connect(self.push_address)
sub_socket = zmq_context.socket(zmq.SUB)
sub_socket.setsockopt_string(zmq.SUBSCRIBE, self.uuid)
sub_socket.connect(self.sub_address)
if type(function) == str:
# Single-stage job
pass
elif type(function) == list:
# Pipelined job.
function = ' '.join(function)
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
push_socket.send(message.SerializeToString())
result = []
for i in range(messages):
[client, message_data] = sub_socket.recv_multipart()
message.ParseFromString(message_data)
result.append(message.payload)
if messages == 1:
return result[0]
else:
return result
def _sender(self, socket, function, generator, cache):
for payload in generator:
message = PalmMessage()
message.function = function
message.stage = 0
message.pipeline = self.pipeline
message.client = self.uuid
message.payload = payload
if cache:
message.cache = cache
socket.send(message.SerializeToString())
def fake_router():
original_message = PalmMessage()
original_message.pipeline = 'pipeline'
original_message.client = 'client'
original_message.stage = 1
original_message.cache = '0'
original_message.function = 'function'
original_message.payload = b'0'
socket = zmq_context.socket(zmq.REQ)
socket.bind(broker_address)
serialized = original_message.SerializeToString()
socket.send(serialized)
socket.recv()
def delete(self, key):
"""
Deletes data in the server's internal cache.
:param key: Key of the data to be deleted
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'delete'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def get(self, key):
"""
Gets a value from server's internal cache
:param key: Key for the data to be selected.
:return: Value
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join([self.server_name, 'get'])
message.payload = key.encode('utf-8')
self.db.send(message.SerializeToString())
return self.db.recv()
def set(self, value, key=None):
"""
Sets a key value pare in the remote database.
:param key:
:param value:
:return:
"""
message = PalmMessage()
message.pipeline = str(uuid4())
message.client = self.uuid
message.stage = 0
message.function = '.'.join(['_', 'set'])
message.payload = value
if key:
message.cache = key
self.db.send(message.SerializeToString())
return self.db.recv().decode('utf-8')
def handle(self):
if self.request.method == 'POST':
try:
message = PalmMessage()
message.ParseFromString(self.request.data)
# This exports the message information
self.message = message
instruction = message.function.split('.')[1]
result = getattr(self, instruction)(message.payload)
message.payload = result
response_body = message.SerializeToString()
status = '200 OK'
except Exception as exc:
status = '500 Internal Server Error'
response_body = b''
else:
status = '405 Method not allowed'
response_body = b''
return status, response_body
def start(self):
self.logger.info('Launch Component {}'.format(self.name))
for i in range(self.messages):
self.logger.debug('Waiting for etcd')
if self.wait_index > 0:
response = self.etcd.wait(self.key, wait_index=self.wait_index)
else:
response = self.etcd.wait(self.key)
self.wait_index = response['node']['modifiedIndex']+1
self.logger.debug('New wait index: {}'.format(self.wait_index))
message = PalmMessage()
message.function = self.function
message.pipeline = ''
message.stage = 0
message.client = 'EtcdPoller'
message.payload = json.dumps(response).encode('utf-8')
# Build the PALM message that tells what to do with the data
self.broker.send(message.SerializeToString())
# Just unblock
self.logger.debug('blocked waiting broker')
self.broker.recv()
self.logger.debug('Got response from broker')