class Process(Loggable):
"""Hosts a number of Blocks, distributing requests between them"""
def __init__(self, name, sync_factory):
self.set_logger_name(name)
self.name = name
self.sync_factory = sync_factory
self.q = self.create_queue()
self._blocks = OrderedDict() # block name -> block
self._block_state_cache = Cache()
self._recv_spawned = None
self._other_spawned = []
self._subscriptions = OrderedDict() # block name -> list of subs
self._last_changes = OrderedDict() # block name -> list of changes
self._client_comms = OrderedDict() # client comms -> list of blocks
self._handle_functions = {
Post: self._forward_block_request,
Put: self._forward_block_request,
Get: self._handle_get,
Subscribe: self._handle_subscribe,
BlockNotify: self._handle_block_notify,
BlockChanged: self._handle_block_changed,
BlockRespond: self._handle_block_respond,
BlockAdd: self._handle_block_add,
BlockList: self._handle_block_list,
}
self.create_process_block()
def recv_loop(self):
"""Service self.q, distributing the requests to the right block"""
while True:
request = self.q.get()
self.log_debug("Received request %s", request)
if request is PROCESS_STOP:
# Got the sentinel, stop immediately
break
try:
self._handle_functions[type(request)](request)
except Exception:
self.log_exception("Exception while handling %s", request)
def start(self):
"""Start the process going"""
self._recv_spawned = self.sync_factory.spawn(self.recv_loop)
def stop(self, timeout=None):
"""Stop the process and wait for it to finish
Args:
timeout (float): Maximum amount of time to wait for each spawned
process. None means forever
"""
assert self._recv_spawned, "Process not started"
self.q.put(PROCESS_STOP)
# Wait for recv_loop to complete first
self._recv_spawned.wait(timeout=timeout)
# Now wait for anything it spawned to complete
for s in self._other_spawned:
s.wait(timeout=timeout)
def _forward_block_request(self, request):
"""Lookup target Block and spawn block.handle_request(request)
Args:
request (Request): The message that should be passed to the Block
"""
block_name = request.endpoint[0]
block = self._blocks[block_name]
self._other_spawned.append(
self.sync_factory.spawn(block.handle_request, request))
def create_queue(self):
"""
Create a queue using sync_factory object
Returns:
Queue: New queue
"""
return self.sync_factory.create_queue()
def create_lock(self):
"""
Create a lock using sync_factory object
Returns:
Lock: New lock
"""
return self.sync_factory.create_lock()
def spawn(self, function, *args, **kwargs):
"""Calls SyncFactory.spawn()"""
spawned = self.sync_factory.spawn(function, *args, **kwargs)
self._other_spawned.append(spawned)
return spawned
def get_client_comms(self, block_name):
for client_comms, blocks in list(self._client_comms.items()):
if block_name in blocks:
return client_comms
def create_process_block(self):
self.process_block = Block()
a = Attribute(StringArrayMeta(
description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(StringArrayMeta(
description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def update_block_list(self, client_comms, blocks):
self.q.put(BlockList(client_comms=client_comms, blocks=blocks))
def _handle_block_list(self, request):
self._client_comms[request.client_comms] = request.blocks
remotes = []
for blocks in self._client_comms.values():
remotes += [b for b in blocks if b not in remotes]
self.process_block.remoteBlocks.set_value(remotes)
def notify_subscribers(self, block_name):
self.q.put(BlockNotify(name=block_name))
def _handle_block_notify(self, request):
"""Update subscribers with changes and applies stored changes to the
cached structure"""
# update cached dict
for delta in self._last_changes.setdefault(request.name, []):
self._block_state_cache.delta_update(delta)
for subscription in self._subscriptions.setdefault(request.name, []):
endpoint = subscription.endpoint
# find stuff that's changed that is relevant to this subscriber
changes = []
for change in self._last_changes[request.name]:
change_path = change[0]
# look for a change_path where the beginning matches the
# endpoint path, then strip away the matching part and add
# to the change set
i = 0
for (cp_element, ep_element) in zip(change_path, endpoint):
if cp_element != ep_element:
break
i += 1
else:
# change has matching path, so keep it
# but strip off the end point path
filtered_change = [change_path[i:]] + change[1:]
changes.append(filtered_change)
if len(changes) > 0:
if subscription.delta:
# respond with the filtered changes
response = Delta(
subscription.id_, subscription.context, changes)
else:
# respond with the structure of everything
# below the endpoint
d = self._block_state_cache.walk_path(endpoint)
response = Update(
subscription.id_, subscription.context, d)
self.log_debug("Responding to subscription %s", response)
subscription.response_queue.put(response)
self._last_changes[request.name] = []
def on_changed(self, change, notify=True):
self.q.put(BlockChanged(change=change))
if notify:
block_name = change[0][0]
self.notify_subscribers(block_name)
def _handle_block_changed(self, request):
"""Record changes to made to a block"""
# update changes
path = request.change[0]
block_changes = self._last_changes.setdefault(path[0], [])
block_changes.append(request.change)
def block_respond(self, response, response_queue):
self.q.put(BlockRespond(response, response_queue))
def _handle_block_respond(self, request):
"""Push the response to the required queue"""
request.response_queue.put(request.response)
def add_block(self, name, block):
"""Add a block to be hosted by this process
Args:
block (Block): The block to be added
"""
assert name not in self._blocks, \
"There is already a block called %s" % name
block.set_parent(self, name)
self.q.put(BlockAdd(block=block))
def _handle_block_add(self, request):
"""Add a block to be hosted by this process"""
block = request.block
assert block.name not in self._blocks, \
"There is already a block called %s" % block.name
self._blocks[block.name] = block
self._block_state_cache[block.name] = block.to_dict()
block.lock = self.create_lock()
# Regenerate list of blocks
self.process_block.blocks.set_value(list(self._blocks))
def _handle_subscribe(self, request):
"""Add a new subscriber and respond with the current
sub-structure state"""
subs = self._subscriptions.setdefault(request.endpoint[0], [])
subs.append(request)
d = self._block_state_cache.walk_path(request.endpoint)
self.log_debug("Initial subscription value %s", d)
if request.delta:
request.respond_with_delta([[[], d]])
else:
request.respond_with_update(d)
def _handle_get(self, request):
d = self._block_state_cache.walk_path(request.endpoint)
response = Return(request.id_, request.context, d)
request.response_queue.put(response)
def test_deletion(self):
c = Cache()
c[1] = 2
c.delta_update([[1]])
self.assertEqual(list(c), [])
def test_change(self):
c = Cache()
c[1] = 3
c.delta_update([[1], 4])
self.assertEqual(c[1], 4)
def test_addition(self):
c = Cache()
c.delta_update([["thing"], {1: 2}])
self.assertEqual(c["thing"][1], 2)
class Process(Loggable):
"""Hosts a number of Blocks, distributing requests between them"""
def __init__(self, name, sync_factory):
self.set_logger_name(name)
self.name = name
self.sync_factory = sync_factory
self.q = self.create_queue()
self._blocks = OrderedDict() # block name -> block
self._block_state_cache = Cache()
self._recv_spawned = None
self._other_spawned = []
self._subscriptions = OrderedDict() # block name -> list of subs
self._last_changes = OrderedDict() # block name -> list of changes
self._client_comms = OrderedDict() # client comms -> list of blocks
self._handle_functions = {
Post: self._forward_block_request,
Put: self._forward_block_request,
Get: self._handle_get,
Subscribe: self._handle_subscribe,
BlockNotify: self._handle_block_notify,
BlockChanged: self._handle_block_changed,
BlockRespond: self._handle_block_respond,
BlockAdd: self._handle_block_add,
BlockList: self._handle_block_list,
}
self.create_process_block()
def recv_loop(self):
"""Service self.q, distributing the requests to the right block"""
while True:
request = self.q.get()
self.log_debug("Received request %s", request)
if request is PROCESS_STOP:
# Got the sentinel, stop immediately
break
try:
self._handle_functions[type(request)](request)
except Exception:
self.log_exception("Exception while handling %s", request)
def start(self):
"""Start the process going"""
self._recv_spawned = self.sync_factory.spawn(self.recv_loop)
def stop(self, timeout=None):
"""Stop the process and wait for it to finish
Args:
timeout (float): Maximum amount of time to wait for each spawned
process. None means forever
"""
assert self._recv_spawned, "Process not started"
self.q.put(PROCESS_STOP)
# Wait for recv_loop to complete first
self._recv_spawned.wait(timeout=timeout)
# Now wait for anything it spawned to complete
for s in self._other_spawned:
s.wait(timeout=timeout)
def _forward_block_request(self, request):
"""Lookup target Block and spawn block.handle_request(request)
Args:
request (Request): The message that should be passed to the Block
"""
block_name = request.endpoint[0]
block = self._blocks[block_name]
self._other_spawned.append(
self.sync_factory.spawn(block.handle_request, request))
def create_queue(self):
"""
Create a queue using sync_factory object
Returns:
Queue: New queue
"""
return self.sync_factory.create_queue()
def create_lock(self):
"""
Create a lock using sync_factory object
Returns:
Lock: New lock
"""
return self.sync_factory.create_lock()
def spawn(self, function, *args, **kwargs):
"""Calls SyncFactory.spawn()"""
spawned = self.sync_factory.spawn(function, *args, **kwargs)
self._other_spawned.append(spawned)
return spawned
def get_client_comms(self, block_name):
for client_comms, blocks in list(self._client_comms.items()):
if block_name in blocks:
return client_comms
def create_process_block(self):
self.process_block = Block()
a = Attribute(
StringArrayMeta(description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(
StringArrayMeta(description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def update_block_list(self, client_comms, blocks):
self.q.put(BlockList(client_comms=client_comms, blocks=blocks))
def _handle_block_list(self, request):
self._client_comms[request.client_comms] = request.blocks
remotes = []
for blocks in self._client_comms.values():
remotes += [b for b in blocks if b not in remotes]
self.process_block.remoteBlocks.set_value(remotes)
def notify_subscribers(self, block_name):
self.q.put(BlockNotify(name=block_name))
def _handle_block_notify(self, request):
"""Update subscribers with changes and applies stored changes to the
cached structure"""
# update cached dict
for delta in self._last_changes.setdefault(request.name, []):
self._block_state_cache.delta_update(delta)
for subscription in self._subscriptions.setdefault(request.name, []):
endpoint = subscription.endpoint
# find stuff that's changed that is relevant to this subscriber
changes = []
for change in self._last_changes[request.name]:
change_path = change[0]
# look for a change_path where the beginning matches the
# endpoint path, then strip away the matching part and add
# to the change set
i = 0
for (cp_element, ep_element) in zip(change_path, endpoint):
if cp_element != ep_element:
break
i += 1
else:
# change has matching path, so keep it
# but strip off the end point path
filtered_change = [change_path[i:]] + change[1:]
changes.append(filtered_change)
if len(changes) > 0:
if subscription.delta:
# respond with the filtered changes
response = Delta(subscription.id_, subscription.context,
changes)
else:
# respond with the structure of everything
# below the endpoint
d = self._block_state_cache.walk_path(endpoint)
response = Update(subscription.id_, subscription.context,
d)
self.log_debug("Responding to subscription %s", response)
subscription.response_queue.put(response)
self._last_changes[request.name] = []
def on_changed(self, change, notify=True):
self.q.put(BlockChanged(change=change))
if notify:
block_name = change[0][0]
self.notify_subscribers(block_name)
def _handle_block_changed(self, request):
"""Record changes to made to a block"""
# update changes
path = request.change[0]
block_changes = self._last_changes.setdefault(path[0], [])
block_changes.append(request.change)
def block_respond(self, response, response_queue):
self.q.put(BlockRespond(response, response_queue))
def _handle_block_respond(self, request):
"""Push the response to the required queue"""
request.response_queue.put(request.response)
def add_block(self, name, block):
"""Add a block to be hosted by this process
Args:
block (Block): The block to be added
"""
assert name not in self._blocks, \
"There is already a block called %s" % name
block.set_parent(self, name)
self.q.put(BlockAdd(block=block))
def _handle_block_add(self, request):
"""Add a block to be hosted by this process"""
block = request.block
assert block.name not in self._blocks, \
"There is already a block called %s" % block.name
self._blocks[block.name] = block
self._block_state_cache[block.name] = block.to_dict()
block.lock = self.create_lock()
# Regenerate list of blocks
self.process_block.blocks.set_value(list(self._blocks))
def _handle_subscribe(self, request):
"""Add a new subscriber and respond with the current
sub-structure state"""
subs = self._subscriptions.setdefault(request.endpoint[0], [])
subs.append(request)
d = self._block_state_cache.walk_path(request.endpoint)
self.log_debug("Initial subscription value %s", d)
if request.delta:
request.respond_with_delta([[[], d]])
else:
request.respond_with_update(d)
def _handle_get(self, request):
d = self._block_state_cache.walk_path(request.endpoint)
response = Return(request.id_, request.context, d)
request.response_queue.put(response)
def test_change(self):
c = Cache()
c[1] = 3
c.delta_update([[1], 4])
self.assertEqual(c[1], 4)
def test_deletion(self):
c = Cache()
c[1] = 2
c.delta_update([[1]])
self.assertEqual(list(c), [])
def test_addition(self):
c = Cache()
c.delta_update([["thing"], {1: 2}])
self.assertEqual(c["thing"][1], 2)
