def hop(self, token, msg, policy):
here = Address(token.namespace, token.user, token.service,
token.application)
if msg.header.hop >= msg.header.hMax:
return (None, None)
msg = Message(msg.header._replace(hop=msg.header.hop + 1, via=here),
msg.payload)
if msg.header.dst == here:
return (None, msg)
hop = next(
Flow.find(token,
application=msg.header.dst.application,
policy=policy), None)
if hop is None:
# TODO: Get next hop from routing table
# mechanism
#search = ((i, Flow.inspect(i)) for i in Flow.find(token, policy="application"))
#query = (
# ref
# for ref, table in search
# for rule in table
# if rule.dst.application == msg.header.via.application
#)
return (None, msg)
poa = Flow.inspect(hop)
return (poa, msg)
def create_from_resource(path: Resource, poa: list, role: list, routing: list, prefix="flow_", suffix=""):
if all(path[:5]) and not any(path[5:]):
parent = os.path.join(*path[:5])
drctry = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=parent)
flow = path._replace(flow=os.path.basename(drctry))
# MRO important here.
for registry, choices in [
("turberfield.ipc.routing", routing),
("turberfield.ipc.poa", poa),
("turberfield.ipc.role", role),
]:
policies = dict(gather_installed(registry))
for option in choices:
try:
typ = policies[option]
if issubclass(typ, Pooled):
others = [Flow.inspect(i) for i in Flow.find(path, application="*", policy=option)]
obj = typ.allocate(others=others)
else:
obj = typ()
flow = flow._replace(policy=option, suffix=".json")
with open(os.path.join(*flow[:-1]) + flow.suffix, "w") as record:
record.write(obj.__json__())
record.flush()
except KeyError:
warnings.warn("No policy found for '{}'.".format(option))
yield None
except Exception as e:
warnings.warn("Create error: {}".format(e))
yield None
else:
yield flow
def create_from_resource(path:Resource, poa:list, role:list, routing:list, prefix="flow_", suffix=""):
if all(path[:5]) and not any(path[5:]):
parent = os.path.join(*path[:5])
drctry = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=parent)
flow = path._replace(flow=os.path.basename(drctry))
# MRO important here.
for registry, choices in [
("turberfield.ipc.routing", routing),
("turberfield.ipc.poa", poa),
("turberfield.ipc.role", role)
]:
policies = dict(gather_installed(registry))
for option in choices:
try:
typ = policies[option]
if issubclass(typ, Pooled):
others = [Flow.inspect(i) for i in Flow.find(path, application="*", policy=option)]
obj = typ.allocate(others=others)
else:
obj = typ()
flow = flow._replace(policy=option, suffix=".json")
with open(os.path.join(*flow[:-1]) + flow.suffix, 'w') as record:
record.write(obj.__json__())
record.flush()
except KeyError:
warnings.warn("No policy found for '{}'.".format(option))
yield None
except Exception as e:
warnings.warn("Create error: {}".format(e))
yield None
else:
yield flow
def __call__(self, token=None):
token = token or self.token
while True:
try:
job = yield from self.down.get()
poa, msg = self.hop(token, job, policy="udp")
if job.header.via is not None:
# User-defined route
hop = next(Flow.find(
token,
application=job.header.via.application,
policy="udp"),
None)
poa = Flow.inspect(hop)
if poa is None:
warnings.warn("Message expired.")
continue
if msg is not None:
remote_addr = (poa.addr, poa.port)
data = Assembly.dumps(msg)
packet = dumpb(data)
self.transport.sendto(packet, remote_addr)
else:
warnings.warn("No message from hop.")
except concurrent.futures.CancelledError:
break
except Exception as e:
warnings.warn(repr(getattr(e, "args", e) or e))
continue
def test_create_routing(self):
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
refs = list(Flow.create(tok, poa=["udp"], role=[], routing=["application"]))
features = references_by_type(refs)
routes = features[turberfield.ipc.policy.Routing.Application]
self.assertEqual(1, len(routes))
table = Flow.inspect(routes[0])
self.assertIsInstance(table, turberfield.ipc.policy.Routing.Application)
self.assertFalse(table)
rule = turberfield.ipc.policy.Routing.Application.Rule(
Address("turberfield", "tundish", "test", "turberfield.ipc.demo.sender"),
Address("turberfield", "tundish", "test", "turberfield.ipc.demo.receiver"),
1,
Address("turberfield", "tundish", "test", "turberfield.ipc.demo.hub")
)
self.assertIs(None, table.replace(rule.src, rule.dst, rule))
self.assertEqual(1, len(table))
Flow.replace(routes[0], table)
rv = Flow.inspect(routes[0])
self.assertEqual(table, rv)
def test_create_routing(self):
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
refs = list(
Flow.create(tok, poa=["udp"], role=[], routing=["application"]))
features = references_by_type(refs)
routes = features[turberfield.ipc.policy.Routing.Application]
self.assertEqual(1, len(routes))
table = Flow.inspect(routes[0])
self.assertIsInstance(table,
turberfield.ipc.policy.Routing.Application)
self.assertFalse(table)
rule = turberfield.ipc.policy.Routing.Application.Rule(
Address("turberfield", "tundish", "test",
"turberfield.ipc.demo.sender"),
Address("turberfield", "tundish", "test",
"turberfield.ipc.demo.receiver"), 1,
Address("turberfield", "tundish", "test",
"turberfield.ipc.demo.hub"))
self.assertIs(None, table.replace(rule.src, rule.dst, rule))
self.assertEqual(1, len(table))
Flow.replace(routes[0], table)
rv = Flow.inspect(routes[0])
self.assertEqual(table, rv)
def __call__(self, token=None):
token = token or self.token
while True:
try:
job = yield from self.down.get()
poa, msg = self.hop(token, job, policy="udp")
if job.header.via is not None:
# User-defined route
hop = next(
Flow.find(token,
application=job.header.via.application,
policy="udp"), None)
poa = Flow.inspect(hop)
if poa is None:
warnings.warn("Message expired.")
continue
if msg is not None:
remote_addr = (poa.addr, poa.port)
data = Assembly.dumps(msg)
packet = dumpb(data)
self.transport.sendto(packet, remote_addr)
else:
warnings.warn("No message from hop.")
except concurrent.futures.CancelledError:
break
except Exception as e:
warnings.warn(repr(getattr(e, "args", e) or e))
continue
def create_udp_node(loop, token, down, up):
"""
Creates a node which uses UDP for inter-application messaging
"""
assert loop.__class__.__name__.endswith("SelectorEventLoop")
services = []
types = Assembly.register(Policy)
policies = Policy(poa=["udp"], role=[], routing=["application"])
refs = match_policy(token, policies) or Flow.create(token, **policies._asdict())
for ref in refs:
obj = Flow.inspect(ref)
key = next(k for k, v in policies._asdict().items() if ref.policy in v)
field = getattr(policies, key)
field[field.index(ref.policy)] = obj
try:
services.append(obj.mechanism)
except AttributeError:
warnings.warn("Policy '{}' lacks a mechanism".format(ref.policy))
udp = next(iter(policies.poa))
Mix = type("UdpNode", tuple(services), {})
transport, protocol = loop.run_until_complete(
loop.create_datagram_endpoint(
lambda:Mix(loop, token, types, down=down, up=up, **policies._asdict()),
local_addr=(udp.addr, udp.port)
)
)
return protocol
def test_route_inspection_use_case(self):
self.test_create_routing()
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
search = ((i, Flow.inspect(i))
for i in Flow.find(tok, policy="application"))
query = (ref for ref, table in search for rule in table
if rule.dst.application == "turberfield.ipc.demo.receiver")
self.assertEqual(1, len(list(query)))
def match_policy(token, policy:Policy):
# MRO important here.
policies = [i for p in policy for i in p]
flows = Flow.find(token)
for flow in flows:
matched = []
for p in policies:
matched.append(next(Flow.find(token, application=token.application, policy=p), None))
if all(matched):
return matched
return None
def test_pool_allocation(self):
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
ports = range(49152, 65536)
for n, p in enumerate(ports):
with self.subTest(n=n, p=p):
flow = list(Flow.create(tok, poa=["udp"], role=[], routing=[]))
query = (Flow.inspect(i) for i in Flow.find(tok, policy="udp"))
alloc = {(i.addr, i.port) for i in query}
self.assertEqual(n + 1, len(alloc))
def test_create_policy_unregistered(self):
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
self.assertIs(None, tok.flow)
rv = next(Flow.find(tok), None)
self.assertIs(None, rv)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
rv = next(Flow.create(tok, poa=["ftp"], role=[], routing=[]))
self.assertIs(None, rv)
self.assertTrue(issubclass(w[-1].category, UserWarning))
self.assertIn("No policy", str(w[-1].message))
def test_find_flow_empty(self):
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
self.assertIs(None, tok.flow)
rv = list(Flow.find(tok))
self.assertFalse(rv)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("ignore")
rv = next(Flow.create(tok, poa=[], role=[], routing=[]), None)
self.assertIs(None, rv)
results = list(Flow.find(tok))
self.assertFalse(results)
def test_pool_allocation(self):
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
ports = range(49152, 65536)
for n, p in enumerate(ports):
with self.subTest(n=n, p=p):
flow = list(Flow.create(tok, poa=["udp"], role=[], routing=[]))
query = (Flow.inspect(i) for i in Flow.find(tok, policy="udp"))
alloc = {(i.addr, i.port) for i in query}
self.assertEqual(n + 1, len(alloc))
def test_route_inspection_use_case(self):
self.test_create_routing()
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
search = ((i, Flow.inspect(i)) for i in Flow.find(tok, policy="application"))
query = (
ref
for ref, table in search
for rule in table
if rule.dst.application == "turberfield.ipc.demo.receiver"
)
self.assertEqual(1, len(list(query)))
def test_create_policy(self):
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
self.assertIs(None, tok.flow)
rv = list(Flow.find(tok))
self.assertFalse(rv)
self.assertTrue(
list(gather_installed("turberfield.ipc.poa")),
"No declared POA endpoints; install package for testing.")
rv = next(Flow.create(tok, poa=["udp"], role=[], routing=[]))
self.assertEqual("udp", rv.policy)
self.assertEqual(".json", rv.suffix)
udp = Flow.inspect(rv)
self.assertIsInstance(udp.port, int)
def test_create_policy_unregistered(self):
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
self.assertIs(None, tok.flow)
rv = next(Flow.find(tok), None)
self.assertIs(None, rv)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
rv = next(Flow.create(tok, poa=["ftp"], role=[], routing=[]))
self.assertIs(None, rv)
self.assertTrue(
issubclass(w[-1].category, UserWarning))
self.assertIn("No policy", str(w[-1].message))
def test_find_flow_empty(self):
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
self.assertIs(None, tok.flow)
rv = list(Flow.find(tok))
self.assertFalse(rv)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("ignore")
rv = next(Flow.create(tok, poa=[], role=[], routing=[]), None)
self.assertIs(None, rv)
results = list(Flow.find(tok))
self.assertFalse(results)
def test_find_application(self):
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
self.assertIs(None, tok.flow)
results = list(Flow.find(tok, application="addisonarches.game"))
self.assertFalse(results)
def test_create_policy(self):
tok = token(
"file://{}".format(self.root.name),
"test",
"addisonarches.web"
)
self.assertIs(None, tok.flow)
rv = list(Flow.find(tok))
self.assertFalse(rv)
self.assertTrue(
list(gather_installed("turberfield.ipc.poa")),
"No declared POA endpoints; install package for testing."
)
rv = next(Flow.create(tok, poa=["udp"], role=[], routing=[]))
self.assertEqual("udp", rv.policy)
self.assertEqual(".json", rv.suffix)
udp = Flow.inspect(rv)
self.assertIsInstance(udp.port, int)
def hop(self, token, msg, policy):
here = Address(
token.namespace, token.user, token.service, token.application)
if msg.header.hop >= msg.header.hMax:
return (None, None)
msg = Message(
msg.header._replace(
hop=msg.header.hop + 1,
via=here
),
msg.payload
)
if msg.header.dst == here:
return (None, msg)
hop = next(Flow.find(
token,
application=msg.header.dst.application,
policy=policy),
None)
if hop is None:
# TODO: Get next hop from routing table
# mechanism
#search = ((i, Flow.inspect(i)) for i in Flow.find(token, policy="application"))
#query = (
# ref
# for ref, table in search
# for rule in table
# if rule.dst.application == msg.header.via.application
#)
return (None, msg)
poa = Flow.inspect(hop)
return (poa, msg)
def references_by_type(refs):
objects = [Flow.inspect(i) for i in refs]
rv = defaultdict(list)
for ref, obj in zip(refs, objects):
rv[type(obj)].append(ref)
return rv
def test_find_application(self):
tok = token("file://{}".format(self.root.name), "test",
"addisonarches.web")
self.assertIs(None, tok.flow)
results = list(Flow.find(tok, application="addisonarches.game"))
self.assertFalse(results)
def references_by_type(refs):
objects = [Flow.inspect(i) for i in refs]
rv = defaultdict(list)
for ref, obj in zip(refs, objects):
rv[type(obj)].append(ref)
return rv
