def test_remove_periodic_action_on_computation():
a = Agent("a", MagicMock())
class TestComputation(MessagePassingComputation):
def __init__(self):
super().__init__("test")
self.mock = MagicMock()
def on_start(self):
self.handle = self.add_periodic_action(0.1, self.action)
def action(self):
self.mock()
def test_remove(self):
self.remove_periodic_action(self.handle)
c = TestComputation()
a.add_computation(c)
a.start()
a.run()
sleep(0.25)
assert c.mock.call_count == 2
c.test_remove()
c.mock.reset_mock()
sleep(0.5)
c.mock.assert_not_called()
a.stop()
def test_several_periodic_action_on_computation():
a = Agent("a", MagicMock())
class TestComputation(MessagePassingComputation):
def __init__(self):
super().__init__("test")
self.mock1 = MagicMock()
self.mock2 = MagicMock()
def on_start(self):
self.add_periodic_action(0.1, self.action1)
self.add_periodic_action(0.2, self.action2)
def action1(self):
self.mock1()
def action2(self):
self.mock2()
c = TestComputation()
a.add_computation(c)
a.start()
a.run()
sleep(0.25)
a.stop()
assert 1 <= c.mock1.call_count <= 2
assert c.mock2.call_count == 1
def test_oneshot_delayed_action_on_computation():
# To implement a one-shot action, add a periodic action and remove it
# the first time it is called:
a = Agent("a", MagicMock())
class TestComputation(MessagePassingComputation):
def __init__(self):
super().__init__("test")
self.mock = MagicMock()
def on_start(self):
self.handle = self.add_periodic_action(0.1, self.action)
def action(self):
self.mock()
self.remove_periodic_action(self.handle)
c = TestComputation()
a.add_computation(c)
a.start()
a.run()
sleep(0.25)
# the action is remove on firts call there must be one single call
assert c.mock.call_count == 1
c.mock.reset_mock()
sleep(0.2)
c.mock.assert_not_called()
a.stop()
def directory_discovery():
# Agent hosting the directory
agt_dir = Agent('agt_dir', InProcessCommunicationLayer())
directory = Directory(agt_dir.discovery)
agt_dir.add_computation(directory.directory_computation)
agt_dir.discovery.use_directory('agt_dir', agt_dir.address)
agt_dir.start()
agt_dir.run(directory.directory_computation.name)
# standard agents
agt1 = Agent('agt1', InProcessCommunicationLayer())
agt1.discovery.use_directory('agt_dir', agt_dir.address)
agt1.start()
agt2 = Agent('agt2', InProcessCommunicationLayer())
agt2.discovery.use_directory('agt_dir', agt_dir.address)
agt2.start()
yield agt_dir, agt1, agt2
for c in agt1.computations():
agt1.remove_computation(c.name)
for c in agt1.discovery.agent_computations(agt1.name):
agt1.discovery.unregister_computation(c)
for c in agt2.computations():
agt2.remove_computation(c.name)
for c in agt2.discovery.agent_computations(agt2.name):
agt2.discovery.unregister_computation(c)
wait_run()
agt1.stop()
agt2.stop()
agt_dir.stop()
def test_periodic_action_not_called_when_paused():
a = Agent("a", MagicMock())
class TestComputation(MessagePassingComputation):
def __init__(self):
super().__init__("test")
self.mock = MagicMock()
def on_start(self):
self.add_periodic_action(0.1, self.action)
def action(self):
self.mock()
c = TestComputation()
a.add_computation(c)
a.start()
a.run()
sleep(0.25)
assert 1 <= c.mock.call_count <= 2
c.mock.reset_mock()
a.pause_computations("test")
sleep(0.25)
assert c.mock.call_count == 0
a.stop()
def distribute_agents(var_facts):
comm = InProcessCommunicationLayer()
dcop_agents = []
factors = [f for _, f in var_facts]
i = 0
for v, f in var_facts:
# Algorithm for variable
# build the list of factors that depend on this variable
f_for_variable = [
f.name for f in factors
if v.name in [i.name for i in f.dimensions]
]
v_a = amaxsum.VariableAlgo(v, f_for_variable)
# Algorithm for the factor
f_a = amaxsum.FactorAlgo(f)
# Agent hosting the factor and variable
a = Agent('a_' + str(i), comm)
a.add_computation(v_a)
a.add_computation(f_a)
i += 1
dcop_agents.append(a)
return dcop_agents, comm
def dpop_unary_constraint():
x0 = Variable('x0', ['a', 'b', 'c'])
x1 = Variable('x1', ['a', 'b', 'c'])
# Unary constraint on x0
@relations.AsNAryFunctionRelation(x0)
def unary_x1(x):
if x == 'a':
return 8
if x == 'b':
return 2
if x == 'c':
return 5
@relations.AsNAryFunctionRelation(x0, x1)
def prefer_different(x, y):
if x == y:
return 0
else:
return 10
al0 = DpopAlgo(x0)
al1 = DpopAlgo(x1)
# Distribution: two agents
comm = InProcessCommunicationLayer()
a0 = Agent('a0', comm)
a0.add_computation(al0)
a1 = Agent('a1', comm)
a1.add_computation(al1)
al0.set_parent(x1)
al0.add_relation(prefer_different)
al1.add_child(x0)
# we represent the unary constraint as a variable having itself as
# pseudo-parent
al0.add_relation(unary_x1)
results, _, _ = synchronous_single_run([a0, a1])
if results == {'x0': 'a', 'x1': 'b'} or \
results == {'x0': 'a', 'x1': 'c'}:
logging.info('SUCCESS !! ')
return 0
else:
logging.info('invalid result found, needs some debugging ...' + str(
results))
return 1
def maxsum_equality_noise():
"""
This sample demonstrates the use of noise to break ties between variables.
"""
l1 = VariableNoisyCostFunc('l1', list(range(10)), lambda x: x)
l2 = VariableNoisyCostFunc('l2', list(range(10)), lambda x: x)
# Scene action
y1 = VariableWithCostFunc('y1', list(range(10)), lambda x: 10 * abs(5 - x))
@AsNAryFunctionRelation(l1, l2, y1)
def scene_rel(x, y, z):
if z == x + y:
return 0
return 10000
# Create computation for factors and variables
# Light 1
algo_l1 = VariableAlgo(l1, [scene_rel.name])
# Light 2
algo_l2 = VariableAlgo(l2, [scene_rel.name])
# Scene
algo_y1 = VariableAlgo(y1, [scene_rel.name])
algo_scene = FactorAlgo(scene_rel)
comm = InProcessCommunicationLayer()
a1 = Agent('A1', comm)
a1.add_computation(algo_l1)
a2 = Agent('A2', comm)
a2.add_computation(algo_l2)
a3 = Agent('A3', comm)
a3.add_computation(algo_y1)
a3.add_computation(algo_scene)
dcop_agents = [a1, a2, a3]
results, _, _ = synchronous_single_run(dcop_agents, 5)
print(results)
if results['y1'] == 5 and results['l1'] + results['l2'] == 5:
logging.info('SUCCESS !! ')
return 0
else:
logging.info('invalid result found, needs some debugging ...' +
str(results))
return 1
def distribute_agents(var_facts, variant, probability):
comm = InProcessCommunicationLayer()
dcop_agents = []
factors = [f for _, f in var_facts]
i = 0
for v, f in var_facts:
# Algorithm for variable
# build the list of factors that depend on this variable
f_for_variable = [
f for f in factors if v.name in [i.name for i in f.dimensions]
]
v_a = dsa.DsaComputation(v, f_for_variable, variant='A')
# Agent hosting computation
a = Agent('a_' + str(i), comm)
a.add_computation(v_a)
i += 1
dcop_agents.append(a)
return dcop_agents, comm
def agents():
# Agent hosting the directory
agt_dir = Agent('agt_dir', InProcessCommunicationLayer())
directory = Directory(agt_dir.discovery)
agt_dir.add_computation(directory.directory_computation)
agt_dir.discovery.use_directory('agt_dir', agt_dir.address)
agt_dir.start()
agt_dir.run(directory.directory_computation.name)
# standard agents
agt1 = Agent('agt1', InProcessCommunicationLayer())
agt1.discovery.use_directory('agt_dir', agt_dir.address)
agt1.start()
agt2 = Agent('agt2', InProcessCommunicationLayer())
agt2.discovery.use_directory('agt_dir', agt_dir.address)
agt2.start()
yield agt_dir, agt1, agt2
agt1.stop()
agt2.stop()
agt_dir.stop()
def test_periodic_action_on_computation():
a = Agent("a", MagicMock())
class TestComputation(MessagePassingComputation):
def __init__(self):
super().__init__("test")
self.mock = MagicMock()
def on_start(self):
self.add_periodic_action(0.1, self.action)
def action(self):
self.mock()
c = TestComputation()
a.add_computation(c)
a.start()
a.run()
sleep(0.25)
a.stop()
assert c.mock.call_count == 2
def test_fix_delayed_message_on_start():
"""
Test case for bug fix:
c1 send a message to c2 during its `on_start` handler. If c2 is not started yet
at that point, the message is stored and injected once c2 starts.
Sometime, when c2 starts in a2, it does not receive the pending message from c1.
This is timing dependent and most of the time, the bug does not happens, especially
if the on_start handlers contain log statements.
The problem is due to the fact that start, and hence, the on_start handler and
pending messages injection, might be done on the main thread, while message reception
(and thus storage of th pending messages) is performed on the agent's thread.
Everything that happens in an agent MUST run on the agent thread, including startup !
To start a computation (on a started agent), one may call the `agt.run()` method,
from any thread, in which case the computation is also started from that thread,
and not from the agent thread.
Currently (10/2019), there are only two ways for avoiding this:
* use an orcherstrated agent, and start computation by sending it messages.
However this also means using an orchestrator, which is not convenient during test
and when working on non-dcop algorithm (the orchestrator is heavily biased
toward DCOPs)
* start all computations when starting the agent,
using `a.start(run_computations=True)`, which is the workaround tested here.
"""
class TestComputation1(MessagePassingComputation):
def __init__(self):
super().__init__("c1")
def on_start(self):
# self.logger.info("Starting c1, send to c2")
self.post_msg("c2", Message("foo"))
class TestComputation2(MessagePassingComputation):
def __init__(self):
super().__init__("c2")
self.received = False
# def on_start(self):
# # self.logger.info("Starting c2")
# pass
@register("foo")
def on_foo(self, sender, msg, t):
self.logger.info(f"Receiving message {msg} from {sender}")
self.received = True
c1 = TestComputation1()
c2 = TestComputation2()
a1 = Agent("a1", InProcessCommunicationLayer())
a1.add_computation(c1)
a2 = Agent("a2", InProcessCommunicationLayer())
a2.add_computation(c2)
a1.discovery.register_computation("c2", "a2", a2.address)
a2.discovery.register_computation("c1", "a1", a1.address)
agts = [a1, a2]
for a in agts:
a.start(run_computations=True)
sleep(0.5)
for a in agts:
a.stop()
for a in agts:
a.join()
# Check that c2 really received the message
assert c2.received
def dpop_nonbinaryrelation_4vars():
x0 = Variable('x0', list(range(10)))
x1 = Variable('x1', list(range(10)))
x2 = Variable('x2', list(range(10)))
x3 = Variable('x3', list(range(10)))
@relations.AsNAryFunctionRelation(x0)
def x0_prefs(x):
if x > 3:
return 0
return 10
@relations.AsNAryFunctionRelation(x1)
def x1_prefs(x):
if 2 < x < 7:
return 0
return 10
@relations.AsNAryFunctionRelation(x2)
def x2_prefs(x):
if x < 5:
return 0
return 10
@relations.AsNAryFunctionRelation(x3)
def x3_prefs(x):
if 0 < x < 5:
return 0
return 10
@relations.AsNAryFunctionRelation(x0, x1, x2, x3)
def four_ary_relation(a, b, c, d):
return abs(10 - (a + b + c + d))
def neutral_relation(x, y):
return 0
comm = InProcessCommunicationLayer()
al0 = DpopAlgo(x0, mode='min')
al1 = DpopAlgo(x1, mode='min')
al2 = DpopAlgo(x2, mode='min')
al3 = DpopAlgo(x3, mode='min')
# unary relation for preferences
al0.add_relation(x0_prefs)
al1.add_relation(x1_prefs)
al2.add_relation(x2_prefs)
al3.add_relation(x3_prefs)
# The 4-ary relation must be introduced only once, in the lowest node in
# the DFS tree (in our case, a3).
# We still need to add relation between the
# We use neutral relation between two variables that are present in the
# tree but that do not share a real constraint : in reality with the
# current implementation, these neutral relation are not needed any-more,
# I just keep them here for historical reasons.
al0.add_child(x1)
al1.set_parent(x0)
al1.add_relation(relations.NAryFunctionRelation(neutral_relation,
[x0, x1]))
al1.add_child(x2)
al2.set_parent(x1)
al2.add_relation(relations.NAryFunctionRelation(neutral_relation,
[x1, x2]))
al2.add_child(x3)
al3.set_parent(x2)
al3.add_relation(relations.NAryFunctionRelation(neutral_relation,
[x2, x3]))
al3.add_relation(four_ary_relation)
a0 = Agent('a0', comm)
a0.add_computation(al0)
a1 = Agent('a1', comm)
a1.add_computation(al1)
a2 = Agent('a2', comm)
a2.add_computation(al2)
a3 = Agent('a3', comm)
a3.add_computation(al3)
results, _, _ = synchronous_single_run([a0, a1, a2, a3])
if results == {'x0': 4, 'x2': 0, 'x1': 3, 'x3': 3}:
logging.info('SUCCESS !! ' + str(results))
return 0
else:
logging.info('invalid result found, needs some debugging ...' +
str(results))
return 1
def test_api_cg_creation_mgm2():
# This time we solve the same graph coloring problem with the MGM2
# algorithm.
# As you can see, the only difference is the creation of the AlgorithmDef
# instance with 'mgm2'
d = Domain('color', '', ['R', 'G'])
v1 = Variable('v1', d)
v2 = Variable('v2', d)
v3 = Variable('v3', d)
# We need to define all agents first, because we will need their address
# when registering neighbors computation
agt1 = Agent('a1', InProcessCommunicationLayer())
agt2 = Agent('a2', InProcessCommunicationLayer())
agt3 = Agent('a3', InProcessCommunicationLayer())
# Agent 1 with Variable v1
cost_v1 = constraint_from_str('cv1', '-0.1 if v1 == "R" else 0.1 ', [v1])
diff_v1_v2 = constraint_from_str('c1', '1 if v1 == v2 else 0', [v1, v2])
node_v1 = chg.VariableComputationNode(v1, [cost_v1, diff_v1_v2])
comp_def = ComputationDef(node_v1,
AlgorithmDef.build_with_default_param('mgm2'))
v1_computation = build_computation(comp_def)
agt1.add_computation(v1_computation)
# We need to register manually as we are not using the directory hosted by
# the orchestrator.
agt1.discovery.register_computation('v2', 'a2', agt2.address)
# Agent 2 with Variable v2
cost_v2 = constraint_from_str('cv2', '-0.1 if v2 == "G" else 0.1 ', [v2])
diff_v2_v3 = constraint_from_str('c2', '1 if v2 == v3 else 0', [v2, v3])
node_v2 = chg.VariableComputationNode(v2, [cost_v2, diff_v2_v3, diff_v1_v2])
comp_def_v2 = ComputationDef(node_v2,
AlgorithmDef.build_with_default_param('mgm2'))
v2_computation = build_computation(comp_def_v2)
agt2.add_computation(v2_computation)
agt2.discovery.register_computation('v1', 'a1', agt1.address)
agt2.discovery.register_computation('v3', 'a3', agt3.address)
# Agent 3 with Variable v3
cost_v3 = constraint_from_str('cv3', '-0.1 if v3 == "G" else 0.1 ', [v3])
node_v3 = chg.VariableComputationNode(v3, [cost_v3, diff_v2_v3])
comp_def_v3 = ComputationDef(node_v3,
AlgorithmDef.build_with_default_param('mgm2'))
v3_computation = build_computation(comp_def_v3)
agt3.add_computation(v3_computation)
agt3.discovery.register_computation('v2', 'a2', agt2.address)
# Start and run the 3 agents manually:
agts = [agt1, agt2, agt3]
for a in agts:
a.start()
for a in agts:
a.run()
sleep(1) # let the system run for 1 second
for a in agts:
a.stop()
# As we do not have an orchestrator, we need to collect results manually.
# As with MGM, MGM2 does not necessarily find the optimal solution,
# depending on the start affectation (which may require a 3-coordinated
# change and thus is not possible with mgm2), so we just check the
# hard constraints
assert agt1.computation('v1').current_value != \
agt2.computation('v2').current_value
assert agt3.computation('v3').current_value != \
agt2.computation('v2').current_value
def test_api_cg_creation_dsa():
# creating a computation graph from API, without relying on a full
# description of the DCOP.
# Two API level
# * DCOP : create a dcop
# * Computation graph:
# DCOP
# works when using an orchestrator that can transform a dcop into a
# computation graph and distribute the computation on the set of agents.
# Efficient and simple solution when there is a single deployement of the
# system at startup.
# Computation Graph
# create the computations directly
# Necessary when there is no central place where the full dcop could be
# created. Each agent build the computation it will run, for example when
# the definition of the dcop changes at run time or when a new dcop must be
# created and solve completely dynamically and runtime without relying on a
# central element like an orchestrator.
# to create a computation instance, one need:
# of course, variable(s) and constraint(s) like for the dcop.
# but also
# an algo_def : given as input
# a comp_node : depends of the type of computation graph, requires a
# variable and or constraints
# Here we define a simple graph coloring problem with 3 variables:
d = Domain('color', '', ['R', 'G'])
v1 = Variable('v1', d)
v2 = Variable('v2', d)
v3 = Variable('v3', d)
# We need to define all agents first, because we will need their address
# when registering neighbors computation
agt1 = Agent('a1', InProcessCommunicationLayer())
agt2 = Agent('a2', InProcessCommunicationLayer())
agt3 = Agent('a3', InProcessCommunicationLayer())
# Agent 1 with Variable v1
# Constraints in which v1 is involved
cost_v1 = constraint_from_str('cv1', '-0.1 if v1 == "R" else 0.1 ', [v1])
diff_v1_v2 = constraint_from_str('c1', '1 if v1 == v2 else 0', [v1, v2])
# Computation node for the variable with these constraints
node_v1 = chg.VariableComputationNode(v1, [cost_v1, diff_v1_v2])
comp_def = ComputationDef(node_v1, AlgorithmDef.build_with_default_param('dsa'))
v1_computation = build_computation(comp_def)
# and register the computation and the agents for the neighboring
# computations.
agt1.add_computation(v1_computation)
agt1.discovery.register_computation('v2', 'a2', agt2.address)
# Agent 2 with Variable v2
cost_v2 = constraint_from_str('cv2', '-0.1 if v2 == "G" else 0.1 ', [v2])
diff_v2_v3 = constraint_from_str('c2', '1 if v2 == v3 else 0', [v2, v3])
node_v2 = chg.VariableComputationNode(v2, [cost_v2, diff_v2_v3, diff_v1_v2])
comp_def_v2 = ComputationDef(node_v2,
AlgorithmDef.build_with_default_param('dsa'))
v2_computation = build_computation(comp_def_v2)
agt2.add_computation(v2_computation)
agt2.discovery.register_computation('v1', 'a1', agt1.address)
agt2.discovery.register_computation('v3', 'a3', agt3.address)
# Agent 3 with Variable v3
cost_v3 = constraint_from_str('cv3', '-0.1 if v3 == "G" else 0.1 ', [v3])
node_v3 = chg.VariableComputationNode(v3, [cost_v3, diff_v2_v3])
comp_def_v3 = ComputationDef(node_v3,
AlgorithmDef.build_with_default_param('dsa'))
v3_computation = build_computation(comp_def_v3)
agt3.add_computation(v3_computation)
agt3.discovery.register_computation('v2', 'a2', agt2.address)
# Start and run the 3 agents manually:
agts = [agt1, agt2, agt3]
for a in agts:
a.start()
for a in agts:
a.run()
sleep(1) # let the system run for 1 second
for a in agts:
a.stop()
# As we do not have an ochestrator, we need to collect results manually:
assert agt1.computation('v1').current_value != \
agt2.computation('v2').current_value
assert agt3.computation('v3').current_value != \
agt2.computation('v2').current_value
def dpop_nonbinaryrelation():
x0 = Variable('x0', list(range(10)))
x1 = Variable('x1', list(range(10)))
x2 = Variable('x2', list(range(10)))
@relations.AsNAryFunctionRelation(x0)
def x0_prefs(x):
if x > 5:
return 0
return 10
@relations.AsNAryFunctionRelation(x1)
def x1_prefs(x):
if 2 < x < 7:
return 0
return 10
@relations.AsNAryFunctionRelation(x2)
def x2_prefs(x):
if x < 5:
return 0
return 10
@relations.AsNAryFunctionRelation(x0, x1, x2)
def three_ary_relation(x, y, z):
return abs(10 - (x+y+z))
comm = InProcessCommunicationLayer()
al0 = DpopAlgo(x0, mode='min')
al1 = DpopAlgo(x1, mode='min')
al2 = DpopAlgo(x2, mode='min')
# unary relation for preferences
al0.add_relation(x0_prefs)
al1.add_relation(x1_prefs)
al2.add_relation(x2_prefs)
al0.add_child(x1)
al1.add_child(x2)
al1.set_parent(x0)
al2.set_parent(x1)
al2.add_relation(three_ary_relation)
a0 = Agent('a0', comm)
a1 = Agent('a1', comm)
a2 = Agent('a2', comm)
a0.add_computation(al0)
a1.add_computation(al1)
a2.add_computation(al2)
results, _, _ = synchronous_single_run([a0, a1, a2])
if results == {'x0': 6, 'x1': 3, 'x2': 1} or \
results == {'x0': 7, 'x1': 3, 'x2': 0}:
logging.info('SUCCESS !! ' + str(results))
return 0
else:
logging.info('invalid result found, needs some debugging ...' + str(
results))
return 1
def dmaxsum_graphcoloring():
v1 = VariableNoisyCostFunc('v1', d, prefer_color('R'))
v2 = VariableNoisyCostFunc('v2', d, prefer_color('G'))
v3 = VariableNoisyCostFunc('v3', d, prefer_color('B'))
v4 = VariableNoisyCostFunc('v4', d, prefer_color('R'))
def r1(v1_, v2_, v3_):
if v1_ != v2_ and v2_ != v3_ and v1_ != v3_:
return 0
return 100
r1 = NAryFunctionRelation(r1, [v1, v2, v3], name='r1')
def r1_2(v1_, v2_, v4_):
if v1_ != v2_ and v2_ != v4_ and v1_ != v4_:
return 0
return 100
r1_2 = NAryFunctionRelation(r1_2, [v1, v2, v4], name='r1_2')
def r2(v2_, v4_):
if v2_ != v4_:
return 0
return 100
r2 = NAryFunctionRelation(r2, [v2, v4], name='r2')
def r3(v3_, v4_):
if v3_ != v4_:
return 0
return 100
r3 = NAryFunctionRelation(r3, [v3, v4], name='r3')
relations = [r1, r2, r3]
r1_computation = DynamicFactorComputation(r1, name='r1')
r2_computation = DynamicFactorComputation(r2)
r3_computation = DynamicFactorComputation(r3)
v1_computation = \
DynamicFactorVariableComputation(
v1,
[r.name for r in find_dependent_relations(v1, relations)])
v2_computation = \
DynamicFactorVariableComputation(
v2,
[r.name for r in find_dependent_relations(v2, relations)])
v3_computation = \
DynamicFactorVariableComputation(
v3,
[r.name for r in find_dependent_relations(v3, relations)])
v4_computation = \
DynamicFactorVariableComputation(
v4,
[r.name for r in find_dependent_relations(v4, relations)])
# Prepare the agents
comm = InProcessCommunicationLayer()
a1 = Agent('a1', comm)
a1.add_computation(v1_computation)
a1.add_computation(r1_computation)
a2 = Agent('a2', comm)
a2.add_computation(v2_computation)
a1.add_computation(r2_computation)
a3 = Agent('a3', comm)
a3.add_computation(v3_computation)
a3.add_computation(v4_computation)
a3.add_computation(r3_computation)
# Expected results to check for success
expected_results = {
'r1': {
'v1': 'R',
'v2': 'G',
'v3': 'B',
'v4': 'R'
},
'r1_2': {
'v1': 'B',
'v2': 'G',
'v3': 'B',
'v4': 'R'
}
}
r1_fcts = [r1, r1_2]
agents = [a1, a2, a3]
for a in agents:
a.start()
# Now change a factor function every two seconds
r1_fct, iteration, fail = 0, 0, False
for _ in range(5):
iteration += 1
time.sleep(2)
print('### Iteration {} - function {}'.format(iteration,
r1_fcts[r1_fct].name))
print(runner.status_string())
results = runner.variable_values()
if not results == expected_results[r1_fcts[r1_fct].name]:
print('Error on results for {} : \nGot {} instead of {} !'.format(
r1_fcts[r1_fct].name, results,
expected_results[r1_fcts[r1_fct].name]))
fail = True
break
r1_fct = (r1_fct + 1) % 2
print('## Changing to function {}'.format(r1_fcts[r1_fct].name))
r1_computation.change_factor_function(r1_fcts[r1_fct])
print('Finished, stopping agents')
runner.request_stop_agents(wait_for_stop=True)
if fail:
print('Failed !')
return 1
else:
print('Success !')
return 0
class Orchestrator(object):
"""
Centralized organisation of the set of agents used to solve a dcop.
Notes
-----
Central orchestration is only used for bootstrapping the system and to
collect metrics.
As the orchestrator will generally run in a separate process, it uses an
agent object and communicates with other agents using messages.
Main responsibilities:
* deploying the computations
* collecting metrics
* running and stopping agents (note that the orchestrator does not
create nor start agents, it just request them to run their computations)
A typical use scenario:
* create and start the agents for the dcop (thread or process based)
* create the orchestrator, giving him the dcop and its distribution on
agents
* deploy the computations
* run the computations
* stop everything
Examples
--------
orchestrator.start()
orchestrator.deploy_computations()
orchestrator.start_replication() # only needed for resilient system
orchestrator.run()`
orchestrator.stop_agents()`
orchestrator.stop()`
Parameters
----------
algo: AlgorithmDef,
algorithm used to solve the dcop
cg: ComputationGraph,
computation graph
agent_mapping: Distribution,
initial distribution of computations on agents
comm: CommunicationLayer,
An instance of communication layer object
dcop: DCOP
The DCOP
infinity=float
infinity
collector: Queue
A queue used to collect metrics
collect_moment: str
metrics collection mode (e.g. 'value_change')
"""
def __init__(self, algo: AlgorithmDef, cg: ComputationGraph,
agent_mapping: Distribution,
comm: CommunicationLayer,
dcop: DCOP,
infinity=float('inf'),
collector: Queue=None,
collect_moment: str='value_change',
collect_period: float=None,
ui_port: int = None):
self._own_agt = Agent(ORCHESTRATOR, comm, ui_port=ui_port)
self.directory = Directory(self._own_agt.discovery)
self._own_agt.add_computation(self.directory.directory_computation)
self._own_agt.discovery.use_directory(ORCHESTRATOR,
self._own_agt.address)
self.discovery = self._own_agt.discovery
self.messaging = comm.messaging
self.logger = self._own_agt.logger
self.dcop = dcop
self.status = 'OK'
# For scenario execution
self._events_iterator = None
self._event_timer = None # type: threading.Timer
self._timeout_timer = None
self._stopping = threading.Event()
self.mgt = AgentsMgt(algo, cg, agent_mapping, dcop,
self._own_agt, self, infinity, collector=collector,
collect_moment=collect_moment,
collect_period=collect_period)
@property
def address(self):
return self._own_agt.address
def set_error_handler(self, callback: Callable):
"""
Set a callback that will be called if the orchestrator thread stops
due to an unexpected error.
Parameters
----------
callback: a signle-argument callable
the callback must accept a single argument, which will the the
exception that caused the orchestrator thread to stop.
"""
self._own_agt.on_fatal_error = callback
def start(self):
"""
Start the orchestrator.
Notes
-----
The orchestrator, and it's directory, must be
started in order to receive registration messages, which means you
must always start the orchestrator before the agents.
"""
self._own_agt.start()
self._own_agt.run(self.directory.directory_computation.name)
self._own_agt.add_computation(self.mgt, ORCHESTRATOR_MGT)
self._own_agt.run(self.mgt.name)
def stop(self):
"""
Stop the orchestrator.
Notes
-----
Once stopped, the orchestrator will not receive
nor send any new message. This means that agents must be stopped before
stopping the orchestrator.
"""
self.logger.info('Requesting orchestrator to stop')
self._own_agt.stop()
if self._event_timer is not None:
self._event_timer.cancel()
self._event_timer = None
def deploy_computations(self, once_registered=True):
"""
Deploy the computation for the dcop.
The computations are deployed according to the Computation Graph
and the initial distribution (given to the Orchestrator's constructor).
Parameters
----------
once_registered: bool
wait until all agents have registered before starting deployment.
"""
if once_registered:
self.logger.info('Waiting for all registration before deploying '
'computations')
self.mgt.all_registered.wait()
self.logger.info('deploying computations')
self._mgt_method('_orchestrator_deploy_computations', None)
def start_replication(self, k_target: int):
"""
Ask all agents to replicate their computations.
Notes
-----
deploy_computations must be called before, otherwise agents have no
computation to replicate !
Parameters
----------
k_target: int
number of replica for each computation (aka resiliency level).
"""
# We must be sure computations have been deployed first
self.logger.info('Waiting until agents are ready to run before '
'starting replication')
self.mgt.ready_to_run.wait()
self.mgt.ready_to_run = threading.Event()
self.logger.info('Starting replication')
self._mgt_method('_orchestrator_start_replication', k_target)
def run(self, scenario: Scenario=None,
timeout: Optional[float]=None, repair_only=False):
"""Run the DCOP, with a scenario if given.
When `run()` is called, the orchestrator asks all orchestrated agents to
start their computations. If the agents are not ready, the orchestrator
automatically waits until agents are ready (i.e. computations have
been deployed).
Parameters
----------
scenario: Scenario
an optional Scenario object whose events will be injected into
the system.
timeout: float
time, in seconds, after which all agents, and the orchestrator
itself, must be stopped.
"""
self.repair_only = repair_only
self.logger.info('Waiting until agents are ready to run')
self.mgt.ready_to_run.wait()
self.logger.info('Requesting agents to run')
self._mgt_method('_orchestrator_run_computations', None)
if timeout is not None:
self.logger.info('Setting timer for %s timeour ', timeout)
self._timeout_timer = threading.Timer(timeout,
self._on_timeout)
self._timeout_timer.daemon = True
self._timeout_timer.start()
self.mgt.ready_to_run = threading.Event()
else:
self.logger.info('Not timeout, stop with ctrl+c or on algo end ')
if scenario is not None:
self.logger.info('Setting scenario ')
self._events_iterator = iter(scenario)
self._process_event()
else:
self.logger.info('No scenario ')
self.mgt.wait_stop_agents()
self._own_agt.clean_shutdown()
self._own_agt.join()
def stop_agents(self, timeout: float):
self.logger.info('Requesting all agents to stop')
self._stopping.set()
# WARNING: must NOT access the mgt directly, all its action
# must be done in the agent's tread. That's the reason we use a msg
# here. It must have MSG_MGT type to have higher priority, in case the
# orchestrator's queue is full of other messages.
if self._event_timer is not None:
self._event_timer.cancel()
self._event_timer = None
self._mgt_method('_orchestrator_stop_agents', None)
self.mgt.wait_stop_agents(timeout)
self.mgt.ready_to_run.set()
def current_global_cost(self):
return self.mgt.current_global_cost()
def current_solution(self):
return self.mgt.current_solution()
def end_metrics(self):
return self.mgt.global_metrics('END', self.mgt.last_agt_stop_time)
def replication_metrics(self):
return self.mgt._replication_metrics
def wait_ready(self):
"""Blocks until the Orchestrator is ready to perform another action.
This can be used to wait until the dcop has finished running,
for example when using a timeout.
Notes
-----
When calling `wait_ready` after `run()` with a timeout, you may be
blocked for a longer time than the timeout, as orchestrator also wait
until all agents have stopped.
Examples
--------
orchestrator.run(timeout=5)
orchestrator.wait_ready()
"""
self.mgt.ready_to_run.wait()
return self._own_agt.is_running and not self._stopping.is_set()
def _process_event(self):
# FIXME: hack too avoid overlapping events
waited = [a for a, state in self.mgt._agts_state.items()
if state != 'running']
if waited:
self.logger.warning(f"Event while agents {waited} are still processing"
f" previous event, wait 20 s ")
self._event_timer = threading.Timer(20, self._process_event)
self._event_timer.start()
return
try:
evt = next(self._events_iterator)
except StopIteration:
self.logger.info("All events processed for scenario")
self._events_iterator = None
return
if evt.is_delay:
self.logger.info('Delay: wait %s s for next event', evt.delay)
self._event_timer = threading.Timer(evt.delay, self._process_event)
self._event_timer.start()
else:
self.logger.info('posting event to mgt %s', evt)
self._mgt_method('_orchestrator_scenario_event', evt)
self._process_event()
def _mgt_method(self, method: str, arg: Any):
self.messaging.post_msg(
ORCHESTRATOR_MGT, ORCHESTRATOR_MGT,
Message(method, arg), msg_type=5)
def _on_timeout(self):
"""Run timeout callback"""
self.status = "TIMEOUT"
self.logger.info("Timeout, requesting agents to stop")
self.stop_agents(5)
self.mgt.ready_to_run.set()
def dpop_graphcoloring_1():
x0 = Variable('x0', ['R', 'G', 'B'])
x1 = Variable('x1', ['R', 'G', 'B'])
x2 = Variable('x2', ['R', 'G', 'B'])
# Unary constraint on x0
@relations.AsNAryFunctionRelation(x0)
def x0_prefers_r(x):
if x == 'R':
return 0
return 10
@relations.AsNAryFunctionRelation(x1)
def x1_prefers_g(x):
if x == 'G':
return 0
return 10
@relations.AsNAryFunctionRelation(x2)
def x2_prefers_b(x):
if x == 'B':
return 0
return 10
def prefer_different(x, y):
if x == y:
return 10
else:
return 0
r1_0 = relations.NAryFunctionRelation(prefer_different, [x0, x1])
r0_2 = relations.NAryFunctionRelation(prefer_different, [x0, x2])
r1_2 = relations.NAryFunctionRelation(prefer_different, [x1, x2])
# Create computations objects to solve this problem
# For this we must define the DFS tree
# x0 ---> x1 ---> x2
# preferences are modeled as unary relation, set directly on each variable
# other constraints are represented by binary relations which are set on
# the lowest variable in the tree.
c0 = DpopAlgo(x0, mode='min')
c0.add_child(x1)
c0.add_relation(x0_prefers_r)
c1 = DpopAlgo(x1, mode='min')
c1.set_parent(x0)
c1.add_child(x2)
c1.add_relation(x1_prefers_g)
c1.add_relation(r1_0)
c2 = DpopAlgo(x2, mode='min')
c2.set_parent(x1)
c2.add_relation(x2_prefers_b)
c2.add_relation(r0_2)
c2.add_relation(r1_2)
# Distribution: 3 agents, one for each variable
comm = InProcessCommunicationLayer()
a0 = Agent('a0', comm)
a1 = Agent('a1', comm)
a2 = Agent('a2', comm)
a0.add_computation(c0)
a1.add_computation(c1)
a2.add_computation(c2)
results, _, _ = synchronous_single_run([a0, a1, a2])
if results == {'x0': 'R', 'x1': 'G', 'x2': 'B'}:
logging.info('SUCCESS !! ')
return 0
else:
logging.info('invalid result found, needs some debugging ...' +
str(results))
return 1