def solve_all(self, data):
"""
This method expects data to be an object of the form {"id": pirate_id, "data": [{"id": clue_id, "data": clue_data}]}
"""
logger.info("Solving all the clues")
pirate_id = data["id"]
logger.debug("pirate_id: %s", pirate_id)
clue_list = data["data"]
logger.debug("num clues: %s", len(clue_list))
results = list(self.solve(clue) for clue in clue_list)
summary = {"id": pirate_id, "data": results}
pirates = discover('Pirate')
first_mate = None
for pirate in pirates:
if pirate != (self.exposed_host(), self.exposed_port()):
c = rpyc.connect(pirate[0], pirate[1])
is_first_mate = c.root.is_leader
answer = is_first_mate()
if answer:
first_mate = pirate
break
while not first_mate:
logger.warn("No leader found, time to get a new one (%s)",
self.exposed_get_id())
self.exposed_elect_leader()
first_mate = leader
c = rpyc.connect(first_mate[0], first_mate[1])
verify_clues = rpyc. async (c.root.verify)
result = verify_clues(summary)
result.wait()
def distribute_work(self, pirate_data):
"""
This method expects pirate_data to be of the form [{"id": pirate_id, "data": [{"id": clue_id, "data": clue_data}]}].
"""
logger.info('Starting work distribution')
logger.info("Number of pirates: %s", len(pirate_data))
all_clues = []
for pirate in pirate_data:
logger.debug("Adding %s to global clue list", len(pirate["data"]))
all_clues.extend(pirate["data"])
logger.info("Number of clues to be distributed: %s", len(all_clues))
pirates = discover("pirate")
logger.info("Found pirates: %s", pirates)
logger.info("Splitting the clues into %s chunks", len(pirates) - 1)
chunks = list(
[all_clues[i::len(pirates) - 1] for i in range(len(pirates) - 1)])
logger.info("Split into %s chunks", len(chunks))
for chunk in chunks:
logger.debug("Chunk length: %s", len(chunk))
logger.debug('All pirates: %s', pirates)
logger.debug('Leader: %s', leader)
logger.debug('My host and port: %s',
(self.exposed_host(), self.exposed_port()))
for pirate in pirates:
if pirate != (self.exposed_host(), self.exposed_port()):
logger.info("Sending clues to pirate at %s", pirate)
c = rpyc.connect(pirate[0], pirate[1])
give_clues = rpyc. async (c.root.set_clues)
clues = chunks.pop()
result = give_clues(clues)
result.wait()
def exposed_elect_leader(self):
global leader
logger.info('Electing a new leader')
pirates = discover('Pirate')
logger.info('Pirates available: %s', pirates)
pirates = sorted(pirates)
logger.debug('Sorted pirates: %s', pirates)
leader = (pirates[0][0], pirates[0][1])
logger.info('Found a new leader: %s', leader)
for pirate in pirates:
if (self.exposed_host(), self.exposed_port()) != pirate:
c = rpyc.connect(pirate[0], pirate[1])
set_leader = c.root.set_leader
set_leader(leader)
return leader
def cluster_connect(self, max_connections=1):
if len(self.service.connections) >= max_connections:
self.logger.debug("Max connections reached ({})! ".format(
len(self.service.connections)))
return True # We are connected with max connections
try:
con_tuples = discover(self.name)
except rpyc.utils.factory.DiscoveryError as de:
self.logger.warning(de)
return False
self.logger.debug("Discovered {} {}s".format(len(con_tuples),
self.name))
for i in range(max_connections):
con_tuple = con_tuples[i]
# If the connection exists do not repeat it
if self._already_connected(con_tuple):
continue
self.logger.debug("Connecting to {}:{}".format(*con_tuple))
rpyc.connect(*con_tuple, service=self.service)
return True
def exposed_verify(self, data):
"""
This method expects data to be of the form {"id": pirate_id, "data": [{"id": clue_id, "data": clue_data}]}
"""
logger.info('Verifying with the captain...')
request = qm.root.verify
results = request(data)
logger.info("%s: %s", results["status"], results["message"])
try:
finished = results["finished"]
except Exception:
finished = False
if finished:
logger.info(
"Finally I'm done. Now to do Captain Rummy's dirty work so that we can split the treasure. Time to kill the crew."
)
pirates = discover('Pirate')
for pirate in pirates:
if pirate != (self.exposed_host(), self.exposed_port()):
c = rpyc.connect(pirate[0], pirate[1])
kill = rpyc. async (c.root.close_server)
kill()
logger.info('And now that bastard of a quartermaster.')
kill = rpyc. async (qm.root.close_server)
kill()
logger.info(
"Wait, Captain Rummy, don't shoot! What about our pla--?")
self.exposed_close_server()
else:
try:
data = results["data"]
except:
logger.info("I'm done, I wonder what the others are up to...")
data = []
if data:
logger.info("Not finished, redistributing failed clues")
self.distribute_work(results["data"])
import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s: (main) %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
DEV_MODE = False
if __name__ == '__main__':
# Start the quartermaster service
qm_servers = discover('QuarterMaster')
qm = qm_servers[0]
logger.info('Quartermaster running on {}:{}'.format(qm[0], qm[1]))
# Connect to the qm service
c = rpyc.connect(qm[0], qm[1], config={"logger": logger})
# Set the netrefs
wake = c.root.wake
unlock = c.root.unlock
gather = c.root.gather
prepare = c.root.prepare
add = c.root.add
remove = c.root.remove
ship_out = c.root.ship_out
clues = c.root.clues
verify = c.root.verify
# Execute the correct netrefs and initialize all agents.
def main_exec(args):
if args.mode == 'discover':
try:
conn_tuples = discover(args.name)
for conn_tuple in conn_tuples:
print("Connecting to {}".format(conn_tuple))
conn = rpyc.connect(*conn_tuple)
serv = conn.root
print("~ Dumping:")
pprint(serv.sync())
conn.close()
except rpyc.utils.factory.DiscoveryError as de:
print(de)
except AttributeError:
conn.close()
if args.mode == 'set':
try:
conn_tuples = discover(args.name)
for conn_tuple in conn_tuples:
print("Connecting to {}".format(conn_tuple))
conn = rpyc.connect(*conn_tuple)
serv = conn.root
# Keep the types of the key and value from CLI
key = eval(args.key)
value = eval(args.value)
print("~ '{}[{}] = {}'".format(args.name, key, value))
serv.set(key, value)
conn.close()
except rpyc.utils.factory.DiscoveryError as de:
print(de)
if args.mode == 'get':
try:
conn_tuples = discover(args.name)
for conn_tuple in conn_tuples:
print("Connecting to {}".format(conn_tuple))
conn = rpyc.connect(*conn_tuple)
serv = conn.root
# Keep the type of the key from CLI
key = eval(args.key)
print("~ '{}[{}]'".format(args.name, key))
print(serv.get(key))
conn.close()
except rpyc.utils.factory.DiscoveryError as de:
print(de)
if args.mode == 'serve':
if args.json_file:
data = json.load(args.json_file)
else:
data = {}
print('{}({})'.format(args.name, pformat(data)))
try:
cd = ClusterDict(store=data, name=args.name)
if args.exit:
cd.sync()
cd.disconnect()
sys.exit(0)
import code
code.interact(local=locals(),
banner="""
================================================
Variable 'cd' contains ClusterDict "{name}"
================================================
""".format(name=args.name))
# Reached here after killing Interactive console
cd.service.close_down()
except rpyc.utils.factory.DiscoveryError as de:
print(de)
except KeyboardInterrupt as ki:
cd.service.close_down()
print("Aborted by the user")
try:
data = results["data"]
except:
logger.info("I'm done, I wonder what the others are up to...")
data = []
if data:
logger.info("Not finished, redistributing failed clues")
self.distribute_work(results["data"])
def exposed_close_server(self):
logger.info('Shutting down...')
server.close()
if __name__ == '__main__':
# Get the local IP address of this pirate. The ThreadedServer#server attribute is not used, due to it always returning '0.0.0.0'.
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
myhost = s.getsockname()[0]
s.close()
leader = tuple()
pirate_id = None
qm_details = discover('QuarterMaster')
if len(qm_details) != 1:
raise Exception(
'One and only one quartermaster should exist before trying to wake a pirate up'
)
qm = rpyc.connect(qm_details[0][0], qm_details[0][1])
server = ThreadedServer(PirateService, auto_register=True, logger=logger)
server.start()