def __configure_channels(self, args):
if args.file is not None:
with open(args.file) as f:
config = f.read()
config = json.loads(config)
LOG.debug("json loaded %s", config)
self.broadcaster.configure(config)
def test_broadcast(self, msg_data, config=None):
if config is None:
config = self.config
if config is None:
return self.broadcast(msg_data)
msg = Message(None, msg_data, None)
msg.time = time.time()
msg.host = self.server_id
server_config = config[str(self.server_id)]
for c in self.channels:
channel_config = server_config[str(c.channel_id)]
if channel_config[0][0] == 0:
LOG.info("skipping c%i at server %i", c.channel_id,
self.server_id)
continue
delay = channel_config[0][1]
if delay > 0:
LOG.info("add %i delay to c%i at server %i", delay,
c.channel_id, self.server_id)
time.sleep(delay)
for id, host in self.hosts.items():
if id == self.server_id:
continue
if channel_config[id + 1] == 0:
# continue
break
c.send(host.ip, host.port, msg)
def kill(self):
"""terminate cluster"""
for _, server in self.servers.items():
server['log'].close()
server['process'].kill()
LOG.info('cluster killed')
def send_new_group(self, t):
""" Sends a reconfigure request for the group consisting of the id """
msg_dict = {'new_group': True, 'gid': t, 'id': self.host.id}
msg_bytes = json.dumps(msg_dict).encode()
msg_size = struct.pack('i', len(msg_bytes))
LOG.debug("host%i sending new_group: %f", self.host.id, t)
# LOG.debug("host%i sending new_group: %s", self.host.id, msg_bytes)
self.atomic_b.broadcast(msg_size + msg_bytes)
def send(self, id, buf):
if id not in self.servers:
LOG.error("server %i does not exists", id)
return False
LOG.debug("sending %s to server %i", buf, id)
buf += '\n'
self.servers[id]['process'].stdin.write(buf.encode())
self.servers[id]['process'].stdin.flush()
return True
def send_broadcast(self, time, new_group=False):
""" Broadcast a message to all hosts """
if new_group:
LOG.debug("Host:%i, Sending new_group:%f", self.host.id, time)
else:
LOG.debug("Host:%i, Sending present gid:%f", self.host.id, time)
msg = struct.pack(self.msg_fmt, new_group, time, self.host.id)
self.atomic_b.broadcast(msg)
def remove_server(c, *args):
"""
:c: cluster object
:id: id of server to remove
"""
# FIXME
id = int(args[0])
LOG.info("removing server %i", id)
c.remove(id)
def __test_broadcast(self, args):
config = None
if args.file is not None:
with open(args.file) as f:
config = f.read()
config = json.loads(config)
LOG.debug("json loaded %s", config)
# self.broadcaster.broadcast(' '.join(args.message).encode())
# self.__configure_channels(self, args)
self.broadcaster.test_broadcast(' '.join(args.message).encode(),
config)
def main():
"""main entry point"""
command_handlers = {
'server': run_server,
'cluster': run_cluster,
}
parser = cli.configure_parser()
args = parser.parse_args()
LOG.setLevel(logging.DEBUG if args.verbose else logging.INFO)
command_handlers[args.subcmd](args)
def __init__(self, args):
self.port = 50000 + 100 * args.id
self.servers = {}
LOG.debug("starting server %i with port %i", args.id, self.port)
LOG.debug("starting server with args %s", args)
for server in args.servers:
split = server.split(':')
id = int(split[1])
ip = split[0]
self.servers[id] = Host(id, ip)
self.servers[args.id] = Host(args.id, split[0])
# LOG.debug("server list %s", self.servers)
self.broadcaster = AtomicBroadcaster(args.id, self.port,
self.servers, 1)
# check if we running an membership protocol
if args.protocol == 'periodic':
self.periodic_group = PeriodicBroadcastGroup(self.broadcaster,
self.servers[args.id],
args.join)
elif args.protocol == 'list':
self.attendance = AttendanceListGroup(self.broadcaster,
self.servers[args.id],
args.join)
elif args.protocol == 'neighbor':
self.attendance = NeighborSurveillanceGroup(self.broadcaster,
self.servers[args.id],
args.join)
self.commands = {
'bc': self.__test_broadcast,
'config': self.__configure_channels,
'destroy': self.__destroy,
}
self.parser = cli.configure_parser()
while True:
cmd = input()
LOG.info("cmd: %s", cmd)
cmd = cmd.split()
args = self.parser.parse_args(cmd)
LOG.info(args)
if args.subcmd not in self.commands:
LOG.info("cmd not found")
self.commands[args.subcmd](args)
def __membership_check(self, check_time):
LOG.debug("host%i membership check task %f", self.host.id, check_time)
# self.members.add(self.host.id)
if self.host.id == max(self.members):
self.send_list([self.host.id])
gamma = len(self.members) * self.sigma
confirm_time = check_time - time.time() + gamma
confirm_task = th.Timer(confirm_time,
self.__membership_confirmation,
args=(check_time + gamma,))
self.scheduled_tasks.append(confirm_task)
confirm_task.start()
mem_check_time = check_time - time.time() + self.period
mem_check_task = th.Timer(mem_check_time,
self.__membership_check,
args=(check_time + self.period,))
self.scheduled_tasks.append(mem_check_task)
mem_check_task.start()
def __membership_confirmation(self, check_time):
#TODO this is probably broken sends in delta check time instead of abs check time
LOG.debug("host%i: membership confirm task", self.host.id)
#time.time() > check_time:
# return
if self.last_r_t + len(self.members) * self.sigma + .1 < check_time:
LOG.debug("%i: SENDING NEW GROUP!!!!!!!!!!", self.host.id)
new_group_time = time.time() + self.delta
self.send_new_group(new_group_time)
# self.send_broadcast(check_time + self.delta, new_group=True)
self.present_members.add(self.host.id)
confirm_time = new_group_time + 2 * self.delta
confirm_task = th.Timer(confirm_time - time.time(),
self.__new_group_confirmation,
args=(new_group_time, ))
# self.scheduled_broadcasts[confirm_time] = confirm_task
self.scheduled_tasks.append(confirm_task)
confirm_task.start()
def __new_group_confirmation(self, check_time):
LOG.info("\033[95 m%i: members arrived before confirm %s\033[0m",
self.host.id, self.present_members)
if self.present_members != self.members:
LOG.debug("confirming: UPDATING mems from %s to %s",
self.members, self.present_members)
self.members = self.present_members
# self.group += self.period
# self.group = check_time
self.present_members = set([self.host.id])
self.check_members = set([self.host.id])
else:
LOG.debug("%i: confirming: OKAY mems at %s", self.host.id,
self.members)
self.check_members = set([self.host.id])
# XXX
# for key, task in self.scheduled_broadcasts.items():
# # if msg[0] <= key:
# task.cancel()
# self.scheduled_broadcasts = {}
for task in self.scheduled_tasks:
task.cancel()
self.scheduled_tasks = []
# self.__membership_check(check_time)
next_check_time = check_time + self.period
next_check_task = th.Timer(next_check_time - time.time(),
self.__membership_check,
args=(next_check_time,))
# self.scheduled_broadcasts[next_check_time] = next_check_task
self.scheduled_tasks.append(next_check_task)
next_check_task.start()
def __membership_confirmation_task(self, check_time):
LOG.info("\033[95 m%i: members arrived before check %s\033[0m",
self.host.id, self.check_members)
if self.check_members != self.cur_members:
LOG.debug("confirming: UPDATING mems from %s to %s",
self.cur_members, self.check_members)
self.cur_members = self.check_members
self.cur_group += self.period
# reset
self.check_members = set([self.host.id])
else:
# XXX
self.cur_group += self.period
LOG.debug("%i: confirming: OKAY mems at %s", self.host.id,
self.cur_members)
self.check_members = set([self.host.id])
for key, task in self.scheduled_broadcasts.items():
# if msg[0] <= key:
task.cancel()
self.scheduled_broadcasts = {}
# schedule next check task; next check at check_time + period
next_check_time = check_time + self.period
next_check_task = th.Timer(next_check_time - time.time() - 1,
self.__membership_check_task,
args=(next_check_time, ))
self.scheduled_broadcasts[next_check_time] = next_check_task
next_check_task.start()
def add(self, id, ip='127.0.1.1', init=False, join=False):
"""
adds server id to the cluster
:id: new server's id
"""
if id in self.servers:
LOG.error("server %i already exists", id)
return False
elif id < 0 or id > 100:
LOG.error("id %i is out of range [0, 100]", id)
return False
# TODO clean up
args = ['python3', '-m', 'membership', 'server', '-s']
# args += [ip + ':' + str(id) for id in self.servers]
# if init:
# args += [ip + ':' + str(id) for id in range(self.args.count)]
args += [ip + ':' + str(i) for i in range(10)]
if init and id == 0:
args += ['-j']
if join:
args += ['-j']
# else:
# args += [ip + ':' + str(id) for id in self.servers]
# LOG.debug("protocol: %s", self.args.protocol)
if self.args.protocol == 'periodic':
args += ['-p', self.args.protocol]
elif self.args.protocol == 'neighbor':
args += ['-p', self.args.protocol]
elif self.args.protocol == 'list':
args += ['-p', self.args.protocol]
if self.args.verbose:
args.append('-v')
args += ['-i', str(id)]
# open log file
log = open(os.path.join('logs', str(id) + '.log'), 'w')
LOG.info('starting server: %i', id)
LOG.debug('arguments: %s', args)
# spawn child process
if self.args.debug:
p = Popen(args, stdout=sys.stdout, stderr=sys.stderr, stdin=PIPE)
else:
p = Popen(args, stdout=log, stderr=log, stdin=PIPE)
self.servers[id] = {'process': p, 'log': log}
return True
def msg_handler(self, msg):
""" Handle received message """
msg_size = struct.unpack('i', msg.data[0:4])[0]
#LOG.debug("size: %i, json: %s", msg_size, msg.data[4:4+msg_size])
# LOG.debug("host%i recieved msg: %s", self.host.id, msg_dict)
msg_dict = json.loads(msg.data[4:4 + msg_size].decode())
# if "new-group" received
if 'new_group' in msg_dict:
# cancel membership_check and membership_confirmation tasks
for task in self.scheduled_tasks:
task.cancel()
self.scheduled_tasks = []
#can try if time.time > ['gid'] later
LOG.info("%i: new_group %f, from %i", self.host.id,
msg_dict['gid'], msg_dict['id'])
self.members = set()
self.group = msg_dict['gid']
self.send_present(msg_dict['gid'])
LOG.debug("timer for %f",
msg_dict['gid'] - time.time() + self.period)
self.present_members = set([self.host.id, msg_dict['id']])
# confirm_time = new_group_time + 2 * self.delta
wait_t = msg_dict['gid'] + 2 * self.delta - time.time()
confirm_task = th.Timer(wait_t,
self.__new_group_confirmation,
args=(msg_dict['gid'], ))
# self.scheduled_broadcasts[confirm_time] = confirm_task
self.scheduled_tasks.append(confirm_task)
confirm_task.start()
# check_task = th.Timer(msg_dict['gid'] - time.time() + self.period,
# self.__membership_check,
# args=(msg_dict['gid'],))
# check_task.start()
# self.scheduled_tasks.append(check_task)
elif 'present' in msg_dict:
#TODO check correct group id
LOG.info("%i: present %f, from %i", self.host.id, msg_dict['gid'],
msg_dict['id'])
# self.members.add(msg_dict['id'])
self.present_members.add(msg_dict['id'])
elif 'list' in msg_dict:
#TODO check time < O and gamma
LOG.info("%i: list received", self.host.id)
self.last_r_t = time.time(
) #TODO this is O not 0 needs to be fixed
def __forward_task(self, msg):
""" determine the channels to forward msg on """
duration = msg.get_timely_deadline(self.sigma) - time.time()
# LOG.debug('sleeping for %f in task', duration)
time.sleep(duration)
self.c_hist_lock.acquire()
highest_chan_recv = self.c_history[(msg.time, msg.host)]['c'] # c
highest_chan_send = self.channel_count - msg.hops # f + 1 - h
# check if c < f + 1 - h
if highest_chan_recv < highest_chan_send:
LOG.info("%i is forwarding msg from %i", self.server_port,
msg.host)
msg.add_hop()
# forward on channels c + 1, ..., f + 1 - h
for channel in self.channels[highest_chan_recv:highest_chan_send]:
for _, host in self.hosts.items():
if host.id == self.server_id:
continue
channel.send(host.ip, host.port, msg)
self.c_hist_lock.release()
def send_list(self, members):
LOG.debug("host%i sending list %s", self.host.id, members)
msg_dict = {'list': True, 'gid': self.group, 'members': list(members)}
msg_bytes = json.dumps(msg_dict).encode()
msg_size = struct.pack('i', len(msg_bytes))
dest = self.get_next_host()
if self.host.id == max(self.members):
dest = min(self.members)
msg = Message(None, msg_size + msg_bytes, -1)
msg.hops = -1
msg.time = 0
msg.host = -1
msg.chan = -1
LOG.debug("host%i, sending list to host%i", self.host.id, dest)
LOG.debug("members: %s", self.members)
port = 50000 + (100 * dest) + 1
# send on the first channel, abuse the system
self.atomic_b.channels[0].socket.sendto(
msg.marshal(), (socket.gethostbyname(socket.gethostname()), port))
def remove(self, id):
"""
removes server id from the cluster
:id: server id to remove
"""
if id not in self.servers:
LOG.error("server %i does not exists", id)
return False
buf = 'destroy'
LOG.debug("sending %s to server %i", buf, id)
buf += '\n'
self.servers[id]['process'].stdin.write(buf.encode())
self.servers[id]['process'].stdin.flush()
time.sleep(1)
LOG.info("forcibly terminating server %i", id)
self.servers[id]['log'].close()
self.servers[id]['process'].kill()
del self.servers[id]
def msg_handler(self, msg):
""" Handle receipt of broadcasts """
msg = struct.unpack(self.msg_fmt, msg.data[:20])
# if on time; myclock > V abort
# if msg[1] < time.time():
# if msg[1] > (time.time() - .2):
if True:
# LOG.debug("TIME, MYCLOCK: %s, %s", msg[1], time.time() - .2)
# if "new-group" received
if msg[0]:
# cancel broadcasts
# LOG.debug("new group %s", self.scheduled_broadcasts)
for key, task in self.scheduled_broadcasts.items():
# if msg[0] <= key:
task.cancel()
self.scheduled_broadcasts = {}
LOG.info("%i: new group requested", self.host.id)
self.cur_group = msg[1]
self.cur_period = 0
self.check_members = set([self.host.id, msg[2]])
self.send_broadcast(msg[1])
# schedule check for the new group req
confirm_task = th.Timer(2 * self.delta,
self.__membership_confirmation_task,
args=(msg[1], ))
confirm_task.start()
# if "present" received
else:
LOG.info("%i: member added %d", self.host.id, msg[2])
# put the member in the group
# if self.cur_group == msg[1]:
self.check_members.add(msg[2])
# LOG.info("%i: members after add %s", self.host.id,
# self.check_members)
else:
LOG.info("LATE %s, %s", msg[1], time.time())
def destroy(self):
for chan in self.channels:
chan.destroy()
LOG.info("destroying broadcaster")