def __init__(self, me):
self.should_shutdown = False
self.sock = None
self.group = parameters.get_property(MULTICAST_GROUP_PROPERTY, DEFAULT_MULTICAST_GROUP)
self.port = int(parameters.get_property(MULTICAST_PORT_PROPERTY, DEFAULT_MULTICAST_PORT))
self.address = parameters.get_property(ADDRESS_PROPERTY, DEFAULT_ADDRESS)
self.me = me
self.listen_thread = None
def __init__(self, me):
self.should_shutdown = False
self.sock = None
self.group = parameters.get_property(MULTICAST_GROUP_PROPERTY,
DEFAULT_MULTICAST_GROUP)
self.port = int(
parameters.get_property(MULTICAST_PORT_PROPERTY,
DEFAULT_MULTICAST_PORT))
self.address = parameters.get_property(ADDRESS_PROPERTY,
DEFAULT_ADDRESS)
self.me = me
self.listen_thread = None
def __init__(self):
super().__init__()
self.interceptors = dict()
self.data_reactor = Reactor()
self.gossip_reactor = Reactor()
protocol = parameters.get_property(PROTOCOL_PROPERTY, DEFAULT_PROTOCOL)
address = parameters.get_property(ADDRESS_PROPERTY, DEFAULT_ADDRESS)
gossip_port = parameters.get_property(GOSSIP_PORT_PROPERTY)
data_port = parameters.get_property(DATA_PORT_PROPERTY)
if not gossip_port:
gossip_port = network_util.get_free_port()
if not data_port:
data_port = network_util.get_free_port()
if protocol == 'tcp':
self.gossip_server = ThreadedTCPServer((address, gossip_port), GossipHandler, self.gossip)
self.gossip_server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self.gossip_server.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.gossip_thread = threading.Thread(target=self.gossip_server.serve_forever)
self.data_server = ThreadedTCPServer((address, data_port), DataHandler, self.send)
self.data_server.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self.data_server.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.data_thread = threading.Thread(target=self.data_server.serve_forever)
logger.info('Starting TCP node on gossip port ' + str(gossip_port) + ' : data port ' + str(data_port))
else:
self.gossip_server = ThreadedUDPServer((address, gossip_port), GossipHandler, self.gossip)
self.gossip_thread = threading.Thread(target=self.gossip_server.serve_forever)
self.data_server = ThreadedUDPServer((address, data_port), DataHandler, self.send)
self.data_thread = threading.Thread(target=self.data_server.serve_forever)
logger.info('Starting UDP node on gossip port ' + str(gossip_port) + ' : data port ' + str(data_port))
self.gossip_thread.daemon = True
self.gossip_thread.start()
self.data_thread.daemon = True
self.data_thread.start()
self.ip = address
self.gossip_port = gossip_port
self.data_port = data_port
logger.info('Node started')
return
def get_ip():
global ip
if ip:
return ip
ip = socket.gethostbyname(socket.gethostname())
return ip
nic = parameters.get_property(NETWORK_INTERFACE_PROPERTY)
# if not nic:
# interfaces = os.networkInterfaces()
# os = parameters.current_os()
#
# switch(parameters.getInstance().currentOS()):
# case OperatingSystem.WIN_64:
# case OperatingSystem.WIN_32:
# match = "Loopback";
# break;
# case OperatingSystem.LINUX_64:
# case OperatingSystem.LINUX_32:
# match = "lo";
# break;
# case OperatingSystem.MAC_64:
# case OperatingSystem.MAC_32:
# match = "lo0";
# break;
# default:
# logger.error("Cannot determine operating system. Cluster cannot form.");
return
def handle_gossip_message(self, message):
if self.shutting_down:
return
sender_key = utils.get_key(ip=message.ip, gossip_port=message.gossip_port, data_port=message.data_port)
update_tags = False
for i in range(0, len(message.members)):
key = message.members[i]
m = self.member_holder.get_member(key)
if m is None:
protocol = parameters.get_property(PROTOCOL_PROPERTY, DEFAULT_PROTOCOL)
if protocol == 'udp':
logger.debug("Discovered new UDP member through gossip: " + str(message.getIp()) + ":" + str(message.getGossipPort()) + ":" + str(message.getDataPort()))
m = RemoteMember(use_tcp=False)
else:
logger.debug("Discovered new TCP member through gossip: " + str(message.ip) + ":" + str(message.gossip_port) + ":" + str(message.data_port))
m = RemoteMember(use_tcp=True)
values = key.split(":")
m.ip = values[0]
m.gossip_port = int(values[1])
m.data_port = int(values[2])
if message.public_key:
m.public_key = message.public_key
m.initialize()
self.member_holder.update_member_status(m)
member_clock = message.clock[i]
known_member_clock = m.sequence
if member_clock > known_member_clock:
if key == sender_key:
update_tags = True
m.sequence = member_clock
if key in message.listeners:
topics = message.listeners[key]
else:
topics = []
for key, topic_list in self.listener_registry.get_all_registrations().items():
if key == sender_key:
# member reporting on itself - take its listener list as canon
self.listener_registry.get_all_registrations()[key] = topics
for topic_string in topics:
topic = topic_utils.get_topic(topic_string)
partition = topic_utils.get_partition(topic_string)
if m not in self.listener_registry.get_registered_members(topic):
self.listener_registry.register_member_for_topic(topic, m)
if update_tags:
m = self.member_holder.get_member(sender_key)
m.tags = message.tags
__author__ = 'atrimble'
import bigio.parameters as parameters
import random
from threading import Timer
from bigio.util.configuration import *
from bigio.util import utils
from bigio.gossip_message import GossipMessage
import bigio.util.time_util as time_util
import logging
logger = logging.getLogger(__name__)
gossip_interval = parameters.get_property(GOSSIP_INTERVAL_PROPERTY,
DEFAULT_GOSSIP_INTERVAL)
cleanup_interval = parameters.get_property(CLEANUP_INTERVAL_PROPERTY,
DEFAULT_CLEANUP_INTERVAL)
should_shutdown = False
class Gossiper:
def __init__(self, me, member_holder, listener_registry):
self.me = me
self.member_holder = member_holder
self.listener_registry = listener_registry
if not should_shutdown:
t = GossipThread(self.send_membership)
t.start()
def shutdown(self):
__author__ = 'atrimble'
import bigio.parameters as parameters
import random
from threading import Timer
from bigio.util.configuration import *
from bigio.util import utils
from bigio.gossip_message import GossipMessage
import bigio.util.time_util as time_util
import logging
logger = logging.getLogger(__name__)
gossip_interval = parameters.get_property(GOSSIP_INTERVAL_PROPERTY, DEFAULT_GOSSIP_INTERVAL)
cleanup_interval = parameters.get_property(CLEANUP_INTERVAL_PROPERTY, DEFAULT_CLEANUP_INTERVAL)
should_shutdown = False
class Gossiper:
def __init__(self, me, member_holder, listener_registry):
self.me = me
self.member_holder = member_holder
self.listener_registry = listener_registry
if not should_shutdown:
t = GossipThread(self.send_membership)
t.start()
def shutdown(self):
def __init__(self):
super().__init__()
self.interceptors = dict()
self.data_reactor = Reactor()
self.gossip_reactor = Reactor()
protocol = parameters.get_property(PROTOCOL_PROPERTY, DEFAULT_PROTOCOL)
address = parameters.get_property(ADDRESS_PROPERTY, DEFAULT_ADDRESS)
gossip_port = parameters.get_property(GOSSIP_PORT_PROPERTY)
data_port = parameters.get_property(DATA_PORT_PROPERTY)
if not gossip_port:
gossip_port = network_util.get_free_port()
if not data_port:
data_port = network_util.get_free_port()
if protocol == 'tcp':
self.gossip_server = ThreadedTCPServer((address, gossip_port),
GossipHandler, self.gossip)
self.gossip_server.socket.setsockopt(socket.SOL_SOCKET,
socket.SO_KEEPALIVE, 1)
self.gossip_server.socket.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)
self.gossip_thread = threading.Thread(
target=self.gossip_server.serve_forever)
self.data_server = ThreadedTCPServer((address, data_port),
DataHandler, self.send)
self.data_server.socket.setsockopt(socket.SOL_SOCKET,
socket.SO_KEEPALIVE, 1)
self.data_server.socket.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)
self.data_thread = threading.Thread(
target=self.data_server.serve_forever)
logger.info('Starting TCP node on gossip port ' +
str(gossip_port) + ' : data port ' + str(data_port))
else:
self.gossip_server = ThreadedUDPServer((address, gossip_port),
GossipHandler, self.gossip)
self.gossip_thread = threading.Thread(
target=self.gossip_server.serve_forever)
self.data_server = ThreadedUDPServer((address, data_port),
DataHandler, self.send)
self.data_thread = threading.Thread(
target=self.data_server.serve_forever)
logger.info('Starting UDP node on gossip port ' +
str(gossip_port) + ' : data port ' + str(data_port))
self.gossip_thread.daemon = True
self.gossip_thread.start()
self.data_thread.daemon = True
self.data_thread.start()
self.ip = address
self.gossip_port = gossip_port
self.data_port = data_port
logger.info('Node started')
return