class Registry(object):
def __init__(self, config, conn, dry_run=False):
self.config = config
self.zk_prefix = self.config["zk_prefix"]
self.zk_iptoid_path = "%s/%s" % (self.zk_prefix, self.config["zk_iptoid_node"])
self.zk_idtoip_path = "%s/%s" % (self.zk_prefix, self.config["zk_idtoip_node"])
self.zk_ip_map_path = "%s/%s" % (self.zk_prefix, self.config["zk_ip_map_node"])
self.zk_max_change_threshold_path = "%s/%s" % (self.zk_prefix, self.config["zk_max_change_threshold_path"])
self.conn = conn
self.dry_run = dry_run
@property
def _zk_id_path(self):
""" :returns: ID to IP mapping path for this instance. """
return "%s/%s" % (self.zk_idtoip_path, self.conn.instance_id)
def _zk_ip_path(self, ip):
""" :returns: IP to ID mapping path for this IP. """
return "%s/%s" % (self.zk_iptoid_path, ip)
def _tag_instance(self):
""" Tag "overlay_ip" and "neti_version" on instance with registered overlay_ip """
self.conn.tag_instance({"overlay_ip": self.overlay_ip, "neti_version": version})
def _choose_overlay_ip(self):
"""
Get list of used IPs from ZK and randomly choose one from the rest of the subnet.
:returns: String containing chosen IP.
"""
try:
used_ips = {IPv4Address(unicode(ip)) for ip in self.conn.zk.get_children(self.zk_iptoid_path)}
except NoNodeError:
used_ips = set()
available_ips = set(self.conn.network.hosts()) - used_ips
if not available_ips:
logger.error("No available IPs found!")
raise NoAvailableIPsError
return str(random.sample(available_ips, 1)[0])
def _find_available_overlay_ip(self):
"""
Get a new IP, and attempt to register it. If it already is taken, get another. Retry this cycle for
MAX_RETRIES. After that, fail on no IPs.
:returns: String containing overlay IP.
"""
retries = 0
ip = self._choose_overlay_ip()
while retries < constants.MAX_IP_TRIES:
retries += 1
try:
self.conn.zk.create(self._zk_id_path, ip)
logger.warn("Creating %s" % ip)
except NoNodeError:
self.conn.zk.ensure_path(self.zk_idtoip_path)
logger.error("Path %s did not exist...creating and trying again" % self.zk_idtoip_path)
continue
except NodeExistsError:
try:
zk_ip, _ = self.conn.zk.get(self._zk_id_path)
logger.error("IP %s already assigned to %s...using that" % (zk_ip, self.conn.instance_id))
except NoNodeError:
logger.error("No IP found...trying again")
continue
return zk_ip
try:
zk_ip, _ = self.conn.zk.get(self._zk_id_path)
except NoNodeError:
logger.error("IP %s did not get associated...trying again" % ip)
continue
if zk_ip == ip:
logger.info("IP %s set" % ip)
return ip
else:
logger.error("IP %s already assigned to %s...using that" % (zk_ip, self.conn.instance_id))
return zk_ip
logger.error("No available IPs found!")
raise NoAvailableIPsError
def _set_ip_to_id_map(self, ip):
""" Sets the reverse map for IP-based lookups. """
try:
self.conn.zk.set(self._zk_ip_path(ip), self.conn.instance_id)
except NoNodeError:
logger.info("No IP to ID map node for %s" % ip)
try:
self.conn.zk.create(self._zk_ip_path(ip), self.conn.instance_id)
except NoNodeError:
self.conn.zk.ensure_path(self._zk_ip_path(ip))
self._set_ip_to_id_map(ip)
def register(self):
""" Attempt to get registered overlay IP by instance ID. If that succeeds, verify that the IP to ID map
is set correctly, set the instance variable, and tag the instance. If it fails, attempt to get a new one.
:returns: String containing overlay IP. """
try:
self.overlay_ip, _ = self.conn.zk.get(self._zk_id_path)
except NoNodeError:
self.overlay_ip = self._find_available_overlay_ip()
self.conn.zk.handler.spawn(self._tag_instance)
self._set_ip_to_id_map(self.overlay_ip)
return self.overlay_ip
def _get_ip_map(self):
""" :returns: Mapping of all IPs for znode """
return "%s|%s|%s|%d" % (self.conn.public_ip, self.conn.private_ip, self.overlay_ip, self.conn._is_vpc)
def _ips_from_entries(self, entries):
""" Builds array of InstanceIPBundles from found ZK nodes.
:returns: List of InstanceIPBundles. """
ips = []
for entry in entries:
ips.append(InstanceIPBundle(entry))
return ips
def _join_party(self):
self.party = ShallowParty(self.conn.zk, self.zk_ip_map_path, identifier=self._get_ip_map())
self.party.join()
def _load_param(self, path, default):
try:
param, _ = self.conn.zk.retry(self.conn.zk.get, path)
return int(param) if param else default
except NoNodeError:
self.conn.zk.ensure_path(path)
self.conn.zk.retry(self.conn.zk.set, (path, str(default)))
return default
def _state_listener(self, state):
logger.error("Connection state change: %s" % state)
if state == KazooState.SUSPENDED:
logger.warn("Suspended connection, setting connected to False.")
self.conn.connected = False
elif state == KazooState.LOST:
logger.warn("Lost connection, setting lost to True.")
self.conn.lost = True
self.conn.zk.handler.spawn(self._rejoin_party)
elif state == KazooState.CONNECTED:
logger.warn("Regained connection, setting connected to True.")
self.conn.connected = True
self.conn.zk.handler.spawn(self._rejoin_party)
def _rejoin_party(self):
logger.warn("Thinking about going back to the party (Lost: %r; Connected %r)..." % (self.conn.lost, self.conn.connected))
if self.conn.lost and self.conn.connected:
logger.warn("Got 86'd. Sneaking back into the party...")
self._join_party()
self.conn.lost = False
def run(self):
""" Connects to both ZKs, inserts an ephemeral node, and starts a watch for changes. """
try:
self._join_party()
self.conn.zk.add_listener(self._state_listener)
@ThrottledChildrenWatch(self.conn.zk, self.zk_ip_map_path, delay_min=constants.TIME_DELAY_MIN, delay_max=constants.TIME_DELAY_MAX)
def update_iptables(hosts):
self.max_change_threshold = self._load_param(self.zk_max_change_threshold_path, constants.DEFAULT_MAX_CHANGE_THRESHOLD)
if getattr(self, "hosts", None):
hosts_to_remove = len(self.hosts) - len(hosts)
if hosts_to_remove > self.max_change_threshold:
logger.warn("Trying to remove %d hosts...untriggering (max is %s)" % (hosts_to_remove, self.max_change_threshold))
return
else:
remove = set(self.hosts) - set(hosts)
add = set(hosts) - set(self.hosts)
if add:
logger.debug("Adding: %s" % add)
if remove:
logger.debug("Removing: %s" % remove)
self.hosts = hosts
bundles = self._ips_from_entries(self.hosts)
builder = IPtables(config=self.config, is_vpc=self.conn._is_vpc, dry_run=self.dry_run)
builder.build(bundles)
while True:
self.conn.zk.handler.sleep_func(constants.DEFAULT_SLEEP)
except ZookeeperError as e:
logger.error("ZookeeperError: %s" % e)
self.run()
class Registry(object):
def __init__(self, config, conn, dry_run=False):
self.config = config
self.zk_prefix = self.config["zk_prefix"]
self.zk_iptoid_path = "%s/%s" % (self.zk_prefix,
self.config["zk_iptoid_node"])
self.zk_idtoip_path = "%s/%s" % (self.zk_prefix,
self.config["zk_idtoip_node"])
self.zk_ip_map_path = "%s/%s" % (self.zk_prefix,
self.config["zk_ip_map_node"])
self.zk_max_change_threshold_path = "%s/%s" % (
self.zk_prefix, self.config["zk_max_change_threshold_path"])
self.conn = conn
self.dry_run = dry_run
@property
def _zk_id_path(self):
""" :returns: ID to IP mapping path for this instance. """
return "%s/%s" % (self.zk_idtoip_path, self.conn.instance_id)
def _zk_ip_path(self, ip):
""" :returns: IP to ID mapping path for this IP. """
return "%s/%s" % (self.zk_iptoid_path, ip)
def _tag_instance(self):
""" Tag "overlay_ip" and "neti_version" on instance with registered overlay_ip """
self.conn.tag_instance({
"overlay_ip": self.overlay_ip,
"neti_version": version
})
def _choose_overlay_ip(self):
"""
Get list of used IPs from ZK and randomly choose one from the rest of the subnet.
:returns: String containing chosen IP.
"""
try:
used_ips = {
IPv4Address(unicode(ip))
for ip in self.conn.zk.get_children(self.zk_iptoid_path)
}
except NoNodeError:
used_ips = set()
available_ips = set(self.conn.network.hosts()) - used_ips
if not available_ips:
logger.error("No available IPs found!")
raise NoAvailableIPsError
return str(random.sample(available_ips, 1)[0])
def _find_available_overlay_ip(self):
"""
Get a new IP, and attempt to register it. If it already is taken, get another. Retry this cycle for
MAX_RETRIES. After that, fail on no IPs.
:returns: String containing overlay IP.
"""
retries = 0
ip = self._choose_overlay_ip()
while retries < constants.MAX_IP_TRIES:
retries += 1
try:
self.conn.zk.create(self._zk_id_path, ip)
logger.warn("Creating %s" % ip)
except NoNodeError:
self.conn.zk.ensure_path(self.zk_idtoip_path)
logger.error(
"Path %s did not exist...creating and trying again" %
self.zk_idtoip_path)
continue
except NodeExistsError:
try:
zk_ip, _ = self.conn.zk.get(self._zk_id_path)
logger.error("IP %s already assigned to %s...using that" %
(zk_ip, self.conn.instance_id))
except NoNodeError:
logger.error("No IP found...trying again")
continue
return zk_ip
try:
zk_ip, _ = self.conn.zk.get(self._zk_id_path)
except NoNodeError:
logger.error("IP %s did not get associated...trying again" %
ip)
continue
if zk_ip == ip:
logger.info("IP %s set" % ip)
return ip
else:
logger.error("IP %s already assigned to %s...using that" %
(zk_ip, self.conn.instance_id))
return zk_ip
logger.error("No available IPs found!")
raise NoAvailableIPsError
def _set_ip_to_id_map(self, ip):
""" Sets the reverse map for IP-based lookups. """
try:
self.conn.zk.set(self._zk_ip_path(ip), self.conn.instance_id)
except NoNodeError:
logger.info("No IP to ID map node for %s" % ip)
try:
self.conn.zk.create(self._zk_ip_path(ip),
self.conn.instance_id)
except NoNodeError:
self.conn.zk.ensure_path(self._zk_ip_path(ip))
self._set_ip_to_id_map(ip)
def register(self):
""" Attempt to get registered overlay IP by instance ID. If that succeeds, verify that the IP to ID map
is set correctly, set the instance variable, and tag the instance. If it fails, attempt to get a new one.
:returns: String containing overlay IP. """
try:
self.overlay_ip, _ = self.conn.zk.get(self._zk_id_path)
except NoNodeError:
self.overlay_ip = self._find_available_overlay_ip()
self.conn.zk.handler.spawn(self._tag_instance)
self._set_ip_to_id_map(self.overlay_ip)
return self.overlay_ip
def _get_ip_map(self):
""" :returns: Mapping of all IPs for znode """
return "%s|%s|%s|%d" % (self.conn.public_ip, self.conn.private_ip,
self.overlay_ip, self.conn._is_vpc)
def _ips_from_entries(self, entries):
""" Builds array of InstanceIPBundles from found ZK nodes.
:returns: List of InstanceIPBundles. """
ips = []
for entry in entries:
ips.append(InstanceIPBundle(entry))
return ips
def _join_party(self):
self.party = ShallowParty(self.conn.zk,
self.zk_ip_map_path,
identifier=self._get_ip_map())
self.party.join()
def _load_param(self, path, default):
try:
param, _ = self.conn.zk.retry(self.conn.zk.get, path)
return int(param) if param else default
except NoNodeError:
self.conn.zk.ensure_path(path)
self.conn.zk.retry(self.conn.zk.set, (path, str(default)))
return default
def _state_listener(self, state):
logger.error("Connection state change: %s" % state)
if state == KazooState.SUSPENDED:
logger.warn("Suspended connection, setting connected to False.")
self.conn.connected = False
elif state == KazooState.LOST:
logger.warn("Lost connection, setting lost to True.")
self.conn.lost = True
self.conn.zk.handler.spawn(self._rejoin_party)
elif state == KazooState.CONNECTED:
logger.warn("Regained connection, setting connected to True.")
self.conn.connected = True
self.conn.zk.handler.spawn(self._rejoin_party)
def _rejoin_party(self):
logger.warn(
"Thinking about going back to the party (Lost: %r; Connected %r)..."
% (self.conn.lost, self.conn.connected))
if self.conn.lost and self.conn.connected:
logger.warn("Got 86'd. Sneaking back into the party...")
self._join_party()
self.conn.lost = False
def run(self):
""" Connects to both ZKs, inserts an ephemeral node, and starts a watch for changes. """
try:
self._join_party()
self.conn.zk.add_listener(self._state_listener)
@ThrottledChildrenWatch(self.conn.zk,
self.zk_ip_map_path,
delay_min=constants.TIME_DELAY_MIN,
delay_max=constants.TIME_DELAY_MAX)
def update_iptables(hosts):
self.max_change_threshold = self._load_param(
self.zk_max_change_threshold_path,
constants.DEFAULT_MAX_CHANGE_THRESHOLD)
if getattr(self, "hosts", None):
hosts_to_remove = len(self.hosts) - len(hosts)
if hosts_to_remove > self.max_change_threshold:
logger.warn(
"Trying to remove %d hosts...untriggering (max is %s)"
% (hosts_to_remove, self.max_change_threshold))
return
else:
remove = set(self.hosts) - set(hosts)
add = set(hosts) - set(self.hosts)
if add:
logger.debug("Adding: %s" % add)
if remove:
logger.debug("Removing: %s" % remove)
self.hosts = hosts
bundles = self._ips_from_entries(self.hosts)
builder = IPtables(config=self.config,
is_vpc=self.conn._is_vpc,
dry_run=self.dry_run)
builder.build(bundles)
while True:
self.conn.zk.handler.sleep_func(constants.DEFAULT_SLEEP)
except ZookeeperError as e:
logger.error("ZookeeperError: %s" % e)
self.run()
def _join_party(self):
self.party = ShallowParty(self.conn.zk, self.zk_ip_map_path, identifier=self._get_ip_map())
self.party.join()
def _join_party(self):
self.party = ShallowParty(self.conn.zk,
self.zk_ip_map_path,
identifier=self._get_ip_map())
self.party.join()
