def __init__(self, zkobj, populate=None, update=None):
super(Cache, self).__init__()
self.zkobj = zkobj
self._index = []
self._cache = {}
# Save the hooks for this cache.
# We allow users to specify a populate hook, which
# will be used to populate the cache when we see a
# new entry, and an update hook, which will be called
# whenever the collection of associated resources
# changes.
if populate is None:
self._populate = self._default_populate
else:
self._populate = utils.callback(populate)
if update is None:
self._update_hook = self._default_update_hook
else:
self._update_hook = utils.callback(update)
# NOTE: We provide some convenience methods
# here if this happens to be a collection.
if hasattr(zkobj, 'add'):
self.add = zkobj.add
if hasattr(zkobj, 'remove'):
self.remove = zkobj.remove
if hasattr(zkobj, 'as_map'):
self.as_map = zkobj.as_map
self._get_child = zkobj._get_child
# Start the watch.
self._update(zkobj._list_children(watch=self.update))
def __init__(self, **kwargs):
super(Connection, self).__init__(**kwargs)
template_file = os.path.join(os.path.dirname(__file__), 'haproxy.template')
self.template = Template(filename=template_file)
self.frontends = {}
self.http_backends = {}
self.tcp_backends = {}
self.error_notify = utils.callback(kwargs.get("error_notify"))
def __init__(self, **kwargs):
super(Connection, self).__init__(**kwargs)
template_file = os.path.join(os.path.dirname(__file__),
'haproxy.template')
self.template = Template(filename=template_file)
self.frontends = {}
self.http_backends = {}
self.tcp_backends = {}
self.error_notify = utils.callback(kwargs.get("error_notify"))
def __init__(self, locks, error_notify, discard_notify, producer):
super(ConnectionConsumer, self).__init__()
self.locks = locks
self.error_notify = utils.callback(error_notify)
self.discard_notify = utils.callback(discard_notify)
self.producer = producer
self.portmap = {}
self.postponed = []
self.standby = {}
self.children = {}
# Subscribe to events generated by the producer.
# NOTE: These are cleaned up in stop().
self.producer.subscribe(self.notify)
# Start the thread.
super(ConnectionConsumer, self).start()
def __init__(self, locks, error_notify, discard_notify, producer):
super(ConnectionConsumer, self).__init__()
self.locks = locks
self.error_notify = utils.callback(error_notify)
self.discard_notify = utils.callback(discard_notify)
self.producer = producer
self.portmap = {}
self.postponed = []
self.standby = {}
self.children = {}
# Subscribe to events generated by the producer.
# NOTE: These are cleaned up in stop().
self.producer.subscribe(self.notify)
# Start the thread.
super(ConnectionConsumer, self).start()
def _list_children(self, watch=None):
client = self._zk_client.connect()
if watch:
with self._lock:
watch = utils.callback(watch)
def _fn(children):
return watch(children or [])
_fn.__name__ = watch.__name__
if self._watch_children:
client.clear_watch_fn(self._watch_children)
self._watch_children = _fn
return client.watch_children(self._path, self._watch_children) or []
else:
return client.list_children(self._path) or []
def _get_data(self, watch=None):
client = self._zk_client.connect()
if watch:
with self._lock:
watch = utils.callback(watch)
def _fn(value):
return watch(self._deserialize(value))
_fn.__name__ = watch.__name__
if self._watch_content:
client.clear_watch_fn(self._watch_content)
self._watch_content = _fn
return self._deserialize(
client.watch_contents(self._path, self._watch_content))
else:
return self._deserialize(client.read(self._path))
def _get_data(self, watch=None):
client = self._zk_client.connect()
if watch:
with self._lock:
watch = utils.callback(watch)
def _fn(value):
return watch(self._deserialize(value))
_fn.__name__ = watch.__name__
if self._watch_content:
client.clear_watch_fn(self._watch_content)
self._watch_content = _fn
return self._deserialize(
client.watch_contents(self._path, self._watch_content))
else:
return self._deserialize(client.read(self._path))
def _list_children(self, watch=None):
client = self._zk_client.connect()
if watch:
with self._lock:
watch = utils.callback(watch)
def _fn(children):
return watch(children or [])
_fn.__name__ = watch.__name__
if self._watch_children:
client.clear_watch_fn(self._watch_children)
self._watch_children = _fn
return client.watch_children(self._path,
self._watch_children) or []
else:
return client.list_children(self._path) or []
def handle(self, connection):
# Index by the destination port.
port = connection.dst[1]
if not(self.portmap.has_key(port)):
connection.drop()
return True
# Grab the information for this port.
(_, exclusive, disposable, reconnect, backends, client_subnets) = self.portmap[port]
# Check the subnet.
if client_subnets:
subnet_okay = False
for subnet in client_subnets:
if netaddr.ip.IPAddress(str(connection.src[0])) in \
netaddr.ip.IPNetwork(str(subnet)):
subnet_okay = True
break
if not subnet_okay:
connection.drop()
return True
# Find a backend IP (exclusive or not).
ip = None
port = None
if exclusive:
# See if we have a VM to reconnect to.
if reconnect > 0:
existing = self.locks.find(connection.src[0])
if len(existing) > 0:
(ip, port) = existing[0].split(":", 1)
port = int(port)
if (ip, port) in self.standby:
# NOTE: We will have a lock representing
# this connection, but is it already held.
del self.standby[(ip, port)]
if not ip:
# Grab the grab the named lock (w/ ip and port).
candidates = ["%s:%d" % backend for backend in backends]
got = self.locks.lock(candidates, value=connection.src[0])
if got:
# We managed to grab the above lock.
(ip, port) = got.split(":", 1)
port = int(port)
else:
# Select an IP at random.
(ip, port) = backends[random.randint(0, len(backends)-1)]
if ip and port:
# Either redirect or drop the connection.
child = connection.redirect(ip, port)
standby_time = (exclusive and reconnect)
if child is not None:
# Start a notification thread. This will
# wake up the main consumer thread (so that
# the child can be reaped) when the child exits,
# but it will also mark the IP as failed if
# an error ultimately lead to the exit.
def closure(child_pid, error_fn, notify_fn):
def fn():
while True:
try:
(pid, status) = os.waitpid(child_pid, 0)
if pid == child_pid:
break
except (TypeError, OSError):
(pid, status) = (child_pid, 0)
break
if os.WEXITSTATUS(status) > 0:
# Notify the high-level manager about an error
# found on this IP. This may ultimately result in
# the instance being terminated, etc.
error_fn()
# Wait up the consumer that it may reap this
# process and accept any new incoming connections.
notify_fn()
return fn
# Start threads that will wait for the child process.
# NOTE: Unlike most other threads in the system, these
# threads are *not* daemon threads. We want them to keep
# everything alive so that our Zookeeper locks are properly
# maintained while the socat processes are alive.
def error_fn():
self.error_notify("%s:%d" % (ip, port))
t = threading.Thread(target=closure(
child, error_fn, utils.callback(self.notify)))
t.start()
self.children[child] = (
ip,
port,
connection,
standby_time,
disposable)
return True
return False
def handle(self, connection):
# Index by the destination port.
port = connection.dst[1]
if not (self.portmap.has_key(port)):
connection.drop()
return True
# Grab the information for this port.
(_, exclusive, disposable, reconnect, backends,
client_subnets) = self.portmap[port]
# Check the subnet.
if client_subnets:
subnet_okay = False
for subnet in client_subnets:
if netaddr.ip.IPAddress(str(connection.src[0])) in \
netaddr.ip.IPNetwork(str(subnet)):
subnet_okay = True
break
if not subnet_okay:
connection.drop()
return True
# Find a backend IP (exclusive or not).
ip = None
port = None
if exclusive:
# See if we have a VM to reconnect to.
if reconnect > 0:
existing = self.locks.find(connection.src[0])
if len(existing) > 0:
(ip, port) = existing[0].split(":", 1)
port = int(port)
if (ip, port) in self.standby:
# NOTE: We will have a lock representing
# this connection, but is it already held.
del self.standby[(ip, port)]
if not ip:
# Grab the grab the named lock (w/ ip and port).
candidates = ["%s:%d" % backend for backend in backends]
got = self.locks.lock(candidates, value=connection.src[0])
if got:
# We managed to grab the above lock.
(ip, port) = got.split(":", 1)
port = int(port)
else:
# Select an IP at random.
(ip, port) = backends[random.randint(0, len(backends) - 1)]
if ip and port:
# Either redirect or drop the connection.
child = connection.redirect(ip, port)
standby_time = (exclusive and reconnect)
# Note: disposable is either None or a mimumum session time.
# So set the time to dispose as now + minimum session time,
# or False.
dispose_time = (disposable is not None
and time.time() + disposable)
if child is not None:
# Start a notification thread. This will
# wake up the main consumer thread (so that
# the child can be reaped) when the child exits,
# but it will also mark the IP as failed if
# an error ultimately lead to the exit.
def closure(child_pid, error_fn, notify_fn):
def fn():
while True:
try:
(pid, status) = os.waitpid(child_pid, 0)
if pid == child_pid:
break
except (TypeError, OSError):
(pid, status) = (child_pid, 0)
break
if os.WEXITSTATUS(status) > 0:
# Notify the high-level manager about an error
# found on this IP. This may ultimately result in
# the instance being terminated, etc.
error_fn()
# Wait up the consumer that it may reap this
# process and accept any new incoming connections.
notify_fn()
return fn
# Start threads that will wait for the child process.
# NOTE: Unlike most other threads in the system, these
# threads are *not* daemon threads. We want them to keep
# everything alive so that our Zookeeper locks are properly
# maintained while the socat processes are alive.
def error_fn():
self.error_notify("%s:%d" % (ip, port))
t = threading.Thread(target=closure(
child, error_fn, utils.callback(self.notify)))
t.start()
self.children[child] = (ip, port, connection, standby_time,
dispose_time)
return True
return False