def __init__(self, zk, path, hostname, hostid):
try:
zk.get_children(path)
except kazoo.exceptions.NoNodeError:
raise RuntimeError("role lock node '%s' must exist" % path)
self.zk = zk
self.path = path
self.hostname = hostname
self.hostid = hostid
def main(args=None):
if args is None:
args = sys.argv[1:]
zkconnection_string, path, host = args
zk = zc.zk.ZK(zkconnection_string)
addrs = [addr for addr in zk.get_children(path)
if addr.startswith(host+':')]
if not addrs:
return error("Server not registered")
if len(addrs) > 1:
return error("Too many servers registered")
addr = addrs[0]
url = 'http://%s/ruok' % addr
try:
r = requests.get(url)
except requests.ConnectionError:
return error("Can't connect to %r" % addr)
if r.status_code != 200 or r.text != "imok":
return error("bad response from %r" % url)
print "cool"
def sync_with_canonical(url, dry_run=False, force=False, tree_directory=None):
zk = zc.zk.ZK(ZK_LOCATION)
zk_version = get_zk_version(zk)
if zk_version is None:
logger.critical("ALL STOP, cluster version is None")
if not force:
return
retries = 0
while True:
try:
if url.startswith('svn') or url.startswith('file://'):
vcs = SVN(url)
else:
vcs = GIT(url, tree_directory)
break
except RuntimeError:
retries += 1
if retries > MAX_VCS_RETRIES:
raise
time.sleep(3)
logger.info("VCS Version: " + str(vcs.version))
logger.info("ZK Version: " + str(zk_version))
if zk_version != vcs.version:
if not (force or zk_version is False):
for child in zk.get_children('/hosts'):
host_version = zk.properties('/hosts/' + child)['version']
if host_version != zk_version:
logger.error(
"Version mismatch detected, can't resync since " +
"host %s has not converged (%s -> %s)" %
(child, host_version, zk_version))
return
cluster_lock = zk.client.Lock('/hosts-lock', str(os.getpid()))
if cluster_lock.acquire(0):
try:
logger.info("Version mismatch detected, resyncing")
# Import changes
for fi, contents in vcs:
output = ' '.join(('Importing', fi))
if dry_run:
output += ' (dry run, no action taken)'
logger.info(output)
if not dry_run:
zk.import_tree(contents, trim=fi.endswith('.zk'))
# bump version number
if not dry_run:
zk.properties('/hosts').update(version=vcs.version)
finally:
cluster_lock.release()
else:
logger.error("Refused to update zookeeper tree, "
"couldn't obtain cluster lock")
zk.close()
def process(self, cb):
try:
zk = self.zk
except AttributeError:
try:
self.zk = zc.zk.ZK('zookeeper:2181')
except zookeeper.ZooKeeperException:
return self.error(cb, self.url, "Can't connect to ZooKeeper")
zk = self.zk
try:
hosts = zk.get_properties('/hosts')
except zookeeper.NoNodeException:
return self.error(cb, self.url, "No node: /hosts")
if 'version' not in hosts:
return self.error(cb, self.url, "No version: /hosts")
now = time.time()
version = hosts['version']
if version != getattr(self, 'version', self):
self.version = version
self.version_time = now
self.version_times = {}
times = self.version_times
late = []
for name in zk.get_children('/hosts'):
properties = zk.get_properties('/hosts/'+name)
if 'version' in properties:
name = properties.get('name', name)
if properties['version'] == version:
if name in times:
del times[name]
if not times:
self.version_time = now
continue
if name in times:
dur = int(now - times[name]) / 60
else:
times[name] = now
dur = 0
late.append((dur, name))
if late:
late.sort()
dur, name = late[-1]
mess = 'Unconverged (%s after %s minutes)' % (name, dur)
if dur > 30:
self.error(cb, self.url, mess)
elif dur > 15:
self.warn(cb, self.url, mess)
else:
self.ok(cb, self.url, mess)
else:
self.ok(cb, self.url, 'Converged (%s since %s)' % (
version, time.ctime(self.version_time)))
def find_server(zookeeper, path, monitor_address):
server = None
if monitor_address:
try:
fp, s = connect(monitor_address)
except socket.error as err:
return print("Can't connect %s" % err)
s.settimeout(1.0)
fp.write('servers %s\n' % path)
fp.flush()
data = fp.read().strip()
fp.close(); s.close()
if data.lower().startswith("invalid "):
return print(data + ' at %r' % monitor_address)
servers = list(set(data.split())) # dedup
if not servers:
return print("No servers at: %r" % monitor_address)
if len(servers) > 1:
return print("Too many servers, %r, at: %r" %
(sorted(servers), monitor_address))
server = servers[0]
zk = zc.zk.ZK(zookeeper)
children = zk.get_children(path)
zk.close()
if server:
host, port = server.split(':')
if host:
children = [c for c in children if c == server]
else:
children = [c for c in children if c.split(':')[1] == port]
if len(children) != 1:
return print("Couldn't find server in ZooKeeper")
addr = children[0]
return addr
def my_func(event):
# check to see what the children are now
path = event.path
addresses = zk.get_children(event.path)
create_cgf(path,addresses)
temp = "/%s-%s-%s-%s" % (datacenter,server_type,location,environment)
zookeeper_path_list.append(temp)
return service_hash, zookeeper_path_list
def get_ip_encode(children):
ip_encode = ''.join(list(children))
ip_encode = base64.b64encode(ip_encode)
return ip_encode
while True:
service_hash, zookeeper_path_list = get_service_hash()
for path in zookeeper_path_list:
try:
exists = zk.exists(path)
except KazooException:
exists = None
zk = get_zk_conn()
if exists:
children = zk.get_children(path, watch=my_func)
print path,children
ip_encode = get_ip_encode(children)
server_type = path.split('-')[1]
if os.path.isfile('/etc/haproxy/conf.d/%s-%s.cfg' % (server_type,ip_encode))==False:
create_cgf(path,list(children))
sys.stdout.flush()
sys.stderr.flush()
time.sleep(.5)
def my_func(event):
# check to see what the children are now
path = event.path
addresses = zk.get_children(event.path)
create_cgf(path, addresses)
return service_hash, zookeeper_path_list
def get_ip_encode(children):
ip_encode = ''.join(list(children))
ip_encode = base64.b64encode(ip_encode)
return ip_encode
while True:
service_hash, zookeeper_path_list = get_service_hash()
for path in zookeeper_path_list:
try:
exists = zk.exists(path)
except KazooException:
exists = None
zk = get_zk_conn()
if exists:
children = zk.get_children(path, watch=my_func)
print path, children
ip_encode = get_ip_encode(children)
server_type = path.split('-')[1]
if os.path.isfile('/etc/haproxy/conf.d/%s-%s.cfg' %
(server_type, ip_encode)) == False:
create_cgf(path, list(children))
sys.stdout.flush()
sys.stderr.flush()
time.sleep(.5)
def clients(path):
logging.basicConfig()
random.seed(0)
import zc.zk
zk = zc.zk.ZooKeeper()
properties = zk.properties(path)
settings = zc.mappingobject.mappingobject(properties)
siteids = []
@properties
def _(*a):
n = settings.sites
siteids[:] = [0]
for i in range(4):
if n:
siteids.extend(range(n))
n /= 2
wpath = path + '/lb/providers'
zope.testing.wait.wait(lambda : zk.get_children(wpath))
[waddr] = zk.get_children(wpath)
waddr = zc.parse_addr.parse_addr(waddr)
stats = zc.mappingobject.mappingobject(dict(
truncated = 0,
requests = 0,
bypid = {},
nobs = 0,
nhits = 0,
))
spath = path + '/stats'
if not zk.exists(spath):
zk.create(spath, '', zc.zk.OPEN_ACL_UNSAFE)
import gevent.socket
def do_request():
siteid = random.choice(siteids)
oids = set(
int(random.gauss(0, settings.objects_per_site/4))
for i in range(settings.objects_per_request)
)
socket = gevent.socket.create_connection(waddr)
try:
socket.sendall(
request_template % dict(
data='_'.join(map(str, oids)),
host='h%s' % siteid,
)
)
response = ''
while '\r\n\r\n' not in response:
data = socket.recv(9999)
if not data:
stats.truncated += 1
return
response += data
headers, body = response.split('\r\n\r\n')
headers = headers.split('\r\n')
status = headers.pop(0)
headers = dict(l.strip().lower().split(':', 1)
for l in headers if ':' in l)
content_length = int(headers['content-length'])
while len(body) < content_length:
data = socket.recv(9999)
if not data:
stats.truncated += 1
return
body += data
pid, n, nhit, nmiss, nevict = map(int, body.strip().split())
stats.requests += 1
stats.nobs += n
stats.nhits += nhit
bypid = stats.bypid.get(pid)
if bypid is None:
bypid = stats.bypid[pid] = dict(nr=0, n=0, nhit=0)
bypid['nr'] += 1
bypid['n'] += n
bypid['nhit'] += nhit
logger.info(' '.join(map(str, (
100*stats.nhits/stats.nobs,
pid, n, nhit, 100*nhit/n,
))))
finally:
socket.close()
def client():
try:
while 1:
do_request()
except:
print 'client error'
logging.getLogger(__name__+'-client').exception('client')
greenlets = [gevent.spawn(client) for i in range(settings.clients)]
# Set up notification of address changes.
watcher = gevent.get_hub().loop.async()
@watcher.start
def _():
print 'got update event'
while settings.clients > len(greenlets):
greenlets.append(gevent.spawn(client))
while settings.clients < len(greenlets):
greenlets.pop().kill()
properties(lambda a: watcher.send())
while 1:
gevent.sleep(60.0)
