def export(args=None):
"""Usage: %prog [options] connection [path]
"""
if args is None:
args = sys.argv[1:]
parser = optparse.OptionParser(export.__doc__)
parser.add_option('-e', '--ephemeral', action='store_true')
parser.add_option('-o', '--output')
options, args = parser.parse_args(args)
connection = args.pop(0)
if args:
[path] = args
else:
path = '/'
logging.basicConfig(level=logging.WARNING)
zk = zc.zk.ZooKeeper(connection)
data = zk.export_tree(path, ephemeral=options.ephemeral)
if options.output:
with open(options.output, 'w') as f:
f.write(data)
else:
print data,
zk.close()
def worker(app, global_conf, zookeeper, path, loggers=None, address=':0',
threads=None, backdoor=False, description=None, version=None,
run=True, **kw):
"""Paste deploy server runner
"""
if loggers:
if re.match(r'\d+$', loggers):
logging.basicConfig(level=int(loggers))
elif loggers in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'):
logging.basicConfig(level=getattr(logging, loggers))
else:
import ZConfig
ZConfig.configureLoggers(loggers)
zk = zc.zk.ZooKeeper(zookeeper)
address = zc.parse_addr.parse_addr(address)
from zc.resumelb.worker import Worker
worker = Worker(app, address, threads=threads and int(threads),
**kw)
# Set up notification of settings changes.
settings = zk.properties(path)
watcher = gevent.get_hub().loop.async()
watcher.start(lambda : worker.update_settings(settings))
settings(lambda _: watcher.send())
registration_data = {}
if backdoor == 'true':
from gevent import backdoor
bd = backdoor.BackdoorServer(('127.0.0.1', 0), locals())
bd.start()
registration_data['backdoor'] = '127.0.0.1:%s' % bd.server_port
worker.__bd = bd
if description:
registration_data['description'] = description
if version:
registration_data['version'] = version
zk.register_server(path+'/providers', worker.addr, **registration_data)
worker.zk = zk
worker.__zksettings = settings
def shutdown():
zk.close()
worker.shutdown()
gevent.signal(signal.SIGTERM, shutdown)
if run:
try:
worker.server.serve_forever()
finally:
logging.getLogger(__name__+'.worker').info('exiting')
zk.close()
else:
gevent.sleep(.01)
return worker
def sync_with_canonical(url, dry_run=False, force=False, tree_directory=None):
zk = zc.zk.ZK(ZK_LOCATION)
zk_version = get_zk_version(zk)
if zk_version is None:
logger.critical("ALL STOP, cluster version is None")
if not force:
return
retries = 0
while True:
try:
if url.startswith('svn') or url.startswith('file://'):
vcs = SVN(url)
else:
vcs = GIT(url, tree_directory)
break
except RuntimeError:
retries += 1
if retries > MAX_VCS_RETRIES:
raise
time.sleep(3)
logger.info("VCS Version: " + str(vcs.version))
logger.info("ZK Version: " + str(zk_version))
if zk_version != vcs.version:
if not (force or zk_version is False):
for child in zk.get_children('/hosts'):
host_version = zk.properties('/hosts/' + child)['version']
if host_version != zk_version:
logger.error(
"Version mismatch detected, can't resync since " +
"host %s has not converged (%s -> %s)" %
(child, host_version, zk_version))
return
cluster_lock = zk.client.Lock('/hosts-lock', str(os.getpid()))
if cluster_lock.acquire(0):
try:
logger.info("Version mismatch detected, resyncing")
# Import changes
for fi, contents in vcs:
output = ' '.join(('Importing', fi))
if dry_run:
output += ' (dry run, no action taken)'
logger.info(output)
if not dry_run:
zk.import_tree(contents, trim=fi.endswith('.zk'))
# bump version number
if not dry_run:
zk.properties('/hosts').update(version=vcs.version)
finally:
cluster_lock.release()
else:
logger.error("Refused to update zookeeper tree, "
"couldn't obtain cluster lock")
zk.close()
def import_(args=None):
"""Usage: %prog [options] connection [import-file [path]]
Import a tree definition from a file.
If no import-file is provided or if the import file is -, then
data are read from standard input.
"""
if args is None:
args = sys.argv[1:]
parser = optparse.OptionParser(import_.__doc__)
parser.add_option('-d', '--dry-run', action='store_true')
parser.add_option('-t', '--trim', action='store_true')
parser.add_option(
'-p', '--permission', type='int',
default=kazoo.security.Permissions.ALL,
help='ZooKeeper permission bits as integer,'
' kazoo.security.Permissions.ALL',
)
options, args = parser.parse_args(args)
if not (1 <= len(args) <= 3):
parser.parse_args(['-h'])
connection = args.pop(0)
if args:
import_file = args.pop(0)
else:
import_file = '-'
if args:
[path] = args
else:
path = '/'
logging.basicConfig(level=logging.WARNING)
zk = zc.zk.ZooKeeper(connection)
if import_file == '-':
import_file = sys.stdin
else:
import_file = open(import_file)
zk.import_tree(
import_file.read(), path,
trim=options.trim,
dry_run=options.dry_run,
acl=[world_acl(options.permission)],
)
zk.close()
def tearDown(test):
"""The matching tearDown for setUp.
The single argument is the test case passed to setUp.
"""
setupstack.tearDown(test)
real_zk = testing_with_real_zookeeper()
if real_zk:
zk = zc.zk.ZooKeeper(real_zk)
root = setupstack.globs(test)['/zc.zk.testing.test-root']
if zk.exists(root):
zk.delete_recursive(root)
zk.close()
def set_property(args=None):
if args is None:
args = sys.argv[1:]
connection = args.pop(0)
path = args.pop(0)
zk = zc.zk.ZooKeeper(connection)
def _property(arg):
name, expr = arg.split('=', 1)
return name, eval(expr, {})
zk.properties(path).update(dict(map(_property, args)))
zk.close()
def main(args=None):
if args is None:
args = sys.argv[1:]
args = parser.parse_args(args)
config = zc.zkdeployment.agent.Configuration(args.configuration)
zk = zc.zk.ZK(args.zookeeper)
try:
host_properties = dict(zk.properties('/hosts/' + config.host_id))
except kazoo.exceptions.NoNodeError:
return error('Host not registered')
zkversion = zk.properties('/hosts', False).get('version')
zk.close()
if zkversion is None:
return warn('Cluster version is None')
try:
with open(os.path.join(config.run_directory, 'status')) as f:
t, _, version, status = f.read().strip().split(None, 3)
except IOError, err:
return error(str(err))
def setup_tree(tree, connection_string, root='/test-root',
zookeeper_node=False):
zk = zc.zk.ZooKeeper(connection_string)
if zk.exists(root):
zk.delete_recursive(root)
zk.create(root, '', zc.zk.OPEN_ACL_UNSAFE)
zk.import_tree(tree or """
/fooservice
/providers
database = '/databases/foomain'
threads = 1
favorite_color = 'red'
""", root)
if zookeeper_node:
zk.import_tree("""
/zookeeper
/quota
""", root)
zk.close()
def find_server(zookeeper, path, monitor_address):
server = None
if monitor_address:
try:
fp, s = connect(monitor_address)
except socket.error as err:
return print("Can't connect %s" % err)
s.settimeout(1.0)
fp.write('servers %s\n' % path)
fp.flush()
data = fp.read().strip()
fp.close(); s.close()
if data.lower().startswith("invalid "):
return print(data + ' at %r' % monitor_address)
servers = list(set(data.split())) # dedup
if not servers:
return print("No servers at: %r" % monitor_address)
if len(servers) > 1:
return print("Too many servers, %r, at: %r" %
(sorted(servers), monitor_address))
server = servers[0]
zk = zc.zk.ZK(zookeeper)
children = zk.get_children(path)
zk.close()
if server:
host, port = server.split(':')
if host:
children = [c for c in children if c == server]
else:
children = [c for c in children if c.split(':')[1] == port]
if len(children) != 1:
return print("Couldn't find server in ZooKeeper")
addr = children[0]
return addr
import zc.zk
zk = None
try:
host = '10.251.22.99:2181'
zk = zc.zk.ZooKeeper(host)
print zk
zk.delete_recursive('brokers/topics/test')
zk.close()
except Exception as e:
zk.close()
def setUp(test, tree=None, connection_string='zookeeper.example.com:2181'):
"""Set up zookeeper emulation.
Standard (mock) testing
-----------------------
The first argument is a test case object (either doctest or unittest).
You can optionally pass:
tree
An initial ZooKeeper tree expressed as an import string.
If not passed, an initial tree will be created with examples
used in the zc.zk doctests.
connection_string
The connection string to use for the emulation server. This
defaults to 'zookeeper.example.com:2181'.
Testing with a real ZooKeeper Server
------------------------------------
You can test against a real ZooKeeper server, instead of a mock by
setting the environment variable TEST_ZOOKEEPER_CONNECTION to the
connection string of a test server.
The tests will create a top-level node with a random name that
starts with 'zc.zk.testing.test-roo', and use that as the virtual
root for your tests. Although this is the virtual root, of the
zookeeper tree in your tests, the presense of the node may be
shown in your tests. In particularm ``zookeeper.create`` returns
the path created and the string returned is real, not virtual.
This node is cleaned up by the ``tearDown``.
A doctest can determine if it's running with a stub ZooKeeper by
checking whether the value of the ZooKeeper gloval variable is None.
A regular unit test can check the ZooKeeper test attribute.
"""
globs = setupstack.globs(test)
faux_zookeeper = None
real_zk = testing_with_real_zookeeper()
if real_zk:
test_root = '/zc.zk.testing.test-root%s' % random.randint(0, sys.maxsize)
globs['/zc.zk.testing.test-root'] = test_root
setup_tree(tree, real_zk, test_root, True)
orig_init = zookeeper.init
@side_effect(
setupstack.context_manager(test, mock.patch('zookeeper.init')))
def init(addr, watch=None, session_timeout=1000):
if addr != connection_string:
return orig_init(addr, watch, session_timeout)
else:
return orig_init(real_zk+test_root, watch, session_timeout)
setupstack.register(
test, lambda : setattr(zc.zk.ZooKeeper, 'test_sleep', 0))
zc.zk.ZooKeeper.test_sleep = .01
time.sleep(float(os.environ.get('TEST_ZOOKEEPER_SLEEP', 0)))
else:
if tree:
faux_zookeeper = ZooKeeper(connection_string, Node())
else:
faux_zookeeper = ZooKeeper(
connection_string,
Node(
fooservice = Node(
json.dumps(dict(
database = "/databases/foomain",
threads = 1,
favorite_color= "red",
)),
providers = Node()
),
zookeeper = Node('', quota=Node()),
),
)
for name in ZooKeeper.__dict__:
if name[0] == '_':
continue
m = setupstack.context_manager(test, mock.patch('zookeeper.'+name))
m.side_effect = getattr(faux_zookeeper, name)
if tree:
zk = zc.zk.ZooKeeper(connection_string)
zk.import_tree(tree)
zk.close()
globs['wait_until'] = wait_until # BBB
globs['ZooKeeper'] = faux_zookeeper
globs.setdefault('assert_', assert_)
def __init__(self, buildout, name, options):
super(Recipe, self).__init__(buildout, name, options)
if len(options) < 2:
zk = zc.zk.ZK('zookeeper:2181')
path = '/' + name.rsplit('.', 1)[0].replace(',', '/')
options.update(flatten(zk, path))
zk.close()
user = self.user = options.get('user', 'root')
self['deployment'] = dict(
recipe='zc.recipe.deployment',
name=name,
user=user,
)
client = docker.Client()
image_spec = options['image']
image_name, tag = image_spec.rsplit(':', 1)
images = [
image for image in client.images(image_name) if image['Tag'] == tag
]
if not images:
container = client.create_container(
'registry',
'/docker-registry/run.sh',
environment=dict(SETTINGS_FLAVOR='prod'),
detach=True,
ports=["5000/tcp"])
try:
client.start(container)
try:
port = client.inspect_container(container)[
'NetworkSettings']['PortMapping']['Tcp']['5000']
repo_name = "127.0.0.1:%s/%s" % (port, image_name)
time.sleep(9) # give registry plenty of time to start
client.pull(repo_name, tag=tag)
finally:
client.stop(container)
finally:
client.remove_container(container)
images = [
image for image in client.images(repo_name)
if tag is None or image['Tag'] == tag
]
if not images:
raise ValueError("Couldn't pull", image_spec)
[image] = images
client.tag(image['Id'], image_name, tag)
else:
[image] = images
image = client.inspect_image(image['Id'])['container_config']
run_command = ['docker run -rm']
hostname = '.'.join(reversed(name.rsplit('.', 1)[0].split(',')))
n = name.rsplit('.', 1)[1]
if n != '0':
hostname = 'n%s.%s' % (n, hostname)
hostname += ".o-o." + socket.getfqdn()
run_command.append("-h " + hostname)
ports = options.get('ports', ())
if ports == '*':
run_command.append('-P')
elif ports:
if ports == '=:*':
ports = [
'%s:%s' % (port, port) for port in sorted(
image['PortSpecs']
# 0.7: parse_exposed(image['ExposedPorts'])
)
]
else:
ports = parse_ports(
ports,
set(image['PortSpecs']
# 0.7: parse_exposed(image['ExposedPorts'])
))
for port in ports:
run_command.extend(('-p', port))
for option, value in sorted(options.items()):
if option.startswith('volumes/'):
run_command.append('-v=%s:%s' % (value, option[7:]))
elif option.startswith('environment/'):
run_command.append('-e=%s=%s' % (option[12:], value))
run_command.append(image_spec)
run_command = ' '.join(run_command)
self['container'] = dict( # TODO: start-test-program
recipe='zc.zdaemonrecipe',
deployment='deployment',
program=run_command)
self['rc'] = dict(recipe='zc.recipe.rhrc',
deployment='deployment',
parts='container',
chkconfig='345 99 10',
digest=hashlib.md5(run_command).hexdigest(),
**{'process-management': 'true'})
def shutdown():
zk.close()
worker.shutdown()
def shutdown():
zk.close()
server.close()
if status_server is not None:
status_server.close()
lb.shutdown()
def lbmain(args=None, run=True):
"""%prog [options] zookeeper_connection path
Run a resume-based load balancer on addr.
"""
if args is None:
args = sys.argv[1:]
elif isinstance(args, str):
args = args.split()
run = False
import optparse
parser = optparse.OptionParser(lbmain.__doc__)
parser.add_option(
'-a', '--address', default=':0',
help="Address to listed on for web requests"
)
parser.add_option(
'-b', '--backlog', type='int',
help="Server backlog setting.")
parser.add_option(
'-d', '--backdoor', action='store_true',
help="Run a backdoor server. Use with caution!")
parser.add_option(
'-e', '--disconnect-message',
help="Path to error page to use when a request is lost due to "
"worker disconnection"
)
parser.add_option(
'-L', '--logger-configuration',
help=
"Read logger configuration from the given configuration file path.\n"
"\n"
"The configuration file must be in ZConfig logger configuration syntax."
"\n"
"Alternatively, you can give a Python logger level name or number."
)
parser.add_option('-l', '--access-logger', help='Access-log logger name.')
parser.add_option(
'-m', '--max-connections', type='int',
help="Maximum number of simultanious accepted connections.")
parser.add_option(
'-r', '--request-classifier', default='zc.resumelb.lb:host_classifier',
help="Request classification function (module:expr)"
)
parser.add_option(
'-s', '--status-server',
help=("Run a status server for getting pool information. "
"The argument is a unix-domain socket path to listen on."))
parser.add_option(
'-t', '--socket-timeout', type='float', default=99.,
help=('HTTP socket timeout.'))
parser.add_option(
'-v', '--single-version', action='store_true',
help=('Only use a single worker version.'))
try:
options, args = parser.parse_args(args)
if len(args) != 2:
print 'Error: must supply a zookeeper connection string and path.'
parser.parse_args(['-h'])
zookeeper, path = args
except SystemExit:
if run:
raise
else:
return
if options.logger_configuration:
logger_config = options.logger_configuration
if re.match(r'\d+$', logger_config):
logging.basicConfig(level=int(logger_config))
elif logger_config in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'):
logging.basicConfig(level=getattr(logging, logger_config))
else:
import ZConfig
with open(logger_config) as f:
ZConfig.configureLoggers(f.read())
zk = zc.zk.ZooKeeper(zookeeper)
addrs = zk.children(path+'/workers/providers')
rcmod, rcexpr = options.request_classifier.split(':')
__import__(rcmod)
rcmod = sys.modules[rcmod]
request_classifier = eval(rcexpr, rcmod.__dict__)
disconnect_message = options.disconnect_message
if disconnect_message:
with open(disconnect_message) as f:
disconnect_message = f.read()
else:
disconnect_message = zc.resumelb.lb.default_disconnect_message
from zc.resumelb.lb import LB
lb = LB(map(zc.parse_addr.parse_addr, ()),
request_classifier, disconnect_message,
single_version=options.single_version)
to_send = [[]]
# Set up notification of address changes.
awatcher = gevent.get_hub().loop.async()
@awatcher.start
def _():
lb.set_worker_addrs(to_send[0])
if options.single_version:
@addrs
def get_addrs(a):
to_send[0] = dict(
(zc.parse_addr.parse_addr(addr),
zk.get_properties(
path+'/workers/providers/'+addr).get('version')
)
for addr in addrs)
awatcher.send()
else:
@addrs
def get_addrs(a):
to_send[0] = map(zc.parse_addr.parse_addr, addrs)
awatcher.send()
# Set up notification of address changes.
settings = zk.properties(path)
swatcher = gevent.get_hub().loop.async()
swatcher.start(lambda : lb.update_settings(settings))
settings(lambda a: swatcher.send())
lb.zk = zk
lb.__zk = addrs, settings
# Now, start a wsgi server
addr = zc.parse_addr.parse_addr(options.address)
if options.max_connections:
spawn= gevent.pool.Pool(options.max_connections)
else:
spawn = 'default'
if options.access_logger:
accesslog = AccessLog(options.access_logger)
else:
accesslog = None
server = WSGIServer(
addr, lb.handle_wsgi, backlog=options.backlog,
spawn=spawn, log=accesslog, socket_timeout=options.socket_timeout)
server.start()
registration_data = {}
if options.backdoor:
from gevent import backdoor
bd = backdoor.BackdoorServer(('127.0.0.1', 0), locals())
bd.start()
registration_data['backdoor'] = '127.0.0.1:%s' % bd.server_port
status_server = None
if options.status_server:
def status(socket, addr):
pool = lb.pool
writer = socket.makefile('w')
writer.write(json.dumps(
dict(
backlog = pool.backlog,
mean_backlog = pool.mbacklog,
workers = [
(worker.__name__,
worker.backlog,
worker.mbacklog,
(int(worker.oldest_time)
if worker.oldest_time else None),
)
for worker in sorted(
pool.workers, key=lambda w: w.__name__)
]
))+'\n')
writer.close()
socket.close()
status_server_address = options.status_server
if os.path.exists(status_server_address):
os.remove(status_server_address)
sock = gevent.socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(status_server_address)
sock.listen(5)
status_server = gevent.server.StreamServer(sock, status)
status_server.start()
zk.register_server(path+'/providers', (addr[0], server.server_port),
**registration_data)
def shutdown():
zk.close()
server.close()
if status_server is not None:
status_server.close()
lb.shutdown()
gevent.signal(signal.SIGTERM, shutdown)
if run:
try:
server.serve_forever()
finally:
logging.getLogger(__name__+'.lbmain').info('exiting')
zk.close()
else:
gevent.sleep(.01)
return lb, server
