def sync_with_canonical(url, dry_run=False, force=False, tree_directory=None):
zk = zc.zk.ZK(ZK_LOCATION)
zk_version = get_zk_version(zk)
if zk_version is None:
logger.critical("ALL STOP, cluster version is None")
if not force:
return
retries = 0
while True:
try:
if url.startswith('svn') or url.startswith('file://'):
vcs = SVN(url)
else:
vcs = GIT(url, tree_directory)
break
except RuntimeError:
retries += 1
if retries > MAX_VCS_RETRIES:
raise
time.sleep(3)
logger.info("VCS Version: " + str(vcs.version))
logger.info("ZK Version: " + str(zk_version))
if zk_version != vcs.version:
if not (force or zk_version is False):
for child in zk.get_children('/hosts'):
host_version = zk.properties('/hosts/' + child)['version']
if host_version != zk_version:
logger.error(
"Version mismatch detected, can't resync since " +
"host %s has not converged (%s -> %s)" %
(child, host_version, zk_version))
return
cluster_lock = zk.client.Lock('/hosts-lock', str(os.getpid()))
if cluster_lock.acquire(0):
try:
logger.info("Version mismatch detected, resyncing")
# Import changes
for fi, contents in vcs:
output = ' '.join(('Importing', fi))
if dry_run:
output += ' (dry run, no action taken)'
logger.info(output)
if not dry_run:
zk.import_tree(contents, trim=fi.endswith('.zk'))
# bump version number
if not dry_run:
zk.properties('/hosts').update(version=vcs.version)
finally:
cluster_lock.release()
else:
logger.error("Refused to update zookeeper tree, "
"couldn't obtain cluster lock")
zk.close()
def import_(args=None):
"""Usage: %prog [options] connection [import-file [path]]
Import a tree definition from a file.
If no import-file is provided or if the import file is -, then
data are read from standard input.
"""
if args is None:
args = sys.argv[1:]
parser = optparse.OptionParser(import_.__doc__)
parser.add_option('-d', '--dry-run', action='store_true')
parser.add_option('-t', '--trim', action='store_true')
parser.add_option(
'-p',
'--permission',
type='int',
default=zookeeper.PERM_ALL,
help='ZooKeeper permission bits as integer,'
' defaulting to zookeeper.PERM_ALL',
)
options, args = parser.parse_args(args)
if not (1 <= len(args) <= 3):
parser.parse_args(['-h'])
connection = args.pop(0)
if args:
import_file = args.pop(0)
else:
import_file = '-'
if args:
[path] = args
else:
path = '/'
logging.basicConfig(level=logging.WARNING)
zk = zc.zk.ZooKeeper(connection)
if import_file == '-':
import_file = sys.stdin
else:
import_file = open(import_file)
zk.import_tree(
import_file.read(),
path,
trim=options.trim,
dry_run=options.dry_run,
acl=None,
)
def import_(args=None):
"""Usage: %prog [options] connection [import-file [path]]
Import a tree definition from a file.
If no import-file is provided or if the import file is -, then
data are read from standard input.
"""
if args is None:
args = sys.argv[1:]
parser = optparse.OptionParser(import_.__doc__)
parser.add_option('-d', '--dry-run', action='store_true')
parser.add_option('-t', '--trim', action='store_true')
parser.add_option(
'-p', '--permission', type='int',
default=kazoo.security.Permissions.ALL,
help='ZooKeeper permission bits as integer,'
' kazoo.security.Permissions.ALL',
)
options, args = parser.parse_args(args)
if not (1 <= len(args) <= 3):
parser.parse_args(['-h'])
connection = args.pop(0)
if args:
import_file = args.pop(0)
else:
import_file = '-'
if args:
[path] = args
else:
path = '/'
logging.basicConfig(level=logging.WARNING)
zk = zc.zk.ZooKeeper(connection)
if import_file == '-':
import_file = sys.stdin
else:
import_file = open(import_file)
zk.import_tree(
import_file.read(), path,
trim=options.trim,
dry_run=options.dry_run,
acl=[world_acl(options.permission)],
)
zk.close()
def import_(args=None):
"""Usage: %prog [options] connection [import-file [path]]
Import a tree definition from a file.
If no import-file is provided or if the import file is -, then
data are read from standard input.
"""
if args is None:
args = sys.argv[1:]
parser = optparse.OptionParser(import_.__doc__)
parser.add_option("-d", "--dry-run", action="store_true")
parser.add_option("-t", "--trim", action="store_true")
parser.add_option(
"-p",
"--permission",
type="int",
default=zookeeper.PERM_ALL,
help="ZooKeeper permission bits as integer," " defaulting to zookeeper.PERM_ALL",
)
options, args = parser.parse_args(args)
if not (1 <= len(args) <= 3):
parser.parse_args(["-h"])
connection = args.pop(0)
if args:
import_file = args.pop(0)
else:
import_file = "-"
if args:
[path] = args
else:
path = "/"
logging.basicConfig(level=logging.WARNING)
zk = zc.zk.ZooKeeper(connection)
if import_file == "-":
import_file = sys.stdin
else:
import_file = open(import_file)
zk.import_tree(import_file.read(), path, trim=options.trim, dry_run=options.dry_run, acl=None)
def setup_tree(tree, connection_string, root='/test-root',
zookeeper_node=False):
zk = zc.zk.ZooKeeper(connection_string)
if zk.exists(root):
zk.delete_recursive(root)
zk.create(root, '', zc.zk.OPEN_ACL_UNSAFE)
zk.import_tree(tree or """
/fooservice
/providers
database = '/databases/foomain'
threads = 1
favorite_color = 'red'
""", root)
if zookeeper_node:
zk.import_tree("""
/zookeeper
/quota
""", root)
zk.close()
def get_zk_version(zk):
try:
return zk.get_properties('/hosts')['version']
except kazoo.exceptions.NoNodeException:
zk.import_tree('/hosts\n version="initial"')
return "initial"
def setUp(test, tree=None, connection_string='zookeeper.example.com:2181'):
"""Set up zookeeper emulation.
Standard (mock) testing
-----------------------
The first argument is a test case object (either doctest or unittest).
You can optionally pass:
tree
An initial ZooKeeper tree expressed as an import string.
If not passed, an initial tree will be created with examples
used in the zc.zk doctests.
connection_string
The connection string to use for the emulation server. This
defaults to 'zookeeper.example.com:2181'.
Testing with a real ZooKeeper Server
------------------------------------
You can test against a real ZooKeeper server, instead of a mock by
setting the environment variable TEST_ZOOKEEPER_CONNECTION to the
connection string of a test server.
The tests will create a top-level node with a random name that
starts with 'zc.zk.testing.test-roo', and use that as the virtual
root for your tests. Although this is the virtual root, of the
zookeeper tree in your tests, the presense of the node may be
shown in your tests. In particularm ``zookeeper.create`` returns
the path created and the string returned is real, not virtual.
This node is cleaned up by the ``tearDown``.
A doctest can determine if it's running with a stub ZooKeeper by
checking whether the value of the ZooKeeper gloval variable is None.
A regular unit test can check the ZooKeeper test attribute.
"""
globs = setupstack.globs(test)
faux_zookeeper = None
real_zk = testing_with_real_zookeeper()
if real_zk:
test_root = '/zc.zk.testing.test-root%s' % random.randint(0, sys.maxsize)
globs['/zc.zk.testing.test-root'] = test_root
setup_tree(tree, real_zk, test_root, True)
orig_init = zookeeper.init
@side_effect(
setupstack.context_manager(test, mock.patch('zookeeper.init')))
def init(addr, watch=None, session_timeout=1000):
if addr != connection_string:
return orig_init(addr, watch, session_timeout)
else:
return orig_init(real_zk+test_root, watch, session_timeout)
setupstack.register(
test, lambda : setattr(zc.zk.ZooKeeper, 'test_sleep', 0))
zc.zk.ZooKeeper.test_sleep = .01
time.sleep(float(os.environ.get('TEST_ZOOKEEPER_SLEEP', 0)))
else:
if tree:
faux_zookeeper = ZooKeeper(connection_string, Node())
else:
faux_zookeeper = ZooKeeper(
connection_string,
Node(
fooservice = Node(
json.dumps(dict(
database = "/databases/foomain",
threads = 1,
favorite_color= "red",
)),
providers = Node()
),
zookeeper = Node('', quota=Node()),
),
)
for name in ZooKeeper.__dict__:
if name[0] == '_':
continue
m = setupstack.context_manager(test, mock.patch('zookeeper.'+name))
m.side_effect = getattr(faux_zookeeper, name)
if tree:
zk = zc.zk.ZooKeeper(connection_string)
zk.import_tree(tree)
zk.close()
globs['wait_until'] = wait_until # BBB
globs['ZooKeeper'] = faux_zookeeper
globs.setdefault('assert_', assert_)
import zc.zk
zk = zc.zk.ZooKeeper()
zk.import_tree("""
/simul
cache_size=20000
clients=12
lambda=0.01
objects_per_site=1000
objects_per_request=100
sites=40
workers=2
/lb
/providers
/workers
history=999
threads=1
/providers
""", trim=True)
