def sync_with_canonical(url, dry_run=False, force=False, tree_directory=None): zk = zc.zk.ZK(ZK_LOCATION) zk_version = get_zk_version(zk) if zk_version is None: logger.critical("ALL STOP, cluster version is None") if not force: return retries = 0 while True: try: if url.startswith('svn') or url.startswith('file://'): vcs = SVN(url) else: vcs = GIT(url, tree_directory) break except RuntimeError: retries += 1 if retries > MAX_VCS_RETRIES: raise time.sleep(3) logger.info("VCS Version: " + str(vcs.version)) logger.info("ZK Version: " + str(zk_version)) if zk_version != vcs.version: if not (force or zk_version is False): for child in zk.get_children('/hosts'): host_version = zk.properties('/hosts/' + child)['version'] if host_version != zk_version: logger.error( "Version mismatch detected, can't resync since " + "host %s has not converged (%s -> %s)" % (child, host_version, zk_version)) return cluster_lock = zk.client.Lock('/hosts-lock', str(os.getpid())) if cluster_lock.acquire(0): try: logger.info("Version mismatch detected, resyncing") # Import changes for fi, contents in vcs: output = ' '.join(('Importing', fi)) if dry_run: output += ' (dry run, no action taken)' logger.info(output) if not dry_run: zk.import_tree(contents, trim=fi.endswith('.zk')) # bump version number if not dry_run: zk.properties('/hosts').update(version=vcs.version) finally: cluster_lock.release() else: logger.error("Refused to update zookeeper tree, " "couldn't obtain cluster lock") zk.close()
def import_(args=None): """Usage: %prog [options] connection [import-file [path]] Import a tree definition from a file. If no import-file is provided or if the import file is -, then data are read from standard input. """ if args is None: args = sys.argv[1:] parser = optparse.OptionParser(import_.__doc__) parser.add_option('-d', '--dry-run', action='store_true') parser.add_option('-t', '--trim', action='store_true') parser.add_option( '-p', '--permission', type='int', default=zookeeper.PERM_ALL, help='ZooKeeper permission bits as integer,' ' defaulting to zookeeper.PERM_ALL', ) options, args = parser.parse_args(args) if not (1 <= len(args) <= 3): parser.parse_args(['-h']) connection = args.pop(0) if args: import_file = args.pop(0) else: import_file = '-' if args: [path] = args else: path = '/' logging.basicConfig(level=logging.WARNING) zk = zc.zk.ZooKeeper(connection) if import_file == '-': import_file = sys.stdin else: import_file = open(import_file) zk.import_tree( import_file.read(), path, trim=options.trim, dry_run=options.dry_run, acl=None, )
def import_(args=None): """Usage: %prog [options] connection [import-file [path]] Import a tree definition from a file. If no import-file is provided or if the import file is -, then data are read from standard input. """ if args is None: args = sys.argv[1:] parser = optparse.OptionParser(import_.__doc__) parser.add_option('-d', '--dry-run', action='store_true') parser.add_option('-t', '--trim', action='store_true') parser.add_option( '-p', '--permission', type='int', default=kazoo.security.Permissions.ALL, help='ZooKeeper permission bits as integer,' ' kazoo.security.Permissions.ALL', ) options, args = parser.parse_args(args) if not (1 <= len(args) <= 3): parser.parse_args(['-h']) connection = args.pop(0) if args: import_file = args.pop(0) else: import_file = '-' if args: [path] = args else: path = '/' logging.basicConfig(level=logging.WARNING) zk = zc.zk.ZooKeeper(connection) if import_file == '-': import_file = sys.stdin else: import_file = open(import_file) zk.import_tree( import_file.read(), path, trim=options.trim, dry_run=options.dry_run, acl=[world_acl(options.permission)], ) zk.close()
def import_(args=None): """Usage: %prog [options] connection [import-file [path]] Import a tree definition from a file. If no import-file is provided or if the import file is -, then data are read from standard input. """ if args is None: args = sys.argv[1:] parser = optparse.OptionParser(import_.__doc__) parser.add_option("-d", "--dry-run", action="store_true") parser.add_option("-t", "--trim", action="store_true") parser.add_option( "-p", "--permission", type="int", default=zookeeper.PERM_ALL, help="ZooKeeper permission bits as integer," " defaulting to zookeeper.PERM_ALL", ) options, args = parser.parse_args(args) if not (1 <= len(args) <= 3): parser.parse_args(["-h"]) connection = args.pop(0) if args: import_file = args.pop(0) else: import_file = "-" if args: [path] = args else: path = "/" logging.basicConfig(level=logging.WARNING) zk = zc.zk.ZooKeeper(connection) if import_file == "-": import_file = sys.stdin else: import_file = open(import_file) zk.import_tree(import_file.read(), path, trim=options.trim, dry_run=options.dry_run, acl=None)
def setup_tree(tree, connection_string, root='/test-root', zookeeper_node=False): zk = zc.zk.ZooKeeper(connection_string) if zk.exists(root): zk.delete_recursive(root) zk.create(root, '', zc.zk.OPEN_ACL_UNSAFE) zk.import_tree(tree or """ /fooservice /providers database = '/databases/foomain' threads = 1 favorite_color = 'red' """, root) if zookeeper_node: zk.import_tree(""" /zookeeper /quota """, root) zk.close()
def get_zk_version(zk): try: return zk.get_properties('/hosts')['version'] except kazoo.exceptions.NoNodeException: zk.import_tree('/hosts\n version="initial"') return "initial"
def setUp(test, tree=None, connection_string='zookeeper.example.com:2181'): """Set up zookeeper emulation. Standard (mock) testing ----------------------- The first argument is a test case object (either doctest or unittest). You can optionally pass: tree An initial ZooKeeper tree expressed as an import string. If not passed, an initial tree will be created with examples used in the zc.zk doctests. connection_string The connection string to use for the emulation server. This defaults to 'zookeeper.example.com:2181'. Testing with a real ZooKeeper Server ------------------------------------ You can test against a real ZooKeeper server, instead of a mock by setting the environment variable TEST_ZOOKEEPER_CONNECTION to the connection string of a test server. The tests will create a top-level node with a random name that starts with 'zc.zk.testing.test-roo', and use that as the virtual root for your tests. Although this is the virtual root, of the zookeeper tree in your tests, the presense of the node may be shown in your tests. In particularm ``zookeeper.create`` returns the path created and the string returned is real, not virtual. This node is cleaned up by the ``tearDown``. A doctest can determine if it's running with a stub ZooKeeper by checking whether the value of the ZooKeeper gloval variable is None. A regular unit test can check the ZooKeeper test attribute. """ globs = setupstack.globs(test) faux_zookeeper = None real_zk = testing_with_real_zookeeper() if real_zk: test_root = '/zc.zk.testing.test-root%s' % random.randint(0, sys.maxsize) globs['/zc.zk.testing.test-root'] = test_root setup_tree(tree, real_zk, test_root, True) orig_init = zookeeper.init @side_effect( setupstack.context_manager(test, mock.patch('zookeeper.init'))) def init(addr, watch=None, session_timeout=1000): if addr != connection_string: return orig_init(addr, watch, session_timeout) else: return orig_init(real_zk+test_root, watch, session_timeout) setupstack.register( test, lambda : setattr(zc.zk.ZooKeeper, 'test_sleep', 0)) zc.zk.ZooKeeper.test_sleep = .01 time.sleep(float(os.environ.get('TEST_ZOOKEEPER_SLEEP', 0))) else: if tree: faux_zookeeper = ZooKeeper(connection_string, Node()) else: faux_zookeeper = ZooKeeper( connection_string, Node( fooservice = Node( json.dumps(dict( database = "/databases/foomain", threads = 1, favorite_color= "red", )), providers = Node() ), zookeeper = Node('', quota=Node()), ), ) for name in ZooKeeper.__dict__: if name[0] == '_': continue m = setupstack.context_manager(test, mock.patch('zookeeper.'+name)) m.side_effect = getattr(faux_zookeeper, name) if tree: zk = zc.zk.ZooKeeper(connection_string) zk.import_tree(tree) zk.close() globs['wait_until'] = wait_until # BBB globs['ZooKeeper'] = faux_zookeeper globs.setdefault('assert_', assert_)
import zc.zk zk = zc.zk.ZooKeeper() zk.import_tree(""" /simul cache_size=20000 clients=12 lambda=0.01 objects_per_site=1000 objects_per_request=100 sites=40 workers=2 /lb /providers /workers history=999 threads=1 /providers """, trim=True)