class KazooServiceRegistryIntegrationTestsWithAuth(KazooTestHarness): # A flag for filtering nose tests integration = True def setUp(self): self.setup_zookeeper() self.sandbox = "/tests/sr-%s" % uuid.uuid4().hex self.server = 'localhost:20000' self.username = '******' self.password = '******' self.ndsr = KazooServiceRegistry(server=self.server, username=self.username, password=self.password, rate_limit_calls=0, rate_limit_time=0) self._zk = self.ndsr._zk def tearDown(self): self.teardown_zookeeper() self.ndsr._initialized = False def test_set_node_with_acl(self): path = '%s/set_node_test_1' % self.sandbox self.ndsr.set_node(path, {}) acls, znode_stat = self.ndsr._zk.get_acls(path) # ACLs are returned back in a slightly different format than # when we set them, so we have to dig into the return value # a bit for tests. self.assertEquals(self.ndsr._acl[0], acls[0]) self.assertEquals(self.ndsr._acl[1], acls[1])
def setUp(self): self.setup_zookeeper() self.sandbox = "/tests/sr-%s" % uuid.uuid4().hex self.server = 'localhost:20000' self.ndsr = KazooServiceRegistry(server=self.server, rate_limit_calls=0, rate_limit_time=0)
def setUp(self): self.setup_zookeeper() self.sandbox = "/tests/sr-%s" % uuid.uuid4().hex self.server = 'localhost:20000' self.username = '******' self.password = '******' self.ndsr = KazooServiceRegistry(server=self.server, username=self.username, password=self.password, rate_limit_calls=0, rate_limit_time=0) self._zk = self.ndsr._zk
def setUp(self): self.setup_zookeeper() self.server = 'localhost:20000' self.sandbox = "/tests/registration-%s" % uuid.uuid4().hex nd = KazooServiceRegistry(server=self.server, rate_limit_calls=0, rate_limit_time=0) self.zk = nd._zk
def _connect(self): """Connects to the ServiceRegistry. If already connected, updates the current connection settings.""" self.log.debug('Checking for ServiceRegistry object...') if not self._server_reg: self.log.debug('Creating new ServiceRegistry object...') # 通过用户名,密码连接到注册服务器 self._server_reg = ServiceRegistry(server=self._server, lazy=True, username=self.user, password=self.password) else: self.log.debug('Updating existing object...') self._server_reg.set_username(self.user) self._server_reg.set_password(self.password)
def _connect(self): """Connects to the ServiceRegistry. If already connected, updates the current connection settings.""" self.log.debug('Checking for ServiceRegistry object...') if not self._sr: self.log.debug('Creating new ServiceRegistry object...') self._sr = ServiceRegistry(server=self._server, lazy=True, username=self.user, password=self.password) else: self.log.debug('Updating existing object...') self._sr.set_username(self.user) self._sr.set_password(self.password)
class WatcherDaemon(threading.Thread): """The main daemon process. This is the main object that defines all of our major functions and connection information.""" LOGGER = 'WatcherDaemon' def __init__(self, server, config_file, verbose=False): """Initilization code for the main WatcherDaemon. Set up our local logger reference, and pid file locations.""" # Initiate our thread super(WatcherDaemon, self).__init__() self.log = logging.getLogger(self.LOGGER) self.log.info('WatcherDaemon %s' % VERSION) self._watchers = [] self._sr = None self._config_file = config_file self._server = server self._verbose = verbose # Get a logger for nd_service_registry and set it to be quiet nd_log = logging.getLogger('nd_service_registry') # Set up our threading environment self._event = threading.Event() # These threads can die with prejudice. Make sure that any time the # python interpreter exits, we exit immediately self.setDaemon(True) # Watch for any signals signal.signal(signal.SIGHUP, self._signal_handler) # Bring in our configuration options self._parse_config() # Create our ServiceRegistry object self._connect() # Start up self.start() def _signal_handler(self, signum, frame): """Watch for certain signals""" self.log.warning('Received signal: %s' % signum) if signum == 1: self.log.warning('Received SIGHUP. Reloading config.') self._parse_config() self._connect() self._setup_watchers() def _parse_config(self): """Read in the supplied config file and update our local settings.""" self.log.debug('Loading config...') self._config = ConfigParser.ConfigParser() self._config.read(self._config_file) # Check if auth data was supplied. If it is, read it in and then remove # it from our configuration object so its not used anywhere else. try: self.user = self._config.get('auth', 'user') self.password = self._config.get('auth', 'password') self._config.remove_section('auth') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): self.user = None self.password = None def _connect(self): """Connects to the ServiceRegistry. If already connected, updates the current connection settings.""" self.log.debug('Checking for ServiceRegistry object...') if not self._sr: self.log.debug('Creating new ServiceRegistry object...') self._sr = ServiceRegistry(server=self._server, lazy=True, username=self.user, password=self.password) else: self.log.debug('Updating existing object...') self._sr.set_username(self.user) self._sr.set_password(self.password) def _setup_watchers(self): # For each watcher, see if we already have one for a given path or not. for service in self._config.sections(): w = self._get_watcher(service) # Gather up the config data for our section into a few local # variables so that we can shorten the statements below. command = self._config.get(service, 'cmd') service_port = self._config.get(service, 'service_port') zookeeper_path = self._config.get(service, 'zookeeper_path') refresh = self._config.get(service, 'refresh') # Gather our optional parameters. If they don't exist, set # some reasonable default. try: zookeeper_data = self._parse_data( self._config.get(service, 'zookeeper_data')) except: zookeeper_data = {} try: service_hostname = self._config.get(service, 'service_hostname') except: service_hostname = socket.getfqdn() if w: # Certain fields cannot be changed without destroying the # object and its registration with Zookeeper. if w._service_port != service_port or \ w._service_hostname != service_hostname or \ w._path != zookeeper_path: w.stop() w = None if w: # We already have a watcher for this service. Update its # object data, and let it keep running. w.set(command=command, data=zookeeper_data, refresh=refresh) # If there's still no 'w' returned (either _get_watcher failed, or # we noticed that certain un-updatable fields were changed, then # create a new object. if not w: w = ServiceWatcher(registry=self._sr, service=service, service_port=service_port, service_hostname=service_hostname, command=command, path=zookeeper_path, data=zookeeper_data, refresh=refresh) self._watchers.append(w) # Check if any watchers need to be destroyed because they're no longer # in our config. for w in self._watchers: if not w._service in list(self._config.sections()): w.stop() self._watchers.remove(w) def _get_watcher(self, service): """Returns a watcher based on the service name.""" for watcher in self._watchers: if watcher._service == service: return watcher return None def _parse_data(self, data): """Convert a string of data from ConfigParse into our dict. The zookeeper_data field supports one of two types of fields. Either a single key=value string, or a JSON-formatted set of key=value pairs: zookeeper_data: foo=bar zookeeper_data: foo=bar, bar=foo zookeeper_data: { "foo": "bar", "bar": "foo" } Args: data: String representing data above""" try: data_dict = json.loads(data) except: data_dict = {} for pair in data.split(','): if pair.split('=').__len__() == 2: key = pair.split('=')[0] value = pair.split('=')[1] data_dict[key] = value return data_dict def run(self): """Start up all of the worker threads and keep an eye on them""" self._setup_watchers() # Now, loop. Wait for a death signal while True and not self._event.is_set(): self._event.wait(1) # At this point we must be exiting. Kill off our above threads for w in self._watchers: w.stop() def stop(self): self._event.set()
class KazooServiceRegistryIntegrationTests(KazooTestHarness): # A flag for filtering nose tests integration = True def setUp(self): self.setup_zookeeper() self.sandbox = "/tests/sr-%s" % uuid.uuid4().hex self.server = 'localhost:20000' self.ndsr = KazooServiceRegistry(server=self.server, rate_limit_calls=0, rate_limit_time=0) def tearDown(self): self.teardown_zookeeper() self.ndsr._initialized = False def test_get_state(self): self.ndsr.start() self.assertTrue(self.ndsr.get_state()) self.ndsr.stop() self.assertFalse(self.ndsr.get_state()) def test_get_state_with_callback(self): # With a callback, the callback should get executed callback_checker = mock.MagicMock() callback_checker.test.return_value = True self.ndsr.start() self.ndsr.get_state(callback_checker.test) self.ndsr.stop() self.assertTrue(mock.call(True) in callback_checker.test.mock_calls) self.assertTrue(mock.call(False) in callback_checker.test.mock_calls) def test_unset_node(self): path = '%s/test_unset_node' % self.sandbox self.ndsr.set_node(path) self.ndsr.unset(path) self.assertRaises(exceptions.NoNodeError, self.ndsr._zk.get, path) def test_unset_data(self): path = '%s/test_unset_data' % self.sandbox self.ndsr.set_data(path) self.ndsr.unset(path) self.assertRaises(exceptions.NoNodeError, self.ndsr._zk.get, path) def test_unset_data_with_missing_reg_object(self): path = '%s/test_unset_data_missing_reg_object' % self.sandbox self.ndsr._zk.create(path, makepath=True) self.ndsr.unset(path) self.assertRaises(exceptions.NoNodeError, self.ndsr._zk.get, path) def test_unset_data_on_absent_path(self): path = '%s/test_unset_data_on_absent_path' % self.sandbox self.ndsr.unset(path) self.assertRaises(exceptions.NoNodeError, self.ndsr._zk.get, path)
class WatcherDaemon(threading.Thread): """The main daemon process. This is the main object that defines all of our major functions and connection information.""" LOGGER = 'WatcherDaemon' def __init__(self, server, config_file=None, verbose=False): """Initilization code for the main WatcherDaemon. Set up our local logger reference, and pid file locations.""" # Initiate our thread super(WatcherDaemon, self).__init__() self.log = logging.getLogger(self.LOGGER) self.log.info('WatcherDaemon %s' % VERSION) self._watchers = [] self._sr = None self._config_file = config_file self._server = server self._verbose = verbose self.done = False # Set up our threading environment self._event = threading.Event() # These threads can die with prejudice. Make sure that any time the # python interpreter exits, we exit immediately self.setDaemon(True) # Watch for any signals signal.signal(signal.SIGHUP, self._signal_handler) signal.signal(signal.SIGTERM, self._signal_handler) # Bring in our configuration options self._parse_config() # Create our ServiceRegistry object self._connect() # Start up self.start() def _signal_handler(self, signum, frame): """Watch for certain signals""" self.log.warning('Received signal: %s' % signum) if signum == signal.SIGHUP: self.log.warning('Reloading config.') self._parse_config() self._connect() self._setup_watchers() if signum == signal.SIGTERM: self.log.warning('Terminating all node registrations.') self.stop() def _parse_config(self): """Read in the supplied config file and update our local settings.""" self.log.debug('Loading config...') self._config = ConfigParser.ConfigParser() self._config.read(self._config_file) # Check if auth data was supplied. If it is, read it in and then remove # it from our configuration object so its not used anywhere else. try: self.user = self._config.get('auth', 'user') self.password = self._config.get('auth', 'password') self._config.remove_section('auth') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): self.user = None self.password = None def _connect(self): """Connects to the ServiceRegistry. If already connected, updates the current connection settings.""" self.log.debug('Checking for ServiceRegistry object...') if not self._sr: self.log.debug('Creating new ServiceRegistry object...') self._sr = ServiceRegistry(server=self._server, lazy=True, username=self.user, password=self.password) else: self.log.debug('Updating existing object...') self._sr.set_username(self.user) self._sr.set_password(self.password) def _setup_watchers(self): # For each watcher, see if we already have one for a given path or not. for service in self._config.sections(): w = self._get_watcher(service) # Gather up the config data for our section into a few local # variables so that we can shorten the statements below. command = self._config.get(service, 'cmd') service_port = self._config.get(service, 'service_port') zookeeper_path = self._config.get(service, 'zookeeper_path') refresh = self._config.get(service, 'refresh') # Gather our optional parameters. If they don't exist, set # some reasonable default. try: zookeeper_data = self._parse_data( self._config.get(service, 'zookeeper_data')) except: zookeeper_data = {} try: service_hostname = self._config.get( service, 'service_hostname') except: service_hostname = socket.getfqdn() if w: # Certain fields cannot be changed without destroying the # object and its registration with Zookeeper. if w._service_port != service_port or \ w._service_hostname != service_hostname or \ w._path != zookeeper_path: w.stop() w = None if w: # We already have a watcher for this service. Update its # object data, and let it keep running. w.set(command=command, data=zookeeper_data, refresh=refresh) # If there's still no 'w' returned (either _get_watcher failed, or # we noticed that certain un-updatable fields were changed, then # create a new object. if not w: w = ServiceWatcher(registry=self._sr, service=service, service_port=service_port, service_hostname=service_hostname, command=command, path=zookeeper_path, data=zookeeper_data, refresh=refresh) self._watchers.append(w) # Check if any watchers need to be destroyed because they're no longer # in our config. for w in self._watchers: if w._service not in list(self._config.sections()): w.stop() self._watchers.remove(w) def _get_watcher(self, service): """Returns a watcher based on the service name.""" for watcher in self._watchers: if watcher._service == service: return watcher return None def _parse_data(self, data): """Convert a string of data from ConfigParse into our dict. The zookeeper_data field supports one of two types of fields. Either a single key=value string, or a JSON-formatted set of key=value pairs: zookeeper_data: foo=bar zookeeper_data: foo=bar, bar=foo zookeeper_data: { "foo": "bar", "bar": "foo" } Args: data: String representing data above""" try: data_dict = json.loads(data) except: data_dict = {} for pair in data.split(','): if pair.split('=').__len__() == 2: key = pair.split('=')[0] value = pair.split('=')[1] data_dict[key] = value return data_dict def run(self): """Start up all of the worker threads and keep an eye on them""" self._setup_watchers() # Now, loop. Wait for a death signal while True and not self._event.is_set(): self._event.wait(1) # At this point we must be exiting. Kill off our above threads self.log.info('Shutting down') for w in self._watchers: self.log.info('Stopping watcher: %s' % w._path) w.stop() # Finally, mark us as done self.done = True def stop(self): self._event.set()
class ZKAdapter(BaseStoreAdapter): # TODO: move the logic to the logical layer # TODO: separate the reader and the writer to different classes, since # they're using different clients # nd object handles all of the connection states # there is no need to start/stop or monitor the connection state at all. nd = KazooServiceRegistry(server=settings.ZK_HOSTS, timeout=settings.ZK_CONNECTION_TIMEOUT, rate_limit_calls=None) @property def key_separator(self): return "/" def get_key(self, *suffixes): """ @suffixes: anything to be added after the prefix and version e.g. application name, key, segments ... etc. """ suffixes = map(lambda s: s.lower(), suffixes) path = super(ZKAdapter, self).get_key(*suffixes) # append a preceeding slash in the beginning, ZK specific format path = "/%s" % path return path def _check_data(self, node): try: data = node["data"] stat = node["stat"] except (TypeError, KeyError) as e: # node is False or malformed # getting the node details from ZK failed. raise ZKConnectionTimeoutError(e) # if stat is None, the node does not exist if stat is None: raise KeyDoesNotExistError return data def _get_children(self, key): node = self.nd.get(key) # check if the node exists self._check_data(node) return node["children"] def connect(self): self.zk = KazooClient(hosts=settings.ZK_HOSTS) try: self.zk.start(timeout=settings.ZK_CONNECTION_TIMEOUT) except TimeoutError: raise ZKConnectionTimeoutError def disconnect(self): self.zk.stop() self.zk.close() def get_applications(self): try: # self.get_key() without params will get the root node path # i.e. "/flags/v1/" apps = self._get_children(self.get_key()) except KeyDoesNotExistError: return [] return apps def get_all_keys(self, *path): return self._get_children(self.get_key(*path)) def get_all_features(self, application): keys = self.get_all_keys(application, settings.FEATURES_KEY) items = dict() for key in keys: items[key] = self.read_feature(application, key) return items def get_all_segments(self, application): segments = self.get_all_keys(application, settings.SEGMENTS_KEY) items = dict() for segment in segments: # Read the options in each segment items[segment] = self.get_all_keys( application, settings.SEGMENTS_KEY, segment ) return items def create_feature(self, application, key, value=None): # Ensure a path, create if necessary app_path = self.get_key(application, settings.FEATURES_KEY) self.zk.ensure_path(app_path) node_path = self.get_key(application, settings.FEATURES_KEY, key) # default segmentation if value is None: value = self._prepare_default_feature_dict(application) try: # Create a node with data self.zk.create(node_path, json.dumps(value)) except NodeExistsError: raise KeyExistsError def _prepare_default_feature_dict(self, application): # Create the segmentation segments = self._get_children(self.get_key( application, settings.SEGMENTS_KEY )) segmentation = { segment: { "toggled": settings.DEFAULT_VALUE, "options": { option: settings.DEFAULT_VALUE for option in self._get_children( self.get_key( application, settings.SEGMENTS_KEY, segment ) ) } } for segment in segments } return { "segmentation": segmentation, "feature_toggled": settings.DEFAULT_VALUE } def create_application(self, application): # ensure that /flags/v1 path exists root_path = self.get_key() self.zk.ensure_path(root_path) node_path = self.get_key(application) try: # Create the application node self.zk.create(node_path) # add the features and segmentation paths to the newly created app self.zk.ensure_path( self.get_key(application, settings.FEATURES_KEY) ) self.zk.ensure_path( self.get_key(application, settings.SEGMENTS_KEY) ) except NodeExistsError: raise KeyExistsError def create_segment(self, application, segment): # Ensure a path, create if necessary app_path = self.get_key(application, settings.SEGMENTS_KEY) self.zk.ensure_path(app_path) node_path = self.get_key(application, settings.SEGMENTS_KEY, segment) try: self.zk.create(node_path) # Update the segmentation of the existing features self._update_new_segment(application, segment) except NodeExistsError: raise KeyExistsError def _update_new_segment(self, application, segment): segment = segment.lower() features = self.get_all_keys(application, settings.FEATURES_KEY) for feature in features: feature_dict = self.read_feature(application, feature) feature_dict["segmentation"][segment] = { "toggled": settings.DEFAULT_VALUE, "options": dict() } self.update_feature(application, feature, feature_dict) def _update_new_segment_option(self, application, segment, option): segment = segment.lower() option = option.lower() features = self.get_all_keys(application, settings.FEATURES_KEY) for feature in features: feature_dict = self.read_feature(application, feature) feature_dict["segmentation"][segment]["options"][option] = settings.DEFAULT_VALUE self.update_feature(application, feature, feature_dict) def _update_deleted_segment_option(self, application, segment, option): segment = segment.lower() option = option.lower() features = self.get_all_keys(application, settings.FEATURES_KEY) for feature in features: feature_dict = self.read_feature(application, feature) del feature_dict["segmentation"][segment]["options"][option] self.update_feature(application, feature, feature_dict) def _update_deleted_segment(self, application, segment): segment = segment.lower() features = self.get_all_keys(application, settings.FEATURES_KEY) for feature in features: feature_dict = self.read_feature(application, feature) del feature_dict["segmentation"][segment] self.update_feature(application, feature, feature_dict) def create_segment_option(self, application, segment, option): # Ensure a path, create if necessary app_path = self.get_key(application, settings.SEGMENTS_KEY, segment) self.zk.ensure_path(app_path) node_path = self.get_key(application, settings.SEGMENTS_KEY, segment, option) try: self.zk.create(node_path) self._update_new_segment_option(application, segment, option) except NodeExistsError: raise KeyExistsError def read_feature(self, application, key): return self.read(application, settings.FEATURES_KEY, key) def read_segment(self, application, key): # TODO: is this needed? return self.read(application, settings.SEGMENTS_KEY, key) def read(self, *key_path): node = self.nd.get(path=self.get_key(*key_path)) data = self._check_data(node) return data def update_feature(self, application, key, value): node_path = self.get_key(application, settings.FEATURES_KEY, key) try: self.zk.set(node_path, json.dumps(value)) except NoNodeError: raise KeyDoesNotExistError def delete_feature(self, application, key): node_path = self.get_key(application, settings.FEATURES_KEY, key) try: self.zk.delete(node_path) except NoNodeError: raise KeyDoesNotExistError