def test_exists(): tendrlNS = init() with patch.object(__builtin__, 'hasattr', has_attr) as mock_hasattr: obj = BaseObject_Child() obj._ns = tendrlNS with patch.object(TendrlNS, 'get_obj_definition', obj_definition) as mock_obj_definition: obj._defs = obj.load_definition() obj.value = 'Test' with patch.object(Client, "write", return_value=True) as mock_write: with patch.object(Client, "read", return_value=maps.NamedDict( value="Test")) as mock_read: obj.exists() NS._int.client = etcd.Client() NS._int.reconnect = type("Dummy", (object, ), {}) with patch.object(Client, "read", read) as mock_read: with pytest.raises(etcd.EtcdConnectionFailed): obj.exists()
def test_constructor(self, monkeypatch): def mock_config_get(package, parameter): if parameter == "etcd_port": return 2379 elif parameter == "etcd_connection": return "0.0.0.0" monkeypatch.setattr(config, 'get', mock_config_get) def mock_uuid4(): return 'aa22a6fe-87f0-45cf-8b70-2d0ff4c02af6' monkeypatch.setattr(uuid, 'uuid4', mock_uuid4) server = EtcdRPC() assert server.bridge_id == 'aa22a6fe-87f0-45cf-8b70-2d0ff4c02af6' local_client = etcd.Client( port=2379, host="0.0.0.0" ) assert server.client.port == local_client.port assert server.client.host == local_client.host
def main_loop(): etcd_host = os.environ.get('ETCD_HOST', 'etcd') etcd_port = int(os.environ.get('ETCD_PORT', '2379')) base_dir = os.environ.get('BASE_DIR', '/services') interval = int(os.environ.get('UPDATE_INTERVAL', '120')) infinite_run = os.environ.get('RUN_ONCE', 'False') == 'False' print('etcd: {0}:{1}, base_dir: {2}, run_once: {3}, update_interval: {4}'. format(etcd_host, etcd_port, base_dir, not infinite_run, interval)) etcd_client = etcd.Client(host=etcd_host, port=etcd_port) docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock') while infinite_run: try: update(etcd_client, docker_client, base_dir) except: print("Failed to update services: ", sys.exc_info()[0]) time.sleep(interval)
def test_setup_common_objects(monkeypatch): tendrlNS = init() obj = importlib.import_module("tendrl.commons.tests.fixtures.config") for obj_cls in inspect.getmembers(obj, inspect.isclass): tendrlNS.current_ns.objects["Config"] = obj_cls[1] with patch.object(etcd, "Client", return_value=etcd.Client()) as client: tendrlNS.current_ns.objects.pop("NodeContext") NS.tendrl.objects.TendrlContext.load = MagicMock() tendrlNS.setup_common_objects() assert NS._int.client is not None assert NS._int.wclient is not None etcd.Client.assert_called_with(host=1, port=1) tendrlNS.current_ns.objects.pop("TendrlContext") tendrlNS.setup_common_objects() def client(**param): raise Exception monkeypatch.setattr(etcd, 'Client', client) with pytest.raises(Exception): tendrlNS.setup_common_objects()
def test_delete(self): """ Can delete a value """ client = etcd.Client() client.api_execute = mock.Mock( return_value=FakeHTTPResponse(200, '{"action":"DELETE",' '"node": {' '"key":"/testkey",' '"prevValue":"test",' '"expiration":"2013-09-14T01:06:35.5242587+02:00",' '"modifiedIndex":189}}') ) result = client.delete('/testkey') self.assertEquals(etcd.EtcdResult( **{u'action': u'DELETE', u'node': { u'expiration': u'2013-09-14T01:06:35.5242587+02:00', u'modifiedIndex': 189, u'key': u'/testkey', u'prevValue': u'test'} }), result)
def test_get(self): """ Can get a value """ client = etcd.Client() client.api_execute = mock.Mock(return_value=FakeHTTPResponse( 200, '{"action":"GET",' '"node": {' '"key":"/testkey",' '"value":"test",' '"modifiedIndex":190}}')) result = client.get('/testkey') self.assertEquals( etcd.EtcdResult( **{ u'action': u'GET', u'node': { u'modifiedIndex': 190, u'key': u'/testkey', u'value': u'test' } }), result)
def test_get_set_authenticated(self): """ INTEGRATION: connecting to server with mutual auth """ # This gives an unexplicable ssl error, as connecting to the same # Etcd cluster where this fails with the exact same code this # doesn't fail client = etcd.Client( port=6001, protocol='https', cert=self.client_all_cert, ca_cert=self.ca_cert_path ) set_result = client.set('/test_set', 'test-key') self.assertEquals(u'set', set_result.action.lower()) self.assertEquals(u'/test_set', set_result.key) self.assertEquals(u'test-key', set_result.value) get_result = client.get('/test_set') self.assertEquals('get', get_result.action.lower()) self.assertEquals('/test_set', get_result.key) self.assertEquals('test-key', get_result.value)
def __init__(self): super(EtcdAgentCommunicator, self).__init__() self.etcd_client = etcd.Client() # TODO(ijw): give this args # We need certain directories to exist self.do_etcd_mkdir(LEADIN + '/state') self.do_etcd_mkdir(LEADIN + '/nodes') # TODO(ijw): .../state/<host> lists all known hosts, and they # heartbeat when they're functioning # Get the physnets the agents know about. This is updated # periodically in the return thread below. self.physical_networks = set() self._find_physnets() self.db_q_ev = eventlet.event.Event() self.return_thread = eventlet.spawn(self._return_worker) self.forward_thread = eventlet.spawn(self._forward_worker)
def get_services(): host, port = get_etcd_addr() client = etcd.Client(host=host, port=int(port)) backends = client.read('/backends', recursive = True) services = {} for i in backends.children: if i.key[1:].count("/") != 2: continue ignore, service, container = i.key[1:].split("/") endpoints = services.setdefault(service, dict(port="", backends=[])) if container == "port": endpoints["port"] = i.value continue endpoints["backends"].append(dict(name=container, addr=i.value)) for service in services: services[service]["backends"].sort() return services
def __init__(self, host: str, port: int, base: str, timeout: int = 60, callbacks: Callbacks = None): super().__init__() self._callbacks = [] if isinstance(callbacks, (list, tuple)): self._callbacks.extend([f for f in callbacks if isFunction(f)]) self._data = QueryDict() self._etcd_client = etcd.Client(host=host, port=port) self._root = base if base.endswith('/') else f'{base}/' self._timeout = timeout self._initial() self._etcd_updater = ReusableThread(target=self._update, kwargs={'wait': True}) self._etcd_updater.setDaemon(True) self._etcd_updater.start()
def main(): client = etcd.Client() sleep_time = 3 wait_index = 0 message = None messages = Queue() # create a thread worker = Thread(target=process_queue, args=(messages,)) worker.setDaemon(True) worker.start() while True: logging.info("watching on waitIndex %s" % wait_index) try: if wait_index: message = client.read('/nuage_shim-gluon-test', recursive=True, wait=True, waitIndex=wait_index) else: message = client.read('/nuage_shim-gluon-test', recursive=True, wait=True) messages.put(message.value) #logging.info("message received %s" % message.value) except KeyboardInterrupt: logging.info("exiting on interrupt") exit(1) except: pass #logging.info("sleeping for %s seconds" % sleep_time) #time.sleep(sleep_time) if (message.modifiedIndex - wait_index) > 1000: wait_index = 0 else: wait_index = message.modifiedIndex + 1
def __init__(self): super(EtcdAgentCommunicator, self).__init__() LOG.debug("Using etcd host:%s port:%s user:%s password:***" % (cfg.CONF.ml2_vpp.etcd_host, cfg.CONF.ml2_vpp.etcd_port, cfg.CONF.ml2_vpp.etcd_user,)) host = nwvpp_utils.parse_host_config(cfg.CONF.ml2_vpp.etcd_host, cfg.CONF.ml2_vpp.etcd_port) self.etcd_client = etcd.Client(host=host, username=cfg.CONF.ml2_vpp.etcd_user, password=cfg.CONF.ml2_vpp.etcd_pass, allow_reconnect=True) # We need certain directories to exist self.state_key_space = LEADIN + '/state' self.port_key_space = LEADIN + '/nodes' self.secgroup_key_space = LEADIN + '/global/secgroups' self.election_key_space = LEADIN + '/election' self.do_etcd_mkdir(self.state_key_space) self.do_etcd_mkdir(self.port_key_space) self.do_etcd_mkdir(self.secgroup_key_space) self.do_etcd_mkdir(self.election_key_space) self.secgroup_enabled = cfg.CONF.SECURITYGROUP.enable_security_group if self.secgroup_enabled: self.register_secgroup_event_handler() # TODO(ijw): .../state/<host> lists all known hosts, and they # heartbeat when they're functioning self.db_q_ev = eventlet.event.Event() try: # Liberty, Mitaka ev = events.AFTER_INIT except Exception: # Newton and on ev = events.AFTER_CREATE # Clear any previously elected master keys from the election key space EtcdHelper(self.etcd_client).clear_state(self.election_key_space) registry.subscribe(self.start_threads, resources.PROCESS, ev)
def start(self, timeout=60): """ starts the server and blocks for timeout seconds until the server is ready. """ # etcd opens two ports: one for clients, and another for peers client_bind_sock = _get_socket_with_port() peer_bind_sock = _get_socket_with_port() with closing(client_bind_sock), closing(peer_bind_sock): self._client_port = client_bind_sock.getsockname()[1] self._peer_port = peer_bind_sock.getsockname()[1] # FIXME: this has a race condition here. Explore a more reliable # way to pass a listening port to a separate process. client_bind_sock.close() peer_bind_sock.close() self._subprocess = subprocess.Popen( self._get_etcd_cmd(), stdin=subprocess.PIPE, stdout=self.file, stderr=self.file, close_fds=True, ) log.info( f"Starting etcd server, tmp dir: {self.tmpdir} " f"host: {self.get_host()}, port: {self.get_port()}" ) self._client = etcd.Client( host=self.get_host(), port=self.get_port(), version_prefix="/v2", read_timeout=timeout, ) # ask etcd server for version until server is ready version = self._get_version(timeout) log.info(f"etcd server v{version} is running")
def _read_etcd_config_key(key, json_object): """ Tries to read from etcd as described in read_config_file(). """ # Log relevant environment variables. # We know we have at least ETCD_MACHINES. def maybe_print_environ(key, secret=False): if key in os.environ: value = '(redacted)' if secret else repr(os.environ[key]) print(' {}={}'.format(key, value)) else: print(' {} not defined'.format(key)) print('Environment variables:') maybe_print_environ('ETCD_MACHINES') maybe_print_environ('ETCD_TLSPEM') maybe_print_environ('ETCD_TLSKEY') maybe_print_environ('ETCD_CACERT') maybe_print_environ('ETCD_USERNAME') maybe_print_environ('ETCD_PASSWORD', secret=True) try: client = etcd.Client(**etcd_client_args()) json_object.update(json.loads(client.get(key).value)) print('Using configuration in etcd at "{}"'.format(key)) except etcd.EtcdConnectionFailed as error: print( 'Connection to etcd failed: {}'.format(error.args[0]), file=sys.stderr) except etcd.EtcdKeyNotFound: print( 'Missing etcd configuration key "{}"'.format(key), file=sys.stderr) except json.JSONDecodeError as error: print( 'Skipping invalid configuration in etcd key "{}": {}'.format( key, error.args[0]), file=sys.stderr) except (TypeError, ValueError): print( 'Skipping invalid configuration in etcd key "{}": {}'.format( key, 'Content must be a JSON object'), file=sys.stderr)
def __init__(self): self.provisioner_only_plugin = True TendrlGlusterfsMonitoringBase.__init__(self) self.brick_path_separator = ":" if not self.etcd_client: _etcd_args = dict(host=self.CONFIG['etcd_host'], port=int(self.CONFIG['etcd_port'])) etcd_ca_cert_file = self.CONFIG.get("etcd_ca_cert_file") etcd_cert_file = self.CONFIG.get("etcd_cert_file") etcd_key_file = self.CONFIG.get("etcd_key_file") if (etcd_ca_cert_file and str(etcd_ca_cert_file) != "" and etcd_cert_file and str(etcd_cert_file) != "" and etcd_key_file and str(etcd_key_file) != ""): _etcd_args.update({ "ca_cert": str(self.CONFIG['etcd_ca_cert_file']), "cert": (str(self.CONFIG['etcd_cert_file']), str(self.CONFIG['etcd_key_file'])), "protocol": "https" }) self.etcd_client = etcd.Client(**_etcd_args)
def test_investigator(self): """ Verify the investigator. """ with mock.patch('commissaire.transport.ansibleapi.Transport') as _tp: _tp().get_info.return_value = (0, { 'os': 'fedora', 'cpus': 2, 'memory': 11989228, 'space': 487652, }) q = Queue() client = etcd.Client() client.get = MagicMock('get') client.get.return_value = MagicMock(value=self.etcd_host) client.set = MagicMock('set') client.set.return_value = self.etcd_host to_investigate = { 'address': '10.0.0.2', } ssh_priv_key = 'dGVzdAo=' connection_config = { 'etcd': { 'uri': urlparse('http://127.0.0.1:2379'), }, 'kubernetes': { 'uri': urlparse('http://127.0.0.1:8080'), 'token': 'token', } } q.put_nowait((to_investigate, ssh_priv_key)) investigator(q, connection_config, client, True) self.assertEquals(1, client.get.call_count) self.assertEquals(2, client.set.call_count)
def update_key(request, ecsn=None): try: print(ecsn) ec = EtcdCluster.objects.get(serial_number=ecsn) ec_endpoint = parseURL(ec.cluster_endpoint) print(ec_endpoint) eClient = etcd.Client(host=ec_endpoint['host'], port=ec_endpoint['port'], protocol=ec_endpoint['scheme'], allow_reconnect=True) key = request.GET.get('key') value = request.GET.get('value') try: eClient.update(key, value) except etcd.EtcdException: print("etcd key update error.") except EtcdCluster.DoesNotExist: print("etcd cluster is not found.") return render(request, 'update_key.html', locals())
def __init__(self, opts, profile=None): opts_pillar = opts.get('pillar', {}) opts_master = opts_pillar.get('master', {}) opts_merged = {} opts_merged.update(opts_master) opts_merged.update(opts_pillar) opts_merged.update(opts) if profile: self.conf = opts_merged.get(profile, {}) else: self.conf = opts_merged host = self.conf.get('etcd.host', '127.0.0.1') port = self.conf.get('etcd.port', 4001) if HAS_LIBS: self.client = etcd.Client(host, port) else: raise CommandExecutionError('(unable to import etcd, ' 'module most likely not installed)')
def __init__(self, machine_id): self._complete = gevent.event.Event() # Initialize the state sync thread which gets the underlying # node details and pushes the same to etcd etcd_kwargs = { 'port': int(config.get("commons", "etcd_port")), 'host': config.get("commons", "etcd_connection") } self.etcd_client = etcd.Client(**etcd_kwargs) local_node_context = utils.set_local_node_context() if local_node_context: if utils.get_node_context(self.etcd_client, local_node_context) \ is None: utils.delete_local_node_context() super(NodeAgentManager, self).__init__("node", utils.get_local_node_context(), utils.get_local_node_context(), config, NodeAgentSyncStateThread(self), NodeAgentEtcdPersister(config), "/tendrl_definitions_node-agent/data") self.register_node(machine_id)
def __init__(self, root: str = ROOT_DIR, act_id: str = ACT_ID) -> None: """ Initialize and start the Actuator Process :param root: rood directory of actuator - default CWD :param act_id: id of the actuator - default UUIDv4 """ config_file = os.path.join(root, "config.json") schema_file = os.path.join(root, "schema.json") config = general.safe_load(config_file) if "actuator_id" not in config.keys(): config.setdefault("actuator_id", act_id) json.dump(config, open(config_file, "w"), indent=4) # Initialize etcd client self._etcd = etcd.Client( host=os.environ.get('ETCD_HOST', 'etcd'), port=safe_cast(os.environ.get('ETCD_PORT', 4001), int, 4001) ) schema = general.safe_load(schema_file) self._config = FrozenDict( **config, schema=schema ) self._dispatch = dispatch.Dispatch(act=self, dispatch_transform=self._dispatch_transform) self._dispatch.register(exceptions.action_not_implemented, "default") # Get valid Actions & Targets from the schema self._profile = self._config.schema.get("title", "N/A").replace(" ", "_").lower() self._validator = general.ValidatorJSON(schema) schema_defs = self._config.schema.get("definitions", {}) profiles = self.nsid if len(self.nsid) > 0 else [self._profile] for profile in profiles: self._etcd.write(f"{self._prefix}/{profile}", self._config.actuator_id) self._valid_actions = tuple(a["const"] for a in schema_defs.get("Action", {}).get("oneOf", [])) self._valid_targets = tuple(schema_defs.get("Target", {}).get("properties", {}).keys())
def Main(args=sys.argv[1:]): parser = argparse.ArgumentParser() parser.add_argument("--etcd", default="localhost") parser.add_argument("--etcd-prefix", default="/services/elasticsearch") parser.add_argument("--node-name") parser.add_argument("--master-node", action="store_true") parser.add_argument("--data-node", action="store_true") parser.add_argument("--publish-host") options = parser.parse_args(args) client = etcd.Client(host=options.etcd) while True: command = GetCommand(options, client=client) if command: break client.read(options.etcd_prefix, recursive=True, wait=True, waitIndex=0) os.execv(command[0], command)
def add_host(): try: logging.debug('add_host') params = None if request.method == 'GET': params = request.args if request.method == 'POST': if request.json: params = request.json else: return 'FAIL\n' done = False host_name = params.get('host_name', settings.default_host) host = params.get('host', settings.default_host) port = params.get('port', settings.default_port) cpus = params.get('cpus', settings.default_cpus_host) memory = params.get('memory', settings.default_memory_host) disk = params.get('disk', settings.default_disk_host) data_list = {} data_list['host'] = host data_list['port'] = port data_list['cpus'] = cpus data_list['memory'] = memory data_list['disk'] = disk data_list['containers'] = [] data = json.dumps(data_list) etcd_client = etcd.Client(host=settings.etcd_host, port=settings.etcd_port) etcd_client.write( settings.etcd_prefix + settings.etcd_hosts_prefix + host_name, data) done = True logging.debug('host ' + host_name + ' added') if request.method == 'GET': return render_template('add_host.html', done=done) return 'OK\n' except Exception as e: logging.error(e, exc_info=True) pass
def __init__(self, host, protocol, cert, ca_cert): """ Initialize the client. Args: host (mixed): a tuple ((host, port), (host, port), ...) protocol (str): Protocol used to connect to etcd. cert (mixed): If a string, the whole ssl client certificate; if a tuple, the cert and key file names. ca_cert (str): The ca certificate. If pressent it will enable validation. """ import etcd self.client = etcd.Client(host=host, protocol=protocol, cert=cert, ca_cert=ca_cert, allow_reconnect=True)
def test__set_version_info(self): """Verify _set_version_info makes the proper call to the server""" with mock.patch('urllib3.PoolManager') as _pm: _request = _pm().request # Return the expected data type _request.return_value = mock.MagicMock( data=b'{"etcdserver": "2.2.3", "etcdcluster": "2.3.0"}') # Create the client and make the call. client = etcd.Client() client._set_version_info() # Verify we call the proper endpoint _request.assert_called_once_with(client._MGET, client._base_uri + '/version', headers=mock.ANY, redirect=mock.ANY, timeout=mock.ANY) # Verify the properties while we are here self.assertEquals('2.2.3', client.version) self.assertEquals('2.3.0', client.cluster_version)
def __init__(self, etcd_options, service_name, service_addr): """ :param etcd_options: Options of etcd. :param service_name: Name of current service. :param service_addr: Address of current service. :type etcd_options: dict :type service_name: str :type service_addr: dict :return: """ super(OctpServer, self).__init__() self.etcd_options = etcd_options self.service_name = service_name self.service_addr = service_addr self.ec = etcd.Client(**self.etcd_options) self._token = '' self._watcher_co = None self._heartbeat_co = None
def __init__(self, host='127.0.0.1', port=4001, srv_domain=None, version_prefix='/v2', read_timeout=60, allow_redirect=True, protocol='http', cert=None, ca_cert=None, username=None, password=None, allow_reconnect=False, use_proxies=False, expected_cluster_id=None, per_host_pool_size=10 ): client = etcd.Client( host='127.0.0.1', port=4003, allow_reconnect=True, protocol='https', )
def get_etcd_peer_ip_list(self, bad_node_config): etcd_cluster_ips_peer = "" separated = "" host_list = list() for host in self.cluster_config['mastermachinelist']: host_list.append( (self.cluster_config['mastermachinelist'][host]['hostip'], 4001)) client = etcd.Client(host=tuple(host_list), allow_reconnect=True) member_dict = client.members for member_hash in member_dict: etcd_id = member_dict[member_hash]['name'] peer_url = member_dict[member_hash]['peerURLs'][0] if etcd_id == "": # new member before announcing, etcdid will be empty. continue ip_peer = "{0}={1}".format(etcd_id, peer_url) etcd_cluster_ips_peer = etcd_cluster_ips_peer + separated + ip_peer separated = "," new_etcd_id = bad_node_config['etcdid'] peer_url = bad_node_config['hostip'] ip_peer = "{0}=http://{1}:2380".format(new_etcd_id, peer_url) etcd_cluster_ips_peer = etcd_cluster_ips_peer + separated + ip_peer self.logger.debug( "New etcd-initial-cluster: {0}".format(etcd_cluster_ips_peer)) return etcd_cluster_ips_peer
def main(): # pragma: no cover import sys import urlparse # TODO: Use argparse try: etcd_uri = urlparse.urlparse(sys.argv[1]) except: sys.stdout.write( 'You must provide an etcd url. EX: http://127.0.0.1:2379\n') raise SystemExit(1) ds = etcd.Client(host=etcd_uri.hostname, port=etcd_uri.port) try: logging.config.dictConfig( json.loads(ds.get('/commissaire/config/logger').value)) logging.info('Using Etcd for logging configuration.') except etcd.EtcdKeyNotFound: with open('./conf/logger.json', 'r') as logging_default_cfg: logging.config.dictConfig(json.loads(logging_default_cfg.read())) logging.warn('No logger configuration in Etcd. Using defaults.') except etcd.EtcdConnectionFailed as ecf: err = 'Unable to connect to Etcd: {0}. Exiting ...'.format(ecf) logging.fatal(err) sys.stderr.write('{0}\n'.format(err)) raise SystemExit(1) POOLS['investigator'].spawn(investigator, INVESTIGATE_QUEUE, ds) # watch_thread = gevent.spawn(host_watcher, ROUTER_QUEUE, ds) # router_thread = gevent.spawn(router, ROUTER_QUEUE) app = create_app(ds) try: WSGIServer(('127.0.0.1', 8000), app).serve_forever() except KeyboardInterrupt: pass POOLS['investigator'].kill()
def get_services(): host, port = get_etcd_addr() client = etcd.Client(host=host, port=int(port)) backends = client.read('/backends', recursive=True) services = {} for i in backends.children: print('children: {i}'.format(i=i)) if i.key[1:].count("/") == 3: _, service, version, container = i.key[1:].split("/") else: print('skipping: {i}'.format(i=i)) print('/ = {count}'.format(count=i.key[1:].count("/"))) continue service_name = service + '_' + version path = '/' + service + '/' + version print('service_name: {srv}'.format(srv=service_name)) print('version: {vers}'.format(vers=version)) print('path: {path}'.format(path=path)) print('container: {cont}'.format(cont=container)) endpoints = services.setdefault( service_name, dict(path=path, weight=100, backends=[])) if container == "weight": endpoints["weight"] = i.value continue endpoints["backends"].append(dict(name=container, addr=i.value)) print('endpoints: {ep}'.format(ep=endpoints)) print("services: {srv}".format(srv=services)) return services
def set_etcd_key(key, value, key_path, host=ETCD_HOST, port=ETCD_PORT, **kwargs): """ 在指定目录下添加或更新key-value :param key: :param value: :param key_path: 指定目录 :param kwargs: 其他可选参数, 更多详细说明请见官方文档 {ttl: int, # 过期时间 dir: bool, # 是否为目录, 默认为False append: bool, # 是否根据value自动创建新的key, 默认为False prevValue: str, # compare key to this value, and swap only if corresponding prevIndex: int, # modify key only if actual modifiedIndex matches the provided one prevExist: bool, # If false, only create key; if true, only update key ...} :return: """ if not key_path.endswith("/"): key_path += "/" client = etcd.Client(host=host, port=port, allow_redirect=False) print "key:", key, "value:", value client.write("{0}{1}".format(key_path, key), value, kwargs)