def __init__(self): self.consul = Consul(host=ConsulConfig.host, port=ConsulConfig.port) self.consul_agent = self.consul.Agent(self.consul) self.consul_service = self.consul_agent.Service(self.consul) self.consul_check = self.consul_agent.Check(self.consul) ConsulConfig.service_host = gethostbyname(gethostname()) ConsulConfig.service_port = self.create_service_port() ConsulConfig.service_id = f"{ConsulConfig.service_name}-{uuid.uuid4()}"
def __init__(self): if Environment.is_prod_environment(): self.consul = Consul(host=Environment.consul_host(), port=Environment.consul_port(), token=Environment.consul_token(), scheme='http', verify=False) else: self.consul = Consul(host=Environment.consul_host(), port=Environment.consul_port(), scheme='http', verify=False) self.service_name = 'SchedulerService'
def init(self): consul_host = self.url.netloc or cfg.consul_host self.meta = Consul_KV.meta_cls( key=self.url.path.lstrip("/"), consul=Consul(host=consul_host), )
def _get_endpoint_from_consul(self, service_name): """ Look up an appropriate grpc endpoint (host, port) from consul, under the service name specified by service-name """ host = self.consul_endpoint.split(':')[0].strip() port = int(self.consul_endpoint.split(':')[1].strip()) while True: log.debug('consul-lookup', host=host, port=port) consul = Consul(host=host, port=port) _, services = consul.catalog.service(service_name) log.debug('consul-response', services=services) if services: break log.warning('no-service', consul_host=host, consul_port=port, service_name=service_name) yield asleep(1.0) # pick local addresses when resolving a service via consul # see CORD-815 (https://jira.opencord.org/browse/CORD-815) service = services[randint(0, len(services) - 1)] endpoint = '{}:{}'.format(service['ServiceAddress'], service['ServicePort']) returnValue(endpoint)
def GetCredentials(): VAULT_SERVER = getenv('VAULT_SERVER') VAULT_TOKEN = getenv('VAULT_TOKEN') CONSUL_SERVER = getenv('CONSUL_SERVER') client = Client(url="http://{}:8200".format(VAULT_SERVER), token=VAULT_TOKEN, verify=False) MONGODB_USER = client.read( "secret/devops-lab/app-devops")['data']['mongodb_user'] MONGODB_PASS = client.read( "secret/devops-lab/app-devops")['data']['mongodb_pass'] RABBIT_USER = client.read( "secret/devops-lab/app-devops")['data']['rabbitmq_user'] RABBIT_PASS = client.read( "secret/devops-lab/app-devops")['data']['rabbitmq_pass'] client = Consul(host=CONSUL_SERVER) index, data = client.kv.get('devops-lab', index=None, recurse=True, separator='lab') CONFIG = [{ "mongo": { "user": MONGODB_USER, "pass": MONGODB_PASS, "host": data[0]['Value'].decode("utf-8") } }, { "rabbit": { "user": RABBIT_USER, "pass": RABBIT_PASS, "host": data[1]['Value'].decode("utf-8") } }] return CONFIG
def HealthConsul(): client = Consul(host=getenv('CONSUL_SERVER')) health = client.health.service('consul')[1][0]['Checks'][0]['Status'] if health == 'passing': return "Healthy" else: return "Unhealthy"
def test_keep_alive(): """ Integration test for keeping service alive in consul cluster """ test_remove_services() session = Consul() tailf = get_tailf_service(session) # assert that there is no checks yet _, checks = session.health.service(tailf.service_name) assert not checks # test that service health check is unknown or critical after registration tailf.register() # small sleep for cluster consensus sleep(SLEEP_WAIT) assert _get_service_status(session, tailf) in ("unknown", "critical") # assert service is healthy back again after keep alive tailf.keep_alive() sleep(SLEEP_WAIT) assert _get_service_status(session, tailf) == "passing" # assert service health check fails after ttl passed sleep(tailf.ttl + SLEEP_WAIT) assert _get_service_status(session, tailf) == "critical"
def HealthConsul(): client = Consul(host='localhost') health = client.health.service('consul')[1][0]['Checks'][0]['Status'] if health == 'passing': return "Healthy" else: return "Unhealthy"
def __init__(self, host='127.0.0.1', port=8500, consistency='stale'): self.host = host self.port = port self.consistency = consistency self.client = Consul(host=self.host, port=self.port, consistency=self.consistency)
def generate_jobs(self, comment): self.add_comment(comment) jobs_list = {} config_api = ConfigAPI() job_path = config_api.get_consul_replication_jobs_path() consul = Consul() kv = KV() key, jobs_info = consul.kv.get(job_path, recurse=True) if jobs_info and len(jobs_info) > 0: for i in jobs_info: kv = KV() kv.load_json(json.dumps(i)) replication_job = ReplicationJob() replication_job.load_json(kv.Value) node = replication_job.node_name status = replication_job.status if node == self.node_name and status == 'started': jobs_list.update({replication_job.job_id: replication_job}) for job_id, job_info in jobs_list.iteritems(): print(job_id) cmd = "/opt/petasan/scripts/backups/replication.py run-replication-job --job_id " + str(job_info.job_id) self.add_job(job_info.schedule, cmd)
def test_ignore_connection_failures(): session = Consul(host="invalid") tailf = get_tailf_service(session) # assert service starts tailf.start() assert tailf.process.poll() is None with mock.patch('ianitor.service.logger') as logger: tailf.register() assert logger.error.called with mock.patch('ianitor.service.logger') as logger: tailf.keep_alive() assert logger.error.called with mock.patch('ianitor.service.logger') as logger: tailf.deregister() assert logger.error.called tailf.deregister() # assert service can be killed tailf.kill() tailf.process.wait()
def parse_consul(value): consul_config = json.loads(os.environ['CONSUL_CONFIG']) if 'CONSUL_CONFIG' in os.environ else {} scheme = consul_config['scheme'] if 'scheme' in consul_config else 'http' host = consul_config['host'] if 'host' in consul_config else '127.0.0.1' port = consul_config['port'] if 'port' in consul_config else 8500 token = consul_config['token'] if 'token' in consul_config else None consul_merger = Merger([(list, ['append']), (dict, ['merge'])], ['override'], ['override']) try: consul = Consul(scheme=scheme, host=host, port=port, token=token) _, kv_entries = consul.kv.get(recurse=True, key=value) except ACLPermissionDenied: print(BRIGHT_RED + 'Access denied connecting to: {}://{}:{}'.format(scheme, host, port) + RESET_ALL) return ERROR if not kv_entries: # Mark as failed if we can't find the consul key return ERROR consul_dict = {} for entry in kv_entries: subkeys = entry['Key'].split('/') value = entry['Value'].decode('utf-8') if hasattr(entry['Value'], 'decode') else entry['Value'] value = '' if value is None else value if '/' in entry['Key']: key = '{"' + entry['Key'].replace('/', '":{"') + '": "' + value + '"}'.ljust(len(subkeys)+1, '}') consul_dict = consul_merger.merge(consul_dict, json.loads(key)) else: consul_dict[entry['Key']] = value # return subkeys relative to rootkey rootkey = list(consul_dict.keys())[0] return consul_dict[rootkey]
def __init__(self): self.dont_ip = os.environ.get('DONT_IP', '').split(' ') self.other_repeaters = [] self.consul = Consul(sys.argv[1]) self.ctx = zmq.Context() self.poller = zmq.Poller() self.repeater_pub_port = REPEATER_PUB_PORT self.pub = self.ctx.socket(zmq.PUB) self.pub.bind('tcp://*:%d' % self.repeater_pub_port) self.sub = self.ctx.socket(zmq.SUB) self.sub.setsockopt(zmq.SUBSCRIBE, b'') self.poller.register(self.sub, zmq.POLLIN) self.beacon = ZActor(self.ctx, ZBeacon) self.beacon.send_unicode('CONFIGURE', zmq.SNDMORE) self.beacon.send(struct.pack('I', ZRE_DISCOVERY_PORT)) self.address = self.beacon.recv_unicode() # Hostname filter_ = struct.pack('ccc', b'Z', b'R', b'E') self.beacon.send_unicode('SUBSCRIBE',zmq.SNDMORE) self.beacon.send(filter_) self.beacon_socket = self.beacon.resolve() self.poller.register(self.beacon_socket, zmq.POLLIN)
def connect_to_consult(consul_endpoint): log.debug('getting-service-endpoint', consul=consul_endpoint) host = consul_endpoint.split(':')[0].strip() port = int(consul_endpoint.split(':')[1].strip()) return Consul(host=host, port=port)
def __init__(self): self.prefix = "gluu/config/" token = None cert = None verify = False if os.path.isfile(GLUU_CONSUL_TOKEN_FILE): with open(GLUU_CONSUL_TOKEN_FILE) as fr: token = fr.read().strip() if GLUU_CONSUL_SCHEME == "https": verify = as_boolean(GLUU_CONSUL_VERIFY) # verify using CA cert (if any) if verify and os.path.isfile(GLUU_CONSUL_CACERT_FILE): verify = GLUU_CONSUL_CACERT_FILE if all([ os.path.isfile(GLUU_CONSUL_CERT_FILE), os.path.isfile(GLUU_CONSUL_KEY_FILE) ]): cert = (GLUU_CONSUL_CERT_FILE, GLUU_CONSUL_KEY_FILE) self._request_warning(GLUU_CONSUL_SCHEME, verify) self.client = Consul( host=GLUU_CONSUL_HOST, port=GLUU_CONSUL_PORT, token=token, scheme=GLUU_CONSUL_SCHEME, consistency=GLUU_CONSUL_CONSISTENCY, verify=verify, cert=cert, )
def register_plugin(): """ Registers this plugin with Consul :return: None """ logger.info("Registering plugin with Consul") consul_host = getenv("CONSUL_HOST", "localhost") consul_port = getenv("CONSUL_PORT", 8500) c = Consul(host=consul_host, port=consul_port) hname = socket.gethostname() ipaddr = socket.gethostbyname(hname) health_check = Check.http(url=f'http://{ipaddr}:{port}/config', interval="20s", deregister=True) service_name = config['serviceName'] service_id = f'{service_name}-{str(uuid.uuid4())}' c.agent.service.register(name=service_name, service_id=service_id, address=ipaddr, port=port, tags=['secure=false'], check=health_check) atexit.register(deregister, c, service_id)
def __init__(self, context, options): super().__init__(context, options) self._consul = Consul( scheme=self.options.scheme, host=self.options.host, port=self.options.port, timeout=self.options.timeout, )
def run( keys: List[Key], exist_ok: bool = False, ): consul = Consul() if not exist_ok and multi_get(consul, keys): raise TribunError("Existing keys")
def __init__(self, host=None, port=None, token=None): # 初始化,指定consul主机,端口,和token self.host = host # consul 主机 self.port = port # consul 端口 self.token = token self.consul = Consul(host=host, port=port)
def register_service(service_name: str, *, address: str = None, port: int = None): consul = Consul() if address is None: address = get_local_ip() consul.agent.service.register(service_name, address=address, port=port)
def __init__(self, host, port, path_prefix): self.log = structlog.get_logger() self._consul = Consul(host=host, port=port) self.host = host self.port = port self._path_prefix = path_prefix self.retries = 0
def __init__(self, options, hostname): self.options = options self.consul = Consul(host=options.consul_host, port=options.consul_port) self.service_name = options.app self.hostname = hostname self.service_id = _make_service_id(options, service_name=self.service_name, hostname=self.hostname)
def test_service_start(): session = Consul() tailf = get_tailf_service(session) with mock.patch.object(service.Service, "register") as register_method: tailf.start() assert bool(tailf.is_up()) register_method.assert_any_call()
def _is_myservice_registered_to_consul(self, ip): port = self.service_port(8500, 'consul') consul = Consul('localhost', port, 'the_one_ring') services = consul.agent.services() for index, service in services.items(): if service['Service'] == 'myservice' and service['Address'] == ip: return True return False
def registry_factory(self) -> Consul: if self.server == None: raise RegistryError("load etcd server error") address = self.server.address if address: return Consul(host=address.split(",")[0].split(":")[0], port=int(address.split(",")[0].split(":")[1])) else: raise RegistryError("load etcd server error")
def __init__(self): self.service_name = django_config_dic['SERVICE_NAME'] self.host = django_config_dic['ip'] # django运行 主机 self.port = django_config_dic['port'] # django运行 端口 print( f"ConsulConf类 __init__()方法开始初始化Consul连接,连接信息:服务名: {self.service_name} IP: {self.host} 端口号:{self.port}" ) self.consul = Consul(host=consul_config_dic['ip'], port=consul_config_dic['port']) print("成功连接到Consul!")
def register_consul(address, port): c = Consul(host='10.10.6.11') c.agent.service.register('hadoop_exporter_dev', service_id='hadoop_exporter_dev2', address='indata-10-10-6-11.indata.com', port=port, tags=['hadoop']) start_http_server(port) # print("Polling %s. Serving at port: %s" % (args.address, port)) print("Polling %s. Serving at port: %s" % (address, port))
def __init__(self, context, options): super().__init__(context, options) self._session_id = None self._refresh_lock_flag = False self._consul = Consul( scheme=self.options.scheme, host=self.options.host, port=self.options.port, timeout=self.options.timeout, )
def get_service(service_id): consul = Consul(host="consul", port=consul_port) agent = consul.agent service_list = agent.services() service_info = service_list[service_id] return service_info['Address'], service_info['Port']
def test_service_register(): session = Consul() agent = session.agent tailf = get_tailf_service(session) tailf.start() test_remove_services() tailf.register() assert agent.services()
if environ.get('USE_ENV_VARS') == 'TRUE': try: # use local settings for env_key, value in config.iteritems(): if not value: config[env_key] = environ[env_key] except KeyError as e: """ Throw an error if a setting is missing """ print "ERR MSG: {}".format(e.message) print ("Some of your settings aren't in the environment." "You probably need to run:" "\n\n\tsource config/<your settings file>") exit(1) else: # use consul kv = Consul().kv # initalize client to KV store for consul_key, config_key in consul_configurations: _, tmp = kv.get("density/{}".format(consul_key)) val = tmp.get('Value') config[config_key] = val if not val: raise Exception(("no value found in Consul for key " "density/{}").format(consul_key)) # mail settings config['MAIL_SERVER'] = 'smtp.gmail.com' config['MAIL_PORT'] = 465 config['MAIL_USE_SSL'] = True config['MAIL_USE_TLS'] = False config['MAIL_DEFAULT_SENDER'] = '*****@*****.**'
if environ.get('USE_ENV_VARS') == 'TRUE': try: # use local settings for env_key, value in config.iteritems(): if not value: config[env_key] = environ[env_key] except KeyError as e: """ Throw an error if a setting is missing """ print "ERR MSG: {}".format(e.message) print ("Some of your settings aren't in the environment." "You probably need to run:" "\n\n\tsource config/<your settings file>") exit(1) else: # use consul kv = Consul().kv # initalize client to KV store for consul_key, config_key in consul_configurations: _, tmp = kv.get("density/{}".format(consul_key)) val = tmp.get('Value') config[config_key] = val if not val: raise Exception(("no value found in Consul for key " "density/{}").format(consul_key)) # mail settings config.update({ 'MAIL_SERVER': 'smtp.googlemail.com', 'MAIL_PORT': 465, 'MAIL_USE_SSL': True, 'MAIL_USE_TLS': False,
'INSTALLED_APP_CREDENTIALS_PATH': 'config/installed_app_credentials.json', 'CLIENT_SECRETS_PATH': 'config/client_secrets.json', 'PRIVATE_CALENDAR_ID': None, 'PUBLIC_CALENDAR_ID': None, # MongoDB configurations 'MONGO_DATABASE': 'eventum', # Logging configurations 'LOG_FILE_MAX_SIZE': '256', 'APP_LOG_NAME': 'app.log', 'WERKZEUG_LOG_NAME': 'werkzeug.log', } from consul import Consul kv = Consul().kv # initalize client to KV store # get values from Consul and set the corresponding config variable to the # value retrieved from Consul. for key, value in config.iteritems(): try: _, consul_value = kv.get(("adi-website/{}").format(key.lower())) except requests.ConnectionError: raise Exception('Failed to connect to Consul. You probably need to ' 'run: \n\n\t./config/run_consul.sh') # We have a good value in Consul if consul_value and consul_value.get('Value'): config[key] = consul_value.get('Value') continue