def __init__(self):
self.consul = Consul(host=ConsulConfig.host, port=ConsulConfig.port)
self.consul_agent = self.consul.Agent(self.consul)
self.consul_service = self.consul_agent.Service(self.consul)
self.consul_check = self.consul_agent.Check(self.consul)
ConsulConfig.service_host = gethostbyname(gethostname())
ConsulConfig.service_port = self.create_service_port()
ConsulConfig.service_id = f"{ConsulConfig.service_name}-{uuid.uuid4()}"
def __init__(self):
if Environment.is_prod_environment():
self.consul = Consul(host=Environment.consul_host(),
port=Environment.consul_port(),
token=Environment.consul_token(),
scheme='http',
verify=False)
else:
self.consul = Consul(host=Environment.consul_host(),
port=Environment.consul_port(),
scheme='http',
verify=False)
self.service_name = 'SchedulerService'
def init(self):
consul_host = self.url.netloc or cfg.consul_host
self.meta = Consul_KV.meta_cls(
key=self.url.path.lstrip("/"),
consul=Consul(host=consul_host),
)
def _get_endpoint_from_consul(self, service_name):
"""
Look up an appropriate grpc endpoint (host, port) from
consul, under the service name specified by service-name
"""
host = self.consul_endpoint.split(':')[0].strip()
port = int(self.consul_endpoint.split(':')[1].strip())
while True:
log.debug('consul-lookup', host=host, port=port)
consul = Consul(host=host, port=port)
_, services = consul.catalog.service(service_name)
log.debug('consul-response', services=services)
if services:
break
log.warning('no-service', consul_host=host, consul_port=port,
service_name=service_name)
yield asleep(1.0)
# pick local addresses when resolving a service via consul
# see CORD-815 (https://jira.opencord.org/browse/CORD-815)
service = services[randint(0, len(services) - 1)]
endpoint = '{}:{}'.format(service['ServiceAddress'],
service['ServicePort'])
returnValue(endpoint)
def GetCredentials():
VAULT_SERVER = getenv('VAULT_SERVER')
VAULT_TOKEN = getenv('VAULT_TOKEN')
CONSUL_SERVER = getenv('CONSUL_SERVER')
client = Client(url="http://{}:8200".format(VAULT_SERVER),
token=VAULT_TOKEN,
verify=False)
MONGODB_USER = client.read(
"secret/devops-lab/app-devops")['data']['mongodb_user']
MONGODB_PASS = client.read(
"secret/devops-lab/app-devops")['data']['mongodb_pass']
RABBIT_USER = client.read(
"secret/devops-lab/app-devops")['data']['rabbitmq_user']
RABBIT_PASS = client.read(
"secret/devops-lab/app-devops")['data']['rabbitmq_pass']
client = Consul(host=CONSUL_SERVER)
index, data = client.kv.get('devops-lab',
index=None,
recurse=True,
separator='lab')
CONFIG = [{
"mongo": {
"user": MONGODB_USER,
"pass": MONGODB_PASS,
"host": data[0]['Value'].decode("utf-8")
}
}, {
"rabbit": {
"user": RABBIT_USER,
"pass": RABBIT_PASS,
"host": data[1]['Value'].decode("utf-8")
}
}]
return CONFIG
def HealthConsul():
client = Consul(host=getenv('CONSUL_SERVER'))
health = client.health.service('consul')[1][0]['Checks'][0]['Status']
if health == 'passing':
return "Healthy"
else:
return "Unhealthy"
def test_keep_alive():
"""
Integration test for keeping service alive in consul cluster
"""
test_remove_services()
session = Consul()
tailf = get_tailf_service(session)
# assert that there is no checks yet
_, checks = session.health.service(tailf.service_name)
assert not checks
# test that service health check is unknown or critical after registration
tailf.register()
# small sleep for cluster consensus
sleep(SLEEP_WAIT)
assert _get_service_status(session, tailf) in ("unknown", "critical")
# assert service is healthy back again after keep alive
tailf.keep_alive()
sleep(SLEEP_WAIT)
assert _get_service_status(session, tailf) == "passing"
# assert service health check fails after ttl passed
sleep(tailf.ttl + SLEEP_WAIT)
assert _get_service_status(session, tailf) == "critical"
def HealthConsul():
client = Consul(host='localhost')
health = client.health.service('consul')[1][0]['Checks'][0]['Status']
if health == 'passing':
return "Healthy"
else:
return "Unhealthy"
def __init__(self, host='127.0.0.1', port=8500, consistency='stale'):
self.host = host
self.port = port
self.consistency = consistency
self.client = Consul(host=self.host,
port=self.port,
consistency=self.consistency)
def generate_jobs(self, comment):
self.add_comment(comment)
jobs_list = {}
config_api = ConfigAPI()
job_path = config_api.get_consul_replication_jobs_path()
consul = Consul()
kv = KV()
key, jobs_info = consul.kv.get(job_path, recurse=True)
if jobs_info and len(jobs_info) > 0:
for i in jobs_info:
kv = KV()
kv.load_json(json.dumps(i))
replication_job = ReplicationJob()
replication_job.load_json(kv.Value)
node = replication_job.node_name
status = replication_job.status
if node == self.node_name and status == 'started':
jobs_list.update({replication_job.job_id: replication_job})
for job_id, job_info in jobs_list.iteritems():
print(job_id)
cmd = "/opt/petasan/scripts/backups/replication.py run-replication-job --job_id " + str(job_info.job_id)
self.add_job(job_info.schedule, cmd)
def test_ignore_connection_failures():
session = Consul(host="invalid")
tailf = get_tailf_service(session)
# assert service starts
tailf.start()
assert tailf.process.poll() is None
with mock.patch('ianitor.service.logger') as logger:
tailf.register()
assert logger.error.called
with mock.patch('ianitor.service.logger') as logger:
tailf.keep_alive()
assert logger.error.called
with mock.patch('ianitor.service.logger') as logger:
tailf.deregister()
assert logger.error.called
tailf.deregister()
# assert service can be killed
tailf.kill()
tailf.process.wait()
def parse_consul(value):
consul_config = json.loads(os.environ['CONSUL_CONFIG']) if 'CONSUL_CONFIG' in os.environ else {}
scheme = consul_config['scheme'] if 'scheme' in consul_config else 'http'
host = consul_config['host'] if 'host' in consul_config else '127.0.0.1'
port = consul_config['port'] if 'port' in consul_config else 8500
token = consul_config['token'] if 'token' in consul_config else None
consul_merger = Merger([(list, ['append']), (dict, ['merge'])], ['override'], ['override'])
try:
consul = Consul(scheme=scheme, host=host, port=port, token=token)
_, kv_entries = consul.kv.get(recurse=True, key=value)
except ACLPermissionDenied:
print(BRIGHT_RED + 'Access denied connecting to: {}://{}:{}'.format(scheme, host, port) + RESET_ALL)
return ERROR
if not kv_entries:
# Mark as failed if we can't find the consul key
return ERROR
consul_dict = {}
for entry in kv_entries:
subkeys = entry['Key'].split('/')
value = entry['Value'].decode('utf-8') if hasattr(entry['Value'], 'decode') else entry['Value']
value = '' if value is None else value
if '/' in entry['Key']:
key = '{"' + entry['Key'].replace('/', '":{"') + '": "' + value + '"}'.ljust(len(subkeys)+1, '}')
consul_dict = consul_merger.merge(consul_dict, json.loads(key))
else:
consul_dict[entry['Key']] = value
# return subkeys relative to rootkey
rootkey = list(consul_dict.keys())[0]
return consul_dict[rootkey]
def __init__(self):
self.dont_ip = os.environ.get('DONT_IP', '').split(' ')
self.other_repeaters = []
self.consul = Consul(sys.argv[1])
self.ctx = zmq.Context()
self.poller = zmq.Poller()
self.repeater_pub_port = REPEATER_PUB_PORT
self.pub = self.ctx.socket(zmq.PUB)
self.pub.bind('tcp://*:%d' % self.repeater_pub_port)
self.sub = self.ctx.socket(zmq.SUB)
self.sub.setsockopt(zmq.SUBSCRIBE, b'')
self.poller.register(self.sub, zmq.POLLIN)
self.beacon = ZActor(self.ctx, ZBeacon)
self.beacon.send_unicode('CONFIGURE', zmq.SNDMORE)
self.beacon.send(struct.pack('I', ZRE_DISCOVERY_PORT))
self.address = self.beacon.recv_unicode() # Hostname
filter_ = struct.pack('ccc', b'Z', b'R', b'E')
self.beacon.send_unicode('SUBSCRIBE',zmq.SNDMORE)
self.beacon.send(filter_)
self.beacon_socket = self.beacon.resolve()
self.poller.register(self.beacon_socket, zmq.POLLIN)
def connect_to_consult(consul_endpoint):
log.debug('getting-service-endpoint', consul=consul_endpoint)
host = consul_endpoint.split(':')[0].strip()
port = int(consul_endpoint.split(':')[1].strip())
return Consul(host=host, port=port)
def __init__(self):
self.prefix = "gluu/config/"
token = None
cert = None
verify = False
if os.path.isfile(GLUU_CONSUL_TOKEN_FILE):
with open(GLUU_CONSUL_TOKEN_FILE) as fr:
token = fr.read().strip()
if GLUU_CONSUL_SCHEME == "https":
verify = as_boolean(GLUU_CONSUL_VERIFY)
# verify using CA cert (if any)
if verify and os.path.isfile(GLUU_CONSUL_CACERT_FILE):
verify = GLUU_CONSUL_CACERT_FILE
if all([
os.path.isfile(GLUU_CONSUL_CERT_FILE),
os.path.isfile(GLUU_CONSUL_KEY_FILE)
]):
cert = (GLUU_CONSUL_CERT_FILE, GLUU_CONSUL_KEY_FILE)
self._request_warning(GLUU_CONSUL_SCHEME, verify)
self.client = Consul(
host=GLUU_CONSUL_HOST,
port=GLUU_CONSUL_PORT,
token=token,
scheme=GLUU_CONSUL_SCHEME,
consistency=GLUU_CONSUL_CONSISTENCY,
verify=verify,
cert=cert,
)
def register_plugin():
"""
Registers this plugin with Consul
:return: None
"""
logger.info("Registering plugin with Consul")
consul_host = getenv("CONSUL_HOST", "localhost")
consul_port = getenv("CONSUL_PORT", 8500)
c = Consul(host=consul_host, port=consul_port)
hname = socket.gethostname()
ipaddr = socket.gethostbyname(hname)
health_check = Check.http(url=f'http://{ipaddr}:{port}/config',
interval="20s",
deregister=True)
service_name = config['serviceName']
service_id = f'{service_name}-{str(uuid.uuid4())}'
c.agent.service.register(name=service_name,
service_id=service_id,
address=ipaddr,
port=port,
tags=['secure=false'],
check=health_check)
atexit.register(deregister, c, service_id)
def __init__(self, context, options):
super().__init__(context, options)
self._consul = Consul(
scheme=self.options.scheme,
host=self.options.host,
port=self.options.port,
timeout=self.options.timeout,
)
def run(
keys: List[Key],
exist_ok: bool = False,
):
consul = Consul()
if not exist_ok and multi_get(consul, keys):
raise TribunError("Existing keys")
def __init__(self,
host=None,
port=None,
token=None): # 初始化，指定consul主机，端口，和token
self.host = host # consul 主机
self.port = port # consul 端口
self.token = token
self.consul = Consul(host=host, port=port)
def register_service(service_name: str,
*,
address: str = None,
port: int = None):
consul = Consul()
if address is None:
address = get_local_ip()
consul.agent.service.register(service_name, address=address, port=port)
def __init__(self, host, port, path_prefix):
self.log = structlog.get_logger()
self._consul = Consul(host=host, port=port)
self.host = host
self.port = port
self._path_prefix = path_prefix
self.retries = 0
def __init__(self, options, hostname):
self.options = options
self.consul = Consul(host=options.consul_host,
port=options.consul_port)
self.service_name = options.app
self.hostname = hostname
self.service_id = _make_service_id(options,
service_name=self.service_name,
hostname=self.hostname)
def test_service_start():
session = Consul()
tailf = get_tailf_service(session)
with mock.patch.object(service.Service, "register") as register_method:
tailf.start()
assert bool(tailf.is_up())
register_method.assert_any_call()
def _is_myservice_registered_to_consul(self, ip):
port = self.service_port(8500, 'consul')
consul = Consul('localhost', port, 'the_one_ring')
services = consul.agent.services()
for index, service in services.items():
if service['Service'] == 'myservice' and service['Address'] == ip:
return True
return False
def registry_factory(self) -> Consul:
if self.server == None:
raise RegistryError("load etcd server error")
address = self.server.address
if address:
return Consul(host=address.split(",")[0].split(":")[0],
port=int(address.split(",")[0].split(":")[1]))
else:
raise RegistryError("load etcd server error")
def __init__(self):
self.service_name = django_config_dic['SERVICE_NAME']
self.host = django_config_dic['ip'] # django运行 主机
self.port = django_config_dic['port'] # django运行 端口
print(
f"ConsulConf类 __init__()方法开始初始化Consul连接,连接信息：服务名: {self.service_name} IP: {self.host} 端口号：{self.port}"
)
self.consul = Consul(host=consul_config_dic['ip'],
port=consul_config_dic['port'])
print("成功连接到Consul!")
def register_consul(address, port):
c = Consul(host='10.10.6.11')
c.agent.service.register('hadoop_exporter_dev',
service_id='hadoop_exporter_dev2',
address='indata-10-10-6-11.indata.com',
port=port,
tags=['hadoop'])
start_http_server(port)
# print("Polling %s. Serving at port: %s" % (args.address, port))
print("Polling %s. Serving at port: %s" % (address, port))
def __init__(self, context, options):
super().__init__(context, options)
self._session_id = None
self._refresh_lock_flag = False
self._consul = Consul(
scheme=self.options.scheme,
host=self.options.host,
port=self.options.port,
timeout=self.options.timeout,
)
def get_service(service_id):
consul = Consul(host="consul", port=consul_port)
agent = consul.agent
service_list = agent.services()
service_info = service_list[service_id]
return service_info['Address'], service_info['Port']
def test_service_register():
session = Consul()
agent = session.agent
tailf = get_tailf_service(session)
tailf.start()
test_remove_services()
tailf.register()
assert agent.services()
if environ.get('USE_ENV_VARS') == 'TRUE':
try: # use local settings
for env_key, value in config.iteritems():
if not value:
config[env_key] = environ[env_key]
except KeyError as e:
""" Throw an error if a setting is missing """
print "ERR MSG: {}".format(e.message)
print ("Some of your settings aren't in the environment."
"You probably need to run:"
"\n\n\tsource config/<your settings file>")
exit(1)
else: # use consul
kv = Consul().kv # initalize client to KV store
for consul_key, config_key in consul_configurations:
_, tmp = kv.get("density/{}".format(consul_key))
val = tmp.get('Value')
config[config_key] = val
if not val:
raise Exception(("no value found in Consul for key "
"density/{}").format(consul_key))
# mail settings
config['MAIL_SERVER'] = 'smtp.gmail.com'
config['MAIL_PORT'] = 465
config['MAIL_USE_SSL'] = True
config['MAIL_USE_TLS'] = False
config['MAIL_DEFAULT_SENDER'] = '*****@*****.**'
if environ.get('USE_ENV_VARS') == 'TRUE':
try: # use local settings
for env_key, value in config.iteritems():
if not value:
config[env_key] = environ[env_key]
except KeyError as e:
""" Throw an error if a setting is missing """
print "ERR MSG: {}".format(e.message)
print ("Some of your settings aren't in the environment."
"You probably need to run:"
"\n\n\tsource config/<your settings file>")
exit(1)
else: # use consul
kv = Consul().kv # initalize client to KV store
for consul_key, config_key in consul_configurations:
_, tmp = kv.get("density/{}".format(consul_key))
val = tmp.get('Value')
config[config_key] = val
if not val:
raise Exception(("no value found in Consul for key "
"density/{}").format(consul_key))
# mail settings
config.update({
'MAIL_SERVER': 'smtp.googlemail.com',
'MAIL_PORT': 465,
'MAIL_USE_SSL': True,
'MAIL_USE_TLS': False,
'INSTALLED_APP_CREDENTIALS_PATH': 'config/installed_app_credentials.json',
'CLIENT_SECRETS_PATH': 'config/client_secrets.json',
'PRIVATE_CALENDAR_ID': None,
'PUBLIC_CALENDAR_ID': None,
# MongoDB configurations
'MONGO_DATABASE': 'eventum',
# Logging configurations
'LOG_FILE_MAX_SIZE': '256',
'APP_LOG_NAME': 'app.log',
'WERKZEUG_LOG_NAME': 'werkzeug.log',
}
from consul import Consul
kv = Consul().kv # initalize client to KV store
# get values from Consul and set the corresponding config variable to the
# value retrieved from Consul.
for key, value in config.iteritems():
try:
_, consul_value = kv.get(("adi-website/{}").format(key.lower()))
except requests.ConnectionError:
raise Exception('Failed to connect to Consul. You probably need to '
'run: \n\n\t./config/run_consul.sh')
# We have a good value in Consul
if consul_value and consul_value.get('Value'):
config[key] = consul_value.get('Value')
continue
