def setUp(self):
self.consul = consulate.Consul(host=os.environ['CONSUL_HOST'],
port=os.environ['CONSUL_PORT'],
token=CONSUL_CONFIG['acl_master_token'])
self.forbidden_consul = consulate.Consul(
host=os.environ['CONSUL_HOST'],
port=os.environ['CONSUL_PORT'],
token=str(uuid.uuid4()))
self.used_keys = list()
def setUp(self):
self.consul = consulate.Consul(
host=os.environ["CONSUL_HOST"],
port=os.environ["CONSUL_PORT"],
token=CONSUL_CONFIG["acl"]["tokens"]["master"],
)
self.forbidden_consul = consulate.Consul(
host=os.environ["CONSUL_HOST"],
port=os.environ["CONSUL_PORT"],
token=str(uuid.uuid4()))
self.used_keys = list()
def client(self):
"""
#Cached property for getting ``Redis`` client to be used to interact with redis.
Not sure if it is ok to cache client, because
Returned client also subclasses from :class:`.EnsuredRedisMixin` which
ensures that all redis commands are executed with retry logic in case
of sentinel failover.
Returns
-------
Redis
Redis client connected to Sentinel via Sentinel connection pool
"""
params = self.connparams
params.update({
'service_name': self.service_name,
'socket_timeout': self.socket_timeout,
})
if self.use_consul and consulate is not None:
consul = consulate.Consul(host=self.consul_ip_addr)
params['sentinels'] = [
(node['Address'], self.sentinel_port) for node in consul.catalog.nodes()
if node['Meta'].get('consul_role') == 'server'
]
else:
params['sentinels'] = self.sentinels
return get_redis_via_sentinel(
redis_class=type(str('Redis'), (EnsuredRedisMixin, Redis), {}),
**params
)
def list():
consul = consulate.Consul()
services = consul.health.service('port-agent', passing=True)
agents = []
for svc in services:
agents.extend(svc.get('Service', {}).get('Tags', []))
click.echo('\n'.join(agents))
def out(basedir, instream):
input = json.load(instream)
# create consul_instance
consul_instance = consulate.Consul(host=input['source']['host'], port=input['source'].get('port', 443), scheme=input['source'].get('scheme', 'https'), token=input['source']['token'])
# see which key we need to monitor
key = input['source']['key']
if not key or len(key) <= 0:
common.msg("[out] consul singlekey resource expected a non-empty key name")
exit(1)
value = None
if "value" in input['params']:
value = input['params']['value']
elif "value_file" in input['params']:
with open(os.path.join(basedir, input['params']['value_file']), 'r') as file:
value = file.read()
else:
common.msg("[out] consul singlekey resource expected either 'value' or 'value_file' specified")
exit(1)
value = value.strip()
common.msg("[out] consul singlekey resource setting {0} = {1}".format(key, value))
consul_instance.kv[key] = value
return {'version': {'value': value}}
def __init__(self,host,port,scheme,token):
self.host = host
self.port = port
self.scheme = scheme
self.token = token
self.session = consulate.Consul(host=host, port=port,
scheme=scheme, token=token)
def main():
"""Entrypoint for the consulate cli application"""
args = parse_cli_args()
if args.api_scheme == "http+unix":
adapter = adapters.UnixSocketRequest
port = None
api_host = os.environ.get("CONSUL_HTTP_ADDR").replace("unix://", "")
if args.api_host:
api_host = args.api_host
else:
adapter = None
port = args.api_port
api_host = "localhost"
if args.api_host:
api_host = args.api_host
consul = consulate.Consul(api_host, port, args.dc, args.token, args.api_scheme, adapter)
if args.command == "acl":
ACL_ACTIONS[args.action](consul, args)
elif args.command == "kv":
KV_ACTIONS[args.action](consul, args)
elif args.command == "register":
register(consul, args)
elif args.command == "deregister":
deregister(consul, args)
elif args.command == "services":
services(consul, args)
elif args.command == "run_once":
run_once(consul, args)
def __init__(self):
self.consul = consulate.Consul(
host=const.host, port=const.consul['port']
)
self.nodes = dict(
node1=dict(
docker_cli=docker.DockerClient(
base_url='unix:///node1/var/run/docker.sock',
**self.__class__.client_default_kwargs
),
),
node2=dict(
docker_cli=docker.DockerClient(
base_url='unix:///node2/var/run/docker.sock',
**self.__class__.client_default_kwargs
),
),
node3=dict(
docker_cli=docker.DockerClient(
base_url='unix:///node3/var/run/docker.sock',
**self.__class__.client_default_kwargs
),
),
node4=dict(
docker_cli=docker.DockerClient(
base_url='unix:///node4/var/run/docker.sock',
**self.__class__.client_default_kwargs
),
),
)
def check(instream):
input = json.load(instream)
# create consul_instance
consul_instance = consulate.Consul(host=input['source']['host'],
port=input['source'].get('port', 443),
scheme=input['source'].get(
'scheme', 'https'),
token=input['source']['token'])
# see which key we need to monitor
key = input['source']['key']
if not key or len(key) <= 0:
common.msg(
"[check] consul singlekey resource expected a non-empty key name")
exit(1)
value = consul_instance.kv[key] if key in consul_instance.kv else ""
common.msg("[check] consul singlekey resource {0} = {1}".format(
key, value))
# see if the same as previous version, or different
version = input.get('version')
valueOld = version.get('value', "") if version is not None else ""
if valueOld is None or value == valueOld:
return [{'value': value}]
return [{'value': valueOld}, {'value': value}]
def notify_device(device_id, version_buf, local_ip, netmask, gateway_ip):
print "Device(id=%s,v=%s,ip=%s/%s,gateway=%s)" % (
device_id, version_buf, local_ip, netmask, gateway_ip)
# register
consul = consulate.Consul()
consul.agent.service.register(device_id, address=local_ip)
def main():
"""Entrypoint for the consulate cli application"""
args = parse_cli_args()
if args.api_scheme == 'http+unix':
adapter = adapters.UnixSocketRequest
port = None
api_host = os.environ.get('CONSUL_HTTP_ADDR').replace('unix://', '')
if args.api_host:
api_host = args.api_host
else:
adapter = None
port = args.api_port
api_host = 'localhost'
if args.api_host:
api_host = args.api_host
consul = consulate.Consul(api_host, port, args.dc, args.token,
args.api_scheme, adapter)
if args.command == 'acl':
ACL_ACTIONS[args.action](consul, args)
elif args.command == 'kv':
KV_ACTIONS[args.action](consul, args)
elif args.command == 'register':
register(consul, args)
elif args.command == 'deregister':
deregister(consul, args)
elif args.command == 'run_once':
run_once(consul, args)
def consul(self):
session = consulate.Consul()
yield session.kv
for k in session.kv.keys():
session.kv.delete(k)
def __init__(self, conf):
super(Reader, self).__init__(conf)
self.session = consulate.Consul(host=self.conf['host'],
port=self.conf['port'],
datacenter=self.conf.get('datacenter'),
token=self.conf.get('token'),
scheme=self.conf['scheme'])
def get_service_checks(args):
"""Request service checks on Consul API."""
consul = consulate.Consul(token=args.token,
port=args.port,
host=args.host,
datacenter=args.datacenter,
scheme='https' if args.tls else 'http')
return consul.health.checks(args.service)
def __init__(self, location, params):
super(ConsulCache, self).__init__(params)
self.location = location
options = params.get('OPTIONS', {})
self._host = options.get('HOST', 'localhost')
self._port = options.get('PORT', 27017)
self.root = options.get('/DJANGO-CACHE/')
self.session = consulate.Consul()
def __init__(self, wrapper, ttl=120):
super(StatusThread, self).__init__()
self.wrapper = wrapper
self.consul = consulate.Consul()
self.ttl = ttl
# sleep for 1/2 ttl, so 2 missed polls will degrade the state
self.sleep_time = ttl / 2.0
self.running = True
def get_host_port(service_id, tag):
consul = consulate.Consul()
svc = consul.health.service(service_id, tag=tag, passing=True)
if len(svc) == 1:
svc = svc[0]
addr = svc['Node']['Address']
port = svc['Service']['Port']
return addr, port
return None, None
def test_provisioning_adsws_service(self):
"""
First run the provisioner and then we can check that some configuration
values have been correctly set in the key/value store
"""
self._provision('adsws')
# Obtain what we expect to find
config_file = '{}/{}/adsws/adsws.config.json'.format(
ConsulProvisioner.template_dir,
ConsulProvisioner.name,
)
with open(config_file) as json_file:
template = Template(json_file.read())
json_config = template.render(
db_host='localhost',
db_port=5432,
cache_host='localhost',
cache_port=6379
)
config = json.loads(json_config)
# Compare with consul
consul = consulate.Consul(port=self.port)
for key in config:
self.assertIn(key, consul.kv.keys())
self.assertEqual(
config[key],
consul.kv.get(key),
msg='Key {} mismatch: {} != {}'.format(
key,
config[key],
consul.kv.get(key)
)
)
cache = consul.kv.get('config/adsws/staging/CACHE')
self.assertIn(
'localhost',
cache,
)
self.assertIn(
'6379',
cache,
)
db_uri = consul.kv.get('config/adsws/staging/SQLALCHEMY_DATABASE_URI')
self.assertEqual(
db_uri,
'"postgresql+psycopg2://postgres:@localhost:5432/adsws"',
msg='Provisioning is not working: {} != '
'postgresql+psycopg2://postgres:@localhost:5432/adsws'.format(db_uri)
)
def registerService(serverName, serverTag, listenPort):
consul = consulate.Consul()
consul.agent.service.register(
serverName,
port=listenPort,
tags=[serverTag],
)
consul.agent.check.register(serverName,
script='nc -z -w5 localhost %d' % listenPort,
interval='30s')
def main():
vms = enc_load()
c = consulate.Consul()
kv_present = c.kv.find('node/')
for deleted in set(kv_present.keys()) - set(vms.keys()):
print('deleting VM %s' % deleted)
del c.kv[deleted]
for updated, vm in vms.items():
print('updating VM %s' % updated)
c.kv[updated] = vm
def consul(self):
if not self._consul:
self._consul = consulate.Consul(
scheme=self._consul_url.scheme,
host=self._consul_url.hostname,
port=self._consul_url.port,
datacenter=None,
token=None,
)
return self._consul
def check(instream):
input = json.load(instream)
# take snapshot of consul key/values
consul_instance = consulate.Consul(host=input['source']['host'],
port=input['source'].get('port', 443),
scheme=input['source'].get(
'scheme', 'https'),
token=input['source']['token'])
# collect all keys from all given prefixes
prefixes = input['source']['prefixes']
if not isinstance(prefixes, (list, )):
common.msg(
"[check] consul resource expected a list of prefixes, but it's not a list"
)
exit(1)
prefixStr = ""
for prefix in prefixes:
if len(prefixStr) > 0:
prefixStr += " "
prefixStr += prefix
common.msg("[check] consul resource searching under {0}".format(prefixStr))
result = {}
for prefix in prefixes:
found = consul_instance.kv.find(prefix)
if found is not None:
for k, v in found.iteritems():
if v is not None:
result[k] = v
else:
result[k] = ""
common.msg("[check] consul resource found {0} key/values under {1}".format(
len(result), prefixStr))
# hash values from all keys
hash = hashlib.sha224()
for k, v in sorted(result.iteritems()):
hash.update(v.encode("utf-8"))
hashNew = hash.hexdigest()
common.msg("[check] consul resource value hash under {0}: {1}".format(
prefixStr, hashNew))
# see if the same as previous version, or different
version = input.get('version')
hashOld = version.get('hash', "") if version is not None else ""
if hashOld is None or hashNew == hashOld:
return [{'hash': hashNew}]
return [{'hash': hashOld}, {'hash': hashNew}]
def register():
host = os.environ['SERVICE_IP']
port = os.environ['SERVICE_PORT']
service_name = os.environ['SERVICE_NAME']
consul = consulate.Consul(host=os.environ['CONSUL_HOST'],
port=os.environ['CONSUL_PORT'])
consul.agent.service.register(name=service_name,
port=int(port),
address=host)
service_info = {'host': host, 'port': port}
consul.kv.set(service_name, service_info)
def __init__(self):
self.consul = consulate.Consul()
self.nodes = dict(
core1=dict(docker_cli=docker.DockerClient(
base_url="unix:///tmp/docker_core1.sock"), ),
core2=dict(docker_cli=docker.DockerClient(
base_url="unix:///tmp/docker_core2.sock"), ),
core3=dict(docker_cli=docker.DockerClient(
base_url="unix:///tmp/docker_core3.sock"), ),
core4=dict(docker_cli=docker.DockerClient(
base_url="unix:///tmp/docker_core4.sock"), ),
)
def in_(destdir, instream):
input = json.load(instream)
# take snapshot of consul key/values
consul_instance = consulate.Consul(host=input['source']['host'], port=input['source'].get('port', 443), scheme=input['source'].get('scheme', 'https'), token=input['source']['token'])
# collect all keys from all given prefixes
prefixes = input['source']['prefixes']
if not isinstance(prefixes, (list,)):
common.msg("[in] consul resource expected a list of prefixes, but it's not a list")
exit(1)
prefixStr = ""
for prefix in prefixes:
if len(prefixStr) > 0:
prefixStr += " "
prefixStr += prefix
common.msg("[in] consul resource searching under {0}".format(prefixStr))
result = {}
for prefix in prefixes:
found = consul_instance.kv.find(prefix)
if found is not None:
for k, v in found.iteritems():
if v is not None:
result[k] = v
else:
result[k] = ""
common.msg("[in] consul resource found {0} key/values under {1}".format(len(result), prefixStr))
# hash values from all keys
hash = hashlib.sha224()
for k, v in sorted(result.iteritems()):
hash.update(v.encode("utf-8"))
hashNew = hash.hexdigest()
common.msg("[in] consul resource value hash under {0}: {1}".format(prefixStr, hashNew))
# see if the same as previous version, or different
version = input.get('version')
hashRequested = version.get('hash', "") if version is not None else ""
if hashRequested != hashNew:
common.msg("[in] some values got changed in consul, so let's just fetch newer values")
# put on a file system
common.msg("[in] consul resource writing key/values to {0}".format(destdir))
for k, v in result.iteritems():
with safe_open(os.path.join(destdir, k), 'w') as f:
f.write(v.encode("utf-8"))
metadata = [{'name': 'keys', "value": str(len(result))}]
return {'version': {'hash': hashNew}, 'metadata': metadata}
def prep_consul(file):
try:
LOGGER.info('Processing %s', file)
consul = consulate.Consul(os.environ.get('CONSUL_HOST', 'localhost'),
int(os.environ.get('CONSUL_PORT', '8500')))
with open(file) as fh:
config = json.load(fh)
LOGGER.debug('%r', config)
for k, v in config.items():
consul.kv[k] = v
except Exception:
LOGGER.exception('Failed to load consul data.')
sys.exit(-1)
def request(kv_url):
r = requests.get(kv_url)
if not r.ok:
raise ValueError(
"Cannot get list of connected nodes requesting {} (got status code: {} due to: {})"
.format(kv_url, r.status_code, r.reason))
nds = r.json()
nds = [data["Node"] for data in nds]
sess = consulate.Consul()
# test
if len(nds) == 0:
raise KeyError("Expect at least one working node to operate")
sess.kv["states/" + nds[0]]
return nds, sess
def create_snapshot(self, name):
consul = consulate.Consul(token=self.consul_acl_token)
snapshot_key = 'snapshot/{}'.format(str(uuid.uuid4()))
logger.info('Consul: requesting consistent snapshot of %s@%s via %s',
self.vm, name, snapshot_key)
consul.kv[snapshot_key] = {'vm': self.vm, 'snapshot': name}
time.sleep(3)
try:
timeout = TimeOut(self.snapshot_timeout,
interval=2,
raise_on_timeout=True)
while timeout.tick():
for snapshot in self.rbd.snap_ls(self._image_name):
if snapshot['name'] == name:
return
except TimeOutError:
# The VM might have been shut down. Try doing a regular Ceph
# snapshot locally.
super(FlyingCircusRootDisk, self).create_snapshot(name)
except KeyboardInterrupt:
raise
finally:
# In case the snapshot still gets created: the general snapshot
# deletion code in ceph/source will clean up unused backy snapshots
# anyway. However, we need to work a little harder to delete old
# snapshot requests, otherwise we've sometimes seen those not
# getting deleted and then re-created all the time.
for key in list(consul.kv.find('snapshot/')):
try:
s = consul.kv[key]
except KeyError:
continue
try:
s = json.loads(s)
except json.decoder.JSONDecodeError:
# Clean up garbage.
logger.warning(
'Consul: removing garbage request {}'.format(key))
del consul.kv[key]
if s['vm'] != self.vm:
continue
# The knowledge about the `backy-` prefix isn't properly
# encapsulated here.
if s['snapshot'].startswith('backy-'):
logger.info(
'Consul: removing snapshot request of %s@%s via %s',
s['vm'], s['snapshot'], key)
del consul.kv[key]
def main(admin_pw, email, domain, org_name, kv_host, kv_port, save, view,
encoded_salt, encoded_ox_ldap_pw, inum_appliance, oxauth_jks_pw):
# generate all config
cfg = generate_config(admin_pw, email, domain, org_name, encoded_salt,
encoded_ox_ldap_pw, inum_appliance, oxauth_jks_pw)
if save:
consul = consulate.Consul(host=kv_host, port=kv_port)
for k, v in cfg.iteritems():
if k not in consul.kv:
consul.kv.set(k, v)
if view:
pprint.pprint(cfg)
def __create_consul_client(self):
self._consul = consulate.Consul(adapter=RequestAdapter,
host=self.host,
port=self.host_port,
datacenter=None,
token=self._token,
scheme=self.host_scheme)
if self.data_center is None:
pass
elif self.data_center in self.catalog.datacenters():
self._consul = consulate.Consul(adapter=RequestAdapter,
host=self.host,
port=self.host_port,
datacenter=self.data_center,
token=self._token,
scheme=self.host_scheme)
else:
raise LookupError(
'unable to find {0} in available data centers: {1}'.format(
self.data_center, ', '.join(self.catalog.datacenters())))
