def __init__(self,
dev_params=None,
prefix='config',
protocol='http',
host='localhost',
port=2379,
username=None,
password=None,
long_polling_timeout=50,
long_polling_safety_delay=5):
self._client = Client(host=host,
port=port,
protocol=protocol,
allow_redirect=True,
username=username,
password=password)
# Overriding retries for urllib3.PoolManager.connection_pool_kw
self._client.http.connection_pool_kw['retries'] = 0
self._base_config_path = prefix
self._dev_params = dev_params
self._base_config_set_path = "{}/extensions" \
.format(self._base_config_path)
r = ('^(?P<path>{}/(?:extensions/)?'
'(?P<envorset>[\w\-\.]+))/(?P<key>.+)$')
self._key_regex = re.compile(r.format(self._base_config_path))
self.long_polling_timeout = long_polling_timeout
self.long_polling_safety_delay = long_polling_safety_delay
self._init_logger()
def remove_listener(client: etcd.Client, alb, identifier):
try:
client.delete("/alb/{alb}/listeners/{identifier}".format(
alb=alb, identifier=identifier),
recursive=True)
except (etcd.EtcdKeyNotFound, KeyError):
pass
def has_certificate(client: etcd.Client, certificate_name: str):
"""
Check if certificate exists
"""
try:
client.read("/certs/{name}".format(name=certificate_name))
except (etcd.EtcdKeyNotFound, KeyError):
return False
return True
def main(etcdctl_peers, services_base, name, output, maps, links):
click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
click.secho("Links: %s" % " ".join(str(i) for i in links), fg='green')
if len(links) == 0:
click.echo('Nothing to do')
sys.exit()
etcd_url = urlparse(etcdctl_peers)
etcd = Client(host=etcd_url.hostname, port=etcd_url.port)
environment = {}
# Get values from etcd for the services specified
# --link <name or id>:alias
for link in links:
lname, _, alias = link.partition(":")
if not alias:
alias = lname.replace('-', '_')
try:
# Try to get the service
service = etcd.read(os.path.join(services_base, lname))
except EtcdKeyNotFound:
click.secho("Service: %s not found" % lname, fg='red')
else:
# Expects the service to have some children
for child in service._children:
value = child['value']
ip, _, port = value.partition(":")
click.secho("Key: %s found" % lname, fg='green')
environment["%s_NAME" %
alias.upper()] = "/%s/%s" % (name, lname)
environment["%s_PORT" % alias.upper()] = "tcp://%s" % value
environment["%s_PORT_%s_TCP" %
(alias.upper(), port)] = "tcp://%s" % value
environment["%s_PORT_%s_TCP_PROTO" %
(alias.upper(), port)] = "tcp"
environment["%s_PORT_%s_TCP_PORT" %
(alias.upper(), port)] = "%s" % port
environment["%s_PORT_%s_TCP_ADDR" %
(alias.upper(), port)] = "%s" % ip
# Do mapping
for item in maps:
mname, _, alias = item.partition(":")
if mname and alias:
try:
environment[alias] = environment[mname]
except KeyError:
click.secho("Missing Key: %s" % mname, fg='red')
with open(output, 'w') as f:
for key, value in environment.iteritems():
f.write("%s=%s\n" % (key, value))
click.secho("%s=%s" % (key, value), fg='yellow')
click.secho("All done.", fg='green')
def unregister_certificate(client: etcd.Client, certificate_name: str):
"""
Removes a registered certificate. If no certificate has been previously registered
nothing happens.
"""
try:
client.delete("/certs/{name}".format(name=certificate_name),
recursive=True,
dir=True)
except KeyError:
pass
def main(etcdctl_peers, services_base, name, output, maps, links):
click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
click.secho("Links: %s" % " ".join(str(i) for i in links), fg='green')
if len(links) == 0:
click.echo('Nothing to do')
sys.exit()
etcd_url = urlparse(etcdctl_peers)
etcd = Client(host=etcd_url.hostname, port=etcd_url.port)
environment = {}
# Get values from etcd for the services specified
# --link <name or id>:alias
for link in links:
lname, _, alias = link.partition(":")
if not alias:
alias = lname.replace('-', '_')
try:
# Try to get the service
service = etcd.read(os.path.join(services_base, lname))
except EtcdKeyNotFound:
click.secho("Service: %s not found" % lname, fg='red')
else:
# Expects the service to have some children
for child in service._children:
value = child['value']
ip, _, port = value.partition(":")
click.secho("Key: %s found" % lname, fg='green')
environment["%s_NAME" % alias.upper()] = "/%s/%s" % (name, lname)
environment["%s_PORT" % alias.upper()] = "tcp://%s" % value
environment["%s_PORT_%s_TCP" % (alias.upper(), port)] = "tcp://%s" % value
environment["%s_PORT_%s_TCP_PROTO" % (alias.upper(), port)] = "tcp"
environment["%s_PORT_%s_TCP_PORT" % (alias.upper(), port)] = "%s" % port
environment["%s_PORT_%s_TCP_ADDR" % (alias.upper(), port)] = "%s" % ip
# Do mapping
for item in maps:
mname, _, alias = item.partition(":")
if mname and alias:
try:
environment[alias] = environment[mname]
except KeyError:
click.secho("Missing Key: %s" % mname, fg='red')
with open(output, 'w') as f:
for key, value in environment.iteritems():
f.write("%s=%s\n" % (key, value))
click.secho("%s=%s" % (key, value), fg='yellow')
click.secho("All done.", fg='green')
def unregister_certbot(client: etcd.Client, alb, listener_id):
"""
Removes a registered certbot for a given listener. If no certbot has been previously registered
nothing happens.
"""
try:
client.delete("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=listener_id),
recursive=True,
dir=True)
except KeyError as e:
logger.exception("Failed to unregister certbot: %s: %s",
type(e).__name__, e)
def __init__(self, config, section):
super(EtcdStore, self).__init__(config, section)
# Initialize the DB by trying to create the default table
try:
self.etcd = Client(self.etcd_server, self.etcd_port)
self.etcd.write(self.namespace, None, dir=True)
except EtcdNotFile:
# Already exists
pass
except EtcdException:
self.logger.exception("Error creating namespace %s",
self.namespace)
raise CSStoreError('Error occurred while trying to init db')
def __create_client(cls, etcd_host, etcd_port) -> Client:
""" Create etcd client and test connection. """
Logger(cls.__name__).info('Connecting to etcd server...')
client = Client(host=etcd_host, port=etcd_port)
# Test connection by trying to read a random value
try:
client.read('nodes')
except EtcdConnectionFailed:
raise EtcdConnectionError(port=etcd_port)
except EtcdKeyNotFound:
# This is to handle the case where etcd did not have the key (we don't care) but it is running
pass
return client
def wait_certbot_ready(client: etcd.Client, alb, listener_id):
try:
client.read("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=listener_id))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=listener_id),
None,
dir=True)
# Wait max 3 seconds for they entry to be ready
entry = client.watch("/alb/{alb}/certbot/{identifier}/ready".format(
alb=alb, identifier=listener_id),
timeout=3 * 60)
return entry.value == 'true'
def __init__(self, keyspace=None, **kwargs):
"""
Initialize the handler data store.
:param keyspace: etcd keyspace for configuration map starting with /
:type key: string
:param kwargs: generic params forwarded from the Configmanager
:type key: dict
"""
super().__init__()
self.client = Client(**kwargs)
self.keyspace = keyspace if keyspace else '/config'
if not self.keyspace.startswith('/'):
self.keyspace = '/' + self.keyspace
self.config = self.load()
def _get_certificate(certificate_name,
with_pem=False,
client: etcd.Client = None):
"""
:param with_pem: If True then it also loads the pem data
:rtype: Optional[Certificate]
"""
if client is None:
host, port = get_etcd_addr()
client = etcd.Client(host=host, port=int(port))
cert_path = '/certs/{name}'.format(name=certificate_name)
try:
client.read(cert_path)
except (etcd.EtcdKeyNotFound, KeyError):
return None
cert_pem = get_value(client, cert_path + '/cert') or None
cert_domains = get_json(client, cert_path + '/domains', default=[])
cert_email = get_value(client, cert_path + '/email') or None
cert_modified = get_value(client, cert_path + '/modified') or None
if cert_modified:
try:
cert_modified = datetime.strptime(cert_modified,
"%Y-%m-%dT%H:%M:%S.%f%z")
except ValueError:
try:
cert_modified = datetime.strptime(cert_modified,
"%Y-%m-%dT%H:%M:%S.%f")
except ValueError:
try:
cert_modified = datetime.strptime(cert_modified,
"%Y-%m-%dT%H:%M:%S%z")
except ValueError:
cert_modified = None
cert_is_valid = get_value(client, cert_path + '/is_valid')
cert_is_valid = bool_lookup.get(cert_is_valid)
certificate = Certificate(certificate_name,
pem_data=cert_pem,
domains=cert_domains,
email=cert_email,
modified=cert_modified,
is_valid=cert_is_valid)
if with_pem:
certificate.pem_data = _load_certificate_data(certificate,
client=client)
return certificate
def unregister_targets(client: etcd.Client, identifier, targets):
try:
client.read(
"/target_group/{identifier}/targets".format(identifier=identifier))
except (etcd.EtcdKeyNotFound, KeyError):
return
for target in targets:
host = target['host']
port = target['port']
alb = target.get('alb')
try:
client.delete("/target_group/{identifier}/targets/{name}".format(
identifier=identifier, name="{}:{}".format(host, port)))
except (etcd.EtcdKeyNotFound, KeyError):
pass
class Registry(object):
SERVICE_DIRECTORY = "/horizon/services"
def __init__(self, ip, port):
self._client = Client(ip, port)
def create_directory(self, k, v):
try:
self._client.write(k, v, dir=True)
except EtcdException, e:
errorMessage = str(e) + "\n"
errorMessage = "%s [Registry] write %s = %s " % (errorMessage, k,
v)
module_logger.fatal("[registry] write %s=%s" %
(errorMessage, k, v))
raise EtcdException(errorMessage)
def __init__(self, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(("localhost", port))
self.listen(1)
self.child_pids = None
self.zk = KazooClient()
self.etcd = Client(port=2379)
self.port = port
if self.prefork(10):
# self.register_zk(port)
self.register_etcd()
self.register_parent_signal()
else:
# in child process, pid = 0
self.register_child_signal()
class EtcdWrapper:
"""
Pack etcd library, send data to etcd
"""
def __init__(self, conf):
self.conf = conf
self.host, self.port = self.conf['host'], self.conf['port']
self.client = Client(host=self.host, port=self.port)
self.test_connect()
def test_connect(self):
"""
test etcd server
:return:
"""
url = 'http://' + self.host + ':' + str(self.port) + '/version'
try:
data = requests.get(url)
if data.status_code == 200:
log.info("etcd client init ok!")
return True
else:
return False
except Exception as e:
log.error("\n%s", e)
log.info("Check etcd server or network")
return False
def write(self, key, value):
try:
self.client.write(key, value)
except Exception as e:
log.error("\n%s", e)
def read(self, key, value):
try:
return self.client.read(key)
except Exception as e:
log.error("\n%s", e)
def delete(self, key):
try:
return self.client.delete(key)
except Exception as e:
log.error("\n%s", e)
def __init__(self, config):
super(EtcdStore, self).__init__(config)
self.server = config.get('etcd_server', '127.0.0.1')
self.port = int(config.get('etcd_port', 4001))
self.namespace = config.get('namespace', "/custodia")
# Initialize the DB by trying to create the default table
try:
self.etcd = Client(self.server, self.port)
self.etcd.write(self.namespace, None, dir=True)
except EtcdNotFile:
# Already exists
pass
except EtcdException:
self.logger.exception("Error creating namespace %s",
self.namespace)
raise CSStoreError('Error occurred while trying to init db')
def EtcdFactory(*args, **kwargs):
"""Factory method, returning a connection to a real etcd if we need one for
FV, or to an in-memory implementation for UT."""
if os.environ.get('ETCD_IP'):
return Client(os.environ.get('ETCD_IP'),
int(os.environ.get('ETCD_PORT', 4001)))
else:
return MockEtcdClient(None, None)
def main(etcdctl_peers, host, service_name, backend):
etcd_url = urlparse(etcdctl_peers)
etcd = Client(host=etcd_url.hostname, port=etcd_url.port)
key = "/vulcand/frontends/%s/frontend" % service_name
click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
try:
etcd.read(key='/')
except EtcdConnectionFailed:
click.secho('ETCD Connection Failed', fg='red')
sys.exit(99)
click.secho("Writing key %s" % key, fg='green')
try:
value = etcd.read(key=key).value
except EtcdKeyNotFound:
value = '{}'
j = json.loads(value)
j['Route'] = "Host(`%s`) && PathRegexp(`/.*`)" % host
j['Type'] = 'http'
if backend:
j['BackendId'] = backend
# Write / Update Key
try:
etcd.write(key=key, value=json.dumps(j))
except Exception as e:
raise e # TODO: Handle specific exceptions
click.secho("All done.", fg='green')
def test_order_of_entries_on_dump(self):
backup = tempfile()
with EtcdWrap() as etcd:
try:
client = Client(etcd.host, etcd.port)
for i in range(10):
client.write("/{}".format(i), i**i)
Dumper(etcd.client_urls[0]).dump(filename=backup)
finally:
etcd.terminate()
self.assertGreater(stat(backup).st_size, 0)
with open(backup, encoding="utf-8") as fp:
lastindex = 0
for i, entry in enumerate(load(fp)):
self.assertEqual(entry["key"], "/{}".format(i))
self.assertEqual(entry["value"], str(i**i))
self.assertTrue(i <= lastindex < entry["index"])
lastindex = entry["index"]
def main(etcdctl_peers, host, service_name, backend):
etcd_url = urlparse(etcdctl_peers)
etcd = Client(host=etcd_url.hostname, port=etcd_url.port)
key = "/vulcand/frontends/%s/frontend" % service_name
click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
try:
etcd.read(key='/')
except EtcdConnectionFailed:
click.secho('ETCD Connection Failed', fg='red')
sys.exit(99)
click.secho("Writing key %s" % key, fg='green')
try:
value = etcd.read(key=key).value
except EtcdKeyNotFound:
value = '{}'
j = json.loads(value)
j['Route'] = "Host(`%s`) && PathRegexp(`/.*`)" % host
j['Type'] = 'http'
if backend:
j['BackendId'] = backend
# Write / Update Key
try:
etcd.write(key=key, value=json.dumps(j))
except Exception as e:
raise e # TODO: Handle specific exceptions
click.secho("All done.", fg='green')
def cluster_key(node_name, *clusters, **client_kwargs):
key_name = '/unclustered/{node_name}'.format(node_name=node_name)
client = Client(**client_kwargs)
client.get_lock(key_name, ttl=60)
value = client.get(key_name).value
for cluster in clusters:
client.set('/{cluster}/{node_name}'.format(cluster=cluster, node_name=node_name), value)
return client.delete(key_name)
def get_servers_with_etcd():
client = Client(port=2379)
# 获取新的服务地址，并监听服务变动
def get_all_addr():
new_addrs = set()
rpc_dir = client.get(zk_rpc)
for child in rpc_dir.leaves:
addr = json.loads(child.value)
new_addrs.add("%s:%d" % (addr["host"], addr["port"]))
return new_addrs
# 当前活跃地址
current_addrs = get_all_addr()
G["servers"] = [RemoteServer(s) for s in current_addrs]
for _ in client.eternal_watch(zk_rpc, recursive=True):
print("listening etcd")
new_addrs = get_all_addr()
# 新增
add_addrs = new_addrs - current_addrs
# 需要删除
del_addrs = current_addrs - new_addrs
del_servers = []
for addr in del_addrs:
for s in G["servers"]:
if s.addr == addr:
del_servers.append(s)
break
for server in del_servers:
G["servers"].remove(server)
current_addrs.remove(server.addr)
# 新增
for addr in add_addrs:
G["servers"].append(RemoteServer(addr))
current_addrs.add(addr)
print("last g[servers]", G["servers"])
return G["servers"]
def _connect(self):
return Client(host=self.host,
port=self.port,
srv_domain=self.srv_domain,
version_prefix=self.version_prefix,
read_timeout=self.read_timeout,
allow_redirect=self.allow_redirect,
protocol=self.protocol,
cert=self.cert,
ca_cert=self.ca_cert,
username=self.username,
password=self.password,
allow_reconnect=self.allow_reconnect,
use_proxies=self.use_proxies,
expected_cluster_id=self.expected_cluster_id,
per_host_pool_size=self.per_host_pool_size)
def __init__(
self, dev_params=None, prefix='config', protocol='http',
host='localhost', port=2379, long_polling_timeout=50,
long_polling_safety_delay=5):
self._client = Client(
host=host, port=port, protocol=protocol, allow_redirect=True)
# Overriding retries for urllib3.PoolManager.connection_pool_kw
self._client.http.connection_pool_kw['retries'] = 0
self._base_config_path = prefix
self._dev_params = dev_params
self._base_config_set_path = "{}/extensions"\
.format(self._base_config_path)
r = ('^(?P<path>{}/(?:extensions/)?'
'(?P<envorset>[\w\-\.]+))/(?P<key>.+)$')
self._key_regex = re.compile(r.format(self._base_config_path))
self._etcd_index = 0
self.long_polling_timeout = long_polling_timeout
self.long_polling_safety_delay = long_polling_safety_delay
self._init_logger()
def mark_certbot_ready(client: etcd.Client, alb, cerbot_id, is_ready=True):
"""
Tell system a certbot is ready
"""
try:
client.read("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=cerbot_id))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=cerbot_id),
None,
dir=True)
client.write(
"/alb/{alb}/certbot/{identifier}/ready".format(alb=alb,
identifier=cerbot_id),
'true' if is_ready else 'false')
def register_target_group(client: etcd.Client,
identifier,
name,
targets,
protocol="http"):
try:
client.read("/target_group/{identifier}".format(identifier=identifier))
except (etcd.EtcdKeyNotFound, KeyError):
client.write(
"/target_group/{identifier}".format(identifier=identifier),
None,
dir=True)
client.write(
"/target_group/{identifier}/name".format(identifier=identifier), name)
client.write("/target_group/{identifier}/id".format(identifier=identifier),
identifier)
# only http for now
client.write(
"/target_group/{identifier}/protocol".format(identifier=identifier),
'http')
# Health check config is hardcoded for now
client.write(
"/target_group/{identifier}/healthcheck".format(identifier=identifier),
json.dumps({
'protocol': 'http',
'path': '/',
# traffic port is the port of the first target
'port': 'traffic',
'healthy': 2,
'unhealthy': 10,
'timeout': 4,
'interval': 5,
'success': 200,
}))
try:
client.read(
"/target_group/{identifier}/targets".format(identifier=identifier))
except (etcd.EtcdKeyNotFound, KeyError):
client.write(
"/target_group/{identifier}/targets".format(identifier=identifier),
None,
dir=True)
for target in targets:
host = target['host']
port = target['port']
alb = target.get('alb')
# TODO: If the target is an ALB, then we need to register this ALB as the listener
# in the target ALB. We also need to transfer any rules from the target to the listener
client.write(
"/target_group/{identifier}/targets/{name}".format(
identifier=identifier, name="{}:{}".format(host, port)),
json.dumps({
'host': host,
'port': port,
}))
def upload_certificate_file(client: etcd.Client,
certificate_name,
certificate_file,
modified: datetime = None):
try:
client.read("/certs/{cert_name}".format(cert_name=certificate_name))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/certs/{cert_name}".format(cert_name=certificate_name),
None,
dir=True)
if isinstance(certificate_file, str):
with open(certificate_file) as cert_fh:
certificate_content = cert_fh.read()
else:
certificate_content = certificate_file.read()
client.write("/certs/{cert_name}/cert".format(cert_name=certificate_name),
certificate_content)
if not modified:
modified = datetime.now()
client.write("/certs/{name}/modified".format(name=certificate_name),
modified.isoformat())
# TODO: Verify certificate data with ssl
if certificate_content:
client.write("/certs/{name}/is_valid".format(name=certificate_name),
'true')
else:
client.write("/certs/{name}/is_valid".format(name=certificate_name),
'false')
def upload_certificate_data(client: etcd.Client,
certificate_name,
data,
modified: datetime = None):
try:
client.read("/certs/{cert_name}".format(cert_name=certificate_name))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/certs/{cert_name}".format(cert_name=certificate_name),
None,
dir=True)
client.write("/certs/{cert_name}/cert".format(cert_name=certificate_name),
data)
if not modified:
modified = datetime.now()
client.write("/certs/{name}/modified".format(name=certificate_name),
modified.isoformat())
# TODO: Verify certificate data with ssl
if data:
client.write("/certs/{name}/is_valid".format(name=certificate_name),
'true')
else:
client.write("/certs/{name}/is_valid".format(name=certificate_name),
'false')
def register_certificate(client: etcd.Client,
certificate_name: str,
domains: list = None,
email: str = None,
data: str = None,
modified: datetime = None):
"""
Register a certificate with optional data, domains, email and modification date.
"""
try:
client.read("/certs/{name}".format(name=certificate_name))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/certs/{name}".format(name=certificate_name),
None,
dir=True)
client.write("/certs/{name}/email".format(name=certificate_name), email)
client.write("/certs/{name}/data".format(name=certificate_name), data)
if not modified:
modified = datetime.now()
client.write("/certs/{name}/modified".format(name=certificate_name),
modified.isoformat())
client.write("/certs/{name}/domains".format(name=certificate_name),
json.dumps(domains))
# TODO: Verify certificate data with ssl
if data:
client.write("/certs/{name}/is_valid".format(name=certificate_name),
'true')
else:
client.write("/certs/{name}/is_valid".format(name=certificate_name),
'false')
import uuid
from etcd import Client
c = Client(host='etcd-cluster.hostd.svc.tutum.io')
username = '******'
userHash = uuid.uuid4()
c.set("/hostd/users/%s" % username, '{"userHash":"%s", "username":"******"}' % (userHash, username))
import uuid
from etcd import Client
c = Client(host='hostd-etcd.paalmoest.svc.tutum.io')
for i in range(10000):
userHash = uuid.uuid4()
c.set("/hostd/users/%s" % userHash, '{"userHash":"%s"}' % userHash)
from etcd import Client
import os
etcd_host = os.environ.get("ETCD_HOST", os.getenv("ETCD_HOST"))
etcd_port = os.environ.get("ETCD_PORT", os.getenv("ETCD_PORT"))
client = Client(host=etcd_host, port=int(etcd_port))
# Execute in app start
# client.write('/user', None, dir=True)
# client.write('/ttl', None, dir=True)
# client.write('/type', None, dir=True)
# client.write('/zone', None, dir=True)
# client.write('/record', None, dir=True)
# client.write('/serial', None, dir=True)
# client.write('/conten', None, dir=True)
# client.write("/user/1", {"key": "1","email": "*****@*****.**", "project_id": "001","state": "inserted", "created_at":"2019-07-20 23:04:22.420505"})
client.write("/ttl/1", {"key": "1", "value": "300"})
client.write("/ttl/2", {"key": "2", "value": "900"})
client.write("/ttl/3", {"key": "3", "value": "1800"})
client.write("/ttl/4", {"key": "4", "value": "3600"})
client.write("/ttl/5", {"key": "5", "value": "7200"})
client.write("/ttl/6", {"key": "6", "value": "14400"})
client.write("/ttl/7", {"key": "7", "value": "28800"})
client.write("/ttl/8", {"key": "8", "value": "43200"})
client.write("/ttl/9", {"key": "9", "value": "86400"})
class EtcdStore(CSStore):
def __init__(self, config):
super(EtcdStore, self).__init__(config)
self.server = config.get('etcd_server', '127.0.0.1')
self.port = int(config.get('etcd_port', 4001))
self.namespace = config.get('namespace', "/custodia")
# Initialize the DB by trying to create the default table
try:
self.etcd = Client(self.server, self.port)
self.etcd.write(self.namespace, None, dir=True)
except EtcdNotFile:
# Already exists
pass
except EtcdException:
self.logger.exception("Error creating namespace %s",
self.namespace)
raise CSStoreError('Error occurred while trying to init db')
def _absolute_key(self, key):
"""Get absolute path to key and validate key"""
if '//' in key:
raise ValueError("Invalid empty components in key '%s'" % key)
parts = key.split('/')
if set(parts).intersection({'.', '..'}):
raise ValueError("Invalid relative components in key '%s'" % key)
return '/'.join([self.namespace] + parts).replace('//', '/')
def get(self, key):
self.logger.debug("Fetching key %s", key)
try:
result = self.etcd.get(self._absolute_key(key))
except EtcdException:
self.logger.exception("Error fetching key %s", key)
raise CSStoreError('Error occurred while trying to get key')
self.logger.debug("Fetched key %s got result: %r", key, result)
return result.value
def set(self, key, value, replace=False):
self.logger.debug("Setting key %s to value %s (replace=%s)",
key, value, replace)
path = self._absolute_key(key)
try:
self.etcd.write(path, value, prevExist=replace)
except EtcdAlreadyExist as err:
raise CSStoreExists(str(err))
except EtcdException:
self.logger.exception("Error storing key %s", key)
raise CSStoreError('Error occurred while trying to store key')
def span(self, key):
path = self._absolute_key(key)
self.logger.debug("Creating directory %s", path)
try:
self.etcd.write(path, None, dir=True, prevExist=False)
except EtcdAlreadyExist as err:
raise CSStoreExists(str(err))
except EtcdException:
self.logger.exception("Error storing key %s", key)
raise CSStoreError('Error occurred while trying to store key')
def list(self, keyfilter='/'):
path = self._absolute_key(keyfilter)
if path != '/':
path = path.rstrip('/')
self.logger.debug("Listing keys matching %s", path)
try:
result = self.etcd.read(path, recursive=True)
except EtcdKeyNotFound:
return None
except EtcdException:
self.logger.exception("Error listing %s", keyfilter)
raise CSStoreError('Error occurred while trying to list keys')
self.logger.debug("Searched for %s got result: %r", path, result)
value = set()
for entry in result.get_subtree():
if entry.key == path:
continue
name = entry.key[len(path):]
if entry.dir and not name.endswith('/'):
name += '/'
value.add(name.lstrip('/'))
return sorted(value)
def cut(self, key):
self.logger.debug("Removing key %s", key)
try:
self.etcd.delete(self._absolute_key(key))
except EtcdKeyNotFound:
self.logger.debug("Key %s not found", key)
return False
except EtcdException:
self.logger.exception("Error removing key %s", key)
raise CSStoreError('Error occurred while trying to cut key')
self.logger.debug("Key %s removed", key)
return True
class EtcdConfigManager():
def __init__(
self, dev_params=None, prefix='config', protocol='http',
host='localhost', port=2379, long_polling_timeout=50,
long_polling_safety_delay=5):
self._client = Client(
host=host, port=port, protocol=protocol, allow_redirect=True)
# Overriding retries for urllib3.PoolManager.connection_pool_kw
self._client.http.connection_pool_kw['retries'] = 0
self._base_config_path = prefix
self._dev_params = dev_params
self._base_config_set_path = "{}/extensions"\
.format(self._base_config_path)
r = ('^(?P<path>{}/(?:extensions/)?'
'(?P<envorset>[\w\-\.]+))/(?P<key>.+)$')
self._key_regex = re.compile(r.format(self._base_config_path))
self._etcd_index = 0
self.long_polling_timeout = long_polling_timeout
self.long_polling_safety_delay = long_polling_safety_delay
self._init_logger()
def _init_logger(self):
self.logger = logging.getLogger('etcd_config_manager')
logger_console_handler = logging.StreamHandler()
logger_console_handler.setLevel(logging.ERROR)
self.logger.addHandler(logger_console_handler)
def _env_defaults_path(self, env='test'):
return "{}/{}".format(self._base_config_path, env)
def _config_set_path(self, set_name):
return "{}/{}".format(self._base_config_set_path, set_name)
def _encode_config_key(self, k):
return k.lower().replace('_', '/')
def _decode_config_key(self, k):
[env_or_set, key_path] = re.sub(
self._key_regex, '\g<envorset>|\g<key>', k).split('|')
return env_or_set, key_path.upper().replace('/', '_')
def _encode_config_value(self, val):
return json.dumps(val, cls=CustomJSONEncoder)
def _decode_config_value(self, val):
decoded = json.loads(val, object_hook=custom_json_decoder_hook)
return byteify(decoded)
def _process_response_set(self, rset, env_defaults=True):
d = {}
for leaf in rset.leaves:
try:
config_set, key = self._decode_config_key(leaf.key)
except ValueError:
info = "An error occurred when processing an EtcdResponse"
if not env_defaults:
info += " (is '{}' a directory?)".format(
self._base_config_set_path)
self.logger.warning(info)
else:
if leaf.value is not None:
try:
value = self._decode_config_value(leaf.value)
except ValueError as e:
raise EtcdConfigInvalidValueError(
leaf.key, leaf.value, e)
if env_defaults:
d[key] = value
else:
if config_set not in d:
d[config_set] = {}
d[config_set][key] = value
return d
@staticmethod
def get_dev_params(mod):
params = {}
if mod:
params = attrs_to_dir(import_module(mod))
return params
def get_env_defaults(self, env):
res = self._client.read(
self._env_defaults_path(env),
recursive=True)
conf = self._process_response_set(res)
conf.update(EtcdConfigManager.get_dev_params(self._dev_params))
return conf
def get_config_sets(self):
conf = {}
try:
res = self._client.read(
self._base_config_set_path,
recursive=True)
conf = self._process_response_set(res, env_defaults=False)
except EtcdKeyNotFound:
self.logger.warning(
"Unable to find config sets at '{}' (expected a dict)",
self._base_config_set_path)
return conf
@threaded(daemon=True)
def monitor_env_defaults(
self, env, conf={}, wsgi_file=None, max_events=None):
processed_events = 0
for event in self._watch(
self._env_defaults_path(env), conf, wsgi_file, max_events):
if event is not None:
self._etcd_index = event.etcd_index
conf.update(self._process_response_set(event))
conf.update(EtcdConfigManager.get_dev_params(self._dev_params))
if wsgi_file:
with open(wsgi_file, 'a'):
utime(wsgi_file, None)
processed_events += 1
return processed_events
@threaded(daemon=True)
def monitor_config_sets(self, conf={}, max_events=None):
processed_events = 0
for event in self._watch(
self._base_config_set_path, conf=conf, max_events=max_events):
if event is not None:
self._etcd_index = event.etcd_index
conf.update(
self._process_response_set(event, env_defaults=False))
processed_events += 1
return processed_events
def _watch(self, path, conf={}, wsgi_file=None, max_events=None):
i = 0
while (max_events is None) or (i < max_events):
try:
i += 1
res = self._client.watch(
path,
index=self._etcd_index,
recursive=True,
timeout=self.long_polling_timeout)
yield res
except Exception as e:
if not (isinstance(e, EtcdException)
and ('timed out' in str(e))):
self.logger.error("Long Polling Error: {}".format(e))
time.sleep(self.long_polling_safety_delay)
yield None
def set_env_defaults(self, env, conf={}):
path = self._env_defaults_path(env)
errors = {}
for k, v in conf.items():
if k.isupper():
try:
encoded_key = self._encode_config_key(k)
self._client.write(
"{}/{}".format(path, encoded_key),
self._encode_config_value(v))
except Exception as e:
errors[k] = str(e)
return errors
def set_config_sets(self, config_sets={}):
errors = {}
for set_name, config_set in config_sets.items():
path = self._config_set_path(set_name)
for k, v in config_set.items():
if k.isupper():
try:
self._client.write(
"{}/{}".format(path, self._encode_config_key(k)),
self._encode_config_value(v))
except Exception as e:
errors[k] = str(e)
return errors
def register_certbot(client: etcd.Client,
alb,
listener_id,
domains,
target,
certificate_name=None):
"""
Register a certbot for a given listener, this creates special rules for
this listener for allowing the certbot to verify the domain.
"""
try:
client.read("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=listener_id))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/alb/{alb}/certbot/{identifier}".format(
alb=alb, identifier=listener_id),
None,
dir=True)
client.write(
"/alb/{alb}/certbot/{identifier}/enabled".format(
alb=alb, identifier=listener_id), 'true')
client.write(
"/alb/{alb}/certbot/{identifier}/ready".format(alb=alb,
identifier=listener_id),
'false')
client.write(
"/alb/{alb}/certbot/{identifier}/certificate_name".format(
alb=alb, identifier=listener_id), certificate_name)
client.write(
"/alb/{alb}/certbot/{identifier}/domains".format(
alb=alb, identifier=listener_id), json.dumps(domains))
client.write(
"/alb/{alb}/certbot/{identifier}/target".format(
alb=alb, identifier=listener_id), json.dumps(target))
def register_listener_group(client: etcd.Client,
alb,
listener_id,
domains=None,
listeners=None,
certificate_name=None,
use_certbot=False):
try:
client.read("/alb/{alb}/listener_groups/{identifier}".format(
alb=alb, identifier=listener_id))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/alb/{alb}/listener_groups/{identifier}".format(
alb=alb, identifier=listener_id),
None,
dir=True)
# client.write("/alb/{alb}/listener_groups/{identifier}/name".format(alb=alb, identifier=listener_id), name)
client.write(
"/alb/{alb}/listener_groups/{identifier}/domains".format(
alb=alb, identifier=listener_id), json.dumps(domains))
client.write(
"/alb/{alb}/listener_groups/{identifier}/listeners".format(
alb=alb, identifier=listener_id), json.dumps(listeners))
client.write(
"/alb/{alb}/listener_groups/{identifier}/certificate_name".format(
alb=alb, identifier=listener_id), certificate_name)
client.write(
"/alb/{alb}/listener_groups/{identifier}/certbot_managed".format(
alb=alb, identifier=listener_id),
'true' if use_certbot else 'false')
def register_listener(client: etcd.Client,
alb,
identifier,
name,
port,
protocol,
rules,
certificate_name=None):
try:
client.read("/alb/{alb}/listeners/{identifier}".format(
alb=alb, identifier=identifier))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/alb/{alb}/listeners/{identifier}".format(
alb=alb, identifier=identifier),
None,
dir=True)
client.write(
"/alb/{alb}/listeners/{identifier}/name".format(alb=alb,
identifier=identifier),
name)
client.write(
"/alb/{alb}/listeners/{identifier}/protocol".format(
alb=alb, identifier=identifier), protocol)
client.write(
"/alb/{alb}/listeners/{identifier}/port".format(alb=alb,
identifier=identifier),
port)
client.write(
"/alb/{alb}/listeners/{identifier}/certificate_name".format(
alb=alb, identifier=identifier), certificate_name)
try:
client.read("/alb/{alb}/listeners/{identifier}/rules".format(
alb=alb, identifier=identifier))
except (etcd.EtcdKeyNotFound, KeyError):
client.write("/alb/{alb}/listeners/{identifier}/rules".format(
alb=alb, identifier=identifier),
None,
dir=True)
for rule in rules:
rule_id = rule['id']
rule_host = rule.get('host')
rule_path = rule.get('path')
action = rule.get('action')
try:
client.read(
"/alb/{alb}/listeners/{identifier}/rules/{rule}".format(
alb=alb, identifier=identifier, rule=rule_id))
except (etcd.EtcdKeyNotFound, KeyError):
client.write(
"/alb/{alb}/listeners/{identifier}/rules/{rule}".format(
alb=alb, identifier=identifier, rule=rule_id),
None,
dir=True)
client.write(
"/alb/{alb}/listeners/{identifier}/rules/{rule}/config".format(
alb=alb, identifier=identifier, rule=rule_id),
json.dumps({
'host': rule_host,
'path': rule_path,
'action': action,
}))