Exemplo n.º 1
0
 def __init__(self,
              dev_params=None,
              prefix='config',
              protocol='http',
              host='localhost',
              port=2379,
              username=None,
              password=None,
              long_polling_timeout=50,
              long_polling_safety_delay=5):
     self._client = Client(host=host,
                           port=port,
                           protocol=protocol,
                           allow_redirect=True,
                           username=username,
                           password=password)
     # Overriding retries for urllib3.PoolManager.connection_pool_kw
     self._client.http.connection_pool_kw['retries'] = 0
     self._base_config_path = prefix
     self._dev_params = dev_params
     self._base_config_set_path = "{}/extensions" \
         .format(self._base_config_path)
     r = ('^(?P<path>{}/(?:extensions/)?'
          '(?P<envorset>[\w\-\.]+))/(?P<key>.+)$')
     self._key_regex = re.compile(r.format(self._base_config_path))
     self.long_polling_timeout = long_polling_timeout
     self.long_polling_safety_delay = long_polling_safety_delay
     self._init_logger()
Exemplo n.º 2
0
def remove_listener(client: etcd.Client, alb, identifier):
    try:
        client.delete("/alb/{alb}/listeners/{identifier}".format(
            alb=alb, identifier=identifier),
                      recursive=True)
    except (etcd.EtcdKeyNotFound, KeyError):
        pass
Exemplo n.º 3
0
def has_certificate(client: etcd.Client, certificate_name: str):
    """
    Check if certificate exists
    """
    try:
        client.read("/certs/{name}".format(name=certificate_name))
    except (etcd.EtcdKeyNotFound, KeyError):
        return False
    return True
Exemplo n.º 4
0
def main(etcdctl_peers, services_base, name, output, maps, links):
    click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
    click.secho("Links: %s" % " ".join(str(i) for i in links), fg='green')

    if len(links) == 0:
        click.echo('Nothing to do')
        sys.exit()

    etcd_url = urlparse(etcdctl_peers)
    etcd = Client(host=etcd_url.hostname, port=etcd_url.port)

    environment = {}

    # Get values from etcd for the services specified
    # --link <name or id>:alias
    for link in links:
        lname, _, alias = link.partition(":")
        if not alias:
            alias = lname.replace('-', '_')
        try:
            # Try to get the service
            service = etcd.read(os.path.join(services_base, lname))
        except EtcdKeyNotFound:
            click.secho("Service: %s not found" % lname, fg='red')
        else:
            # Expects the service to have some children
            for child in service._children:
                value = child['value']
                ip, _, port = value.partition(":")
                click.secho("Key: %s found" % lname, fg='green')
                environment["%s_NAME" %
                            alias.upper()] = "/%s/%s" % (name, lname)
                environment["%s_PORT" % alias.upper()] = "tcp://%s" % value
                environment["%s_PORT_%s_TCP" %
                            (alias.upper(), port)] = "tcp://%s" % value
                environment["%s_PORT_%s_TCP_PROTO" %
                            (alias.upper(), port)] = "tcp"
                environment["%s_PORT_%s_TCP_PORT" %
                            (alias.upper(), port)] = "%s" % port
                environment["%s_PORT_%s_TCP_ADDR" %
                            (alias.upper(), port)] = "%s" % ip

    # Do mapping
    for item in maps:
        mname, _, alias = item.partition(":")
        if mname and alias:
            try:
                environment[alias] = environment[mname]
            except KeyError:
                click.secho("Missing Key: %s" % mname, fg='red')

    with open(output, 'w') as f:
        for key, value in environment.iteritems():
            f.write("%s=%s\n" % (key, value))
            click.secho("%s=%s" % (key, value), fg='yellow')

    click.secho("All done.", fg='green')
Exemplo n.º 5
0
def unregister_certificate(client: etcd.Client, certificate_name: str):
    """
    Removes a registered certificate. If no certificate has been previously registered
    nothing happens.
    """
    try:
        client.delete("/certs/{name}".format(name=certificate_name),
                      recursive=True,
                      dir=True)
    except KeyError:
        pass
Exemplo n.º 6
0
def main(etcdctl_peers, services_base, name, output, maps, links):
    click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
    click.secho("Links: %s" % " ".join(str(i) for i in links), fg='green')

    if len(links) == 0:
        click.echo('Nothing to do')
        sys.exit()

    etcd_url = urlparse(etcdctl_peers)
    etcd = Client(host=etcd_url.hostname, port=etcd_url.port)

    environment = {}

    # Get values from etcd for the services specified
    # --link <name or id>:alias
    for link in links:
        lname, _, alias = link.partition(":")
        if not alias:
            alias = lname.replace('-', '_')
        try:
            # Try to get the service
            service = etcd.read(os.path.join(services_base, lname))
        except EtcdKeyNotFound:
            click.secho("Service: %s not found" % lname, fg='red')
        else:
            # Expects the service to have some children
            for child in service._children:
                value = child['value']
                ip, _, port = value.partition(":")
                click.secho("Key: %s found" % lname, fg='green')
                environment["%s_NAME" % alias.upper()] = "/%s/%s" % (name, lname)
                environment["%s_PORT" % alias.upper()] = "tcp://%s" % value
                environment["%s_PORT_%s_TCP" % (alias.upper(), port)] = "tcp://%s" % value
                environment["%s_PORT_%s_TCP_PROTO" % (alias.upper(), port)] = "tcp"
                environment["%s_PORT_%s_TCP_PORT" % (alias.upper(), port)] = "%s" % port
                environment["%s_PORT_%s_TCP_ADDR" % (alias.upper(), port)] = "%s" % ip

    # Do mapping
    for item in maps:
        mname, _, alias = item.partition(":")
        if mname and alias:
            try:
                environment[alias] = environment[mname]
            except KeyError:
                click.secho("Missing Key: %s" % mname, fg='red')

    with open(output, 'w') as f:
        for key, value in environment.iteritems():
            f.write("%s=%s\n" % (key, value))
            click.secho("%s=%s" % (key, value), fg='yellow')

    click.secho("All done.", fg='green')
Exemplo n.º 7
0
def unregister_certbot(client: etcd.Client, alb, listener_id):
    """
    Removes a registered certbot for a given listener. If no certbot has been previously registered
    nothing happens.
    """
    try:
        client.delete("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=listener_id),
                      recursive=True,
                      dir=True)
    except KeyError as e:
        logger.exception("Failed to unregister certbot: %s: %s",
                         type(e).__name__, e)
Exemplo n.º 8
0
 def __init__(self, config, section):
     super(EtcdStore, self).__init__(config, section)
     # Initialize the DB by trying to create the default table
     try:
         self.etcd = Client(self.etcd_server, self.etcd_port)
         self.etcd.write(self.namespace, None, dir=True)
     except EtcdNotFile:
         # Already exists
         pass
     except EtcdException:
         self.logger.exception("Error creating namespace %s",
                               self.namespace)
         raise CSStoreError('Error occurred while trying to init db')
 def __create_client(cls, etcd_host, etcd_port) -> Client:
     """ Create etcd client and test connection. """
     Logger(cls.__name__).info('Connecting to etcd server...')
     client = Client(host=etcd_host, port=etcd_port)
     # Test connection by trying to read a random value
     try:
         client.read('nodes')
     except EtcdConnectionFailed:
         raise EtcdConnectionError(port=etcd_port)
     except EtcdKeyNotFound:
         # This is to handle the case where etcd did not have the key (we don't care) but it is running
         pass
     return client
Exemplo n.º 10
0
def wait_certbot_ready(client: etcd.Client, alb, listener_id):
    try:
        client.read("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=listener_id))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=listener_id),
                     None,
                     dir=True)
    # Wait max 3 seconds for they entry to be ready
    entry = client.watch("/alb/{alb}/certbot/{identifier}/ready".format(
        alb=alb, identifier=listener_id),
                         timeout=3 * 60)
    return entry.value == 'true'
Exemplo n.º 11
0
 def __init__(self, keyspace=None, **kwargs):
     """
         Initialize the handler data store.
         :param keyspace: etcd keyspace for configuration map starting with /
         :type key: string
         :param kwargs: generic params forwarded from the Configmanager
         :type key: dict
     """
     super().__init__()
     self.client = Client(**kwargs)
     self.keyspace = keyspace if keyspace else '/config'
     if not self.keyspace.startswith('/'):
         self.keyspace = '/' + self.keyspace
     self.config = self.load()
Exemplo n.º 12
0
def _get_certificate(certificate_name,
                     with_pem=False,
                     client: etcd.Client = None):
    """
    :param with_pem: If True then it also loads the pem data
    :rtype: Optional[Certificate]
    """
    if client is None:
        host, port = get_etcd_addr()
        client = etcd.Client(host=host, port=int(port))

    cert_path = '/certs/{name}'.format(name=certificate_name)
    try:
        client.read(cert_path)
    except (etcd.EtcdKeyNotFound, KeyError):
        return None

    cert_pem = get_value(client, cert_path + '/cert') or None
    cert_domains = get_json(client, cert_path + '/domains', default=[])
    cert_email = get_value(client, cert_path + '/email') or None
    cert_modified = get_value(client, cert_path + '/modified') or None
    if cert_modified:
        try:
            cert_modified = datetime.strptime(cert_modified,
                                              "%Y-%m-%dT%H:%M:%S.%f%z")
        except ValueError:
            try:
                cert_modified = datetime.strptime(cert_modified,
                                                  "%Y-%m-%dT%H:%M:%S.%f")
            except ValueError:
                try:
                    cert_modified = datetime.strptime(cert_modified,
                                                      "%Y-%m-%dT%H:%M:%S%z")
                except ValueError:
                    cert_modified = None
    cert_is_valid = get_value(client, cert_path + '/is_valid')
    cert_is_valid = bool_lookup.get(cert_is_valid)

    certificate = Certificate(certificate_name,
                              pem_data=cert_pem,
                              domains=cert_domains,
                              email=cert_email,
                              modified=cert_modified,
                              is_valid=cert_is_valid)
    if with_pem:
        certificate.pem_data = _load_certificate_data(certificate,
                                                      client=client)

    return certificate
Exemplo n.º 13
0
def unregister_targets(client: etcd.Client, identifier, targets):
    try:
        client.read(
            "/target_group/{identifier}/targets".format(identifier=identifier))
    except (etcd.EtcdKeyNotFound, KeyError):
        return
    for target in targets:
        host = target['host']
        port = target['port']
        alb = target.get('alb')
        try:
            client.delete("/target_group/{identifier}/targets/{name}".format(
                identifier=identifier, name="{}:{}".format(host, port)))
        except (etcd.EtcdKeyNotFound, KeyError):
            pass
Exemplo n.º 14
0
class Registry(object):
    SERVICE_DIRECTORY = "/horizon/services"

    def __init__(self, ip, port):
        self._client = Client(ip, port)

    def create_directory(self, k, v):
        try:
            self._client.write(k, v, dir=True)
        except EtcdException, e:
            errorMessage = str(e) + "\n"
            errorMessage = "%s  [Registry] write %s = %s " % (errorMessage, k,
                                                              v)
            module_logger.fatal("[registry] write %s=%s" %
                                (errorMessage, k, v))
            raise EtcdException(errorMessage)
Exemplo n.º 15
0
 def __init__(self, port):
     asyncore.dispatcher.__init__(self)
     self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
     self.set_reuse_addr()
     self.bind(("localhost", port))
     self.listen(1)
     self.child_pids = None
     self.zk = KazooClient()
     self.etcd = Client(port=2379)
     self.port = port
     if self.prefork(10):
         # self.register_zk(port)
         self.register_etcd()
         self.register_parent_signal()
     else:
         # in child process, pid = 0
         self.register_child_signal()
Exemplo n.º 16
0
class EtcdWrapper:
    """
    Pack etcd library, send data to etcd
    """

    def __init__(self, conf):
        self.conf = conf
        self.host, self.port = self.conf['host'], self.conf['port']
        self.client = Client(host=self.host, port=self.port)
        self.test_connect()

    def test_connect(self):
        """
        test etcd server
        :return: 
        """
        url = 'http://' + self.host + ':' + str(self.port) + '/version'
        try:
            data = requests.get(url)
            if data.status_code == 200:
                log.info("etcd client init ok!")
                return True
            else:
                return False
        except Exception as e:
            log.error("\n%s", e)
            log.info("Check etcd server or network")
            return False

    def write(self, key, value):
        try:
            self.client.write(key, value)
        except Exception as e:
            log.error("\n%s", e)

    def read(self, key, value):
        try:
            return self.client.read(key)
        except Exception as e:
            log.error("\n%s", e)

    def delete(self, key):
        try:
            return self.client.delete(key)
        except Exception as e:
            log.error("\n%s", e)
Exemplo n.º 17
0
    def __init__(self, config):
        super(EtcdStore, self).__init__(config)
        self.server = config.get('etcd_server', '127.0.0.1')
        self.port = int(config.get('etcd_port', 4001))
        self.namespace = config.get('namespace', "/custodia")

        # Initialize the DB by trying to create the default table
        try:
            self.etcd = Client(self.server, self.port)
            self.etcd.write(self.namespace, None, dir=True)
        except EtcdNotFile:
            # Already exists
            pass
        except EtcdException:
            self.logger.exception("Error creating namespace %s",
                                  self.namespace)
            raise CSStoreError('Error occurred while trying to init db')
Exemplo n.º 18
0
def EtcdFactory(*args, **kwargs):
    """Factory method, returning a connection to a real etcd if we need one for
    FV, or to an in-memory implementation for UT."""
    if os.environ.get('ETCD_IP'):
        return Client(os.environ.get('ETCD_IP'),
                      int(os.environ.get('ETCD_PORT', 4001)))
    else:
        return MockEtcdClient(None, None)
Exemplo n.º 19
0
def main(etcdctl_peers, host, service_name, backend):
    etcd_url = urlparse(etcdctl_peers)
    etcd = Client(host=etcd_url.hostname, port=etcd_url.port)
    key = "/vulcand/frontends/%s/frontend" % service_name

    click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
    try:
        etcd.read(key='/')
    except EtcdConnectionFailed:
        click.secho('ETCD Connection Failed', fg='red')
        sys.exit(99)

    click.secho("Writing key %s" % key, fg='green')
    try:
        value = etcd.read(key=key).value
    except EtcdKeyNotFound:
        value = '{}'

    j = json.loads(value)
    j['Route'] = "Host(`%s`) && PathRegexp(`/.*`)" % host
    j['Type'] = 'http'

    if backend:
        j['BackendId'] = backend

    # Write / Update Key
    try:
        etcd.write(key=key, value=json.dumps(j))
    except Exception as e:
        raise e  # TODO: Handle specific exceptions

    click.secho("All done.", fg='green')
Exemplo n.º 20
0
    def test_order_of_entries_on_dump(self):
        backup = tempfile()

        with EtcdWrap() as etcd:
            try:
                client = Client(etcd.host, etcd.port)
                for i in range(10):
                    client.write("/{}".format(i), i**i)
                Dumper(etcd.client_urls[0]).dump(filename=backup)
            finally:
                etcd.terminate()

        self.assertGreater(stat(backup).st_size, 0)
        with open(backup, encoding="utf-8") as fp:
            lastindex = 0
            for i, entry in enumerate(load(fp)):
                self.assertEqual(entry["key"], "/{}".format(i))
                self.assertEqual(entry["value"], str(i**i))
                self.assertTrue(i <= lastindex < entry["index"])
                lastindex = entry["index"]
Exemplo n.º 21
0
def main(etcdctl_peers, host, service_name, backend):
    etcd_url = urlparse(etcdctl_peers)
    etcd = Client(host=etcd_url.hostname, port=etcd_url.port)
    key = "/vulcand/frontends/%s/frontend" % service_name

    click.secho("Connecting to ECTD: %s" % etcdctl_peers, fg='green')
    try:
        etcd.read(key='/')
    except EtcdConnectionFailed:
        click.secho('ETCD Connection Failed', fg='red')
        sys.exit(99)

    click.secho("Writing key %s" % key, fg='green')
    try:
        value = etcd.read(key=key).value
    except EtcdKeyNotFound:
        value = '{}'

    j = json.loads(value)
    j['Route'] = "Host(`%s`) && PathRegexp(`/.*`)" % host
    j['Type'] = 'http'

    if backend:
        j['BackendId'] = backend

    # Write / Update Key
    try:
        etcd.write(key=key, value=json.dumps(j))
    except Exception as e:
        raise e  # TODO: Handle specific exceptions

    click.secho("All done.", fg='green')
Exemplo n.º 22
0
def cluster_key(node_name, *clusters, **client_kwargs):
    key_name = '/unclustered/{node_name}'.format(node_name=node_name)

    client = Client(**client_kwargs)
    client.get_lock(key_name, ttl=60)
    value = client.get(key_name).value

    for cluster in clusters:
        client.set('/{cluster}/{node_name}'.format(cluster=cluster, node_name=node_name), value)

    return client.delete(key_name)
Exemplo n.º 23
0
def get_servers_with_etcd():
    client = Client(port=2379)

    # 获取新的服务地址,并监听服务变动

    def get_all_addr():
        new_addrs = set()
        rpc_dir = client.get(zk_rpc)
        for child in rpc_dir.leaves:
            addr = json.loads(child.value)
            new_addrs.add("%s:%d" % (addr["host"], addr["port"]))
        return new_addrs

    # 当前活跃地址
    current_addrs = get_all_addr()
    G["servers"] = [RemoteServer(s) for s in current_addrs]
    for _ in client.eternal_watch(zk_rpc, recursive=True):
        print("listening etcd")
        new_addrs = get_all_addr()
        # 新增
        add_addrs = new_addrs - current_addrs
        # 需要删除
        del_addrs = current_addrs - new_addrs
        del_servers = []
        for addr in del_addrs:
            for s in G["servers"]:
                if s.addr == addr:
                    del_servers.append(s)
                    break
        for server in del_servers:
            G["servers"].remove(server)
            current_addrs.remove(server.addr)
        # 新增
        for addr in add_addrs:
            G["servers"].append(RemoteServer(addr))
            current_addrs.add(addr)
        print("last g[servers]", G["servers"])
    return G["servers"]
Exemplo n.º 24
0
 def _connect(self):
     return Client(host=self.host,
                   port=self.port,
                   srv_domain=self.srv_domain,
                   version_prefix=self.version_prefix,
                   read_timeout=self.read_timeout,
                   allow_redirect=self.allow_redirect,
                   protocol=self.protocol,
                   cert=self.cert,
                   ca_cert=self.ca_cert,
                   username=self.username,
                   password=self.password,
                   allow_reconnect=self.allow_reconnect,
                   use_proxies=self.use_proxies,
                   expected_cluster_id=self.expected_cluster_id,
                   per_host_pool_size=self.per_host_pool_size)
Exemplo n.º 25
0
 def __init__(
         self, dev_params=None, prefix='config', protocol='http',
         host='localhost', port=2379, long_polling_timeout=50,
         long_polling_safety_delay=5):
     self._client = Client(
         host=host, port=port, protocol=protocol, allow_redirect=True)
     # Overriding retries for urllib3.PoolManager.connection_pool_kw
     self._client.http.connection_pool_kw['retries'] = 0
     self._base_config_path = prefix
     self._dev_params = dev_params
     self._base_config_set_path = "{}/extensions"\
         .format(self._base_config_path)
     r = ('^(?P<path>{}/(?:extensions/)?'
          '(?P<envorset>[\w\-\.]+))/(?P<key>.+)$')
     self._key_regex = re.compile(r.format(self._base_config_path))
     self._etcd_index = 0
     self.long_polling_timeout = long_polling_timeout
     self.long_polling_safety_delay = long_polling_safety_delay
     self._init_logger()
Exemplo n.º 26
0
def mark_certbot_ready(client: etcd.Client, alb, cerbot_id, is_ready=True):
    """
    Tell system a certbot is ready
    """
    try:
        client.read("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=cerbot_id))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=cerbot_id),
                     None,
                     dir=True)
    client.write(
        "/alb/{alb}/certbot/{identifier}/ready".format(alb=alb,
                                                       identifier=cerbot_id),
        'true' if is_ready else 'false')
Exemplo n.º 27
0
def register_target_group(client: etcd.Client,
                          identifier,
                          name,
                          targets,
                          protocol="http"):
    try:
        client.read("/target_group/{identifier}".format(identifier=identifier))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write(
            "/target_group/{identifier}".format(identifier=identifier),
            None,
            dir=True)
    client.write(
        "/target_group/{identifier}/name".format(identifier=identifier), name)
    client.write("/target_group/{identifier}/id".format(identifier=identifier),
                 identifier)
    # only http for now
    client.write(
        "/target_group/{identifier}/protocol".format(identifier=identifier),
        'http')
    # Health check config is hardcoded for now
    client.write(
        "/target_group/{identifier}/healthcheck".format(identifier=identifier),
        json.dumps({
            'protocol': 'http',
            'path': '/',
            # traffic port is the port of the first target
            'port': 'traffic',
            'healthy': 2,
            'unhealthy': 10,
            'timeout': 4,
            'interval': 5,
            'success': 200,
        }))

    try:
        client.read(
            "/target_group/{identifier}/targets".format(identifier=identifier))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write(
            "/target_group/{identifier}/targets".format(identifier=identifier),
            None,
            dir=True)
    for target in targets:
        host = target['host']
        port = target['port']
        alb = target.get('alb')
        # TODO: If the target is an ALB, then we need to register this ALB as the listener
        # in the target ALB. We also need to transfer any rules from the target to the listener
        client.write(
            "/target_group/{identifier}/targets/{name}".format(
                identifier=identifier, name="{}:{}".format(host, port)),
            json.dumps({
                'host': host,
                'port': port,
            }))
Exemplo n.º 28
0
def upload_certificate_file(client: etcd.Client,
                            certificate_name,
                            certificate_file,
                            modified: datetime = None):
    try:
        client.read("/certs/{cert_name}".format(cert_name=certificate_name))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/certs/{cert_name}".format(cert_name=certificate_name),
                     None,
                     dir=True)
    if isinstance(certificate_file, str):
        with open(certificate_file) as cert_fh:
            certificate_content = cert_fh.read()
    else:
        certificate_content = certificate_file.read()
    client.write("/certs/{cert_name}/cert".format(cert_name=certificate_name),
                 certificate_content)
    if not modified:
        modified = datetime.now()
    client.write("/certs/{name}/modified".format(name=certificate_name),
                 modified.isoformat())
    # TODO: Verify certificate data with ssl
    if certificate_content:
        client.write("/certs/{name}/is_valid".format(name=certificate_name),
                     'true')
    else:
        client.write("/certs/{name}/is_valid".format(name=certificate_name),
                     'false')
Exemplo n.º 29
0
def upload_certificate_data(client: etcd.Client,
                            certificate_name,
                            data,
                            modified: datetime = None):
    try:
        client.read("/certs/{cert_name}".format(cert_name=certificate_name))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/certs/{cert_name}".format(cert_name=certificate_name),
                     None,
                     dir=True)
    client.write("/certs/{cert_name}/cert".format(cert_name=certificate_name),
                 data)
    if not modified:
        modified = datetime.now()
    client.write("/certs/{name}/modified".format(name=certificate_name),
                 modified.isoformat())
    # TODO: Verify certificate data with ssl
    if data:
        client.write("/certs/{name}/is_valid".format(name=certificate_name),
                     'true')
    else:
        client.write("/certs/{name}/is_valid".format(name=certificate_name),
                     'false')
Exemplo n.º 30
0
def register_certificate(client: etcd.Client,
                         certificate_name: str,
                         domains: list = None,
                         email: str = None,
                         data: str = None,
                         modified: datetime = None):
    """
    Register a certificate with optional data, domains, email and modification date.
    """
    try:
        client.read("/certs/{name}".format(name=certificate_name))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/certs/{name}".format(name=certificate_name),
                     None,
                     dir=True)
    client.write("/certs/{name}/email".format(name=certificate_name), email)
    client.write("/certs/{name}/data".format(name=certificate_name), data)
    if not modified:
        modified = datetime.now()
    client.write("/certs/{name}/modified".format(name=certificate_name),
                 modified.isoformat())
    client.write("/certs/{name}/domains".format(name=certificate_name),
                 json.dumps(domains))
    # TODO: Verify certificate data with ssl
    if data:
        client.write("/certs/{name}/is_valid".format(name=certificate_name),
                     'true')
    else:
        client.write("/certs/{name}/is_valid".format(name=certificate_name),
                     'false')
Exemplo n.º 31
0
import uuid
from etcd import Client
c = Client(host='etcd-cluster.hostd.svc.tutum.io')
username = '******'
userHash = uuid.uuid4()
c.set("/hostd/users/%s" % username, '{"userHash":"%s", "username":"******"}' % (userHash, username))
Exemplo n.º 32
0
import uuid
from etcd import Client
c = Client(host='hostd-etcd.paalmoest.svc.tutum.io')

for i in range(10000):
    userHash = uuid.uuid4()
    c.set("/hostd/users/%s" % userHash, '{"userHash":"%s"}' % userHash)
Exemplo n.º 33
0
from etcd import Client
import os

etcd_host = os.environ.get("ETCD_HOST", os.getenv("ETCD_HOST"))
etcd_port = os.environ.get("ETCD_PORT", os.getenv("ETCD_PORT"))
client = Client(host=etcd_host, port=int(etcd_port))

# Execute in app start
# client.write('/user', None, dir=True)
# client.write('/ttl', None, dir=True)
# client.write('/type', None, dir=True)
# client.write('/zone', None, dir=True)
# client.write('/record', None, dir=True)
# client.write('/serial', None, dir=True)
# client.write('/conten', None, dir=True)

# client.write("/user/1", {"key": "1","email": "*****@*****.**", "project_id": "001","state": "inserted", "created_at":"2019-07-20 23:04:22.420505"})
client.write("/ttl/1", {"key": "1", "value": "300"})
client.write("/ttl/2", {"key": "2", "value": "900"})
client.write("/ttl/3", {"key": "3", "value": "1800"})
client.write("/ttl/4", {"key": "4", "value": "3600"})
client.write("/ttl/5", {"key": "5", "value": "7200"})
client.write("/ttl/6", {"key": "6", "value": "14400"})
client.write("/ttl/7", {"key": "7", "value": "28800"})
client.write("/ttl/8", {"key": "8", "value": "43200"})
client.write("/ttl/9", {"key": "9", "value": "86400"})
Exemplo n.º 34
0
class EtcdStore(CSStore):

    def __init__(self, config):
        super(EtcdStore, self).__init__(config)
        self.server = config.get('etcd_server', '127.0.0.1')
        self.port = int(config.get('etcd_port', 4001))
        self.namespace = config.get('namespace', "/custodia")

        # Initialize the DB by trying to create the default table
        try:
            self.etcd = Client(self.server, self.port)
            self.etcd.write(self.namespace, None, dir=True)
        except EtcdNotFile:
            # Already exists
            pass
        except EtcdException:
            self.logger.exception("Error creating namespace %s",
                                  self.namespace)
            raise CSStoreError('Error occurred while trying to init db')

    def _absolute_key(self, key):
        """Get absolute path to key and validate key"""
        if '//' in key:
            raise ValueError("Invalid empty components in key '%s'" % key)
        parts = key.split('/')
        if set(parts).intersection({'.', '..'}):
            raise ValueError("Invalid relative components in key '%s'" % key)
        return '/'.join([self.namespace] + parts).replace('//', '/')

    def get(self, key):
        self.logger.debug("Fetching key %s", key)
        try:
            result = self.etcd.get(self._absolute_key(key))
        except EtcdException:
            self.logger.exception("Error fetching key %s", key)
            raise CSStoreError('Error occurred while trying to get key')
        self.logger.debug("Fetched key %s got result: %r", key, result)
        return result.value

    def set(self, key, value, replace=False):
        self.logger.debug("Setting key %s to value %s (replace=%s)",
                          key, value, replace)
        path = self._absolute_key(key)
        try:
            self.etcd.write(path, value, prevExist=replace)
        except EtcdAlreadyExist as err:
            raise CSStoreExists(str(err))
        except EtcdException:
            self.logger.exception("Error storing key %s", key)
            raise CSStoreError('Error occurred while trying to store key')

    def span(self, key):
        path = self._absolute_key(key)
        self.logger.debug("Creating directory %s", path)
        try:
            self.etcd.write(path, None, dir=True, prevExist=False)
        except EtcdAlreadyExist as err:
            raise CSStoreExists(str(err))
        except EtcdException:
            self.logger.exception("Error storing key %s", key)
            raise CSStoreError('Error occurred while trying to store key')

    def list(self, keyfilter='/'):
        path = self._absolute_key(keyfilter)
        if path != '/':
            path = path.rstrip('/')
        self.logger.debug("Listing keys matching %s", path)
        try:
            result = self.etcd.read(path, recursive=True)
        except EtcdKeyNotFound:
            return None
        except EtcdException:
            self.logger.exception("Error listing %s", keyfilter)
            raise CSStoreError('Error occurred while trying to list keys')
        self.logger.debug("Searched for %s got result: %r", path, result)
        value = set()
        for entry in result.get_subtree():
            if entry.key == path:
                continue
            name = entry.key[len(path):]
            if entry.dir and not name.endswith('/'):
                name += '/'
            value.add(name.lstrip('/'))
        return sorted(value)

    def cut(self, key):
        self.logger.debug("Removing key %s", key)
        try:
            self.etcd.delete(self._absolute_key(key))
        except EtcdKeyNotFound:
            self.logger.debug("Key %s not found", key)
            return False
        except EtcdException:
            self.logger.exception("Error removing key %s", key)
            raise CSStoreError('Error occurred while trying to cut key')
        self.logger.debug("Key %s removed", key)
        return True
Exemplo n.º 35
0
class EtcdConfigManager():

    def __init__(
            self, dev_params=None, prefix='config', protocol='http',
            host='localhost', port=2379, long_polling_timeout=50,
            long_polling_safety_delay=5):
        self._client = Client(
            host=host, port=port, protocol=protocol, allow_redirect=True)
        # Overriding retries for urllib3.PoolManager.connection_pool_kw
        self._client.http.connection_pool_kw['retries'] = 0
        self._base_config_path = prefix
        self._dev_params = dev_params
        self._base_config_set_path = "{}/extensions"\
            .format(self._base_config_path)
        r = ('^(?P<path>{}/(?:extensions/)?'
             '(?P<envorset>[\w\-\.]+))/(?P<key>.+)$')
        self._key_regex = re.compile(r.format(self._base_config_path))
        self._etcd_index = 0
        self.long_polling_timeout = long_polling_timeout
        self.long_polling_safety_delay = long_polling_safety_delay
        self._init_logger()

    def _init_logger(self):
        self.logger = logging.getLogger('etcd_config_manager')
        logger_console_handler = logging.StreamHandler()
        logger_console_handler.setLevel(logging.ERROR)
        self.logger.addHandler(logger_console_handler)

    def _env_defaults_path(self, env='test'):
        return "{}/{}".format(self._base_config_path, env)

    def _config_set_path(self, set_name):
        return "{}/{}".format(self._base_config_set_path, set_name)

    def _encode_config_key(self, k):
        return k.lower().replace('_', '/')

    def _decode_config_key(self, k):
        [env_or_set, key_path] = re.sub(
            self._key_regex, '\g<envorset>|\g<key>', k).split('|')
        return env_or_set, key_path.upper().replace('/', '_')

    def _encode_config_value(self, val):
        return json.dumps(val, cls=CustomJSONEncoder)

    def _decode_config_value(self, val):
        decoded = json.loads(val, object_hook=custom_json_decoder_hook)
        return byteify(decoded)

    def _process_response_set(self, rset, env_defaults=True):
        d = {}
        for leaf in rset.leaves:
            try:
                config_set, key = self._decode_config_key(leaf.key)
            except ValueError:
                info = "An error occurred when processing an EtcdResponse"
                if not env_defaults:
                    info += " (is '{}' a directory?)".format(
                        self._base_config_set_path)
                self.logger.warning(info)
            else:
                if leaf.value is not None:
                    try:
                        value = self._decode_config_value(leaf.value)
                    except ValueError as e:
                        raise EtcdConfigInvalidValueError(
                            leaf.key, leaf.value, e)

                    if env_defaults:
                        d[key] = value
                    else:
                        if config_set not in d:
                            d[config_set] = {}
                        d[config_set][key] = value
        return d

    @staticmethod
    def get_dev_params(mod):
        params = {}
        if mod:
            params = attrs_to_dir(import_module(mod))
        return params

    def get_env_defaults(self, env):
        res = self._client.read(
            self._env_defaults_path(env),
            recursive=True)
        conf = self._process_response_set(res)
        conf.update(EtcdConfigManager.get_dev_params(self._dev_params))
        return conf

    def get_config_sets(self):
        conf = {}
        try:
            res = self._client.read(
                self._base_config_set_path,
                recursive=True)
            conf = self._process_response_set(res, env_defaults=False)
        except EtcdKeyNotFound:
            self.logger.warning(
                "Unable to find config sets at '{}' (expected a dict)",
                self._base_config_set_path)
        return conf

    @threaded(daemon=True)
    def monitor_env_defaults(
            self, env, conf={}, wsgi_file=None, max_events=None):
        processed_events = 0
        for event in self._watch(
                self._env_defaults_path(env), conf, wsgi_file, max_events):
            if event is not None:
                self._etcd_index = event.etcd_index
                conf.update(self._process_response_set(event))
                conf.update(EtcdConfigManager.get_dev_params(self._dev_params))
                if wsgi_file:
                    with open(wsgi_file, 'a'):
                        utime(wsgi_file, None)
            processed_events += 1
        return processed_events

    @threaded(daemon=True)
    def monitor_config_sets(self, conf={}, max_events=None):
        processed_events = 0
        for event in self._watch(
                self._base_config_set_path, conf=conf, max_events=max_events):
            if event is not None:
                self._etcd_index = event.etcd_index
                conf.update(
                    self._process_response_set(event, env_defaults=False))
            processed_events += 1
        return processed_events

    def _watch(self, path, conf={}, wsgi_file=None, max_events=None):
        i = 0
        while (max_events is None) or (i < max_events):
            try:
                i += 1
                res = self._client.watch(
                    path,
                    index=self._etcd_index,
                    recursive=True,
                    timeout=self.long_polling_timeout)
                yield res
            except Exception as e:
                if not (isinstance(e, EtcdException)
                        and ('timed out' in str(e))):
                    self.logger.error("Long Polling Error: {}".format(e))
                    time.sleep(self.long_polling_safety_delay)
                yield None

    def set_env_defaults(self, env, conf={}):
        path = self._env_defaults_path(env)
        errors = {}
        for k, v in conf.items():
            if k.isupper():
                try:
                    encoded_key = self._encode_config_key(k)
                    self._client.write(
                        "{}/{}".format(path, encoded_key),
                        self._encode_config_value(v))
                except Exception as e:
                    errors[k] = str(e)
        return errors

    def set_config_sets(self, config_sets={}):
        errors = {}
        for set_name, config_set in config_sets.items():
            path = self._config_set_path(set_name)
            for k, v in config_set.items():
                if k.isupper():
                    try:
                        self._client.write(
                            "{}/{}".format(path, self._encode_config_key(k)),
                            self._encode_config_value(v))
                    except Exception as e:
                        errors[k] = str(e)
        return errors
Exemplo n.º 36
0
def register_certbot(client: etcd.Client,
                     alb,
                     listener_id,
                     domains,
                     target,
                     certificate_name=None):
    """
    Register a certbot for a given listener, this creates special rules for
    this listener for allowing the certbot to verify the domain.
    """
    try:
        client.read("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=listener_id))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/alb/{alb}/certbot/{identifier}".format(
            alb=alb, identifier=listener_id),
                     None,
                     dir=True)
    client.write(
        "/alb/{alb}/certbot/{identifier}/enabled".format(
            alb=alb, identifier=listener_id), 'true')
    client.write(
        "/alb/{alb}/certbot/{identifier}/ready".format(alb=alb,
                                                       identifier=listener_id),
        'false')
    client.write(
        "/alb/{alb}/certbot/{identifier}/certificate_name".format(
            alb=alb, identifier=listener_id), certificate_name)
    client.write(
        "/alb/{alb}/certbot/{identifier}/domains".format(
            alb=alb, identifier=listener_id), json.dumps(domains))
    client.write(
        "/alb/{alb}/certbot/{identifier}/target".format(
            alb=alb, identifier=listener_id), json.dumps(target))
Exemplo n.º 37
0
def register_listener_group(client: etcd.Client,
                            alb,
                            listener_id,
                            domains=None,
                            listeners=None,
                            certificate_name=None,
                            use_certbot=False):
    try:
        client.read("/alb/{alb}/listener_groups/{identifier}".format(
            alb=alb, identifier=listener_id))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/alb/{alb}/listener_groups/{identifier}".format(
            alb=alb, identifier=listener_id),
                     None,
                     dir=True)
    # client.write("/alb/{alb}/listener_groups/{identifier}/name".format(alb=alb, identifier=listener_id), name)
    client.write(
        "/alb/{alb}/listener_groups/{identifier}/domains".format(
            alb=alb, identifier=listener_id), json.dumps(domains))
    client.write(
        "/alb/{alb}/listener_groups/{identifier}/listeners".format(
            alb=alb, identifier=listener_id), json.dumps(listeners))
    client.write(
        "/alb/{alb}/listener_groups/{identifier}/certificate_name".format(
            alb=alb, identifier=listener_id), certificate_name)
    client.write(
        "/alb/{alb}/listener_groups/{identifier}/certbot_managed".format(
            alb=alb, identifier=listener_id),
        'true' if use_certbot else 'false')
Exemplo n.º 38
0
def register_listener(client: etcd.Client,
                      alb,
                      identifier,
                      name,
                      port,
                      protocol,
                      rules,
                      certificate_name=None):
    try:
        client.read("/alb/{alb}/listeners/{identifier}".format(
            alb=alb, identifier=identifier))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/alb/{alb}/listeners/{identifier}".format(
            alb=alb, identifier=identifier),
                     None,
                     dir=True)
    client.write(
        "/alb/{alb}/listeners/{identifier}/name".format(alb=alb,
                                                        identifier=identifier),
        name)
    client.write(
        "/alb/{alb}/listeners/{identifier}/protocol".format(
            alb=alb, identifier=identifier), protocol)
    client.write(
        "/alb/{alb}/listeners/{identifier}/port".format(alb=alb,
                                                        identifier=identifier),
        port)
    client.write(
        "/alb/{alb}/listeners/{identifier}/certificate_name".format(
            alb=alb, identifier=identifier), certificate_name)
    try:
        client.read("/alb/{alb}/listeners/{identifier}/rules".format(
            alb=alb, identifier=identifier))
    except (etcd.EtcdKeyNotFound, KeyError):
        client.write("/alb/{alb}/listeners/{identifier}/rules".format(
            alb=alb, identifier=identifier),
                     None,
                     dir=True)

    for rule in rules:
        rule_id = rule['id']
        rule_host = rule.get('host')
        rule_path = rule.get('path')
        action = rule.get('action')
        try:
            client.read(
                "/alb/{alb}/listeners/{identifier}/rules/{rule}".format(
                    alb=alb, identifier=identifier, rule=rule_id))
        except (etcd.EtcdKeyNotFound, KeyError):
            client.write(
                "/alb/{alb}/listeners/{identifier}/rules/{rule}".format(
                    alb=alb, identifier=identifier, rule=rule_id),
                None,
                dir=True)
        client.write(
            "/alb/{alb}/listeners/{identifier}/rules/{rule}/config".format(
                alb=alb, identifier=identifier, rule=rule_id),
            json.dumps({
                'host': rule_host,
                'path': rule_path,
                'action': action,
            }))