def __init__(self, m2m_ep, use_xml, ca_certs=None, cert_file=None, key_file=None,
insecure=False):
super(OneM2MHTTPClient, self).__init__()
self.parsed_url = urlparse(m2m_ep)
is_https = self.parsed_url.scheme[-1].lower() == "s"
port = self.parsed_url.port or (is_https and 443 or 80)
host = self.parsed_url.hostname
self.path = self.parsed_url.path.rstrip('/')
if self.path and not self.path.endswith('/'):
self.path += '/'
# TODO(rst): handle IPv6 host here
# geventhttpclient sets incorrect host header
# i.e "host: ::1:8000" instead of "host: [::1]:8000
if is_https:
ssl_options = {
'ssl_version': self.DEF_SSL_VERSION
}
if ca_certs:
ssl_options['ca_certs'] = ca_certs
if cert_file and key_file:
ssl_options['certfile'] = cert_file
ssl_options['keyfile'] = key_file
else:
ssl_options = None
client = HTTPClient(host, port, connection_timeout=120.0,
concurrency=50, ssl=is_https,
ssl_options=ssl_options, insecure=insecure)
self.request = client.request
self.content_type = 'application/' + ('xml' if use_xml else 'json')
def get_http_client():
return HTTPClient(host,
port,
connection_timeout=120.0,
concurrency=50,
ssl=is_https,
ssl_options=ssl_options,
insecure=insecure)
def gevent_from_pool(self, url, http_request):
"""
Get a gevent client from url and request
:param url: geventhttpclient.url.URL
:type url: geventhttpclient.url.URL
:param http_request: HttpRequest
:type http_request: HttpRequest
:return HTTPClient
:rtype HTTPClient
"""
# Compute key
key = "{0}#{1}#{2}#{3}#{4}#{5}#{6}#{7}#{8}#{9}#".format(
# host and port
url.host,
url.port,
# Ssl
url.scheme == PROTO_HTTPS,
# Other dynamic stuff
http_request.https_insecure,
http_request.disable_ipv6,
http_request.connection_timeout_ms / 1000,
http_request.network_timeout_ms / 1000,
http_request.http_concurrency,
http_request.http_proxy_host,
http_request.http_proxy_port,
)
# Check
if key in self._gevent_pool:
SolBase.sleep(0)
return self._gevent_pool[key]
# Allocate (in lock)
with self._gevent_locker:
# Check maxed
if len(self._gevent_pool) >= self._gevent_pool_max:
raise Exception("gevent pool maxed, cur={0}, max={1}".format(
len(self._gevent_pool), self._gevent_pool_max
))
# Ok, allocate
http = HTTPClient.from_url(
url,
insecure=http_request.https_insecure,
disable_ipv6=http_request.disable_ipv6,
connection_timeout=http_request.connection_timeout_ms / 1000,
network_timeout=http_request.network_timeout_ms / 1000,
concurrency=http_request.http_concurrency,
proxy_host=http_request.http_proxy_host,
proxy_port=http_request.http_proxy_port,
headers={},
)
self._gevent_pool[key] = http
logger.info("Started new pool for key=%s", key)
SolBase.sleep(0)
return http
def __init__(self, api, config, *args, **kw):
super(Analytics, self).__init__(api=api, config=config, *args, **kw)
self._mongo_http = {
'host': config.get('host', '192.168.149.59'),
'port': config.get('port', 3000),
'method': config.get('method', 'POST'),
'path': config.get('path', '/store/measurement'),
'headers': config.get('headers',
{'content-type': 'application/json'})
}
self.__rest = HTTPClient(self._mongo_http['host'],
self._mongo_http['port'],
connection_timeout=120.0,
concurrency=50,
ssl=False)
self.fmt_json_regex = re.compile(r'application/([\w]+\+)?json')
def _get_client(self, parsed_url):
https = parsed_url.scheme[-1].lower() == "s"
port = parsed_url.port or (https and 443 or 80)
host = parsed_url.hostname
key = (host, port, https)
try:
return self._clients[key]
except KeyError:
# TODO: make connection_timeout and concurrency configurable
client = self._clients[key] = HTTPClient(host,
port,
connection_timeout=120.0,
concurrency=50,
ssl=https)
return client
def __init__(self, m2m_ep, use_xml):
super(OneM2MHTTPClient, self).__init__()
self.parsed_url = urlparse(m2m_ep)
https = self.parsed_url.scheme[-1].lower() == "s"
port = self.parsed_url.port or (https and 443 or 80)
host = self.parsed_url.hostname
self.path = self.parsed_url.path.rstrip('/')
if self.path and not self.path.endswith('/'):
self.path += '/'
# TODO(rst): handle IPv6 host here
# geventhttpclient sets incorrect host header
# i.e "host: ::1:8000" instead of "host: [::1]:8000
client = HTTPClient(host,
port,
connection_timeout=120.0,
concurrency=50,
ssl=https)
self.request = client.request
self.content_type = 'application/' + ('xml' if use_xml else 'json')
class Analytics(Plugin):
def __init__(self, api, config, *args, **kw):
super(Analytics, self).__init__(api=api, config=config, *args, **kw)
self._mongo_http = {
'host': config.get('host', '192.168.149.59'),
'port': config.get('port', 3000),
'method': config.get('method', 'POST'),
'path': config.get('path', '/store/measurement'),
'headers': config.get('headers',
{'content-type': 'application/json'})
}
self.__rest = HTTPClient(self._mongo_http['host'],
self._mongo_http['port'],
connection_timeout=120.0,
concurrency=50,
ssl=False)
self.fmt_json_regex = re.compile(r'application/([\w]+\+)?json')
def _init(self):
# resource created
self.events.resource_created.register_handler(self._handle_con_created,
Con)
self.events.resource_created.register_handler(self._handle_cin_created,
Cin)
self.events.resource_created.register_handler(
self._handle_container_created, Container)
self.events.resource_created.register_handler(
self._handle_content_instance_created, ContentInstance)
self._initialized()
def _handle_con_created(self, con, req):
pass
def _handle_cin_created(self, cin, req):
self.send_to_mongo_http(cin)
def _handle_container_created(self, con, req):
pass
def _handle_content_instance_created(self, cin, req):
pass
def send_to_mongo_http(self, cin):
response = None
try:
if "sensor_data" in cin.path:
if hasattr(cin, 'contentInfo'):
fmt, encoding_type = cin.contentInfo.split(':')
if not re.search(self.fmt_json_regex, fmt):
# TODO(rst): handle non json data
return
if int(encoding_type) == EncodingType.plain.value:
data = cin.content
else:
try:
data = b64decode(cin.content)
except Exception as e:
self.logger.error("Error decoding content:" +
str(e))
else:
data = b64decode(cin.content)
response = self.__rest.request(self._mongo_http['method'],
self._mongo_http['path'], data,
self._mongo_http['headers'])
except Exception as e:
self.logger.debug(e)
else:
if response is not None:
response.release()
def __init__(
self,
host='127.0.0.1',
port=4001,
read_timeout=60,
allow_redirect=True,
protocol='http',
cert=None,
ca_cert=None,
allow_reconnect=False,
):
"""
Initialize the client.
Args:
host (mixed):
If a string, IP to connect to.
If a tuple ((host, port), (host, port), ...)
port (int): Port used to connect to etcd.
read_timeout (int): max seconds to wait for a read.
allow_redirect (bool): allow the client to connect to other nodes.
protocol (str): Protocol used to connect to etcd.
cert (mixed): If a string, the whole ssl client certificate;
if a tuple, the cert and key file names.
ca_cert (str): The ca certificate. If pressent it will enable
validation.
allow_reconnect (bool): allow the client to reconnect to another
etcd server in the cluster in the case the
default one does not respond.
"""
self._machines_cache = []
self._protocol = protocol
def uri(protocol, host, port):
return '%s://%s:%d' % (protocol, host, port)
if not isinstance(host, tuple):
self._host = host
self._port = port
else:
self._host, self._port = host[0]
self._machines_cache.extend(
[uri(self._protocol, *conn) for conn in host])
self._base_uri = uri(self._protocol, self._host, self._port)
self.version_prefix = '/v2'
self._read_timeout = read_timeout
self._allow_redirect = allow_redirect
self._allow_reconnect = allow_reconnect
# SSL Client certificate support
kw = {}
ssl_options = {}
if self._read_timeout > 0:
kw['timeout'] = self._read_timeout
if protocol == 'https':
# If we don't allow TLSv1, clients using older version of OpenSSL
# (<1.0) won't be able to connect.
kw['ssl_version'] = ssl.PROTOCOL_TLSv1
ssl_options['ssl_version'] = ssl.PROTOCOL_TLSv1
if cert:
if isinstance(cert, tuple):
# Key and cert are separate
kw['cert_file'] = cert[0]
kw['key_file'] = cert[1]
ssl_options['certfile'] = cert[0]
ssl_options['keyfile'] = cert[1]
else:
# combined certificate
kw['cert_file'] = cert
ssl_options['certfile'] = cert
if ca_cert:
kw['ca_certs'] = ca_cert
kw['cert_reqs'] = ssl.CERT_REQUIRED
ssl_options['ca_certs'] = ca_cert
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
# self.http = urllib3.PoolManager(num_pools=10, **kw)
from geventhttpclient.client import HTTPClient
kw__ = {
'network_timeout': kw['timeout'],
'ssl_options': ssl_options,
'ssl': bool(protocol == 'https')
}
self.http = HTTPClient(self._host, self._port, concurrency=10, **kw__)
if self._allow_reconnect:
# we need the set of servers in the cluster in order to try
# reconnecting upon error.
self._machines_cache = self.machines
self._machines_cache.remove(self._base_uri)
else:
self._machines_cache = []
class Client(object):
"""
Client for etcd, the distributed log service using raft.
"""
_MGET = 'GET'
_MPUT = 'PUT'
_MPOST = 'POST'
_MDELETE = 'DELETE'
_comparison_conditions = set(('prevValue', 'prevIndex', 'prevExist'))
_read_options = set(
('recursive', 'wait', 'waitIndex', 'sorted', 'consistent'))
_del_conditions = set(('prevValue', 'prevIndex'))
def __init__(
self,
host='127.0.0.1',
port=4001,
read_timeout=60,
allow_redirect=True,
protocol='http',
cert=None,
ca_cert=None,
allow_reconnect=False,
):
"""
Initialize the client.
Args:
host (mixed):
If a string, IP to connect to.
If a tuple ((host, port), (host, port), ...)
port (int): Port used to connect to etcd.
read_timeout (int): max seconds to wait for a read.
allow_redirect (bool): allow the client to connect to other nodes.
protocol (str): Protocol used to connect to etcd.
cert (mixed): If a string, the whole ssl client certificate;
if a tuple, the cert and key file names.
ca_cert (str): The ca certificate. If pressent it will enable
validation.
allow_reconnect (bool): allow the client to reconnect to another
etcd server in the cluster in the case the
default one does not respond.
"""
self._machines_cache = []
self._protocol = protocol
def uri(protocol, host, port):
return '%s://%s:%d' % (protocol, host, port)
if not isinstance(host, tuple):
self._host = host
self._port = port
else:
self._host, self._port = host[0]
self._machines_cache.extend(
[uri(self._protocol, *conn) for conn in host])
self._base_uri = uri(self._protocol, self._host, self._port)
self.version_prefix = '/v2'
self._read_timeout = read_timeout
self._allow_redirect = allow_redirect
self._allow_reconnect = allow_reconnect
# SSL Client certificate support
kw = {}
ssl_options = {}
if self._read_timeout > 0:
kw['timeout'] = self._read_timeout
if protocol == 'https':
# If we don't allow TLSv1, clients using older version of OpenSSL
# (<1.0) won't be able to connect.
kw['ssl_version'] = ssl.PROTOCOL_TLSv1
ssl_options['ssl_version'] = ssl.PROTOCOL_TLSv1
if cert:
if isinstance(cert, tuple):
# Key and cert are separate
kw['cert_file'] = cert[0]
kw['key_file'] = cert[1]
ssl_options['certfile'] = cert[0]
ssl_options['keyfile'] = cert[1]
else:
# combined certificate
kw['cert_file'] = cert
ssl_options['certfile'] = cert
if ca_cert:
kw['ca_certs'] = ca_cert
kw['cert_reqs'] = ssl.CERT_REQUIRED
ssl_options['ca_certs'] = ca_cert
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
# self.http = urllib3.PoolManager(num_pools=10, **kw)
from geventhttpclient.client import HTTPClient
kw__ = {
'network_timeout': kw['timeout'],
'ssl_options': ssl_options,
'ssl': bool(protocol == 'https')
}
self.http = HTTPClient(self._host, self._port, concurrency=10, **kw__)
if self._allow_reconnect:
# we need the set of servers in the cluster in order to try
# reconnecting upon error.
self._machines_cache = self.machines
self._machines_cache.remove(self._base_uri)
else:
self._machines_cache = []
@property
def base_uri(self):
"""URI used by the client to connect to etcd."""
return self._base_uri
@property
def host(self):
"""Node to connect etcd."""
return self._host
@property
def port(self):
"""Port to connect etcd."""
return self._port
@property
def protocol(self):
"""Protocol used to connect etcd."""
return self._protocol
@property
def read_timeout(self):
"""Max seconds to wait for a read."""
return self._read_timeout
@property
def allow_redirect(self):
"""Allow the client to connect to other nodes."""
return self._allow_redirect
@property
def machines(self):
"""
Members of the cluster.
Returns:
list. str with all the nodes in the cluster.
>>> print client.machines
['http://127.0.0.1:4001', 'http://127.0.0.1:4002']
"""
return [
node.strip() for node in self.api_execute(
self.version_prefix +
'/machines', self._MGET).data.decode('utf-8').split(',')
]
@property
def leader(self):
"""
Returns:
str. the leader of the cluster.
>>> print client.leader
'http://127.0.0.1:4001'
"""
return self.api_execute(self.version_prefix + '/leader',
self._MGET).data.decode('ascii')
@property
def key_endpoint(self):
"""
REST key endpoint.
"""
return self.version_prefix + '/keys'
def __contains__(self, key):
"""
Check if a key is available in the cluster.
>>> print 'key' in client
True
"""
try:
self.get(key)
return True
except KeyError:
return False
def _sanitize_key(self, key):
if not key.startswith('/'):
key = "/{}".format(key)
return key
def write(self, key, value, ttl=None, dir=False, append=False, **kwdargs):
"""
Writes the value for a key, possibly doing atomit Compare-and-Swap
Args:
key (str): Key.
value (object): value to set
ttl (int): Time in seconds of expiration (optional).
dir (bool): Set to true if we are writing a directory; default is false.
append (bool): If true, it will post to append the new value to the dir, creating a sequential key. Defaults to false.
Other parameters modifying the write method are accepted:
prevValue (str): compare key to this value, and swap only if corresponding (optional).
prevIndex (int): modify key only if actual modifiedIndex matches the provided one (optional).
prevExist (bool): If false, only create key; if true, only update key.
Returns:
client.EtcdResult
>>> print client.write('/key', 'newValue', ttl=60, prevExist=False).value
'newValue'
"""
key = self._sanitize_key(key)
params = {}
if value is not None:
params['value'] = value
if ttl:
params['ttl'] = ttl
if dir:
if value:
raise etcd.EtcdException(
'Cannot create a directory with a value')
params['dir'] = "true"
for (k, v) in kwdargs.items():
if k in self._comparison_conditions:
if type(v) == bool:
params[k] = v and "true" or "false"
else:
params[k] = v
method = append and self._MPOST or self._MPUT
if '_endpoint' in kwdargs:
path = kwdargs['_endpoint'] + key
else:
path = self.key_endpoint + key
response = self.api_execute(path, method, params=params)
return self._result_from_response(response)
def update(self, obj):
"""
Updates the value for a key atomically. Typical usage would be:
c = etcd.Client()
o = c.read("/somekey")
o.value += 1
c.update(o)
Args:
obj (etcd.EtcdResult): The object that needs updating.
"""
kwdargs = {'dir': obj.dir, 'ttl': obj.ttl, 'prevExist': True}
if not obj.dir:
# prevIndex on a dir causes a 'not a file' error. d'oh!
kwdargs['prevIndex'] = obj.modifiedIndex
return self.write(obj.key, obj.value, **kwdargs)
def read(self, key, **kwdargs):
"""
Returns the value of the key 'key'.
Args:
key (str): Key.
Recognized kwd args
recursive (bool): If you should fetch recursively a dir
wait (bool): If we should wait and return next time the key is changed
waitIndex (int): The index to fetch results from.
sorted (bool): Sort the output keys (alphanumerically)
timeout (int): max seconds to wait for a read.
Returns:
client.EtcdResult (or an array of client.EtcdResult if a
subtree is queried)
Raises:
KeyValue: If the key doesn't exists.
urllib3.exceptions.TimeoutError: If timeout is reached.
>>> print client.get('/key').value
'value'
"""
key = self._sanitize_key(key)
params = {}
for (k, v) in kwdargs.items():
if k in self._read_options:
if type(v) == bool:
params[k] = v and "true" or "false"
else:
params[k] = v
timeout = kwdargs.get('timeout', None)
response = self.api_execute(self.key_endpoint + key,
self._MGET,
params=params,
timeout=timeout)
return self._result_from_response(response)
def delete(self, key, recursive=None, dir=None, **kwdargs):
"""
Removed a key from etcd.
Args:
key (str): Key.
recursive (bool): if we want to recursively delete a directory, set
it to true
dir (bool): if we want to delete a directory, set it to true
prevValue (str): compare key to this value, and swap only if
corresponding (optional).
prevIndex (int): modify key only if actual modifiedIndex matches the
provided one (optional).
Returns:
client.EtcdResult
Raises:
KeyValue: If the key doesn't exists.
>>> print client.delete('/key').key
'/key'
"""
key = self._sanitize_key(key)
kwds = {}
if recursive is not None:
kwds['recursive'] = recursive and "true" or "false"
if dir is not None:
kwds['dir'] = dir and "true" or "false"
for k in self._del_conditions:
if k in kwdargs:
kwds[k] = kwdargs[k]
response = self.api_execute(self.key_endpoint + key,
self._MDELETE,
params=kwds)
return self._result_from_response(response)
# Higher-level methods on top of the basic primitives
def test_and_set(self, key, value, prev_value, ttl=None):
"""
Atomic test & set operation.
It will check if the value of 'key' is 'prev_value',
if the the check is correct will change the value for 'key' to 'value'
if the the check is false an exception will be raised.
Args:
key (str): Key.
value (object): value to set
prev_value (object): previous value.
ttl (int): Time in seconds of expiration (optional).
Returns:
client.EtcdResult
Raises:
ValueError: When the 'prev_value' is not the current value.
>>> print client.test_and_set('/key', 'new', 'old', ttl=60).value
'new'
"""
return self.write(key, value, prevValue=prev_value, ttl=ttl)
def set(self, key, value, ttl=None):
"""
Compatibility: sets the value of the key 'key' to the value 'value'
Args:
key (str): Key.
value (object): value to set
ttl (int): Time in seconds of expiration (optional).
Returns:
client.EtcdResult
Raises:
etcd.EtcdException: when something weird goes wrong.
"""
return self.write(key, value, ttl=ttl)
def get(self, key):
"""
Returns the value of the key 'key'.
Args:
key (str): Key.
Returns:
client.EtcdResult
Raises:
KeyError: If the key doesn't exists.
>>> print client.get('/key').value
'value'
"""
return self.read(key)
def watch(self, key, index=None, timeout=None):
"""
Blocks until a new event has been received, starting at index 'index'
Args:
key (str): Key.
index (int): Index to start from.
timeout (int): max seconds to wait for a read.
Returns:
client.EtcdResult
Raises:
KeyValue: If the key doesn't exists.
urllib3.exceptions.TimeoutError: If timeout is reached.
>>> print client.watch('/key').value
'value'
"""
if index:
return self.read(key, wait=True, waitIndex=index, timeout=timeout)
else:
return self.read(key, wait=True, timeout=timeout)
def eternal_watch(self, key, index=None):
"""
Generator that will yield changes from a key.
Note that this method will block forever until an event is generated.
Args:
key (str): Key to subcribe to.
index (int): Index from where the changes will be received.
Yields:
client.EtcdResult
>>> for event in client.eternal_watch('/subcription_key'):
... print event.value
...
value1
value2
"""
local_index = index
while True:
response = self.watch(key, index=local_index, timeout=0)
if local_index is not None:
local_index += 1
yield response
def get_lock(self, *args, **kwargs):
return etcd.Lock(self, *args, **kwargs)
@property
def election(self):
return etcd.LeaderElection(self)
def _result_from_response(self, response):
""" Creates an EtcdResult from json dictionary """
try:
res = json.loads(response.read().decode('utf-8'))
r = etcd.EtcdResult(**res)
if response.status_code == 201:
r.newKey = True
r.parse_headers(response)
return r
except Exception as e:
raise etcd.EtcdException('Unable to decode server response: %s' %
e)
def _next_server(self):
""" Selects the next server in the list, refreshes the server list. """
try:
return self._machines_cache.pop()
except IndexError:
raise etcd.EtcdException('No more machines in the cluster')
def api_execute(self, path, method, params=None, timeout=None):
""" Executes the query. """
some_request_failed = False
response = False
if timeout is None:
timeout = self.read_timeout
if timeout == 0:
timeout = None
if not path.startswith('/'):
raise ValueError('Path does not start with /')
while not response:
try:
url = self._base_uri + path
# if (method == self._MGET) or (method == self._MDELETE):
# response = self.http.request(
# method,
# url,
# timeout=timeout,
# fields=params,
# redirect=self.allow_redirect)
# elif (method == self._MPUT) or (method == self._MPOST):
# response = self.http.request_encode_body(
# method,
# url,
# fields=params,
# timeout=timeout,
# encode_multipart=False,
# redirect=self.allow_redirect)
if method in (self._MGET, self._MPUT, self._MPOST,
self._MDELETE):
from urllib import urlencode
response = self.http.request(method,
url,
body=urlencode(params))
if method in (self._MGET, self._MPUT):
response = self.http.request(
method, "?".join([url, urlencode(params)]))
elif method in (self._MPOST, self._MDELETE):
response = self.http.request(method,
url,
body=urlencode(params))
else:
raise etcd.EtcdException(
'HTTP method {} not supported'.format(method))
except urllib3.exceptions.MaxRetryError:
self._base_uri = self._next_server()
some_request_failed = True
if some_request_failed:
self._machines_cache = self.machines
self._machines_cache.remove(self._base_uri)
return self._handle_server_response(response)
def _handle_server_response(self, response):
""" Handles the server response """
if response.status_code in [200, 201]:
return response
else:
resp = response.read().decode('utf-8')
# throw the appropriate exception
try:
r = json.loads(resp)
except ValueError:
r = None
if r:
etcd.EtcdError.handle(**r)
else:
raise etcd.EtcdException(resp)