def __init__(self, conf, **kwargs):
super(AdminClient, self).__init__(conf,
request_prefix="/admin",
**kwargs)
self.forwarder = ProxyClient(conf,
request_prefix="/forward",
pool_manager=self.pool_manager,
no_ns_in_url=True,
**kwargs)
def test_endpoint(self):
proxy_client = ProxyClient({"namespace": "OPENIO"},
endpoint="127.0.0.1:4444")
self.assertEqual(proxy_client.proxy_netloc, "127.0.0.1:4444")
self.assertEqual(proxy_client.endpoint,
"http://127.0.0.1:4444/v3.0/OPENIO")
proxy_client = ProxyClient({"namespace": "OPENIO"},
endpoint="http://127.0.0.1:4444")
self.assertEqual(proxy_client.proxy_netloc, "127.0.0.1:4444")
self.assertEqual(proxy_client.endpoint,
"http://127.0.0.1:4444/v3.0/OPENIO")
def _match(self, obj_meta, now=None, **kwargs):
"""
Check if versioning is enabled.
"""
if self.versioning is None:
data = self.lifecycle.api.container_get_properties(
self.lifecycle.account, self.lifecycle.container)
sys = data['system']
version = sys.get('sys.m2.policy.version', None)
if version is None:
from oio.common.client import ProxyClient
proxy_client = ProxyClient(
{"namespace": self.lifecycle.api.namespace},
no_ns_in_url=True)
_, data = proxy_client._request('GET', "config")
version = data['meta2.max_versions']
version = int(version)
self.versioning = version > 1 or version < 0
return self.versioning
def forwarder(self):
"""
Instanciate a client object for '/forward/*' proxy routes.
"""
if self._forwarder is None:
self._forwarder = ProxyClient(
self.conf, request_prefix="/forward",
pool_manager=self.pool_manager,
no_ns_in_url=True, **self._kwargs)
return self._forwarder
def cache_client(self):
"""
Instanciate a client object for '/cache/*' proxy routes.
"""
if self._cache_client is None:
self._cache_client = ProxyClient(
self.conf, request_prefix="/cache",
pool_manager=self.pool_manager,
no_ns_in_url=True, **self._kwargs)
return self._cache_client
def __init__(self, conf, service, **kwargs):
self.conf = conf
self.running = False
for k in ['host', 'port', 'type']:
if k not in service:
raise Exception('Missing field "%s" in service configuration' %
k)
self.name = '%s|%s|%s' % \
(service['type'], service['host'], service['port'])
self.service = service
self.rise = int_value(self._load_item_config('rise'), 1)
self.fall = int_value(self._load_item_config('fall'), 1)
self.check_interval = float_value(
self._load_item_config('check_interval'), 1)
self.deregister_on_exit = true_value(
self._load_item_config('deregister_on_exit', False))
self.logger = get_logger(self.conf)
self.pool_manager = get_pool_manager()
self.cs = ConscienceClient(self.conf,
pool_manager=self.pool_manager,
logger=self.logger)
# FIXME: explain that
self.client = ProxyClient(self.conf,
pool_manager=self.pool_manager,
no_ns_in_url=True,
logger=self.logger)
self.last_status = False
self.status = False
self.failed = False
self.service_definition = {
'ns': self.conf['namespace'],
'type': self.service['type'],
'addr': '%s:%s' % (self.service['host'], self.service['port']),
'score': 0,
'tags': {}
}
if self.service.get('slots', None):
self.service_definition['tags']['tag.slots'] = \
','.join(self.service['slots'])
for name, tag in (('location', 'tag.loc'),
('service_id', 'tag.service_id'), ('tls',
'tag.tls')):
if self.service.get(name):
self.service_definition['tags'][tag] = \
self.service[name]
self.service_checks = list()
self.service_stats = list()
self.init_checkers(service)
self.init_stats(service)
class MetaRebuilderWorker(RebuilderWorker):
def __init__(self, conf, logger, type_, max_attempts=5, **kwargs):
super(MetaRebuilderWorker, self).__init__(conf, logger, **kwargs)
self.type = type_
self.max_attempts = max_attempts
self.proxy_client = ProxyClient(
self.conf, request_prefix='/admin', logger=self.logger)
def _rebuild_one(self, cid, **kwargs):
attempts = 0
while True:
attempts += 1
params = {'cid': cid, 'type': self.type}
try:
properties = {'properties': {'sys.last_rebuild':
str(int(time.time()))}}
# Setting a property will trigger a database replication
self.proxy_client._request('POST', '/set_properties',
params=params, json=properties)
self.passes += 1
break
except Exception as err:
if attempts < self.max_attempts:
if isinstance(err, NotFound):
try:
self.proxy_client._request('POST', '/leave',
params=params)
except Exception:
pass
continue
if isinstance(err, ServiceBusy):
time.sleep(attempts * 0.5)
continue
self.logger.error('ERROR while rebuilding %s: %s', cid, err)
self.errors += 1
sys.stdout.write(cid+'\n')
break
class AdminClient(ProxyClient):
"""Low level database administration client."""
def __init__(self, conf, **kwargs):
super(AdminClient, self).__init__(
conf, request_prefix="/admin", **kwargs)
self.forwarder = ProxyClient(
conf, request_prefix="/forward", pool_manager=self.pool_manager,
no_ns_in_url=True, **kwargs)
@loc_params
def election_debug(self, params, **kwargs):
"""
Get debugging information about an election.
"""
_, body = self._request('POST', '/debug', params=params, **kwargs)
return body
@loc_params
def election_leave(self, params, **kwargs):
"""
Force all peers to leave the election.
"""
_, body = self._request('POST', '/leave', params=params, **kwargs)
return body
@loc_params
def election_ping(self, params, **kwargs):
"""
Trigger or refresh an election.
"""
_, body = self._request('POST', '/ping', params=params, **kwargs)
return body
@loc_params
def election_status(self, params, **kwargs):
"""
Get the status of an election (trigger it if necessary).
:returns: a `dict` with 'master' (`str`), 'slaves' (`list`),
'peers' (`dict`) and 'type' (`str`)
.. py:data:: example
{
'peers': {
'127.0.0.3:6014': {
'status':
{'status': 303,
'message': '127.0.0.1:6015'},
'body': u''},
'127.0.0.1:6015': {
'status':
{'status': 200,
'message': 'OK'},
'body': u''},
'127.0.0.2:6016': {
'status':
{'status': 303,
'message': '127.0.0.1:6015'},
'body': u''}
},
'master': '127.0.0.1:6015',
'slaves': ['127.0.0.3:6014', '127.0.0.2:6016'],
'type': 'meta1'
}
"""
_, body = self._request('POST', '/status', params=params, **kwargs)
resp = {'peers': body, 'type': params['type']}
for svc_id in body.keys():
if body[svc_id]['status']['status'] == 200:
resp['master'] = svc_id
elif body[svc_id]['status']['status'] == 303:
slaves = resp.get('slaves', [])
slaves.append(svc_id)
resp['slaves'] = slaves
return resp
@loc_params
def election_sync(self, params, **kwargs):
"""Try to synchronize a dubious election."""
_, body = self._request('POST', '/sync', params=params, **kwargs)
return body
@loc_params
def set_properties(self, params,
properties=None, system=None, **kwargs):
"""
Set user or system properties in the admin table of an sqliterepo base.
"""
data = dict()
if properties:
data['properties'] = properties
if system:
data['system'] = dict()
for k, v in system:
data['system'][k if k.startswith('sys.') else 'sys.' + k] = v
self._request('POST', "/set_properties",
params=params, json=data, **kwargs)
@loc_params
def get_properties(self, params, **kwargs):
"""
Get user and system properties from the admin table of an
sqliterepo base.
"""
_resp, body = self._request('POST', "/get_properties",
params=params, data='', **kwargs)
return body
@loc_params
def set_peers(self, params, peers, **kwargs):
"""
Force the new peer set in the replicas of the old peer set.
"""
data = {'system': {'sys.peers': ','.join(peers)}}
self._request('POST', "/set_properties",
params=params, json=data, **kwargs)
@loc_params
def copy_base_from(self, params, svc_from, svc_to, **kwargs):
"""
Copy a base to another service, using DB_PIPEFROM.
:param svc_from: id of the source service.
:param svc_to: id of the destination service.
"""
data = {'to': svc_to, 'from': svc_from}
self._request('POST', "/copy",
params=params, json=data, **kwargs)
@loc_params
def copy_base_to(self, params, svc_to, **kwargs):
"""
Copy a base to another service, using DB_PIPETO.
Source service is looked after in service directory.
:param svc_to: id of the destination service.
"""
self._request('POST', "/copy",
params=params, json={'to': svc_to}, **kwargs)
def _forward_service_action(self, svc_id, action, **kwargs):
"""Execute service-specific actions."""
self.forwarder._request('POST', action,
params={'id': svc_id}, **kwargs)
def service_flush_cache(self, svc_id, **kwargs):
"""Flush the resolver cache of an sqlx-bases service."""
self._forward_service_action(svc_id, '/flush', **kwargs)
def setUp(self):
super(TestProxyClient, self).setUp()
self.proxy_client = ProxyClient({"namespace": self.ns},
request_attempts=2)
def __init__(self, conf, logger, type_, max_attempts=5, **kwargs):
super(MetaRebuilderWorker, self).__init__(conf, logger, **kwargs)
self.type = type_
self.max_attempts = max_attempts
self.proxy_client = ProxyClient(
self.conf, request_prefix='/admin', logger=self.logger)