class AccountClient(Client):
def __init__(self, conf, **kwargs):
super(AccountClient, self).__init__(conf, **kwargs)
self.cs = ConscienceClient(self.conf)
# TODO keep account srv addr in local cache to avoid lookup requests
def _get_account_addr(self):
try:
acct_instance = self.cs.next_instance('account')
acct_addr = acct_instance.get('addr')
except Exception:
raise ClientException("No Account service found")
return acct_addr
def _make_uri(self, action):
account_addr = self._get_account_addr()
uri = 'http://%s/v1.0/account/%s' % (account_addr, action)
return uri
def _account_request(self, account, method, action, params={}):
uri = self._make_uri(action)
params['id'] = account
resp, body = self._direct_request(method, uri, params=params)
return resp, body
def account_create(self, account):
self._account_request(account, 'PUT', 'create')
def account_delete(self, account):
self._account_request(account, 'POST', 'delete')
def containers_list(self, account, marker=None, limit=None):
params = {}
if marker is not None:
params['marker'] = marker
if limit is not None:
params['limit'] = limit
resp, body = self._account_request(account,
'GET', 'containers', params)
return body
class AccountClient(HttpApi):
def __init__(self, conf, **kwargs):
super(AccountClient, self).__init__(**kwargs)
self.cs = ConscienceClient(conf, **kwargs)
# TODO keep account srv addr in local cache to avoid lookup requests
def _get_account_addr(self):
try:
acct_instance = self.cs.next_instance('account')
acct_addr = acct_instance.get('addr')
except Exception:
raise ClientException("No Account service found")
return acct_addr
def _make_uri(self, action):
account_addr = self._get_account_addr()
uri = 'http://%s/v1.0/account/%s' % (account_addr, action)
return uri
def _account_request(self, account, method, action, params={}, **kwargs):
uri = self._make_uri(action)
# FIXME: account must be urlencoded (utils.quote)
params['id'] = account
resp, body = self._direct_request(method, uri, params=params, **kwargs)
return resp, body
def account_create(self, account, **kwargs):
"""
Create an account.
:param account: name of the account to create
:type account: `str`
:returns: `True` if the account has been created
"""
resp, _body = self._account_request(account, 'PUT', 'create', **kwargs)
return resp.status_code == 201
def account_delete(self, account, **kwargs):
"""
Delete an account.
"""
self._account_request(account, 'POST', 'delete', **kwargs)
def account_list(self, **kwargs):
"""
List accounts.
"""
_resp, body = self._account_request(None, 'GET', 'list', **kwargs)
return body
def account_show(self, account, **kwargs):
"""
Get information about an account.
"""
_resp, body = self._account_request(account, 'GET', 'show', **kwargs)
return body
# FIXME: document this
def account_update(self, account, metadata, to_delete, **kwargs):
data = json.dumps({"metadata": metadata, "to_delete": to_delete})
self._account_request(account, 'POST', 'update', data=data, **kwargs)
def container_list(self,
account,
limit=None,
marker=None,
end_marker=None,
prefix=None,
delimiter=None,
**kwargs):
"""
Get the list of containers of an account.
:param account: account from which to get the container list
:type account: `str`
:keyword limit: maximum number of results to return
:type limit: `int`
:keyword marker: name of the container from where to start the listing
:type marker: `str`
:keyword end_marker:
:keyword prefix:
:keyword delimiter:
"""
params = {
"id": account,
"limit": limit,
"marker": marker,
"end_marker": end_marker,
"prefix": prefix,
"delimiter": delimiter
}
_resp, body = self._account_request(account,
'GET',
'containers',
params=params,
**kwargs)
return body
class AccountClient(HttpApi):
"""Simple client API for the account service."""
def __init__(self,
conf,
endpoint=None,
proxy_endpoint=None,
refresh_delay=3600.0,
logger=None,
**kwargs):
"""
Initialize a client for the account service.
:param conf: dictionary with at least the namespace name
:type conf: `dict`
:param endpoint: URL of an account service
:param proxy_endpoint: URL of the proxy
:param refresh_interval: time between refreshes of the
account service endpoint (if not provided at instantiation)
:type refresh_interval: `float` seconds
"""
super(AccountClient, self).__init__(endpoint=endpoint,
service_type='account-service',
**kwargs)
self.logger = logger or get_logger(conf)
self.cs = ConscienceClient(conf,
endpoint=proxy_endpoint,
logger=self.logger,
**kwargs)
self._global_kwargs = {
tok: float_value(tov, None)
for tok, tov in kwargs.items() if tok in TIMEOUT_KEYS
}
self._refresh_delay = refresh_delay if not self.endpoint else -1.0
self._last_refresh = 0.0
def _get_account_addr(self, **kwargs):
"""Fetch IP and port of an account service from Conscience."""
acct_instance = self.cs.next_instance('account', **kwargs)
acct_addr = acct_instance.get('addr')
return acct_addr
def _refresh_endpoint(self, now=None, **kwargs):
"""Refresh account service endpoint."""
addr = self._get_account_addr(**kwargs)
self.endpoint = '/'.join(("http:/", addr, "v1.0/account"))
if not now:
now = time.time()
self._last_refresh = now
def _maybe_refresh_endpoint(self, **kwargs):
"""Refresh account service endpoint if delay has been reached."""
if self._refresh_delay >= 0.0 or not self.endpoint:
now = time.time()
if now - self._last_refresh > self._refresh_delay:
try:
self._refresh_endpoint(now, **kwargs)
except OioNetworkException as exc:
if not self.endpoint:
# Cannot use the previous one
raise
self.logger.warn("Failed to refresh account endpoint: %s",
exc)
except OioException:
if not self.endpoint:
# Cannot use the previous one
raise
self.logger.exception("Failed to refresh account endpoint")
# Since all operations implemented in this class (as of 2019-08-08) result
# in only one request to the account service, we can patch the keyword
# arguments here. If this is changed, put the decorator on each public
# method of this class.
@patch_kwargs
def account_request(self, account, method, action, params=None, **kwargs):
"""Make a request to the account service."""
self._maybe_refresh_endpoint(**kwargs)
if not params:
params = dict()
if account:
# Do not quote account, _request() will urlencode query string
params['id'] = account
try:
resp, body = self._request(method, action, params=params, **kwargs)
except OioNetworkException as exc:
exc_info = sys.exc_info()
if self._refresh_delay >= 0.0:
self.logger.info("Refreshing account endpoint after error %s",
exc)
try:
self._refresh_endpoint(**kwargs)
except Exception as exc:
self.logger.warn("%s", exc)
raise exc_info[0], exc_info[1], exc_info[2]
return resp, body
def account_create(self, account, **kwargs):
"""
Create an account.
:param account: name of the account to create
:type account: `str`
:returns: `True` if the account has been created
"""
resp, _body = self.account_request(account, 'PUT', 'create', **kwargs)
return resp.status == 201
def account_delete(self, account, **kwargs):
"""
Delete an account.
:param account: name of the account to delete
:type account: `str`
"""
self.account_request(account, 'POST', 'delete', **kwargs)
def account_list(self, **kwargs):
"""
List accounts.
"""
_resp, body = self.account_request(None, 'GET', 'list', **kwargs)
return body
def account_show(self, account, **kwargs):
"""
Get information about an account.
"""
_resp, body = self.account_request(account, 'GET', 'show', **kwargs)
return body
def account_update(self, account, metadata, to_delete, **kwargs):
"""
Update metadata of the specified account.
:param metadata: dictionary of properties that must be set or updated.
:type metadata: `dict`
:param to_delete: list of property keys that must be removed.
:type to_delete: `list`
"""
data = json.dumps({"metadata": metadata, "to_delete": to_delete})
self.account_request(account, 'PUT', 'update', data=data, **kwargs)
def container_list(self,
account,
limit=None,
marker=None,
end_marker=None,
prefix=None,
delimiter=None,
s3_buckets_only=False,
**kwargs):
"""
Get the list of containers of an account.
:param account: account from which to get the container list
:type account: `str`
:keyword limit: maximum number of results to return
:type limit: `int`
:keyword marker: name of the container from where to start the listing
:type marker: `str`
:keyword end_marker:
:keyword prefix:
:keyword delimiter:
:keyword s3_buckets_only: list only S3 buckets.
:type s3_buckets_only: `bool`
:rtype: `dict` with 'ctime' (`float`), 'bytes' (`int`),
'objects' (`int`), 'containers' (`int`), 'id' (`str`),
'metadata' (`dict`) and 'listing' (`list`).
'listing' contains lists of container metadata (name,
number of objects, number of bytes, whether it is a prefix,
and modification time).
"""
params = {
"id": account,
"limit": limit,
"marker": marker,
"end_marker": end_marker,
"prefix": prefix,
"delimiter": delimiter,
"s3_buckets_only": s3_buckets_only
}
_resp, body = self.account_request(account,
'GET',
'containers',
params=params,
**kwargs)
return body
def container_update(self, account, container, metadata=None, **kwargs):
"""
Update account with container-related metadata.
:param account: name of the account to update
:type account: `str`
:param container: name of the container whose metadata has changed
:type container: `str`
:param metadata: container metadata ("bytes", "objects",
"mtime", "dtime")
:type metadata: `dict`
"""
metadata['name'] = container
_resp, body = self.account_request(account,
'PUT',
'container/update',
data=json.dumps(metadata),
**kwargs)
return body
def container_reset(self, account, container, mtime, **kwargs):
"""
Reset container of an account
:param account: name of the account
:type account: `str`
:param container: name of the container to reset
:type container: `str`
:param mtime: time of the modification
"""
metadata = dict()
metadata["name"] = container
metadata["mtime"] = mtime
self.account_request(account,
'PUT',
'container/reset',
data=json.dumps(metadata),
**kwargs)
def account_refresh(self, account, **kwargs):
"""
Refresh counters of an account
:param account: name of the account to refresh
:type account: `str`
"""
self.account_request(account, 'POST', 'refresh', **kwargs)
def account_flush(self, account, **kwargs):
"""
Flush all containers of an account
:param account: name of the account to flush
:type account: `str`
"""
self.account_request(account, 'POST', 'flush', **kwargs)
class EventWorker(Worker):
def __init__(self, *args, **kwargs):
super(EventWorker, self).__init__(*args, **kwargs)
self.app_env = dict()
def init(self):
self.tube = self.conf.get("tube", DEFAULT_TUBE)
self.cs = ConscienceClient(self.conf, logger=self.logger)
self.rdir = RdirClient(self.conf, logger=self.logger)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60)
self.app_env['acct_addr'] = self.acct_addr
if 'handlers_conf' not in self.conf:
raise ValueError("'handlers_conf' path not defined in conf")
self.handlers = loadhandlers(self.conf.get('handlers_conf'),
global_conf=self.conf,
app=self)
for opt in ('acct_update', 'rdir_update', 'retries_per_second',
'batch_size'):
if opt in self.conf:
self.logger.warn('Deprecated option: %s', opt)
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job_id, data):
try:
env = json.loads(data)
env['job_id'] = job_id
return env
except Exception as exc:
self.logger.warn('Failed to decode job %s: "%s"', job_id,
str(exc.message))
return None
def run(self):
coros = []
queue_url = self.conf.get('queue_url', 'beanstalk://127.0.0.1:11300')
concurrency = int_value(self.conf.get('concurrency'), 10)
server_gt = greenthread.getcurrent()
for url in queue_url.split(';'):
for i in range(concurrency):
beanstalk = Beanstalk.from_url(url)
gt = eventlet.spawn(self.handle, beanstalk)
gt.link(_eventlet_stop, server_gt, beanstalk)
coros.append(gt)
beanstalk, gt = None, None
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
[c.kill(StopServe()) for c in coros]
[c.wait() for c in coros]
except Timeout as te:
if te != t:
raise
[c.kill() for c in coros]
def handle(self, beanstalk):
conn_error = False
try:
if self.tube:
beanstalk.use(self.tube)
beanstalk.watch(self.tube)
while True:
try:
job_id, data = beanstalk.reserve()
if conn_error:
self.logger.warn("beanstalk reconnected")
conn_error = False
except ConnectionError:
if not conn_error:
self.logger.warn("beanstalk connection error")
conn_error = True
eventlet.sleep(BEANSTALK_RECONNECTION)
continue
event = self.safe_decode_job(job_id, data)
if not event:
self.logger.warn("Burying event %s: %s", job_id,
"malformed")
beanstalk.bury(job_id)
else:
try:
self.process_event(job_id, event, beanstalk)
except (ClientException, OioNetworkException) as exc:
self.logger.warn("Burying event %s (%s): %s", job_id,
event.get('event'), exc)
beanstalk.bury(job_id)
except ExplicitBury:
self.logger.info("Burying event %s (%s)", job_id,
event.get('event'))
beanstalk.bury(job_id)
except Exception:
self.logger.exception("Burying event %s: %s", job_id,
event)
beanstalk.bury(job_id)
except StopServe:
pass
def process_event(self, job_id, event, beanstalk):
handler = self.get_handler(event)
if not handler:
self.logger.warn('no handler found for %r' % event)
beanstalk.delete(job_id)
return
def cb(status, msg):
if is_success(status):
beanstalk.delete(job_id)
elif is_error(status):
self.logger.warn(
'event %s handling failure (release with delay): %s',
event['job_id'], msg)
beanstalk.release(job_id, delay=RELEASE_DELAY)
handler(event, cb)
def get_handler(self, event):
return self.handlers.get(event.get('event'), None)
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def __init__(self, conf, tool):
super(_DistributedDispatcher, self).__init__(conf, tool)
self.sending = None
self.max_items_per_second = int_value(
self.conf.get('items_per_second'),
self.tool.DEFAULT_ITEM_PER_SECOND)
# All available beanstalkd
conscience_client = ConscienceClient(self.conf)
all_beanstalkd = conscience_client.all_services('beanstalkd')
all_available_beanstalkd = dict()
for beanstalkd in all_beanstalkd:
if beanstalkd['score'] <= 0:
continue
all_available_beanstalkd[beanstalkd['addr']] = beanstalkd
if not all_available_beanstalkd:
raise OioException('No beanstalkd available')
# Beanstalkd workers
workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \
or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE
self.beanstalkd_workers = dict()
for beanstalkd in locate_tube(all_available_beanstalkd.values(),
workers_tube):
beanstalkd_worker = BeanstalkdSender(beanstalkd['addr'],
workers_tube, self.logger)
self.beanstalkd_workers[beanstalkd['addr']] = beanstalkd_worker
self.logger.info(
'Beanstalkd %s using tube %s is selected as a worker',
beanstalkd_worker.addr, beanstalkd_worker.tube)
if not self.beanstalkd_workers:
raise OioException('No beanstalkd worker available')
nb_workers = len(self.beanstalkd_workers)
if self.max_items_per_second > 0:
# Max 2 seconds in advance
queue_size_per_worker = self.max_items_per_second * 2 / nb_workers
else:
queue_size_per_worker = 64
for _, beanstalkd_worker in self.beanstalkd_workers.items():
beanstalkd_worker.low_limit = queue_size_per_worker / 2
beanstalkd_worker.high_limit = queue_size_per_worker
# Beanstalkd reply
beanstalkd_reply = dict()
try:
local_services = conscience_client.local_services()
for local_service in local_services:
if local_service['type'] != 'beanstalkd':
continue
beanstalkd = all_available_beanstalkd.get(
local_service['addr'])
if beanstalkd is None:
continue
if beanstalkd_reply \
and beanstalkd_reply['score'] >= beanstalkd['score']:
continue
beanstalkd_reply = beanstalkd
except Exception as exc: # pylint: disable=broad-except
self.logger.warning(
'ERROR when searching for beanstalkd locally: %s', exc)
if not beanstalkd_reply:
self.logger.warn('No beanstalkd available locally')
try:
beanstalkd = conscience_client.next_instance('beanstalkd')
beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']]
except Exception as exc: # pylint: disable=broad-except
self.logger.warning('ERROR when searching for beanstalkd: %s',
exc)
beanstalkd_reply_addr = beanstalkd_reply['addr']
# If the tube exists, another service must have already used this tube
tube_reply = workers_tube + '.reply.' + str(time.time())
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_reply_addr).tubes()
if tube_reply in tubes:
raise OioException('Beanstalkd %s using tube %s is already used')
self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr,
tube_reply, self.logger)
self.logger.info(
'Beanstalkd %s using tube %s is selected for the replies',
self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)
class EventWorker(Worker):
def init(self):
eventlet.monkey_patch(os=False)
self.tube = self.conf.get("tube", DEFAULT_TUBE)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60
)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
if 'handlers_conf' not in self.conf:
raise ValueError("'handlers_conf' path not defined in conf")
self.handlers = loadhandlers(
self.conf.get('handlers_conf'), evt_types, app=self)
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job_id, data):
try:
env = json.loads(data)
env['job_id'] = job_id
return env
except Exception as e:
self.logger.warn('decoding job "%s"', str(e.message))
return None
def run(self):
coros = []
queue_url = self.conf.get('queue_url', '127.0.0.1:11300')
concurrency = int_value(self.conf.get('concurrency'), 10)
server_gt = greenthread.getcurrent()
for i in range(concurrency):
beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle, beanstalk)
gt.link(_eventlet_stop, server_gt, beanstalk)
coros.append(gt)
beanstalk, gt = None, None
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
[c.kill(StopServe()) for c in coros]
[c.wait() for c in coros]
except Timeout as te:
if te != t:
raise
[c.kill() for c in coros]
def handle(self, beanstalk):
conn_error = False
try:
if self.tube:
beanstalk.use(self.tube)
beanstalk.watch(self.tube)
while True:
try:
job_id, data = beanstalk.reserve()
if conn_error:
self.logger.warn("beanstalk reconnected")
conn_error = False
except ConnectionError:
if not conn_error:
self.logger.warn("beanstalk connection error")
conn_error = True
eventlet.sleep(BEANSTALK_RECONNECTION)
continue
try:
event = self.safe_decode_job(job_id, data)
self.process_event(job_id, event, beanstalk)
except ConnectionError:
self.logger.warn(
"beanstalk connection error during processing")
except Exception:
beanstalk.bury(job_id)
self.logger.exception("handling event %s (bury)", job_id)
except StopServe:
pass
def process_event(self, job_id, event, beanstalk):
handler = self.get_handler(event)
if not handler:
self.logger.warn('no handler found for %r' % event)
beanstalk.delete(job_id)
return
def cb(status, msg):
if is_success(status):
beanstalk.delete(job_id)
elif is_error(status):
self.logger.warn('bury event %r' % event)
beanstalk.bury(job_id)
else:
self.logger.warn('release event %r' % event)
beanstalk.release(job_id)
handler(event, cb)
def get_handler(self, event):
return self.handlers.get(event.get('event'), None)
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
class XcuteClient(HttpApi):
"""Simple client API for the xcute service."""
def __init__(self, conf, endpoint=None, proxy_endpoint=None,
refresh_delay=3600.0, logger=None, **kwargs):
"""
Initialize a client for the xcute service.
:param conf: dictionary with at least the namespace name
:type conf: `dict`
:param endpoint: URL of an xcute service
:param proxy_endpoint: URL of the proxy
:param refresh_interval: time between refreshes of the
xcute service endpoint (if not provided at instantiation)
:type refresh_interval: `float` seconds
"""
super(XcuteClient, self).__init__(
endpoint=endpoint, service_type='xcute-service', **kwargs)
self.conf = conf
self.logger = logger or get_logger(self.conf)
self.conscience = ConscienceClient(conf, endpoint=proxy_endpoint,
logger=self.logger, **kwargs)
self._refresh_delay = refresh_delay if not self.endpoint else -1.0
self._last_refresh = 0.0
def _get_xcute_addr(self, **kwargs):
"""Fetch IP and port of an xcute service from Conscience."""
acct_instance = self.conscience.next_instance('xcute', **kwargs)
acct_addr = acct_instance.get('addr')
return acct_addr
def _refresh_endpoint(self, now=None, **kwargs):
"""Refresh xcute service endpoint."""
addr = self._get_xcute_addr(**kwargs)
self.endpoint = '/'. join(("http:/", addr, "v1.0/xcute"))
if not now:
now = time.time()
self._last_refresh = now
def _maybe_refresh_endpoint(self, **kwargs):
"""Refresh xcute service endpoint if delay has been reached."""
if self._refresh_delay >= 0.0 or not self.endpoint:
now = time.time()
if now - self._last_refresh > self._refresh_delay:
try:
self._refresh_endpoint(now, **kwargs)
except OioNetworkException as exc:
if not self.endpoint:
# Cannot use the previous one
raise
self.logger.warn(
"Failed to refresh xcute endpoint: %s", exc)
except OioException:
if not self.endpoint:
# Cannot use the previous one
raise
self.logger.exception("Failed to refresh xcute endpoint")
def xcute_request(self, method, action, params=None, **kwargs):
"""Make a request to the xcute service."""
self._maybe_refresh_endpoint(**kwargs)
if not params:
params = dict()
try:
resp, body = self._request(method, action, params=params, **kwargs)
except OioNetworkException as exc:
exc_info = sys.exc_info()
if self._refresh_delay >= 0.0:
self.logger.info(
"Refreshing xcute endpoint after error %s", exc)
try:
self._refresh_endpoint(**kwargs)
except Exception as exc:
self.logger.warn("%s", exc)
reraise(exc_info[0], exc_info[1], exc_info[2])
return resp, body
def job_list(self, limit=None, marker=None):
_, data = self.xcute_request(
'GET', '/job/list', params={'limit': limit, 'marker': marker})
return data
def job_create(self, job_type, job_config=None):
_, data = self.xcute_request(
'POST', '/job/create', params={'type': job_type}, json=job_config)
return data
def job_show(self, job_id):
_, data = self.xcute_request(
'GET', '/job/show', params={'id': job_id})
return data
def job_pause(self, job_id):
_, data = self.xcute_request(
'POST', '/job/pause', params={'id': job_id})
return data
def job_resume(self, job_id):
_, data = self.xcute_request(
'POST', '/job/resume', params={'id': job_id})
return data
def job_delete(self, job_id):
self.xcute_request(
'DELETE', '/job/delete', params={'id': job_id})
def lock_list(self):
_, data = self.xcute_request(
'GET', '/lock/list')
return data
def lock_show(self, lock):
_, data = self.xcute_request(
'GET', '/lock/show', params={'lock': lock})
return data
class EventWorker(object):
def __init__(self, conf, name, context, **kwargs):
self.conf = conf
self.name = name
verbose = kwargs.pop('verbose', False)
self.logger = get_logger(self.conf, verbose=verbose)
self.init_zmq(context)
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.acct_refresh_interval = int_value(
conf.get('acct_refresh_interval'), 60
)
self.acct_update = true_value(
conf.get('acct_update', True))
self.session = requests.Session()
self.failed = False
def start(self):
self.logger.info('worker "%s" starting', self.name)
self.running = True
self.run()
def stop(self):
self.logger.info('worker "%s" stopping', self.name)
self.running = False
def init_zmq(self, context):
socket = context.socket(zmq.REP)
socket.connect('inproc://event-front')
self.socket = socket
def safe_ack(self, msg):
try:
self.socket.send_multipart(msg)
except Exception:
self.logger.warn('Unable to ack event')
def run(self):
try:
while self.running:
msg = self.socket.recv_multipart()
self.logger.debug("msg received: %s" % msg)
event = decode_msg(msg)
success = self.process_event(event)
f = "0" if success else ""
self.safe_ack([msg[0], f])
except Exception as e:
self.logger.warn('ERROR in worker "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('worker "%s" stopped', self.name)
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("No handler found")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker "%s" handle container put', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker "%s" handle container update', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker "%s" handle container destroy', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker "%s" handle object delete', self.name)
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.exception(e)
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.info('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker "%s" handle object put', self.name)
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker "%s" handle reference update', self.name)
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
self.logger.debug('worker "%s" handle chunk creation', self.name)
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
self.logger.debug('worker "%s" handle chunk deletion', self.name)
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker "%s" handle ping', self.name)
class EventWorker(Worker):
def init(self):
eventlet.monkey_patch(os=False)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60)
self.concurrency = int_value(self.conf.get('concurrency'), 1000)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job):
try:
return json.loads(job)
except Exception as e:
self.logger.warn('ERROR decoding job "%s"', str(e.message))
return None
def run(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle)
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
gt.kill(StopServe())
gt.wait()
except Timeout as te:
if te != t:
raise
gt.kill()
def handle(self):
try:
while True:
job_id, data = self.beanstalk.reserve()
try:
event = self.safe_decode_job(data)
if event:
self.process_event(event)
self.beanstalk.delete(job_id)
except Exception:
self.logger.exception("ERROR handling event %s", job_id)
except StopServe:
self.logger.info('Stopping event handler')
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("ERROR no handler found for event")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker handle container put')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker handle container update')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker handle container destroy')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker handle object delete')
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.warn('error while deleting chunk %s "%s"',
chunk['id'], str(e.message))
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.debug('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker handle object put')
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker handle reference update')
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk creation')
return
self.logger.debug('worker handle chunk creation')
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk deletion')
return
self.logger.debug('worker handle chunk deletion')
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker handle ping')
class AccountClient(HttpApi):
"""Simple client API for the account service."""
def __init__(self,
conf,
endpoint=None,
proxy_endpoint=None,
refresh_delay=3600.0,
logger=None,
**kwargs):
"""
Initialize a client for the account service.
:param conf: dictionary with at least the namespace name
:type conf: `dict`
:param endpoint: URL of an account service
:param proxy_endpoint: URL of the proxy
:param refresh_interval: time between refreshes of the
account service endpoint (if not provided at instantiation)
:type refresh_interval: `float` seconds
"""
super(AccountClient, self).__init__(endpoint=endpoint, **kwargs)
self.logger = logger or get_logger(conf)
self.cs = ConscienceClient(conf,
endpoint=proxy_endpoint,
logger=self.logger,
**kwargs)
self._refresh_delay = refresh_delay if not self.endpoint else -1.0
self._last_refresh = 0.0
def _get_account_addr(self):
"""Fetch IP and port of an account service from Conscience."""
try:
acct_instance = self.cs.next_instance('account')
acct_addr = acct_instance.get('addr')
except Exception:
raise ClientException("No Account service found")
return acct_addr
def _refresh_endpoint(self, now=None):
"""Refresh account service endpoint."""
addr = self._get_account_addr()
self.endpoint = '/'.join(("http:/", addr, "v1.0/account"))
if not now:
now = time.time()
self._last_refresh = now
def _maybe_refresh_endpoint(self):
"""Refresh account service endpoint if delay has been reached."""
if self._refresh_delay >= 0.0 or not self.endpoint:
now = time.time()
if now - self._last_refresh > self._refresh_delay:
try:
self._refresh_endpoint(now)
except ClientException:
if not self.endpoint:
# Cannot use the previous one
raise
self.logger.exception("Failed to refresh account endpoint")
def account_request(self, account, method, action, params=None, **kwargs):
"""Make a request to the account service."""
self._maybe_refresh_endpoint()
if not params:
params = dict()
if account:
params['id'] = quote(account)
try:
resp, body = self._request(method, action, params=params, **kwargs)
except OioNetworkException as exc:
exc_info = sys.exc_info()
if self._refresh_delay >= 0.0:
self.logger.info("Refreshing account endpoint after error %s",
exc)
try:
self._refresh_endpoint()
except Exception as exc:
self.logger.warn("%s", exc)
raise exc_info[0], exc_info[1], exc_info[2]
return resp, body
def account_create(self, account, **kwargs):
"""
Create an account.
:param account: name of the account to create
:type account: `str`
:returns: `True` if the account has been created
"""
resp, _body = self.account_request(account, 'PUT', 'create', **kwargs)
return resp.status == 201
def account_delete(self, account, **kwargs):
"""
Delete an account.
:param account: name of the account to delete
:type account: `str`
"""
self.account_request(account, 'POST', 'delete', **kwargs)
def account_list(self, **kwargs):
"""
List accounts.
"""
_resp, body = self.account_request(None, 'GET', 'list', **kwargs)
return body
def account_show(self, account, **kwargs):
"""
Get information about an account.
"""
_resp, body = self.account_request(account, 'GET', 'show', **kwargs)
return body
# FIXME: document this
def account_update(self, account, metadata, to_delete, **kwargs):
data = json.dumps({"metadata": metadata, "to_delete": to_delete})
self.account_request(account, 'POST', 'update', data=data, **kwargs)
def container_list(self,
account,
limit=None,
marker=None,
end_marker=None,
prefix=None,
delimiter=None,
**kwargs):
"""
Get the list of containers of an account.
:param account: account from which to get the container list
:type account: `str`
:keyword limit: maximum number of results to return
:type limit: `int`
:keyword marker: name of the container from where to start the listing
:type marker: `str`
:keyword end_marker:
:keyword prefix:
:keyword delimiter:
"""
params = {
"id": account,
"limit": limit,
"marker": marker,
"end_marker": end_marker,
"prefix": prefix,
"delimiter": delimiter
}
_resp, body = self.account_request(account,
'GET',
'containers',
params=params,
**kwargs)
return body
def container_update(self, account, container, metadata=None, **kwargs):
"""
Update account with container-related metadata.
:param account: name of the account to update
:type account: `str`
:param container: name of the container whose metadata has changed
:type container: `str`
:param metadata: container metadata ("bytes", "objects",
"mtime", "dtime")
:type metadata: `dict`
"""
metadata['name'] = container
_resp, body = self.account_request(account,
'POST',
'container/update',
data=json.dumps(metadata))
return body
def container_reset(self, account, container, mtime, **kwargs):
"""
Reset container of an account
:param account: name of the account
:type account: `str`
:param container: name of the container to reset
:type container: `str`
:param mtime: time of the modification
"""
metadata = dict()
metadata["name"] = container
metadata["mtime"] = mtime
self.account_request(account,
'POST',
'container/reset',
data=json.dumps(metadata))
def account_refresh(self, account, **kwargs):
"""
Refresh counters of an account
:param account: name of the account to refresh
:type account: `str`
"""
self.account_request(account, 'POST', 'refresh')
def account_flush(self, account, **kwargs):
"""
Flush all containers of an account
:param account: name of the account to flush
:type account: `str`
"""
self.account_request(account, 'POST', 'flush')
class EventWorker(Worker):
def init(self):
eventlet.monkey_patch(os=False)
self.tube = self.conf.get("tube", DEFAULT_TUBE)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
if 'handlers_conf' not in self.conf:
raise ValueError("'handlers_conf' path not defined in conf")
self.handlers = loadhandlers(self.conf.get('handlers_conf'),
evt_types,
app=self)
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job_id, data):
try:
env = json.loads(data)
env['job_id'] = job_id
return env
except Exception as e:
self.logger.warn('decoding job "%s"', str(e.message))
return None
def run(self):
coros = []
queue_url = self.conf.get('queue_url', '127.0.0.1:11300')
concurrency = int_value(self.conf.get('concurrency'), 10)
server_gt = greenthread.getcurrent()
for i in range(concurrency):
beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(self.handle, beanstalk)
gt.link(_eventlet_stop, server_gt, beanstalk)
coros.append(gt)
beanstalk, gt = None, None
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
[c.kill(StopServe()) for c in coros]
[c.wait() for c in coros]
except Timeout as te:
if te != t:
raise
[c.kill() for c in coros]
def handle(self, beanstalk):
conn_error = False
try:
beanstalk.use(self.tube)
beanstalk.watch(self.tube)
while True:
try:
job_id, data = beanstalk.reserve()
if conn_error:
self.logger.warn("beanstalk reconnected")
conn_error = False
except ConnectionError:
if not conn_error:
self.logger.warn("beanstalk connection error")
conn_error = True
eventlet.sleep(BEANSTALK_RECONNECTION)
continue
try:
event = self.safe_decode_job(job_id, data)
self.process_event(job_id, event, beanstalk)
except ConnectionError:
self.logger.warn(
"beanstalk connection error during processing")
except Exception:
beanstalk.bury(job_id)
self.logger.exception("handling event %s (bury)", job_id)
except StopServe:
pass
def process_event(self, job_id, event, beanstalk):
handler = self.get_handler(event)
if not handler:
self.logger.warn('no handler found for %r' % event)
beanstalk.delete(job_id)
return
def cb(status, msg):
if is_success(status):
beanstalk.delete(job_id)
elif is_error(status):
self.logger.warn('bury event %r' % event)
beanstalk.bury(job_id)
else:
self.logger.warn('release event %r' % event)
beanstalk.release(job_id)
handler(event, cb)
def get_handler(self, event):
return self.handlers.get(event.get('event'), None)
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
class EventWorker(Worker):
def init(self):
eventlet.monkey_patch(os=False)
self.session = requests.Session()
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.graceful_timeout = 1
self.acct_refresh_interval = int_value(
self.conf.get('acct_refresh_interval'), 60
)
self.concurrency = int_value(self.conf.get('concurrency'), 1000)
self.acct_update = true_value(self.conf.get('acct_update', True))
self.rdir_update = true_value(self.conf.get('rdir_update', True))
super(EventWorker, self).init()
def notify(self):
"""TODO"""
pass
def safe_decode_job(self, job):
try:
return json.loads(job)
except Exception as e:
self.logger.warn('ERROR decoding job "%s"', str(e.message))
return None
def run(self):
queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300')
self.beanstalk = Beanstalk.from_url(queue_url)
gt = eventlet.spawn(
self.handle)
while self.alive:
self.notify()
try:
eventlet.sleep(1.0)
except AssertionError:
self.alive = False
break
self.notify()
try:
with Timeout(self.graceful_timeout) as t:
gt.kill(StopServe())
gt.wait()
except Timeout as te:
if te != t:
raise
gt.kill()
def handle(self):
try:
while True:
job_id, data = self.beanstalk.reserve()
try:
event = self.safe_decode_job(data)
if event:
self.process_event(event)
self.beanstalk.delete(job_id)
except Exception:
self.logger.exception("ERROR handling event %s", job_id)
except StopServe:
self.logger.info('Stopping event handler')
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("ERROR no handler found for event")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker handle container put')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker handle container update')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, json=event)
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker handle container destroy')
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker handle object delete')
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.warn('error while deleting chunk %s "%s"',
chunk['id'], str(e.message))
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.debug('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker handle object put')
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker handle reference update')
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk creation')
return
self.logger.debug('worker handle chunk creation')
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker skip chunk deletion')
return
self.logger.debug('worker handle chunk deletion')
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker handle ping')
class EventWorker(object):
def __init__(self, conf, name, context, **kwargs):
self.conf = conf
self.name = name
verbose = kwargs.pop('verbose', False)
self.logger = get_logger(self.conf, verbose=verbose)
self.init_zmq(context)
self.cs = ConscienceClient(self.conf)
self.rdir = RdirClient(self.conf)
self._acct_addr = None
self.acct_update = 0
self.acct_refresh_interval = int_value(
conf.get('acct_refresh_interval'), 60)
self.acct_update = true_value(conf.get('acct_update', True))
self.rdir_update = true_value(conf.get('rdir_update', True))
self.session = requests.Session()
self.failed = False
def start(self):
self.logger.info('worker "%s" starting', self.name)
self.running = True
self.run()
def stop(self):
self.logger.info('worker "%s" stopping', self.name)
self.running = False
def init_zmq(self, context):
socket = context.socket(zmq.REP)
socket.connect('inproc://event-front')
self.socket = socket
def safe_ack(self, msg):
try:
self.socket.send_multipart(msg)
except Exception:
self.logger.warn('Unable to ack event')
def run(self):
try:
while self.running:
msg = self.socket.recv_multipart()
self.logger.debug("msg received: %s" % msg)
event = decode_msg(msg)
success = self.process_event(event)
f = "0" if success else ""
self.safe_ack([msg[0], f])
except Exception as e:
self.logger.warn('ERROR in worker "%s"', e)
self.failed = True
raise e
finally:
self.logger.info('worker "%s" stopped', self.name)
def process_event(self, event):
handler = self.get_handler(event)
if not handler:
self.logger.warn("No handler found")
# mark as success
return True
success = True
try:
handler(event)
except Exception:
success = False
finally:
return success
def get_handler(self, event):
event_type = event.get('event')
if not event_type:
return None
if event_type == EventType.CONTAINER_PUT:
return self.handle_container_put
elif event_type == EventType.CONTAINER_DESTROY:
return self.handle_container_destroy
elif event_type == EventType.CONTAINER_UPDATE:
return self.handle_container_update
elif event_type == EventType.OBJECT_PUT:
return self.handle_object_put
elif event_type == EventType.OBJECT_DELETE:
return self.handle_object_delete
elif event_type == EventType.REFERENCE_UPDATE:
return self.handle_reference_update
elif event_type == EventType.CHUNK_PUT:
return self.handle_chunk_put
elif event_type == EventType.CHUNK_DELETE:
return self.handle_chunk_delete
elif event_type == EventType.PING:
return self.handle_ping
else:
return None
@property
def acct_addr(self):
if not self._acct_addr or self.acct_refresh():
try:
acct_instance = self.cs.next_instance(ACCOUNT_SERVICE)
self._acct_addr = acct_instance.get('addr')
self.acct_update = time.time()
except Exception:
self.logger.warn('Unable to find account instance')
return self._acct_addr
def acct_refresh(self):
return (time.time() - self.acct_update) > self.acct_refresh_interval
def handle_container_put(self, event):
"""
Handle container creation.
:param event:
"""
self.logger.debug('worker "%s" handle container put', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'mtime': mtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_update(self, event):
"""
Handle container update.
:param event:
"""
self.logger.debug('worker "%s" handle container update', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
mtime = event.get('when')
data = event.get('data')
name = event.get('url').get('user')
account = event.get('url').get('account')
bytes_count = data.get('bytes-count', 0)
object_count = data.get('object-count', 0)
event = {
'mtime': mtime,
'name': name,
'bytes': bytes_count,
'objects': object_count
}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_container_destroy(self, event):
"""
Handle container destroy.
:param event:
"""
self.logger.debug('worker "%s" handle container destroy', self.name)
if not self.acct_update:
return
uri = 'http://%s/v1.0/account/container/update' % self.acct_addr
dtime = event.get('when')
data = event.get('data')
name = data.get('url').get('user')
account = data.get('url').get('account')
event = {'dtime': dtime, 'name': name}
self.session.post(uri, params={'id': account}, data=json.dumps(event))
def handle_object_delete(self, event):
"""
Handle object deletion.
Delete the chunks of the object.
:param event:
"""
self.logger.debug('worker "%s" handle object delete', self.name)
pile = GreenPile(PARALLEL_CHUNKS_DELETE)
chunks = []
for item in event.get('data'):
if item.get('type') == 'chunks':
chunks.append(item)
if not len(chunks):
self.logger.warn('No chunks found in event data')
return
def delete_chunk(chunk):
resp = None
try:
with Timeout(CHUNK_TIMEOUT):
resp = self.session.delete(chunk['id'])
except (Exception, Timeout) as e:
self.logger.exception(e)
return resp
for chunk in chunks:
pile.spawn(delete_chunk, chunk)
resps = [resp for resp in pile if resp]
for resp in resps:
if resp.status_code == 204:
self.logger.info('deleted chunk %s' % resp.url)
else:
self.logger.warn('failed to delete chunk %s' % resp.url)
def handle_object_put(self, event):
"""
Handle object creation.
TODO
:param event:
"""
self.logger.debug('worker "%s" handle object put', self.name)
def handle_reference_update(self, event):
"""
Handle reference update.
TODO
:param event
"""
self.logger.debug('worker "%s" handle reference update', self.name)
def handle_chunk_put(self, event):
"""
Handle chunk creation.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker "%s" skip chunk creation', self.name)
return
self.logger.debug('worker "%s" handle chunk creation', self.name)
when = event.get('when')
data = event.get('data')
volume_id = data.get('volume_id')
del data['volume_id']
container_id = data.get('container_id')
del data['container_id']
content_id = data.get('content_id')
del data['content_id']
chunk_id = data.get('chunk_id')
del data['chunk_id']
data['mtime'] = when
self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id,
**data)
def handle_chunk_delete(self, event):
"""
Handle chunk deletion.
:param event
"""
if not self.rdir_update:
self.logger.debug('worker "%s" skip chunk deletion', self.name)
return
self.logger.debug('worker "%s" handle chunk deletion', self.name)
data = event.get('data')
volume_id = data.get('volume_id')
container_id = data.get('container_id')
content_id = data.get('content_id')
chunk_id = data.get('chunk_id')
self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id)
def handle_ping(self, event):
"""
Handle ping
:param event
"""
self.logger.debug('worker "%s" handle ping', self.name)
def __init__(self, conf, tool):
super(_DistributedDispatcher, self).__init__(conf, tool)
self.sending = False
# All available beanstalkd
conscience_client = ConscienceClient(self.conf)
all_beanstalkd = conscience_client.all_services('beanstalkd')
all_available_beanstalkd = dict()
for beanstalkd in all_beanstalkd:
if beanstalkd['score'] <= 0:
continue
all_available_beanstalkd[beanstalkd['addr']] = beanstalkd
if not all_available_beanstalkd:
raise OioException('No beanstalkd available')
# Beanstalkd workers
workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \
or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE
self.beanstalkd_workers = dict()
for _, beanstalkd in all_available_beanstalkd.items():
beanstalkd_worker_addr = beanstalkd['addr']
# If the tube exists,
# there should be a service that listens to this tube
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_worker_addr).tubes()
if workers_tube not in tubes:
continue
beanstalkd_worker = BeanstalkdSender(beanstalkd_worker_addr,
workers_tube, self.logger)
self.beanstalkd_workers[beanstalkd_worker_addr] = beanstalkd_worker
self.logger.info(
'Beanstalkd %s using tube %s is selected as a worker',
beanstalkd_worker.addr, beanstalkd_worker.tube)
if not self.beanstalkd_workers:
raise OioException('No beanstalkd worker available')
# Beanstalkd reply
beanstalkd_reply = dict()
try:
local_services = conscience_client.local_services()
for local_service in local_services:
if local_service['type'] != 'beanstalkd':
continue
beanstalkd = all_available_beanstalkd.get(
local_service['addr'])
if beanstalkd is None:
continue
if beanstalkd_reply \
and beanstalkd_reply['score'] >= beanstalkd['score']:
continue
beanstalkd_reply = beanstalkd
except Exception as exc: # pylint: disable=broad-except
self.logger.warning(
'ERROR when searching for beanstalkd locally: %s', exc)
if not beanstalkd_reply:
self.logger.warn('No beanstalkd available locally')
try:
beanstalkd = conscience_client.next_instance('beanstalkd')
beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']]
except Exception as exc: # pylint: disable=broad-except
self.logger.warning('ERROR when searching for beanstalkd: %s',
exc)
beanstalkd_reply_addr = beanstalkd_reply['addr']
# If the tube exists, another service must have already used this tube
tube_reply = workers_tube + '.reply.' + str(time.time())
tubes = Beanstalk.from_url('beanstalk://' +
beanstalkd_reply_addr).tubes()
if tube_reply in tubes:
raise OioException('Beanstalkd %s using tube %s is already used')
self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr,
tube_reply, self.logger)
self.logger.info(
'Beanstalkd %s using tube %s is selected for the replies',
self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)