class AccountClient(Client): def __init__(self, conf, **kwargs): super(AccountClient, self).__init__(conf, **kwargs) self.cs = ConscienceClient(self.conf) # TODO keep account srv addr in local cache to avoid lookup requests def _get_account_addr(self): try: acct_instance = self.cs.next_instance('account') acct_addr = acct_instance.get('addr') except Exception: raise ClientException("No Account service found") return acct_addr def _make_uri(self, action): account_addr = self._get_account_addr() uri = 'http://%s/v1.0/account/%s' % (account_addr, action) return uri def _account_request(self, account, method, action, params={}): uri = self._make_uri(action) params['id'] = account resp, body = self._direct_request(method, uri, params=params) return resp, body def account_create(self, account): self._account_request(account, 'PUT', 'create') def account_delete(self, account): self._account_request(account, 'POST', 'delete') def containers_list(self, account, marker=None, limit=None): params = {} if marker is not None: params['marker'] = marker if limit is not None: params['limit'] = limit resp, body = self._account_request(account, 'GET', 'containers', params) return body
class AccountClient(HttpApi): def __init__(self, conf, **kwargs): super(AccountClient, self).__init__(**kwargs) self.cs = ConscienceClient(conf, **kwargs) # TODO keep account srv addr in local cache to avoid lookup requests def _get_account_addr(self): try: acct_instance = self.cs.next_instance('account') acct_addr = acct_instance.get('addr') except Exception: raise ClientException("No Account service found") return acct_addr def _make_uri(self, action): account_addr = self._get_account_addr() uri = 'http://%s/v1.0/account/%s' % (account_addr, action) return uri def _account_request(self, account, method, action, params={}, **kwargs): uri = self._make_uri(action) # FIXME: account must be urlencoded (utils.quote) params['id'] = account resp, body = self._direct_request(method, uri, params=params, **kwargs) return resp, body def account_create(self, account, **kwargs): """ Create an account. :param account: name of the account to create :type account: `str` :returns: `True` if the account has been created """ resp, _body = self._account_request(account, 'PUT', 'create', **kwargs) return resp.status_code == 201 def account_delete(self, account, **kwargs): """ Delete an account. """ self._account_request(account, 'POST', 'delete', **kwargs) def account_list(self, **kwargs): """ List accounts. """ _resp, body = self._account_request(None, 'GET', 'list', **kwargs) return body def account_show(self, account, **kwargs): """ Get information about an account. """ _resp, body = self._account_request(account, 'GET', 'show', **kwargs) return body # FIXME: document this def account_update(self, account, metadata, to_delete, **kwargs): data = json.dumps({"metadata": metadata, "to_delete": to_delete}) self._account_request(account, 'POST', 'update', data=data, **kwargs) def container_list(self, account, limit=None, marker=None, end_marker=None, prefix=None, delimiter=None, **kwargs): """ Get the list of containers of an account. :param account: account from which to get the container list :type account: `str` :keyword limit: maximum number of results to return :type limit: `int` :keyword marker: name of the container from where to start the listing :type marker: `str` :keyword end_marker: :keyword prefix: :keyword delimiter: """ params = { "id": account, "limit": limit, "marker": marker, "end_marker": end_marker, "prefix": prefix, "delimiter": delimiter } _resp, body = self._account_request(account, 'GET', 'containers', params=params, **kwargs) return body
class AccountClient(HttpApi): """Simple client API for the account service.""" def __init__(self, conf, endpoint=None, proxy_endpoint=None, refresh_delay=3600.0, logger=None, **kwargs): """ Initialize a client for the account service. :param conf: dictionary with at least the namespace name :type conf: `dict` :param endpoint: URL of an account service :param proxy_endpoint: URL of the proxy :param refresh_interval: time between refreshes of the account service endpoint (if not provided at instantiation) :type refresh_interval: `float` seconds """ super(AccountClient, self).__init__(endpoint=endpoint, service_type='account-service', **kwargs) self.logger = logger or get_logger(conf) self.cs = ConscienceClient(conf, endpoint=proxy_endpoint, logger=self.logger, **kwargs) self._global_kwargs = { tok: float_value(tov, None) for tok, tov in kwargs.items() if tok in TIMEOUT_KEYS } self._refresh_delay = refresh_delay if not self.endpoint else -1.0 self._last_refresh = 0.0 def _get_account_addr(self, **kwargs): """Fetch IP and port of an account service from Conscience.""" acct_instance = self.cs.next_instance('account', **kwargs) acct_addr = acct_instance.get('addr') return acct_addr def _refresh_endpoint(self, now=None, **kwargs): """Refresh account service endpoint.""" addr = self._get_account_addr(**kwargs) self.endpoint = '/'.join(("http:/", addr, "v1.0/account")) if not now: now = time.time() self._last_refresh = now def _maybe_refresh_endpoint(self, **kwargs): """Refresh account service endpoint if delay has been reached.""" if self._refresh_delay >= 0.0 or not self.endpoint: now = time.time() if now - self._last_refresh > self._refresh_delay: try: self._refresh_endpoint(now, **kwargs) except OioNetworkException as exc: if not self.endpoint: # Cannot use the previous one raise self.logger.warn("Failed to refresh account endpoint: %s", exc) except OioException: if not self.endpoint: # Cannot use the previous one raise self.logger.exception("Failed to refresh account endpoint") # Since all operations implemented in this class (as of 2019-08-08) result # in only one request to the account service, we can patch the keyword # arguments here. If this is changed, put the decorator on each public # method of this class. @patch_kwargs def account_request(self, account, method, action, params=None, **kwargs): """Make a request to the account service.""" self._maybe_refresh_endpoint(**kwargs) if not params: params = dict() if account: # Do not quote account, _request() will urlencode query string params['id'] = account try: resp, body = self._request(method, action, params=params, **kwargs) except OioNetworkException as exc: exc_info = sys.exc_info() if self._refresh_delay >= 0.0: self.logger.info("Refreshing account endpoint after error %s", exc) try: self._refresh_endpoint(**kwargs) except Exception as exc: self.logger.warn("%s", exc) raise exc_info[0], exc_info[1], exc_info[2] return resp, body def account_create(self, account, **kwargs): """ Create an account. :param account: name of the account to create :type account: `str` :returns: `True` if the account has been created """ resp, _body = self.account_request(account, 'PUT', 'create', **kwargs) return resp.status == 201 def account_delete(self, account, **kwargs): """ Delete an account. :param account: name of the account to delete :type account: `str` """ self.account_request(account, 'POST', 'delete', **kwargs) def account_list(self, **kwargs): """ List accounts. """ _resp, body = self.account_request(None, 'GET', 'list', **kwargs) return body def account_show(self, account, **kwargs): """ Get information about an account. """ _resp, body = self.account_request(account, 'GET', 'show', **kwargs) return body def account_update(self, account, metadata, to_delete, **kwargs): """ Update metadata of the specified account. :param metadata: dictionary of properties that must be set or updated. :type metadata: `dict` :param to_delete: list of property keys that must be removed. :type to_delete: `list` """ data = json.dumps({"metadata": metadata, "to_delete": to_delete}) self.account_request(account, 'PUT', 'update', data=data, **kwargs) def container_list(self, account, limit=None, marker=None, end_marker=None, prefix=None, delimiter=None, s3_buckets_only=False, **kwargs): """ Get the list of containers of an account. :param account: account from which to get the container list :type account: `str` :keyword limit: maximum number of results to return :type limit: `int` :keyword marker: name of the container from where to start the listing :type marker: `str` :keyword end_marker: :keyword prefix: :keyword delimiter: :keyword s3_buckets_only: list only S3 buckets. :type s3_buckets_only: `bool` :rtype: `dict` with 'ctime' (`float`), 'bytes' (`int`), 'objects' (`int`), 'containers' (`int`), 'id' (`str`), 'metadata' (`dict`) and 'listing' (`list`). 'listing' contains lists of container metadata (name, number of objects, number of bytes, whether it is a prefix, and modification time). """ params = { "id": account, "limit": limit, "marker": marker, "end_marker": end_marker, "prefix": prefix, "delimiter": delimiter, "s3_buckets_only": s3_buckets_only } _resp, body = self.account_request(account, 'GET', 'containers', params=params, **kwargs) return body def container_update(self, account, container, metadata=None, **kwargs): """ Update account with container-related metadata. :param account: name of the account to update :type account: `str` :param container: name of the container whose metadata has changed :type container: `str` :param metadata: container metadata ("bytes", "objects", "mtime", "dtime") :type metadata: `dict` """ metadata['name'] = container _resp, body = self.account_request(account, 'PUT', 'container/update', data=json.dumps(metadata), **kwargs) return body def container_reset(self, account, container, mtime, **kwargs): """ Reset container of an account :param account: name of the account :type account: `str` :param container: name of the container to reset :type container: `str` :param mtime: time of the modification """ metadata = dict() metadata["name"] = container metadata["mtime"] = mtime self.account_request(account, 'PUT', 'container/reset', data=json.dumps(metadata), **kwargs) def account_refresh(self, account, **kwargs): """ Refresh counters of an account :param account: name of the account to refresh :type account: `str` """ self.account_request(account, 'POST', 'refresh', **kwargs) def account_flush(self, account, **kwargs): """ Flush all containers of an account :param account: name of the account to flush :type account: `str` """ self.account_request(account, 'POST', 'flush', **kwargs)
class EventWorker(Worker): def __init__(self, *args, **kwargs): super(EventWorker, self).__init__(*args, **kwargs) self.app_env = dict() def init(self): self.tube = self.conf.get("tube", DEFAULT_TUBE) self.cs = ConscienceClient(self.conf, logger=self.logger) self.rdir = RdirClient(self.conf, logger=self.logger) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60) self.app_env['acct_addr'] = self.acct_addr if 'handlers_conf' not in self.conf: raise ValueError("'handlers_conf' path not defined in conf") self.handlers = loadhandlers(self.conf.get('handlers_conf'), global_conf=self.conf, app=self) for opt in ('acct_update', 'rdir_update', 'retries_per_second', 'batch_size'): if opt in self.conf: self.logger.warn('Deprecated option: %s', opt) super(EventWorker, self).init() def notify(self): """TODO""" pass def safe_decode_job(self, job_id, data): try: env = json.loads(data) env['job_id'] = job_id return env except Exception as exc: self.logger.warn('Failed to decode job %s: "%s"', job_id, str(exc.message)) return None def run(self): coros = [] queue_url = self.conf.get('queue_url', 'beanstalk://127.0.0.1:11300') concurrency = int_value(self.conf.get('concurrency'), 10) server_gt = greenthread.getcurrent() for url in queue_url.split(';'): for i in range(concurrency): beanstalk = Beanstalk.from_url(url) gt = eventlet.spawn(self.handle, beanstalk) gt.link(_eventlet_stop, server_gt, beanstalk) coros.append(gt) beanstalk, gt = None, None while self.alive: self.notify() try: eventlet.sleep(1.0) except AssertionError: self.alive = False break self.notify() try: with Timeout(self.graceful_timeout) as t: [c.kill(StopServe()) for c in coros] [c.wait() for c in coros] except Timeout as te: if te != t: raise [c.kill() for c in coros] def handle(self, beanstalk): conn_error = False try: if self.tube: beanstalk.use(self.tube) beanstalk.watch(self.tube) while True: try: job_id, data = beanstalk.reserve() if conn_error: self.logger.warn("beanstalk reconnected") conn_error = False except ConnectionError: if not conn_error: self.logger.warn("beanstalk connection error") conn_error = True eventlet.sleep(BEANSTALK_RECONNECTION) continue event = self.safe_decode_job(job_id, data) if not event: self.logger.warn("Burying event %s: %s", job_id, "malformed") beanstalk.bury(job_id) else: try: self.process_event(job_id, event, beanstalk) except (ClientException, OioNetworkException) as exc: self.logger.warn("Burying event %s (%s): %s", job_id, event.get('event'), exc) beanstalk.bury(job_id) except ExplicitBury: self.logger.info("Burying event %s (%s)", job_id, event.get('event')) beanstalk.bury(job_id) except Exception: self.logger.exception("Burying event %s: %s", job_id, event) beanstalk.bury(job_id) except StopServe: pass def process_event(self, job_id, event, beanstalk): handler = self.get_handler(event) if not handler: self.logger.warn('no handler found for %r' % event) beanstalk.delete(job_id) return def cb(status, msg): if is_success(status): beanstalk.delete(job_id) elif is_error(status): self.logger.warn( 'event %s handling failure (release with delay): %s', event['job_id'], msg) beanstalk.release(job_id, delay=RELEASE_DELAY) handler(event, cb) def get_handler(self, event): return self.handlers.get(event.get('event'), None) def acct_addr(self): if not self._acct_addr or self.acct_refresh(): acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval
def __init__(self, conf, tool): super(_DistributedDispatcher, self).__init__(conf, tool) self.sending = None self.max_items_per_second = int_value( self.conf.get('items_per_second'), self.tool.DEFAULT_ITEM_PER_SECOND) # All available beanstalkd conscience_client = ConscienceClient(self.conf) all_beanstalkd = conscience_client.all_services('beanstalkd') all_available_beanstalkd = dict() for beanstalkd in all_beanstalkd: if beanstalkd['score'] <= 0: continue all_available_beanstalkd[beanstalkd['addr']] = beanstalkd if not all_available_beanstalkd: raise OioException('No beanstalkd available') # Beanstalkd workers workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \ or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE self.beanstalkd_workers = dict() for beanstalkd in locate_tube(all_available_beanstalkd.values(), workers_tube): beanstalkd_worker = BeanstalkdSender(beanstalkd['addr'], workers_tube, self.logger) self.beanstalkd_workers[beanstalkd['addr']] = beanstalkd_worker self.logger.info( 'Beanstalkd %s using tube %s is selected as a worker', beanstalkd_worker.addr, beanstalkd_worker.tube) if not self.beanstalkd_workers: raise OioException('No beanstalkd worker available') nb_workers = len(self.beanstalkd_workers) if self.max_items_per_second > 0: # Max 2 seconds in advance queue_size_per_worker = self.max_items_per_second * 2 / nb_workers else: queue_size_per_worker = 64 for _, beanstalkd_worker in self.beanstalkd_workers.items(): beanstalkd_worker.low_limit = queue_size_per_worker / 2 beanstalkd_worker.high_limit = queue_size_per_worker # Beanstalkd reply beanstalkd_reply = dict() try: local_services = conscience_client.local_services() for local_service in local_services: if local_service['type'] != 'beanstalkd': continue beanstalkd = all_available_beanstalkd.get( local_service['addr']) if beanstalkd is None: continue if beanstalkd_reply \ and beanstalkd_reply['score'] >= beanstalkd['score']: continue beanstalkd_reply = beanstalkd except Exception as exc: # pylint: disable=broad-except self.logger.warning( 'ERROR when searching for beanstalkd locally: %s', exc) if not beanstalkd_reply: self.logger.warn('No beanstalkd available locally') try: beanstalkd = conscience_client.next_instance('beanstalkd') beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']] except Exception as exc: # pylint: disable=broad-except self.logger.warning('ERROR when searching for beanstalkd: %s', exc) beanstalkd_reply_addr = beanstalkd_reply['addr'] # If the tube exists, another service must have already used this tube tube_reply = workers_tube + '.reply.' + str(time.time()) tubes = Beanstalk.from_url('beanstalk://' + beanstalkd_reply_addr).tubes() if tube_reply in tubes: raise OioException('Beanstalkd %s using tube %s is already used') self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr, tube_reply, self.logger) self.logger.info( 'Beanstalkd %s using tube %s is selected for the replies', self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)
class EventWorker(Worker): def init(self): eventlet.monkey_patch(os=False) self.tube = self.conf.get("tube", DEFAULT_TUBE) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60 ) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) if 'handlers_conf' not in self.conf: raise ValueError("'handlers_conf' path not defined in conf") self.handlers = loadhandlers( self.conf.get('handlers_conf'), evt_types, app=self) super(EventWorker, self).init() def notify(self): """TODO""" pass def safe_decode_job(self, job_id, data): try: env = json.loads(data) env['job_id'] = job_id return env except Exception as e: self.logger.warn('decoding job "%s"', str(e.message)) return None def run(self): coros = [] queue_url = self.conf.get('queue_url', '127.0.0.1:11300') concurrency = int_value(self.conf.get('concurrency'), 10) server_gt = greenthread.getcurrent() for i in range(concurrency): beanstalk = Beanstalk.from_url(queue_url) gt = eventlet.spawn(self.handle, beanstalk) gt.link(_eventlet_stop, server_gt, beanstalk) coros.append(gt) beanstalk, gt = None, None while self.alive: self.notify() try: eventlet.sleep(1.0) except AssertionError: self.alive = False break self.notify() try: with Timeout(self.graceful_timeout) as t: [c.kill(StopServe()) for c in coros] [c.wait() for c in coros] except Timeout as te: if te != t: raise [c.kill() for c in coros] def handle(self, beanstalk): conn_error = False try: if self.tube: beanstalk.use(self.tube) beanstalk.watch(self.tube) while True: try: job_id, data = beanstalk.reserve() if conn_error: self.logger.warn("beanstalk reconnected") conn_error = False except ConnectionError: if not conn_error: self.logger.warn("beanstalk connection error") conn_error = True eventlet.sleep(BEANSTALK_RECONNECTION) continue try: event = self.safe_decode_job(job_id, data) self.process_event(job_id, event, beanstalk) except ConnectionError: self.logger.warn( "beanstalk connection error during processing") except Exception: beanstalk.bury(job_id) self.logger.exception("handling event %s (bury)", job_id) except StopServe: pass def process_event(self, job_id, event, beanstalk): handler = self.get_handler(event) if not handler: self.logger.warn('no handler found for %r' % event) beanstalk.delete(job_id) return def cb(status, msg): if is_success(status): beanstalk.delete(job_id) elif is_error(status): self.logger.warn('bury event %r' % event) beanstalk.bury(job_id) else: self.logger.warn('release event %r' % event) beanstalk.release(job_id) handler(event, cb) def get_handler(self, event): return self.handlers.get(event.get('event'), None) @property def acct_addr(self): if not self._acct_addr or self.acct_refresh(): try: acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() except Exception: self.logger.warn('Unable to find account instance') return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval
class XcuteClient(HttpApi): """Simple client API for the xcute service.""" def __init__(self, conf, endpoint=None, proxy_endpoint=None, refresh_delay=3600.0, logger=None, **kwargs): """ Initialize a client for the xcute service. :param conf: dictionary with at least the namespace name :type conf: `dict` :param endpoint: URL of an xcute service :param proxy_endpoint: URL of the proxy :param refresh_interval: time between refreshes of the xcute service endpoint (if not provided at instantiation) :type refresh_interval: `float` seconds """ super(XcuteClient, self).__init__( endpoint=endpoint, service_type='xcute-service', **kwargs) self.conf = conf self.logger = logger or get_logger(self.conf) self.conscience = ConscienceClient(conf, endpoint=proxy_endpoint, logger=self.logger, **kwargs) self._refresh_delay = refresh_delay if not self.endpoint else -1.0 self._last_refresh = 0.0 def _get_xcute_addr(self, **kwargs): """Fetch IP and port of an xcute service from Conscience.""" acct_instance = self.conscience.next_instance('xcute', **kwargs) acct_addr = acct_instance.get('addr') return acct_addr def _refresh_endpoint(self, now=None, **kwargs): """Refresh xcute service endpoint.""" addr = self._get_xcute_addr(**kwargs) self.endpoint = '/'. join(("http:/", addr, "v1.0/xcute")) if not now: now = time.time() self._last_refresh = now def _maybe_refresh_endpoint(self, **kwargs): """Refresh xcute service endpoint if delay has been reached.""" if self._refresh_delay >= 0.0 or not self.endpoint: now = time.time() if now - self._last_refresh > self._refresh_delay: try: self._refresh_endpoint(now, **kwargs) except OioNetworkException as exc: if not self.endpoint: # Cannot use the previous one raise self.logger.warn( "Failed to refresh xcute endpoint: %s", exc) except OioException: if not self.endpoint: # Cannot use the previous one raise self.logger.exception("Failed to refresh xcute endpoint") def xcute_request(self, method, action, params=None, **kwargs): """Make a request to the xcute service.""" self._maybe_refresh_endpoint(**kwargs) if not params: params = dict() try: resp, body = self._request(method, action, params=params, **kwargs) except OioNetworkException as exc: exc_info = sys.exc_info() if self._refresh_delay >= 0.0: self.logger.info( "Refreshing xcute endpoint after error %s", exc) try: self._refresh_endpoint(**kwargs) except Exception as exc: self.logger.warn("%s", exc) reraise(exc_info[0], exc_info[1], exc_info[2]) return resp, body def job_list(self, limit=None, marker=None): _, data = self.xcute_request( 'GET', '/job/list', params={'limit': limit, 'marker': marker}) return data def job_create(self, job_type, job_config=None): _, data = self.xcute_request( 'POST', '/job/create', params={'type': job_type}, json=job_config) return data def job_show(self, job_id): _, data = self.xcute_request( 'GET', '/job/show', params={'id': job_id}) return data def job_pause(self, job_id): _, data = self.xcute_request( 'POST', '/job/pause', params={'id': job_id}) return data def job_resume(self, job_id): _, data = self.xcute_request( 'POST', '/job/resume', params={'id': job_id}) return data def job_delete(self, job_id): self.xcute_request( 'DELETE', '/job/delete', params={'id': job_id}) def lock_list(self): _, data = self.xcute_request( 'GET', '/lock/list') return data def lock_show(self, lock): _, data = self.xcute_request( 'GET', '/lock/show', params={'lock': lock}) return data
class EventWorker(object): def __init__(self, conf, name, context, **kwargs): self.conf = conf self.name = name verbose = kwargs.pop('verbose', False) self.logger = get_logger(self.conf, verbose=verbose) self.init_zmq(context) self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.acct_refresh_interval = int_value( conf.get('acct_refresh_interval'), 60 ) self.acct_update = true_value( conf.get('acct_update', True)) self.session = requests.Session() self.failed = False def start(self): self.logger.info('worker "%s" starting', self.name) self.running = True self.run() def stop(self): self.logger.info('worker "%s" stopping', self.name) self.running = False def init_zmq(self, context): socket = context.socket(zmq.REP) socket.connect('inproc://event-front') self.socket = socket def safe_ack(self, msg): try: self.socket.send_multipart(msg) except Exception: self.logger.warn('Unable to ack event') def run(self): try: while self.running: msg = self.socket.recv_multipart() self.logger.debug("msg received: %s" % msg) event = decode_msg(msg) success = self.process_event(event) f = "0" if success else "" self.safe_ack([msg[0], f]) except Exception as e: self.logger.warn('ERROR in worker "%s"', e) self.failed = True raise e finally: self.logger.info('worker "%s" stopped', self.name) def process_event(self, event): handler = self.get_handler(event) if not handler: self.logger.warn("No handler found") # mark as success return True success = True try: handler(event) except Exception: success = False finally: return success def get_handler(self, event): event_type = event.get('event') if not event_type: return None if event_type == EventType.CONTAINER_PUT: return self.handle_container_put elif event_type == EventType.CONTAINER_DESTROY: return self.handle_container_destroy elif event_type == EventType.CONTAINER_UPDATE: return self.handle_container_update elif event_type == EventType.OBJECT_PUT: return self.handle_object_put elif event_type == EventType.OBJECT_DELETE: return self.handle_object_delete elif event_type == EventType.REFERENCE_UPDATE: return self.handle_reference_update elif event_type == EventType.CHUNK_PUT: return self.handle_chunk_put elif event_type == EventType.CHUNK_DELETE: return self.handle_chunk_delete elif event_type == EventType.PING: return self.handle_ping else: return None @property def acct_addr(self): if not self._acct_addr or self.acct_refresh(): try: acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() except Exception: self.logger.warn('Unable to find account instance') return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval def handle_container_put(self, event): """ Handle container creation. :param event: """ self.logger.debug('worker "%s" handle container put', self.name) if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'mtime': mtime, 'name': name} self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_container_update(self, event): """ Handle container update. :param event: """ self.logger.debug('worker "%s" handle container update', self.name) if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = event.get('url').get('user') account = event.get('url').get('account') bytes_count = data.get('bytes-count', 0) object_count = data.get('object-count', 0) event = { 'mtime': mtime, 'name': name, 'bytes': bytes_count, 'objects': object_count } self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_container_destroy(self, event): """ Handle container destroy. :param event: """ self.logger.debug('worker "%s" handle container destroy', self.name) if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr dtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'dtime': dtime, 'name': name} self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_object_delete(self, event): """ Handle object deletion. Delete the chunks of the object. :param event: """ self.logger.debug('worker "%s" handle object delete', self.name) pile = GreenPile(PARALLEL_CHUNKS_DELETE) chunks = [] for item in event.get('data'): if item.get('type') == 'chunks': chunks.append(item) if not len(chunks): self.logger.warn('No chunks found in event data') return def delete_chunk(chunk): resp = None try: with Timeout(CHUNK_TIMEOUT): resp = self.session.delete(chunk['id']) except (Exception, Timeout) as e: self.logger.exception(e) return resp for chunk in chunks: pile.spawn(delete_chunk, chunk) resps = [resp for resp in pile if resp] for resp in resps: if resp.status_code == 204: self.logger.info('deleted chunk %s' % resp.url) else: self.logger.warn('failed to delete chunk %s' % resp.url) def handle_object_put(self, event): """ Handle object creation. TODO :param event: """ self.logger.debug('worker "%s" handle object put', self.name) def handle_reference_update(self, event): """ Handle reference update. TODO :param event """ self.logger.debug('worker "%s" handle reference update', self.name) def handle_chunk_put(self, event): """ Handle chunk creation. :param event """ self.logger.debug('worker "%s" handle chunk creation', self.name) when = event.get('when') data = event.get('data') volume_id = data.get('volume_id') del data['volume_id'] container_id = data.get('container_id') del data['container_id'] content_id = data.get('content_id') del data['content_id'] chunk_id = data.get('chunk_id') del data['chunk_id'] data['mtime'] = when self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id, **data) def handle_chunk_delete(self, event): """ Handle chunk deletion. :param event """ self.logger.debug('worker "%s" handle chunk deletion', self.name) data = event.get('data') volume_id = data.get('volume_id') container_id = data.get('container_id') content_id = data.get('content_id') chunk_id = data.get('chunk_id') self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id) def handle_ping(self, event): """ Handle ping :param event """ self.logger.debug('worker "%s" handle ping', self.name)
class EventWorker(Worker): def init(self): eventlet.monkey_patch(os=False) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60) self.concurrency = int_value(self.conf.get('concurrency'), 1000) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) super(EventWorker, self).init() def notify(self): """TODO""" pass def safe_decode_job(self, job): try: return json.loads(job) except Exception as e: self.logger.warn('ERROR decoding job "%s"', str(e.message)) return None def run(self): queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300') self.beanstalk = Beanstalk.from_url(queue_url) gt = eventlet.spawn(self.handle) while self.alive: self.notify() try: eventlet.sleep(1.0) except AssertionError: self.alive = False break self.notify() try: with Timeout(self.graceful_timeout) as t: gt.kill(StopServe()) gt.wait() except Timeout as te: if te != t: raise gt.kill() def handle(self): try: while True: job_id, data = self.beanstalk.reserve() try: event = self.safe_decode_job(data) if event: self.process_event(event) self.beanstalk.delete(job_id) except Exception: self.logger.exception("ERROR handling event %s", job_id) except StopServe: self.logger.info('Stopping event handler') def process_event(self, event): handler = self.get_handler(event) if not handler: self.logger.warn("ERROR no handler found for event") # mark as success return True success = True try: handler(event) except Exception: success = False finally: return success def get_handler(self, event): event_type = event.get('event') if not event_type: return None if event_type == EventType.CONTAINER_PUT: return self.handle_container_put elif event_type == EventType.CONTAINER_DESTROY: return self.handle_container_destroy elif event_type == EventType.CONTAINER_UPDATE: return self.handle_container_update elif event_type == EventType.OBJECT_PUT: return self.handle_object_put elif event_type == EventType.OBJECT_DELETE: return self.handle_object_delete elif event_type == EventType.REFERENCE_UPDATE: return self.handle_reference_update elif event_type == EventType.CHUNK_PUT: return self.handle_chunk_put elif event_type == EventType.CHUNK_DELETE: return self.handle_chunk_delete elif event_type == EventType.PING: return self.handle_ping else: return None @property def acct_addr(self): if not self._acct_addr or self.acct_refresh(): try: acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() except Exception: self.logger.warn('Unable to find account instance') return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval def handle_container_put(self, event): """ Handle container creation. :param event: """ self.logger.debug('worker handle container put') if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'mtime': mtime, 'name': name} self.session.post(uri, params={'id': account}, json=event) def handle_container_update(self, event): """ Handle container update. :param event: """ self.logger.debug('worker handle container update') if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = event.get('url').get('user') account = event.get('url').get('account') bytes_count = data.get('bytes-count', 0) object_count = data.get('object-count', 0) event = { 'mtime': mtime, 'name': name, 'bytes': bytes_count, 'objects': object_count } self.session.post(uri, params={'id': account}, json=event) def handle_container_destroy(self, event): """ Handle container destroy. :param event: """ self.logger.debug('worker handle container destroy') if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr dtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'dtime': dtime, 'name': name} self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_object_delete(self, event): """ Handle object deletion. Delete the chunks of the object. :param event: """ self.logger.debug('worker handle object delete') pile = GreenPile(PARALLEL_CHUNKS_DELETE) chunks = [] for item in event.get('data'): if item.get('type') == 'chunks': chunks.append(item) if not len(chunks): self.logger.warn('No chunks found in event data') return def delete_chunk(chunk): resp = None try: with Timeout(CHUNK_TIMEOUT): resp = self.session.delete(chunk['id']) except (Exception, Timeout) as e: self.logger.warn('error while deleting chunk %s "%s"', chunk['id'], str(e.message)) return resp for chunk in chunks: pile.spawn(delete_chunk, chunk) resps = [resp for resp in pile if resp] for resp in resps: if resp.status_code == 204: self.logger.debug('deleted chunk %s' % resp.url) else: self.logger.warn('failed to delete chunk %s' % resp.url) def handle_object_put(self, event): """ Handle object creation. TODO :param event: """ self.logger.debug('worker handle object put') def handle_reference_update(self, event): """ Handle reference update. TODO :param event """ self.logger.debug('worker handle reference update') def handle_chunk_put(self, event): """ Handle chunk creation. :param event """ if not self.rdir_update: self.logger.debug('worker skip chunk creation') return self.logger.debug('worker handle chunk creation') when = event.get('when') data = event.get('data') volume_id = data.get('volume_id') del data['volume_id'] container_id = data.get('container_id') del data['container_id'] content_id = data.get('content_id') del data['content_id'] chunk_id = data.get('chunk_id') del data['chunk_id'] data['mtime'] = when self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id, **data) def handle_chunk_delete(self, event): """ Handle chunk deletion. :param event """ if not self.rdir_update: self.logger.debug('worker skip chunk deletion') return self.logger.debug('worker handle chunk deletion') data = event.get('data') volume_id = data.get('volume_id') container_id = data.get('container_id') content_id = data.get('content_id') chunk_id = data.get('chunk_id') self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id) def handle_ping(self, event): """ Handle ping :param event """ self.logger.debug('worker handle ping')
class AccountClient(HttpApi): """Simple client API for the account service.""" def __init__(self, conf, endpoint=None, proxy_endpoint=None, refresh_delay=3600.0, logger=None, **kwargs): """ Initialize a client for the account service. :param conf: dictionary with at least the namespace name :type conf: `dict` :param endpoint: URL of an account service :param proxy_endpoint: URL of the proxy :param refresh_interval: time between refreshes of the account service endpoint (if not provided at instantiation) :type refresh_interval: `float` seconds """ super(AccountClient, self).__init__(endpoint=endpoint, **kwargs) self.logger = logger or get_logger(conf) self.cs = ConscienceClient(conf, endpoint=proxy_endpoint, logger=self.logger, **kwargs) self._refresh_delay = refresh_delay if not self.endpoint else -1.0 self._last_refresh = 0.0 def _get_account_addr(self): """Fetch IP and port of an account service from Conscience.""" try: acct_instance = self.cs.next_instance('account') acct_addr = acct_instance.get('addr') except Exception: raise ClientException("No Account service found") return acct_addr def _refresh_endpoint(self, now=None): """Refresh account service endpoint.""" addr = self._get_account_addr() self.endpoint = '/'.join(("http:/", addr, "v1.0/account")) if not now: now = time.time() self._last_refresh = now def _maybe_refresh_endpoint(self): """Refresh account service endpoint if delay has been reached.""" if self._refresh_delay >= 0.0 or not self.endpoint: now = time.time() if now - self._last_refresh > self._refresh_delay: try: self._refresh_endpoint(now) except ClientException: if not self.endpoint: # Cannot use the previous one raise self.logger.exception("Failed to refresh account endpoint") def account_request(self, account, method, action, params=None, **kwargs): """Make a request to the account service.""" self._maybe_refresh_endpoint() if not params: params = dict() if account: params['id'] = quote(account) try: resp, body = self._request(method, action, params=params, **kwargs) except OioNetworkException as exc: exc_info = sys.exc_info() if self._refresh_delay >= 0.0: self.logger.info("Refreshing account endpoint after error %s", exc) try: self._refresh_endpoint() except Exception as exc: self.logger.warn("%s", exc) raise exc_info[0], exc_info[1], exc_info[2] return resp, body def account_create(self, account, **kwargs): """ Create an account. :param account: name of the account to create :type account: `str` :returns: `True` if the account has been created """ resp, _body = self.account_request(account, 'PUT', 'create', **kwargs) return resp.status == 201 def account_delete(self, account, **kwargs): """ Delete an account. :param account: name of the account to delete :type account: `str` """ self.account_request(account, 'POST', 'delete', **kwargs) def account_list(self, **kwargs): """ List accounts. """ _resp, body = self.account_request(None, 'GET', 'list', **kwargs) return body def account_show(self, account, **kwargs): """ Get information about an account. """ _resp, body = self.account_request(account, 'GET', 'show', **kwargs) return body # FIXME: document this def account_update(self, account, metadata, to_delete, **kwargs): data = json.dumps({"metadata": metadata, "to_delete": to_delete}) self.account_request(account, 'POST', 'update', data=data, **kwargs) def container_list(self, account, limit=None, marker=None, end_marker=None, prefix=None, delimiter=None, **kwargs): """ Get the list of containers of an account. :param account: account from which to get the container list :type account: `str` :keyword limit: maximum number of results to return :type limit: `int` :keyword marker: name of the container from where to start the listing :type marker: `str` :keyword end_marker: :keyword prefix: :keyword delimiter: """ params = { "id": account, "limit": limit, "marker": marker, "end_marker": end_marker, "prefix": prefix, "delimiter": delimiter } _resp, body = self.account_request(account, 'GET', 'containers', params=params, **kwargs) return body def container_update(self, account, container, metadata=None, **kwargs): """ Update account with container-related metadata. :param account: name of the account to update :type account: `str` :param container: name of the container whose metadata has changed :type container: `str` :param metadata: container metadata ("bytes", "objects", "mtime", "dtime") :type metadata: `dict` """ metadata['name'] = container _resp, body = self.account_request(account, 'POST', 'container/update', data=json.dumps(metadata)) return body def container_reset(self, account, container, mtime, **kwargs): """ Reset container of an account :param account: name of the account :type account: `str` :param container: name of the container to reset :type container: `str` :param mtime: time of the modification """ metadata = dict() metadata["name"] = container metadata["mtime"] = mtime self.account_request(account, 'POST', 'container/reset', data=json.dumps(metadata)) def account_refresh(self, account, **kwargs): """ Refresh counters of an account :param account: name of the account to refresh :type account: `str` """ self.account_request(account, 'POST', 'refresh') def account_flush(self, account, **kwargs): """ Flush all containers of an account :param account: name of the account to flush :type account: `str` """ self.account_request(account, 'POST', 'flush')
class EventWorker(Worker): def init(self): eventlet.monkey_patch(os=False) self.tube = self.conf.get("tube", DEFAULT_TUBE) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) if 'handlers_conf' not in self.conf: raise ValueError("'handlers_conf' path not defined in conf") self.handlers = loadhandlers(self.conf.get('handlers_conf'), evt_types, app=self) super(EventWorker, self).init() def notify(self): """TODO""" pass def safe_decode_job(self, job_id, data): try: env = json.loads(data) env['job_id'] = job_id return env except Exception as e: self.logger.warn('decoding job "%s"', str(e.message)) return None def run(self): coros = [] queue_url = self.conf.get('queue_url', '127.0.0.1:11300') concurrency = int_value(self.conf.get('concurrency'), 10) server_gt = greenthread.getcurrent() for i in range(concurrency): beanstalk = Beanstalk.from_url(queue_url) gt = eventlet.spawn(self.handle, beanstalk) gt.link(_eventlet_stop, server_gt, beanstalk) coros.append(gt) beanstalk, gt = None, None while self.alive: self.notify() try: eventlet.sleep(1.0) except AssertionError: self.alive = False break self.notify() try: with Timeout(self.graceful_timeout) as t: [c.kill(StopServe()) for c in coros] [c.wait() for c in coros] except Timeout as te: if te != t: raise [c.kill() for c in coros] def handle(self, beanstalk): conn_error = False try: beanstalk.use(self.tube) beanstalk.watch(self.tube) while True: try: job_id, data = beanstalk.reserve() if conn_error: self.logger.warn("beanstalk reconnected") conn_error = False except ConnectionError: if not conn_error: self.logger.warn("beanstalk connection error") conn_error = True eventlet.sleep(BEANSTALK_RECONNECTION) continue try: event = self.safe_decode_job(job_id, data) self.process_event(job_id, event, beanstalk) except ConnectionError: self.logger.warn( "beanstalk connection error during processing") except Exception: beanstalk.bury(job_id) self.logger.exception("handling event %s (bury)", job_id) except StopServe: pass def process_event(self, job_id, event, beanstalk): handler = self.get_handler(event) if not handler: self.logger.warn('no handler found for %r' % event) beanstalk.delete(job_id) return def cb(status, msg): if is_success(status): beanstalk.delete(job_id) elif is_error(status): self.logger.warn('bury event %r' % event) beanstalk.bury(job_id) else: self.logger.warn('release event %r' % event) beanstalk.release(job_id) handler(event, cb) def get_handler(self, event): return self.handlers.get(event.get('event'), None) @property def acct_addr(self): if not self._acct_addr or self.acct_refresh(): try: acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() except Exception: self.logger.warn('Unable to find account instance') return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval
class EventWorker(Worker): def init(self): eventlet.monkey_patch(os=False) self.session = requests.Session() self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.graceful_timeout = 1 self.acct_refresh_interval = int_value( self.conf.get('acct_refresh_interval'), 60 ) self.concurrency = int_value(self.conf.get('concurrency'), 1000) self.acct_update = true_value(self.conf.get('acct_update', True)) self.rdir_update = true_value(self.conf.get('rdir_update', True)) super(EventWorker, self).init() def notify(self): """TODO""" pass def safe_decode_job(self, job): try: return json.loads(job) except Exception as e: self.logger.warn('ERROR decoding job "%s"', str(e.message)) return None def run(self): queue_url = self.conf.get('queue_url', 'tcp://127.0.0.1:11300') self.beanstalk = Beanstalk.from_url(queue_url) gt = eventlet.spawn( self.handle) while self.alive: self.notify() try: eventlet.sleep(1.0) except AssertionError: self.alive = False break self.notify() try: with Timeout(self.graceful_timeout) as t: gt.kill(StopServe()) gt.wait() except Timeout as te: if te != t: raise gt.kill() def handle(self): try: while True: job_id, data = self.beanstalk.reserve() try: event = self.safe_decode_job(data) if event: self.process_event(event) self.beanstalk.delete(job_id) except Exception: self.logger.exception("ERROR handling event %s", job_id) except StopServe: self.logger.info('Stopping event handler') def process_event(self, event): handler = self.get_handler(event) if not handler: self.logger.warn("ERROR no handler found for event") # mark as success return True success = True try: handler(event) except Exception: success = False finally: return success def get_handler(self, event): event_type = event.get('event') if not event_type: return None if event_type == EventType.CONTAINER_PUT: return self.handle_container_put elif event_type == EventType.CONTAINER_DESTROY: return self.handle_container_destroy elif event_type == EventType.CONTAINER_UPDATE: return self.handle_container_update elif event_type == EventType.OBJECT_PUT: return self.handle_object_put elif event_type == EventType.OBJECT_DELETE: return self.handle_object_delete elif event_type == EventType.REFERENCE_UPDATE: return self.handle_reference_update elif event_type == EventType.CHUNK_PUT: return self.handle_chunk_put elif event_type == EventType.CHUNK_DELETE: return self.handle_chunk_delete elif event_type == EventType.PING: return self.handle_ping else: return None @property def acct_addr(self): if not self._acct_addr or self.acct_refresh(): try: acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() except Exception: self.logger.warn('Unable to find account instance') return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval def handle_container_put(self, event): """ Handle container creation. :param event: """ self.logger.debug('worker handle container put') if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'mtime': mtime, 'name': name} self.session.post(uri, params={'id': account}, json=event) def handle_container_update(self, event): """ Handle container update. :param event: """ self.logger.debug('worker handle container update') if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = event.get('url').get('user') account = event.get('url').get('account') bytes_count = data.get('bytes-count', 0) object_count = data.get('object-count', 0) event = { 'mtime': mtime, 'name': name, 'bytes': bytes_count, 'objects': object_count } self.session.post(uri, params={'id': account}, json=event) def handle_container_destroy(self, event): """ Handle container destroy. :param event: """ self.logger.debug('worker handle container destroy') if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr dtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'dtime': dtime, 'name': name} self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_object_delete(self, event): """ Handle object deletion. Delete the chunks of the object. :param event: """ self.logger.debug('worker handle object delete') pile = GreenPile(PARALLEL_CHUNKS_DELETE) chunks = [] for item in event.get('data'): if item.get('type') == 'chunks': chunks.append(item) if not len(chunks): self.logger.warn('No chunks found in event data') return def delete_chunk(chunk): resp = None try: with Timeout(CHUNK_TIMEOUT): resp = self.session.delete(chunk['id']) except (Exception, Timeout) as e: self.logger.warn('error while deleting chunk %s "%s"', chunk['id'], str(e.message)) return resp for chunk in chunks: pile.spawn(delete_chunk, chunk) resps = [resp for resp in pile if resp] for resp in resps: if resp.status_code == 204: self.logger.debug('deleted chunk %s' % resp.url) else: self.logger.warn('failed to delete chunk %s' % resp.url) def handle_object_put(self, event): """ Handle object creation. TODO :param event: """ self.logger.debug('worker handle object put') def handle_reference_update(self, event): """ Handle reference update. TODO :param event """ self.logger.debug('worker handle reference update') def handle_chunk_put(self, event): """ Handle chunk creation. :param event """ if not self.rdir_update: self.logger.debug('worker skip chunk creation') return self.logger.debug('worker handle chunk creation') when = event.get('when') data = event.get('data') volume_id = data.get('volume_id') del data['volume_id'] container_id = data.get('container_id') del data['container_id'] content_id = data.get('content_id') del data['content_id'] chunk_id = data.get('chunk_id') del data['chunk_id'] data['mtime'] = when self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id, **data) def handle_chunk_delete(self, event): """ Handle chunk deletion. :param event """ if not self.rdir_update: self.logger.debug('worker skip chunk deletion') return self.logger.debug('worker handle chunk deletion') data = event.get('data') volume_id = data.get('volume_id') container_id = data.get('container_id') content_id = data.get('content_id') chunk_id = data.get('chunk_id') self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id) def handle_ping(self, event): """ Handle ping :param event """ self.logger.debug('worker handle ping')
class EventWorker(object): def __init__(self, conf, name, context, **kwargs): self.conf = conf self.name = name verbose = kwargs.pop('verbose', False) self.logger = get_logger(self.conf, verbose=verbose) self.init_zmq(context) self.cs = ConscienceClient(self.conf) self.rdir = RdirClient(self.conf) self._acct_addr = None self.acct_update = 0 self.acct_refresh_interval = int_value( conf.get('acct_refresh_interval'), 60) self.acct_update = true_value(conf.get('acct_update', True)) self.rdir_update = true_value(conf.get('rdir_update', True)) self.session = requests.Session() self.failed = False def start(self): self.logger.info('worker "%s" starting', self.name) self.running = True self.run() def stop(self): self.logger.info('worker "%s" stopping', self.name) self.running = False def init_zmq(self, context): socket = context.socket(zmq.REP) socket.connect('inproc://event-front') self.socket = socket def safe_ack(self, msg): try: self.socket.send_multipart(msg) except Exception: self.logger.warn('Unable to ack event') def run(self): try: while self.running: msg = self.socket.recv_multipart() self.logger.debug("msg received: %s" % msg) event = decode_msg(msg) success = self.process_event(event) f = "0" if success else "" self.safe_ack([msg[0], f]) except Exception as e: self.logger.warn('ERROR in worker "%s"', e) self.failed = True raise e finally: self.logger.info('worker "%s" stopped', self.name) def process_event(self, event): handler = self.get_handler(event) if not handler: self.logger.warn("No handler found") # mark as success return True success = True try: handler(event) except Exception: success = False finally: return success def get_handler(self, event): event_type = event.get('event') if not event_type: return None if event_type == EventType.CONTAINER_PUT: return self.handle_container_put elif event_type == EventType.CONTAINER_DESTROY: return self.handle_container_destroy elif event_type == EventType.CONTAINER_UPDATE: return self.handle_container_update elif event_type == EventType.OBJECT_PUT: return self.handle_object_put elif event_type == EventType.OBJECT_DELETE: return self.handle_object_delete elif event_type == EventType.REFERENCE_UPDATE: return self.handle_reference_update elif event_type == EventType.CHUNK_PUT: return self.handle_chunk_put elif event_type == EventType.CHUNK_DELETE: return self.handle_chunk_delete elif event_type == EventType.PING: return self.handle_ping else: return None @property def acct_addr(self): if not self._acct_addr or self.acct_refresh(): try: acct_instance = self.cs.next_instance(ACCOUNT_SERVICE) self._acct_addr = acct_instance.get('addr') self.acct_update = time.time() except Exception: self.logger.warn('Unable to find account instance') return self._acct_addr def acct_refresh(self): return (time.time() - self.acct_update) > self.acct_refresh_interval def handle_container_put(self, event): """ Handle container creation. :param event: """ self.logger.debug('worker "%s" handle container put', self.name) if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'mtime': mtime, 'name': name} self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_container_update(self, event): """ Handle container update. :param event: """ self.logger.debug('worker "%s" handle container update', self.name) if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr mtime = event.get('when') data = event.get('data') name = event.get('url').get('user') account = event.get('url').get('account') bytes_count = data.get('bytes-count', 0) object_count = data.get('object-count', 0) event = { 'mtime': mtime, 'name': name, 'bytes': bytes_count, 'objects': object_count } self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_container_destroy(self, event): """ Handle container destroy. :param event: """ self.logger.debug('worker "%s" handle container destroy', self.name) if not self.acct_update: return uri = 'http://%s/v1.0/account/container/update' % self.acct_addr dtime = event.get('when') data = event.get('data') name = data.get('url').get('user') account = data.get('url').get('account') event = {'dtime': dtime, 'name': name} self.session.post(uri, params={'id': account}, data=json.dumps(event)) def handle_object_delete(self, event): """ Handle object deletion. Delete the chunks of the object. :param event: """ self.logger.debug('worker "%s" handle object delete', self.name) pile = GreenPile(PARALLEL_CHUNKS_DELETE) chunks = [] for item in event.get('data'): if item.get('type') == 'chunks': chunks.append(item) if not len(chunks): self.logger.warn('No chunks found in event data') return def delete_chunk(chunk): resp = None try: with Timeout(CHUNK_TIMEOUT): resp = self.session.delete(chunk['id']) except (Exception, Timeout) as e: self.logger.exception(e) return resp for chunk in chunks: pile.spawn(delete_chunk, chunk) resps = [resp for resp in pile if resp] for resp in resps: if resp.status_code == 204: self.logger.info('deleted chunk %s' % resp.url) else: self.logger.warn('failed to delete chunk %s' % resp.url) def handle_object_put(self, event): """ Handle object creation. TODO :param event: """ self.logger.debug('worker "%s" handle object put', self.name) def handle_reference_update(self, event): """ Handle reference update. TODO :param event """ self.logger.debug('worker "%s" handle reference update', self.name) def handle_chunk_put(self, event): """ Handle chunk creation. :param event """ if not self.rdir_update: self.logger.debug('worker "%s" skip chunk creation', self.name) return self.logger.debug('worker "%s" handle chunk creation', self.name) when = event.get('when') data = event.get('data') volume_id = data.get('volume_id') del data['volume_id'] container_id = data.get('container_id') del data['container_id'] content_id = data.get('content_id') del data['content_id'] chunk_id = data.get('chunk_id') del data['chunk_id'] data['mtime'] = when self.rdir.chunk_push(volume_id, container_id, content_id, chunk_id, **data) def handle_chunk_delete(self, event): """ Handle chunk deletion. :param event """ if not self.rdir_update: self.logger.debug('worker "%s" skip chunk deletion', self.name) return self.logger.debug('worker "%s" handle chunk deletion', self.name) data = event.get('data') volume_id = data.get('volume_id') container_id = data.get('container_id') content_id = data.get('content_id') chunk_id = data.get('chunk_id') self.rdir.chunk_delete(volume_id, container_id, content_id, chunk_id) def handle_ping(self, event): """ Handle ping :param event """ self.logger.debug('worker "%s" handle ping', self.name)
def __init__(self, conf, tool): super(_DistributedDispatcher, self).__init__(conf, tool) self.sending = False # All available beanstalkd conscience_client = ConscienceClient(self.conf) all_beanstalkd = conscience_client.all_services('beanstalkd') all_available_beanstalkd = dict() for beanstalkd in all_beanstalkd: if beanstalkd['score'] <= 0: continue all_available_beanstalkd[beanstalkd['addr']] = beanstalkd if not all_available_beanstalkd: raise OioException('No beanstalkd available') # Beanstalkd workers workers_tube = self.conf.get('distributed_beanstalkd_worker_tube') \ or self.tool.DEFAULT_DISTRIBUTED_BEANSTALKD_WORKER_TUBE self.beanstalkd_workers = dict() for _, beanstalkd in all_available_beanstalkd.items(): beanstalkd_worker_addr = beanstalkd['addr'] # If the tube exists, # there should be a service that listens to this tube tubes = Beanstalk.from_url('beanstalk://' + beanstalkd_worker_addr).tubes() if workers_tube not in tubes: continue beanstalkd_worker = BeanstalkdSender(beanstalkd_worker_addr, workers_tube, self.logger) self.beanstalkd_workers[beanstalkd_worker_addr] = beanstalkd_worker self.logger.info( 'Beanstalkd %s using tube %s is selected as a worker', beanstalkd_worker.addr, beanstalkd_worker.tube) if not self.beanstalkd_workers: raise OioException('No beanstalkd worker available') # Beanstalkd reply beanstalkd_reply = dict() try: local_services = conscience_client.local_services() for local_service in local_services: if local_service['type'] != 'beanstalkd': continue beanstalkd = all_available_beanstalkd.get( local_service['addr']) if beanstalkd is None: continue if beanstalkd_reply \ and beanstalkd_reply['score'] >= beanstalkd['score']: continue beanstalkd_reply = beanstalkd except Exception as exc: # pylint: disable=broad-except self.logger.warning( 'ERROR when searching for beanstalkd locally: %s', exc) if not beanstalkd_reply: self.logger.warn('No beanstalkd available locally') try: beanstalkd = conscience_client.next_instance('beanstalkd') beanstalkd_reply = all_available_beanstalkd[beanstalkd['addr']] except Exception as exc: # pylint: disable=broad-except self.logger.warning('ERROR when searching for beanstalkd: %s', exc) beanstalkd_reply_addr = beanstalkd_reply['addr'] # If the tube exists, another service must have already used this tube tube_reply = workers_tube + '.reply.' + str(time.time()) tubes = Beanstalk.from_url('beanstalk://' + beanstalkd_reply_addr).tubes() if tube_reply in tubes: raise OioException('Beanstalkd %s using tube %s is already used') self.beanstalkd_reply = BeanstalkdListener(beanstalkd_reply_addr, tube_reply, self.logger) self.logger.info( 'Beanstalkd %s using tube %s is selected for the replies', self.beanstalkd_reply.addr, self.beanstalkd_reply.tube)