def run(args): pool = get_pool_manager() v = vars(args) dirclient = DirectoryClient(v) backend = AccountBackend(v) for entry, _, _, partial in full_list(backend, prefix=args.prefix): if partial: if args.verbose: print(":%s: partial, skip" % entry) continue try: dirclient.show(account=ACCOUNT, reference=entry) if args.verbose: print("%s: OK" % entry) continue except NotFound: pass except Exception as exc: print("Exception not managed for %s: %s" % (entry, str(exc))) continue print("%s: meta2 not found" % entry) if args.dry_run: continue data = {"dtime": time(), "name": entry} # post event to Account service res = pool.request('POST', HOST + '/v1.0/account/container/update?id=%s' % ACCOUNT, headers={'Content-Type': 'application/json'}, body=json.dumps(data)) if res.status_int / 100 != 2: print(res.status)
class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.directory_client = DirectoryClient(conf, **kwargs) def _lookup_rdir_host(self, resp): host = None for srv in resp.get('srv', {}): if srv['type'] == 'rdir': host = srv['host'] if not host: raise ClientException("No rdir service found") return host def _link_rdir(self, volume_id): self.directory_client.link(acct=RDIR_ACCT, ref=volume_id, srv_type='rdir', autocreate=True) return self.directory_client.show( acct=RDIR_ACCT, ref=volume_id, srv_type='rdir') # TODO keep rdir addr in local cache to avoid lookup requests def _get_rdir_addr(self, volume_id, create=False): resp = {} try: resp = self.directory_client.show(acct=RDIR_ACCT, ref=volume_id, srv_type='rdir') except NotFound: if not create: raise try: host = self._lookup_rdir_host(resp) except ClientException: # Reference exists but no rdir linked if not create: raise resp = self._link_rdir(volume_id) host = self._lookup_rdir_host(resp) return host def _make_uri(self, action, volume_id, create=False): rdir_host = self._get_rdir_addr(volume_id, create=create) uri = 'http://%s/v1/%s/%s' % (rdir_host, self.ns, action) return uri def _rdir_request(self, volume, method, action, create=False, **kwargs): uri = self._make_uri(action, volume, create=create) params = {'vol': volume} if create: params['create'] = '1' resp, body = self._direct_request(method, uri, params=params, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, 'POST', 'rdir/push', create=True, json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): req_body = {'limit': limit} if rebuild: req_body['rebuild'] = True while True: resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch', json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split('|') yield container, content, chunk, value req_body['start_after'] = key def admin_incident_set(self, volume, date): body = {'date': date} self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/incident') return resp_body.get('date') def admin_lock(self, volume, who): body = {'who': who} self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body) def admin_unlock(self, volume): self._rdir_request(volume, 'POST', 'rdir/admin/unlock') def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show') return resp_body def status(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/status') return resp_body
class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.autocreate = true_value(conf.get('autocreate', True)) self.directory_client = DirectoryClient(conf) # TODO keep rdir addr in local cache to avoid lookup requests def _get_rdir_addr(self, volume_id): try: resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') except NotFound as e: if self.autocreate: self.directory_client.link('_RDIR', volume_id, 'rdir', autocreate=True) resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') else: raise e for srv in resp['srv']: if srv['type'] == 'rdir': return srv['host'] raise ClientException("No rdir service found") def _make_uri(self, action, volume_id): rdir_host = self._get_rdir_addr(volume_id) uri = 'http://%s/v1/%s/%s?vol=%s' % (rdir_host, self.ns, action, volume_id) return uri def _rdir_request(self, volume, method, action, **kwargs): uri = self._make_uri(action, volume) resp, body = self._direct_request(method, uri, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): body = { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, 'POST', 'rdir/push', json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): body = { 'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id } self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): req_body = {'limit': limit} if rebuild: req_body['rebuild'] = True while True: resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch', json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split('|') yield container, content, chunk, value req_body['start_after'] = key def admin_incident_set(self, volume, date): body = {'date': date} self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/incident') return resp_body.get('date') def admin_lock(self, volume, who): body = {'who': who} self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body) def admin_unlock(self, volume): self._rdir_request(volume, 'POST', 'rdir/admin/unlock') def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show') return resp_body
class RdirClient(Client): def __init__(self, conf, **kwargs): super(RdirClient, self).__init__(conf, **kwargs) self.autocreate = true_value(conf.get('autocreate', True)) self.directory_client = DirectoryClient(conf) # TODO keep rdir addr in local cache to avoid lookup requests def _get_rdir_addr(self, volume_id): try: resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') except NotFound as e: if self.autocreate: self.directory_client.link('_RDIR', volume_id, 'rdir', autocreate=True) resp = self.directory_client.show(acct='_RDIR', ref=volume_id, srv_type='rdir') else: raise e for srv in resp['srv']: if srv['type'] == 'rdir': return srv['host'] raise ClientException("No rdir service found") def _make_uri(self, action, volume_id): rdir_host = self._get_rdir_addr(volume_id) uri = 'http://%s/v1/%s/%s?vol=%s' % ( rdir_host, self.ns, action, volume_id) return uri def _rdir_request(self, volume, method, action, **kwargs): uri = self._make_uri(action, volume) resp, body = self._direct_request(method, uri, **kwargs) return resp, body def chunk_push(self, volume_id, container_id, content_id, chunk_id, **data): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} for key, value in data.iteritems(): body[key] = value headers = {} self._rdir_request(volume_id, 'POST', 'rdir/push', json=body, headers=headers) def chunk_delete(self, volume_id, container_id, content_id, chunk_id): body = {'container_id': container_id, 'content_id': content_id, 'chunk_id': chunk_id} self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body) def chunk_fetch(self, volume, limit=100, rebuild=False): req_body = {'limit': limit} if rebuild: req_body['rebuild'] = True while True: resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch', json=req_body) resp.raise_for_status() if len(resp_body) == 0: break for (key, value) in resp_body: container, content, chunk = key.split('|') yield container, content, chunk, value req_body['start_after'] = key def admin_incident_set(self, volume, date): body = {'date': date} self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body) def admin_incident_get(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/incident') return resp_body.get('date') def admin_lock(self, volume, who): body = {'who': who} self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body) def admin_unlock(self, volume): self._rdir_request(volume, 'POST', 'rdir/admin/unlock') def admin_show(self, volume): resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show') return resp_body