def run(args):
pool = get_pool_manager()
v = vars(args)
dirclient = DirectoryClient(v)
backend = AccountBackend(v)
for entry, _, _, partial in full_list(backend, prefix=args.prefix):
if partial:
if args.verbose:
print(":%s: partial, skip" % entry)
continue
try:
dirclient.show(account=ACCOUNT, reference=entry)
if args.verbose:
print("%s: OK" % entry)
continue
except NotFound:
pass
except Exception as exc:
print("Exception not managed for %s: %s" % (entry, str(exc)))
continue
print("%s: meta2 not found" % entry)
if args.dry_run:
continue
data = {"dtime": time(), "name": entry}
# post event to Account service
res = pool.request('POST',
HOST +
'/v1.0/account/container/update?id=%s' % ACCOUNT,
headers={'Content-Type': 'application/json'},
body=json.dumps(data))
if res.status_int / 100 != 2:
print(res.status)
class RdirClient(Client):
def __init__(self, conf, **kwargs):
super(RdirClient, self).__init__(conf, **kwargs)
self.directory_client = DirectoryClient(conf, **kwargs)
def _lookup_rdir_host(self, resp):
host = None
for srv in resp.get('srv', {}):
if srv['type'] == 'rdir':
host = srv['host']
if not host:
raise ClientException("No rdir service found")
return host
def _link_rdir(self, volume_id):
self.directory_client.link(acct=RDIR_ACCT, ref=volume_id,
srv_type='rdir', autocreate=True)
return self.directory_client.show(
acct=RDIR_ACCT, ref=volume_id, srv_type='rdir')
# TODO keep rdir addr in local cache to avoid lookup requests
def _get_rdir_addr(self, volume_id, create=False):
resp = {}
try:
resp = self.directory_client.show(acct=RDIR_ACCT, ref=volume_id,
srv_type='rdir')
except NotFound:
if not create:
raise
try:
host = self._lookup_rdir_host(resp)
except ClientException:
# Reference exists but no rdir linked
if not create:
raise
resp = self._link_rdir(volume_id)
host = self._lookup_rdir_host(resp)
return host
def _make_uri(self, action, volume_id, create=False):
rdir_host = self._get_rdir_addr(volume_id, create=create)
uri = 'http://%s/v1/%s/%s' % (rdir_host, self.ns, action)
return uri
def _rdir_request(self, volume, method, action, create=False, **kwargs):
uri = self._make_uri(action, volume, create=create)
params = {'vol': volume}
if create:
params['create'] = '1'
resp, body = self._direct_request(method, uri, params=params, **kwargs)
return resp, body
def chunk_push(self, volume_id, container_id, content_id, chunk_id,
**data):
body = {'container_id': container_id,
'content_id': content_id,
'chunk_id': chunk_id}
for key, value in data.iteritems():
body[key] = value
headers = {}
self._rdir_request(volume_id, 'POST', 'rdir/push', create=True,
json=body, headers=headers)
def chunk_delete(self, volume_id, container_id, content_id, chunk_id):
body = {'container_id': container_id,
'content_id': content_id,
'chunk_id': chunk_id}
self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body)
def chunk_fetch(self, volume, limit=100, rebuild=False):
req_body = {'limit': limit}
if rebuild:
req_body['rebuild'] = True
while True:
resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch',
json=req_body)
resp.raise_for_status()
if len(resp_body) == 0:
break
for (key, value) in resp_body:
container, content, chunk = key.split('|')
yield container, content, chunk, value
req_body['start_after'] = key
def admin_incident_set(self, volume, date):
body = {'date': date}
self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body)
def admin_incident_get(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET',
'rdir/admin/incident')
return resp_body.get('date')
def admin_lock(self, volume, who):
body = {'who': who}
self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body)
def admin_unlock(self, volume):
self._rdir_request(volume, 'POST', 'rdir/admin/unlock')
def admin_show(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show')
return resp_body
def status(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/status')
return resp_body
class RdirClient(Client):
def __init__(self, conf, **kwargs):
super(RdirClient, self).__init__(conf, **kwargs)
self.autocreate = true_value(conf.get('autocreate', True))
self.directory_client = DirectoryClient(conf)
# TODO keep rdir addr in local cache to avoid lookup requests
def _get_rdir_addr(self, volume_id):
try:
resp = self.directory_client.show(acct='_RDIR',
ref=volume_id,
srv_type='rdir')
except NotFound as e:
if self.autocreate:
self.directory_client.link('_RDIR',
volume_id,
'rdir',
autocreate=True)
resp = self.directory_client.show(acct='_RDIR',
ref=volume_id,
srv_type='rdir')
else:
raise e
for srv in resp['srv']:
if srv['type'] == 'rdir':
return srv['host']
raise ClientException("No rdir service found")
def _make_uri(self, action, volume_id):
rdir_host = self._get_rdir_addr(volume_id)
uri = 'http://%s/v1/%s/%s?vol=%s' % (rdir_host, self.ns, action,
volume_id)
return uri
def _rdir_request(self, volume, method, action, **kwargs):
uri = self._make_uri(action, volume)
resp, body = self._direct_request(method, uri, **kwargs)
return resp, body
def chunk_push(self, volume_id, container_id, content_id, chunk_id,
**data):
body = {
'container_id': container_id,
'content_id': content_id,
'chunk_id': chunk_id
}
for key, value in data.iteritems():
body[key] = value
headers = {}
self._rdir_request(volume_id,
'POST',
'rdir/push',
json=body,
headers=headers)
def chunk_delete(self, volume_id, container_id, content_id, chunk_id):
body = {
'container_id': container_id,
'content_id': content_id,
'chunk_id': chunk_id
}
self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body)
def chunk_fetch(self, volume, limit=100, rebuild=False):
req_body = {'limit': limit}
if rebuild:
req_body['rebuild'] = True
while True:
resp, resp_body = self._rdir_request(volume,
'POST',
'rdir/fetch',
json=req_body)
resp.raise_for_status()
if len(resp_body) == 0:
break
for (key, value) in resp_body:
container, content, chunk = key.split('|')
yield container, content, chunk, value
req_body['start_after'] = key
def admin_incident_set(self, volume, date):
body = {'date': date}
self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body)
def admin_incident_get(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET',
'rdir/admin/incident')
return resp_body.get('date')
def admin_lock(self, volume, who):
body = {'who': who}
self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body)
def admin_unlock(self, volume):
self._rdir_request(volume, 'POST', 'rdir/admin/unlock')
def admin_show(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show')
return resp_body
class RdirClient(Client):
def __init__(self, conf, **kwargs):
super(RdirClient, self).__init__(conf, **kwargs)
self.autocreate = true_value(conf.get('autocreate', True))
self.directory_client = DirectoryClient(conf)
# TODO keep rdir addr in local cache to avoid lookup requests
def _get_rdir_addr(self, volume_id):
try:
resp = self.directory_client.show(acct='_RDIR', ref=volume_id,
srv_type='rdir')
except NotFound as e:
if self.autocreate:
self.directory_client.link('_RDIR', volume_id, 'rdir',
autocreate=True)
resp = self.directory_client.show(acct='_RDIR', ref=volume_id,
srv_type='rdir')
else:
raise e
for srv in resp['srv']:
if srv['type'] == 'rdir':
return srv['host']
raise ClientException("No rdir service found")
def _make_uri(self, action, volume_id):
rdir_host = self._get_rdir_addr(volume_id)
uri = 'http://%s/v1/%s/%s?vol=%s' % (
rdir_host, self.ns, action, volume_id)
return uri
def _rdir_request(self, volume, method, action, **kwargs):
uri = self._make_uri(action, volume)
resp, body = self._direct_request(method, uri, **kwargs)
return resp, body
def chunk_push(self, volume_id, container_id, content_id, chunk_id,
**data):
body = {'container_id': container_id,
'content_id': content_id,
'chunk_id': chunk_id}
for key, value in data.iteritems():
body[key] = value
headers = {}
self._rdir_request(volume_id, 'POST', 'rdir/push',
json=body, headers=headers)
def chunk_delete(self, volume_id, container_id, content_id, chunk_id):
body = {'container_id': container_id,
'content_id': content_id,
'chunk_id': chunk_id}
self._rdir_request(volume_id, 'DELETE', 'rdir/delete', json=body)
def chunk_fetch(self, volume, limit=100, rebuild=False):
req_body = {'limit': limit}
if rebuild:
req_body['rebuild'] = True
while True:
resp, resp_body = self._rdir_request(volume, 'POST', 'rdir/fetch',
json=req_body)
resp.raise_for_status()
if len(resp_body) == 0:
break
for (key, value) in resp_body:
container, content, chunk = key.split('|')
yield container, content, chunk, value
req_body['start_after'] = key
def admin_incident_set(self, volume, date):
body = {'date': date}
self._rdir_request(volume, 'POST', 'rdir/admin/incident', json=body)
def admin_incident_get(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET',
'rdir/admin/incident')
return resp_body.get('date')
def admin_lock(self, volume, who):
body = {'who': who}
self._rdir_request(volume, 'POST', 'rdir/admin/lock', json=body)
def admin_unlock(self, volume):
self._rdir_request(volume, 'POST', 'rdir/admin/unlock')
def admin_show(self, volume):
resp, resp_body = self._rdir_request(volume, 'GET', 'rdir/admin/show')
return resp_body