def test_single_score(self):
srv0 = self._srv('echo', ip='127.0.0.3')
def check(code):
params = {'type': 'echo', 'service_id': srv0['addr']}
resp = self.request('GET',
self._url_cs("score"),
None,
params=params,
headers=self.TEST_HEADERS)
self.assertEqual(code, resp.status)
# Service not found
self._reload()
check(404)
# Registration -> found
self._register_srv([srv0])
self._reload_proxy()
check(200)
# lock to 0 -> found
resp = self.request('POST', self._url_cs('lock'), json.dumps(srv0))
self.assertIn(resp.status, (200, 204))
self._reload_proxy()
check(200)
# removal -> not found
resp = self.request('POST', self._url_cs('unlock'), json.dumps(srv0))
self._deregister_srv(srv0)
self._flush_proxy()
self._reload_proxy()
check(404)
def test_services_pool_actions_unlock(self):
srv = self._srv('echo')
resp = self.request('POST', self._url_cs("lock"), json.dumps(srv))
self.assertIn(resp.status, (200, 204))
resp = self.request('POST', self._url_cs("unlock"), json.dumps(srv))
self.assertIn(resp.status, (200, 204))
resp = self.request('GET',
self._url_cs('list'),
params={"type": "echo"})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
self.assertIsInstance(body, list)
def _reply_task_res(self, beanstalkd_reply, task_res):
self.queue_reply.put(task_res)
if beanstalkd_reply is None:
return
res_event = self.tool.res_event_from_task_res(task_res)
if self.tool.beanstalkd is not None:
res_event['beanstalkd_worker'] = \
{
'addr': self.tool.beanstalkd.addr,
'tube': self.tool.beanstalkd.tube
}
try:
if self.beanstalkd_reply is None \
or self.beanstalkd_reply.addr != beanstalkd_reply['addr'] \
or self.beanstalkd_reply.tube != beanstalkd_reply['tube']:
if self.beanstalkd_reply is not None:
self.beanstalkd_reply.close()
self.beanstalkd_reply = BeanstalkdSender(
beanstalkd_reply['addr'], beanstalkd_reply['tube'],
self.logger)
self.beanstalkd_reply.send_job(json.dumps(res_event))
except Exception as exc: # pylint: disable=broad-except
item, info, error = task_res
self.logger.warn(
'Beanstalkd reply failed %s (info=%s error=%s): %s',
self.tool.string_from_item(item), str(info), error, exc)
def put_user_policy(self, account, user, policy, policy_name=''):
"""
Save an IAM policy for the specified user.
:param policy: JSON-formatted string
:type policy: str
:param policy_name: name of the policy (empty string if not set)
"""
if not isinstance(policy, str):
raise TypeError("policy parameter must be a string")
if not policy_name and not self.allow_empty_policy_name:
raise ValueError('policy name cannot be empty')
if policy_name and not self.name_regex.fullmatch(policy_name):
raise ValueError('policy name does not match %s' %
(self.name_regex.pattern))
# XXX: we should also match user name, but unfortunately, when using
# tempauth, user names have the ':' character between the project name
# and the actual user name.
try:
policy_obj = json.loads(policy)
policy_obj['UpdateDate'] = time.strftime('%Y-%m-%dT%H:%M:%SZ',
time.gmtime())
# Strip spaces and new lines
policy = json.dumps(policy_obj, separators=(',', ':'))
except ValueError as err:
raise ValueError('policy is not JSON-formatted: %s' % err)
acct_key = self.key_for_account(account)
policy_key = self.subkey_for_policy(user, policy_name)
self.redis.conn.hset(acct_key, policy_key, policy.encode('utf-8'))
def on_account_container_show(self, req, **kwargs):
account_id = self._get_account_id(req)
cname = self._get_item_id(req, key='container', what='container')
raw = self.backend.get_container_info(account_id, cname, **kwargs)
if raw is not None:
return Response(json.dumps(raw), mimetype=HTTP_CONTENT_TYPE_JSON)
return NotFound('Container not found')
def handle_container_listing(self, req, start_response):
resp = req.get_response(self.app)
if not resp.is_success or resp.content_type != 'application/json':
return resp(req.environ, start_response)
if resp.content_length is None:
return resp(req.environ, start_response)
if resp.content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
self.logger.warn(
'The content length (%d) of the listing is too long (max=%d)',
resp.content_length, MAX_CONTAINER_LISTING_CONTENT_LENGTH)
return resp(req.environ, start_response)
try:
listing = json.loads(resp.body)
except ValueError:
return resp(req.environ, start_response)
for item in listing:
if 'subdir' in item \
or item.get('content_type') == DELETE_MARKER_CONTENT_TYPE:
continue
etag, params = parse_header(item['hash'])
if 'slo_etag' in params:
item['slo_etag'] = '"%s"' % params.pop('slo_etag')
item['hash'] = etag + ''.join('; %s=%s' % kv
for kv in params.items())
resp.body = json.dumps(listing)
return resp(req.environ, start_response)
def on_job_list(self, req):
limit = int_value(req.args.get('limit'), None)
marker = req.args.get('marker')
job_infos = self.backend.list_jobs(limit=limit, marker=marker)
return Response(
json.dumps(job_infos), mimetype='application/json')
def on_job_resume(self, req):
job_id = self._get_job_id(req)
self.backend.resume(job_id)
job_info = self.backend.get_job_info(job_id)
return Response(json.dumps(job_info),
mimetype='application/json',
status=202)
def request(self,
method,
url,
data=None,
params=None,
headers=None,
json=None):
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, unicode):
v = unicode(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
# Convert json and add Content-Type
headers = headers if headers else {}
if json:
headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
out_kwargs = {}
out_kwargs['headers'] = headers
out_kwargs['body'] = data
return self.http_pool.request(method, url, **out_kwargs)
def content_get_properties(
self, account=None, reference=None, path=None, properties=None,
cid=None, content=None, version=None, params=None, **kwargs):
"""
Get a description of the content along with its user properties.
"""
obj_meta, _ = get_cached_object_metadata(
account=account, reference=reference, path=path,
cid=cid, version=version, properties=True, **kwargs)
if obj_meta is not None:
return obj_meta
uri = self._make_uri('content/get_properties')
data = json.dumps(properties) if properties else None
resp, body = self._direct_request(
'POST', uri, data=data, params=params, **kwargs)
obj_meta = extract_content_headers_meta(resp.headers)
obj_meta.update(body)
set_cached_object_metadata(
obj_meta, None,
account=account, reference=reference, path=path,
cid=cid, version=version, properties=True, **kwargs)
return obj_meta
def container_del_properties(self, account=None, reference=None,
properties=[], cid=None, **kwargs):
params = self._make_params(account, reference, cid=cid)
data = json.dumps(properties)
_resp, body = self._request(
'POST', '/del_properties', data=data, params=params, **kwargs)
return body
def test_service_pool_actions_lock_and_reput(self):
srv = self._srv('echo')
resp = self.request('POST', self._url_cs('lock'), json.dumps(srv))
self.assertIn(resp.status, (200, 204))
resp = self.request('GET',
self._url_cs('list'),
params={"type": "echo"})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
self.assertIsInstance(body, list)
self.assertIn(srv['addr'], [x['addr'] for x in body])
self._register_srv(srv)
resp = self.request('GET',
self._url_cs('list'),
params={'type': 'echo'})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
self.assertIsInstance(body, list)
self.assertIn(srv['addr'], [x['addr'] for x in body])
srv2 = dict(srv)
srv2['score'] = -1
self._register_srv(srv2)
self.assertEqual(resp.status, 200)
resp = self.request('GET',
self._url_cs('list'),
params={'type': 'echo'})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
self.assertIsInstance(body, list)
self.assertIn(srv['addr'], [x['addr'] for x in body])
def container_snapshot(self,
account=None,
reference=None,
dst_account=None,
dst_reference=None,
cid=None,
**kwargs):
"""
Create a snapshot of a the container.
This function duplicates only the database. It doesn't duplicate the
chunks of the contents.
:param account: account in which the container is
:type account: `str`
:param reference: name of the container
:type reference: `str`
:param cid: container id that can be used instead of account
and reference
:type cid: `str`
:param dst_account: account in which the snapshot will be created
:type dst_account: `str`
:param dst_reference: name of the snapshot
:type dst_reference: `str`
"""
params = self._make_params(account, reference, cid=cid)
data = json.dumps({"account": dst_account, "container": dst_reference})
resp, _ = self._request('POST',
'/snapshot',
params=params,
data=data,
**kwargs)
return resp
def container_get_properties(self,
account=None,
reference=None,
properties=None,
cid=None,
params=None,
**kwargs):
"""
Get information about a container (user and system properties).
:param account: account in which the container is
:type account: `str`
:param reference: name of the container
:type reference: `str`
:param cid: container id that can be used instead of account
and reference
:type cid: `str`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: a `dict` with "properties" and "system" entries,
containing respectively a `dict` of user properties and
a `dict` of system properties.
"""
if not properties:
properties = list()
data = json.dumps(properties)
_resp, body = self._request('POST',
'/get_properties',
data=data,
params=params,
**kwargs)
return body
def on_account_container_show(self, req):
account_id = self._get_account_id(req)
cname = self._get_item_id(req, key='container', what='container')
raw = self.backend.get_container_info(account_id, cname)
if raw is not None:
return Response(json.dumps(raw), mimetype='text/json')
return NotFound('Container not found')
def container_create(self,
account,
reference,
properties=None,
system=None,
**kwargs):
"""
Create a container.
:param account: account in which to create the container
:type account: `str`
:param reference: name of the container
:type reference: `str`
:param properties: properties to set on the container
:type properties: `dict`
:param system: system properties to set on the container
:type system: `dict`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: True if the container has been created,
False if it already exists
"""
params = self._make_params(account, reference)
data = json.dumps({
'properties': properties or {},
'system': system or {}
})
resp, body = self._request('POST',
'/create',
params=params,
data=data,
**kwargs)
if resp.status not in (204, 201):
raise exceptions.from_response(resp, body)
return resp.status == 201
def content_prepare(self,
account=None,
reference=None,
path=None,
size=None,
cid=None,
stgpol=None,
**kwargs):
"""
Prepare an upload: get URLs of chunks on available rawx.
:keyword autocreate: create container if it doesn't exist
"""
uri = self._make_uri('content/prepare')
params = self._make_params(account, reference, path, cid=cid)
data = {'size': size}
if stgpol:
data['policy'] = stgpol
data = json.dumps(data)
resp, body = self._direct_request('POST',
uri,
data=data,
params=params,
**kwargs)
resp_headers = extract_content_headers_meta(resp.headers)
return resp_headers, body
def on_account_containers(self, req):
account_id = self._get_account_id(req)
info = self.backend.info_account(account_id)
if not info:
return NotFound('Account not found')
marker = req.args.get('marker', '')
end_marker = req.args.get('end_marker', '')
prefix = req.args.get('prefix', '')
limit = int(req.args.get('limit', '1000'))
limit = max(0, min(ACCOUNT_LISTING_MAX_LIMIT, int_value(
req.args.get('limit'), 0)))
if limit <= 0:
limit = ACCOUNT_LISTING_DEFAULT_LIMIT
delimiter = req.args.get('delimiter', '')
s3_buckets_only = true_value(req.args.get('s3_buckets_only', False))
user_list = self.backend.list_containers(
account_id, limit=limit, marker=marker, end_marker=end_marker,
prefix=prefix, delimiter=delimiter,
s3_buckets_only=s3_buckets_only)
info['listing'] = user_list
# TODO(FVE): add "truncated" entry telling if the listing is truncated
result = json.dumps(info)
return Response(result, mimetype='text/json')
def content_prepare(self, account=None, reference=None, path=None,
size=None, cid=None, stgpol=None, content_id=None,
version=None, params=None, **kwargs):
"""
Prepare an upload: get URLs of chunks on available rawx.
:keyword autocreate: create container if it doesn't exist
"""
uri = self._make_uri('content/prepare')
data = {'size': size}
if stgpol:
data['policy'] = stgpol
data = json.dumps(data)
try:
resp, body = self._direct_request(
'POST', uri + '2', data=data, params=params, **kwargs)
chunks = body['chunks']
obj_meta = extract_content_headers_meta(resp.headers)
obj_meta['properties'] = dict()
# pylint: disable=no-member
obj_meta['properties'].update(body.get('properties', {}))
except exceptions.NotFound:
# Proxy does not support v2 request (oio < 4.3)
resp, chunks = self._direct_request(
'POST', uri, data=data, params=params, **kwargs)
obj_meta = extract_content_headers_meta(resp.headers)
return obj_meta, chunks
def content_del_properties(self,
account=None,
reference=None,
path=None,
properties=[],
cid=None,
version=None,
**kwargs):
"""
Delete some properties from an object.
:param properties: list of property keys to delete
:type properties: `list`
:returns: True is the property has been deleted
"""
uri = self._make_uri('content/del_properties')
params = self._make_params(account,
reference,
path,
cid=cid,
version=version)
data = json.dumps(properties)
resp, _body = self._direct_request('POST',
uri,
data=data,
params=params,
**kwargs)
return resp.status == 204
def force(self,
account=None,
reference=None,
service_type=None,
services=None,
cid=None,
autocreate=False,
replace=False,
**kwargs):
"""
Associate the specified services to the reference.
:param replace: do not require the list of services
of the specified type to be empty, overwrite it.
:type replace: `bool`
"""
params = self._make_params(account, reference, service_type, cid=cid)
if replace:
params["replace"] = "yes"
data = json.dumps(services)
_resp, _body = self._request('POST',
'/force',
data=data,
params=params,
autocreate=autocreate,
**kwargs)
def container_set_properties(self,
account=None,
reference=None,
properties=None,
clear=False,
cid=None,
system=None,
**kwargs):
params = self._make_params(account, reference, cid=cid)
if clear:
params["flush"] = 1
data = json.dumps({
'properties': properties or {},
'system': system or {}
})
del_cached_container_metadata(account=account,
reference=reference,
cid=cid,
**kwargs)
_resp, body = self._request('POST',
'/set_properties',
data=data,
params=params,
**kwargs)
return body
def content_show(self,
account=None,
reference=None,
path=None,
properties=None,
cid=None,
content=None,
version=None,
**kwargs):
"""
Get a description of the content along with its user properties.
"""
uri = self._make_uri('content/get_properties')
params = self._make_params(account,
reference,
path,
cid=cid,
content=content,
version=version)
data = json.dumps(properties) if properties else None
resp, body = self._direct_request('POST',
uri,
data=data,
params=params,
**kwargs)
obj_meta = extract_content_headers_meta(resp.headers)
obj_meta.update(body)
return obj_meta
def content_set_properties(self,
account=None,
reference=None,
path=None,
properties={},
cid=None,
version=None,
clear=False,
**kwargs):
"""
Set properties on an object.
:param properties: dictionary of properties
"""
uri = self._make_uri('content/set_properties')
params = self._make_params(account,
reference,
path,
cid=cid,
version=version)
if clear:
params['flush'] = 1
data = json.dumps(properties)
_resp, _body = self._direct_request('POST',
uri,
data=data,
params=params,
**kwargs)
def on_account_buckets(self, req, **kwargs):
account_id = self._get_account_id(req)
info = self.backend.info_account(account_id, **kwargs)
if not info:
return NotFound('Account not found')
marker = req.args.get('marker', '')
end_marker = req.args.get('end_marker', '')
prefix = req.args.get('prefix', '')
limit = int(req.args.get('limit', '1000'))
bucket_list, next_marker = self.backend.list_buckets(
account_id,
limit=limit,
marker=marker,
end_marker=end_marker,
prefix=prefix,
**kwargs)
info['listing'] = bucket_list
info['truncated'] = next_marker is not None
if next_marker is not None:
info['next_marker'] = next_marker
result = json.dumps(info)
return Response(result, mimetype='text/json')
def test_not_polled_when_score_is_zero(self):
self._flush_cs('echo')
srv = self._srv('echo')
def check_service_known(body):
self.assertIsInstance(body, list)
self.assertListEqual([srv['addr']], [s['addr'] for s in body])
# register the service with a positive score
srv['score'] = 1
resp = self.request('POST', self._url_cs("lock"), json.dumps(srv))
self.assertIn(resp.status, (200, 204))
# Ensure the proxy reloads its LB pool
self._reload()
# check it appears
resp = self.request('GET',
self._url_cs('list'),
params={"type": "echo"})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
check_service_known(body)
# check it is polled
resp = self.request('POST',
self._url_lb('poll'),
params={"pool": "echo"})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
check_service_known(body)
# register the service locked to 0
srv['score'] = 0
resp = self.request('POST', self._url_cs("lock"), json.dumps(srv))
self.assertIn(resp.status, (200, 204))
# Ensure the proxy reloads its LB pool
self._reload()
# check it appears
resp = self.request('GET',
self._url_cs('list'),
params={"type": "echo"})
self.assertEqual(resp.status, 200)
body = self.json_loads(resp.data)
check_service_known(body)
# the service must not be polled
resp = self.request('POST',
self._url_lb('poll'),
params={"pool": "echo"})
self.assertError(resp, 500, 481)
def container_create_many(self, account, containers, properties=None,
**kwargs):
"""
Create several containers.
:param account: account in which to create the containers
:type account: `str`
:param containers: names of the containers
:type containers: iterable of `str`
:param properties: properties to set on the containers
:type properties: `dict`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: a list of tuples with the name of the container and
a boolean telling if the container has been created
:rtype: `list` of `tuple`
"""
results = list()
try:
params = self._make_params(account)
unformatted_data = list()
for container in containers:
unformatted_data.append({'name': container,
'properties': properties or {},
'system': kwargs.get('system', {})})
data = json.dumps({"containers": unformatted_data})
resp, body = self._request('POST', '/create_many', params=params,
data=data, autocreate=True,
**kwargs)
if resp.status not in (204, 200):
raise exceptions.from_response(resp, body)
for container in body["containers"]:
results.append((container["name"], container["status"] == 201))
return results
except exceptions.TooLarge:
# Batch too large for the proxy
pivot = len(containers) / 2
head = containers[:pivot]
tail = containers[pivot:]
if head:
results += self.container_create_many(
account, head, properties=properties,
**kwargs)
if tail:
results += self.container_create_many(
account, tail, properties=properties,
**kwargs)
return results
except exceptions.NotFound:
# Batches not supported by the proxy
for container in containers:
try:
rc = self.container_create(
account, container, properties=properties,
**kwargs)
results.append((container, rc))
except Exception:
results.append((container, False))
return results
def _restore_object(self, hdrs, account, container):
kwargs = {}
if not self.append and hdrs and 'mime_type' in hdrs:
kwargs['mime_type'] = hdrs['mime_type']
del hdrs['mime_type']
data = LimitedStream(self.req.stream, self.inf.size,
entry=self.cur_state.get('entry'),
offset=self.cur_state.get('offset'))
try:
_, size, _ = self.proxy.object_create(
account, container, obj_name=self.inf.name, append=self.append,
file_or_path=data, **kwargs)
except Exception:
# No data is written if an error occurs during object_create.
# We just have to update our state_machine offset regarding
# the current object.
if self.cur_state.get('manifest') is None:
raise
entry = None
for entry in self.cur_state['manifest']:
if entry['name'] == self.inf.name:
break
else: # it should not happen
raise BadRequest("Invalid internal state")
# Since an error has occured, we have to reset the
# current offset to the start of the current chunk
# and remove the stored checksum if set.
if data.invalid_checksum:
self.logger.error("Invalid checksum detected for %s",
self.inf.name)
raise BadRequest("Checksum error for %s" % self.inf.name)
self.cur_state['end'] = (entry['start_block']
+ self.cur_state['offset_block'])
self.redis.set("restore:%s:%s" % (account, container),
json.dumps(self.cur_state, sort_keys=True),
ex=ContainerBackup.REDIS_TIMEOUT)
raise
# save properties before checking size, otherwise they'll be lost
if hdrs:
self.proxy.object_set_properties(account, container,
self.inf.name,
properties=hdrs)
if size != self.inf.size:
raise UnprocessableEntity(
"Object created is smaller than expected")
self.state['consumed'] += size
if self.mode == self.MODE_RANGE:
self.cur_state['offset_block'] = 0
self.cur_state['offset'] = 0
self.append = False
def container_raw_insert(self, bean, account=None, reference=None,
cid=None, **kwargs):
params = self._make_params(account, reference, cid=cid)
data = json.dumps((bean,))
if kwargs.pop("frozen", None):
params["frozen"] = 1
self._request(
'POST', '/raw_insert', data=data, params=params, **kwargs)
def container_raw_update(self, old, new, account=None, reference=None,
cid=None, **kwargs):
params = self._make_params(account, reference, cid=cid)
data = json.dumps({"old": old, "new": new})
if kwargs.pop("frozen", None):
params["frozen"] = 1
self._request(
'POST', '/raw_update', data=data, params=params, **kwargs)
