def test_single_score(self): srv0 = self._srv('echo', ip='127.0.0.3') def check(code): params = {'type': 'echo', 'service_id': srv0['addr']} resp = self.request('GET', self._url_cs("score"), None, params=params, headers=self.TEST_HEADERS) self.assertEqual(code, resp.status) # Service not found self._reload() check(404) # Registration -> found self._register_srv([srv0]) self._reload_proxy() check(200) # lock to 0 -> found resp = self.request('POST', self._url_cs('lock'), json.dumps(srv0)) self.assertIn(resp.status, (200, 204)) self._reload_proxy() check(200) # removal -> not found resp = self.request('POST', self._url_cs('unlock'), json.dumps(srv0)) self._deregister_srv(srv0) self._flush_proxy() self._reload_proxy() check(404)
def test_services_pool_actions_unlock(self): srv = self._srv('echo') resp = self.request('POST', self._url_cs("lock"), json.dumps(srv)) self.assertIn(resp.status, (200, 204)) resp = self.request('POST', self._url_cs("unlock"), json.dumps(srv)) self.assertIn(resp.status, (200, 204)) resp = self.request('GET', self._url_cs('list'), params={"type": "echo"}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) self.assertIsInstance(body, list)
def _reply_task_res(self, beanstalkd_reply, task_res): self.queue_reply.put(task_res) if beanstalkd_reply is None: return res_event = self.tool.res_event_from_task_res(task_res) if self.tool.beanstalkd is not None: res_event['beanstalkd_worker'] = \ { 'addr': self.tool.beanstalkd.addr, 'tube': self.tool.beanstalkd.tube } try: if self.beanstalkd_reply is None \ or self.beanstalkd_reply.addr != beanstalkd_reply['addr'] \ or self.beanstalkd_reply.tube != beanstalkd_reply['tube']: if self.beanstalkd_reply is not None: self.beanstalkd_reply.close() self.beanstalkd_reply = BeanstalkdSender( beanstalkd_reply['addr'], beanstalkd_reply['tube'], self.logger) self.beanstalkd_reply.send_job(json.dumps(res_event)) except Exception as exc: # pylint: disable=broad-except item, info, error = task_res self.logger.warn( 'Beanstalkd reply failed %s (info=%s error=%s): %s', self.tool.string_from_item(item), str(info), error, exc)
def put_user_policy(self, account, user, policy, policy_name=''): """ Save an IAM policy for the specified user. :param policy: JSON-formatted string :type policy: str :param policy_name: name of the policy (empty string if not set) """ if not isinstance(policy, str): raise TypeError("policy parameter must be a string") if not policy_name and not self.allow_empty_policy_name: raise ValueError('policy name cannot be empty') if policy_name and not self.name_regex.fullmatch(policy_name): raise ValueError('policy name does not match %s' % (self.name_regex.pattern)) # XXX: we should also match user name, but unfortunately, when using # tempauth, user names have the ':' character between the project name # and the actual user name. try: policy_obj = json.loads(policy) policy_obj['UpdateDate'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) # Strip spaces and new lines policy = json.dumps(policy_obj, separators=(',', ':')) except ValueError as err: raise ValueError('policy is not JSON-formatted: %s' % err) acct_key = self.key_for_account(account) policy_key = self.subkey_for_policy(user, policy_name) self.redis.conn.hset(acct_key, policy_key, policy.encode('utf-8'))
def on_account_container_show(self, req, **kwargs): account_id = self._get_account_id(req) cname = self._get_item_id(req, key='container', what='container') raw = self.backend.get_container_info(account_id, cname, **kwargs) if raw is not None: return Response(json.dumps(raw), mimetype=HTTP_CONTENT_TYPE_JSON) return NotFound('Container not found')
def handle_container_listing(self, req, start_response): resp = req.get_response(self.app) if not resp.is_success or resp.content_type != 'application/json': return resp(req.environ, start_response) if resp.content_length is None: return resp(req.environ, start_response) if resp.content_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH: self.logger.warn( 'The content length (%d) of the listing is too long (max=%d)', resp.content_length, MAX_CONTAINER_LISTING_CONTENT_LENGTH) return resp(req.environ, start_response) try: listing = json.loads(resp.body) except ValueError: return resp(req.environ, start_response) for item in listing: if 'subdir' in item \ or item.get('content_type') == DELETE_MARKER_CONTENT_TYPE: continue etag, params = parse_header(item['hash']) if 'slo_etag' in params: item['slo_etag'] = '"%s"' % params.pop('slo_etag') item['hash'] = etag + ''.join('; %s=%s' % kv for kv in params.items()) resp.body = json.dumps(listing) return resp(req.environ, start_response)
def on_job_list(self, req): limit = int_value(req.args.get('limit'), None) marker = req.args.get('marker') job_infos = self.backend.list_jobs(limit=limit, marker=marker) return Response( json.dumps(job_infos), mimetype='application/json')
def on_job_resume(self, req): job_id = self._get_job_id(req) self.backend.resume(job_id) job_info = self.backend.get_job_info(job_id) return Response(json.dumps(job_info), mimetype='application/json', status=202)
def request(self, method, url, data=None, params=None, headers=None, json=None): # Add query string if params: out_param = [] for k, v in params.items(): if v is not None: if isinstance(v, unicode): v = unicode(v).encode('utf-8') out_param.append((k, v)) encoded_args = urlencode(out_param) url += '?' + encoded_args # Convert json and add Content-Type headers = headers if headers else {} if json: headers["Content-Type"] = "application/json" data = jsonlib.dumps(json) out_kwargs = {} out_kwargs['headers'] = headers out_kwargs['body'] = data return self.http_pool.request(method, url, **out_kwargs)
def content_get_properties( self, account=None, reference=None, path=None, properties=None, cid=None, content=None, version=None, params=None, **kwargs): """ Get a description of the content along with its user properties. """ obj_meta, _ = get_cached_object_metadata( account=account, reference=reference, path=path, cid=cid, version=version, properties=True, **kwargs) if obj_meta is not None: return obj_meta uri = self._make_uri('content/get_properties') data = json.dumps(properties) if properties else None resp, body = self._direct_request( 'POST', uri, data=data, params=params, **kwargs) obj_meta = extract_content_headers_meta(resp.headers) obj_meta.update(body) set_cached_object_metadata( obj_meta, None, account=account, reference=reference, path=path, cid=cid, version=version, properties=True, **kwargs) return obj_meta
def container_del_properties(self, account=None, reference=None, properties=[], cid=None, **kwargs): params = self._make_params(account, reference, cid=cid) data = json.dumps(properties) _resp, body = self._request( 'POST', '/del_properties', data=data, params=params, **kwargs) return body
def test_service_pool_actions_lock_and_reput(self): srv = self._srv('echo') resp = self.request('POST', self._url_cs('lock'), json.dumps(srv)) self.assertIn(resp.status, (200, 204)) resp = self.request('GET', self._url_cs('list'), params={"type": "echo"}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) self.assertIsInstance(body, list) self.assertIn(srv['addr'], [x['addr'] for x in body]) self._register_srv(srv) resp = self.request('GET', self._url_cs('list'), params={'type': 'echo'}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) self.assertIsInstance(body, list) self.assertIn(srv['addr'], [x['addr'] for x in body]) srv2 = dict(srv) srv2['score'] = -1 self._register_srv(srv2) self.assertEqual(resp.status, 200) resp = self.request('GET', self._url_cs('list'), params={'type': 'echo'}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) self.assertIsInstance(body, list) self.assertIn(srv['addr'], [x['addr'] for x in body])
def container_snapshot(self, account=None, reference=None, dst_account=None, dst_reference=None, cid=None, **kwargs): """ Create a snapshot of a the container. This function duplicates only the database. It doesn't duplicate the chunks of the contents. :param account: account in which the container is :type account: `str` :param reference: name of the container :type reference: `str` :param cid: container id that can be used instead of account and reference :type cid: `str` :param dst_account: account in which the snapshot will be created :type dst_account: `str` :param dst_reference: name of the snapshot :type dst_reference: `str` """ params = self._make_params(account, reference, cid=cid) data = json.dumps({"account": dst_account, "container": dst_reference}) resp, _ = self._request('POST', '/snapshot', params=params, data=data, **kwargs) return resp
def container_get_properties(self, account=None, reference=None, properties=None, cid=None, params=None, **kwargs): """ Get information about a container (user and system properties). :param account: account in which the container is :type account: `str` :param reference: name of the container :type reference: `str` :param cid: container id that can be used instead of account and reference :type cid: `str` :keyword headers: extra headers to send to the proxy :type headers: `dict` :returns: a `dict` with "properties" and "system" entries, containing respectively a `dict` of user properties and a `dict` of system properties. """ if not properties: properties = list() data = json.dumps(properties) _resp, body = self._request('POST', '/get_properties', data=data, params=params, **kwargs) return body
def on_account_container_show(self, req): account_id = self._get_account_id(req) cname = self._get_item_id(req, key='container', what='container') raw = self.backend.get_container_info(account_id, cname) if raw is not None: return Response(json.dumps(raw), mimetype='text/json') return NotFound('Container not found')
def container_create(self, account, reference, properties=None, system=None, **kwargs): """ Create a container. :param account: account in which to create the container :type account: `str` :param reference: name of the container :type reference: `str` :param properties: properties to set on the container :type properties: `dict` :param system: system properties to set on the container :type system: `dict` :keyword headers: extra headers to send to the proxy :type headers: `dict` :returns: True if the container has been created, False if it already exists """ params = self._make_params(account, reference) data = json.dumps({ 'properties': properties or {}, 'system': system or {} }) resp, body = self._request('POST', '/create', params=params, data=data, **kwargs) if resp.status not in (204, 201): raise exceptions.from_response(resp, body) return resp.status == 201
def content_prepare(self, account=None, reference=None, path=None, size=None, cid=None, stgpol=None, **kwargs): """ Prepare an upload: get URLs of chunks on available rawx. :keyword autocreate: create container if it doesn't exist """ uri = self._make_uri('content/prepare') params = self._make_params(account, reference, path, cid=cid) data = {'size': size} if stgpol: data['policy'] = stgpol data = json.dumps(data) resp, body = self._direct_request('POST', uri, data=data, params=params, **kwargs) resp_headers = extract_content_headers_meta(resp.headers) return resp_headers, body
def on_account_containers(self, req): account_id = self._get_account_id(req) info = self.backend.info_account(account_id) if not info: return NotFound('Account not found') marker = req.args.get('marker', '') end_marker = req.args.get('end_marker', '') prefix = req.args.get('prefix', '') limit = int(req.args.get('limit', '1000')) limit = max(0, min(ACCOUNT_LISTING_MAX_LIMIT, int_value( req.args.get('limit'), 0))) if limit <= 0: limit = ACCOUNT_LISTING_DEFAULT_LIMIT delimiter = req.args.get('delimiter', '') s3_buckets_only = true_value(req.args.get('s3_buckets_only', False)) user_list = self.backend.list_containers( account_id, limit=limit, marker=marker, end_marker=end_marker, prefix=prefix, delimiter=delimiter, s3_buckets_only=s3_buckets_only) info['listing'] = user_list # TODO(FVE): add "truncated" entry telling if the listing is truncated result = json.dumps(info) return Response(result, mimetype='text/json')
def content_prepare(self, account=None, reference=None, path=None, size=None, cid=None, stgpol=None, content_id=None, version=None, params=None, **kwargs): """ Prepare an upload: get URLs of chunks on available rawx. :keyword autocreate: create container if it doesn't exist """ uri = self._make_uri('content/prepare') data = {'size': size} if stgpol: data['policy'] = stgpol data = json.dumps(data) try: resp, body = self._direct_request( 'POST', uri + '2', data=data, params=params, **kwargs) chunks = body['chunks'] obj_meta = extract_content_headers_meta(resp.headers) obj_meta['properties'] = dict() # pylint: disable=no-member obj_meta['properties'].update(body.get('properties', {})) except exceptions.NotFound: # Proxy does not support v2 request (oio < 4.3) resp, chunks = self._direct_request( 'POST', uri, data=data, params=params, **kwargs) obj_meta = extract_content_headers_meta(resp.headers) return obj_meta, chunks
def content_del_properties(self, account=None, reference=None, path=None, properties=[], cid=None, version=None, **kwargs): """ Delete some properties from an object. :param properties: list of property keys to delete :type properties: `list` :returns: True is the property has been deleted """ uri = self._make_uri('content/del_properties') params = self._make_params(account, reference, path, cid=cid, version=version) data = json.dumps(properties) resp, _body = self._direct_request('POST', uri, data=data, params=params, **kwargs) return resp.status == 204
def force(self, account=None, reference=None, service_type=None, services=None, cid=None, autocreate=False, replace=False, **kwargs): """ Associate the specified services to the reference. :param replace: do not require the list of services of the specified type to be empty, overwrite it. :type replace: `bool` """ params = self._make_params(account, reference, service_type, cid=cid) if replace: params["replace"] = "yes" data = json.dumps(services) _resp, _body = self._request('POST', '/force', data=data, params=params, autocreate=autocreate, **kwargs)
def container_set_properties(self, account=None, reference=None, properties=None, clear=False, cid=None, system=None, **kwargs): params = self._make_params(account, reference, cid=cid) if clear: params["flush"] = 1 data = json.dumps({ 'properties': properties or {}, 'system': system or {} }) del_cached_container_metadata(account=account, reference=reference, cid=cid, **kwargs) _resp, body = self._request('POST', '/set_properties', data=data, params=params, **kwargs) return body
def content_show(self, account=None, reference=None, path=None, properties=None, cid=None, content=None, version=None, **kwargs): """ Get a description of the content along with its user properties. """ uri = self._make_uri('content/get_properties') params = self._make_params(account, reference, path, cid=cid, content=content, version=version) data = json.dumps(properties) if properties else None resp, body = self._direct_request('POST', uri, data=data, params=params, **kwargs) obj_meta = extract_content_headers_meta(resp.headers) obj_meta.update(body) return obj_meta
def content_set_properties(self, account=None, reference=None, path=None, properties={}, cid=None, version=None, clear=False, **kwargs): """ Set properties on an object. :param properties: dictionary of properties """ uri = self._make_uri('content/set_properties') params = self._make_params(account, reference, path, cid=cid, version=version) if clear: params['flush'] = 1 data = json.dumps(properties) _resp, _body = self._direct_request('POST', uri, data=data, params=params, **kwargs)
def on_account_buckets(self, req, **kwargs): account_id = self._get_account_id(req) info = self.backend.info_account(account_id, **kwargs) if not info: return NotFound('Account not found') marker = req.args.get('marker', '') end_marker = req.args.get('end_marker', '') prefix = req.args.get('prefix', '') limit = int(req.args.get('limit', '1000')) bucket_list, next_marker = self.backend.list_buckets( account_id, limit=limit, marker=marker, end_marker=end_marker, prefix=prefix, **kwargs) info['listing'] = bucket_list info['truncated'] = next_marker is not None if next_marker is not None: info['next_marker'] = next_marker result = json.dumps(info) return Response(result, mimetype='text/json')
def test_not_polled_when_score_is_zero(self): self._flush_cs('echo') srv = self._srv('echo') def check_service_known(body): self.assertIsInstance(body, list) self.assertListEqual([srv['addr']], [s['addr'] for s in body]) # register the service with a positive score srv['score'] = 1 resp = self.request('POST', self._url_cs("lock"), json.dumps(srv)) self.assertIn(resp.status, (200, 204)) # Ensure the proxy reloads its LB pool self._reload() # check it appears resp = self.request('GET', self._url_cs('list'), params={"type": "echo"}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) check_service_known(body) # check it is polled resp = self.request('POST', self._url_lb('poll'), params={"pool": "echo"}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) check_service_known(body) # register the service locked to 0 srv['score'] = 0 resp = self.request('POST', self._url_cs("lock"), json.dumps(srv)) self.assertIn(resp.status, (200, 204)) # Ensure the proxy reloads its LB pool self._reload() # check it appears resp = self.request('GET', self._url_cs('list'), params={"type": "echo"}) self.assertEqual(resp.status, 200) body = self.json_loads(resp.data) check_service_known(body) # the service must not be polled resp = self.request('POST', self._url_lb('poll'), params={"pool": "echo"}) self.assertError(resp, 500, 481)
def container_create_many(self, account, containers, properties=None, **kwargs): """ Create several containers. :param account: account in which to create the containers :type account: `str` :param containers: names of the containers :type containers: iterable of `str` :param properties: properties to set on the containers :type properties: `dict` :keyword headers: extra headers to send to the proxy :type headers: `dict` :returns: a list of tuples with the name of the container and a boolean telling if the container has been created :rtype: `list` of `tuple` """ results = list() try: params = self._make_params(account) unformatted_data = list() for container in containers: unformatted_data.append({'name': container, 'properties': properties or {}, 'system': kwargs.get('system', {})}) data = json.dumps({"containers": unformatted_data}) resp, body = self._request('POST', '/create_many', params=params, data=data, autocreate=True, **kwargs) if resp.status not in (204, 200): raise exceptions.from_response(resp, body) for container in body["containers"]: results.append((container["name"], container["status"] == 201)) return results except exceptions.TooLarge: # Batch too large for the proxy pivot = len(containers) / 2 head = containers[:pivot] tail = containers[pivot:] if head: results += self.container_create_many( account, head, properties=properties, **kwargs) if tail: results += self.container_create_many( account, tail, properties=properties, **kwargs) return results except exceptions.NotFound: # Batches not supported by the proxy for container in containers: try: rc = self.container_create( account, container, properties=properties, **kwargs) results.append((container, rc)) except Exception: results.append((container, False)) return results
def _restore_object(self, hdrs, account, container): kwargs = {} if not self.append and hdrs and 'mime_type' in hdrs: kwargs['mime_type'] = hdrs['mime_type'] del hdrs['mime_type'] data = LimitedStream(self.req.stream, self.inf.size, entry=self.cur_state.get('entry'), offset=self.cur_state.get('offset')) try: _, size, _ = self.proxy.object_create( account, container, obj_name=self.inf.name, append=self.append, file_or_path=data, **kwargs) except Exception: # No data is written if an error occurs during object_create. # We just have to update our state_machine offset regarding # the current object. if self.cur_state.get('manifest') is None: raise entry = None for entry in self.cur_state['manifest']: if entry['name'] == self.inf.name: break else: # it should not happen raise BadRequest("Invalid internal state") # Since an error has occured, we have to reset the # current offset to the start of the current chunk # and remove the stored checksum if set. if data.invalid_checksum: self.logger.error("Invalid checksum detected for %s", self.inf.name) raise BadRequest("Checksum error for %s" % self.inf.name) self.cur_state['end'] = (entry['start_block'] + self.cur_state['offset_block']) self.redis.set("restore:%s:%s" % (account, container), json.dumps(self.cur_state, sort_keys=True), ex=ContainerBackup.REDIS_TIMEOUT) raise # save properties before checking size, otherwise they'll be lost if hdrs: self.proxy.object_set_properties(account, container, self.inf.name, properties=hdrs) if size != self.inf.size: raise UnprocessableEntity( "Object created is smaller than expected") self.state['consumed'] += size if self.mode == self.MODE_RANGE: self.cur_state['offset_block'] = 0 self.cur_state['offset'] = 0 self.append = False
def container_raw_insert(self, bean, account=None, reference=None, cid=None, **kwargs): params = self._make_params(account, reference, cid=cid) data = json.dumps((bean,)) if kwargs.pop("frozen", None): params["frozen"] = 1 self._request( 'POST', '/raw_insert', data=data, params=params, **kwargs)
def container_raw_update(self, old, new, account=None, reference=None, cid=None, **kwargs): params = self._make_params(account, reference, cid=cid) data = json.dumps({"old": old, "new": new}) if kwargs.pop("frozen", None): params["frozen"] = 1 self._request( 'POST', '/raw_update', data=data, params=params, **kwargs)