def test_get_range_last_byte(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=24-24'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "1") self.assertEqual(body, b'e')
def test_get_range_on_segment_boundaries(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=10-19'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") self.assertEqual(body, b'cccccddddd')
def __init__(self, status, body='', headers=None, frag_index=None): self.status = status self.body = body self.readable = BytesIO(body) self.headers = HeaderKeyDict(headers) if frag_index is not None: self.headers['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index fake_reason = ('Fake', 'This response is a lie.') self.reason = swob.RESPONSE_REASONS.get(status, fake_reason)[0]
def gen_headers(hdrs_in=None, add_ts=True): """ Get the headers ready for a request. All requests should have a User-Agent string, but if one is passed in don't over-write it. Not all requests will need an X-Timestamp, but if one is passed in do not over-write it. :param headers: dict or None, base for HTTP headers :param add_ts: boolean, should be True for any "unsafe" HTTP request :returns: HeaderKeyDict based on headers and ready for the request """ hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict() if add_ts and 'X-Timestamp' not in hdrs_out: hdrs_out['X-Timestamp'] = Timestamp.now().internal if 'user-agent' not in hdrs_out: hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() hdrs_out.setdefault('X-Backend-Allow-Reserved-Names', 'true') return hdrs_out
def test_get_suffix_range(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-40'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "25") self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def test_setup(self): bucket_name = self.create_name('new-bucket') resp = self.client.create_bucket(Bucket=bucket_name) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) expected_location = '/%s' % bucket_name self.assertEqual(expected_location, resp['Location']) headers = HeaderKeyDict(resp['ResponseMetadata']['HTTPHeaders']) self.assertEqual('0', headers['content-length']) self.assertEqual(expected_location, headers['location']) # get versioning resp = self.client.get_bucket_versioning(Bucket=bucket_name) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) self.assertNotIn('Status', resp) # put versioning versioning_config = { 'Status': 'Enabled', } resp = self.client.put_bucket_versioning( Bucket=bucket_name, VersioningConfiguration=versioning_config) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) # ... now it's enabled def check_status(): resp = self.client.get_bucket_versioning(Bucket=bucket_name) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) try: self.assertEqual('Enabled', resp['Status']) except KeyError: self.fail('Status was not in %r' % resp) retry(check_status) # send over some bogus junk versioning_config['Status'] = 'Disabled' with self.assertRaises(ClientError) as ctx: self.client.put_bucket_versioning( Bucket=bucket_name, VersioningConfiguration=versioning_config) expected_err = 'An error occurred (MalformedXML) when calling the ' \ 'PutBucketVersioning operation: The XML you provided was ' \ 'not well-formed or did not validate against our published schema' self.assertEqual(expected_err, str(ctx.exception)) # disable it versioning_config['Status'] = 'Suspended' resp = self.client.put_bucket_versioning( Bucket=bucket_name, VersioningConfiguration=versioning_config) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) # ... now it's disabled again def check_status(): resp = self.client.get_bucket_versioning(Bucket=bucket_name) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) self.assertEqual('Suspended', resp['Status']) retry(check_status)
def _get_footers(self, req): """ Get extra metadata that may be generated during upload by some middlewares (e.g. checksum of cyphered data). """ footers = HeaderKeyDict() footer_callback = req.environ.get('swift.callback.update_footers', lambda _footer: None) footer_callback(footers) return footers
def test_get_range_overlapping_end(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=18-30'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "7") self.assertEqual(headers["Content-Range"], "bytes 18-24/25") self.assertEqual(body, b'ddeeeee')
def test_head_large_object(self): expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(headers["Etag"], expected_etag) self.assertEqual(headers["Content-Length"], "25")
def test_head_large_object_too_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'HEAD'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) # etag is manifest's etag self.assertEqual(headers["Etag"], "etag-manyseg") self.assertIsNone(headers.get("Content-Length"))
def process_object_update(self, update_path, device, policy): """ Process the object information to be updated and update. :param update_path: path to pickled object update file :param device: path to device :param policy: storage policy of object update """ try: update = pickle.load(open(update_path, 'rb')) except Exception: self.logger.exception( _('ERROR Pickle problem, quarantining %s'), update_path) self.logger.increment('quarantines') target_path = os.path.join(device, 'quarantined', 'objects', os.path.basename(update_path)) renamer(update_path, target_path, fsync=False) return successes = update.get('successes', []) part, nodes = self.get_container_ring().get_nodes( update['account'], update['container']) obj = '/%s/%s/%s' % \ (update['account'], update['container'], update['obj']) headers_out = HeaderKeyDict(update['headers']) headers_out['user-agent'] = 'object-updater %s' % os.getpid() headers_out.setdefault('X-Backend-Storage-Policy-Index', str(int(policy))) events = [spawn(self.object_update, node, part, update['op'], obj, headers_out) for node in nodes if node['id'] not in successes] success = True new_successes = False for event in events: event_success, node_id = event.wait() if event_success is True: successes.append(node_id) new_successes = True else: success = False if success: self.successes += 1 self.logger.increment('successes') self.logger.debug('Update sent for %(obj)s %(path)s', {'obj': obj, 'path': update_path}) self.logger.increment("unlinks") os.unlink(update_path) else: self.failures += 1 self.logger.increment('failures') self.logger.debug('Update failed for %(obj)s %(path)s', {'obj': obj, 'path': update_path}) if new_successes: update['successes'] = successes write_pickle(update, update_path, os.path.join( device, get_tmp_dir(policy)))
def test_obj_put_legacy_updates(self): ts = (normalize_timestamp(t) for t in itertools.count(int(time()))) policy = POLICIES.get_by_index(0) # setup updater conf = { 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, } async_dir = os.path.join(self.sda1, get_async_dir(policy)) os.mkdir(async_dir) account, container, obj = 'a', 'c', 'o' # write an async for op in ('PUT', 'DELETE'): self.logger._clear() daemon = object_updater.ObjectUpdater(conf, logger=self.logger) dfmanager = DiskFileManager(conf, daemon.logger) # don't include storage-policy-index in headers_out pickle headers_out = HeaderKeyDict({ 'x-size': 0, 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-timestamp': next(ts), }) data = { 'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out } dfmanager.pickle_async_update(self.sda1, account, container, obj, data, next(ts), policy) request_log = [] def capture(*args, **kwargs): request_log.append((args, kwargs)) # run once fake_status_codes = [200, 200, 200] with mocked_http_conn(*fake_status_codes, give_connect=capture): daemon.run_once() self.assertEqual(len(fake_status_codes), len(request_log)) for request_args, request_kwargs in request_log: ip, part, method, path, headers, qs, ssl = request_args self.assertEqual(method, op) self.assertEqual(headers['X-Backend-Storage-Policy-Index'], str(int(policy))) self.assertEqual(daemon.logger.get_increment_counts(), { 'successes': 1, 'unlinks': 1, 'async_pendings': 1 })
def test_get_response_headers_with_legacy_data(self): broker = backend.AccountBroker(':memory:', account='a') now = time.time() with mock.patch('time.time', new=lambda: now): broker.initialize(Timestamp(now).internal) # add some container data ts = (Timestamp(t).internal for t in itertools.count(int(now))) total_containers = 0 total_objects = 0 total_bytes = 0 for policy in POLICIES: delete_timestamp = next(ts) put_timestamp = next(ts) object_count = int(policy) bytes_used = int(policy) * 10 broker.put_container('c-%s' % policy.name, put_timestamp, delete_timestamp, object_count, bytes_used, int(policy)) total_containers += 1 total_objects += object_count total_bytes += bytes_used expected = HeaderKeyDict({ 'X-Account-Container-Count': total_containers, 'X-Account-Object-Count': total_objects, 'X-Account-Bytes-Used': total_bytes, 'X-Timestamp': Timestamp(now).normal, 'X-PUT-Timestamp': Timestamp(now).normal, }) for policy in POLICIES: prefix = 'X-Account-Storage-Policy-%s-' % policy.name expected[prefix + 'Object-Count'] = int(policy) expected[prefix + 'Bytes-Used'] = int(policy) * 10 orig_policy_stats = broker.get_policy_stats def stub_policy_stats(*args, **kwargs): policy_stats = orig_policy_stats(*args, **kwargs) for stats in policy_stats.values(): # legacy db's won't return container_count del stats['container_count'] return policy_stats broker.get_policy_stats = stub_policy_stats resp_headers = utils.get_response_headers(broker) per_policy_container_headers = [ h for h in resp_headers if h.lower().startswith('x-account-storage-policy-') and h.lower().endswith('-container-count') ] self.assertFalse(per_policy_container_headers) for key, value in resp_headers.items(): expected_value = expected.pop(key) self.assertEqual( expected_value, str(value), 'value for %r was %r not %r' % (key, value, expected_value)) self.assertFalse(expected)
def test_get_suffix_range_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-5'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Range")) self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def _get_direct_account_container(path, stype, node, part, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15, end_marker=None, reverse=None, headers=None): """Base class for get direct account and container. Do not use directly use the get_direct_account or get_direct_container instead. """ params = ['format=json'] if marker: params.append('marker=%s' % quote(marker)) if limit: params.append('limit=%d' % limit) if prefix: params.append('prefix=%s' % quote(prefix)) if delimiter: params.append('delimiter=%s' % quote(delimiter)) if end_marker: params.append('end_marker=%s' % quote(end_marker)) if reverse: params.append('reverse=%s' % quote(reverse)) qs = '&'.join(params) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, query_string=qs, headers=gen_headers(hdrs_in=headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise DirectClientException(stype, 'GET', node, part, path, resp) resp_headers = HeaderKeyDict() for header, value in resp.getheaders(): resp_headers[header] = value if resp.status == HTTP_NO_CONTENT: resp.read() return resp_headers, [] return resp_headers, json.loads(resp.read().decode('ascii'))
def test_if_none_match_does_not_match(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': 'not it'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '25') self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')
def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.etag = None
def test_if_match_does_not_match(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': 'not it'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, '412 Precondition Failed') self.assertEqual(headers['Content-Length'], '0') self.assertEqual(body, b'')
def handle_get_head_symlink(self, req): """ Handle get/head request when client sent parameter ?symlink=get :param req: HTTP GET or HEAD object request with param ?symlink=get :returns: Response Iterator """ resp = self._app_call(req.environ) response_header_dict = HeaderKeyDict(self._response_headers) symlink_sysmeta_to_usermeta(response_header_dict) self._response_headers = response_header_dict.items() return resp
def __init__(self, stype, method, node, part, path, resp, host=None): # host can be used to override the node ip and port reported in # the exception host = host if host is not None else node full_path = quote('/%s/%s%s' % (node['device'], part, path)) msg = '%s server %s:%s direct %s %r gave status %s' % ( stype, host['ip'], host['port'], method, full_path, resp.status) headers = HeaderKeyDict(resp.getheaders()) super(DirectClientException, self).__init__( msg, http_host=host['ip'], http_port=host['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason, http_headers=headers)
def test_get_range_many_segments_satisfiability_unknown(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=10-22'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") # this requires multiple pages of container listing, so we can't send # a Content-Length header self.assertIsNone(headers.get("Content-Length")) self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")
def __init__(self, stype, method, node, part, path, resp): full_path = quote('/%s/%s%s' % (node['device'], part, path)) msg = '%s server %s:%s direct %s %r gave status %s' % ( stype, node['ip'], node['port'], method, full_path, resp.status) headers = HeaderKeyDict(resp.getheaders()) super(DirectClientException, self).__init__(msg, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason, http_headers=headers)
def test_retry(self): headers = HeaderKeyDict({'key': 'value'}) with mocked_http_conn(200, headers) as conn: attempts, resp = direct_client.retry( direct_client.direct_head_object, self.node, self.part, self.account, self.container, self.obj) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.obj_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp) self.assertEqual(attempts, 1)
def test_direct_put_object_header_content_length(self): contents = six.StringIO('123456') stub_headers = HeaderKeyDict({ 'Content-Length': '6'}) with mocked_http_conn(200) as conn: resp = direct_client.direct_put_object( self.node, self.part, self.account, self.container, self.obj, contents, headers=stub_headers) self.assertEqual('PUT', conn.method) self.assertEqual(conn.req_headers['Content-length'], '6') self.assertEqual(md5('123456').hexdigest(), resp)
def test_direct_head_container(self): headers = HeaderKeyDict(key='value') with mocked_http_conn(200, headers) as conn: resp = direct_client.direct_head_container( self.node, self.part, self.account, self.container) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.container_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp)
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path, etag): if req.environ.get('swift.symlink_override'): req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \ req.headers[TGT_BYTES_SYMLINK_HDR] return # next we'll make sure the E-Tag matches a real object new_req = make_subrequest(req.environ, path=wsgi_quote(symlink_target_path), method='HEAD', swift_source='SYM') if req.allow_reserved_names: new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true' self._last_target_path = symlink_target_path resp = self._recursive_get_head(new_req, target_etag=etag, follow_softlinks=False) if self._get_status_int() == HTTP_NOT_FOUND: raise HTTPConflict(body='X-Symlink-Target does not exist', request=req, headers={ 'Content-Type': 'text/plain', 'Content-Location': self._last_target_path }) if not is_success(self._get_status_int()): drain_and_close(resp) raise status_map[self._get_status_int()](request=req) response_headers = HeaderKeyDict(self._response_headers) # carry forward any etag update params (e.g. "slo_etag"), we'll append # symlink_target_* params to this header after this method returns override_header = get_container_update_override_key('etag') if override_header in response_headers and \ override_header not in req.headers: sep, params = response_headers[override_header].partition(';')[1:] req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params # It's troublesome that there's so much leakage with SLO if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \ override_header not in req.headers: req.headers[override_header] = '%s; slo_etag=%s' % ( MD5_OF_EMPTY_STRING, response_headers['X-Object-Sysmeta-Slo-Etag']) req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = ( response_headers.get('x-object-sysmeta-slo-size') or response_headers['Content-Length']) req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag if not req.headers.get('Content-Type'): req.headers['Content-Type'] = response_headers['Content-Type']
def __init__(self, method, node, path, resp): if not isinstance(path, six.text_type): path = path.decode("utf-8") msg = 'server %s:%s direct %s %r gave status %s' % ( node['ip'], node['port'], method, path, resp.status) headers = HeaderKeyDict(resp.getheaders()) super(DirectClientReconException, self).__init__(msg, http_host=node['ip'], http_port=node['port'], http_status=resp.status, http_reason=resp.reason, http_headers=headers)
def test_get_range(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=8-17'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") self.assertEqual(body, b'bbcccccddd') expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) self.assertEqual(headers.get("Etag"), expected_etag)
def test_mismatched_etag_fetching_second_segment(self): self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")}, 'bbWRONGbb') req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error
def test_get_multi_range(self): # DLO doesn't support multi-range GETs. The way that you express that # in HTTP is to return a 200 response containing the whole entity. req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=5-9,15-19'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertIsNone(headers.get("Content-Length")) self.assertIsNone(headers.get("Content-Range")) self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')