def reap_object(self, account, container, container_partition, container_nodes, obj): """ Deletes the given object by issuing a delete request to each node for the object. The format of the delete request is such that each object server will update a corresponding container server, removing the object from the container's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the object. :param container: The name of the container for the object. :param container_partition: The partition for the container on the container ring. :param container_nodes: The primary node dicts for the container. :param obj: The name of the object to delete. * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the container node dicts. """ container_nodes = list(container_nodes) part, nodes = self.get_object_ring().get_nodes(account, container, obj) successes = 0 failures = 0 for node in nodes: cnode = container_nodes.pop() try: direct_delete_object(node, part, account, container, obj, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={ 'X-Container-Host': '%(ip)s:%(port)s' % cnode, 'X-Container-Partition': str(container_partition), 'X-Container-Device': cnode['device'] }) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 except ClientException, err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 if successes > failures: self.stats_objects_deleted += 1 elif not successes: self.stats_objects_remaining += 1 else: self.stats_objects_possibly_remaining += 1
def reap_object(self, account, container, container_partition, container_nodes, obj): """ Deletes the given object by issuing a delete request to each node for the object. The format of the delete request is such that each object server will update a corresponding container server, removing the object from the container's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the object. :param container: The name of the container for the object. :param container_partition: The partition for the container on the container ring. :param container_nodes: The primary node dicts for the container. :param obj: The name of the object to delete. * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the container node dicts. """ container_nodes = list(container_nodes) part, nodes = self.get_object_ring().get_nodes(account, container, obj) successes = 0 failures = 0 for node in nodes: cnode = container_nodes.pop() try: direct_delete_object( node, part, account, container, obj, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode, 'X-Container-Partition': str(container_partition), 'X-Container-Device': cnode['device']}) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException, err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.logger.increment('objects_failures') self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status / 100,)) if successes > failures: self.stats_objects_deleted += 1 self.logger.increment('objects_deleted') elif not successes: self.stats_objects_remaining += 1 self.logger.increment('objects_remaining') else: self.stats_objects_possibly_remaining += 1 self.logger.increment('objects_possibly_remaining')
def test_direct_delete_object_error(self): with mocked_http_conn(503) as conn: with self.assertRaises(ClientException) as raised: direct_client.direct_delete_object(self.node, self.part, self.account, self.container, self.obj) self.assertEqual(conn.method, 'DELETE') self.assertEqual(conn.path, self.obj_path) self.assertEqual(raised.exception.http_status, 503) self.assertTrue('DELETE' in str(raised.exception))
def test_direct_delete_object_error(self): with mocked_http_conn(503) as conn: with self.assertRaises(ClientException) as raised: direct_client.direct_delete_object( self.node, self.part, self.account, self.container, self.obj) self.assertEqual(conn.method, 'DELETE') self.assertEqual(conn.path, self.obj_path) self.assertEqual(raised.exception.http_status, 503) self.assertTrue('DELETE' in str(raised.exception))
def test_direct_delete_object_with_timestamp(self): # ensure timestamp is different from any that might be auto-generated timestamp = Timestamp(time.time() - 100) headers = {'X-Timestamp': timestamp.internal} with mocked_http_conn(200) as conn: direct_client.direct_delete_object( self.node, self.part, self.account, self.container, self.obj, headers=headers) self.assertEqual(conn.method, 'DELETE') self.assertEqual(conn.path, self.obj_path) self.assertTrue('X-Timestamp' in conn.req_headers) self.assertEqual(timestamp, conn.req_headers['X-Timestamp'])
def test_direct_delete_object(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200) direct_client.direct_delete_object(node, part, account, container, name) direct_client.http_connect = was_http_connector
def test_direct_delete_object_error(self): with mocked_http_conn(503) as conn: try: direct_client.direct_delete_object( self.node, self.part, self.account, self.container, self.obj) except ClientException as err: pass else: self.fail('ClientException not raised') self.assertEqual(conn.method, 'DELETE') self.assertEqual(conn.path, self.obj_path) self.assertEqual(err.http_status, 503) self.assertTrue('DELETE' in str(err))
def _run(self, thread): if time.time() - self.heartbeat >= 15: self.heartbeat = time.time() self._log_status("DEL") device, partition, name, container_name = self.names.pop() with self.connection() as conn: try: if self.use_proxy: client.delete_object(self.url, self.token, container_name, name, http_conn=conn) else: node = {"ip": self.ip, "port": self.port, "device": device} direct_client.direct_delete_object(node, partition, self.account, container_name, name) except client.ClientException, e: self.logger.debug(str(e)) self.failures += 1
def test_direct_delete_object(self): with mocked_http_conn(200) as conn: resp = direct_client.direct_delete_object( self.node, self.part, self.account, self.container, self.obj) self.assertEqual(conn.method, 'DELETE') self.assertEqual(conn.path, self.obj_path) self.assertEqual(resp, None)
def _run(self, thread): if time.time() - self.heartbeat >= 15: self.heartbeat = time.time() self._log_status('DEL') device, partition, name, container_name = self.names.pop() with self.connection() as conn: try: if self.use_proxy: client.delete_object(self.url, self.token, container_name, name, http_conn=conn) else: node = {'ip': self.ip, 'port': self.port, 'device': device} direct_client.direct_delete_object(node, partition, self.account, container_name, name) except client.ClientException, e: self.logger.debug(str(e)) self.failures += 1
def reap_object(self, account, container, container_partition, container_nodes, obj, policy_index): """ Deletes the given object by issuing a delete request to each node for the object. The format of the delete request is such that each object server will update a corresponding container server, removing the object from the container's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the object. :param container: The name of the container for the object. :param container_partition: The partition for the container on the container ring. :param container_nodes: The primary node dicts for the container. :param obj: The name of the object to delete. :param policy_index: The storage policy index of the object's container * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the container node dicts. """ cnodes = itertools.cycle(container_nodes) try: ring = self.get_object_ring(policy_index) except PolicyError: self.stats_objects_remaining += 1 self.logger.increment('objects_remaining') return part, nodes = ring.get_nodes(account, container, obj) successes = 0 failures = 0 timestamp = Timestamp.now() for node in nodes: cnode = next(cnodes) try: direct_delete_object( node, part, account, container, obj, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode, 'X-Container-Partition': str(container_partition), 'X-Container-Device': cnode['device'], 'X-Backend-Storage-Policy-Index': policy_index, 'X-Timestamp': timestamp.internal}) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.logger.increment('objects_failures') self.stats_return_codes[err.http_status // 100] = \ self.stats_return_codes.get(err.http_status // 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status // 100,)) except (Timeout, socket.error) as err: failures += 1 self.logger.increment('objects_failures') self.logger.error( _('Timeout Exception with %(ip)s:%(port)s/%(device)s'), node) if successes > failures: self.stats_objects_deleted += 1 self.logger.increment('objects_deleted') elif not successes: self.stats_objects_remaining += 1 self.logger.increment('objects_remaining') else: self.stats_objects_possibly_remaining += 1 self.logger.increment('objects_possibly_remaining')
def reap_object(self, account, container, container_partition, container_nodes, obj, policy_index): """ Deletes the given object by issuing a delete request to each node for the object. The format of the delete request is such that each object server will update a corresponding container server, removing the object from the container's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the object. :param container: The name of the container for the object. :param container_partition: The partition for the container on the container ring. :param container_nodes: The primary node dicts for the container. :param obj: The name of the object to delete. :param policy_index: The storage policy index of the object's container * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the container node dicts. """ cnodes = itertools.cycle(container_nodes) try: ring = self.get_object_ring(policy_index) except PolicyError: self.stats_objects_remaining += 1 self.logger.increment('objects_remaining') return part, nodes = ring.get_nodes(account, container, obj) successes = 0 failures = 0 timestamp = Timestamp.now() for node in nodes: cnode = next(cnodes) try: direct_delete_object(node, part, account, container, obj, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={ 'X-Container-Host': '%(ip)s:%(port)s' % cnode, 'X-Container-Partition': str(container_partition), 'X-Container-Device': cnode['device'], 'X-Backend-Storage-Policy-Index': policy_index, 'X-Timestamp': timestamp.internal }) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.logger.increment('objects_failures') self.stats_return_codes[err.http_status // 100] = \ self.stats_return_codes.get(err.http_status // 100, 0) + 1 self.logger.increment('return_codes.%d' % (err.http_status // 100, )) except (Timeout, socket.error) as err: failures += 1 self.logger.increment('objects_failures') self.logger.error( _('Timeout Exception with %(ip)s:%(port)s/%(device)s'), node) if successes > failures: self.stats_objects_deleted += 1 self.logger.increment('objects_deleted') elif not successes: self.stats_objects_remaining += 1 self.logger.increment('objects_remaining') else: self.stats_objects_possibly_remaining += 1 self.logger.increment('objects_possibly_remaining')