def test_retry_client_exception(self): logger = debug_logger('direct-client-test') with mock.patch('swift.common.direct_client.sleep') as mock_sleep, \ mocked_http_conn(500) as conn: with self.assertRaises(direct_client.ClientException) as err_ctx: direct_client.retry(direct_client.direct_delete_object, self.node, self.part, self.account, self.container, self.obj, retries=2, error_log=logger.error) self.assertEqual('DELETE', conn.method) self.assertEqual(err_ctx.exception.http_status, 500) self.assertIn('DELETE', err_ctx.exception.message) self.assertIn(quote('/%s/%s/%s/%s/%s' % (self.node['device'], self.part, self.account, self.container, self.obj)), err_ctx.exception.message) self.assertIn(self.node['ip'], err_ctx.exception.message) self.assertIn(self.node['port'], err_ctx.exception.message) self.assertEqual(self.node['ip'], err_ctx.exception.http_host) self.assertEqual(self.node['port'], err_ctx.exception.http_port) self.assertEqual(self.node['device'], err_ctx.exception.http_device) self.assertEqual(500, err_ctx.exception.http_status) self.assertEqual([mock.call(1), mock.call(2)], mock_sleep.call_args_list) error_lines = logger.get_lines_for_level('error') self.assertEqual(3, len(error_lines)) for line in error_lines: self.assertIn('500 Internal Error', line)
def test_retry_http_exception(self): err_log_file = six.StringIO() def mock_err_logger(err): err_log_file.write(err) def mock_direct_delete_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, headers=None): resp = "Unable to delete object" raise HTTPException('Object', 'DELETE', resp) with mocked_http_conn(500): with mock.patch('swift.common.direct_client.direct_delete_object', mock_direct_delete_object): try: attempts, resp = direct_client.retry( direct_client.direct_delete_object, self.node, self.part, self.account, self.container, self.obj, retries=2, error_log=mock_err_logger) except HTTPException: self.assertTrue(err_log_file.len) err_log_file.close()
def test_retry_http_exception(self): logger = debug_logger('direct-client-test') with mock.patch('swift.common.direct_client.sleep') as mock_sleep, \ mocked_http_conn(HTTPException('Kaboom!')) as conn: with self.assertRaises(HTTPException) as err_ctx: direct_client.retry(direct_client.direct_delete_object, self.node, self.part, self.account, self.container, self.obj, retries=2, error_log=logger.error) self.assertEqual('DELETE', conn.method) self.assertEqual('Kaboom!', str(err_ctx.exception)) self.assertEqual([mock.call(1), mock.call(2)], mock_sleep.call_args_list) error_lines = logger.get_lines_for_level('error') self.assertEqual(3, len(error_lines)) for line in error_lines: self.assertIn('Kaboom!', line)
def test_retry(self): headers = HeaderKeyDict({'key': 'value'}) with mocked_http_conn(200, headers) as conn: attempts, resp = direct_client.retry( direct_client.direct_head_object, self.node, self.part, self.account, self.container, self.obj) self.assertEqual(conn.method, 'HEAD') self.assertEqual(conn.path, self.obj_path) self.assertEqual(conn.req_headers['user-agent'], self.user_agent) self.assertEqual(headers, resp) self.assertEqual(attempts, 1)
def direct(obj, part, nodes): found_count = 0 for node in nodes: error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node) try: attempts, _junk = direct_client.retry( direct_client.direct_head_object, node, part, account, container, obj, error_log=error_log, retries=retries, headers=headers) retries_done[0] += attempts - 1 found_count += 1 except ClientException as err: if err.http_status not in (404, 507): error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account, container, obj, err)) except (Exception, Timeout) as err: error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account, container, obj, err)) if output_missing_partitions and \ found_count < len(nodes): missing = len(nodes) - found_count print('\r\x1B[K', end='') stdout.flush() print('# Object partition %s missing %s cop%s' % (part, missing, 'y' if missing == 1 else 'ies'), file=stderr) object_copies_found[0] += found_count object_copies_missing[len(nodes) - found_count] += 1 objects_queried[0] += 1 if time() >= next_report[0]: next_report[0] = time() + 5 eta, eta_unit = compute_eta(begun, objects_queried[0], objects_listed) if not json_output: print('\r\x1B[KQuerying objects: %d of %d, %d%s left, %d ' 'retries' % (objects_queried[0], objects_listed, round(eta), eta_unit, retries_done[0]), end='') stdout.flush()
def test_retry_client_exception(self): err_log_file = six.StringIO() def mock_err_logger(err): err_log_file.write(err) with mocked_http_conn(500) as conn: try: attempts, resp = direct_client.retry( direct_client.direct_delete_object, self.node, self.part, self.account, self.container, self.obj, retries=2, error_log=mock_err_logger) except ClientException as err: pass self.assertEqual('DELETE', conn.method) self.assertTrue(err_log_file.len) self.assertEqual(err.http_status, 500) err_log_file.close()
def test_retry(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' headers = {'key': 'value'} was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, headers) attempts, resp = direct_client.retry(direct_client.direct_head_object, node, part, account, container, name) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp) self.assertEqual(attempts, 1) direct_client.http_connect = was_http_connector
def direct(obj, part, nodes): found_count = 0 for node in nodes: error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node) try: attempts, _junk = direct_client.retry( direct_client.direct_head_object, node, part, account, container, obj, error_log=error_log, retries=retries, headers=headers) retries_done[0] += attempts - 1 found_count += 1 except ClientException as err: if err.http_status not in (404, 507): error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account, container, obj, err)) except (Exception, Timeout) as err: error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account, container, obj, err)) if output_missing_partitions and \ found_count < len(nodes): missing = len(nodes) - found_count print('\r\x1B[K', end='') stdout.flush() print('# Object partition %s missing %s cop%s' % ( part, missing, 'y' if missing == 1 else 'ies'), file=stderr) object_copies_found[0] += found_count object_copies_missing[len(nodes) - found_count] += 1 objects_queried[0] += 1 if time() >= next_report[0]: next_report[0] = time() + 5 eta, eta_unit = compute_eta(begun, objects_queried[0], objects_listed) if not json_output: print('\r\x1B[KQuerying objects: %d of %d, %d%s left, %d ' 'retries' % (objects_queried[0], objects_listed, round(eta), eta_unit, retries_done[0]), end='') stdout.flush()