def test_retry_client_exception(self):
logger = debug_logger('direct-client-test')
with mock.patch('swift.common.direct_client.sleep') as mock_sleep, \
mocked_http_conn(500) as conn:
with self.assertRaises(direct_client.ClientException) as err_ctx:
direct_client.retry(direct_client.direct_delete_object,
self.node, self.part,
self.account, self.container, self.obj,
retries=2, error_log=logger.error)
self.assertEqual('DELETE', conn.method)
self.assertEqual(err_ctx.exception.http_status, 500)
self.assertIn('DELETE', err_ctx.exception.message)
self.assertIn(quote('/%s/%s/%s/%s/%s'
% (self.node['device'], self.part, self.account,
self.container, self.obj)),
err_ctx.exception.message)
self.assertIn(self.node['ip'], err_ctx.exception.message)
self.assertIn(self.node['port'], err_ctx.exception.message)
self.assertEqual(self.node['ip'], err_ctx.exception.http_host)
self.assertEqual(self.node['port'], err_ctx.exception.http_port)
self.assertEqual(self.node['device'], err_ctx.exception.http_device)
self.assertEqual(500, err_ctx.exception.http_status)
self.assertEqual([mock.call(1), mock.call(2)],
mock_sleep.call_args_list)
error_lines = logger.get_lines_for_level('error')
self.assertEqual(3, len(error_lines))
for line in error_lines:
self.assertIn('500 Internal Error', line)
def test_retry_http_exception(self):
err_log_file = six.StringIO()
def mock_err_logger(err):
err_log_file.write(err)
def mock_direct_delete_object(node,
part,
account,
container,
obj,
conn_timeout=5,
response_timeout=15,
headers=None):
resp = "Unable to delete object"
raise HTTPException('Object', 'DELETE', resp)
with mocked_http_conn(500):
with mock.patch('swift.common.direct_client.direct_delete_object',
mock_direct_delete_object):
try:
attempts, resp = direct_client.retry(
direct_client.direct_delete_object,
self.node,
self.part,
self.account,
self.container,
self.obj,
retries=2,
error_log=mock_err_logger)
except HTTPException:
self.assertTrue(err_log_file.len)
err_log_file.close()
def test_retry_http_exception(self):
logger = debug_logger('direct-client-test')
with mock.patch('swift.common.direct_client.sleep') as mock_sleep, \
mocked_http_conn(HTTPException('Kaboom!')) as conn:
with self.assertRaises(HTTPException) as err_ctx:
direct_client.retry(direct_client.direct_delete_object,
self.node, self.part,
self.account, self.container, self.obj,
retries=2, error_log=logger.error)
self.assertEqual('DELETE', conn.method)
self.assertEqual('Kaboom!', str(err_ctx.exception))
self.assertEqual([mock.call(1), mock.call(2)],
mock_sleep.call_args_list)
error_lines = logger.get_lines_for_level('error')
self.assertEqual(3, len(error_lines))
for line in error_lines:
self.assertIn('Kaboom!', line)
def test_retry(self):
headers = HeaderKeyDict({'key': 'value'})
with mocked_http_conn(200, headers) as conn:
attempts, resp = direct_client.retry(
direct_client.direct_head_object, self.node, self.part,
self.account, self.container, self.obj)
self.assertEqual(conn.method, 'HEAD')
self.assertEqual(conn.path, self.obj_path)
self.assertEqual(conn.req_headers['user-agent'], self.user_agent)
self.assertEqual(headers, resp)
self.assertEqual(attempts, 1)
def direct(obj, part, nodes):
found_count = 0
for node in nodes:
error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
try:
attempts, _junk = direct_client.retry(
direct_client.direct_head_object,
node,
part,
account,
container,
obj,
error_log=error_log,
retries=retries,
headers=headers)
retries_done[0] += attempts - 1
found_count += 1
except ClientException as err:
if err.http_status not in (404, 507):
error_log('Giving up on /%s/%s/%s/%s: %s' %
(part, account, container, obj, err))
except (Exception, Timeout) as err:
error_log('Giving up on /%s/%s/%s/%s: %s' %
(part, account, container, obj, err))
if output_missing_partitions and \
found_count < len(nodes):
missing = len(nodes) - found_count
print('\r\x1B[K', end='')
stdout.flush()
print('# Object partition %s missing %s cop%s' %
(part, missing, 'y' if missing == 1 else 'ies'),
file=stderr)
object_copies_found[0] += found_count
object_copies_missing[len(nodes) - found_count] += 1
objects_queried[0] += 1
if time() >= next_report[0]:
next_report[0] = time() + 5
eta, eta_unit = compute_eta(begun, objects_queried[0],
objects_listed)
if not json_output:
print('\r\x1B[KQuerying objects: %d of %d, %d%s left, %d '
'retries' % (objects_queried[0], objects_listed,
round(eta), eta_unit, retries_done[0]),
end='')
stdout.flush()
def test_retry_client_exception(self):
err_log_file = six.StringIO()
def mock_err_logger(err):
err_log_file.write(err)
with mocked_http_conn(500) as conn:
try:
attempts, resp = direct_client.retry(
direct_client.direct_delete_object, self.node, self.part,
self.account, self.container, self.obj, retries=2,
error_log=mock_err_logger)
except ClientException as err:
pass
self.assertEqual('DELETE', conn.method)
self.assertTrue(err_log_file.len)
self.assertEqual(err.http_status, 500)
err_log_file.close()
def test_retry(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
headers = {'key': 'value'}
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers)
attempts, resp = direct_client.retry(direct_client.direct_head_object,
node, part, account, container, name)
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(headers, resp)
self.assertEqual(attempts, 1)
direct_client.http_connect = was_http_connector
def test_retry_client_exception(self):
err_log_file = six.StringIO()
def mock_err_logger(err):
err_log_file.write(err)
with mocked_http_conn(500) as conn:
try:
attempts, resp = direct_client.retry(
direct_client.direct_delete_object, self.node, self.part,
self.account, self.container, self.obj, retries=2,
error_log=mock_err_logger)
except ClientException as err:
pass
self.assertEqual('DELETE', conn.method)
self.assertTrue(err_log_file.len)
self.assertEqual(err.http_status, 500)
err_log_file.close()
def test_retry(self):
node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'}
part = '0'
account = 'a'
container = 'c'
name = 'o'
headers = {'key': 'value'}
was_http_connector = direct_client.http_connect
direct_client.http_connect = mock_http_connect(200, headers)
attempts, resp = direct_client.retry(direct_client.direct_head_object,
node, part, account, container, name)
headers.update({'user-agent': 'direct-client %s' % os.getpid()})
self.assertEqual(headers, resp)
self.assertEqual(attempts, 1)
direct_client.http_connect = was_http_connector
def direct(obj, part, nodes):
found_count = 0
for node in nodes:
error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
try:
attempts, _junk = direct_client.retry(
direct_client.direct_head_object, node, part, account,
container, obj, error_log=error_log, retries=retries,
headers=headers)
retries_done[0] += attempts - 1
found_count += 1
except ClientException as err:
if err.http_status not in (404, 507):
error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account,
container, obj, err))
except (Exception, Timeout) as err:
error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account,
container, obj, err))
if output_missing_partitions and \
found_count < len(nodes):
missing = len(nodes) - found_count
print('\r\x1B[K', end='')
stdout.flush()
print('# Object partition %s missing %s cop%s' % (
part, missing, 'y' if missing == 1 else 'ies'), file=stderr)
object_copies_found[0] += found_count
object_copies_missing[len(nodes) - found_count] += 1
objects_queried[0] += 1
if time() >= next_report[0]:
next_report[0] = time() + 5
eta, eta_unit = compute_eta(begun, objects_queried[0],
objects_listed)
if not json_output:
print('\r\x1B[KQuerying objects: %d of %d, %d%s left, %d '
'retries' % (objects_queried[0], objects_listed,
round(eta), eta_unit, retries_done[0]),
end='')
stdout.flush()
def test_retry_http_exception(self):
err_log_file = six.StringIO()
def mock_err_logger(err):
err_log_file.write(err)
def mock_direct_delete_object(node, part, account, container, obj,
conn_timeout=5, response_timeout=15,
headers=None):
resp = "Unable to delete object"
raise HTTPException('Object', 'DELETE', resp)
with mocked_http_conn(500):
with mock.patch('swift.common.direct_client.direct_delete_object',
mock_direct_delete_object):
try:
attempts, resp = direct_client.retry(
direct_client.direct_delete_object, self.node,
self.part, self.account, self.container, self.obj,
retries=2, error_log=mock_err_logger)
except HTTPException:
self.assertTrue(err_log_file.len)
err_log_file.close()