def test_upload_object_job_identical_etag(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() mock_conn = mock.Mock() mock_conn.head_object.return_value = { 'content-length': 30, 'etag': md5(b'a' * 30).hexdigest()} type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': True, 'leave_segments': True, 'header': '', 'segment_size': 0}) self.assertTrue(r['success']) self.assertIn('status', r) self.assertEqual(r['status'], 'skipped-identical') self.assertEqual(mock_conn.put_object.call_count, 0) self.assertEqual(mock_conn.head_object.call_count, 1) mock_conn.head_object.assert_called_with('test_c', 'test_o')
def test_upload_object_job_file(self): # Uploading a file results in the file object being wrapped in a # LengthWrapper. This test sets the options in such a way that much # of _upload_object_job is skipped bringing the critical path down # to around 60 lines to ease testing. with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, } expected_mtime = float(os.path.getmtime(f.name)) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True}) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self.assertEqual(r['path'], f.name) del r['path'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 30) # This read forces the LengthWrapper to calculate the md5 # for the read content. This also checks that LengthWrapper was # initialized with md5=True self.assertEqual(contents.read(), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_object_job_stream(self): # Streams are wrapped as ReadableToIterable with tempfile.TemporaryFile() as f: f.write(b'a' * 30) f.flush() f.seek(0) expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, 'path': None, } expected_mtime = float(time.time()) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f, obj='test_o', options={ 'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True }) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=None, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.ReadableToIterable) self.assertEqual(contents.chunk_size, 65536) # next retrieves the first chunk of the stream or len(chunk_size) # or less, it also forces the md5 to be calculated. self.assertEqual(next(contents), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_object_job_stream(self): # Streams are wrapped as ReadableToIterable with tempfile.TemporaryFile() as f: f.write(b'a' * 30) f.flush() f.seek(0) expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, 'path': None, } expected_mtime = float(time.time()) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f, obj='test_o', options={'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True}) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=None, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.ReadableToIterable) self.assertEqual(contents.chunk_size, 65536) # next retrieves the first chunk of the stream or len(chunk_size) # or less, it also forces the md5 to be calculated. self.assertEqual(next(contents), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_object_job_identical_dlo(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() segment_etag = md5(b'a' * 10).hexdigest() mock_conn = mock.Mock() mock_conn.head_object.return_value = { 'x-object-manifest': 'test_c_segments/test_o/prefix', 'content-length': 30, 'etag': md5(segment_etag.encode('ascii') * 3).hexdigest()} mock_conn.get_container.side_effect = [ (None, [{"bytes": 10, "hash": segment_etag, "name": "test_o/prefix/00"}, {"bytes": 10, "hash": segment_etag, "name": "test_o/prefix/01"}]), (None, [{"bytes": 10, "hash": segment_etag, "name": "test_o/prefix/02"}]), (None, {})] type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() with mock.patch('swiftclient.service.get_conn', return_value=mock_conn): r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': True, 'leave_segments': True, 'header': '', 'segment_size': 10}) self.assertIsNone(r.get('error')) self.assertTrue(r['success']) self.assertEqual('skipped-identical', r.get('status')) self.assertEqual(0, mock_conn.put_object.call_count) self.assertEqual(1, mock_conn.head_object.call_count) self.assertEqual(3, mock_conn.get_container.call_count) mock_conn.head_object.assert_called_with('test_c', 'test_o') expected = [ mock.call('test_c_segments', prefix='test_o/prefix', marker='', delimiter=None), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/01", delimiter=None), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/02", delimiter=None), ] mock_conn.get_container.assert_has_calls(expected)
def test_upload_object_job_identical_slo_with_nesting(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() seg_etag = md5(b'a' * 10).hexdigest() submanifest = "[%s]" % ",".join( ['{"bytes":10,"hash":"%s"}' % seg_etag] * 2) submanifest_etag = md5(seg_etag.encode('ascii') * 2).hexdigest() manifest = "[%s]" % ",".join([ '{"sub_slo":true,"name":"/test_c_segments/test_sub_slo",' '"bytes":20,"hash":"%s"}' % submanifest_etag, '{"bytes":10,"hash":"%s"}' % seg_etag]) mock_conn = mock.Mock() mock_conn.head_object.return_value = { 'x-static-large-object': True, 'content-length': 30, 'etag': md5(submanifest_etag.encode('ascii') + seg_etag.encode('ascii')).hexdigest()} mock_conn.get_object.side_effect = [ ({}, manifest.encode('ascii')), ({}, submanifest.encode('ascii'))] type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': True, 'leave_segments': True, 'header': '', 'segment_size': 10}) self.assertIsNone(r.get('error')) self.assertTrue(r['success']) self.assertEqual('skipped-identical', r.get('status')) self.assertEqual(0, mock_conn.put_object.call_count) self.assertEqual([mock.call('test_c', 'test_o')], mock_conn.head_object.mock_calls) self.assertEqual([ mock.call('test_c', 'test_o', query_string='multipart-manifest=get'), mock.call('test_c_segments', 'test_sub_slo', query_string='multipart-manifest=get'), ], mock_conn.get_object.mock_calls)
def test_upload_object_job_etag_mismatch(self): # The etag test for both streams and files use the same code # so only one test should be needed. def _consuming_conn(*a, **kw): contents = a[2] contents.read() # Force md5 calculation return 'badresponseetag' with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() mock_conn = mock.Mock() mock_conn.put_object.side_effect = _consuming_conn type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={ 'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True }) self.assertEqual(r['success'], False) self.assertIn('error', r) self.assertIn('md5 mismatch', str(r['error'])) self.assertEqual(mock_conn.put_object.call_count, 1) expected_headers = {'x-object-meta-mtime': mock.ANY} mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers=expected_headers, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_object_job_etag_mismatch(self): # The etag test for both streams and files use the same code # so only one test should be needed. def _consuming_conn(*a, **kw): contents = a[2] contents.read() # Force md5 calculation return 'badresponseetag' with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() mock_conn = mock.Mock() mock_conn.put_object.side_effect = _consuming_conn type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True}) self.assertEqual(r['success'], False) self.assertIn('error', r) self.assertIn('md5 mismatch', str(r['error'])) self.assertEqual(mock_conn.put_object.call_count, 1) expected_headers = {'x-object-meta-mtime': mock.ANY} mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers=expected_headers, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())
def test_upload_object_job_file(self): # Uploading a file results in the file object being wrapped in a # LengthWrapper. This test sets the options in such a way that much # of _upload_object_job is skipped bringing the critical path down # to around 60 lines to ease testing. with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) f.flush() expected_r = { 'action': 'upload_object', 'attempts': 2, 'container': 'test_c', 'headers': {}, 'large_object': False, 'object': 'test_o', 'response_dict': {}, 'status': 'uploaded', 'success': True, } expected_mtime = float(os.path.getmtime(f.name)) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) s = SwiftService() r = s._upload_object_job(conn=mock_conn, container='test_c', source=f.name, obj='test_o', options={ 'changed': False, 'skip_identical': False, 'leave_segments': True, 'header': '', 'segment_size': 0, 'checksum': True }) mtime = float(r['headers']['x-object-meta-mtime']) self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) del r['headers']['x-object-meta-mtime'] self.assertEqual(r['path'], f.name) del r['path'] self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, content_length=30, headers={}, response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 30) # This read forces the LengthWrapper to calculate the md5 # for the read content. This also checks that LengthWrapper was # initialized with md5=True self.assertEqual(contents.read(), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest())