def setUp(self): super(TestRadosDiskFile, self).setUp() self.mock_rados = mock.MagicMock(name='rados') self.Rados = self.mock_rados.Rados.return_value self.ioctx = self.Rados.open_ioctx.return_value self.rados_ceph_conf = 'xxx-ceph.conf' self.rados_name = 'xxx-rados-name' self.rados_pool = 'xxx-rados-pool' self.device = 'device' self.partition = '0' self.account = 'account' self.container = 'container' self.obj_name = 'myobject' self.rdf = RadosFileSystem(self.rados_ceph_conf, self.rados_name, self.rados_pool, rados=self.mock_rados) self.df = self.rdf.get_diskfile(self.device, self.partition, self.account, self.container, self.obj_name)
def setUp(self): super(TestRadosDiskFile, self).setUp() self.mock_rados = mock.MagicMock(name="rados") self.Rados = self.mock_rados.Rados.return_value self.ioctx = self.Rados.open_ioctx.return_value self.rados_ceph_conf = "xxx-ceph.conf" self.rados_name = "xxx-rados-name" self.rados_pool = "xxx-rados-pool" self.device = "device" self.partition = "0" self.account = "account" self.container = "container" self.obj_name = "myobject" self.rdf = RadosFileSystem(self.rados_ceph_conf, self.rados_name, self.rados_pool, rados=self.mock_rados) self.df = self.rdf.get_diskfile(self.device, self.partition, self.account, self.container, self.obj_name)
def setUp(self): super(TestRadosDiskFile, self).setUp() self.mock_rados = mock.MagicMock(name='rados') self.Rados = self.mock_rados.Rados.return_value self.ioctx = self.Rados.open_ioctx.return_value self.rados_ceph_conf = 'xxx-ceph.conf' self.rados_name = 'xxx-rados-name' self.rados_pool = 'xxx-rados-pool' self.device = 'device' self.partition = '0' self.account = 'account' self.container = 'container' self.obj_name = 'myobject' self.logger = mock.Mock() self.rdf = RadosFileSystem(self.rados_ceph_conf, self.rados_name, self.rados_pool, self.logger, rados=self.mock_rados) self.df = self.rdf.get_diskfile(self.device, self.partition, self.account, self.container, self.obj_name)
def setup(self, conf): ceph_conf = conf.get("rados_ceph_conf", None) rados_user = conf.get("rados_user", None) rados_pool = conf.get("rados_pool", None) self._filesystem = RadosFileSystem(ceph_conf, rados_user, rados_pool)
class ObjectController(server.ObjectController): """ Implements the WSGI application for the Swift Rados Object Server. """ def setup(self, conf): ceph_conf = conf.get("rados_ceph_conf", None) rados_user = conf.get("rados_user", None) rados_pool = conf.get("rados_pool", None) self._filesystem = RadosFileSystem(ceph_conf, rados_user, rados_pool) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. """ return self._filesystem.get_diskfile(device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy_idx): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed: %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s'), {'ip': ip, 'port': port, 'dev': contdevice}) # FIXME: For now don't handle async updates def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ pass
class TestRadosDiskFile(unittest.TestCase): def setUp(self): super(TestRadosDiskFile, self).setUp() self.mock_rados = mock.MagicMock(name="rados") self.Rados = self.mock_rados.Rados.return_value self.ioctx = self.Rados.open_ioctx.return_value self.rados_ceph_conf = "xxx-ceph.conf" self.rados_name = "xxx-rados-name" self.rados_pool = "xxx-rados-pool" self.device = "device" self.partition = "0" self.account = "account" self.container = "container" self.obj_name = "myobject" self.rdf = RadosFileSystem(self.rados_ceph_conf, self.rados_name, self.rados_pool, rados=self.mock_rados) self.df = self.rdf.get_diskfile(self.device, self.partition, self.account, self.container, self.obj_name) def tearDown(self): super(TestRadosDiskFile, self).tearDown() self.mock_rados.reset_mock() self.Rados.reset_mock() self.ioctx.reset_mock() del self.rdf del self.df def _obj_name(self): return "/" + "/".join((self.device, self.partition, self.account, self.container, self.obj_name)) def _assert_if_rados_not_opened(self): self.mock_rados.Rados.assert_called_once_with(conffile=self.rados_ceph_conf, rados_id=self.rados_name) self.Rados.connect.assert_called_once_with() self.Rados.open_ioctx.assert_called_once_with(self.rados_pool) def _assert_if_rados_not_closed(self): self.ioctx.close.assert_called_once_with() def _assert_if_rados_opened(self): assert ( (self.mock_rados.Rados.call_count == 0) and (self.Rados.connect.call_count == 0) and (self.Rados.open_ioctx.call_count == 0) ) def _assert_if_rados_closed(self): assert (self.ioctx.close.call_count == 0) and (self.Rados.shutdown.call_count == 0) def _assert_if_rados_opened_closed(self): assert ( (self.mock_rados.Rados.call_count > 0) and (self.Rados.connect.call_count > 0) and (self.Rados.open_ioctx.call_count > 0) ) assert (self.Rados.connect.call_count > 0) and (self.Rados.open_ioctx.call_count == self.ioctx.close.call_count) def test_df_open_1(self): meta = {"name": self._obj_name(), "Content-Length": 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) with self.df.open(): pass self._assert_if_rados_not_opened() self.ioctx.get_xattr.assert_called_once_with(self._obj_name(), METADATA_KEY) self._assert_if_rados_not_closed() def test_df_open_invalid_name(self): meta = {"name": "invalid", "Content-Length": 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.open() except DiskFileCollision: success = True except Exception: pass finally: assert success def test_df_open_invalid_content_length(self): meta = {"name": self._obj_name(), "Content-Length": 100} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.open() except DiskFileQuarantined: success = True except Exception: pass finally: assert success def test_df_notopen_check(self): success = False try: with self.df: pass except DiskFileNotOpen: success = True except Exception: pass finally: assert success self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_notopen_get_metadata(self): success = False try: self.df.get_metadata() except DiskFileNotOpen: success = True except Exception: pass finally: assert success self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_get_metadata(self): meta = {"name": self._obj_name(), "Content-Length": 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False ret_meta = None try: with self.df.open(): ret_meta = self.df.get_metadata() success = True except Exception: pass finally: assert success assert ret_meta == meta self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() def test_df_read_metadata(self): meta = {"name": self._obj_name(), "Content-Length": 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False ret_meta = None try: ret_meta = self.df.read_metadata() success = True except Exception: pass finally: assert success assert ret_meta == meta self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() def test_df_notopen_reader(self): success = False try: self.df.reader() except DiskFileNotOpen: success = True except Exception: pass finally: assert success self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_open_reader_1(self): meta = {"name": self._obj_name(), "Content-Length": 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: with self.df.open(): self.df.reader() except KeyError: success = True pass finally: assert success self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() def test_df_open_reader_2(self): meta = {"name": self._obj_name(), "Content-Length": 0, "ETag": ""} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: with self.df.open(): rdr = self.df.reader() rdr.close() success = True except Exception: pass finally: assert success self._assert_if_rados_opened_closed() def test_df_reader_iter_invalid_cont_len(self): etag = md5() fcont = "123456789" etag.update(fcont) meta = {"name": self._obj_name(), "Content-Length": len(fcont), "ETag": etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (len(fcont), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert chunk == fcont if num_chunks == 3: self.ioctx.read.return_value = None assert num_chunks == 3 success = True except Exception: pass finally: assert success self._assert_if_rados_opened_closed() # check read calls call_list = [ call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont))), ] self.ioctx.assert_has_calls(call_list) self.ioctx.remove_object.assert_called_once_with(self._obj_name()) def test_df_reader_iter_invalid_etag(self): etag = md5() fcont = "123456789" etag.update(fcont) meta = {"name": self._obj_name(), "Content-Length": (3 * len(fcont)), "ETag": etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = ((len(fcont) * 3), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert chunk == fcont if num_chunks == 3: self.ioctx.read.return_value = None assert num_chunks == 3 success = True except Exception: pass finally: assert success self._assert_if_rados_opened_closed() # check read calls call_list = [ call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont))), ] self.ioctx.assert_has_calls(call_list) self.ioctx.remove_object.assert_called_once_with(self._obj_name()) def test_df_reader_iter_all_ok(self): etag = md5() fcont = "123456789" etag.update(fcont) etag.update(fcont) etag.update(fcont) meta = {"name": self._obj_name(), "Content-Length": (3 * len(fcont)), "ETag": etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = ((3 * len(fcont)), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert chunk == fcont if num_chunks == 3: self.ioctx.read.return_value = None assert num_chunks == 3 success = True except Exception: pass finally: assert success self._assert_if_rados_opened_closed() # check read calls call_list = [ call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont))), ] self.ioctx.assert_has_calls(call_list) # if everything is perfect, the object will not be deleted assert self.ioctx.remove_object.call_count == 0 def test_df_reader_iter_range(self): etag = md5() fcont = "0123456789" etag.update(fcont) meta = {"name": self._obj_name(), "Content-Length": len(fcont), "ETag": etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (len(fcont), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 def ioctx_read(obj_name, length=8192, offset=0): assert obj_name == self._obj_name() return fcont[offset:] self.ioctx.read = ioctx_read for chunk in rdr.app_iter_range(1, 8): num_chunks += 1 assert chunk == "1234567" assert num_chunks == 1 success = True except Exception: pass finally: assert success self._assert_if_rados_opened_closed() assert self.ioctx.remove_object.call_count == 0 def test_df_writer_1(self): with self.df.create(): pass assert self.ioctx.trunc.call_count == 0 self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() with self.df.create(500): pass self.ioctx.trunc.assert_called_once_with(self._obj_name(), 500) def test_df_writer_write(self): fcont = "0123456789" writes = [] def ioctx_write(obj, data, offset): writes.append((data, offset)) return 2 self.ioctx.write = ioctx_write with self.df.create() as writer: assert writer.write(fcont) == len(fcont) check_list = [(fcont, 0), (fcont[2:], 2), (fcont[4:], 4), (fcont[6:], 6), (fcont[8:], 8)] assert writes == check_list self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() def test_df_writer_put(self): meta = {"Content-Length": 0, "ETag": ""} with self.df.create() as writer: writer.put(meta) old_metadata = pickle.dumps(meta) ca = self.ioctx.set_xattr.call_args check_1 = call(self._obj_name(), METADATA_KEY, old_metadata) assert ca == check_1 assert meta["name"] == self._obj_name() self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() def test_df_write_metadata(self): meta = {"Content-Length": 0, "ETag": ""} self.df.write_metadata(meta) old_metadata = pickle.dumps(meta) ca = self.ioctx.set_xattr.call_args check_1 = call(self._obj_name(), METADATA_KEY, old_metadata) assert ca == check_1 assert meta["name"] == self._obj_name() self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() def test_df_delete(self): meta = {"name": self._obj_name(), "Content-Length": 0, "X-Timestamp": 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.delete(1) success = True except Exception: pass finally: assert success self._assert_if_rados_not_opened() self._assert_if_rados_not_closed() self.ioctx.remove_object.assert_called_once_with(self._obj_name())
def setup(self, conf): ceph_conf = conf.get("rados_ceph_conf", None) rados_user = conf.get("rados_user", None) rados_pool = conf.get("rados_pool", None) self._filesystem = RadosFileSystem(ceph_conf, rados_user, rados_pool, self.logger)
class ObjectController(server.ObjectController): """ Implements the WSGI application for the Swift Rados Object Server. """ def setup(self, conf): ceph_conf = conf.get("rados_ceph_conf", None) rados_user = conf.get("rados_user", None) rados_pool = conf.get("rados_pool", None) self._filesystem = RadosFileSystem(ceph_conf, rados_user, rados_pool, self.logger) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. """ return self._filesystem.get_diskfile(device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice, policy, logger_thread_locals=None): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'object-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error( _('ERROR Container update failed: %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), { 'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice }) except (Exception, Timeout): self.logger.exception( _('ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s'), { 'ip': ip, 'port': port, 'dev': contdevice }) # FIXME: For now don't handle async updates def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ pass
class TestRadosDiskFile(unittest.TestCase): def setUp(self): super(TestRadosDiskFile, self).setUp() self.mock_rados = mock.MagicMock(name='rados') self.Rados = self.mock_rados.Rados.return_value self.ioctx = self.Rados.open_ioctx.return_value self.rados_ceph_conf = 'xxx-ceph.conf' self.rados_name = 'xxx-rados-name' self.rados_pool = 'xxx-rados-pool' self.device = 'device' self.partition = '0' self.account = 'account' self.container = 'container' self.obj_name = 'myobject' self.logger = mock.Mock() self.rdf = RadosFileSystem(self.rados_ceph_conf, self.rados_name, self.rados_pool, self.logger, rados=self.mock_rados) self.df = self.rdf.get_diskfile(self.device, self.partition, self.account, self.container, self.obj_name) def tearDown(self): super(TestRadosDiskFile, self).tearDown() self.mock_rados.reset_mock() self.Rados.reset_mock() self.ioctx.reset_mock() del self.rdf del self.df def _obj_name(self): return '/' + '/'.join((self.device, self.partition, self.account, self.container, self.obj_name)) def _assert_if_rados_not_opened(self): self.mock_rados.Rados.assert_called_once_with( conffile=self.rados_ceph_conf, rados_id=self.rados_name) self.Rados.connect.assert_called_once_with() self.Rados.open_ioctx.assert_called_once_with(self.rados_pool) def _assert_if_rados_opened(self): assert ((self.mock_rados.Rados.call_count == 0) and (self.Rados.connect.call_count == 0) and (self.Rados.open_ioctx.call_count == 0)) def _assert_if_rados_closed(self): assert ((self.ioctx.close.call_count == 0) and (self.Rados.shutdown.call_count == 0)) def _assert_if_rados_opened_closed(self): assert ((self.mock_rados.Rados.call_count > 0) and (self.Rados.connect.call_count > 0) and (self.Rados.open_ioctx.call_count > 0)) assert ((self.Rados.connect.call_count > 0) and (self.Rados.open_ioctx.call_count == self.Rados.connect.call_count)) def test_df_open_1(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) with self.df.open(): pass self._assert_if_rados_not_opened() self.ioctx.get_xattr.assert_called_once_with(self._obj_name(), METADATA_KEY) def test_df_open_invalid_name(self): meta = {'name': 'invalid', 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.open() except DiskFileCollision: success = True except Exception: pass finally: assert (success) def test_df_open_invalid_content_length(self): meta = {'name': self._obj_name(), 'Content-Length': 100} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.open() except DiskFileQuarantined: success = True except Exception: pass finally: assert (success) def test_df_notopen_check(self): success = False try: with self.df: pass except DiskFileNotOpen: success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_notopen_get_metadata(self): success = False try: self.df.get_metadata() except DiskFileNotOpen: success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_get_metadata(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False ret_meta = None try: with self.df.open(): ret_meta = self.df.get_metadata() success = True except Exception: pass finally: assert (success) assert (ret_meta == meta) self._assert_if_rados_not_opened() def test_df_read_metadata(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False ret_meta = None try: ret_meta = self.df.read_metadata() success = True except Exception: pass finally: assert (success) assert (ret_meta == meta) self._assert_if_rados_not_opened() def test_df_notopen_reader(self): success = False try: self.df.reader() except DiskFileNotOpen: success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_open_reader_1(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: with self.df.open(): self.df.reader() except KeyError: success = True pass finally: assert (success) self._assert_if_rados_not_opened() def test_df_open_reader_2(self): meta = {'name': self._obj_name(), 'Content-Length': 0, 'ETag': ''} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: with self.df.open(): rdr = self.df.reader() rdr.close() success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened_closed() def test_df_reader_iter_invalid_cont_len(self): etag = md5() fcont = '123456789' etag.update(fcont) meta = { 'name': self._obj_name(), 'Content-Length': len(fcont), 'ETag': etag.hexdigest() } self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (len(fcont), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert (chunk == fcont) if num_chunks == 3: self.ioctx.read.return_value = None assert (num_chunks == 3) success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened_closed() # check read calls call_list = [ call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont))) ] self.ioctx.assert_has_calls(call_list) self.ioctx.remove_object.assert_called_once_with(self._obj_name()) def test_df_reader_iter_invalid_etag(self): etag = md5() fcont = '123456789' etag.update(fcont) meta = { 'name': self._obj_name(), 'Content-Length': (3 * len(fcont)), 'ETag': etag.hexdigest() } self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = ((len(fcont) * 3), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert (chunk == fcont) if num_chunks == 3: self.ioctx.read.return_value = None assert (num_chunks == 3) success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened_closed() # check read calls call_list = [ call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont))) ] self.ioctx.assert_has_calls(call_list) self.ioctx.remove_object.assert_called_once_with(self._obj_name()) def test_df_reader_iter_all_ok(self): etag = md5() fcont = '123456789' etag.update(fcont) etag.update(fcont) etag.update(fcont) meta = { 'name': self._obj_name(), 'Content-Length': (3 * len(fcont)), 'ETag': etag.hexdigest() } self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = ((3 * len(fcont)), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert (chunk == fcont) if num_chunks == 3: self.ioctx.read.return_value = None assert (num_chunks == 3) success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened_closed() # check read calls call_list = [ call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont))) ] self.ioctx.assert_has_calls(call_list) # if everything is perfect, the object will not be deleted assert (self.ioctx.remove_object.call_count == 0) def test_df_reader_iter_range(self): etag = md5() fcont = '0123456789' etag.update(fcont) meta = { 'name': self._obj_name(), 'Content-Length': len(fcont), 'ETag': etag.hexdigest() } self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (len(fcont), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 def ioctx_read(obj_name, length=8192, offset=0): assert (obj_name == self._obj_name()) return fcont[offset:] self.ioctx.read = ioctx_read for chunk in rdr.app_iter_range(1, 8): num_chunks += 1 assert (chunk == '1234567') assert (num_chunks == 1) success = True except Exception: pass finally: assert (success) self._assert_if_rados_opened_closed() assert (self.ioctx.remove_object.call_count == 0) def test_df_writer_1(self): with self.df.create(): pass assert (self.ioctx.trunc.call_count == 0) self._assert_if_rados_not_opened() with self.df.create(500): pass self.ioctx.trunc.assert_called_once_with(self._obj_name(), 500) def test_df_writer_write(self): fcont = '0123456789' writes = [] def mock_write(self, obj, offset, data): writes.append((data, offset)) return 2 with patch( 'swift_ceph_backend.rados_diskfile.' 'RadosFileSystem._radosfs.write', mock_write): with self.df.create() as writer: assert (writer.write(fcont) == len(fcont)) check_list = [(fcont, 0), (fcont[2:], 2), (fcont[4:], 4), (fcont[6:], 6), (fcont[8:], 8)] assert (writes == check_list) self._assert_if_rados_not_opened() def test_df_writer_put(self): meta = {'Content-Length': 0, 'ETag': ''} with self.df.create() as writer: writer.put(meta) old_metadata = pickle.dumps(meta) ca = self.ioctx.set_xattr.call_args check_1 = call(self._obj_name(), METADATA_KEY, old_metadata) assert (ca == check_1) assert (meta['name'] == self._obj_name()) self._assert_if_rados_not_opened() def test_df_write_metadata(self): meta = {'Content-Length': 0, 'ETag': ''} self.df.write_metadata(meta) old_metadata = pickle.dumps(meta) ca = self.ioctx.set_xattr.call_args check_1 = call(self._obj_name(), METADATA_KEY, old_metadata) assert (ca == check_1) assert (meta['name'] == self._obj_name()) self._assert_if_rados_not_opened() def test_df_delete(self): meta = { 'name': self._obj_name(), 'Content-Length': 0, 'X-Timestamp': 0 } self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.delete(1) success = True except Exception: pass finally: assert (success) self._assert_if_rados_not_opened() self.ioctx.remove_object.assert_called_once_with(self._obj_name())
class TestRadosDiskFile(unittest.TestCase): def setUp(self): super(TestRadosDiskFile, self).setUp() self.mock_rados = mock.MagicMock(name='rados') self.Rados = self.mock_rados.Rados.return_value self.ioctx = self.Rados.open_ioctx.return_value self.rados_ceph_conf = 'xxx-ceph.conf' self.rados_name = 'xxx-rados-name' self.rados_pool = 'xxx-rados-pool' self.device = 'device' self.partition = '0' self.account = 'account' self.container = 'container' self.obj_name = 'myobject' self.logger = mock.Mock() self.rdf = RadosFileSystem(self.rados_ceph_conf, self.rados_name, self.rados_pool, self.logger, rados=self.mock_rados) self.df = self.rdf.get_diskfile(self.device, self.partition, self.account, self.container, self.obj_name) def tearDown(self): super(TestRadosDiskFile, self).tearDown() self.mock_rados.reset_mock() self.Rados.reset_mock() self.ioctx.reset_mock() del self.rdf del self.df def _obj_name(self): return '/' + '/'.join((self.device, self.partition, self.account, self.container, self.obj_name)) def _assert_if_rados_not_opened(self): self.mock_rados.Rados.assert_called_once_with( conffile=self.rados_ceph_conf, rados_id=self.rados_name) self.Rados.connect.assert_called_once_with() self.Rados.open_ioctx.assert_called_once_with(self.rados_pool) def _assert_if_rados_opened(self): assert((self.mock_rados.Rados.call_count == 0) and (self.Rados.connect.call_count == 0) and (self.Rados.open_ioctx.call_count == 0)) def _assert_if_rados_closed(self): assert((self.ioctx.close.call_count == 0) and (self.Rados.shutdown.call_count == 0)) def _assert_if_rados_opened_closed(self): assert((self.mock_rados.Rados.call_count > 0) and (self.Rados.connect.call_count > 0) and (self.Rados.open_ioctx.call_count > 0)) assert((self.Rados.connect.call_count > 0) and (self.Rados.open_ioctx.call_count == self.Rados.connect.call_count)) def test_df_open_1(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) with self.df.open(): pass self._assert_if_rados_not_opened() self.ioctx.get_xattr.assert_called_once_with(self._obj_name(), METADATA_KEY) def test_df_open_invalid_name(self): meta = {'name': 'invalid', 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.open() except DiskFileCollision: success = True except Exception: pass finally: assert(success) def test_df_open_invalid_content_length(self): meta = {'name': self._obj_name(), 'Content-Length': 100} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.open() except DiskFileQuarantined: success = True except Exception: pass finally: assert(success) def test_df_notopen_check(self): success = False try: with self.df: pass except DiskFileNotOpen: success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_notopen_get_metadata(self): success = False try: self.df.get_metadata() except DiskFileNotOpen: success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_get_metadata(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False ret_meta = None try: with self.df.open(): ret_meta = self.df.get_metadata() success = True except Exception: pass finally: assert(success) assert(ret_meta == meta) self._assert_if_rados_not_opened() def test_df_read_metadata(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False ret_meta = None try: ret_meta = self.df.read_metadata() success = True except Exception: pass finally: assert(success) assert(ret_meta == meta) self._assert_if_rados_not_opened() def test_df_notopen_reader(self): success = False try: self.df.reader() except DiskFileNotOpen: success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened() self._assert_if_rados_closed() def test_df_open_reader_1(self): meta = {'name': self._obj_name(), 'Content-Length': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: with self.df.open(): self.df.reader() except KeyError: success = True pass finally: assert(success) self._assert_if_rados_not_opened() def test_df_open_reader_2(self): meta = {'name': self._obj_name(), 'Content-Length': 0, 'ETag': ''} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: with self.df.open(): rdr = self.df.reader() rdr.close() success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened_closed() def test_df_reader_iter_invalid_cont_len(self): etag = md5() fcont = '123456789' etag.update(fcont) meta = {'name': self._obj_name(), 'Content-Length': len(fcont), 'ETag': etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (len(fcont), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert(chunk == fcont) if num_chunks == 3: self.ioctx.read.return_value = None assert(num_chunks == 3) success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened_closed() # check read calls call_list = [call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont)))] self.ioctx.assert_has_calls(call_list) self.ioctx.remove_object.assert_called_once_with(self._obj_name()) def test_df_reader_iter_invalid_etag(self): etag = md5() fcont = '123456789' etag.update(fcont) meta = {'name': self._obj_name(), 'Content-Length': (3 * len(fcont)), 'ETag': etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = ((len(fcont) * 3), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert(chunk == fcont) if num_chunks == 3: self.ioctx.read.return_value = None assert(num_chunks == 3) success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened_closed() # check read calls call_list = [call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont)))] self.ioctx.assert_has_calls(call_list) self.ioctx.remove_object.assert_called_once_with(self._obj_name()) def test_df_reader_iter_all_ok(self): etag = md5() fcont = '123456789' etag.update(fcont) etag.update(fcont) etag.update(fcont) meta = {'name': self._obj_name(), 'Content-Length': (3 * len(fcont)), 'ETag': etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = ((3 * len(fcont)), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 self.ioctx.read.return_value = fcont for chunk in rdr: num_chunks += 1 assert(chunk == fcont) if num_chunks == 3: self.ioctx.read.return_value = None assert(num_chunks == 3) success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened_closed() # check read calls call_list = [call.read(self._obj_name(), offset=0), call.read(self._obj_name(), offset=len(fcont)), call.read(self._obj_name(), offset=(2 * len(fcont))), call.read(self._obj_name(), offset=(3 * len(fcont)))] self.ioctx.assert_has_calls(call_list) # if everything is perfect, the object will not be deleted assert(self.ioctx.remove_object.call_count == 0) def test_df_reader_iter_range(self): etag = md5() fcont = '0123456789' etag.update(fcont) meta = {'name': self._obj_name(), 'Content-Length': len(fcont), 'ETag': etag.hexdigest()} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (len(fcont), 0) success = False try: with self.df.open(): rdr = self.df.reader() num_chunks = 0 def ioctx_read(obj_name, length=8192, offset=0): assert(obj_name == self._obj_name()) return fcont[offset:] self.ioctx.read = ioctx_read for chunk in rdr.app_iter_range(1, 8): num_chunks += 1 assert(chunk == '1234567') assert(num_chunks == 1) success = True except Exception: pass finally: assert(success) self._assert_if_rados_opened_closed() assert(self.ioctx.remove_object.call_count == 0) def test_df_writer_1(self): with self.df.create(): pass assert(self.ioctx.trunc.call_count == 0) self._assert_if_rados_not_opened() with self.df.create(500): pass self.ioctx.trunc.assert_called_once_with(self._obj_name(), 500) def test_df_writer_write(self): fcont = '0123456789' writes = [] def mock_write(self, obj, offset, data): writes.append((data, offset)) return 2 with patch('swift_ceph_backend.rados_diskfile.' 'RadosFileSystem._radosfs.write', mock_write): with self.df.create() as writer: assert(writer.write(fcont) == len(fcont)) check_list = [ (fcont, 0), (fcont[2:], 2), (fcont[4:], 4), (fcont[6:], 6), (fcont[8:], 8)] assert(writes == check_list) self._assert_if_rados_not_opened() def test_df_writer_put(self): meta = {'Content-Length': 0, 'ETag': ''} with self.df.create() as writer: writer.put(meta) old_metadata = pickle.dumps(meta) ca = self.ioctx.set_xattr.call_args check_1 = call(self._obj_name(), METADATA_KEY, old_metadata) assert(ca == check_1) assert(meta['name'] == self._obj_name()) self._assert_if_rados_not_opened() def test_df_write_metadata(self): meta = {'Content-Length': 0, 'ETag': ''} self.df.write_metadata(meta) old_metadata = pickle.dumps(meta) ca = self.ioctx.set_xattr.call_args check_1 = call(self._obj_name(), METADATA_KEY, old_metadata) assert(ca == check_1) assert(meta['name'] == self._obj_name()) self._assert_if_rados_not_opened() def test_df_delete(self): meta = {'name': self._obj_name(), 'Content-Length': 0, 'X-Timestamp': 0} self.ioctx.get_xattr.return_value = pickle.dumps(meta) self.ioctx.stat.return_value = (0, 0) success = False try: self.df.delete(1) success = True except Exception: pass finally: assert(success) self._assert_if_rados_not_opened() self.ioctx.remove_object.assert_called_once_with(self._obj_name())