def test_reader_disk_chunk_size(self):
conf = dict(disk_chunk_size=64)
conf.update(self.conf)
self.mgr = DiskFileManager(conf, self.lg)
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader()
try:
assert reader._disk_chunk_size == 64
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 4, repr(chunks)
for chunk in chunks:
assert len(chunk) == 64, repr(chunks)
def setup(self, conf):
"""
Implementation specific setup. This method is called at the very end
by the constructor to allow a specific implementation to modify
existing attributes or add its own attributes.
:param conf: WSGI configuration parameter
"""
# Common on-disk hierarchy shared across account, container and object
# servers.
self._diskfile_mgr = DiskFileManager(conf, self.logger)
def setUp(self):
self._orig_tpool_exc = tpool.execute
tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.obj.diskfile.write_metadata
self._saved_df_rm = gluster.swift.obj.diskfile.read_metadata
gluster.swift.obj.diskfile.write_metadata = _mock_write_metadata
gluster.swift.obj.diskfile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
self._saved_do_fsync = gluster.swift.obj.diskfile.do_fsync
gluster.swift.obj.diskfile.do_fsync = _mock_do_fsync
self.td = tempfile.mkdtemp()
self.conf = dict(devices=self.td,
mb_per_sync=2,
keep_cache_size=(1024 * 1024),
mount_check=False)
self.mgr = DiskFileManager(self.conf, self.lg)
def test_reader_disk_chunk_size(self):
conf = dict(disk_chunk_size=64)
conf.update(self.conf)
self.mgr = DiskFileManager(conf, self.lg)
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader()
try:
assert reader._disk_chunk_size == 64
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 4, repr(chunks)
for chunk in chunks:
assert len(chunk) == 64, repr(chunks)
class ObjectController(server.ObjectController):
"""
Subclass of the object server's ObjectController which replaces the
container_update method with one that is a no-op (information is simply
stored on disk and already updated by virtue of performing the file system
operations directly).
"""
def setup(self, conf):
"""
Implementation specific setup. This method is called at the very end
by the constructor to allow a specific implementation to modify
existing attributes or add its own attributes.
:param conf: WSGI configuration parameter
"""
# Common on-disk hierarchy shared across account, container and object
# servers.
self._diskfile_mgr = DiskFileManager(conf, self.logger)
def get_diskfile(self, device, partition, account, container, obj,
policy_idx, **kwargs):
"""
Utility method for instantiating a DiskFile object supporting a given
REST API.
An implementation of the object server that wants to use a different
DiskFile class would simply over-ride this method to provide that
behavior.
"""
return self._diskfile_mgr.get_diskfile(
device, partition, account, container, obj, policy_idx, **kwargs)
@public
@timing_stats()
def PUT(self, request):
try:
server.check_object_creation = \
gluster.swift.common.constraints.sof_check_object_creation
return server.ObjectController.PUT(self, request)
except (AlreadyExistsAsFile, AlreadyExistsAsDir):
device = \
split_and_validate_path(request, 1, 5, True)
return HTTPConflict(drive=device, request=request)
def setUp(self):
self._orig_tpool_exc = tpool.execute
tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.obj.diskfile.write_metadata
self._saved_df_rm = gluster.swift.obj.diskfile.read_metadata
gluster.swift.obj.diskfile.write_metadata = _mock_write_metadata
gluster.swift.obj.diskfile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
self._saved_do_fsync = gluster.swift.obj.diskfile.do_fsync
gluster.swift.obj.diskfile.do_fsync = _mock_do_fsync
self.td = tempfile.mkdtemp()
self.conf = dict(devices=self.td, mb_per_sync=2,
keep_cache_size=(1024 * 1024), mount_check=False)
self.mgr = DiskFileManager(self.conf, self.lg)
class TestDiskFile(unittest.TestCase):
""" Tests for gluster.swift.obj.diskfile """
def setUp(self):
self._orig_tpool_exc = tpool.execute
tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.obj.diskfile.write_metadata
self._saved_df_rm = gluster.swift.obj.diskfile.read_metadata
gluster.swift.obj.diskfile.write_metadata = _mock_write_metadata
gluster.swift.obj.diskfile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
self._saved_do_fsync = gluster.swift.obj.diskfile.do_fsync
gluster.swift.obj.diskfile.do_fsync = _mock_do_fsync
self.td = tempfile.mkdtemp()
self.conf = dict(devices=self.td, mb_per_sync=2,
keep_cache_size=(1024 * 1024), mount_check=False)
self.mgr = DiskFileManager(self.conf, self.lg)
def tearDown(self):
tpool.execute = self._orig_tpool_exc
self.lg = None
_destroyxattr()
gluster.swift.obj.diskfile.write_metadata = self._saved_df_wm
gluster.swift.obj.diskfile.read_metadata = self._saved_df_rm
gluster.swift.common.utils.write_metadata = self._saved_ut_wm
gluster.swift.common.utils.read_metadata = self._saved_ut_rm
gluster.swift.obj.diskfile.do_fsync = self._saved_do_fsync
shutil.rmtree(self.td)
def _get_diskfile(self, d, p, a, c, o, **kwargs):
return self.mgr.get_diskfile(d, p, a, c, o, **kwargs)
def test_constructor_no_slash(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._mgr is self.mgr
assert gdf._device_path == os.path.join(self.td, "vol0")
assert isinstance(gdf._threadpool, ThreadPool)
assert gdf._uid == DEFAULT_UID
assert gdf._gid == DEFAULT_GID
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar"), gdf._put_datadir
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
assert gdf._fd is None
def test_constructor_leadtrail_slash(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "/b/a/z/")
assert gdf._obj == "z"
assert gdf._obj_path == "b/a"
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "b", "a"), gdf._put_datadir
def test_open_no_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'Content-Type': 'application/octet-stream'}
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
with gdf.open():
assert gdf._data_file == the_file
assert gdf._fd is not None
assert gdf._metadata == exp_md
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
self.assertRaises(DiskFileNotOpen, gdf.get_metadata)
self.assertRaises(DiskFileNotOpen, gdf.reader)
self.assertRaises(DiskFileNotOpen, gdf.__enter__)
def test_read_metadata_optimize_open_close(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
init_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': md5("1234").hexdigest(),
'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime),
'Content-Type': 'application/octet-stream'}
_metadata[_mapit(the_file)] = init_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
# Case 1
# Ensure that reading metadata for non-GET requests
# does not incur opening and closing the file when
# metadata is NOT stale.
mock_open = Mock()
mock_close = Mock()
with mock.patch("gluster.swift.obj.diskfile.do_open", mock_open):
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
md = gdf.read_metadata()
self.assertEqual(md, init_md)
self.assertFalse(mock_open.called)
self.assertFalse(mock_close.called)
# Case 2
# Ensure that reading metadata for non-GET requests
# still opens and reads the file when metadata is stale
with open(the_file, "a") as fd:
# Append to the existing file to make the stored metadata
# invalid/stale.
fd.write("5678")
md = gdf.read_metadata()
# Check that the stale metadata is recalculated to account for
# change in file content
self.assertNotEqual(md, init_md)
self.assertEqual(md['Content-Length'], 8)
self.assertEqual(md['ETag'], md5("12345678").hexdigest())
def test_open_and_close(self):
mock_close = Mock()
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47",
"bar", "z")
with gdf.open():
assert gdf._fd is not None
self.assertTrue(mock_close.called)
def test_open_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_file)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._metadata is None
assert gdf._disk_file_open is False
with gdf.open():
assert gdf._data_file == the_file
assert gdf._fd is not None
assert gdf._metadata == exp_md, "%r != %r" % (gdf._metadata, exp_md)
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def test_open_invalid_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
inv_md = {
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_file)] = inv_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
with gdf.open():
assert gdf._data_file == the_file
assert gdf._metadata != inv_md
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def test_open_isdir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'dir',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_dir)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._obj == "d"
assert gdf._disk_file_open is False
with gdf.open():
assert gdf._data_file == the_dir
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256):
# FIXME: assumes account === volume
the_path = os.path.join(self.td, dev, con)
the_file = os.path.join(the_path, obj)
base_obj = os.path.basename(the_file)
base_dir = os.path.dirname(the_file)
os.makedirs(base_dir)
with open(the_file, "wb") as fd:
fd.write("y" * fsize)
gdf = self._get_diskfile(dev, par, acc, con, obj)
assert gdf._obj == base_obj
assert gdf._fd is None
return gdf
def test_reader(self):
closed = [False]
fd = [-1]
def mock_close(*args, **kwargs):
closed[0] = True
os.close(fd[0])
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
assert gdf._fd is not None
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
reader = gdf.reader()
assert reader._fd is not None
fd[0] = reader._fd
chunks = [ck for ck in reader]
assert reader._fd is None
assert closed[0]
assert len(chunks) == 1, repr(chunks)
def test_reader_disk_chunk_size(self):
conf = dict(disk_chunk_size=64)
conf.update(self.conf)
self.mgr = DiskFileManager(conf, self.lg)
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader()
try:
assert reader._disk_chunk_size == 64
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 4, repr(chunks)
for chunk in chunks:
assert len(chunk) == 64, repr(chunks)
def test_reader_iter_hook(self):
called = [0]
def mock_sleep(*args, **kwargs):
called[0] += 1
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader(iter_hook=mock_sleep)
try:
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 1, repr(chunks)
assert called[0] == 1, called
def test_reader_larger_file(self):
closed = [False]
fd = [-1]
def mock_close(*args, **kwargs):
closed[0] = True
os.close(fd[0])
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2)
with gdf.open():
assert gdf._fd is not None
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
reader = gdf.reader()
assert reader._fd is not None
fd[0] = reader._fd
chunks = [ck for ck in reader]
assert reader._fd is None
assert closed[0]
def test_reader_dir_object(self):
called = [False]
def our_do_close(fd):
called[0] = True
os.close(fd)
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(os.path.join(the_cont, "dir"))
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
reader = gdf.reader()
try:
chunks = [ck for ck in reader]
assert len(chunks) == 0, repr(chunks)
with mock.patch("gluster.swift.obj.diskfile.do_close",
our_do_close):
reader.close()
assert not called[0]
finally:
reader.close()
def test_create_dir_object_no_md(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = "dir"
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"))
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
gdf._create_dir_object(the_dir)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert _mapit(full_dir_path) not in _metadata
def test_create_dir_object_with_md(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = "dir"
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"))
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
dir_md = {'Content-Type': 'application/directory',
X_OBJECT_TYPE: DIR_OBJECT}
gdf._create_dir_object(the_dir, dir_md)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert _mapit(full_dir_path) in _metadata
def test_create_dir_object_exists(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.obj.diskfile.do_chown
gluster.swift.obj.diskfile.do_chown = _mock_do_chown
self.assertRaises(
AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(_mapit(the_dir) in _metadata)
def test_create_dir_object_do_stat_failure(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.obj.diskfile.do_chown
gluster.swift.obj.diskfile.do_chown = _mock_do_chown
self.assertRaises(
AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(_mapit(the_dir) in _metadata)
def test_write_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "z")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy())
self.assertEqual(None, gdf._metadata)
fmd = _metadata[_mapit(the_dir)]
md.update({'X-Object-Type': 'file', 'X-Type': 'Object'})
self.assertTrue(fmd['a'], md['a'])
self.assertTrue(fmd['Content-Type'], md['Content-Type'])
def test_add_metadata_to_existing_file(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_file)] = ini_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy())
self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
newmd = {'X-Object-Meta-test':'1234'}
gdf.write_metadata(newmd.copy())
on_disk_md = _metadata[_mapit(the_file)]
self.assertTrue(on_disk_md['Content-Length'], 4)
self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
self.assertTrue(on_disk_md['X-Type'], 'Object')
self.assertTrue(on_disk_md['X-Object-Type'], 'file')
self.assertTrue(on_disk_md['ETag'], 'etag')
self.assertFalse('a' in on_disk_md)
def test_add_md_to_existing_file_with_md_in_gdf(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'name': 'z',
'ETag': 'etag',
'X-Timestamp': 'ts'}
_metadata[_mapit(the_file)] = ini_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
# make sure gdf has the _metadata
gdf.open()
md = {'a': 'b'}
gdf.write_metadata(md.copy())
self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
newmd = {'X-Object-Meta-test':'1234'}
gdf.write_metadata(newmd.copy())
on_disk_md = _metadata[_mapit(the_file)]
self.assertTrue(on_disk_md['Content-Length'], 4)
self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
self.assertFalse('a' in on_disk_md)
def test_add_metadata_to_existing_dir(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
self.assertEquals(gdf._metadata, None)
init_md = {
'X-Type': 'Object',
'Content-Length': 0,
'ETag': 'etag',
'X-Timestamp': 'ts',
'X-Object-Meta-test':'test',
'Content-Type': 'application/directory'}
_metadata[_mapit(the_dir)] = init_md
md = {'X-Object-Meta-test':'test'}
gdf.write_metadata(md.copy())
self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Meta-test'],
'test')
self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
'application/directory')
# set new metadata
newmd = {'X-Object-Meta-test2':'1234'}
gdf.write_metadata(newmd.copy())
self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
'application/directory')
self.assertEqual(_metadata[_mapit(the_dir)]["X-Object-Meta-test2"],
'1234')
self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Type'],
DIR_OBJECT)
self.assertFalse('X-Object-Meta-test' in _metadata[_mapit(the_dir)])
def test_write_metadata_w_meta_file(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_file)] == newmd
def test_write_metadata_w_meta_file_no_content_type(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = deepcopy(gdf.read_metadata())
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_file)] == newmd
def test_write_metadata_w_meta_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_dir)] == newmd
def test_write_metadata_w_marker_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_dir)] == newmd
def test_put_w_marker_dir_create(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
assert gdf._metadata is None
newmd = {
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/directory'}
with gdf.create() as dw:
dw.put(newmd)
assert gdf._data_file == the_dir
for key, val in newmd.items():
assert _metadata[_mapit(the_dir)][key] == val
assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT
def test_put_is_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
origmd = gdf.get_metadata()
origfmd = _metadata[_mapit(the_dir)]
newmd = deepcopy(origmd)
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
with gdf.create() as dw:
try:
# FIXME: We should probably be able to detect in .create()
# when the target file name already exists as a directory to
# avoid reading the data off the wire only to fail as a
# directory.
dw.write('12345\n')
dw.put(newmd)
except AlreadyExistsAsDir:
pass
else:
self.fail("Expected to encounter"
" 'already-exists-as-dir' exception")
with gdf.open():
assert gdf.get_metadata() == origmd
assert _metadata[_mapit(the_dir)] == origfmd, "was: %r, is: %r" % (
origfmd, _metadata[_mapit(the_dir)])
def test_put(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_put_ENOSPC(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_open(*args, **kwargs):
raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))
with mock.patch("os.open", mock_open):
try:
with gdf.create() as dw:
assert dw._tmppath is not None
dw.write(body)
dw.put(metadata)
except DiskFileNoSpace:
pass
else:
self.fail("Expected exception DiskFileNoSpace")
def test_put_rename_ENOENT(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_sleep(*args, **kwargs):
# Return without sleep, no need to dely unit tests
return
def mock_rename(*args, **kwargs):
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep):
with mock.patch("os.rename", mock_rename):
try:
with gdf.create() as dw:
assert dw._tmppath is not None
dw.write(body)
dw.put(metadata)
except OSError:
pass
else:
self.fail("Expected exception DiskFileError")
def test_put_obj_path(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
assert gdf._obj == "z"
assert gdf._obj_path == the_obj_path
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "b", "a")
assert gdf._data_file == os.path.join(
self.td, "vol0", "bar", "b", "a", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_delete(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_same_timestamp(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
now = float(gdf.read_metadata()['X-Timestamp'])
gdf.delete(normalize_timestamp(now))
assert os.path.isdir(gdf._put_datadir)
assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_file_not_found(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
later = float(gdf.read_metadata()['X-Timestamp']) + 1
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_file_unlink_error(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
later = float(gdf.read_metadata()['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
stats = os.stat(the_path)
try:
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.delete(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf._put_datadir)
assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_is_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._data_file == the_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_create(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
saved_fd = None
with gdf.create() as dw:
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "dir")
assert os.path.isdir(gdf._put_datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._put_datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
saved_fd = dw._fd
# At the end of previous with block a close on fd is called.
# Calling os.close on the same fd will raise an OSError
# exception and we must catch it.
try:
os.close(saved_fd)
except OSError:
pass
else:
self.fail("Exception expected")
assert not os.path.exists(saved_tmppath)
def test_create_err_on_close(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.create() as dw:
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "dir")
assert os.path.isdir(gdf._put_datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._put_datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
# Closing the fd prematurely should not raise any exceptions.
dw.close()
assert not os.path.exists(saved_tmppath)
def test_create_err_on_unlink(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.create() as dw:
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "dir")
assert os.path.isdir(gdf._put_datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._put_datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
os.unlink(saved_tmppath)
assert not os.path.exists(saved_tmppath)
def test_unlink_not_called_after_rename(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
body = '1234\n'
etag = md5(body).hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
_mock_do_unlink = Mock() # Shouldn't be called
with patch("gluster.swift.obj.diskfile.do_unlink", _mock_do_unlink):
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
# do_unlink is not called if dw._tmppath is set to None
assert dw._tmppath is None
self.assertFalse(_mock_do_unlink.called)
assert os.path.exists(gdf._data_file) # Real file exists
assert not os.path.exists(tmppath) # Temp file does not exist
def test_fd_closed_when_diskfile_open_raises_exception_race(self):
# do_open() succeeds but read_metadata() fails(GlusterFS)
_m_do_open = Mock(return_value=999)
_m_do_fstat = Mock(return_value=
os.stat_result((33261, 2753735, 2053, 1, 1000,
1000, 6873, 1431415969,
1376895818, 1433139196)))
_m_rmd = Mock(side_effect=IOError(errno.ENOENT,
os.strerror(errno.ENOENT)))
_m_do_close = Mock()
_m_log = Mock()
with nested(
patch("gluster.swift.obj.diskfile.do_open", _m_do_open),
patch("gluster.swift.obj.diskfile.do_fstat", _m_do_fstat),
patch("gluster.swift.obj.diskfile.read_metadata", _m_rmd),
patch("gluster.swift.obj.diskfile.do_close", _m_do_close),
patch("gluster.swift.obj.diskfile.logging.warn", _m_log)):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
try:
with gdf.open():
pass
except DiskFileNotExist:
pass
else:
self.fail("Expecting DiskFileNotExist")
_m_do_fstat.assert_called_once_with(999)
_m_rmd.assert_called_once_with(999)
_m_do_close.assert_called_once_with(999)
self.assertFalse(gdf._fd)
# Make sure ENOENT failure is logged
self.assertTrue("failed with ENOENT" in _m_log.call_args[0][0])
def test_fd_closed_when_diskfile_open_raises_DiskFileExpired(self):
# A GET/DELETE on an expired object should close fd
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "w") as fd:
fd.write("1234")
md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': str(os.path.getsize(the_file)),
'ETag': md5("1234").hexdigest(),
'X-Timestamp': os.stat(the_file).st_mtime,
'X-Delete-At': 0, # This is in the past
'Content-Type': 'application/octet-stream'}
_metadata[_mapit(the_file)] = md
_m_do_close = Mock()
with patch("gluster.swift.obj.diskfile.do_close", _m_do_close):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
try:
with gdf.open():
pass
except DiskFileExpired:
# Confirm that original exception is re-raised
pass
else:
self.fail("Expecting DiskFileExpired")
self.assertEqual(_m_do_close.call_count, 1)
self.assertFalse(gdf._fd)
# Close the actual fd, as we had mocked do_close
os.close(_m_do_close.call_args[0][0])
def make_directory_chown_call(self):
path = os.path.join(self.td, "a/b/c")
_m_do_chown = Mock()
with patch("gluster.swift.obj.diskfile.do_chown", _m_do_chown):
diskfile.make_directory(path, -1, -1)
self.assertFalse(_m_do_chown.called)
self.assertTrue(os.path.isdir(path))
path = os.path.join(self.td, "d/e/f")
_m_do_chown.reset_mock()
with patch("gluster.swift.obj.diskfile.do_chown", _m_do_chown):
diskfile.make_directory(path, -1, 99)
self.assertEqual(_m_do_chown.call_count, 3)
self.assertTrue(os.path.isdir(path))
path = os.path.join(self.td, "g/h/i")
_m_do_chown.reset_mock()
with patch("gluster.swift.obj.diskfile.do_chown", _m_do_chown):
diskfile.make_directory(path, 99, -1)
self.assertEqual(_m_do_chown.call_count, 3)
self.assertTrue(os.path.isdir(path))
def test_fchown_not_called_on_default_uid_gid_values(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
body = '1234'
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': md5(body).hexdigest(),
'Content-Length': len(body),
}
_m_do_fchown = Mock()
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
with patch("gluster.swift.obj.diskfile.do_fchown", _m_do_fchown):
dw.put(metadata)
self.assertFalse(_m_do_fchown.called)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_unlink_not_called_on_non_existent_object(self):
# Create container dir
cpath = os.path.join(self.td, "vol0", "container")
os.makedirs(cpath)
self.assertTrue(os.path.exists(cpath))
# This file does not exist
obj_path = os.path.join(cpath, "object")
self.assertFalse(os.path.exists(obj_path))
# Create diskfile instance and check attribute initialization
gdf = self._get_diskfile("vol0", "p57", "ufo47", "container", "object")
assert gdf._obj == "object"
assert gdf._data_file == obj_path
self.assertFalse(gdf._disk_file_does_not_exist)
# Simulate disk file call sequence issued during DELETE request.
# And confirm that read_metadata() and unlink() is not called.
self.assertRaises(DiskFileNotExist, gdf.read_metadata)
self.assertTrue(gdf._disk_file_does_not_exist)
_m_rmd = Mock()
_m_do_unlink = Mock()
with patch("gluster.swift.obj.diskfile.read_metadata", _m_rmd):
with patch("gluster.swift.obj.diskfile.do_unlink", _m_do_unlink):
gdf.delete(0)
self.assertFalse(_m_rmd.called)
self.assertFalse(_m_do_unlink.called)
class TestDiskFile(unittest.TestCase):
""" Tests for gluster.swift.obj.diskfile """
def setUp(self):
self._orig_tpool_exc = tpool.execute
tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.obj.diskfile.write_metadata
self._saved_df_rm = gluster.swift.obj.diskfile.read_metadata
gluster.swift.obj.diskfile.write_metadata = _mock_write_metadata
gluster.swift.obj.diskfile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
self._saved_do_fsync = gluster.swift.obj.diskfile.do_fsync
gluster.swift.obj.diskfile.do_fsync = _mock_do_fsync
self.td = tempfile.mkdtemp()
self.conf = dict(devices=self.td, mb_per_sync=2,
keep_cache_size=(1024 * 1024), mount_check=False)
self.mgr = DiskFileManager(self.conf, self.lg)
def tearDown(self):
tpool.execute = self._orig_tpool_exc
self.lg = None
_destroyxattr()
gluster.swift.obj.diskfile.write_metadata = self._saved_df_wm
gluster.swift.obj.diskfile.read_metadata = self._saved_df_rm
gluster.swift.common.utils.write_metadata = self._saved_ut_wm
gluster.swift.common.utils.read_metadata = self._saved_ut_rm
gluster.swift.obj.diskfile.do_fsync = self._saved_do_fsync
shutil.rmtree(self.td)
def _get_diskfile(self, d, p, a, c, o, **kwargs):
return self.mgr.get_diskfile(d, p, a, c, o, **kwargs)
def test_constructor_no_slash(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._mgr is self.mgr
assert gdf._device_path == os.path.join(self.td, "vol0")
assert isinstance(gdf._threadpool, ThreadPool)
assert gdf._uid == DEFAULT_UID
assert gdf._gid == DEFAULT_GID
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar"), gdf._datadir
assert gdf._datadir == gdf._put_datadir
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
assert gdf._is_dir is False
assert gdf._logger == self.lg
assert gdf._fd is None
def test_constructor_leadtrail_slash(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "/b/a/z/")
assert gdf._obj == "z"
assert gdf._obj_path == "b/a"
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a"), gdf._datadir
def test_open_no_metadata(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'Content-Type': 'application/octet-stream'}
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._metadata is None
assert not gdf._is_dir
with gdf.open():
assert gdf._data_file == the_file
assert not gdf._is_dir
assert gdf._fd is not None
assert gdf._metadata == exp_md
self.assertRaises(DiskFileNotOpen, gdf.get_metadata)
self.assertRaises(DiskFileNotOpen, gdf.reader)
self.assertRaises(DiskFileNotOpen, gdf.__enter__)
def test_open_and_close(self):
mock_close = Mock()
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47",
"bar", "z")
with gdf.open():
assert gdf._fd is not None
self.assertTrue(mock_close.called)
def test_open_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_file)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._metadata is None
assert not gdf._is_dir
with gdf.open():
assert not gdf._is_dir
assert gdf._data_file == the_file
assert gdf._fd is not None
assert gdf._metadata == exp_md, "%r != %r" % (gdf._metadata, exp_md)
def test_open_invalid_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
inv_md = {
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_file)] = inv_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert not gdf._is_dir
assert gdf._fd is None
with gdf.open():
assert gdf._data_file == the_file
assert gdf._metadata != inv_md
def test_open_isdir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'dir',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_dir)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._obj == "d"
assert gdf._is_dir is False
with gdf.open():
assert gdf._is_dir
assert gdf._data_file == the_dir
assert gdf._metadata == exp_md
def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256):
# FIXME: assumes account === volume
the_path = os.path.join(self.td, dev, acc, con)
the_file = os.path.join(the_path, obj)
base_obj = os.path.basename(the_file)
base_dir = os.path.dirname(the_file)
os.makedirs(base_dir)
with open(the_file, "wb") as fd:
fd.write("y" * fsize)
gdf = self._get_diskfile(dev, par, acc, con, obj)
assert gdf._obj == base_obj
assert not gdf._is_dir
assert gdf._fd is None
return gdf
def test_reader(self):
closed = [False]
fd = [-1]
def mock_close(*args, **kwargs):
closed[0] = True
os.close(fd[0])
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
assert gdf._fd is not None
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
reader = gdf.reader()
assert reader._fd is not None
fd[0] = reader._fd
chunks = [ck for ck in reader]
assert reader._fd is None
assert closed[0]
assert len(chunks) == 1, repr(chunks)
def test_reader_disk_chunk_size(self):
conf = dict(disk_chunk_size=64)
conf.update(self.conf)
self.mgr = DiskFileManager(conf, self.lg)
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader()
try:
assert reader._disk_chunk_size == 64
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 4, repr(chunks)
for chunk in chunks:
assert len(chunk) == 64, repr(chunks)
def test_reader_iter_hook(self):
called = [0]
def mock_sleep(*args, **kwargs):
called[0] += 1
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader(iter_hook=mock_sleep)
try:
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 1, repr(chunks)
assert called[0] == 1, called
def test_reader_larger_file(self):
closed = [False]
fd = [-1]
def mock_close(*args, **kwargs):
closed[0] = True
os.close(fd[0])
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2)
with gdf.open():
assert gdf._fd is not None
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
reader = gdf.reader()
assert reader._fd is not None
fd[0] = reader._fd
chunks = [ck for ck in reader]
assert reader._fd is None
assert closed[0]
def test_reader_dir_object(self):
called = [False]
def our_do_close(fd):
called[0] = True
os.close(fd)
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
os.makedirs(os.path.join(the_cont, "dir"))
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
reader = gdf.reader()
try:
chunks = [ck for ck in reader]
assert len(chunks) == 0, repr(chunks)
with mock.patch("gluster.swift.obj.diskfile.do_close",
our_do_close):
reader.close()
assert not called[0]
finally:
reader.close()
def test_create_dir_object_no_md(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = "dir"
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"))
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
gdf._create_dir_object(the_dir)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert _mapit(full_dir_path) not in _metadata
def test_create_dir_object_with_md(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = "dir"
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"))
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
dir_md = {'Content-Type': 'application/directory',
X_OBJECT_TYPE: DIR_OBJECT}
gdf._create_dir_object(the_dir, dir_md)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert _mapit(full_dir_path) in _metadata
def test_create_dir_object_exists(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.obj.diskfile.do_chown
gluster.swift.obj.diskfile.do_chown = _mock_do_chown
self.assertRaises(
AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(_mapit(the_dir) in _metadata)
def test_create_dir_object_do_stat_failure(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.obj.diskfile.do_chown
gluster.swift.obj.diskfile.do_chown = _mock_do_chown
self.assertRaises(
AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(_mapit(the_dir) in _metadata)
def test_write_metadata(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "z")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy())
self.assertEqual(None, gdf._metadata)
fmd = _metadata[_mapit(the_dir)]
md.update({'X-Object-Type': 'file', 'X-Type': 'Object'})
self.assertTrue(fmd['a'], md['a'])
self.assertTrue(fmd['Content-Type'], md['Content-Type'])
def test_add_metadata_to_existing_file(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[_mapit(the_file)] = ini_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy())
self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
newmd = {'X-Object-Meta-test':'1234'}
gdf.write_metadata(newmd.copy())
on_disk_md = _metadata[_mapit(the_file)]
self.assertTrue(on_disk_md['Content-Length'], 4)
self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
self.assertTrue(on_disk_md['X-Type'], 'Object')
self.assertTrue(on_disk_md['X-Object-Type'], 'file')
self.assertTrue(on_disk_md['ETag'], 'etag')
self.assertFalse('a' in on_disk_md)
def test_add_md_to_existing_file_with_md_in_gdf(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'name': 'z',
'ETag': 'etag',
'X-Timestamp': 'ts'}
_metadata[_mapit(the_file)] = ini_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
# make sure gdf has the _metadata
gdf.open()
md = {'a': 'b'}
gdf.write_metadata(md.copy())
self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
newmd = {'X-Object-Meta-test':'1234'}
gdf.write_metadata(newmd.copy())
on_disk_md = _metadata[_mapit(the_file)]
self.assertTrue(on_disk_md['Content-Length'], 4)
self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
self.assertFalse('a' in on_disk_md)
def test_add_metadata_to_existing_dir(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
self.assertEquals(gdf._metadata, None)
init_md = {
'X-Type': 'Object',
'Content-Length': 0,
'ETag': 'etag',
'X-Timestamp': 'ts',
'X-Object-Meta-test':'test',
'Content-Type': 'application/directory'}
_metadata[_mapit(the_dir)] = init_md
md = {'X-Object-Meta-test':'test'}
gdf.write_metadata(md.copy())
self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Meta-test'],
'test')
self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
'application/directory')
# set new metadata
newmd = {'X-Object-Meta-test2':'1234'}
gdf.write_metadata(newmd.copy())
self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
'application/directory')
self.assertEqual(_metadata[_mapit(the_dir)]["X-Object-Meta-test2"],
'1234')
self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Type'],
DIR_OBJECT)
self.assertFalse('X-Object-Meta-test' in _metadata[_mapit(the_dir)])
def test_write_metadata_w_meta_file(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_file)] == newmd
def test_write_metadata_w_meta_file_no_content_type(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = deepcopy(gdf.read_metadata())
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_file)] == newmd
def test_write_metadata_w_meta_dir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_dir)] == newmd
def test_write_metadata_w_marker_dir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_dir)] == newmd
def test_put_w_marker_dir_create(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
assert gdf._metadata is None
newmd = {
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/directory'}
with gdf.create() as dw:
dw.put(newmd)
assert gdf._data_file == the_dir
for key, val in newmd.items():
assert _metadata[_mapit(the_dir)][key] == val
assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT
def test_put_is_dir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
origmd = gdf.get_metadata()
origfmd = _metadata[_mapit(the_dir)]
newmd = deepcopy(origmd)
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
with gdf.create() as dw:
try:
# FIXME: We should probably be able to detect in .create()
# when the target file name already exists as a directory to
# avoid reading the data off the wire only to fail as a
# directory.
dw.write('12345\n')
dw.put(newmd)
except AlreadyExistsAsDir:
pass
else:
self.fail("Expected to encounter"
" 'already-exists-as-dir' exception")
with gdf.open():
assert gdf.get_metadata() == origmd
assert _metadata[_mapit(the_dir)] == origfmd, "was: %r, is: %r" % (
origfmd, _metadata[_mapit(the_dir)])
def test_put(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
assert gdf._datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_put_ENOSPC(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
assert gdf._datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_open(*args, **kwargs):
raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))
with mock.patch("os.open", mock_open):
try:
with gdf.create() as dw:
assert dw._tmppath is not None
dw.write(body)
dw.put(metadata)
except DiskFileNoSpace:
pass
else:
self.fail("Expected exception DiskFileNoSpace")
def test_put_rename_ENOENT(self):
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
assert gdf._datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_sleep(*args, **kwargs):
# Return without sleep, no need to dely unit tests
return
def mock_rename(*args, **kwargs):
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep):
with mock.patch("os.rename", mock_rename):
try:
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
except GlusterFileSystemOSError:
pass
else:
self.fail("Expected exception DiskFileError")
def test_put_obj_path(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
assert gdf._obj == "z"
assert gdf._obj_path == the_obj_path
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a")
assert gdf._data_file == os.path.join(
self.td, "vol0", "ufo47", "bar", "b", "a", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_delete(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._datadir)
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_same_timestamp(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
now = float(gdf.read_metadata()['X-Timestamp'])
gdf.delete(normalize_timestamp(now))
assert os.path.isdir(gdf._datadir)
assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_file_not_found(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._datadir)
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_file_unlink_error(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
stats = os.stat(the_path)
try:
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.delete(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf._datadir)
assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_is_dir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._data_file == the_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._datadir)
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_create(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
saved_fd = None
with gdf.create() as dw:
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
assert os.path.isdir(gdf._datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
saved_fd = dw._fd
# At the end of previous with block a close on fd is called.
# Calling os.close on the same fd will raise an OSError
# exception and we must catch it.
try:
os.close(saved_fd)
except OSError:
pass
else:
self.fail("Exception expected")
assert not os.path.exists(saved_tmppath)
def test_create_err_on_close(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.create() as dw:
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
assert os.path.isdir(gdf._datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
# Closing the fd prematurely should not raise any exceptions.
dw.close()
assert not os.path.exists(saved_tmppath)
def test_create_err_on_unlink(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.create() as dw:
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
assert os.path.isdir(gdf._datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
os.unlink(saved_tmppath)
assert not os.path.exists(saved_tmppath)
def __init__(self, *args, **kwargs):
self.manager_cls = DiskFileManager(*args, **kwargs)
class TestDiskFile(unittest.TestCase):
""" Tests for gluster.swift.obj.diskfile """
def setUp(self):
self._orig_tpool_exc = tpool.execute
tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.obj.diskfile.write_metadata
self._saved_df_rm = gluster.swift.obj.diskfile.read_metadata
gluster.swift.obj.diskfile.write_metadata = _mock_write_metadata
gluster.swift.obj.diskfile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
self._saved_do_fsync = gluster.swift.obj.diskfile.do_fsync
gluster.swift.obj.diskfile.do_fsync = _mock_do_fsync
self.td = tempfile.mkdtemp()
self.conf = dict(devices=self.td,
mb_per_sync=2,
keep_cache_size=(1024 * 1024),
mount_check=False)
self.mgr = DiskFileManager(self.conf, self.lg)
def tearDown(self):
tpool.execute = self._orig_tpool_exc
self.lg = None
_destroyxattr()
gluster.swift.obj.diskfile.write_metadata = self._saved_df_wm
gluster.swift.obj.diskfile.read_metadata = self._saved_df_rm
gluster.swift.common.utils.write_metadata = self._saved_ut_wm
gluster.swift.common.utils.read_metadata = self._saved_ut_rm
gluster.swift.obj.diskfile.do_fsync = self._saved_do_fsync
shutil.rmtree(self.td)
def _get_diskfile(self, d, p, a, c, o, **kwargs):
return self.mgr.get_diskfile(d, p, a, c, o, **kwargs)
def test_constructor_no_slash(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._mgr is self.mgr
assert gdf._device_path == os.path.join(self.td, "vol0")
assert isinstance(gdf._threadpool, ThreadPool)
assert gdf._uid == DEFAULT_UID
assert gdf._gid == DEFAULT_GID
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._put_datadir == os.path.join(self.td, "vol0",
"bar"), gdf._put_datadir
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
assert gdf._fd is None
def test_constructor_leadtrail_slash(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "/b/a/z/")
assert gdf._obj == "z"
assert gdf._obj_path == "b/a"
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "b",
"a"), gdf._put_datadir
def test_open_no_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'Content-Type': 'application/octet-stream'
}
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
with gdf.open():
assert gdf._data_file == the_file
assert gdf._fd is not None
assert gdf._metadata == exp_md
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
self.assertRaises(DiskFileNotOpen, gdf.get_metadata)
self.assertRaises(DiskFileNotOpen, gdf.reader)
self.assertRaises(DiskFileNotOpen, gdf.__enter__)
def test_read_metadata_optimize_open_close(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
init_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': md5("1234").hexdigest(),
'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime),
'Content-Type': 'application/octet-stream'
}
_metadata[_mapit(the_file)] = init_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
# Case 1
# Ensure that reading metadata for non-GET requests
# does not incur opening and closing the file when
# metadata is NOT stale.
mock_open = Mock()
mock_close = Mock()
with mock.patch("gluster.swift.obj.diskfile.do_open", mock_open):
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
md = gdf.read_metadata()
self.assertEqual(md, init_md)
self.assertFalse(mock_open.called)
self.assertFalse(mock_close.called)
# Case 2
# Ensure that reading metadata for non-GET requests
# still opens and reads the file when metadata is stale
with open(the_file, "a") as fd:
# Append to the existing file to make the stored metadata
# invalid/stale.
fd.write("5678")
md = gdf.read_metadata()
# Check that the stale metadata is recalculated to account for
# change in file content
self.assertNotEqual(md, init_md)
self.assertEqual(md['Content-Length'], 8)
self.assertEqual(md['ETag'], md5("12345678").hexdigest())
def test_open_and_close(self):
mock_close = Mock()
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar",
"z")
with gdf.open():
assert gdf._fd is not None
self.assertTrue(mock_close.called)
def test_open_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'
}
_metadata[_mapit(the_file)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._metadata is None
assert gdf._disk_file_open is False
with gdf.open():
assert gdf._data_file == the_file
assert gdf._fd is not None
assert gdf._metadata == exp_md, "%r != %r" % (gdf._metadata,
exp_md)
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def test_open_invalid_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
inv_md = {
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'
}
_metadata[_mapit(the_file)] = inv_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
with gdf.open():
assert gdf._data_file == the_file
assert gdf._metadata != inv_md
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def test_open_isdir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'dir',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'
}
_metadata[_mapit(the_dir)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._obj == "d"
assert gdf._disk_file_open is False
with gdf.open():
assert gdf._data_file == the_dir
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256):
# FIXME: assumes account === volume
the_path = os.path.join(self.td, dev, con)
the_file = os.path.join(the_path, obj)
base_obj = os.path.basename(the_file)
base_dir = os.path.dirname(the_file)
os.makedirs(base_dir)
with open(the_file, "wb") as fd:
fd.write("y" * fsize)
gdf = self._get_diskfile(dev, par, acc, con, obj)
assert gdf._obj == base_obj
assert gdf._fd is None
return gdf
def test_reader(self):
closed = [False]
fd = [-1]
def mock_close(*args, **kwargs):
closed[0] = True
os.close(fd[0])
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar",
"z")
with gdf.open():
assert gdf._fd is not None
assert gdf._data_file == os.path.join(self.td, "vol0", "bar",
"z")
reader = gdf.reader()
assert reader._fd is not None
fd[0] = reader._fd
chunks = [ck for ck in reader]
assert reader._fd is None
assert closed[0]
assert len(chunks) == 1, repr(chunks)
def test_reader_disk_chunk_size(self):
conf = dict(disk_chunk_size=64)
conf.update(self.conf)
self.mgr = DiskFileManager(conf, self.lg)
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader()
try:
assert reader._disk_chunk_size == 64
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 4, repr(chunks)
for chunk in chunks:
assert len(chunk) == 64, repr(chunks)
def test_reader_iter_hook(self):
called = [0]
def mock_sleep(*args, **kwargs):
called[0] += 1
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
reader = gdf.reader(iter_hook=mock_sleep)
try:
chunks = [ck for ck in reader]
finally:
reader.close()
assert len(chunks) == 1, repr(chunks)
assert called[0] == 1, called
def test_reader_larger_file(self):
closed = [False]
fd = [-1]
def mock_close(*args, **kwargs):
closed[0] = True
os.close(fd[0])
with mock.patch("gluster.swift.obj.diskfile.do_close", mock_close):
gdf = self._create_and_get_diskfile("vol0",
"p57",
"ufo47",
"bar",
"z",
fsize=1024 * 1024 * 2)
with gdf.open():
assert gdf._fd is not None
assert gdf._data_file == os.path.join(self.td, "vol0", "bar",
"z")
reader = gdf.reader()
assert reader._fd is not None
fd[0] = reader._fd
chunks = [ck for ck in reader]
assert reader._fd is None
assert closed[0]
def test_reader_dir_object(self):
called = [False]
def our_do_close(fd):
called[0] = True
os.close(fd)
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(os.path.join(the_cont, "dir"))
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
reader = gdf.reader()
try:
chunks = [ck for ck in reader]
assert len(chunks) == 0, repr(chunks)
with mock.patch("gluster.swift.obj.diskfile.do_close",
our_do_close):
reader.close()
assert not called[0]
finally:
reader.close()
def test_create_dir_object_no_md(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = "dir"
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"))
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
gdf._create_dir_object(the_dir)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert _mapit(full_dir_path) not in _metadata
def test_create_dir_object_with_md(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = "dir"
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"))
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
dir_md = {
'Content-Type': 'application/directory',
X_OBJECT_TYPE: DIR_OBJECT
}
gdf._create_dir_object(the_dir, dir_md)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert _mapit(full_dir_path) in _metadata
def test_create_dir_object_exists(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.obj.diskfile.do_chown
gluster.swift.obj.diskfile.do_chown = _mock_do_chown
self.assertRaises(AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(_mapit(the_dir) in _metadata)
def test_create_dir_object_do_stat_failure(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.obj.diskfile.do_chown
gluster.swift.obj.diskfile.do_chown = _mock_do_chown
self.assertRaises(AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(_mapit(the_dir) in _metadata)
def test_write_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "z")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy())
self.assertEqual(None, gdf._metadata)
fmd = _metadata[_mapit(the_dir)]
md.update({'X-Object-Type': 'file', 'X-Type': 'Object'})
self.assertTrue(fmd['a'], md['a'])
self.assertTrue(fmd['Content-Type'], md['Content-Type'])
def test_add_metadata_to_existing_file(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'
}
_metadata[_mapit(the_file)] = ini_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy())
self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
newmd = {'X-Object-Meta-test': '1234'}
gdf.write_metadata(newmd.copy())
on_disk_md = _metadata[_mapit(the_file)]
self.assertTrue(on_disk_md['Content-Length'], 4)
self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
self.assertTrue(on_disk_md['X-Type'], 'Object')
self.assertTrue(on_disk_md['X-Object-Type'], 'file')
self.assertTrue(on_disk_md['ETag'], 'etag')
self.assertFalse('a' in on_disk_md)
def test_add_md_to_existing_file_with_md_in_gdf(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'name': 'z',
'ETag': 'etag',
'X-Timestamp': 'ts'
}
_metadata[_mapit(the_file)] = ini_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
# make sure gdf has the _metadata
gdf.open()
md = {'a': 'b'}
gdf.write_metadata(md.copy())
self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
newmd = {'X-Object-Meta-test': '1234'}
gdf.write_metadata(newmd.copy())
on_disk_md = _metadata[_mapit(the_file)]
self.assertTrue(on_disk_md['Content-Length'], 4)
self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
self.assertFalse('a' in on_disk_md)
def test_add_metadata_to_existing_dir(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
self.assertEquals(gdf._metadata, None)
init_md = {
'X-Type': 'Object',
'Content-Length': 0,
'ETag': 'etag',
'X-Timestamp': 'ts',
'X-Object-Meta-test': 'test',
'Content-Type': 'application/directory'
}
_metadata[_mapit(the_dir)] = init_md
md = {'X-Object-Meta-test': 'test'}
gdf.write_metadata(md.copy())
self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Meta-test'],
'test')
self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
'application/directory')
# set new metadata
newmd = {'X-Object-Meta-test2': '1234'}
gdf.write_metadata(newmd.copy())
self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
'application/directory')
self.assertEqual(_metadata[_mapit(the_dir)]["X-Object-Meta-test2"],
'1234')
self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Type'],
DIR_OBJECT)
self.assertFalse('X-Object-Meta-test' in _metadata[_mapit(the_dir)])
def test_write_metadata_w_meta_file(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_file)] == newmd
def test_write_metadata_w_meta_file_no_content_type(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = deepcopy(gdf.read_metadata())
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_file)] == newmd
def test_write_metadata_w_meta_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_dir)] == newmd
def test_write_metadata_w_marker_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = deepcopy(gdf.read_metadata())
newmd['X-Object-Meta-test'] = '1234'
gdf.write_metadata(newmd)
assert _metadata[_mapit(the_dir)] == newmd
def test_put_w_marker_dir_create(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
assert gdf._metadata is None
newmd = {
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/directory'
}
with gdf.create() as dw:
dw.put(newmd)
assert gdf._data_file == the_dir
for key, val in newmd.items():
assert _metadata[_mapit(the_dir)][key] == val
assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT
def test_put_is_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
origmd = gdf.get_metadata()
origfmd = _metadata[_mapit(the_dir)]
newmd = deepcopy(origmd)
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
with gdf.create() as dw:
try:
# FIXME: We should probably be able to detect in .create()
# when the target file name already exists as a directory to
# avoid reading the data off the wire only to fail as a
# directory.
dw.write('12345\n')
dw.put(newmd)
except AlreadyExistsAsDir:
pass
else:
self.fail("Expected to encounter"
" 'already-exists-as-dir' exception")
with gdf.open():
assert gdf.get_metadata() == origmd
assert _metadata[_mapit(the_dir)] == origfmd, "was: %r, is: %r" % (
origfmd, _metadata[_mapit(the_dir)])
def test_put(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_put_ENOSPC(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_open(*args, **kwargs):
raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))
with mock.patch("os.open", mock_open):
try:
with gdf.create() as dw:
assert dw._tmppath is not None
dw.write(body)
dw.put(metadata)
except DiskFileNoSpace:
pass
else:
self.fail("Expected exception DiskFileNoSpace")
def test_put_rename_ENOENT(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == the_cont
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_sleep(*args, **kwargs):
# Return without sleep, no need to dely unit tests
return
def mock_rename(*args, **kwargs):
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep):
with mock.patch("os.rename", mock_rename):
try:
with gdf.create() as dw:
assert dw._tmppath is not None
dw.write(body)
dw.put(metadata)
except OSError:
pass
else:
self.fail("Expected exception DiskFileError")
def test_put_obj_path(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
assert gdf._obj == "z"
assert gdf._obj_path == the_obj_path
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar", "b",
"a")
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "b", "a",
"z")
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_delete(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_same_timestamp(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
now = float(gdf.read_metadata()['X-Timestamp'])
gdf.delete(normalize_timestamp(now))
assert os.path.isdir(gdf._put_datadir)
assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_file_not_found(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
later = float(gdf.read_metadata()['X-Timestamp']) + 1
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_file_unlink_error(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
later = float(gdf.read_metadata()['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
stats = os.stat(the_path)
try:
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.delete(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf._put_datadir)
assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_is_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._data_file == the_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_create(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
saved_fd = None
with gdf.create() as dw:
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar",
"dir")
assert os.path.isdir(gdf._put_datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._put_datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
saved_fd = dw._fd
# At the end of previous with block a close on fd is called.
# Calling os.close on the same fd will raise an OSError
# exception and we must catch it.
try:
os.close(saved_fd)
except OSError:
pass
else:
self.fail("Exception expected")
assert not os.path.exists(saved_tmppath)
def test_create_err_on_close(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.create() as dw:
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar",
"dir")
assert os.path.isdir(gdf._put_datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._put_datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
# Closing the fd prematurely should not raise any exceptions.
dw.close()
assert not os.path.exists(saved_tmppath)
def test_create_err_on_unlink(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.create() as dw:
assert gdf._put_datadir == os.path.join(self.td, "vol0", "bar",
"dir")
assert os.path.isdir(gdf._put_datadir)
saved_tmppath = dw._tmppath
assert os.path.dirname(saved_tmppath) == gdf._put_datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
os.unlink(saved_tmppath)
assert not os.path.exists(saved_tmppath)
def test_unlink_not_called_after_rename(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
body = '1234\n'
etag = md5(body).hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
_mock_do_unlink = Mock() # Shouldn't be called
with patch("gluster.swift.obj.diskfile.do_unlink", _mock_do_unlink):
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
dw.put(metadata)
# do_unlink is not called if dw._tmppath is set to None
assert dw._tmppath is None
self.assertFalse(_mock_do_unlink.called)
assert os.path.exists(gdf._data_file) # Real file exists
assert not os.path.exists(tmppath) # Temp file does not exist
def test_fd_closed_when_diskfile_open_raises_exception_race(self):
# do_open() succeeds but read_metadata() fails(GlusterFS)
_m_do_open = Mock(return_value=999)
_m_do_fstat = Mock(return_value=os.stat_result((33261, 2753735, 2053,
1, 1000, 1000, 6873,
1431415969, 1376895818,
1433139196)))
_m_rmd = Mock(
side_effect=IOError(errno.ENOENT, os.strerror(errno.ENOENT)))
_m_do_close = Mock()
_m_log = Mock()
with nested(patch("gluster.swift.obj.diskfile.do_open", _m_do_open),
patch("gluster.swift.obj.diskfile.do_fstat", _m_do_fstat),
patch("gluster.swift.obj.diskfile.read_metadata", _m_rmd),
patch("gluster.swift.obj.diskfile.do_close", _m_do_close),
patch("gluster.swift.obj.diskfile.logging.warn", _m_log)):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
try:
with gdf.open():
pass
except DiskFileNotExist:
pass
else:
self.fail("Expecting DiskFileNotExist")
_m_do_fstat.assert_called_once_with(999)
_m_rmd.assert_called_once_with(999)
_m_do_close.assert_called_once_with(999)
self.assertFalse(gdf._fd)
# Make sure ENOENT failure is logged
self.assertTrue("failed with ENOENT" in _m_log.call_args[0][0])
def test_fd_closed_when_diskfile_open_raises_DiskFileExpired(self):
# A GET/DELETE on an expired object should close fd
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "w") as fd:
fd.write("1234")
md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': str(os.path.getsize(the_file)),
'ETag': md5("1234").hexdigest(),
'X-Timestamp': os.stat(the_file).st_mtime,
'X-Delete-At': 0, # This is in the past
'Content-Type': 'application/octet-stream'
}
_metadata[_mapit(the_file)] = md
_m_do_close = Mock()
with patch("gluster.swift.obj.diskfile.do_close", _m_do_close):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
try:
with gdf.open():
pass
except DiskFileExpired:
# Confirm that original exception is re-raised
pass
else:
self.fail("Expecting DiskFileExpired")
self.assertEqual(_m_do_close.call_count, 1)
self.assertFalse(gdf._fd)
# Close the actual fd, as we had mocked do_close
os.close(_m_do_close.call_args[0][0])
def make_directory_chown_call(self):
path = os.path.join(self.td, "a/b/c")
_m_do_chown = Mock()
with patch("gluster.swift.obj.diskfile.do_chown", _m_do_chown):
diskfile.make_directory(path, -1, -1)
self.assertFalse(_m_do_chown.called)
self.assertTrue(os.path.isdir(path))
path = os.path.join(self.td, "d/e/f")
_m_do_chown.reset_mock()
with patch("gluster.swift.obj.diskfile.do_chown", _m_do_chown):
diskfile.make_directory(path, -1, 99)
self.assertEqual(_m_do_chown.call_count, 3)
self.assertTrue(os.path.isdir(path))
path = os.path.join(self.td, "g/h/i")
_m_do_chown.reset_mock()
with patch("gluster.swift.obj.diskfile.do_chown", _m_do_chown):
diskfile.make_directory(path, 99, -1)
self.assertEqual(_m_do_chown.call_count, 3)
self.assertTrue(os.path.isdir(path))
def test_fchown_not_called_on_default_uid_gid_values(self):
the_cont = os.path.join(self.td, "vol0", "bar")
os.makedirs(the_cont)
body = '1234'
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': md5(body).hexdigest(),
'Content-Length': len(body),
}
_m_do_fchown = Mock()
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.create() as dw:
assert dw._tmppath is not None
tmppath = dw._tmppath
dw.write(body)
with patch("gluster.swift.obj.diskfile.do_fchown", _m_do_fchown):
dw.put(metadata)
self.assertFalse(_m_do_fchown.called)
assert os.path.exists(gdf._data_file)
assert not os.path.exists(tmppath)
def test_unlink_not_called_on_non_existent_object(self):
# Create container dir
cpath = os.path.join(self.td, "vol0", "container")
os.makedirs(cpath)
self.assertTrue(os.path.exists(cpath))
# This file does not exist
obj_path = os.path.join(cpath, "object")
self.assertFalse(os.path.exists(obj_path))
# Create diskfile instance and check attribute initialization
gdf = self._get_diskfile("vol0", "p57", "ufo47", "container", "object")
assert gdf._obj == "object"
assert gdf._data_file == obj_path
self.assertFalse(gdf._disk_file_does_not_exist)
# Simulate disk file call sequence issued during DELETE request.
# And confirm that read_metadata() and unlink() is not called.
self.assertRaises(DiskFileNotExist, gdf.read_metadata)
self.assertTrue(gdf._disk_file_does_not_exist)
_m_rmd = Mock()
_m_do_unlink = Mock()
with patch("gluster.swift.obj.diskfile.read_metadata", _m_rmd):
with patch("gluster.swift.obj.diskfile.do_unlink", _m_do_unlink):
gdf.delete(0)
self.assertFalse(_m_rmd.called)
self.assertFalse(_m_do_unlink.called)