def test_get_data_file_size_os_err(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
stats = os.stat(the_path)
try:
os.chmod(the_path, 0)
def _mock_getsize_eaccess_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
with patch("os.path.getsize", _mock_getsize_eaccess_err):
try:
gdf.get_data_file_size()
except OSError as err:
assert err.errno == errno.EACCES
else:
self.fail("Expected OSError exception")
finally:
os.chmod(the_path, stats.st_mode)
finally:
shutil.rmtree(td)
def test_put_obj_path(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
td = tempfile.mkdtemp()
try:
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
the_file, self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == the_obj_path
assert gdf.name == os.path.join("bar", "b", "a")
assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a")
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.writer() as dw:
assert dw.tmppath is not None
tmppath = dw.tmppath
dw.write(body)
dw.put(metadata)
assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z")
assert os.path.exists(gdf.data_file)
assert not os.path.exists(tmppath)
finally:
shutil.rmtree(td)
def test_put_is_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
origmd = gdf.metadata.copy()
origfmd = _metadata[the_dir]
newmd = gdf.metadata.copy()
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
with gdf.writer() as dw:
try:
dw.put(newmd, extension='.data')
except DiskFileError:
pass
else:
self.fail("Expected to encounter"
" 'already-exists-as-dir' exception")
assert gdf.metadata == origmd
assert _metadata[the_dir] == origfmd
finally:
shutil.rmtree(td)
def test_put_w_tombstone(self):
assert not os.path.exists("/tmp/foo")
gdf = DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", "z", self.lg)
assert gdf.metadata == {}
gdf.put_metadata({'x': '1'}, tombstone=True)
assert gdf.metadata == {}
def test_writer(self):
td = tempfile.mkdtemp()
try:
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
saved_fd = None
with gdf.writer() as dw:
assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = dw.tmppath
assert os.path.dirname(saved_tmppath) == gdf.datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
saved_fd = dw.fd
# At the end of previous with block a close on fd is called.
# Calling os.close on the same fd will raise an OSError
# exception and we must catch it.
try:
os.close(saved_fd)
except OSError:
pass
else:
self.fail("Exception expected")
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_get_data_file_size_dne(self):
assert not os.path.exists("/tmp/foo")
gdf = DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"/b/a/z/", self.lg)
try:
gdf.get_data_file_size()
except DiskFileNotExist:
pass
else:
self.fail("Expected DiskFileNotExist exception")
def test_put_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "z")
try:
os.makedirs(the_dir)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg)
md = { 'Content-Type': 'application/octet-stream', 'a': 'b' }
gdf.put_metadata(md.copy())
assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (gdf.metadata, md)
assert _metadata[the_dir] == md
finally:
shutil.rmtree(td)
def test_unlinkold_no_metadata(self):
assert not os.path.exists("/tmp/foo")
gdf = DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
_saved_rmobjdir = gluster.swift.obj.diskfile.rmobjdir
gluster.swift.obj.diskfile.rmobjdir = _mock_rmobjdir
try:
gdf.unlinkold(None)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.obj.diskfile.rmobjdir = _saved_rmobjdir
def test_get_data_file_size_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf._obj == "d"
assert gdf.data_file == the_dir
assert gdf._is_dir
assert 0 == gdf.get_data_file_size()
finally:
shutil.rmtree(td)
def test_close_no_open_fp(self):
assert not os.path.exists("/tmp/foo")
gdf = DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg, keep_data_fp=True)
gdf._is_dir = False
self.called = False
def our_do_close(fp):
self.called = True
with mock.patch("gluster.swift.obj.diskfile.do_close", our_do_close):
gdf.close()
assert not self.called
assert gdf.fp is None
def test_put_w_marker_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
finally:
shutil.rmtree(td)
def test_create_dir_object_no_md(self):
td = tempfile.mkdtemp()
the_cont = os.path.join(td, "vol0", "bar")
the_dir = "dir"
try:
os.makedirs(the_cont)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"), self.lg)
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
gdf._create_dir_object(the_dir)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert full_dir_path not in _metadata
finally:
shutil.rmtree(td)
def test_get_data_file_size(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert 4 == gdf.get_data_file_size()
finally:
shutil.rmtree(td)
def test_put_w_meta_file(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
finally:
shutil.rmtree(td)
def test_unlinkold_is_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.data_file == the_dir
assert gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_create_dir_object_with_md(self):
td = tempfile.mkdtemp()
the_cont = os.path.join(td, "vol0", "bar")
the_dir = "dir"
try:
os.makedirs(the_cont)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
os.path.join(the_dir, "z"), self.lg)
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
dir_md = {'Content-Type': 'application/directory',
X_OBJECT_TYPE: DIR_OBJECT}
gdf._create_dir_object(the_dir, dir_md)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert full_dir_path in _metadata
finally:
shutil.rmtree(td)
def test_writer_err_on_unlink(self):
td = tempfile.mkdtemp()
try:
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.writer() as dw:
assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = dw.tmppath
assert os.path.dirname(saved_tmppath) == gdf.datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
os.unlink(saved_tmppath)
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_put_rename_ENOENT(self):
td = tempfile.mkdtemp()
the_cont = os.path.join(td, "vol0", "bar")
try:
os.makedirs(the_cont)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar", "z", self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf.name == "bar"
assert gdf.datadir == the_cont
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_sleep(*args, **kwargs):
# Return without sleep, no need to dely unit tests
return
def mock_rename(*args, **kwargs):
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep):
with mock.patch("os.rename", mock_rename):
try:
with gdf.writer() as dw:
assert dw.tmppath is not None
tmppath = dw.tmppath
dw.write(body)
dw.put(metadata)
except GlusterFileSystemOSError:
pass
else:
self.fail("Expected exception DiskFileError")
finally:
shutil.rmtree(td)
def test_close_dir_object(self):
td = tempfile.mkdtemp()
the_cont = os.path.join(td, "vol0", "bar")
the_dir = "dir"
self.called = False
try:
os.makedirs(os.path.join(the_cont, "dir"))
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg, keep_data_fp=True)
def our_do_close(fp):
self.called = True
with mock.patch("gluster.swift.obj.diskfile.do_close",
our_do_close):
gdf.close()
assert self.called
finally:
shutil.rmtree(td)
def test_writer_err_on_close(self):
td = tempfile.mkdtemp()
try:
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.writer() as dw:
assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = dw.tmppath
assert os.path.dirname(saved_tmppath) == gdf.datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
dw.write("123")
# Closing the fd prematurely should not raise any exceptions.
os.close(dw.fd)
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_unlinkold_file(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_put_ENOSPC(self):
td = tempfile.mkdtemp()
the_cont = os.path.join(td, "vol0", "bar")
try:
os.makedirs(the_cont)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf.name == "bar"
assert gdf.datadir == the_cont
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
def mock_open(*args, **kwargs):
raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))
with mock.patch("os.open", mock_open):
try:
with gdf.writer() as dw:
assert dw.tmppath is not None
dw.write(body)
dw.put(metadata)
except DiskFileNoSpace:
pass
else:
self.fail("Expected exception DiskFileNoSpace")
finally:
shutil.rmtree(td)
def test_get_data_file_size_dne_os_err(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
gdf.data_file = gdf.data_file + ".dne"
try:
gdf.get_data_file_size()
except DiskFileNotExist:
pass
else:
self.fail("Expected DiskFileNotExist exception")
finally:
shutil.rmtree(td)
def test_filter_metadata(self):
assert not os.path.exists("/tmp/foo")
gdf = DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf._filter_metadata()
assert gdf.metadata == {}
gdf.metadata[X_TYPE] = 'a'
gdf.metadata[X_OBJECT_TYPE] = 'b'
gdf.metadata['foobar'] = 'c'
gdf._filter_metadata()
assert X_TYPE not in gdf.metadata
assert X_OBJECT_TYPE not in gdf.metadata
assert 'foobar' in gdf.metadata
def test_unlinkold_file_unlink_error(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
stats = os.stat(the_path)
try:
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.unlinkold(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf.datadir)
assert os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_put_w_marker_dir_create(self):
td = tempfile.mkdtemp()
the_cont = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_cont, "dir")
try:
os.makedirs(the_cont)
gdf = DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
assert gdf.metadata == {}
newmd = {
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/directory'}
with gdf.writer() as dw:
dw.put(newmd, extension='.dir')
assert gdf.data_file == the_dir
for key,val in newmd.items():
assert gdf.metadata[key] == val
assert _metadata[the_dir][key] == val
assert gdf.metadata[X_OBJECT_TYPE] == DIR_OBJECT
assert _metadata[the_dir][X_OBJECT_TYPE] == DIR_OBJECT
finally:
shutil.rmtree(td)
def test_is_deleted(self):
assert not os.path.exists("/tmp/foo")
gdf = DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar", "z", self.lg)
assert gdf.is_deleted()
gdf.data_file = "/tmp/foo/bar"
assert not gdf.is_deleted()
