def test_open_no_metadata(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'X-Object-PUT-Mtime': normalize_timestamp(stats.st_mtime),
'Content-Type': 'application/octet-stream'}
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
assert not gdf._is_dir
with gdf.open():
assert gdf._data_file == the_file
assert not gdf._is_dir
assert gdf._fd is not None
assert gdf._metadata == exp_md
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def test_open_no_metadata(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'X-Object-PUT-Mtime': normalize_timestamp(stats.st_mtime),
'Content-Type': 'application/octet-stream'
}
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
assert not gdf._is_dir
with gdf.open():
assert gdf._data_file == the_file
assert not gdf._is_dir
assert gdf._fd is not None
assert gdf._metadata == exp_md
assert gdf._disk_file_open is True
assert gdf._disk_file_open is False
def test_delete_file_unlink_error(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
stats = os.stat(the_path)
try:
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.delete(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf._put_datadir)
assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_file_unlink_error(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
stats = os.stat(the_path)
try:
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.delete(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf._datadir)
assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_is_dir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._data_file == the_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_is_dir(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._data_file == the_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._datadir)
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_get_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('123')
tf.file.flush()
md = utils.get_object_metadata(tf.name)
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_get_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
md = utils.get_object_metadata(td)
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)
def test_delete_same_timestamp(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
now = float(gdf.read_metadata()['X-Timestamp'])
gdf.delete(normalize_timestamp(now))
assert os.path.isdir(gdf._put_datadir)
assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_same_timestamp(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
now = float(gdf.read_metadata()['X-Timestamp'])
gdf.delete(normalize_timestamp(now))
assert os.path.isdir(gdf._datadir)
assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_read_metadata_optimize_open_close(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
init_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': md5("1234").hexdigest(),
'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime),
'Content-Type': 'application/octet-stream'
}
_metadata[_mapit(the_file)] = init_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
assert not gdf._is_dir
# Case 1
# Ensure that reading metadata for non-GET requests
# does not incur opening and closing the file when
# metadata is NOT stale.
mock_open = Mock()
mock_close = Mock()
with mock.patch("swiftonfile.swift.obj.diskfile.do_open", mock_open):
with mock.patch("swiftonfile.swift.obj.diskfile.do_close",
mock_close):
md = gdf.read_metadata()
self.assertEqual(md, init_md)
self.assertFalse(mock_open.called)
self.assertFalse(mock_close.called)
# Case 2
# Ensure that reading metadata for non-GET requests
# still opens and reads the file when metadata is stale
with open(the_file, "a") as fd:
# Append to the existing file to make the stored metadata
# invalid/stale.
fd.write("5678")
md = gdf.read_metadata()
# Check that the stale metadata is recalculated to account for
# change in file content
self.assertNotEqual(md, init_md)
self.assertEqual(md['Content-Length'], 8)
self.assertEqual(md['ETag'], md5("12345678").hexdigest())
def test_read_metadata_optimize_open_close(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
init_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 4,
'ETag': md5("1234").hexdigest(),
'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime),
'Content-Type': 'application/octet-stream'}
_metadata[_mapit(the_file)] = init_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._fd is None
assert gdf._disk_file_open is False
assert gdf._metadata is None
assert not gdf._is_dir
# Case 1
# Ensure that reading metadata for non-GET requests
# does not incur opening and closing the file when
# metadata is NOT stale.
mock_open = Mock()
mock_close = Mock()
with mock.patch("swiftonfile.swift.obj.diskfile.do_open", mock_open):
with mock.patch("swiftonfile.swift.obj.diskfile.do_close",
mock_close):
md = gdf.read_metadata()
self.assertEqual(md, init_md)
self.assertFalse(mock_open.called)
self.assertFalse(mock_close.called)
# Case 2
# Ensure that reading metadata for non-GET requests
# still opens and reads the file when metadata is stale
with open(the_file, "a") as fd:
# Append to the existing file to make the stored metadata
# invalid/stale.
fd.write("5678")
md = gdf.read_metadata()
# Check that the stale metadata is recalculated to account for
# change in file content
self.assertNotEqual(md, init_md)
self.assertEqual(md['Content-Length'], 8)
self.assertEqual(md['ETag'], md5("12345678").hexdigest())
def test_delete_file_not_found(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._datadir)
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_file_not_found(self):
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
assert gdf._data_file == the_file
assert not gdf._is_dir
later = float(gdf.read_metadata()['X-Timestamp']) + 1
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf._put_datadir)
assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_create_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('4567')
tf.file.flush()
r_md = utils.create_object_metadata(tf.name)
xkey = _xkey(tf.name, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('4567')
tf.file.flush()
r_md = utils.create_object_metadata(tf.name)
xkey = _xkey(tf.name, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['get'] == 1
assert _xattr_op_cnt['set'] == 1
md = pickle.loads(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = utils.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = utils.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['get'] == 1
assert _xattr_op_cnt['set'] == 1
md = pickle.loads(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)
