def test_open_no_metadata(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") stats = os.stat(the_file) ts = normalize_timestamp(stats.st_ctime) etag = md5() etag.update("1234") etag = etag.hexdigest() exp_md = { 'Content-Length': 4, 'ETag': etag, 'X-Timestamp': ts, 'X-Object-PUT-Mtime': normalize_timestamp(stats.st_mtime), 'Content-Type': 'application/octet-stream'} gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._fd is None assert gdf._disk_file_open is False assert gdf._metadata is None assert not gdf._is_dir with gdf.open(): assert gdf._data_file == the_file assert not gdf._is_dir assert gdf._fd is not None assert gdf._metadata == exp_md assert gdf._disk_file_open is True assert gdf._disk_file_open is False
def test_open_no_metadata(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") stats = os.stat(the_file) ts = normalize_timestamp(stats.st_ctime) etag = md5() etag.update("1234") etag = etag.hexdigest() exp_md = { 'Content-Length': 4, 'ETag': etag, 'X-Timestamp': ts, 'X-Object-PUT-Mtime': normalize_timestamp(stats.st_mtime), 'Content-Type': 'application/octet-stream' } gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._fd is None assert gdf._disk_file_open is False assert gdf._metadata is None assert not gdf._is_dir with gdf.open(): assert gdf._data_file == the_file assert not gdf._is_dir assert gdf._fd is not None assert gdf._metadata == exp_md assert gdf._disk_file_open is True assert gdf._disk_file_open is False
def test_delete_file_unlink_error(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._data_file == the_file assert not gdf._is_dir later = float(gdf.read_metadata()['X-Timestamp']) + 1 def _mock_os_unlink_eacces_err(f): raise OSError(errno.EACCES, os.strerror(errno.EACCES)) stats = os.stat(the_path) try: os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR)) # Handle the case os_unlink() raises an OSError with patch("os.unlink", _mock_os_unlink_eacces_err): try: gdf.delete(normalize_timestamp(later)) except OSError as e: assert e.errno == errno.EACCES else: self.fail("Excepted an OSError when unlinking file") finally: os.chmod(the_path, stats.st_mode) assert os.path.isdir(gdf._put_datadir) assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_file_unlink_error(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._data_file == the_file assert not gdf._is_dir later = float(gdf.read_metadata()['X-Timestamp']) + 1 def _mock_os_unlink_eacces_err(f): raise OSError(errno.EACCES, os.strerror(errno.EACCES)) stats = os.stat(the_path) try: os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR)) # Handle the case os_unlink() raises an OSError with patch("os.unlink", _mock_os_unlink_eacces_err): try: gdf.delete(normalize_timestamp(later)) except OSError as e: assert e.errno == errno.EACCES else: self.fail("Excepted an OSError when unlinking file") finally: os.chmod(the_path, stats.st_mode) assert os.path.isdir(gdf._datadir) assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_is_dir(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d") assert gdf._data_file == the_dir later = float(gdf.read_metadata()['X-Timestamp']) + 1 gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf._put_datadir) assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_is_dir(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d") assert gdf._data_file == the_dir later = float(gdf.read_metadata()['X-Timestamp']) + 1 gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf._datadir) assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_get_object_metadata_file(self): tf = tempfile.NamedTemporaryFile() tf.file.write('123') tf.file.flush() md = utils.get_object_metadata(tf.name) for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.FILE assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name)) assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_get_object_metadata_dir(self): td = tempfile.mkdtemp() try: md = utils.get_object_metadata(td) for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)
def test_delete_same_timestamp(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._data_file == the_file assert not gdf._is_dir now = float(gdf.read_metadata()['X-Timestamp']) gdf.delete(normalize_timestamp(now)) assert os.path.isdir(gdf._put_datadir) assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_delete_same_timestamp(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._data_file == the_file assert not gdf._is_dir now = float(gdf.read_metadata()['X-Timestamp']) gdf.delete(normalize_timestamp(now)) assert os.path.isdir(gdf._datadir) assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_read_metadata_optimize_open_close(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") init_md = { 'X-Type': 'Object', 'X-Object-Type': 'file', 'Content-Length': 4, 'ETag': md5("1234").hexdigest(), 'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime), 'Content-Type': 'application/octet-stream' } _metadata[_mapit(the_file)] = init_md gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._fd is None assert gdf._disk_file_open is False assert gdf._metadata is None assert not gdf._is_dir # Case 1 # Ensure that reading metadata for non-GET requests # does not incur opening and closing the file when # metadata is NOT stale. mock_open = Mock() mock_close = Mock() with mock.patch("swiftonfile.swift.obj.diskfile.do_open", mock_open): with mock.patch("swiftonfile.swift.obj.diskfile.do_close", mock_close): md = gdf.read_metadata() self.assertEqual(md, init_md) self.assertFalse(mock_open.called) self.assertFalse(mock_close.called) # Case 2 # Ensure that reading metadata for non-GET requests # still opens and reads the file when metadata is stale with open(the_file, "a") as fd: # Append to the existing file to make the stored metadata # invalid/stale. fd.write("5678") md = gdf.read_metadata() # Check that the stale metadata is recalculated to account for # change in file content self.assertNotEqual(md, init_md) self.assertEqual(md['Content-Length'], 8) self.assertEqual(md['ETag'], md5("12345678").hexdigest())
def test_read_metadata_optimize_open_close(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") init_md = { 'X-Type': 'Object', 'X-Object-Type': 'file', 'Content-Length': 4, 'ETag': md5("1234").hexdigest(), 'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime), 'Content-Type': 'application/octet-stream'} _metadata[_mapit(the_file)] = init_md gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._fd is None assert gdf._disk_file_open is False assert gdf._metadata is None assert not gdf._is_dir # Case 1 # Ensure that reading metadata for non-GET requests # does not incur opening and closing the file when # metadata is NOT stale. mock_open = Mock() mock_close = Mock() with mock.patch("swiftonfile.swift.obj.diskfile.do_open", mock_open): with mock.patch("swiftonfile.swift.obj.diskfile.do_close", mock_close): md = gdf.read_metadata() self.assertEqual(md, init_md) self.assertFalse(mock_open.called) self.assertFalse(mock_close.called) # Case 2 # Ensure that reading metadata for non-GET requests # still opens and reads the file when metadata is stale with open(the_file, "a") as fd: # Append to the existing file to make the stored metadata # invalid/stale. fd.write("5678") md = gdf.read_metadata() # Check that the stale metadata is recalculated to account for # change in file content self.assertNotEqual(md, init_md) self.assertEqual(md['Content-Length'], 8) self.assertEqual(md['ETag'], md5("12345678").hexdigest())
def test_delete_file_not_found(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._data_file == the_file assert not gdf._is_dir later = float(gdf.read_metadata()['X-Timestamp']) + 1 # Handle the case the file is not in the directory listing. os.unlink(the_file) gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf._datadir) assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
def test_delete_file_not_found(self): the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: fd.write("1234") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._data_file == the_file assert not gdf._is_dir later = float(gdf.read_metadata()['X-Timestamp']) + 1 # Handle the case the file is not in the directory listing. os.unlink(the_file) gdf.delete(normalize_timestamp(later)) assert os.path.isdir(gdf._put_datadir) assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))
def test_create_object_metadata_file(self): tf = tempfile.NamedTemporaryFile() tf.file.write('4567') tf.file.flush() r_md = utils.create_object_metadata(tf.name) xkey = _xkey(tf.name, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['set'] == 1 md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.FILE assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name)) assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_file(self): tf = tempfile.NamedTemporaryFile() tf.file.write('4567') tf.file.flush() r_md = utils.create_object_metadata(tf.name) xkey = _xkey(tf.name, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.FILE assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name) assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name)) assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self): td = tempfile.mkdtemp() try: r_md = utils.create_object_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['set'] == 1 md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)
def test_create_object_metadata_dir(self): td = tempfile.mkdtemp() try: r_md = utils.create_object_metadata(td) xkey = _xkey(td, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 md = pickle.loads(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: assert key in md, "Expected key %s in %r" % (key, md) assert md[utils.X_TYPE] == utils.OBJECT assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE assert md[utils.X_CONTENT_LENGTH] == 0 assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td)) assert md[utils.X_ETAG] == hashlib.md5().hexdigest() finally: os.rmdir(td)