def test_deserialize_metadata_pickle(self):
orig_md = {'key1': 'value1', 'key2': 'value2'}
pickled_md = pickle.dumps(orig_md, PICKLE_PROTOCOL)
_m_pickle_loads = Mock(return_value={})
utils.read_pickled_metadata = True
with patch('swiftonfile.swift.common.utils.pickle.loads',
_m_pickle_loads):
# pickled
md = utils.deserialize_metadata(pickled_md)
self.assertTrue(_m_pickle_loads.called)
self.assertTrue(isinstance(md, dict))
_m_pickle_loads.reset_mock()
# not pickled
utils.deserialize_metadata("not_pickle")
self.assertFalse(_m_pickle_loads.called)
_m_pickle_loads.reset_mock()
# pickled but conf does not allow loading
utils.read_pickled_metadata = False
md = utils.deserialize_metadata(pickled_md)
self.assertFalse(_m_pickle_loads.called)
# malformed pickle
_m_pickle_loads.side_effect = pickle.UnpicklingError
md = utils.deserialize_metadata("malformed_pickle")
self.assertTrue(isinstance(md, dict))
def test_deserialize_metadata_pickle(self):
orig_md = {'key1': 'value1', 'key2': 'value2'}
pickled_md = pickle.dumps(orig_md, PICKLE_PROTOCOL)
_m_pickle_loads = Mock(return_value={})
utils.read_pickled_metadata = True
with patch('swiftonfile.swift.common.utils.pickle.loads',
_m_pickle_loads):
# pickled
md = utils.deserialize_metadata(pickled_md)
self.assertTrue(_m_pickle_loads.called)
self.assertTrue(isinstance(md, dict))
_m_pickle_loads.reset_mock()
# not pickled
utils.deserialize_metadata("not_pickle")
self.assertFalse(_m_pickle_loads.called)
_m_pickle_loads.reset_mock()
# pickled but conf does not allow loading
utils.read_pickled_metadata = False
md = utils.deserialize_metadata(pickled_md)
self.assertFalse(_m_pickle_loads.called)
# malformed pickle
_m_pickle_loads.side_effect = pickle.UnpicklingError
md = utils.deserialize_metadata("malformed_pickle")
self.assertTrue(isinstance(md, dict))
def test_deserialize_metadata_json(self):
_m_json_loads = Mock(return_value={})
with patch('swiftonfile.swift.common.utils.json.loads', _m_json_loads):
utils.deserialize_metadata("not_json")
self.assertFalse(_m_json_loads.called)
_m_json_loads.reset_mock()
utils.deserialize_metadata("{fake_valid_json}")
self.assertTrue(_m_json_loads.called)
def test_deserialize_metadata_json(self):
_m_json_loads = Mock(return_value={})
with patch('swiftonfile.swift.common.utils.json.loads',
_m_json_loads):
utils.deserialize_metadata("not_json")
self.assertFalse(_m_json_loads.called)
_m_json_loads.reset_mock()
utils.deserialize_metadata("{fake_valid_json}")
self.assertTrue(_m_json_loads.called)
def test_write_metadata(self):
path = "/tmp/foo/w"
orig_d = {'bar': 'foo'}
utils.write_metadata(path, orig_d)
xkey = _xkey(path, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert orig_d == deserialize_metadata(_xattrs[xkey])
assert _xattr_op_cnt['set'] == 1
def test_write_metadata(self):
path = "/tmp/foo/w"
orig_d = {'bar': 'foo'}
utils.write_metadata(path, orig_d)
xkey = _xkey(path, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert orig_d == deserialize_metadata(_xattrs[xkey])
assert _xattr_op_cnt['set'] == 1
def test_write_metadata_multiple(self):
# At 64 KB an xattr key/value pair, this should generate three keys.
path = "/tmp/foo/w"
orig_d = {'bar': 'x' * 150000}
utils.write_metadata(path, orig_d)
assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(_xattrs.keys())
payload = ''
for i in range(0, 3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
assert xkey in _xattrs
assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE
payload += _xattrs[xkey]
assert orig_d == deserialize_metadata(payload)
assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt
def test_write_metadata_multiple(self):
# At 64 KB an xattr key/value pair, this should generate three keys.
path = "/tmp/foo/w"
orig_d = {'bar': 'x' * 150000}
utils.write_metadata(path, orig_d)
assert len(_xattrs.keys()) == 3, "Expected 3 keys, found %d" % len(
_xattrs.keys())
payload = ''
for i in range(0, 3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
assert xkey in _xattrs
assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE
payload += _xattrs[xkey]
assert orig_d == deserialize_metadata(payload)
assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt
def test_create_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('4567')
tf.file.flush()
r_md = utils.create_object_metadata(tf.name)
xkey = _xkey(tf.name, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = utils.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)
def test_create_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('4567')
tf.file.flush()
r_md = utils.create_object_metadata(tf.name)
xkey = _xkey(tf.name, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(
os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = utils.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(
os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)