def test_write_metadata_etag(self):
# don't store fake etag
path = "/tmp/foo/cont/obj"
md_file = os.path.join('/tmp/foo/cont', self.mp.meta_dir,
'obj', self.mp.obj_meta)
orig_d = {'bar': 'foo', 'ETag': 'fcetag'}
self.mp.write_metadata(path, orig_d)
try:
with open(md_file, 'rt') as f:
md = utils.deserialize_metadata(f.read())
except Exception as e:
self.fail(e)
# but store a real etag
expected_md = {'bar': 'foo'}
self.assertEqual(expected_md, md)
orig_d = {'bar': 'foo', 'ETag': 'realetag'}
self.mp.write_metadata(path, orig_d)
try:
with open(md_file, 'rt') as f:
md = utils.deserialize_metadata(f.read())
except Exception as e:
self.fail(e)
expected_md = {'bar': 'foo', 'ETag': 'realetag'}
self.assertEqual(expected_md, md)
def test_deserialize_metadata_json(self):
orig_md = {'key1': 'value1', 'key2': 'value2'}
json_md = json.dumps(orig_md, separators=(',', ':'))
_m_json_loads = Mock(return_value={})
with patch('file_connector.swift.common.utils.json.loads',
_m_json_loads):
utils.deserialize_metadata("not_json")
self.assertFalse(_m_json_loads.called)
_m_json_loads.reset_mock()
utils.deserialize_metadata("{fake_valid_json}")
self.assertTrue(_m_json_loads.called)
_m_json_loads.reset_mock()
def test_create_account_metadata(self):
td = tempfile.mkdtemp()
try:
r_md = self.mp.create_account_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['get'] == 1
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.acct_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == (utils.ACCOUNT, 0)
assert md[utils.X_TIMESTAMP] == (utils.normalize_timestamp(
os.path.getctime(td)), 0)
assert md[utils.X_PUT_TIMESTAMP] == (utils.normalize_timestamp(
os.path.getmtime(td)), 0)
assert md[utils.X_OBJECTS_COUNT] == (0, 0)
assert md[utils.X_BYTES_USED] == (0, 0)
assert md[utils.X_CONTAINER_COUNT] == (0, 0)
finally:
os.rmdir(td)
def test_write_metadata(self):
path = "/tmp/foo/w"
orig_d = {'bar': 'foo'}
self.mp.write_metadata(path, orig_d)
xkey = _xkey(path, utils.METADATA_KEY)
self.assertEqual(1, len(_xattrs))
self.assertIn(xkey, _xattrs)
self.assertEqual(orig_d, deserialize_metadata(_xattrs[xkey]))
self.assertEqual(_xattr_op_cnt['set'], 1)
def test_write_metadata(self):
path = os.path.join(self.td, 'cont/obj')
md_file = os.path.join(self.td, 'cont', self.mp.meta_dir,
'obj', self.mp.obj_meta)
orig_d = {'bar': 'foo'}
self.mp.write_metadata(path, orig_d)
try:
with open(md_file, 'rt') as f:
md = utils.deserialize_metadata(f.read())
except Exception as e:
self.fail(e)
expected_md = {'bar': 'foo'}
self.assertEqual(expected_md, md)
def test_write_metadata_multiple(self):
# At 64 KB an xattr key/value pair, this should generate three keys.
path = "/tmp/foo/w"
orig_d = {'bar': 'x' * 150000}
self.mp.write_metadata(path, orig_d)
self.assertEqual(len(_xattrs.keys()), 3,
"Expected 3 keys, found %d" % len(_xattrs.keys()))
payload = ''
for i in range(0, 3):
xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or ''))
assert xkey in _xattrs
assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE
payload += _xattrs[xkey]
assert orig_d == deserialize_metadata(payload)
assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt
def test_deserialize_metadata_pickle(self):
orig__read_pickled_metadata = utils._read_pickled_metadata
orig_md = {'key1': 'value1', 'key2': 'value2'}
pickled_md = pickle.dumps(orig_md, PICKLE_PROTOCOL)
_m_pickle_loads = Mock(return_value={})
try:
with patch('file_connector.swift.common.utils.pickle.loads',
_m_pickle_loads):
# Conf option turned off
utils._read_pickled_metadata = False
# pickled
utils.deserialize_metadata(pickled_md)
self.assertFalse(_m_pickle_loads.called)
_m_pickle_loads.reset_mock()
# not pickled
utils.deserialize_metadata("not_pickle")
self.assertFalse(_m_pickle_loads.called)
_m_pickle_loads.reset_mock()
# Conf option turned on
utils._read_pickled_metadata = True
# pickled
md = utils.deserialize_metadata(pickled_md)
self.assertTrue(_m_pickle_loads.called)
self.assertTrue(isinstance(md, dict))
_m_pickle_loads.reset_mock()
# not pickled
utils.deserialize_metadata("not_pickle")
self.assertFalse(_m_pickle_loads.called)
_m_pickle_loads.reset_mock()
# malformed pickle
_m_pickle_loads.side_effect = pickle.UnpicklingError
md = utils.deserialize_metadata("malformed_pickle")
self.assertTrue(isinstance(md, dict))
finally:
utils._read_pickled_metadata = orig__read_pickled_metadata
def test_create_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('4567')
tf.file.flush()
r_md = self.mp.create_object_metadata(tf.name)
xkey = _xkey(tf.name, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
self.assertEqual(r_md, md)
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(
os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = self.mp.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(
os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)