def test_GET_DLO_outside_container(self):
seg1 = self.env.container.file(
"get-dlo-outside-seg1" + Utils.create_name())
seg2 = self.env.container.file(
"get-dlo-outside-seg2" + Utils.create_name())
seg1.write("one fish two fish ")
seg2.write("red fish blue fish")
container2 = self.env.account.container(Utils.create_name())
container2.create()
manifest = container2.file("manifest" + Utils.create_name())
manifest.write(
'',
hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
(self.env.container.name,)})
expires = int(time()) + 86400
parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(manifest.path),
self.env.tempurl_key)
# cross container tempurl works fine for account tempurl key
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
self.assertEqual(contents, "one fish two fish red fish blue fish")
self.assert_status([200])
def test_GET_DLO_outside_container(self):
container2 = self.env.account.container(Utils.create_name())
container2.create()
seg1 = container2.file("get-dlo-outside-seg1" + Utils.create_name())
seg2 = container2.file("get-dlo-outside-seg2" + Utils.create_name())
seg1.write("one fish two fish ")
seg2.write("red fish blue fish")
manifest = self.env.container.file("manifest" + Utils.create_name())
manifest.write('',
hdrs={
"X-Object-Manifest":
"%s/get-dlo-outside-seg" % (container2.name, )
})
expires = int(time()) + 86400
sig = self.tempurl_sig('GET', expires,
self.env.conn.make_path(manifest.path),
self.env.tempurl_key)
parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)}
# cross container tempurl does not work for container tempurl key
try:
manifest.read(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
try:
manifest.info(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.account.update_metadata({
'temp-url-key': cls.tempurl_key,
'temp-url-key-2': cls.tempurl_key2
})
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write("obj contents")
cls.other_obj = cls.container.file(Utils.create_name())
cls.other_obj.write("other obj contents")
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.account.update_metadata({
'temp-url-key': cls.tempurl_key,
'temp-url-key-2': cls.tempurl_key2
})
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write("obj contents")
cls.other_obj = cls.container.file(Utils.create_name())
cls.other_obj.write("other obj contents")
def setUp(cls):
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
super(TestContainerTempurlEnv, cls).setUp()
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
# creating another account and connection
# for ACL tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account2 = Account(
cls.conn2, config2.get('account', config2['username']))
cls.account2 = cls.conn2.get_account()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create({
'x-container-meta-temp-url-key': cls.tempurl_key,
'x-container-meta-temp-url-key-2': cls.tempurl_key2,
'x-container-read': cls.account2.name}):
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write("obj contents")
cls.other_obj = cls.container.file(Utils.create_name())
cls.other_obj.write("other obj contents")
def test_GET_DLO_outside_container(self):
seg1 = self.env.container.file("get-dlo-outside-seg1" +
Utils.create_name())
seg2 = self.env.container.file("get-dlo-outside-seg2" +
Utils.create_name())
seg1.write("one fish two fish ")
seg2.write("red fish blue fish")
container2 = self.env.account.container(Utils.create_name())
container2.create()
manifest = container2.file("manifest" + Utils.create_name())
manifest.write('',
hdrs={
"X-Object-Manifest":
"%s/get-dlo-outside-seg" %
(self.env.container.name, )
})
expires = int(time()) + 86400
parms = self.tempurl_parms('GET', expires,
self.env.conn.make_path(manifest.path),
self.env.tempurl_key)
# cross container tempurl works fine for account tempurl key
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
self.assertEqual(contents, "one fish two fish red fish blue fish")
self.assert_status([200])
def setUp(self):
if 'slo' not in cluster_info:
raise SkipTest("SLO not enabled")
self.conn = Connection(tf.config)
self.conn.authenticate()
self.account = Account(self.conn,
tf.config.get('account', tf.config['username']))
self.account.delete_containers()
# create a container with versioning
self.versions_container = self.account.container(Utils.create_name())
self.container = self.account.container(Utils.create_name())
self.segments_container = self.account.container(Utils.create_name())
if not self.container.create(
hdrs={'X-Versions-Location': self.versions_container.name}):
raise ResponseError(self.conn.response)
if 'versions' not in self.container.info():
raise SkipTest("Object versioning not enabled")
for cont in (self.versions_container, self.segments_container):
if not cont.create():
raise ResponseError(self.conn.response)
# create some segments
self.seg_info = {}
for letter, size in (('a', 1024 * 1024), ('b', 1024 * 1024)):
seg_name = letter
file_item = self.segments_container.file(seg_name)
file_item.write(letter * size)
self.seg_info[seg_name] = {
'size_bytes': size,
'etag': file_item.md5,
'path': '/%s/%s' % (self.segments_container.name, seg_name)
}
def setUp(cls):
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
super(TestContainerTempurlEnv, cls).setUp()
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
# creating another account and connection
# for ACL tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account2 = Account(cls.conn2,
config2.get('account', config2['username']))
cls.account2 = cls.conn2.get_account()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create({
'x-container-meta-temp-url-key': cls.tempurl_key,
'x-container-meta-temp-url-key-2': cls.tempurl_key2,
'x-container-read': cls.account2.name
}):
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write("obj contents")
cls.other_obj = cls.container.file(Utils.create_name())
cls.other_obj.write("other obj contents")
def setUp(self):
if 'slo' not in cluster_info:
raise SkipTest("SLO not enabled")
self.conn = Connection(tf.config)
self.conn.authenticate()
self.account = Account(
self.conn, tf.config.get('account', tf.config['username']))
self.account.delete_containers()
# create a container with versioning
self.versions_container = self.account.container(Utils.create_name())
self.container = self.account.container(Utils.create_name())
self.segments_container = self.account.container(Utils.create_name())
if not self.container.create(
hdrs={'X-Versions-Location': self.versions_container.name}):
raise ResponseError(self.conn.response)
if 'versions' not in self.container.info():
raise SkipTest("Object versioning not enabled")
for cont in (self.versions_container, self.segments_container):
if not cont.create():
raise ResponseError(self.conn.response)
# create some segments
self.seg_info = {}
for letter, size in (('a', 1024 * 1024),
('b', 1024 * 1024)):
seg_name = letter
file_item = self.segments_container.file(seg_name)
file_item.write(letter * size)
self.seg_info[seg_name] = {
'size_bytes': size,
'etag': file_item.md5,
'path': '/%s/%s' % (self.segments_container.name, seg_name)}
def setUp(cls):
super(TestDloEnv, cls).setUp()
config2 = tf.config.copy()
config2['username'] = tf.config['username3']
config2['password'] = tf.config['password3']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.container = cls.account.container(Utils.create_name())
cls.container2 = cls.account.container(Utils.create_name())
for cont in (cls.container, cls.container2):
if not cont.create():
raise ResponseError(cls.conn.response)
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
cls.segment_prefix = prefix
for letter in ('a', 'b', 'c', 'd', 'e'):
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
file_item.write(letter * 10)
file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
file_item.write(letter.upper() * 10)
for letter in ('f', 'g', 'h', 'i', 'j'):
file_item = cls.container2.file("%s/seg_lower%s" %
(prefix, letter))
file_item.write(letter * 10)
man1 = cls.container.file("man1")
man1.write('man1-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg_lower" % (cls.container.name, prefix)
})
man2 = cls.container.file("man2")
man2.write('man2-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg_upper" % (cls.container.name, prefix)
})
manall = cls.container.file("manall")
manall.write('manall-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg" % (cls.container.name, prefix)
})
mancont2 = cls.container.file("mancont2")
mancont2.write('mancont2-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg_lower" % (cls.container2.name, prefix)
})
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
config2 = tf.config.copy()
config2['username'] = tf.config['username3']
config2['password'] = tf.config['password3']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
cls.container2 = cls.account.container(Utils.create_name())
for cont in (cls.container, cls.container2):
if not cont.create():
raise ResponseError(cls.conn.response)
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
cls.segment_prefix = prefix
for letter in ('a', 'b', 'c', 'd', 'e'):
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
file_item.write(letter * 10)
file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
file_item.write(letter.upper() * 10)
for letter in ('f', 'g', 'h', 'i', 'j'):
file_item = cls.container2.file("%s/seg_lower%s" %
(prefix, letter))
file_item.write(letter * 10)
man1 = cls.container.file("man1")
man1.write('man1-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
(cls.container.name, prefix)})
man2 = cls.container.file("man2")
man2.write('man2-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
(cls.container.name, prefix)})
manall = cls.container.file("manall")
manall.write('manall-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg" %
(cls.container.name, prefix)})
mancont2 = cls.container.file("mancont2")
mancont2.write(
'mancont2-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
(cls.container2.name, prefix)})
def setUp(cls):
super(TestObjectVersioningEnv, cls).setUp()
if not tf.skip2:
# Second connection for ACL tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
if six.PY2:
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
else:
prefix = Utils.create_name()[:10]
cls.versions_container = cls.account.container(prefix + "-versions")
if not cls.versions_container.create():
raise ResponseError(cls.conn.response)
cls.container = cls.account.container(prefix + "-objs")
container_headers = {
cls.location_header_key: quote(cls.versions_container.name)
}
if not cls.container.create(hdrs=container_headers):
if cls.conn.response.status == 412:
cls.versioning_enabled = False
return
raise ResponseError(cls.conn.response)
container_info = cls.container.info()
# if versioning is off, then cls.location_header_key won't persist
cls.versioning_enabled = 'versions' in container_info
if not tf.skip2:
# setup another account to test ACLs
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate()
cls.account2 = cls.conn2.get_account()
cls.account2.delete_containers()
if not tf.skip3:
# setup another account with no access to anything to test ACLs
config3 = deepcopy(tf.config)
config3['account'] = tf.config['account']
config3['username'] = tf.config['username3']
config3['password'] = tf.config['password3']
cls.conn3 = Connection(config3)
cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate()
cls.account3 = cls.conn3.get_account()
def setUp(cls):
super(TestDloEnv, cls).setUp()
cls.container = cls.account.container(Utils.create_name())
cls.container2 = cls.account.container(Utils.create_name())
for cont in (cls.container, cls.container2):
if not cont.create():
raise ResponseError(cls.conn.response)
prefix = Utils.create_name(10)
cls.segment_prefix = prefix
for letter in ('a', 'b', 'c', 'd', 'e'):
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
file_item.write(letter.encode('ascii') * 10)
file_item = cls.container.file("%s/seg_upper_%%ff%s" %
(prefix, letter))
file_item.write(letter.upper().encode('ascii') * 10)
for letter in ('f', 'g', 'h', 'i', 'j'):
file_item = cls.container2.file("%s/seg_lower%s" %
(prefix, letter))
file_item.write(letter.encode('ascii') * 10)
man1 = cls.container.file("man1")
man1.write(b'man1-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg_lower" % (cls.container.name, prefix)
})
man2 = cls.container.file("man2")
man2.write(b'man2-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg_upper_%%25ff" % (cls.container.name, prefix)
})
manall = cls.container.file("manall")
manall.write(b'manall-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg" % (cls.container.name, prefix)
})
mancont2 = cls.container.file("mancont2")
mancont2.write(b'mancont2-contents',
hdrs={
"X-Object-Manifest":
"%s/%s/seg_lower" % (cls.container2.name, prefix)
})
def test_PUT_manifest_access(self):
new_obj = self.env.container.file(Utils.create_name())
# give out a signature which allows a PUT to new_obj
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'PUT', expires, self.env.conn.make_path(new_obj.path),
self.env.tempurl_key)
put_parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
# try to create manifest pointing to some random container
try:
new_obj.write('', {
'x-object-manifest': '%s/foo' % 'some_random_container'
}, parms=put_parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
# create some other container
other_container = self.env.account.container(Utils.create_name())
if not other_container.create():
raise ResponseError(self.conn.response)
# try to create manifest pointing to new container
try:
new_obj.write('', {
'x-object-manifest': '%s/foo' % other_container
}, parms=put_parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
# try again using a tempurl POST to an already created object
new_obj.write('', {}, parms=put_parms, cfg={'no_auth_token': True})
expires = int(time.time()) + 86400
sig = self.tempurl_sig(
'POST', expires, self.env.conn.make_path(new_obj.path),
self.env.tempurl_key)
post_parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
try:
new_obj.post({'x-object-manifest': '%s/foo' % other_container},
parms=post_parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 400)
else:
self.fail('request did not error')
def setUp(cls):
super(TestDloEnv, cls).setUp()
cls.container = cls.account.container(Utils.create_name())
cls.container2 = cls.account.container(Utils.create_name())
for cont in (cls.container, cls.container2):
if not cont.create():
raise ResponseError(cls.conn.response)
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
cls.segment_prefix = prefix
for letter in ('a', 'b', 'c', 'd', 'e'):
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
file_item.write(letter * 10)
file_item = cls.container.file(
"%s/seg_upper_%%ff%s" % (prefix, letter))
file_item.write(letter.upper() * 10)
for letter in ('f', 'g', 'h', 'i', 'j'):
file_item = cls.container2.file("%s/seg_lower%s" %
(prefix, letter))
file_item.write(letter * 10)
man1 = cls.container.file("man1")
man1.write('man1-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
(cls.container.name, prefix)})
man2 = cls.container.file("man2")
man2.write('man2-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
(cls.container.name, prefix)})
manall = cls.container.file("manall")
manall.write('manall-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg" %
(cls.container.name, prefix)})
mancont2 = cls.container.file("mancont2")
mancont2.write(
'mancont2-contents',
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
(cls.container2.name, prefix)})
def test_versioning_dlo(self):
container = self.env.container
versions_container = self.env.versions_container
obj_name = Utils.create_name()
for i in ('1', '2', '3'):
time.sleep(.01) # guarantee that the timestamp changes
obj_name_seg = obj_name + '/' + i
versioned_obj = container.file(obj_name_seg)
versioned_obj.write(i)
versioned_obj.write(i + i)
self.assertEqual(3, versions_container.info()['object_count'])
man_file = container.file(obj_name)
man_file.write('',
hdrs={
"X-Object-Manifest":
"%s/%s/" % (self.env.container.name, obj_name)
})
# guarantee that the timestamp changes
time.sleep(.01)
# write manifest file again
man_file.write('',
hdrs={
"X-Object-Manifest":
"%s/%s/" % (self.env.container.name, obj_name)
})
self.assertEqual(3, versions_container.info()['object_count'])
self.assertEqual("112233", man_file.read())
def testObjectMetadataWhenFileModified(self):
data = "I'm whatever Gotham needs me to be "
data_hash = hashlib.md5(data).hexdigest()
# Create an object through object interface
object_name = Utils.create_name()
object_item = self.env.container.file(object_name)
object_item.write(data)
# Make sure GET works
self.assertEqual(data, object_item.read())
self.assert_status(200)
# Check Etag is right
self.assertEqual(data_hash, object_item.info()['etag'])
self.assert_status(200)
# Extend/append more data to file from filesystem interface
file_path = os.path.join(self.env.root_dir,
'AUTH_' + self.env.account.name,
self.env.container.name,
object_name)
more_data = "- Batman"
with open(file_path, 'a') as f:
f.write(more_data)
total_data = data + more_data
total_data_hash = hashlib.md5(total_data).hexdigest()
# Make sure GET works
self.assertEqual(total_data, object_item.read())
self.assert_status(200)
# Check Etag and content-length is right
metadata = object_item.info()
self.assert_status(200)
self.assertEqual(total_data_hash, metadata['etag'])
self.assertEqual(len(total_data), int(metadata['content_length']))
# Re-write the file to be shorter
new_data = "I am Batman"
new_data_hash = hashlib.md5(new_data).hexdigest()
with open(file_path, 'w') as f:
f.write(new_data)
# Make sure GET works
self.assertEqual(new_data, object_item.read())
self.assert_status(200)
# Check Etag and content-length is right
metadata = object_item.info()
self.assert_status(200)
self.assertEqual(new_data_hash, metadata['etag'])
self.assertEqual(len(new_data), int(metadata['content_length']))
# Modify the file but let the length remain same
new_data = "I am Antman"
new_data_hash = hashlib.md5(new_data).hexdigest()
with open(file_path, 'w') as f:
f.write(new_data)
# Make sure GET works
self.assertEqual(new_data, object_item.read())
self.assert_status(200)
# Check Etag and content-length is right
metadata = object_item.info()
self.assert_status(200)
self.assertEqual(new_data_hash, metadata['etag'])
self.assertEqual(len(new_data), int(metadata['content_length']))
def test_PUT_large_object(self):
file_item = self.env.container.file(Utils.create_name())
data = File.random_data(1024 * 1024 * 2)
self.assertTrue(file_item.write(data))
self.assert_status(201)
self.assertTrue(data == file_item.read())
self.assert_status(200)
def test_versioning_dlo(self):
container = self.env.container
versions_container = self.env.versions_container
obj_name = Utils.create_name()
for i in ('1', '2', '3'):
time.sleep(.01) # guarantee that the timestamp changes
obj_name_seg = obj_name + '/' + i
versioned_obj = container.file(obj_name_seg)
versioned_obj.write(i)
versioned_obj.write(i + i)
self.assertEqual(3, versions_container.info()['object_count'])
man_file = container.file(obj_name)
man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
(self.env.container.name, obj_name)})
# guarantee that the timestamp changes
time.sleep(.01)
# write manifest file again
man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
(self.env.container.name, obj_name)})
self.assertEqual(3, versions_container.info()['object_count'])
self.assertEqual("112233", man_file.read())
def test_dlo_post_with_manifest_regular_object(self):
# verify that performing a POST to a regular object
# with a manifest header will create a DLO.
# Put a regular object
file_item = self.env.container.file(Utils.create_name())
file_item.write('file contents', hdrs={})
# sanity checks
file_contents = file_item.read(parms={})
self.assertEqual('file contents', file_contents)
# get the path associated with man1
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
# POST a x-object-manifest value to the regular object
file_item.post({'x-object-manifest': x_o_m})
# verify that the file is now a manifest
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('file contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
file_item.info()
resp_headers = file_item.conn.response.getheaders()
self.assertIn(('x-object-manifest', x_o_m), resp_headers)
def test_dlo_post_without_manifest_header(self):
# verify that a POST to a DLO manifest object with no
# x-object-manifest header will cause the existing x-object-manifest
# header to be lost
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST with no x-object-manifest header
file_item.post({})
# verify that existing x-object-manifest was removed
info = file_item.info()
self.assertNotIn('x_object_manifest', info)
# verify that object content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
# verify that object is no longer a manifest
contents = file_item.read(parms={})
self.assertEqual('manifest-contents', contents)
def test_versioning_check_acl(self):
container = self.env.container
versions_container = self.env.versions_container
versions_container.create(hdrs={'X-Container-Read': '.r:*,.rlistings'})
obj_name = Utils.create_name()
versioned_obj = container.file(obj_name)
versioned_obj.write("aaaaa")
self.assertEqual("aaaaa", versioned_obj.read())
versioned_obj.write("bbbbb")
self.assertEqual("bbbbb", versioned_obj.read())
# Use token from second account and try to delete the object
org_token = self.env.account.conn.storage_token
self.env.account.conn.storage_token = self.env.conn2.storage_token
try:
with self.assertRaises(ResponseError) as cm:
versioned_obj.delete()
self.assertEqual(403, cm.exception.status)
finally:
self.env.account.conn.storage_token = org_token
# Verify with token from first account
self.assertEqual("bbbbb", versioned_obj.read())
versioned_obj.delete()
self.assertEqual("aaaaa", versioned_obj.read())
def _test_versioning_check_acl_setup(self):
container = self.env.container
versions_container = self.env.versions_container
versions_container.create(hdrs={'X-Container-Read': '.r:*,.rlistings'})
obj_name = Utils.create_name()
versioned_obj = container.file(obj_name)
versioned_obj.write("aaaaa")
self.assertEqual("aaaaa", versioned_obj.read())
versioned_obj.write("bbbbb")
self.assertEqual("bbbbb", versioned_obj.read())
# Use token from second account and try to delete the object
org_token = self.env.account.conn.storage_token
self.env.account.conn.storage_token = self.env.conn2.storage_token
try:
with self.assertRaises(ResponseError) as cm:
versioned_obj.delete()
self.assertEqual(403, cm.exception.status)
finally:
self.env.account.conn.storage_token = org_token
# Verify with token from first account
self.assertEqual("bbbbb", versioned_obj.read())
return versioned_obj
def test_object_expiry_X_Delete_At_PUT(self):
obj = self.env.container.file(Utils.create_name())
x_delete_at = str(int(time.time()) + 2)
obj.write_random(self.env.file_size,
hdrs={'X-Delete-At': x_delete_at})
# Object is not expired. Should still be accessible.
obj.read()
self.assert_status(200)
# Ensure X-Delete-At is saved as object metadata.
self.assertEqual(x_delete_at, str(obj.info()['x_delete_at']))
# Wait for object to be expired.
time.sleep(3)
# Object has expired. Should no longer be accessible.
self.assertRaises(ResponseError, obj.read)
self.assert_status(404)
# Object should still be present on filesystem.
self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
self.env.container.name,
obj.name)))
# But, GET on container should list the expired object.
result = self.env.container.files()
self.assertTrue(obj.name in self.env.container.files())
# Check existence of corresponding tracker object in gsexpiring
# account.
enteredLoop = False
for c in self.env.client.iter_containers("gsexpiring"):
for o in self.env.client.iter_objects("gsexpiring", c['name']):
enteredLoop = True
l = o['name'].split('/')
self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
self.assertEqual(l[1], self.env.container.name)
self.assertEqual(l[2], obj.name)
if not enteredLoop:
self.fail("Tracker object not found.")
# Run expirer daemon once.
self.env.expirer.once()
# Ensure object is physically deleted from filesystem.
self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
self.env.container.name,
obj.name)))
# Ensure tracker object is consumed.
try:
self.env.client.iter_containers("gsexpiring").next()
except StopIteration:
pass
else:
self.fail("Tracker object persists!")
# GET on container should no longer list the object.
self.assertFalse(obj.name in self.env.container.files())
def test_dlo_post_with_manifest_regular_object(self):
# verify that performing a POST to a regular object
# with a manifest header will create a DLO.
# Put a regular object
file_item = self.env.container.file(Utils.create_name())
file_item.write(b'file contents', hdrs={})
# sanity checks
file_contents = file_item.read(parms={})
self.assertEqual(b'file contents', file_contents)
# get the path associated with man1
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
# POST a x-object-manifest value to the regular object
file_item.post({'x-object-manifest': x_o_m})
# verify that the file is now a manifest
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'file contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde']).encode()
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
file_item.info()
resp_headers = [(h.lower(), v)
for h, v in file_item.conn.response.getheaders()]
self.assertIn(('x-object-manifest', str_to_wsgi(x_o_m)), resp_headers)
def test_dlo_post_without_manifest_header(self):
# verify that a POST to a DLO manifest object with no
# x-object-manifest header will cause the existing x-object-manifest
# header to be lost
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write(b'manifest-contents',
hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST with no x-object-manifest header
file_item.post({})
# verify that existing x-object-manifest was removed
info = file_item.info()
self.assertNotIn('x_object_manifest', info)
# verify that object content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that object is no longer a manifest
contents = file_item.read(parms={})
self.assertEqual(b'manifest-contents', contents)
def test_dlo_post_with_manifest_header(self):
# verify that performing a POST to a DLO manifest
# preserves the fact that it is a manifest file.
# verify that the x-object-manifest header may be updated.
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write(b'manifest-contents',
hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST a modified x-object-manifest value
new_x_o_m = x_o_m.rstrip('lower') + 'upper'
file_item.post({
'x-object-meta-foo': 'bar',
'x-object-manifest': new_x_o_m
})
# verify that x-object-manifest was updated
file_item.info()
resp_headers = [(h.lower(), v)
for h, v in file_item.conn.response.getheaders()]
self.assertIn(('x-object-manifest', str_to_wsgi(new_x_o_m)),
resp_headers)
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that updated manifest points to new content
expected_contents = ''.join((c * 10) for c in 'ABCDE').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# Now revert the manifest to point to original segments, including a
# multipart-manifest=get param just to check that has no effect
file_item.post({'x-object-manifest': x_o_m},
parms={'multipart-manifest': 'get'})
# verify that x-object-manifest was reverted
info = file_item.info()
self.assertIn('x_object_manifest', info)
self.assertEqual(x_o_m, info['x_object_manifest'])
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual(b'manifest-contents', manifest_contents)
# verify that updated manifest points new content
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
def test_overwriting_with_url_encoded_object_name(self):
versions_container = self.env.versions_container
obj_name = Utils.create_name() + '%25ff'
versioned_obj, expected_headers, expected_content_types = \
self._test_overwriting_setup(obj_name)
# test delete
# at first, delete will succeed with 204
versioned_obj.delete()
expected_content_types.append(
'application/x-deleted;swift_versions_deleted=1')
# after that, any time the delete doesn't restore the old version
# and we will get 404 NotFound
for x in range(3):
with self.assertRaises(ResponseError) as cm:
versioned_obj.delete()
self.assertEqual(404, cm.exception.status)
expected_content_types.append(
'application/x-deleted;swift_versions_deleted=1')
# finally, we have 4 versioned items and 4 delete markers total in
# the versions container
self.assertEqual(8, versions_container.info()['object_count'])
self.assertEqual(expected_content_types, [
o['content_type']
for o in versions_container.files(parms={'format': 'json'})
])
# update versioned_obj
versioned_obj.write("eeee",
hdrs={
'Content-Type': 'text/thanksgiving',
'X-Object-Meta-Bar': 'foo'
})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
self.assertEqual('text/thanksgiving', obj_info['content_type'])
# we still have delete-marker there
self.assertEqual(8, versions_container.info()['object_count'])
# update versioned_obj
versioned_obj.write("ffff",
hdrs={
'Content-Type': 'text/teriyaki',
'X-Object-Meta-Food': 'chickin'
})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
self.assertEqual('text/teriyaki', obj_info['content_type'])
# new obj will be inserted after delete-marker there
self.assertEqual(9, versions_container.info()['object_count'])
versioned_obj.delete()
with self.assertRaises(ResponseError) as cm:
versioned_obj.read()
self.assertEqual(404, cm.exception.status)
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.static_web_enabled is None:
cls.static_web_enabled = 'staticweb' in cluster_info
if not cls.static_web_enabled:
return
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create(
hdrs={'X-Container-Read': '.r:*,.rlistings'}):
raise ResponseError(cls.conn.response)
objects = ['index',
'error',
'listings_css',
'dir/',
'dir/obj',
'dir/subdir/',
'dir/subdir/obj']
cls.objects = {}
for item in sorted(objects):
parent = None
if '/' in item.rstrip('/'):
parent, _ = item.rstrip('/').rsplit('/', 1)
path = '%s/%s' % (cls.objects[parent + '/'].name,
Utils.create_name())
else:
path = Utils.create_name()
if item[-1] == '/':
cls.objects[item] = cls.container.file(path)
cls.objects[item].write(hdrs={
'Content-Type': 'application/directory'})
else:
cls.objects[item] = cls.container.file(path)
cls.objects[item].write('%s contents' % item)
def setUp(cls):
cls.conn = Connection(config)
cls.conn.authenticate()
cls.account = Account(cls.conn,
config.get('account', config['username']))
cls.root_dir = os.path.join(
'/mnt/gluster-object',
cls.account.conn.storage_url.split('/')[2].split('_')[1])
cls.account.delete_containers()
cls.file_size = 8
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.dirs = [
'dir1',
'dir2',
'dir1/subdir1',
'dir1/subdir2',
'dir1/subdir1/subsubdir1',
'dir1/subdir1/subsubdir2',
'dir1/subdir with spaces',
'dir1/subdir+with{whatever',
]
cls.files = [
'file1',
'file A',
'dir1/file2',
'dir1/subdir1/file2',
'dir1/subdir1/file3',
'dir1/subdir1/file4',
'dir1/subdir1/subsubdir1/file5',
'dir1/subdir1/subsubdir1/file6',
'dir1/subdir1/subsubdir1/file7',
'dir1/subdir1/subsubdir1/file8',
'dir1/subdir1/subsubdir2/file9',
'dir1/subdir1/subsubdir2/file0',
'dir1/subdir with spaces/file B',
'dir1/subdir+with{whatever/file D',
]
stored_files = set()
for d in cls.dirs:
file = cls.container.file(d)
file.write(hdrs={'Content-Type': 'application/directory'})
for f in cls.files:
file = cls.container.file(f)
file.write_random(
cls.file_size,
hdrs={'Content-Type': 'application/octet-stream'})
stored_files.add(f)
cls.stored_files = sorted(stored_files)
cls.sorted_objects = sorted(set(cls.dirs + cls.files))
def test_dlo_post_with_manifest_header(self):
# verify that performing a POST to a DLO manifest
# preserves the fact that it is a manifest file.
# verify that the x-object-manifest header may be updated.
# create a new manifest for this test to avoid test coupling.
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
file_item = self.env.container.file(Utils.create_name())
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
# sanity checks
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# POST a modified x-object-manifest value
new_x_o_m = x_o_m.rstrip('lower') + 'upper'
file_item.post({'x-object-meta-foo': 'bar',
'x-object-manifest': new_x_o_m})
# verify that x-object-manifest was updated
file_item.info()
resp_headers = file_item.conn.response.getheaders()
self.assertIn(('x-object-manifest', new_x_o_m), resp_headers)
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
# verify that updated manifest points to new content
expected_contents = ''.join([(c * 10) for c in 'ABCDE'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
# Now revert the manifest to point to original segments, including a
# multipart-manifest=get param just to check that has no effect
file_item.post({'x-object-manifest': x_o_m},
parms={'multipart-manifest': 'get'})
# verify that x-object-manifest was reverted
info = file_item.info()
self.assertIn('x_object_manifest', info)
self.assertEqual(x_o_m, info['x_object_manifest'])
# verify that manifest content was not changed
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
self.assertEqual('manifest-contents', manifest_contents)
# verify that updated manifest points new content
expected_contents = ''.join([(c * 10) for c in 'abcde'])
contents = file_item.read(parms={})
self.assertEqual(expected_contents, contents)
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.static_web_enabled is None:
cls.static_web_enabled = 'staticweb' in tf.cluster_info
if not cls.static_web_enabled:
return
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create(
hdrs={'X-Container-Read': '.r:*,.rlistings'}):
raise ResponseError(cls.conn.response)
objects = ['index',
'error',
'listings_css',
'dir/',
'dir/obj',
'dir/some sub%dir/',
'dir/some sub%dir/obj']
cls.objects = {}
for item in sorted(objects):
if '/' in item.rstrip('/'):
parent, _ = item.rstrip('/').rsplit('/', 1)
path = '%s/%s' % (cls.objects[parent + '/'].name,
Utils.create_name())
else:
path = Utils.create_name()
if item[-1] == '/':
cls.objects[item] = cls.container.file(path)
cls.objects[item].write(hdrs={
'Content-Type': 'application/directory'})
else:
cls.objects[item] = cls.container.file(path)
cls.objects[item].write(('%s contents' % item).encode('utf8'))
def testObjectContentFromMountPoint(self):
file_name = Utils.create_name()
file_item = self.env.container.file(file_name)
data = file_item.write_random()
self.assert_status(201)
file_info = file_item.info()
fhOnMountPoint = open(os.path.join(self.env.root_dir,self.env.container.name,file_name),'r')
data_read_from_mountP = fhOnMountPoint.read()
md5_returned = hashlib.md5(data_read_from_mountP).hexdigest()
self.assertEquals(md5_returned,file_info['etag'])
fhOnMountPoint.close()
def setUp(cls):
cls.conn = Connection(tf.config)
cls.storage_url, cls.storage_token = cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
# Second connection for ACL tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
# avoid getting a prefix that stops halfway through an encoded
# character
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
cls.versions_container = cls.account.container(prefix + "-versions")
if not cls.versions_container.create():
raise ResponseError(cls.conn.response)
cls.container = cls.account.container(prefix + "-objs")
container_headers = {
cls.location_header_key: cls.versions_container.name}
if not cls.container.create(hdrs=container_headers):
if cls.conn.response.status == 412:
cls.versioning_enabled = False
return
raise ResponseError(cls.conn.response)
container_info = cls.container.info()
# if versioning is off, then cls.location_header_key won't persist
cls.versioning_enabled = 'versions' in container_info
# setup another account to test ACLs
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate()
cls.account2 = cls.conn2.get_account()
cls.account2.delete_containers()
# setup another account with no access to anything to test ACLs
config3 = deepcopy(tf.config)
config3['account'] = tf.config['account']
config3['username'] = tf.config['username3']
config3['password'] = tf.config['password3']
cls.conn3 = Connection(config3)
cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate()
cls.account3 = cls.conn3.get_account()
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
#cls.root_dir = os.path.join('/mnt/swiftonhpss/test')
cls.root_dir = os.path.join('/srv/hpss')
cls.account.delete_containers()
cls.file_size = 8
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create(hdrs={'X-Storage-Policy': 'swiftonhpss'}):
raise ResponseError(cls.conn.response)
cls.dirs = [
'dir1',
'dir2',
'dir1/subdir1',
'dir1/subdir2',
'dir1/subdir1/subsubdir1',
'dir1/subdir1/subsubdir2',
'dir1/subdir with spaces',
'dir1/subdir+with{whatever',
]
cls.files = [
'file1',
'file A',
'dir1/file2',
'dir1/subdir1/file2',
'dir1/subdir1/file3',
'dir1/subdir1/file4',
'dir1/subdir1/subsubdir1/file5',
'dir1/subdir1/subsubdir1/file6',
'dir1/subdir1/subsubdir1/file7',
'dir1/subdir1/subsubdir1/file8',
'dir1/subdir1/subsubdir2/file9',
'dir1/subdir1/subsubdir2/file0',
'dir1/subdir with spaces/file B',
'dir1/subdir+with{whatever/file D',
]
stored_files = set()
for d in cls.dirs:
file = cls.container.file(d)
file.write(hdrs={'Content-Type': 'application/directory'})
for f in cls.files:
file = cls.container.file(f)
file.write_random(cls.file_size, hdrs={'Content-Type':
'application/octet-stream'})
stored_files.add(f)
cls.stored_files = sorted(stored_files)
cls.sorted_objects = sorted(set(cls.dirs + cls.files))
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get("account", tf.config["username"]))
# PGB - change hardcoded SoF mountpoint
# cls.root_dir = os.path.join('/mnt/swiftonhpss/test')
cls.root_dir = os.path.join("/srv/swift/hpss")
cls.account.delete_containers()
cls.file_size = 8
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create(None, None):
raise ResponseError(cls.conn.response)
cls.dirs = [
"dir1",
"dir2",
"dir1/subdir1",
"dir1/subdir2",
"dir1/subdir1/subsubdir1",
"dir1/subdir1/subsubdir2",
"dir1/subdir with spaces",
"dir1/subdir+with{whatever",
]
cls.files = [
"file1",
"file A",
"dir1/file2",
"dir1/subdir1/file2",
"dir1/subdir1/file3",
"dir1/subdir1/file4",
"dir1/subdir1/subsubdir1/file5",
"dir1/subdir1/subsubdir1/file6",
"dir1/subdir1/subsubdir1/file7",
"dir1/subdir1/subsubdir1/file8",
"dir1/subdir1/subsubdir2/file9",
"dir1/subdir1/subsubdir2/file0",
"dir1/subdir with spaces/file B",
"dir1/subdir+with{whatever/file D",
]
stored_files = set()
for d in cls.dirs:
file = cls.container.file(d)
file.write(hdrs={"Content-Type": "application/directory"})
for f in cls.files:
file = cls.container.file(f)
file.write_random(cls.file_size, hdrs={"Content-Type": "application/octet-stream"})
stored_files.add(f)
cls.stored_files = sorted(stored_files)
cls.sorted_objects = sorted(set(cls.dirs + cls.files))
def testInvalidHeadersPUT(self):
file = self.env.container.file(Utils.create_name())
self.assertRaises(ResponseError,
file.write_random,
self.env.file_size,
hdrs={'X-Delete-At': '9876545321'})
self.assert_status(400)
self.assertRaises(ResponseError,
file.write_random,
self.env.file_size,
hdrs={'X-Delete-After': '60'})
self.assert_status(400)
def test_PUT_remapped_container(self):
for domain in (self.cont_domain_dash, self.cont_domain_underscore):
new_obj_name = Utils.create_name()
path = '/'.join(['', new_obj_name])
self.env.account.conn.make_request('PUT', path,
data='new obj contents',
hdrs={'Host': domain},
cfg={'absolute_path': True})
self.assert_status(201)
new_obj = self.env.container.file(new_obj_name)
self.assertEqual(new_obj.read(), 'new obj contents')
def test_GET_DLO_inside_container(self):
seg1 = self.env.container.file(
"get-dlo-inside-seg1" + Utils.create_name())
seg2 = self.env.container.file(
"get-dlo-inside-seg2" + Utils.create_name())
seg1.write(b"one fish two fish ")
seg2.write(b"red fish blue fish")
manifest = self.env.container.file("manifest" + Utils.create_name())
manifest.write(
b'',
hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
(self.env.container.name,)})
expires = int(time()) + 86400
parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(manifest.path),
self.env.tempurl_key)
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
self.assertEqual(contents, b"one fish two fish red fish blue fish")
def test_GET_DLO_inside_container(self):
seg1 = self.env.container.file(
"get-dlo-inside-seg1" + Utils.create_name())
seg2 = self.env.container.file(
"get-dlo-inside-seg2" + Utils.create_name())
seg1.write("one fish two fish ")
seg2.write("red fish blue fish")
manifest = self.env.container.file("manifest" + Utils.create_name())
manifest.write(
'',
hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
(self.env.container.name,)})
expires = int(time.time()) + 86400
parms = self.tempurl_parms(
'GET', expires, self.env.conn.make_path(manifest.path),
self.env.tempurl_key)
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
self.assertEqual(contents, "one fish two fish red fish blue fish")
def setUp(cls):
super(TestSloTempurlEnv, cls).setUp()
if cls.enabled is None:
cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info
cls.tempurl_key = Utils.create_name()
cls.account.update_metadata({'temp-url-key': cls.tempurl_key})
cls.manifest_container = cls.account.container(Utils.create_name())
cls.segments_container = cls.account.container(Utils.create_name())
if not cls.manifest_container.create():
raise ResponseError(cls.conn.response)
if not cls.segments_container.create():
raise ResponseError(cls.conn.response)
seg1 = cls.segments_container.file(Utils.create_name())
seg1.write(b'1' * 1024 * 1024)
seg2 = cls.segments_container.file(Utils.create_name())
seg2.write(b'2' * 1024 * 1024)
cls.manifest_data = [{'size_bytes': 1024 * 1024,
'etag': seg1.md5,
'path': '/%s/%s' % (cls.segments_container.name,
seg1.name)},
{'size_bytes': 1024 * 1024,
'etag': seg2.md5,
'path': '/%s/%s' % (cls.segments_container.name,
seg2.name)}]
cls.manifest = cls.manifest_container.file(Utils.create_name())
cls.manifest.write(
json.dumps(cls.manifest_data).encode('ascii'),
parms={'multipart-manifest': 'put'})
def setUp(cls):
super(TestSloTempurlEnv, cls).setUp()
if cls.enabled is None:
cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info
cls.tempurl_key = Utils.create_name()
cls.account.update_metadata({'temp-url-key': cls.tempurl_key})
cls.manifest_container = cls.account.container(Utils.create_name())
cls.segments_container = cls.account.container(Utils.create_name())
if not cls.manifest_container.create():
raise ResponseError(cls.conn.response)
if not cls.segments_container.create():
raise ResponseError(cls.conn.response)
seg1 = cls.segments_container.file(Utils.create_name())
seg1.write('1' * 1024 * 1024)
seg2 = cls.segments_container.file(Utils.create_name())
seg2.write('2' * 1024 * 1024)
cls.manifest_data = [{'size_bytes': 1024 * 1024,
'etag': seg1.md5,
'path': '/%s/%s' % (cls.segments_container.name,
seg1.name)},
{'size_bytes': 1024 * 1024,
'etag': seg2.md5,
'path': '/%s/%s' % (cls.segments_container.name,
seg2.name)}]
cls.manifest = cls.manifest_container.file(Utils.create_name())
cls.manifest.write(
json.dumps(cls.manifest_data),
parms={'multipart-manifest': 'put'})
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.domain_remap_enabled is None:
cls.domain_remap_enabled = 'domain_remap' in cluster_info
if not cls.domain_remap_enabled:
return
cls.account = Account(
cls.conn, tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write('obj contents')
cls.obj_slash = cls.container.file('/v1')
cls.obj_slash.write('obj contents')
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
if cls.domain_remap_enabled is None:
cls.domain_remap_enabled = 'domain_remap' in cluster_info
if not cls.domain_remap_enabled:
return
cls.account = Account(cls.conn,
tf.config.get('account', tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write('obj contents')
cls.obj_slash = cls.container.file('/v1')
cls.obj_slash.write('obj contents')
def test_PUT_remapped_container(self):
for domain in (self.cont_domain_dash, self.cont_domain_underscore):
new_obj_name = Utils.create_name()
path = '/'.join(['', new_obj_name])
self.env.account.conn.make_request('PUT',
path,
data='new obj contents',
hdrs={'Host': domain},
cfg={'absolute_path': True})
self.assert_status(201)
new_obj = self.env.container.file(new_obj_name)
self.assertEqual(new_obj.read(), 'new obj contents')
def test_overwriting_with_url_encoded_object_name(self):
versions_container = self.env.versions_container
obj_name = Utils.create_name() + '%25ff'
versioned_obj, expected_headers, expected_content_types = \
self._test_overwriting_setup(obj_name)
# test delete
# at first, delete will succeed with 204
versioned_obj.delete()
expected_content_types.append(
'application/x-deleted;swift_versions_deleted=1')
# after that, any time the delete doesn't restore the old version
# and we will get 404 NotFound
for x in range(3):
with self.assertRaises(ResponseError) as cm:
versioned_obj.delete()
self.assertEqual(404, cm.exception.status)
expected_content_types.append(
'application/x-deleted;swift_versions_deleted=1')
# finally, we have 4 versioned items and 4 delete markers total in
# the versions container
self.assertEqual(8, versions_container.info()['object_count'])
self.assertEqual(expected_content_types, [
o['content_type'] for o in versions_container.files(
parms={'format': 'json'})])
# update versioned_obj
versioned_obj.write("eeee", hdrs={'Content-Type': 'text/thanksgiving',
'X-Object-Meta-Bar': 'foo'})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
self.assertEqual('text/thanksgiving', obj_info['content_type'])
# we still have delete-marker there
self.assertEqual(8, versions_container.info()['object_count'])
# update versioned_obj
versioned_obj.write("ffff", hdrs={'Content-Type': 'text/teriyaki',
'X-Object-Meta-Food': 'chickin'})
# verify the PUT object is kept successfully
obj_info = versioned_obj.info()
self.assertEqual('text/teriyaki', obj_info['content_type'])
# new obj will be inserted after delete-marker there
self.assertEqual(9, versions_container.info()['object_count'])
versioned_obj.delete()
with self.assertRaises(ResponseError) as cm:
versioned_obj.read()
self.assertEqual(404, cm.exception.status)
def setUp(cls):
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
super(TestTempurlEnv, cls).setUp()
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account.update_metadata({
'temp-url-key': cls.tempurl_key,
'temp-url-key-2': cls.tempurl_key2
})
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write(b"obj contents")
cls.other_obj = cls.container.file(Utils.create_name())
cls.other_obj.write(b"other obj contents")
def test_PUT(self):
new_obj = self.env.container.file(Utils.create_name())
expires = int(time.time()) + 86400
put_parms = self.tempurl_parms(
'PUT', expires, self.env.conn.make_path(new_obj.path),
self.env.tempurl_key)
new_obj.write('new obj contents',
parms=put_parms, cfg={'no_auth_token': True})
self.assertEqual(new_obj.read(), "new obj contents")
# PUT tempurls also allow HEAD requests
self.assertTrue(new_obj.info(parms=put_parms,
cfg={'no_auth_token': True}))
def test_PUT(self):
new_obj = self.env.container.file(Utils.create_name())
expires = int(time.time()) + 86400
put_parms = self.tempurl_parms(
'PUT', expires, self.env.conn.make_path(new_obj.path),
self.env.tempurl_key)
new_obj.write('new obj contents',
parms=put_parms, cfg={'no_auth_token': True})
self.assertEqual(new_obj.read(), "new obj contents")
# PUT tempurls also allow HEAD requests
self.assertTrue(new_obj.info(parms=put_parms,
cfg={'no_auth_token': True}))
def test_GET_DLO_outside_container(self):
container2 = self.env.account.container(Utils.create_name())
container2.create()
seg1 = container2.file(
"get-dlo-outside-seg1" + Utils.create_name())
seg2 = container2.file(
"get-dlo-outside-seg2" + Utils.create_name())
seg1.write("one fish two fish ")
seg2.write("red fish blue fish")
manifest = self.env.container.file("manifest" + Utils.create_name())
manifest.write(
'',
hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
(container2.name,)})
expires = int(time()) + 86400
sig = self.tempurl_sig(
'GET', expires, self.env.conn.make_path(manifest.path),
self.env.tempurl_key)
parms = {'temp_url_sig': sig,
'temp_url_expires': str(expires)}
# cross container tempurl does not work for container tempurl key
try:
manifest.read(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
try:
manifest.info(parms=parms, cfg={'no_auth_token': True})
except ResponseError as e:
self.assertEqual(e.status, 401)
else:
self.fail('request did not error')
def setUp(cls):
if cls.tempurl_enabled is None:
cls.tempurl_enabled = 'tempurl' in cluster_info
if not cls.tempurl_enabled:
return
super(TestTempurlEnv, cls).setUp()
cls.tempurl_key = Utils.create_name()
cls.tempurl_key2 = Utils.create_name()
cls.account.update_metadata({
'temp-url-key': cls.tempurl_key,
'temp-url-key-2': cls.tempurl_key2
})
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.obj = cls.container.file(Utils.create_name())
cls.obj.write("obj contents")
cls.other_obj = cls.container.file(Utils.create_name())
cls.other_obj.write("other obj contents")
def _check_overwriting_symlink(self):
# assertions common to x-versions-location and x-history-location modes
container = self.env.container
versions_container = self.env.versions_container
tgt_a_name = Utils.create_name()
tgt_b_name = Utils.create_name()
tgt_a = container.file(tgt_a_name)
tgt_a.write("aaaaa")
tgt_b = container.file(tgt_b_name)
tgt_b.write("bbbbb")
symlink_name = Utils.create_name()
sym_tgt_header = '%s/%s' % (container.name, tgt_a_name)
sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
symlink = container.file(symlink_name)
symlink.write("", hdrs=sym_headers_a)
self.assertEqual("aaaaa", symlink.read())
sym_headers_b = {
'X-Symlink-Target': '%s/%s' % (container.name, tgt_b_name)
}
symlink.write("", hdrs=sym_headers_b)
self.assertEqual("bbbbb", symlink.read())
# the old version got saved off
self.assertEqual(1, versions_container.info()['object_count'])
versioned_obj_name = versions_container.files()[0]
prev_version = versions_container.file(versioned_obj_name)
prev_version_info = prev_version.info(parms={'symlink': 'get'})
self.assertEqual("aaaaa", prev_version.read())
self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag'])
self.assertEqual(sym_tgt_header, prev_version_info['x_symlink_target'])
return symlink, tgt_a
def _check_overwriting_symlink(self):
# assertions common to x-versions-location and x-history-location modes
container = self.env.container
versions_container = self.env.versions_container
tgt_a_name = Utils.create_name()
tgt_b_name = Utils.create_name()
tgt_a = container.file(tgt_a_name)
tgt_a.write("aaaaa")
tgt_b = container.file(tgt_b_name)
tgt_b.write("bbbbb")
symlink_name = Utils.create_name()
sym_tgt_header = '%s/%s' % (container.name, tgt_a_name)
sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
symlink = container.file(symlink_name)
symlink.write("", hdrs=sym_headers_a)
self.assertEqual("aaaaa", symlink.read())
sym_headers_b = {'X-Symlink-Target': '%s/%s' % (container.name,
tgt_b_name)}
symlink.write("", hdrs=sym_headers_b)
self.assertEqual("bbbbb", symlink.read())
# the old version got saved off
self.assertEqual(1, versions_container.info()['object_count'])
versioned_obj_name = versions_container.files()[0]
prev_version = versions_container.file(versioned_obj_name)
prev_version_info = prev_version.info(parms={'symlink': 'get'})
self.assertEqual("aaaaa", prev_version.read())
self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag'])
self.assertEqual(sym_tgt_header,
prev_version_info['x_symlink_target'])
return symlink, tgt_a
def testInvalidHeadersPUT(self):
#TODO: Although we now support x-delete-at and x-delete-after,
#retained this test case as we may add some other header to
#unsupported list in future
raise SkipTest()
file = self.env.container.file(Utils.create_name())
self.assertRaises(ResponseError,
file.write_random,
self.env.file_size,
hdrs={'X-Delete-At': '9876545321'})
self.assert_status(400)
self.assertRaises(ResponseError,
file.write_random,
self.env.file_size,
hdrs={'X-Delete-After': '60'})
self.assert_status(400)
def _test_versioning_dlo_setup(self):
if tf.in_process:
tf.skip_if_no_xattrs()
container = self.env.container
versions_container = self.env.versions_container
obj_name = Utils.create_name()
for i in ('1', '2', '3'):
time.sleep(.01) # guarantee that the timestamp changes
obj_name_seg = obj_name + '/' + i
versioned_obj = container.file(obj_name_seg)
versioned_obj.write(i.encode('ascii'))
# immediately overwrite
versioned_obj.write((i + i).encode('ascii'))
self.assertEqual(3, versions_container.info()['object_count'])
man_file = container.file(obj_name)
# write a normal file first
man_file.write(b'old content')
# guarantee that the timestamp changes
time.sleep(.01)
# overwrite with a dlo manifest
man_file.write(b'',
hdrs={
"X-Object-Manifest":
"%s/%s/" % (self.env.container.name, obj_name)
})
self.assertEqual(4, versions_container.info()['object_count'])
self.assertEqual(b"112233", man_file.read())
self.assert_most_recent_version(obj_name, b'old content')
# overwrite the manifest with a normal file
man_file.write(b'new content')
self.assertEqual(5, versions_container.info()['object_count'])
# new most-recent archive is the dlo
self.assert_most_recent_version(obj_name,
b'112233',
should_be_dlo=True)
return obj_name, man_file
