def set_container_quota(args):
try:
utils = ManageAccounts()
try:
quota = int(args.quota)
except:
raise ValueError('Invalid quota')
accounts = [args.account] if args.account \
else utils.existing_accounts()
failed = []
def update_container_policy(account):
trans = utils.backend.wrapper.conn.begin()
try:
utils.backend.update_container_policy(account, account,
args.container,
{'quota': quota})
if args.dry:
print "Skipping database commit."
trans.rollback()
else:
trans.commit()
except Exception as e:
failed.append((account, e))
map(update_container_policy, accounts)
if failed and args.report:
sys.stdout.write('Failed for the following accounts:\n')
pprint_table(sys.stdout, failed, headers=[])
except Exception as e:
sys.stderr.write('%s\n' % e)
finally:
utils.cleanup()
def set_container_quota(args):
try:
utils = ManageAccounts()
try:
quota = int(args.quota)
except:
raise ValueError('Invalid quota')
accounts = [args.account] if args.account \
else utils.existing_accounts()
failed = []
def update_container_policy(account):
trans = utils.backend.wrapper.conn.begin()
try:
utils.backend.update_container_policy(
account, account, args.container, {'quota': quota}
)
if args.dry:
print "Skipping database commit."
trans.rollback()
else:
trans.commit()
except Exception, e:
failed.append((account, e))
map(update_container_policy, accounts)
if failed and args.report:
sys.stdout.write(
'Failed for the following accounts:\n'
)
pprint_table(sys.stdout, failed, headers=[])
def merge(args):
try:
utils = ManageAccounts()
utils.merge_account(args.src_account, args.dest_account, only_stats=True)
confirm = raw_input("Type 'yes' if you are sure you want" " to move those entries to %s: " % args.dest_account)
if not confirm == "yes":
return
else:
utils.merge_account(args.src_account, args.dest_account, only_stats=False, dry=args.dry)
except Exception, e:
sys.stderr.write("%s\n" % e)
def list(args):
try:
utils = ManageAccounts()
if args.only_duplicate:
accounts = utils.duplicate_accounts()
else:
accounts = utils.existing_accounts()
headers = ['uuid']
table = [(a,) for a in accounts]
if args.output_format != "json" and not args.headers:
headers = None
pprint_table(sys.stdout, table, headers, args.output_format)
except Exception, e:
sys.stderr.write('%s\n' % e)
def list(args):
try:
utils = ManageAccounts()
if args.only_duplicate:
accounts = utils.duplicate_accounts()
else:
accounts = utils.existing_accounts()
headers = ['uuid']
table = [(a, ) for a in accounts]
if args.output_format != "json" and not args.headers:
headers = None
pprint_table(sys.stdout, table, headers, args.output_format)
except Exception, e:
sys.stderr.write('%s\n' % e)
def export_quota(args):
try:
utils = ManageAccounts()
d = utils.backend.node.node_account_quotas()
location = os.path.abspath(os.path.expanduser(args.location))
f = open(location, 'w')
for uuid, capacity in d.iteritems():
f.write(' '.join([uuid, 'pithos.diskspace', capacity]))
f.write('\n')
except Exception as e:
sys.stderr.write('%s\n' % e)
finally:
utils.cleanup()
def export_quota(args):
try:
utils = ManageAccounts()
d = utils.backend.node.node_account_quotas()
location = os.path.abspath(os.path.expanduser(args.location))
f = open(location, 'w')
for uuid, capacity in d.iteritems():
f.write(' '.join([uuid, 'pithos.diskspace', capacity]))
f.write('\n')
except Exception as e:
sys.stderr.write('%s\n' % e)
finally:
utils.cleanup()
def delete(args):
try:
utils = ManageAccounts()
utils.delete_account(args.delete_account, only_stats=True)
confirm = raw_input(
"Type 'yes' if you are sure you want "
"to remove those entries: "
)
if not confirm == 'yes':
return
else:
utils.delete_account(
args.delete_account, only_stats=False, dry=args.dry
)
except Exception, e:
sys.stderr.write('%s\n' % e)
def delete(args):
try:
utils = ManageAccounts()
utils.delete_account(args.delete_account, only_stats=True)
confirm = raw_input("Type 'yes' if you are sure you want "
"to remove those entries: ")
if not confirm == 'yes':
return
else:
utils.delete_account(args.delete_account,
only_stats=False,
dry=args.dry)
except Exception as e:
sys.stderr.write('%s\n' % e)
finally:
utils.cleanup()
def merge(args):
try:
utils = ManageAccounts()
utils.merge_account(args.src_account,
args.dest_account,
only_stats=True)
confirm = raw_input("Type 'yes' if you are sure you want"
" to move those entries to %s: " %
args.dest_account)
if not confirm == 'yes':
return
else:
utils.merge_account(args.src_account,
args.dest_account,
only_stats=False,
dry=args.dry)
except Exception, e:
sys.stderr.write('%s\n' % e)
class ManageAccountsTests(unittest.TestCase):
def setUp(self):
self.utils = ManageAccounts()
self.accounts = ('account1', 'Account1', 'account2', 'account3')
for i in self.accounts:
self.utils.create_account(i)
def tearDown(self):
for i in self.accounts:
self.utils._delete_account(i)
self.utils.cleanup()
def _verify_object(self, account, container, object, expected=None,
strict=True):
expected = expected or {}
self._verify_object_metadata(account, container, object,
expected.get('meta'))
self._verify_object_history(account, container, object,
expected.get('versions'),
strict=strict)
self._verify_object_permissions(account, container, object,
expected.get('permissions'))
def _verify_object_metadata(self, account, container, object, expected):
object_meta = self.utils.backend.get_object_meta(
account, account, container, object, 'pithos')
for k in expected:
self.assertTrue(k in object_meta)
self.assertEquals(object_meta[k], expected[k])
def _verify_object_history(self, account, container, object, expected,
strict=True):
history = self.utils.list_past_versions(account, container, object)
if strict:
self.assertEquals(sorted(expected), history)
else:
self.assertTrue(set(expected) <= set(history))
def _verify_object_permissions(self, account, container, object, expected):
expected = expected or {}
perms_tuple = self.utils.backend.get_object_permissions(
account, account, container, object)
self.assertEqual(len(perms_tuple), 3)
object_perms = perms_tuple[2]
for k in expected:
self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k)))
for holder in expected.get('read', []):
if holder == '*':
continue
try:
# check first for a group permission
owner, group = holder.split(':', 1)
except ValueError:
holders = [holder]
else:
holders = self.utils.backend.permissions.group_members(owner,
group)
for h in holders:
try:
self.utils.backend.get_object_meta(
holder, account, container, object, 'pithos')
except Exception, e:
self.fail(e)
def setUp(self):
self.utils = ManageAccounts()
self.accounts = ('account1', 'Account1', 'account2', 'account3')
for i in self.accounts:
self.utils.create_account(i)
class TestPublic(unittest.TestCase):
def setUp(self):
self.utils = ManageAccounts()
self.backend = self.utils.backend
self.utils.create_account('account')
def tearDown(self):
self.utils._delete_account('account')
self.utils.cleanup()
def assert_not_public_object(self, account, container, object):
public = self.backend.get_object_public(
account, account, container, object
)
self.assertTrue(public == None)
self.assertRaises(
NameError,
self.backend.get_public,
'$$account$$',
public
)
self.assertRaises(
Exception, self.backend._can_read,
'$$account$$', account, container, object
)
return public
def assert_public_object(self, account, container, object):
public = self.backend.get_object_public(
account, account, container, object
)
self.assertTrue(public != None)
self.assertTrue(len(public) >= settings.PUBLIC_URL_SECURITY)
self.assertTrue(set(public) <= set(settings.PUBLIC_URL_ALPHABET))
self.assertEqual(
self.backend.get_public('$$account$$', public),
(account, container, object)
)
try:
self.backend._can_read('$$account$$', account, container, object)
except Exception:
self.fail('Public object should be readable.')
return public
def test_set_object_public(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
self.assert_public_object('account', 'container', 'object')
def test_set_twice(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public2 = self.assert_public_object('account', 'container', 'object')
self.assertEqual(public, public2)
def test_set_unset_set(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_unset('account/container/object')
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public3 = self.assert_public_object('account', 'container', 'object')
self.assertTrue(public != public3)
def test_update_object_public(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=False
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=True
)
public = self.assert_public_object('account', 'container', 'object')
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=False
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.update_object_public(
'account', 'account', 'container', 'object', public=True
)
new_public = self.assert_public_object('account', 'container', 'object')
self.assertTrue(public != new_public)
def test_delete_not_public_object(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.delete_object('account', 'account', 'container', 'object')
self.assert_not_public_object('account', 'container', 'object')
def test_delete_public_object(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
self.assert_public_object('account', 'container', 'object')
self.backend.delete_object('account', 'account', 'container', 'object')
self.assert_not_public_object('account', 'container', 'object')
def test_delete_public_object_history(self):
self.utils.backend.put_container('account', 'account', 'container')
for i in range(random.randint(1, 10)):
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account',
'container',
'object',
'application/octet-stream',
data
)
_time.sleep(1)
versions = self.backend.list_versions(
'account', 'account', 'container', 'object'
)
mtime = [int(i[1]) for i in versions]
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set(
'account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet
)
public = self.assert_public_object('account', 'container', 'object')
i = random.randrange(len(mtime))
self.backend.delete_object(
'account', 'account', 'container', 'object', until=mtime[i]
)
self.assert_public_object('account', 'container', 'object')
public = self.assert_public_object('account', 'container', 'object')
_time.sleep(1)
t = datetime.datetime.utcnow()
now = int(_time.mktime(t.timetuple()))
self.backend.delete_object(
'account', 'account', 'container', 'object', until=now
)
self.assertRaises(
NameError,
self.backend.get_public,
'$$account$$',
public
)
def setUp(self):
self.utils = ManageAccounts()
self.backend = self.utils.backend
self.utils.create_account('account')
def setUp(self):
self.utils = ManageAccounts()
self.backend = self.utils.backend
self.utils.create_account('account')
class ManageAccountsTests(unittest.TestCase):
def setUp(self):
self.utils = ManageAccounts()
self.accounts = ('account1', 'Account1', 'account2', 'account3')
for i in self.accounts:
self.utils.create_account(i)
def tearDown(self):
for i in self.accounts:
self.utils._delete_account(i)
self.utils.cleanup()
def _verify_object(self,
account,
container,
object,
expected=None,
strict=True):
expected = expected or {}
self._verify_object_metadata(account, container, object,
expected.get('meta'))
self._verify_object_history(account,
container,
object,
expected.get('versions'),
strict=strict)
self._verify_object_permissions(account, container, object,
expected.get('permissions'))
def _verify_object_metadata(self, account, container, object, expected):
object_meta = self.utils.backend.get_object_meta(
account, account, container, object, 'pithos')
for k in expected:
self.assertTrue(k in object_meta)
self.assertEquals(object_meta[k], expected[k])
def _verify_object_history(self,
account,
container,
object,
expected,
strict=True):
history = self.utils.list_past_versions(account, container, object)
if strict:
self.assertEquals(sorted(expected), history)
else:
self.assertTrue(set(expected) <= set(history))
def _verify_object_permissions(self, account, container, object, expected):
expected = expected or {}
perms_tuple = self.utils.backend.get_object_permissions(
account, account, container, object)
self.assertEqual(len(perms_tuple), 3)
object_perms = perms_tuple[2]
for k in expected:
self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k)))
for holder in expected.get('read', []):
if holder == '*':
continue
try:
# check first for a group permission
owner, group = holder.split(':', 1)
except ValueError:
holders = [holder]
else:
holders = self.utils.backend.permissions.group_members(
owner, group)
for h in holders:
try:
self.utils.backend.get_object_meta(holder, account,
container, object,
'pithos')
except Exception as e:
self.fail(e)
def test_existing_accounts(self):
accounts = self.utils.existing_accounts()
self.assertEquals(sorted(accounts), accounts)
self.assertTrue(set(['account1', 'account2']) <= set(accounts))
def test_duplicate_accounts(self):
duplicates = self.utils.duplicate_accounts()
self.assertTrue(['Account1', 'account1'] in duplicates)
def test_list_all_containers(self):
step = 10
containers = []
append = containers.append
for i in range(3 * step + 1):
while 1:
cname = unicode(uuid.uuid4())
if cname not in containers:
append(cname)
break
self.utils.backend.put_container('account1', 'account1', cname)
self.assertEquals(
sorted(containers),
self.utils.list_all_containers('account1', step=step))
def test_list_all_container_objects(self):
containers = ('container1', 'container2')
objects = defaultdict(list)
for c in containers:
self.utils.backend.put_container('account1', 'account1', c)
step = 10
append = objects[c].append
content_type = 'application/octet-stream'
for i in range(3 * step + 1):
while 1:
oname = unicode(uuid.uuid4())
if oname not in objects:
append(oname)
break
data = get_random_data(int(random.random()))
self.utils.create_update_object('account1', c, oname,
content_type, data)
(self.assertEquals(
sorted(objects.get(c)),
self.utils.list_all_container_objects('account1', c))
for c in containers)
def test_list_all_objects(self):
containers = ('container1', 'container2')
objects = []
append = objects.append
for c in containers:
self.utils.backend.put_container('account1', 'account1', c)
step = 10
content_type = 'application/octet-stream'
for i in range(3 * step + 1):
while 1:
oname = unicode(uuid.uuid4())
if oname not in objects:
append(os.path.join(c, oname))
break
data = get_random_data(int(random.random()))
self.utils.create_update_object('account1', c, oname,
content_type, data)
self.assertEquals(len(objects),
len(self.utils.list_all_objects('account1')))
self.assertEquals(sorted(objects),
self.utils.list_all_objects('account1'))
def test_list_past_versions(self):
self.utils.backend.put_container('account1', 'account1', 'container1')
versions = []
append = versions.append
for i in range(5):
data = get_random_data(int(random.random()))
append(
self.utils.create_update_object('account1', 'container1',
'object1',
'application/octet-stream',
data))
self.assertEquals(
sorted([i[0] for i in versions[:-1]]),
self.utils.list_past_versions('account1', 'container1', 'object1'))
def test_move(self):
# create containers
self.utils.backend.put_container('account1', 'account1', 'container1')
self.utils.backend.put_container('Account1', 'Account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload object and update it several times
versions = []
append = versions.append
meta = {'foo': 'bar'}
permissions = {
'read': ['account1', 'account2', 'Account1:test'],
'write': ['account2', 'Account1:test']
}
for i in range(5):
data = get_random_data(int(random.random()))
append(
self.utils.create_update_object('Account1', 'container1',
'object1',
'application/octet-stream',
data, meta, permissions))
self.utils.move_object('Account1',
'container1',
'object1',
'account1',
dry=False,
silent=True)
expected = {
'meta': meta,
'versions': [i[0] for i in versions[:-1]],
'permissions': permissions
}
self._verify_object('account1', 'container1', 'object1', expected)
def test_merge(self):
# create container
self.utils.backend.put_container('Account1', 'Account1', 'container0')
self.utils.backend.put_container('Account1', 'Account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload objects and update them several times
versions = defaultdict(list)
meta = {'foo': 'bar'}
permissions = {
'read': ['account2', 'Account1:test'],
'write': ['account2', 'Account1:test']
}
for i in range(2):
container = 'container%s' % i
versions[container] = {}
for j in range(3):
object = 'object%s' % j
versions[container][object] = []
append = versions[container][object].append
for k in range(5):
data = get_random_data(int(random.random()))
append(
self.utils.create_update_object(
'Account1', container, object,
'application/octet-stream', data, meta,
permissions))
self.utils.merge_account('Account1',
'account1',
only_stats=False,
dry=False,
silent=True)
self.assertTrue('Account1' in self.utils.existing_accounts())
self.assertTrue('account1' in self.utils.existing_accounts())
# assert container has been created
try:
self.utils.backend.get_container_meta('account1', 'account1',
'container1', 'pithos')
except NameError as e:
self.fail(e)
expected = {'meta': meta, 'permissions': permissions}
for c, o_dict in versions.iteritems():
for o, versions in o_dict.iteritems():
expected['versions'] = [i[0] for i in versions[:-1]]
self._verify_object('account1', c, o, expected)
def test_merge_existing_dest_container(self):
# create container
self.utils.backend.put_container('Account1', 'Account1', 'container1')
self.utils.backend.put_container('account1', 'account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload objects and update them several times
versions = defaultdict(list)
meta = {'foo': 'bar'}
permissions = {
'read': ['account2', 'Account1:test'],
'write': ['account2', 'Account1:test']
}
versions = []
append = versions.append
for k in range(5):
data = get_random_data(int(random.random()))
append(
self.utils.create_update_object('Account1', 'container1',
'object1',
'application/octet-stream',
data, meta, permissions))
self.utils.merge_account('Account1',
'account1',
only_stats=False,
dry=False,
silent=True)
self.assertTrue('Account1' in self.utils.existing_accounts())
self.assertTrue('account1' in self.utils.existing_accounts())
try:
self.utils.backend.get_container_meta('account1', 'account1',
'container1', 'pithos')
except NameError as e:
self.fail(e)
expected = {
'meta': meta,
'versions': [i[0] for i in versions[:-1]],
'permissions': permissions
}
self._verify_object('account1', 'container1', 'object1', expected)
def test_merge_existing_dest_object(self):
# create container
self.utils.backend.put_container('Account1', 'Account1', 'container1')
self.utils.backend.put_container('account1', 'account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload objects and update them several times
versions = defaultdict(list)
meta = {'foo': 'bar'}
permissions = {
'read': ['account2', 'Account1:test'],
'write': ['account2', 'Account1:test']
}
container = 'container1'
object = 'object1'
versions = []
append = versions.append
for k in range(5):
data = get_random_data(int(random.random()))
append(
self.utils.create_update_object('Account1', container, object,
'application/octet-stream',
data, meta, permissions))
data = get_random_data(int(random.random()))
self.utils.create_update_object('account1', container, object,
'application/octet-stream', data,
meta, permissions)
self.utils.merge_account('Account1',
'account1',
only_stats=False,
dry=False,
silent=True)
self.assertTrue('Account1' in self.utils.existing_accounts())
self.assertTrue('account1' in self.utils.existing_accounts())
try:
self.utils.backend.get_container_meta('account1', 'account1',
'container1', 'pithos')
except NameError as e:
self.fail(e)
expected = {
'meta': meta,
'permissions': permissions,
'versions': [i[0] for i in versions[:-1]]
}
self._verify_object('account1',
container,
object,
expected,
strict=False)
class ManageAccountsTests(unittest.TestCase):
def setUp(self):
self.utils = ManageAccounts()
self.accounts = ('account1', 'Account1', 'account2', 'account3')
for i in self.accounts:
self.utils.create_account(i)
def tearDown(self):
for i in self.accounts:
self.utils._delete_account(i)
self.utils.cleanup()
def _verify_object(self, account, container, object, expected=None,
strict=True):
expected = expected or {}
self._verify_object_metadata(account, container, object,
expected.get('meta'))
self._verify_object_history(account, container, object,
expected.get('versions'),
strict=strict)
self._verify_object_permissions(account, container, object,
expected.get('permissions'))
def _verify_object_metadata(self, account, container, object, expected):
object_meta = self.utils.backend.get_object_meta(
account, account, container, object, 'pithos')
for k in expected:
self.assertTrue(k in object_meta)
self.assertEquals(object_meta[k], expected[k])
def _verify_object_history(self, account, container, object, expected,
strict=True):
history = self.utils.list_past_versions(account, container, object)
if strict:
self.assertEquals(sorted(expected), history)
else:
self.assertTrue(set(expected) <= set(history))
def _verify_object_permissions(self, account, container, object, expected):
expected = expected or {}
perms_tuple = self.utils.backend.get_object_permissions(
account, account, container, object)
self.assertEqual(len(perms_tuple), 3)
object_perms = perms_tuple[2]
for k in expected:
self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k)))
for holder in expected.get('read', []):
if holder == '*':
continue
try:
# check first for a group permission
owner, group = holder.split(':', 1)
except ValueError:
holders = [holder]
else:
holders = self.utils.backend.permissions.group_members(owner,
group)
for h in holders:
try:
self.utils.backend.get_object_meta(
holder, account, container, object, 'pithos')
except Exception as e:
self.fail(e)
def test_existing_accounts(self):
accounts = self.utils.existing_accounts()
self.assertEquals(sorted(accounts), accounts)
self.assertTrue(set(['account1', 'account2']) <= set(accounts))
def test_duplicate_accounts(self):
duplicates = self.utils.duplicate_accounts()
self.assertTrue(['Account1', 'account1'] in duplicates)
def test_list_all_containers(self):
step = 10
containers = []
append = containers.append
for i in range(3 * step + 1):
while 1:
cname = unicode(uuid.uuid4())
if cname not in containers:
append(cname)
break
self.utils.backend.put_container('account1', 'account1', cname)
self.assertEquals(sorted(containers),
self.utils.list_all_containers('account1',
step=step))
def test_list_all_container_objects(self):
containers = ('container1', 'container2')
objects = defaultdict(list)
for c in containers:
self.utils.backend.put_container('account1', 'account1', c)
step = 10
append = objects[c].append
content_type = 'application/octet-stream'
for i in range(3 * step + 1):
while 1:
oname = unicode(uuid.uuid4())
if oname not in objects:
append(oname)
break
data = get_random_data(int(random.random()))
self.utils.create_update_object('account1', c, oname,
content_type, data)
(self.assertEquals(sorted(objects.get(c)),
self.utils.list_all_container_objects('account1', c)
) for c in containers)
def test_list_all_objects(self):
containers = ('container1', 'container2')
objects = []
append = objects.append
for c in containers:
self.utils.backend.put_container('account1', 'account1', c)
step = 10
content_type = 'application/octet-stream'
for i in range(3 * step + 1):
while 1:
oname = unicode(uuid.uuid4())
if oname not in objects:
append(os.path.join(c, oname))
break
data = get_random_data(int(random.random()))
self.utils.create_update_object('account1', c, oname,
content_type, data)
self.assertEquals(len(objects),
len(self.utils.list_all_objects('account1')))
self.assertEquals(sorted(objects),
self.utils.list_all_objects('account1'))
def test_list_past_versions(self):
self.utils.backend.put_container('account1', 'account1', 'container1')
versions = []
append = versions.append
for i in range(5):
data = get_random_data(int(random.random()))
append(self.utils.create_update_object('account1', 'container1',
'object1',
'application/octet-stream',
data))
self.assertEquals(sorted([i[0] for i in versions[:-1]]),
self.utils.list_past_versions('account1',
'container1',
'object1'))
def test_move(self):
# create containers
self.utils.backend.put_container('account1', 'account1', 'container1')
self.utils.backend.put_container('Account1', 'Account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload object and update it several times
versions = []
append = versions.append
meta = {'foo': 'bar'}
permissions = {'read': ['account1', 'account2', 'Account1:test'],
'write': ['account2', 'Account1:test']}
for i in range(5):
data = get_random_data(int(random.random()))
append(self.utils.create_update_object('Account1', 'container1',
'object1',
'application/octet-stream',
data, meta, permissions))
self.utils.move_object('Account1', 'container1', 'object1', 'account1',
dry=False, silent=True)
expected = {'meta': meta,
'versions': [i[0] for i in versions[:-1]],
'permissions': permissions}
self._verify_object('account1', 'container1', 'object1', expected)
def test_merge(self):
# create container
self.utils.backend.put_container('Account1', 'Account1', 'container0')
self.utils.backend.put_container('Account1', 'Account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload objects and update them several times
versions = defaultdict(list)
meta = {'foo': 'bar'}
permissions = {'read': ['account2', 'Account1:test'],
'write': ['account2', 'Account1:test']}
for i in range(2):
container = 'container%s' % i
versions[container] = {}
for j in range(3):
object = 'object%s' % j
versions[container][object] = []
append = versions[container][object].append
for k in range(5):
data = get_random_data(int(random.random()))
append(self.utils.create_update_object(
'Account1', container, object,
'application/octet-stream', data, meta, permissions))
self.utils.merge_account('Account1', 'account1', only_stats=False,
dry=False, silent=True)
self.assertTrue('Account1' in self.utils.existing_accounts())
self.assertTrue('account1' in self.utils.existing_accounts())
# assert container has been created
try:
self.utils.backend.get_container_meta('account1', 'account1',
'container1', 'pithos')
except NameError as e:
self.fail(e)
expected = {'meta': meta,
'permissions': permissions}
for c, o_dict in versions.iteritems():
for o, versions in o_dict.iteritems():
expected['versions'] = [i[0] for i in versions[:-1]]
self._verify_object('account1', c, o, expected)
def test_merge_existing_dest_container(self):
# create container
self.utils.backend.put_container('Account1', 'Account1', 'container1')
self.utils.backend.put_container('account1', 'account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload objects and update them several times
versions = defaultdict(list)
meta = {'foo': 'bar'}
permissions = {'read': ['account2', 'Account1:test'],
'write': ['account2', 'Account1:test']}
versions = []
append = versions.append
for k in range(5):
data = get_random_data(int(random.random()))
append(self.utils.create_update_object(
'Account1', 'container1', 'object1',
'application/octet-stream', data, meta, permissions))
self.utils.merge_account('Account1', 'account1', only_stats=False,
dry=False, silent=True)
self.assertTrue('Account1' in self.utils.existing_accounts())
self.assertTrue('account1' in self.utils.existing_accounts())
try:
self.utils.backend.get_container_meta('account1', 'account1',
'container1', 'pithos')
except NameError as e:
self.fail(e)
expected = {'meta': meta,
'versions': [i[0] for i in versions[:-1]],
'permissions': permissions}
self._verify_object('account1', 'container1', 'object1', expected)
def test_merge_existing_dest_object(self):
# create container
self.utils.backend.put_container('Account1', 'Account1', 'container1')
self.utils.backend.put_container('account1', 'account1', 'container1')
# add group
self.utils.backend.update_account_groups('Account1', 'Account1',
{'test': ['account3']})
# upload objects and update them several times
versions = defaultdict(list)
meta = {'foo': 'bar'}
permissions = {'read': ['account2', 'Account1:test'],
'write': ['account2', 'Account1:test']}
container = 'container1'
object = 'object1'
versions = []
append = versions.append
for k in range(5):
data = get_random_data(int(random.random()))
append(self.utils.create_update_object(
'Account1', container, object,
'application/octet-stream', data, meta, permissions))
data = get_random_data(int(random.random()))
self.utils.create_update_object(
'account1', container, object, 'application/octet-stream',
data, meta, permissions)
self.utils.merge_account('Account1', 'account1', only_stats=False,
dry=False, silent=True)
self.assertTrue('Account1' in self.utils.existing_accounts())
self.assertTrue('account1' in self.utils.existing_accounts())
try:
self.utils.backend.get_container_meta('account1', 'account1',
'container1', 'pithos')
except NameError as e:
self.fail(e)
expected = {'meta': meta,
'permissions': permissions,
'versions': [i[0] for i in versions[:-1]]}
self._verify_object('account1', container, object, expected,
strict=False)
def setUp(self):
self.utils = ManageAccounts()
self.accounts = ('account1', 'Account1', 'account2', 'account3')
for i in self.accounts:
self.utils.create_account(i)
class ManageAccountsTests(unittest.TestCase):
def setUp(self):
self.utils = ManageAccounts()
self.accounts = ('account1', 'Account1', 'account2', 'account3')
for i in self.accounts:
self.utils.create_account(i)
def tearDown(self):
for i in self.accounts:
self.utils._delete_account(i)
self.utils.cleanup()
def _verify_object(self,
account,
container,
object,
expected=None,
strict=True):
expected = expected or {}
self._verify_object_metadata(account, container, object,
expected.get('meta'))
self._verify_object_history(account,
container,
object,
expected.get('versions'),
strict=strict)
self._verify_object_permissions(account, container, object,
expected.get('permissions'))
def _verify_object_metadata(self, account, container, object, expected):
object_meta = self.utils.backend.get_object_meta(
account, account, container, object, 'pithos')
for k in expected:
self.assertTrue(k in object_meta)
self.assertEquals(object_meta[k], expected[k])
def _verify_object_history(self,
account,
container,
object,
expected,
strict=True):
history = self.utils.list_past_versions(account, container, object)
if strict:
self.assertEquals(sorted(expected), history)
else:
self.assertTrue(set(expected) <= set(history))
def _verify_object_permissions(self, account, container, object, expected):
expected = expected or {}
perms_tuple = self.utils.backend.get_object_permissions(
account, account, container, object)
self.assertEqual(len(perms_tuple), 3)
object_perms = perms_tuple[2]
for k in expected:
self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k)))
for holder in expected.get('read', []):
if holder == '*':
continue
try:
# check first for a group permission
owner, group = holder.split(':', 1)
except ValueError:
holders = [holder]
else:
holders = self.utils.backend.permissions.group_members(
owner, group)
for h in holders:
try:
self.utils.backend.get_object_meta(holder, account,
container, object,
'pithos')
except Exception, e:
self.fail(e)
class TestPublic(unittest.TestCase):
def setUp(self):
self.utils = ManageAccounts()
self.backend = self.utils.backend
self.utils.create_account('account')
def tearDown(self):
self.utils._delete_account('account')
self.utils.cleanup()
def assert_not_public_object(self, account, container, object):
public = self.backend.get_object_public(account, account, container,
object)
self.assertTrue(public == None)
self.assertRaises(NameError, self.backend.get_public, '$$account$$',
public)
self.assertRaises(Exception, self.backend._can_read, '$$account$$',
account, container, object)
return public
def assert_public_object(self, account, container, object):
public = self.backend.get_object_public(account, account, container,
object)
self.assertTrue(public != None)
self.assertTrue(len(public) >= settings.PUBLIC_URL_SECURITY)
self.assertTrue(set(public) <= set(settings.PUBLIC_URL_ALPHABET))
self.assertEqual(self.backend.get_public('$$account$$', public),
(account, container, object))
try:
self.backend._can_read('$$account$$', account, container, object)
except Exception:
self.fail('Public object should be readable.')
return public
def test_set_object_public(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
self.assert_public_object('account', 'container', 'object')
def test_set_twice(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
public2 = self.assert_public_object('account', 'container', 'object')
self.assertEqual(public, public2)
def test_set_unset_set(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
public = self.assert_public_object('account', 'container', 'object')
self.backend.permissions.public_unset('account/container/object')
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
public3 = self.assert_public_object('account', 'container', 'object')
self.assertTrue(public != public3)
def test_update_object_public(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
self.backend.update_object_public('account',
'account',
'container',
'object',
public=False)
self.assert_not_public_object('account', 'container', 'object')
self.backend.update_object_public('account',
'account',
'container',
'object',
public=True)
public = self.assert_public_object('account', 'container', 'object')
self.backend.update_object_public('account',
'account',
'container',
'object',
public=False)
self.assert_not_public_object('account', 'container', 'object')
self.backend.update_object_public('account',
'account',
'container',
'object',
public=True)
new_public = self.assert_public_object('account', 'container',
'object')
self.assertTrue(public != new_public)
def test_delete_not_public_object(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
self.assert_not_public_object('account', 'container', 'object')
self.backend.delete_object('account', 'account', 'container', 'object')
self.assert_not_public_object('account', 'container', 'object')
def test_delete_public_object(self):
self.utils.backend.put_container('account', 'account', 'container')
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
self.assert_public_object('account', 'container', 'object')
self.backend.delete_object('account', 'account', 'container', 'object')
self.assert_not_public_object('account', 'container', 'object')
def test_delete_public_object_history(self):
self.utils.backend.put_container('account', 'account', 'container')
for i in range(random.randint(1, 10)):
data = get_random_data(int(random.random()))
self.utils.create_update_object('account', 'container', 'object',
'application/octet-stream', data)
_time.sleep(1)
versions = self.backend.list_versions('account', 'account',
'container', 'object')
mtime = [int(i[1]) for i in versions]
self.assert_not_public_object('account', 'container', 'object')
self.backend.permissions.public_set('account/container/object',
self.backend.public_url_security,
self.backend.public_url_alphabet)
public = self.assert_public_object('account', 'container', 'object')
i = random.randrange(len(mtime))
self.backend.delete_object('account',
'account',
'container',
'object',
until=mtime[i])
self.assert_public_object('account', 'container', 'object')
public = self.assert_public_object('account', 'container', 'object')
_time.sleep(1)
t = datetime.datetime.utcnow()
now = int(_time.mktime(t.timetuple()))
self.backend.delete_object('account',
'account',
'container',
'object',
until=now)
self.assertRaises(NameError, self.backend.get_public, '$$account$$',
public)