def create(args):
"""Create a user."""
username = args.username.decode("utf8")
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
pass
else:
print "ERROR: There is already an user with that username"
return
# let's create it
user = User(username=username,
email=args.email.decode("utf8"),
first_name=args.firstname.decode("utf8"),
last_name=args.lastname.decode("utf8"))
user.set_password(args.password.decode("utf8"))
user.save()
# refresh the user object to ensure permissions caches are reloaded
user = User.objects.get(username=username)
# create also the storage user
visible_name = "%s %s" % (user.first_name, user.last_name)
services.make_storage_user(user.id, username, visible_name, 2 ** 20)
print "Success: User created ok"
def test_requests_leak(self):
"""Test that the server waits for pending requests."""
# create a server
self.service = StorageServerService(
0,
"localhost",
self.s4_conn.getHost().port,
False,
s4.AWS_DEFAULT_ACCESS_KEY_ID,
s4.AWS_DEFAULT_SECRET_ACCESS_KEY,
auth_provider_class=DummyAuthProvider,
heartbeat_interval=0)
# start it
yield self.service.startService()
make_storage_user(0, u"dummy", u"", 2**20)
client_d = defer.Deferred()
f = TestClientFactory()
f.connected = client_d.callback
reactor.connectTCP("localhost", self.service.port, f)
client = yield client_d
# start with a client
yield client.dummy_authenticate("open sesame")
root_id = yield client.get_root()
# create a bunch of files
mk_deferreds = []
def handle_conn_done(f):
"""Ignore ConnectionDone errors."""
if f.check(error.ConnectionDone):
return
return f
for i in range(10):
mk = client.make_file(request.ROOT, root_id, "hola_%s" % i)
mk.addErrback(handle_conn_done)
mk_deferreds.append(mk)
try:
reactor.callLater(0.1, client.transport.loseConnection)
yield defer.DeferredList(mk_deferreds)
finally:
self.assertTrue(self.service.factory.protocols[0].requests)
# wait for protocol shutdown (this was hanging waiting for requests
# to finish)
yield self.service.factory.protocols[0].wait_for_shutdown()
# see that the server has not protocols alive
yield self.service.factory.wait_for_shutdown()
# cleanup
yield self.service.stopService()
def benchmark_one(bench, user_id):
"""Run a benchmark on a bunch of things."""
user = get_storage_user(int(user_id), active_only=False)
if user is None:
user = make_storage_user(int(user_id), u"TestUser", u"", 100 * (2**30))
# setup a test directory with loads of files.
testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S')
root = user.volume().root.make_subdirectory(testroot)
for i in range(10):
d = root.make_subdirectory(unicode(i))
make_deep_tree_with_files(d)
#move the children on the test root
moved_child = user.volume().get_node_by_path(u"%s/1" % testroot)
dest_dir = root.make_subdirectory(u"test-move")
with bench("big_move"):
moved_child.move(dest_dir.id, moved_child.name)
with bench("deltafull"):
generation, _, _ = user.volume().get_delta(0)
with bench("deltaempty"):
user.volume().get_delta(generation - 1000)
with bench("getmusic"):
user.volume().get_all_nodes(mimetypes=MIME_TYPES)
with bench("get_shared_volumes"):
user.get_share_volumes()
with bench("get_udfs"):
user.get_udfs()
def create_test_user(username=u"fred", email=u"*****@*****.**",
first_name=u"Fredrick", last_name=u"Flintsone",
id=None, password=None, shard_id=None):
"""Create a user used for testing."""
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
now = datetime.utcnow()
user = User(id=id, username=unicode(username), email=unicode(email),
is_staff=False, is_active=True, is_superuser=False,
last_login=now, date_joined=now)
user.set_password(password)
user.save()
user.first_name = unicode(first_name)
user.last_name = unicode(last_name)
user.save()
# refresh the user object to ensure permissions caches are reloaded
account_user = User.objects.get(username=username)
# create also the storage user
visible_name = "%s %s" % (user.first_name, user.last_name)
storage_user = services.make_storage_user(
account_user.id, username, visible_name, 2 ** 20, shard_id=shard_id)
return storage_user
def benchmark_one(bench, user_id):
"""Run a benchmark on a bunch of things."""
user = get_storage_user(int(user_id), active_only=False)
if user is None:
user = make_storage_user(int(user_id), u"TestUser",
u"", 100 * (2 ** 30))
# setup a test directory with loads of files.
testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S')
root = user.volume().root.make_subdirectory(testroot)
for i in range(10):
d = root.make_subdirectory(unicode(i))
make_deep_tree_with_files(d)
# move the children on the test root
moved_child = user.volume().get_node_by_path(u"%s/1" % testroot)
dest_dir = root.make_subdirectory(u"test-move")
with bench("big_move"):
moved_child.move(dest_dir.id, moved_child.name)
with bench("deltafull"):
generation, _, _ = user.volume().get_delta(0)
with bench("deltaempty"):
user.volume().get_delta(generation - 1000)
with bench("getmusic"):
user.volume().get_all_nodes(mimetypes=MIME_TYPES)
with bench("get_shared_volumes"):
user.get_share_volumes()
with bench("get_udfs"):
user.get_udfs()
def setUp(self):
"""Setup for testing."""
# make sure we start with clean state
yield super(TestShutdown, self).setUp()
self.s4_site = site = s4.server.Site(s4.Root())
self.s4_conn = reactor.listenTCP(0, site)
# since storageusers are not automatically created, we need to create
self.usr0 = make_storage_user(0, u"dummy", u"", 2**20)
def setUp(self):
"""Setup for testing."""
# make sure we start with clean state
yield super(TestShutdown, self).setUp()
self.s4_site = site = s4.server.Site(s4.Root())
self.s4_conn = reactor.listenTCP(0, site)
# since storageusers are not automatically created, we need to create
self.usr0 = make_storage_user(0, u"dummy", u"", 2 ** 20)
def test_requests_leak(self):
"""Test that the server waits for pending requests."""
# create a server
self.service = StorageServerService(
0, "localhost", self.s4_conn.getHost().port, False,
s4.AWS_DEFAULT_ACCESS_KEY_ID,
s4.AWS_DEFAULT_SECRET_ACCESS_KEY,
auth_provider_class=DummyAuthProvider,
heartbeat_interval=0)
# start it
yield self.service.startService()
make_storage_user(0, u"dummy", u"", 2 ** 20)
client_d = defer.Deferred()
f = TestClientFactory()
f.connected = client_d.callback
reactor.connectTCP("localhost", self.service.port, f)
client = yield client_d
# start with a client
yield client.dummy_authenticate("open sesame")
root_id = yield client.get_root()
# create a bunch of files
mk_deferreds = []
def handle_conn_done(f):
"""Ignore ConnectionDone errors."""
if f.check(error.ConnectionDone):
return
return f
for i in range(10):
mk = client.make_file(request.ROOT, root_id, "hola_%s" % i)
mk.addErrback(handle_conn_done)
mk_deferreds.append(mk)
try:
reactor.callLater(0.1, client.transport.loseConnection)
yield defer.DeferredList(mk_deferreds)
finally:
self.assertTrue(self.service.factory.protocols[0].requests)
# wait for protocol shutdown (this was hanging waiting for requests
# to finish)
yield self.service.factory.protocols[0].wait_for_shutdown()
# see that the server has not protocols alive
yield self.service.factory.wait_for_shutdown()
# cleanup
yield self.service.stopService()
def test__check_stores_and_invalidate(self):
"""Test _check_stores_and_invalidate invalidate case."""
logger = logging.getLogger('storage.server.noninv')
h = MementoHandler()
logger.addHandler(h)
make_storage_user(1, 'foo', 'foo', 10000)
store = get_filesync_store()
self._sto = store # for later cleanup
obj = StorageObject(1, 'foo', StorageObject.FILE)
store.add(obj)
store.flush()
self.assertFalse(obj.__storm_object_info__.get("invalidated", False))
_check_stores_and_invalidate(filesync_zstorm)
self.assertTrue(obj.__storm_object_info__.get("invalidated", False))
self.assertEqual(1, len(h.records))
self.assertEqual((obj,), h.records[0].args)
def test__check_stores_and_invalidate(self):
"""Test _check_stores_and_invalidate invalidate case."""
from backends.filesync.data.services import make_storage_user
from backends.filesync.data.model import StorageObject
logger = logging.getLogger('storage.server.noninv')
h = MementoHandler()
logger.addHandler(h)
make_storage_user(1, u'foo', u'foo', 10000, u'shard2')
sto = get_store('shard2', storage_zstorm)
self._sto = sto # for later cleanup
obj = StorageObject(1, u'foo', u'File')
sto.add(obj)
sto.flush()
self.assertFalse(obj.__storm_object_info__.get("invalidated", False))
_check_stores_and_invalidate(storage_zstorm)
self.assertTrue(obj.__storm_object_info__.get("invalidated", False))
self.assertEqual(1, len(h.records))
self.assertEqual((obj,), h.records[0].args)
def test__check_stores_and_invalidate(self):
"""Test _check_stores_and_invalidate invalidate case."""
from backends.filesync.data.services import make_storage_user
from backends.filesync.data.model import StorageObject
logger = logging.getLogger('storage.server.noninv')
h = MementoHandler()
logger.addHandler(h)
make_storage_user(1, u'foo', u'foo', 10000, u'shard2')
sto = get_store('shard2', storage_zstorm)
self._sto = sto # for later cleanup
obj = StorageObject(1, u'foo', u'File')
sto.add(obj)
sto.flush()
self.assertFalse(obj.__storm_object_info__.get("invalidated", False))
_check_stores_and_invalidate(storage_zstorm)
self.assertTrue(obj.__storm_object_info__.get("invalidated", False))
self.assertEqual(1, len(h.records))
self.assertEqual((obj, ), h.records[0].args)
def test_get_storage_user(self):
"""Test the get_storage_user function."""
user = make_storage_user(1, u"Cool UserName", u"Visible Name",
MAX_STORAGE_BYTES)
user = get_storage_user(1)
self.assertTrue(isinstance(user, dao.StorageUser))
user.update(subscription=False)
self.assertRaises(errors.DoesNotExist, get_storage_user, 1)
user = get_storage_user(1, active_only=False)
user.update(subscription=True)
# now check a locked user.
suser = self.user_store.get(model.StorageUser, user.id)
suser.locked = True
self.user_store.commit()
self.assertRaises(errors.LockedUserError, get_storage_user, user.id)
# and ignore the lock too
user = get_storage_user(user.id, readonly=True)
self.assertTrue(isinstance(user, dao.StorageUser))
def test_get_storage_user(self):
"""Test the get_storage_user function."""
user = make_storage_user(
1, "Cool UserName", "Visible Name", MAX_STORAGE_BYTES)
user = get_storage_user(1)
self.assertTrue(isinstance(user, dao.StorageUser))
user.update(subscription=False)
self.assertRaises(errors.DoesNotExist, get_storage_user, 1)
user = get_storage_user(1, active_only=False)
user.update(subscription=True)
# now check a locked user.
suser = self.store.get(StorageUser, user.id)
suser.locked = True
self.store.commit()
self.assertRaises(errors.LockedUserError, get_storage_user, user.id)
# and ignore the lock too
user = get_storage_user(user.id, readonly=True)
self.assertTrue(isinstance(user, dao.StorageUser))
def test_shutdown_upload(self):
"""Stop and restart the server."""
# create a server
service = StorageServerService(0,
"localhost",
self.s4_conn.getHost().port,
False,
s4.AWS_DEFAULT_ACCESS_KEY_ID,
s4.AWS_DEFAULT_SECRET_ACCESS_KEY,
auth_provider_class=DummyAuthProvider,
heartbeat_interval=0)
yield service.startService()
# create a user, connect a client
usr0 = make_storage_user(0, u"dummy", u"", 2**20)
d = defer.Deferred()
f = TestClientFactory()
f.connected = d.callback
reactor.connectTCP("localhost", service.port, f)
client = yield d
# auth, get root, create a file
yield client.dummy_authenticate("open sesame")
root = yield client.get_root()
mkfile_req = yield client.make_file(request.ROOT, root, "hola")
# try to upload something that will fail when sending data
empty_hash = content_hash_factory().content_hash()
# lose the connection if something wrong
try:
yield client.put_content(request.ROOT, mkfile_req.new_id,
empty_hash, "fake_hash", 1234, 1000, None)
except:
client.transport.loseConnection()
# see that the server has not protocols alive
yield service.factory.wait_for_shutdown()
ujobs = usr0.get_uploadjobs(node_id=mkfile_req.new_id)
self.assertEqual(ujobs, [])
yield service.stopService()
def test_shutdown_upload(self):
"""Stop and restart the server."""
# create a server
service = StorageServerService(
0, "localhost", self.s4_conn.getHost().port, False,
s4.AWS_DEFAULT_ACCESS_KEY_ID,
s4.AWS_DEFAULT_SECRET_ACCESS_KEY,
auth_provider_class=DummyAuthProvider,
heartbeat_interval=0)
yield service.startService()
# create a user, connect a client
usr0 = make_storage_user(0, u"dummy", u"", 2 ** 20)
d = defer.Deferred()
f = TestClientFactory()
f.connected = d.callback
reactor.connectTCP("localhost", service.port, f)
client = yield d
# auth, get root, create a file
yield client.dummy_authenticate("open sesame")
root = yield client.get_root()
mkfile_req = yield client.make_file(request.ROOT, root, "hola")
# try to upload something that will fail when sending data
empty_hash = content_hash_factory().content_hash()
# lose the connection if something wrong
try:
yield client.put_content(request.ROOT, mkfile_req.new_id,
empty_hash, "fake_hash", 1234, 1000, None)
except:
client.transport.loseConnection()
# see that the server has not protocols alive
yield service.factory.wait_for_shutdown()
ujobs = usr0.get_uploadjobs(node_id=mkfile_req.new_id)
self.assertEqual(ujobs, [])
yield service.stopService()
def test_quota(self):
"""Test quota info """
self.usr0.update(max_storage_bytes=2**16)
usr2 = services.make_storage_user(1, u"otheruser", u"Other User",
2**17)
share = usr2.root.share(self.usr0.id, u"a share", readonly=True)
@defer.inlineCallbacks
def do_test(client):
"""Do the actual test."""
usr1 = self.usr0
quota = usr1.get_quota()
yield client.dummy_authenticate("open sesame")
result = yield client.get_free_space(request.ROOT)
self.assertEqual(quota.free_bytes, result.free_bytes)
self.assertEqual(request.ROOT, result.share_id)
result = yield client.get_free_space(str(share.id))
quota = usr2.get_quota()
self.assertEqual(quota.free_bytes, result.free_bytes)
self.assertEqual(str(share.id), result.share_id)
return self.callback_test(do_test, add_default_callbacks=True)
def test_quota(self):
"""Test quota info """
self.usr0.update(max_storage_bytes=2 ** 16)
usr2 = services.make_storage_user(1, u"otheruser",
u"Other User", 2 ** 17)
share = usr2.root.share(self.usr0.id, u"a share", readonly=True)
@defer.inlineCallbacks
def do_test(client):
"""Do the actual test."""
usr1 = self.usr0
quota = usr1.get_quota()
yield client.dummy_authenticate("open sesame")
result = yield client.get_free_space(request.ROOT)
self.assertEqual(quota.free_bytes, result.free_bytes)
self.assertEqual(request.ROOT, result.share_id)
result = yield client.get_free_space(str(share.id))
quota = usr2.get_quota()
self.assertEqual(quota.free_bytes, result.free_bytes)
self.assertEqual(str(share.id), result.share_id)
return self.callback_test(do_test,
add_default_callbacks=True)
def create_test_user(username=u"fred",
email=u"*****@*****.**",
first_name=u"Fredrick",
last_name=u"Flintsone",
id=None,
password=None,
shard_id=None):
"""Create a user used for testing."""
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
now = datetime.utcnow()
user = User(id=id,
username=unicode(username),
email=unicode(email),
is_staff=False,
is_active=True,
is_superuser=False,
last_login=now,
date_joined=now)
user.set_password(password)
user.save()
user.first_name = unicode(first_name)
user.last_name = unicode(last_name)
user.save()
# refresh the user object to ensure permissions caches are reloaded
account_user = User.objects.get(username=username)
# create also the storage user
visible_name = "%s %s" % (user.first_name, user.last_name)
storage_user = services.make_storage_user(account_user.id,
username,
visible_name,
2**20,
shard_id=shard_id)
return storage_user
def create_user(shard_id):
"""Create a new random user."""
id = datetime.utcnow().toordinal()
username = unicode(uuid.uuid4())
return make_storage_user(id, username, u"", 2 * (2 ** 30),
shard_id=shard_id)
def _make_users(self):
"""Create users for tests."""
usernames = ["bob", "bobby", "inez", "juan", "tim"]
for i, name in zip(range(5), usernames):
services.make_storage_user(i, name, name, 2 ** 30)
def _make_users(self):
"""Create users for tests."""
usernames = [u'bob', u'bobby', u'inez', u'juan', u'tim']
for i, name in zip(range(5), usernames):
services.make_storage_user(i, name, name, 2**30)
def test_make_storage_user(self):
"""Test the make_storage_user function."""
storage_user = make_storage_user(
1, "Cool UserName", "Visible Name", MAX_STORAGE_BYTES)
self.assert_storage_user(
storage_user, 1, "Visible Name", MAX_STORAGE_BYTES)
parser.add_option("--count", dest="count", default="100",
help="number of users to create")
(options, args) = parser.parse_args()
from utilities import userutils
import uuid
from backends.filesync.data.services import make_storage_user
token_data = {}
for i in range(int(options.count)):
username = "******" % i
userinfo = {
'username': unicode(uuid.uuid4()),
'full_name': "name %s" % i,
'active': True,
'email': "*****@*****.**",
}
# create the user account
user = userutils.create_user(userinfo)
# create the storage account
make_storage_user(
user.id, user.username,
"%s %s" % (user.first_name, user.last_name), 2 * (2 ** 30))
# get an oauth token
token = userutils.make_oauth_token(user)
token_data[username] = (user.id, str(token))
token_file = os.path.join('testoauthkeys.json')
with open(token_file, 'w') as f:
json.dump(token_data, f)
(options, args) = parser.parse_args()
from utilities import userutils
import uuid
from backends.filesync.data.services import make_storage_user
from backends.filesync.data.dbmanager import get_new_user_shard_id
token_data = {}
for i in range(int(options.count)):
username = u"testuser%s" % i
userinfo = {
'username': unicode(uuid.uuid4()),
'full_name': u"name %s" % i,
'active': True,
'email': u"*****@*****.**",
}
# create the user account
user = userutils.create_user(userinfo)
# create the storage account
make_storage_user(
user.id, user.username,
"%s %s" % (user.first_name, user.last_name), 2 * (2 ** 30),
shard_id=get_new_user_shard_id(user.id))
# get an oauth token
token = userutils.make_oauth_token(user)
token_data[username] = (user.id, str(token))
token_file = os.path.join('testoauthkeys.json')
with open(token_file, 'w') as f:
json.dump(token_data, f)
def _make_users(self):
"""Create users for tests."""
usernames = [u'bob', u'bobby', u'inez', u'juan', u'tim']
for i, name in zip(range(5), usernames):
services.make_storage_user(i, name, name, 2 ** 30)
def test_make_storage_user(self):
"""Test the make_storage_user function."""
storage_user = make_storage_user(1, u"Cool UserName", u"Visible Name",
MAX_STORAGE_BYTES)
self.assert_storage_user(storage_user, 1, u"Visible Name",
MAX_STORAGE_BYTES)