def create(args): """Create a user.""" username = args.username.decode("utf8") try: user = User.objects.get(username=username) except User.DoesNotExist: pass else: print "ERROR: There is already an user with that username" return # let's create it user = User(username=username, email=args.email.decode("utf8"), first_name=args.firstname.decode("utf8"), last_name=args.lastname.decode("utf8")) user.set_password(args.password.decode("utf8")) user.save() # refresh the user object to ensure permissions caches are reloaded user = User.objects.get(username=username) # create also the storage user visible_name = "%s %s" % (user.first_name, user.last_name) services.make_storage_user(user.id, username, visible_name, 2 ** 20) print "Success: User created ok"
def test_requests_leak(self): """Test that the server waits for pending requests.""" # create a server self.service = StorageServerService( 0, "localhost", self.s4_conn.getHost().port, False, s4.AWS_DEFAULT_ACCESS_KEY_ID, s4.AWS_DEFAULT_SECRET_ACCESS_KEY, auth_provider_class=DummyAuthProvider, heartbeat_interval=0) # start it yield self.service.startService() make_storage_user(0, u"dummy", u"", 2**20) client_d = defer.Deferred() f = TestClientFactory() f.connected = client_d.callback reactor.connectTCP("localhost", self.service.port, f) client = yield client_d # start with a client yield client.dummy_authenticate("open sesame") root_id = yield client.get_root() # create a bunch of files mk_deferreds = [] def handle_conn_done(f): """Ignore ConnectionDone errors.""" if f.check(error.ConnectionDone): return return f for i in range(10): mk = client.make_file(request.ROOT, root_id, "hola_%s" % i) mk.addErrback(handle_conn_done) mk_deferreds.append(mk) try: reactor.callLater(0.1, client.transport.loseConnection) yield defer.DeferredList(mk_deferreds) finally: self.assertTrue(self.service.factory.protocols[0].requests) # wait for protocol shutdown (this was hanging waiting for requests # to finish) yield self.service.factory.protocols[0].wait_for_shutdown() # see that the server has not protocols alive yield self.service.factory.wait_for_shutdown() # cleanup yield self.service.stopService()
def benchmark_one(bench, user_id): """Run a benchmark on a bunch of things.""" user = get_storage_user(int(user_id), active_only=False) if user is None: user = make_storage_user(int(user_id), u"TestUser", u"", 100 * (2**30)) # setup a test directory with loads of files. testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S') root = user.volume().root.make_subdirectory(testroot) for i in range(10): d = root.make_subdirectory(unicode(i)) make_deep_tree_with_files(d) #move the children on the test root moved_child = user.volume().get_node_by_path(u"%s/1" % testroot) dest_dir = root.make_subdirectory(u"test-move") with bench("big_move"): moved_child.move(dest_dir.id, moved_child.name) with bench("deltafull"): generation, _, _ = user.volume().get_delta(0) with bench("deltaempty"): user.volume().get_delta(generation - 1000) with bench("getmusic"): user.volume().get_all_nodes(mimetypes=MIME_TYPES) with bench("get_shared_volumes"): user.get_share_volumes() with bench("get_udfs"): user.get_udfs()
def create_test_user(username=u"fred", email=u"*****@*****.**", first_name=u"Fredrick", last_name=u"Flintsone", id=None, password=None, shard_id=None): """Create a user used for testing.""" try: user = User.objects.get(username=username) except User.DoesNotExist: now = datetime.utcnow() user = User(id=id, username=unicode(username), email=unicode(email), is_staff=False, is_active=True, is_superuser=False, last_login=now, date_joined=now) user.set_password(password) user.save() user.first_name = unicode(first_name) user.last_name = unicode(last_name) user.save() # refresh the user object to ensure permissions caches are reloaded account_user = User.objects.get(username=username) # create also the storage user visible_name = "%s %s" % (user.first_name, user.last_name) storage_user = services.make_storage_user( account_user.id, username, visible_name, 2 ** 20, shard_id=shard_id) return storage_user
def benchmark_one(bench, user_id): """Run a benchmark on a bunch of things.""" user = get_storage_user(int(user_id), active_only=False) if user is None: user = make_storage_user(int(user_id), u"TestUser", u"", 100 * (2 ** 30)) # setup a test directory with loads of files. testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S') root = user.volume().root.make_subdirectory(testroot) for i in range(10): d = root.make_subdirectory(unicode(i)) make_deep_tree_with_files(d) # move the children on the test root moved_child = user.volume().get_node_by_path(u"%s/1" % testroot) dest_dir = root.make_subdirectory(u"test-move") with bench("big_move"): moved_child.move(dest_dir.id, moved_child.name) with bench("deltafull"): generation, _, _ = user.volume().get_delta(0) with bench("deltaempty"): user.volume().get_delta(generation - 1000) with bench("getmusic"): user.volume().get_all_nodes(mimetypes=MIME_TYPES) with bench("get_shared_volumes"): user.get_share_volumes() with bench("get_udfs"): user.get_udfs()
def setUp(self): """Setup for testing.""" # make sure we start with clean state yield super(TestShutdown, self).setUp() self.s4_site = site = s4.server.Site(s4.Root()) self.s4_conn = reactor.listenTCP(0, site) # since storageusers are not automatically created, we need to create self.usr0 = make_storage_user(0, u"dummy", u"", 2**20)
def setUp(self): """Setup for testing.""" # make sure we start with clean state yield super(TestShutdown, self).setUp() self.s4_site = site = s4.server.Site(s4.Root()) self.s4_conn = reactor.listenTCP(0, site) # since storageusers are not automatically created, we need to create self.usr0 = make_storage_user(0, u"dummy", u"", 2 ** 20)
def test_requests_leak(self): """Test that the server waits for pending requests.""" # create a server self.service = StorageServerService( 0, "localhost", self.s4_conn.getHost().port, False, s4.AWS_DEFAULT_ACCESS_KEY_ID, s4.AWS_DEFAULT_SECRET_ACCESS_KEY, auth_provider_class=DummyAuthProvider, heartbeat_interval=0) # start it yield self.service.startService() make_storage_user(0, u"dummy", u"", 2 ** 20) client_d = defer.Deferred() f = TestClientFactory() f.connected = client_d.callback reactor.connectTCP("localhost", self.service.port, f) client = yield client_d # start with a client yield client.dummy_authenticate("open sesame") root_id = yield client.get_root() # create a bunch of files mk_deferreds = [] def handle_conn_done(f): """Ignore ConnectionDone errors.""" if f.check(error.ConnectionDone): return return f for i in range(10): mk = client.make_file(request.ROOT, root_id, "hola_%s" % i) mk.addErrback(handle_conn_done) mk_deferreds.append(mk) try: reactor.callLater(0.1, client.transport.loseConnection) yield defer.DeferredList(mk_deferreds) finally: self.assertTrue(self.service.factory.protocols[0].requests) # wait for protocol shutdown (this was hanging waiting for requests # to finish) yield self.service.factory.protocols[0].wait_for_shutdown() # see that the server has not protocols alive yield self.service.factory.wait_for_shutdown() # cleanup yield self.service.stopService()
def test__check_stores_and_invalidate(self): """Test _check_stores_and_invalidate invalidate case.""" logger = logging.getLogger('storage.server.noninv') h = MementoHandler() logger.addHandler(h) make_storage_user(1, 'foo', 'foo', 10000) store = get_filesync_store() self._sto = store # for later cleanup obj = StorageObject(1, 'foo', StorageObject.FILE) store.add(obj) store.flush() self.assertFalse(obj.__storm_object_info__.get("invalidated", False)) _check_stores_and_invalidate(filesync_zstorm) self.assertTrue(obj.__storm_object_info__.get("invalidated", False)) self.assertEqual(1, len(h.records)) self.assertEqual((obj,), h.records[0].args)
def test__check_stores_and_invalidate(self): """Test _check_stores_and_invalidate invalidate case.""" from backends.filesync.data.services import make_storage_user from backends.filesync.data.model import StorageObject logger = logging.getLogger('storage.server.noninv') h = MementoHandler() logger.addHandler(h) make_storage_user(1, u'foo', u'foo', 10000, u'shard2') sto = get_store('shard2', storage_zstorm) self._sto = sto # for later cleanup obj = StorageObject(1, u'foo', u'File') sto.add(obj) sto.flush() self.assertFalse(obj.__storm_object_info__.get("invalidated", False)) _check_stores_and_invalidate(storage_zstorm) self.assertTrue(obj.__storm_object_info__.get("invalidated", False)) self.assertEqual(1, len(h.records)) self.assertEqual((obj,), h.records[0].args)
def test__check_stores_and_invalidate(self): """Test _check_stores_and_invalidate invalidate case.""" from backends.filesync.data.services import make_storage_user from backends.filesync.data.model import StorageObject logger = logging.getLogger('storage.server.noninv') h = MementoHandler() logger.addHandler(h) make_storage_user(1, u'foo', u'foo', 10000, u'shard2') sto = get_store('shard2', storage_zstorm) self._sto = sto # for later cleanup obj = StorageObject(1, u'foo', u'File') sto.add(obj) sto.flush() self.assertFalse(obj.__storm_object_info__.get("invalidated", False)) _check_stores_and_invalidate(storage_zstorm) self.assertTrue(obj.__storm_object_info__.get("invalidated", False)) self.assertEqual(1, len(h.records)) self.assertEqual((obj, ), h.records[0].args)
def test_get_storage_user(self): """Test the get_storage_user function.""" user = make_storage_user(1, u"Cool UserName", u"Visible Name", MAX_STORAGE_BYTES) user = get_storage_user(1) self.assertTrue(isinstance(user, dao.StorageUser)) user.update(subscription=False) self.assertRaises(errors.DoesNotExist, get_storage_user, 1) user = get_storage_user(1, active_only=False) user.update(subscription=True) # now check a locked user. suser = self.user_store.get(model.StorageUser, user.id) suser.locked = True self.user_store.commit() self.assertRaises(errors.LockedUserError, get_storage_user, user.id) # and ignore the lock too user = get_storage_user(user.id, readonly=True) self.assertTrue(isinstance(user, dao.StorageUser))
def test_get_storage_user(self): """Test the get_storage_user function.""" user = make_storage_user( 1, "Cool UserName", "Visible Name", MAX_STORAGE_BYTES) user = get_storage_user(1) self.assertTrue(isinstance(user, dao.StorageUser)) user.update(subscription=False) self.assertRaises(errors.DoesNotExist, get_storage_user, 1) user = get_storage_user(1, active_only=False) user.update(subscription=True) # now check a locked user. suser = self.store.get(StorageUser, user.id) suser.locked = True self.store.commit() self.assertRaises(errors.LockedUserError, get_storage_user, user.id) # and ignore the lock too user = get_storage_user(user.id, readonly=True) self.assertTrue(isinstance(user, dao.StorageUser))
def test_shutdown_upload(self): """Stop and restart the server.""" # create a server service = StorageServerService(0, "localhost", self.s4_conn.getHost().port, False, s4.AWS_DEFAULT_ACCESS_KEY_ID, s4.AWS_DEFAULT_SECRET_ACCESS_KEY, auth_provider_class=DummyAuthProvider, heartbeat_interval=0) yield service.startService() # create a user, connect a client usr0 = make_storage_user(0, u"dummy", u"", 2**20) d = defer.Deferred() f = TestClientFactory() f.connected = d.callback reactor.connectTCP("localhost", service.port, f) client = yield d # auth, get root, create a file yield client.dummy_authenticate("open sesame") root = yield client.get_root() mkfile_req = yield client.make_file(request.ROOT, root, "hola") # try to upload something that will fail when sending data empty_hash = content_hash_factory().content_hash() # lose the connection if something wrong try: yield client.put_content(request.ROOT, mkfile_req.new_id, empty_hash, "fake_hash", 1234, 1000, None) except: client.transport.loseConnection() # see that the server has not protocols alive yield service.factory.wait_for_shutdown() ujobs = usr0.get_uploadjobs(node_id=mkfile_req.new_id) self.assertEqual(ujobs, []) yield service.stopService()
def test_shutdown_upload(self): """Stop and restart the server.""" # create a server service = StorageServerService( 0, "localhost", self.s4_conn.getHost().port, False, s4.AWS_DEFAULT_ACCESS_KEY_ID, s4.AWS_DEFAULT_SECRET_ACCESS_KEY, auth_provider_class=DummyAuthProvider, heartbeat_interval=0) yield service.startService() # create a user, connect a client usr0 = make_storage_user(0, u"dummy", u"", 2 ** 20) d = defer.Deferred() f = TestClientFactory() f.connected = d.callback reactor.connectTCP("localhost", service.port, f) client = yield d # auth, get root, create a file yield client.dummy_authenticate("open sesame") root = yield client.get_root() mkfile_req = yield client.make_file(request.ROOT, root, "hola") # try to upload something that will fail when sending data empty_hash = content_hash_factory().content_hash() # lose the connection if something wrong try: yield client.put_content(request.ROOT, mkfile_req.new_id, empty_hash, "fake_hash", 1234, 1000, None) except: client.transport.loseConnection() # see that the server has not protocols alive yield service.factory.wait_for_shutdown() ujobs = usr0.get_uploadjobs(node_id=mkfile_req.new_id) self.assertEqual(ujobs, []) yield service.stopService()
def test_quota(self): """Test quota info """ self.usr0.update(max_storage_bytes=2**16) usr2 = services.make_storage_user(1, u"otheruser", u"Other User", 2**17) share = usr2.root.share(self.usr0.id, u"a share", readonly=True) @defer.inlineCallbacks def do_test(client): """Do the actual test.""" usr1 = self.usr0 quota = usr1.get_quota() yield client.dummy_authenticate("open sesame") result = yield client.get_free_space(request.ROOT) self.assertEqual(quota.free_bytes, result.free_bytes) self.assertEqual(request.ROOT, result.share_id) result = yield client.get_free_space(str(share.id)) quota = usr2.get_quota() self.assertEqual(quota.free_bytes, result.free_bytes) self.assertEqual(str(share.id), result.share_id) return self.callback_test(do_test, add_default_callbacks=True)
def test_quota(self): """Test quota info """ self.usr0.update(max_storage_bytes=2 ** 16) usr2 = services.make_storage_user(1, u"otheruser", u"Other User", 2 ** 17) share = usr2.root.share(self.usr0.id, u"a share", readonly=True) @defer.inlineCallbacks def do_test(client): """Do the actual test.""" usr1 = self.usr0 quota = usr1.get_quota() yield client.dummy_authenticate("open sesame") result = yield client.get_free_space(request.ROOT) self.assertEqual(quota.free_bytes, result.free_bytes) self.assertEqual(request.ROOT, result.share_id) result = yield client.get_free_space(str(share.id)) quota = usr2.get_quota() self.assertEqual(quota.free_bytes, result.free_bytes) self.assertEqual(str(share.id), result.share_id) return self.callback_test(do_test, add_default_callbacks=True)
def create_test_user(username=u"fred", email=u"*****@*****.**", first_name=u"Fredrick", last_name=u"Flintsone", id=None, password=None, shard_id=None): """Create a user used for testing.""" try: user = User.objects.get(username=username) except User.DoesNotExist: now = datetime.utcnow() user = User(id=id, username=unicode(username), email=unicode(email), is_staff=False, is_active=True, is_superuser=False, last_login=now, date_joined=now) user.set_password(password) user.save() user.first_name = unicode(first_name) user.last_name = unicode(last_name) user.save() # refresh the user object to ensure permissions caches are reloaded account_user = User.objects.get(username=username) # create also the storage user visible_name = "%s %s" % (user.first_name, user.last_name) storage_user = services.make_storage_user(account_user.id, username, visible_name, 2**20, shard_id=shard_id) return storage_user
def create_user(shard_id): """Create a new random user.""" id = datetime.utcnow().toordinal() username = unicode(uuid.uuid4()) return make_storage_user(id, username, u"", 2 * (2 ** 30), shard_id=shard_id)
def _make_users(self): """Create users for tests.""" usernames = ["bob", "bobby", "inez", "juan", "tim"] for i, name in zip(range(5), usernames): services.make_storage_user(i, name, name, 2 ** 30)
def _make_users(self): """Create users for tests.""" usernames = [u'bob', u'bobby', u'inez', u'juan', u'tim'] for i, name in zip(range(5), usernames): services.make_storage_user(i, name, name, 2**30)
def test_make_storage_user(self): """Test the make_storage_user function.""" storage_user = make_storage_user( 1, "Cool UserName", "Visible Name", MAX_STORAGE_BYTES) self.assert_storage_user( storage_user, 1, "Visible Name", MAX_STORAGE_BYTES)
parser.add_option("--count", dest="count", default="100", help="number of users to create") (options, args) = parser.parse_args() from utilities import userutils import uuid from backends.filesync.data.services import make_storage_user token_data = {} for i in range(int(options.count)): username = "******" % i userinfo = { 'username': unicode(uuid.uuid4()), 'full_name': "name %s" % i, 'active': True, 'email': "*****@*****.**", } # create the user account user = userutils.create_user(userinfo) # create the storage account make_storage_user( user.id, user.username, "%s %s" % (user.first_name, user.last_name), 2 * (2 ** 30)) # get an oauth token token = userutils.make_oauth_token(user) token_data[username] = (user.id, str(token)) token_file = os.path.join('testoauthkeys.json') with open(token_file, 'w') as f: json.dump(token_data, f)
(options, args) = parser.parse_args() from utilities import userutils import uuid from backends.filesync.data.services import make_storage_user from backends.filesync.data.dbmanager import get_new_user_shard_id token_data = {} for i in range(int(options.count)): username = u"testuser%s" % i userinfo = { 'username': unicode(uuid.uuid4()), 'full_name': u"name %s" % i, 'active': True, 'email': u"*****@*****.**", } # create the user account user = userutils.create_user(userinfo) # create the storage account make_storage_user( user.id, user.username, "%s %s" % (user.first_name, user.last_name), 2 * (2 ** 30), shard_id=get_new_user_shard_id(user.id)) # get an oauth token token = userutils.make_oauth_token(user) token_data[username] = (user.id, str(token)) token_file = os.path.join('testoauthkeys.json') with open(token_file, 'w') as f: json.dump(token_data, f)
def _make_users(self): """Create users for tests.""" usernames = [u'bob', u'bobby', u'inez', u'juan', u'tim'] for i, name in zip(range(5), usernames): services.make_storage_user(i, name, name, 2 ** 30)
def test_make_storage_user(self): """Test the make_storage_user function.""" storage_user = make_storage_user(1, u"Cool UserName", u"Visible Name", MAX_STORAGE_BYTES) self.assert_storage_user(storage_user, 1, u"Visible Name", MAX_STORAGE_BYTES)