def benchmark_one(bench, user_id): """Run a benchmark on a bunch of things.""" user = get_storage_user(int(user_id), active_only=False) if user is None: user = make_storage_user(int(user_id), u"TestUser", u"", 100 * (2**30)) # setup a test directory with loads of files. testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S') root = user.volume().root.make_subdirectory(testroot) for i in range(10): d = root.make_subdirectory(unicode(i)) make_deep_tree_with_files(d) #move the children on the test root moved_child = user.volume().get_node_by_path(u"%s/1" % testroot) dest_dir = root.make_subdirectory(u"test-move") with bench("big_move"): moved_child.move(dest_dir.id, moved_child.name) with bench("deltafull"): generation, _, _ = user.volume().get_delta(0) with bench("deltaempty"): user.volume().get_delta(generation - 1000) with bench("getmusic"): user.volume().get_all_nodes(mimetypes=MIME_TYPES) with bench("get_shared_volumes"): user.get_share_volumes() with bench("get_udfs"): user.get_udfs()
def benchmark_one(bench, user_id): """Run a benchmark on a bunch of things.""" user = get_storage_user(int(user_id), active_only=False) if user is None: user = make_storage_user(int(user_id), u"TestUser", u"", 100 * (2 ** 30)) # setup a test directory with loads of files. testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S') root = user.volume().root.make_subdirectory(testroot) for i in range(10): d = root.make_subdirectory(unicode(i)) make_deep_tree_with_files(d) # move the children on the test root moved_child = user.volume().get_node_by_path(u"%s/1" % testroot) dest_dir = root.make_subdirectory(u"test-move") with bench("big_move"): moved_child.move(dest_dir.id, moved_child.name) with bench("deltafull"): generation, _, _ = user.volume().get_delta(0) with bench("deltaempty"): user.volume().get_delta(generation - 1000) with bench("getmusic"): user.volume().get_all_nodes(mimetypes=MIME_TYPES) with bench("get_shared_volumes"): user.get_share_volumes() with bench("get_udfs"): user.get_udfs()
def make_user_data(user): """Make some user data.""" u = services.get_storage_user(username=user_names[random.randint(0, 4)]) s = user.root.share(u.id, u"%s to %s" % (user.username, u.username)) u.get_share(s.id).accept() for i in range(10): d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i)) for f in range(10): f = d.make_file(u"%s file%s.txt" % (user.username, f)) uj = f.make_uploadjob(f.content_hash, get_fake_hash(str(random.random())), random.randint(1, 100), random.randint(100, 1000000), random.randint(100, 1000000)) uj.commit_content(f.content_hash, uuid.uuid4(), None) services.add_music_metadata(uj.file.content_hash, TEST_MUSIC_DATA, user.shard_id) for i in range(10): user.make_musicstore_download(random.randint(100, 1000000), random.randint(1, 100), u"https://fake.com/%s/%s" % (user.id, i), u"%s" % user.username, u"%s Greatest Hits" % user.username, u"Track %s" % i, "US") for i in range(10): downloadservices.get_or_make_download( user.id, user.root_volume_id, u"https://fake.com/%s/%s" % (user.id, i), u"/a/b", u"%s%s" % (user.id, i))
def make_user_data(user): """Make some user data.""" u = services.get_storage_user(username=user_names[random.randint(0, 4)]) s = user.root.share(u.id, u"%s to %s" % (user.username, u.username)) u.get_share(s.id).accept() for i in range(10): d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i)) for f in range(10): f = d.make_file(u"%s file%s.txt" % (user.username, f)) uj = f.make_uploadjob( f.content_hash, get_fake_hash(str(random.random())), random.randint(1, 100), random.randint(100, 1000000), random.randint(100, 1000000)) uj.commit_content(f.content_hash, uuid.uuid4(), None) services.add_music_metadata( uj.file.content_hash, TEST_MUSIC_DATA, user.shard_id) for i in range(10): user.make_musicstore_download( random.randint(100, 1000000), random.randint(1, 100), u"https://fake.com/%s/%s" % (user.id, i), u"%s" % user.username, u"%s Greatest Hits" % user.username, u"Track %s" % i, "US") for i in range(10): downloadservices.get_or_make_download( user.id, user.root_volume_id, u"https://fake.com/%s/%s" % (user.id, i), u"/a/b", u"%s%s" % (user.id, i))
def create_share(self, user_id, node_id, to_username, share_name, readonly): """Create a share.""" user = self._get_user(user_id) to_user = services.get_storage_user(username=to_username) share = user.volume().dir(node_id).share(to_user.id, share_name, readonly) return dict(share_id=share.id)
def test_get_storage_user(self): """Test the get_storage_user function.""" user = make_storage_user( 1, "Cool UserName", "Visible Name", MAX_STORAGE_BYTES) user = get_storage_user(1) self.assertTrue(isinstance(user, dao.StorageUser)) user.update(subscription=False) self.assertRaises(errors.DoesNotExist, get_storage_user, 1) user = get_storage_user(1, active_only=False) user.update(subscription=True) # now check a locked user. suser = self.store.get(StorageUser, user.id) suser.locked = True self.store.commit() self.assertRaises(errors.LockedUserError, get_storage_user, user.id) # and ignore the lock too user = get_storage_user(user.id, readonly=True) self.assertTrue(isinstance(user, dao.StorageUser))
def test_get_storage_user(self): """Test the get_storage_user function.""" user = make_storage_user(1, u"Cool UserName", u"Visible Name", MAX_STORAGE_BYTES) user = get_storage_user(1) self.assertTrue(isinstance(user, dao.StorageUser)) user.update(subscription=False) self.assertRaises(errors.DoesNotExist, get_storage_user, 1) user = get_storage_user(1, active_only=False) user.update(subscription=True) # now check a locked user. suser = self.user_store.get(model.StorageUser, user.id) suser.locked = True self.user_store.commit() self.assertRaises(errors.LockedUserError, get_storage_user, user.id) # and ignore the lock too user = get_storage_user(user.id, readonly=True) self.assertTrue(isinstance(user, dao.StorageUser))
def _create_share(self, _, accept=False, dead=False, from_id=1): """Create the share to me.""" fromusr = get_storage_user(from_id) node = fromusr.root.load() share = node.share(self.usr0.id, u"name", readonly=True) self._state.subtree_id = node.id if accept: self.usr0.get_share(share.id).accept() self._state.share_id = share.id if dead: share.delete() return share
def main(username, sharer, wlist, num): """Create UDFs, folders and files for the given user using a wordlist.""" user = get_storage_user(None, username=utf2unicode(username)) sharer = get_storage_user(None, username=utf2unicode(sharer)) folders = [user.root] names = [utf2unicode(i.strip()) for i in file(wlist) if i.strip()] sample = random.sample(names, num) if sys.stdout.isatty(): import curses curses.setupterm() cols = curses.tigetnum('cols') - 1 progbar = makeStatBar(cols, cols - 2) home = curses.tigetstr('cr') def progress(l): """progress bar writer.""" sys.stdout.write(home + progbar((cols - 2) * (num - len(l)) / num)) sys.stdout.flush() else: progress = lambda l: None # UDF udf = user.make_udf('~/abundant-files') folders.append(user.volume(udf.id).get_node(udf.root_id)) # some UDFs for i in range(num / 100): progress(sample) folders.append(make_udf(user, sample)) for i in range(num / 4): progress(sample) name = sample.pop() folders.append(random.choice(folders).make_subdirectory(name)) sh_folders = [sharer.root] for i in range(num / 10): progress(sample) sh_folders.append(make_udf(sharer, sample)) for i in range(num / 10): progress(sample) name = sample.pop() sh_folders.append(random.choice(sh_folders).make_subdirectory(name)) for i in range(num / 20): progress(sample) name = sample.pop() filename = 'shared by ' + sharer.username readonly = random.choice((False, True)) if readonly: name += ' (ro)' filename += ' (ro)' folder = random.choice(sh_folders).make_subdirectory(name) folder.make_file(filename) share = folder.share(user.id, folder.name, readonly) user.get_share(share.id).accept() for i in random.sample(folders, len(folders) / 4): progress(sample) name = sample.pop() filename = 'shared by ' + user.username readonly = random.choice((False, True)) if readonly: name += ' (ro)' filename += ' (ro)' folder = random.choice(folders).make_subdirectory(name) folder.make_file(filename) share = folder.share(sharer.id, folder.name, readonly) sharer.get_share(share.id).accept() for i in range(num / 20): progress(sample) name = sample.pop() random.choice(folders).make_file(name) fake_hash = get_fake_hash() while sample: progress(sample) name = sample.pop() random.choice(folders).make_file_with_content( name, fake_hash, 12345, 100, 10000, uuid.uuid4(), 'image/tiff') if sys.stdout.isatty(): sys.stdout.write(home + curses.tigetstr('el'))
def _get_user(self, user_id, session_id=None): """Return a storage user for the given id.""" return services.get_storage_user(user_id, session_id=session_id)
def main(username, sharer, wlist, num): """Create UDFs, folders and files for the given user using a wordlist.""" user = get_storage_user(None, username=utf2unicode(username)) sharer = get_storage_user(None, username=utf2unicode(sharer)) folders = [user.root] names = [utf2unicode(i.strip()) for i in file(wlist) if i.strip()] sample = random.sample(names, num) if sys.stdout.isatty(): import curses curses.setupterm() cols = curses.tigetnum('cols') - 1 progbar = makeStatBar(cols, cols - 2) home = curses.tigetstr('cr') def progress(l): """progress bar writer.""" sys.stdout.write(home + progbar((cols - 2) * (num - len(l)) / num)) sys.stdout.flush() else: progress = lambda l: None # UDF udf = user.make_udf(u'~/abundant-files') folders.append(user.volume(udf.id).get_node(udf.root_id)) # some UDFs for i in range(num / 100): progress(sample) folders.append(make_udf(user, sample)) for i in range(num / 4): progress(sample) name = sample.pop() folders.append(random.choice(folders).make_subdirectory(name)) sh_folders = [sharer.root] for i in range(num / 10): progress(sample) sh_folders.append(make_udf(sharer, sample)) for i in range(num / 10): progress(sample) name = sample.pop() sh_folders.append(random.choice(sh_folders).make_subdirectory(name)) for i in range(num / 20): progress(sample) name = sample.pop() filename = u'shared by ' + sharer.username readonly = random.choice((False, True)) if readonly: name += ' (ro)' filename += ' (ro)' folder = random.choice(sh_folders).make_subdirectory(name) folder.make_file(filename) share = folder.share(user.id, folder.name, readonly) user.get_share(share.id).accept() for i in random.sample(folders, len(folders) / 4): progress(sample) name = sample.pop() filename = u'shared by ' + user.username readonly = random.choice((False, True)) if readonly: name += ' (ro)' filename += ' (ro)' folder = random.choice(folders).make_subdirectory(name) folder.make_file(filename) share = folder.share(sharer.id, folder.name, readonly) sharer.get_share(share.id).accept() for i in range(num / 20): progress(sample) name = sample.pop() random.choice(folders).make_file(name) fake_hash = get_fake_hash() while sample: progress(sample) name = sample.pop() random.choice(folders).make_file_with_content(name, fake_hash, 12345, 100, 10000, uuid.uuid4(), u'image/tiff') if sys.stdout.isatty(): sys.stdout.write(home + curses.tigetstr('el'))
d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i)) for f in range(10): f = d.make_file(u"%s file%s.txt" % (user.username, f)) uj = f.make_uploadjob(f.content_hash, get_fake_hash(str(random.random())), random.randint(1, 100), random.randint(100, 1000000), random.randint(100, 1000000)) uj.commit_content(f.content_hash, uuid.uuid4(), None) services.add_music_metadata(uj.file.content_hash, TEST_MUSIC_DATA, user.shard_id) for i in range(10): user.make_musicstore_download(random.randint(100, 1000000), random.randint(1, 100), u"https://fake.com/%s/%s" % (user.id, i), u"%s" % user.username, u"%s Greatest Hits" % user.username, u"Track %s" % i, "US") for i in range(10): downloadservices.get_or_make_download( user.id, user.root_volume_id, u"https://fake.com/%s/%s" % (user.id, i), u"/a/b", u"%s%s" % (user.id, i)) for username in user_names: user = services.get_storage_user(username=username) make_user_data(user) store = get_storage_store() store.commit()
u.get_share(s.id).accept() for i in range(10): d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i)) for f in range(10): f = d.make_file(u"%s file%s.txt" % (user.username, f)) uj = f.make_uploadjob( f.content_hash, get_fake_hash(str(random.random())), random.randint(1, 100), random.randint(100, 1000000), random.randint(100, 1000000)) uj.commit_content(f.content_hash, uuid.uuid4(), None) services.add_music_metadata( uj.file.content_hash, TEST_MUSIC_DATA, user.shard_id) for i in range(10): user.make_musicstore_download( random.randint(100, 1000000), random.randint(1, 100), u"https://fake.com/%s/%s" % (user.id, i), u"%s" % user.username, u"%s Greatest Hits" % user.username, u"Track %s" % i, "US") for i in range(10): downloadservices.get_or_make_download( user.id, user.root_volume_id, u"https://fake.com/%s/%s" % (user.id, i), u"/a/b", u"%s%s" % (user.id, i)) for username in user_names: user = services.get_storage_user(username=username) make_user_data(user) store = get_storage_store() store.commit()