def benchmark_one(bench, user_id):
"""Run a benchmark on a bunch of things."""
user = get_storage_user(int(user_id), active_only=False)
if user is None:
user = make_storage_user(int(user_id), u"TestUser", u"", 100 * (2**30))
# setup a test directory with loads of files.
testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S')
root = user.volume().root.make_subdirectory(testroot)
for i in range(10):
d = root.make_subdirectory(unicode(i))
make_deep_tree_with_files(d)
#move the children on the test root
moved_child = user.volume().get_node_by_path(u"%s/1" % testroot)
dest_dir = root.make_subdirectory(u"test-move")
with bench("big_move"):
moved_child.move(dest_dir.id, moved_child.name)
with bench("deltafull"):
generation, _, _ = user.volume().get_delta(0)
with bench("deltaempty"):
user.volume().get_delta(generation - 1000)
with bench("getmusic"):
user.volume().get_all_nodes(mimetypes=MIME_TYPES)
with bench("get_shared_volumes"):
user.get_share_volumes()
with bench("get_udfs"):
user.get_udfs()
def benchmark_one(bench, user_id):
"""Run a benchmark on a bunch of things."""
user = get_storage_user(int(user_id), active_only=False)
if user is None:
user = make_storage_user(int(user_id), u"TestUser",
u"", 100 * (2 ** 30))
# setup a test directory with loads of files.
testroot = u"stormbench-%s" % datetime.now().strftime('%Y%m%d%H%M%S')
root = user.volume().root.make_subdirectory(testroot)
for i in range(10):
d = root.make_subdirectory(unicode(i))
make_deep_tree_with_files(d)
# move the children on the test root
moved_child = user.volume().get_node_by_path(u"%s/1" % testroot)
dest_dir = root.make_subdirectory(u"test-move")
with bench("big_move"):
moved_child.move(dest_dir.id, moved_child.name)
with bench("deltafull"):
generation, _, _ = user.volume().get_delta(0)
with bench("deltaempty"):
user.volume().get_delta(generation - 1000)
with bench("getmusic"):
user.volume().get_all_nodes(mimetypes=MIME_TYPES)
with bench("get_shared_volumes"):
user.get_share_volumes()
with bench("get_udfs"):
user.get_udfs()
def make_user_data(user):
"""Make some user data."""
u = services.get_storage_user(username=user_names[random.randint(0, 4)])
s = user.root.share(u.id, u"%s to %s" % (user.username, u.username))
u.get_share(s.id).accept()
for i in range(10):
d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i))
for f in range(10):
f = d.make_file(u"%s file%s.txt" % (user.username, f))
uj = f.make_uploadjob(f.content_hash,
get_fake_hash(str(random.random())),
random.randint(1, 100),
random.randint(100, 1000000),
random.randint(100, 1000000))
uj.commit_content(f.content_hash, uuid.uuid4(), None)
services.add_music_metadata(uj.file.content_hash, TEST_MUSIC_DATA,
user.shard_id)
for i in range(10):
user.make_musicstore_download(random.randint(100, 1000000),
random.randint(1, 100),
u"https://fake.com/%s/%s" % (user.id, i),
u"%s" % user.username,
u"%s Greatest Hits" % user.username,
u"Track %s" % i, "US")
for i in range(10):
downloadservices.get_or_make_download(
user.id, user.root_volume_id,
u"https://fake.com/%s/%s" % (user.id, i), u"/a/b",
u"%s%s" % (user.id, i))
def make_user_data(user):
"""Make some user data."""
u = services.get_storage_user(username=user_names[random.randint(0, 4)])
s = user.root.share(u.id, u"%s to %s" % (user.username, u.username))
u.get_share(s.id).accept()
for i in range(10):
d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i))
for f in range(10):
f = d.make_file(u"%s file%s.txt" % (user.username, f))
uj = f.make_uploadjob(
f.content_hash, get_fake_hash(str(random.random())),
random.randint(1, 100), random.randint(100, 1000000),
random.randint(100, 1000000))
uj.commit_content(f.content_hash, uuid.uuid4(), None)
services.add_music_metadata(
uj.file.content_hash, TEST_MUSIC_DATA, user.shard_id)
for i in range(10):
user.make_musicstore_download(
random.randint(100, 1000000),
random.randint(1, 100), u"https://fake.com/%s/%s" % (user.id, i),
u"%s" % user.username, u"%s Greatest Hits" % user.username,
u"Track %s" % i, "US")
for i in range(10):
downloadservices.get_or_make_download(
user.id, user.root_volume_id,
u"https://fake.com/%s/%s" % (user.id, i),
u"/a/b", u"%s%s" % (user.id, i))
def create_share(self, user_id, node_id,
to_username, share_name, readonly):
"""Create a share."""
user = self._get_user(user_id)
to_user = services.get_storage_user(username=to_username)
share = user.volume().dir(node_id).share(to_user.id,
share_name, readonly)
return dict(share_id=share.id)
def create_share(self, user_id, node_id, to_username, share_name,
readonly):
"""Create a share."""
user = self._get_user(user_id)
to_user = services.get_storage_user(username=to_username)
share = user.volume().dir(node_id).share(to_user.id, share_name,
readonly)
return dict(share_id=share.id)
def test_get_storage_user(self):
"""Test the get_storage_user function."""
user = make_storage_user(
1, "Cool UserName", "Visible Name", MAX_STORAGE_BYTES)
user = get_storage_user(1)
self.assertTrue(isinstance(user, dao.StorageUser))
user.update(subscription=False)
self.assertRaises(errors.DoesNotExist, get_storage_user, 1)
user = get_storage_user(1, active_only=False)
user.update(subscription=True)
# now check a locked user.
suser = self.store.get(StorageUser, user.id)
suser.locked = True
self.store.commit()
self.assertRaises(errors.LockedUserError, get_storage_user, user.id)
# and ignore the lock too
user = get_storage_user(user.id, readonly=True)
self.assertTrue(isinstance(user, dao.StorageUser))
def test_get_storage_user(self):
"""Test the get_storage_user function."""
user = make_storage_user(1, u"Cool UserName", u"Visible Name",
MAX_STORAGE_BYTES)
user = get_storage_user(1)
self.assertTrue(isinstance(user, dao.StorageUser))
user.update(subscription=False)
self.assertRaises(errors.DoesNotExist, get_storage_user, 1)
user = get_storage_user(1, active_only=False)
user.update(subscription=True)
# now check a locked user.
suser = self.user_store.get(model.StorageUser, user.id)
suser.locked = True
self.user_store.commit()
self.assertRaises(errors.LockedUserError, get_storage_user, user.id)
# and ignore the lock too
user = get_storage_user(user.id, readonly=True)
self.assertTrue(isinstance(user, dao.StorageUser))
def _create_share(self, _, accept=False, dead=False, from_id=1):
"""Create the share to me."""
fromusr = get_storage_user(from_id)
node = fromusr.root.load()
share = node.share(self.usr0.id, u"name", readonly=True)
self._state.subtree_id = node.id
if accept:
self.usr0.get_share(share.id).accept()
self._state.share_id = share.id
if dead:
share.delete()
return share
def main(username, sharer, wlist, num):
"""Create UDFs, folders and files for the given user using a wordlist."""
user = get_storage_user(None, username=utf2unicode(username))
sharer = get_storage_user(None, username=utf2unicode(sharer))
folders = [user.root]
names = [utf2unicode(i.strip()) for i in file(wlist) if i.strip()]
sample = random.sample(names, num)
if sys.stdout.isatty():
import curses
curses.setupterm()
cols = curses.tigetnum('cols') - 1
progbar = makeStatBar(cols, cols - 2)
home = curses.tigetstr('cr')
def progress(l):
"""progress bar writer."""
sys.stdout.write(home + progbar((cols - 2) * (num - len(l)) / num))
sys.stdout.flush()
else:
progress = lambda l: None
# UDF
udf = user.make_udf('~/abundant-files')
folders.append(user.volume(udf.id).get_node(udf.root_id))
# some UDFs
for i in range(num / 100):
progress(sample)
folders.append(make_udf(user, sample))
for i in range(num / 4):
progress(sample)
name = sample.pop()
folders.append(random.choice(folders).make_subdirectory(name))
sh_folders = [sharer.root]
for i in range(num / 10):
progress(sample)
sh_folders.append(make_udf(sharer, sample))
for i in range(num / 10):
progress(sample)
name = sample.pop()
sh_folders.append(random.choice(sh_folders).make_subdirectory(name))
for i in range(num / 20):
progress(sample)
name = sample.pop()
filename = 'shared by ' + sharer.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(sh_folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(user.id, folder.name, readonly)
user.get_share(share.id).accept()
for i in random.sample(folders, len(folders) / 4):
progress(sample)
name = sample.pop()
filename = 'shared by ' + user.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(sharer.id, folder.name, readonly)
sharer.get_share(share.id).accept()
for i in range(num / 20):
progress(sample)
name = sample.pop()
random.choice(folders).make_file(name)
fake_hash = get_fake_hash()
while sample:
progress(sample)
name = sample.pop()
random.choice(folders).make_file_with_content(
name, fake_hash, 12345, 100, 10000, uuid.uuid4(), 'image/tiff')
if sys.stdout.isatty():
sys.stdout.write(home + curses.tigetstr('el'))
def _get_user(self, user_id, session_id=None):
"""Return a storage user for the given id."""
return services.get_storage_user(user_id, session_id=session_id)
def _get_user(self, user_id, session_id=None):
"""Return a storage user for the given id."""
return services.get_storage_user(user_id, session_id=session_id)
def main(username, sharer, wlist, num):
"""Create UDFs, folders and files for the given user using a wordlist."""
user = get_storage_user(None, username=utf2unicode(username))
sharer = get_storage_user(None, username=utf2unicode(sharer))
folders = [user.root]
names = [utf2unicode(i.strip()) for i in file(wlist) if i.strip()]
sample = random.sample(names, num)
if sys.stdout.isatty():
import curses
curses.setupterm()
cols = curses.tigetnum('cols') - 1
progbar = makeStatBar(cols, cols - 2)
home = curses.tigetstr('cr')
def progress(l):
"""progress bar writer."""
sys.stdout.write(home + progbar((cols - 2) * (num - len(l)) / num))
sys.stdout.flush()
else:
progress = lambda l: None
# UDF
udf = user.make_udf(u'~/abundant-files')
folders.append(user.volume(udf.id).get_node(udf.root_id))
# some UDFs
for i in range(num / 100):
progress(sample)
folders.append(make_udf(user, sample))
for i in range(num / 4):
progress(sample)
name = sample.pop()
folders.append(random.choice(folders).make_subdirectory(name))
sh_folders = [sharer.root]
for i in range(num / 10):
progress(sample)
sh_folders.append(make_udf(sharer, sample))
for i in range(num / 10):
progress(sample)
name = sample.pop()
sh_folders.append(random.choice(sh_folders).make_subdirectory(name))
for i in range(num / 20):
progress(sample)
name = sample.pop()
filename = u'shared by ' + sharer.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(sh_folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(user.id, folder.name, readonly)
user.get_share(share.id).accept()
for i in random.sample(folders, len(folders) / 4):
progress(sample)
name = sample.pop()
filename = u'shared by ' + user.username
readonly = random.choice((False, True))
if readonly:
name += ' (ro)'
filename += ' (ro)'
folder = random.choice(folders).make_subdirectory(name)
folder.make_file(filename)
share = folder.share(sharer.id, folder.name, readonly)
sharer.get_share(share.id).accept()
for i in range(num / 20):
progress(sample)
name = sample.pop()
random.choice(folders).make_file(name)
fake_hash = get_fake_hash()
while sample:
progress(sample)
name = sample.pop()
random.choice(folders).make_file_with_content(name, fake_hash, 12345,
100, 10000, uuid.uuid4(),
u'image/tiff')
if sys.stdout.isatty():
sys.stdout.write(home + curses.tigetstr('el'))
d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i))
for f in range(10):
f = d.make_file(u"%s file%s.txt" % (user.username, f))
uj = f.make_uploadjob(f.content_hash,
get_fake_hash(str(random.random())),
random.randint(1, 100),
random.randint(100, 1000000),
random.randint(100, 1000000))
uj.commit_content(f.content_hash, uuid.uuid4(), None)
services.add_music_metadata(uj.file.content_hash, TEST_MUSIC_DATA,
user.shard_id)
for i in range(10):
user.make_musicstore_download(random.randint(100, 1000000),
random.randint(1, 100),
u"https://fake.com/%s/%s" % (user.id, i),
u"%s" % user.username,
u"%s Greatest Hits" % user.username,
u"Track %s" % i, "US")
for i in range(10):
downloadservices.get_or_make_download(
user.id, user.root_volume_id,
u"https://fake.com/%s/%s" % (user.id, i), u"/a/b",
u"%s%s" % (user.id, i))
for username in user_names:
user = services.get_storage_user(username=username)
make_user_data(user)
store = get_storage_store()
store.commit()
u.get_share(s.id).accept()
for i in range(10):
d = user.root.make_subdirectory(u"%s dir%s" % (user.username, i))
for f in range(10):
f = d.make_file(u"%s file%s.txt" % (user.username, f))
uj = f.make_uploadjob(
f.content_hash, get_fake_hash(str(random.random())),
random.randint(1, 100), random.randint(100, 1000000),
random.randint(100, 1000000))
uj.commit_content(f.content_hash, uuid.uuid4(), None)
services.add_music_metadata(
uj.file.content_hash, TEST_MUSIC_DATA, user.shard_id)
for i in range(10):
user.make_musicstore_download(
random.randint(100, 1000000),
random.randint(1, 100), u"https://fake.com/%s/%s" % (user.id, i),
u"%s" % user.username, u"%s Greatest Hits" % user.username,
u"Track %s" % i, "US")
for i in range(10):
downloadservices.get_or_make_download(
user.id, user.root_volume_id,
u"https://fake.com/%s/%s" % (user.id, i),
u"/a/b", u"%s%s" % (user.id, i))
for username in user_names:
user = services.get_storage_user(username=username)
make_user_data(user)
store = get_storage_store()
store.commit()
