def auth(client):
yield client.dummy_authenticate('open sesame')
d = defer.Deferred()
client.connectionLostHandler = d.callback
# add the log handler
logger = logging.getLogger('storage.server')
hdlr = MementoHandler()
hdlr.setLevel(logging.INFO)
logger.addHandler(hdlr)
# patch the looping ping values
server = self.service.factory.protocols[0]
server.ping_loop.interval = 0.1
server.ping_loop.idle_timeout = 0.3
# reschedule the ping loop
server.ping_loop.reset()
try:
yield d
except ConnectionDone:
msg = "Disconnecting - idle timeout"
self.assertTrue(hdlr.check_info(msg))
else:
self.fail("Should get disconnected.")
finally:
logger.removeHandler(hdlr)
def auth(client):
yield client.dummy_authenticate('open sesame')
d = defer.Deferred()
client.connectionLostHandler = d.callback
# add the log handler
logger = logging.getLogger('storage.server')
hdlr = MementoHandler()
hdlr.setLevel(logging.INFO)
logger.addHandler(hdlr)
# patch the looping ping values
server = self.service.factory.protocols[0]
server.ping_loop.interval = 0.1
server.ping_loop.idle_timeout = 0.3
# reschedule the ping loop
server.ping_loop.reset()
try:
yield d
except ConnectionDone:
msg = "Disconnecting - idle timeout"
self.assertTrue(hdlr.check_info(msg))
else:
self.fail("Should get disconnected.")
finally:
logger.removeHandler(hdlr)
class MainTests(BaseTwistedTestCase):
""" Basic tests to check main.Main """
@defer.inlineCallbacks
def setUp(self):
""" Sets up a test. """
yield super(MainTests, self).setUp()
self.root = self.mktemp('root')
self.shares = self.mktemp('shares')
self.data = self.mktemp('data')
self.partials_dir = self.mktemp('partials_dir')
self.patch(main_mod, 'SyncdaemonService', FakedExternalInterface)
# no status listener by default
self.patch(main_mod.status_listener, "get_listener", lambda *a: None)
self.handler = MementoHandler()
self.handler.setLevel(logging.DEBUG)
self._logger = logging.getLogger('ubuntuone.SyncDaemon')
self._logger.addHandler(self.handler)
self.addCleanup(self._logger.removeHandler, self.handler)
def _get_main_common_params(self):
"""Return the parameters used by the all platforms."""
return dict(root_dir=self.root,
shares_dir=self.shares,
data_dir=self.data,
partials_dir=self.partials_dir,
host='localhost', port=0,
dns_srv=False, ssl=False,
mark_interval=60,
handshake_timeout=2,
auth_credentials=FAKED_CREDENTIALS,
monitor_class=FakeMonitor)
def build_main(self, **kwargs):
"""Build and return a Main object.
Use reasonable defaults for the tests, plus whatever extra kwargs are
passed in.
"""
# get the params using the platform code to ensure they are correct
params = self._get_main_common_params()
params.update(kwargs)
m = main_mod.Main(**params)
self.addCleanup(m.shutdown)
m.local_rescan = lambda *_: m.event_q.push('SYS_LOCAL_RESCAN_DONE')
return m
def test_main_initialization(self):
"""test that creating a Main instance works as expected."""
main = self.build_main()
self.assertIsInstance(main, main_mod.Main)
def test_main_start(self):
"""Test that Main.start works."""
main = self.build_main()
main.start()
def test_main_restarts_on_critical_error(self):
"""Test that Main restarts when syncdaemon gets into UNKNOWN_ERROR."""
self.restarted = False
main = self.build_main()
main.restart = lambda: setattr(self, 'restarted', True)
main.start()
main.event_q.push('SYS_UNKNOWN_ERROR')
self.assertTrue(self.restarted)
@defer.inlineCallbacks
def test_shutdown_pushes_sys_quit(self):
"""When shutting down, the SYS_QUIT event is pushed."""
params = self._get_main_common_params()
main = main_mod.Main(**params)
events = []
self.patch(main.event_q, 'push',
lambda *a, **kw: events.append((a, kw)))
yield main.shutdown()
expected = [(('SYS_USER_DISCONNECT',), {}), (('SYS_QUIT',), {})]
self.assertEqual(expected, events)
def test_handshake_timeout(self):
"""Check connecting times out."""
d0 = defer.Deferred()
class Handler:
"""Trivial event handler."""
def handle_SYS_HANDSHAKE_TIMEOUT(self):
"""Pass the test when we get this event."""
reactor.callLater(0, d0.callback, None)
main = self.build_main(handshake_timeout=0)
def fake_connect(*a):
"""Only connect when States told so."""
main.event_q.push('SYS_CONNECTION_MADE')
return defer.Deferred()
main.action_q.connect = fake_connect
# fake the following to not be executed
main.get_root = lambda *_: defer.Deferred()
main.action_q.check_version = lambda *_: defer.Deferred()
main.event_q.subscribe(Handler())
main.start()
main.event_q.push('SYS_NET_CONNECTED')
main.event_q.push('SYS_USER_CONNECT', access_token='')
return d0
def test_create_dirs_already_exists_dirs(self):
"""test that creating a Main instance works as expected."""
link = os.path.join(self.root, 'Shared With Me')
self.assertFalse(is_link(link))
self.assertTrue(path_exists(self.shares))
self.assertTrue(path_exists(self.root))
main = self.build_main()
# check that the shares link is actually a link
self.assertTrue(is_link(main.shares_dir_link))
self.assertEquals(link, main.shares_dir_link)
def test_create_dirs_already_exists_symlink_too(self):
"""test that creating a Main instance works as expected."""
link = os.path.join(self.root, 'Shared With Me')
make_link(self.shares, link)
self.assertTrue(is_link(link))
self.assertTrue(path_exists(self.shares))
self.assertTrue(path_exists(self.root))
main = self.build_main()
# check that the shares link is actually a link
self.assertTrue(is_link(main.shares_dir_link))
def test_create_dirs_already_exists_but_not_symlink(self):
"""test that creating a Main instance works as expected."""
link = os.path.join(self.root, 'Shared With Me')
make_dir(link, recursive=True)
self.assertTrue(path_exists(link))
self.assertFalse(is_link(link))
self.assertTrue(path_exists(self.shares))
self.assertTrue(path_exists(self.root))
main = self.build_main()
# check that the shares link is actually a link
self.assertEquals(main.shares_dir_link, link)
self.assertFalse(is_link(main.shares_dir_link))
def test_create_dirs_none_exists(self):
"""test that creating a Main instance works as expected."""
# remove the existing dirs
remove_dir(self.root)
remove_dir(self.shares)
main = self.build_main()
# check that the shares link is actually a link
self.assertTrue(is_link(main.shares_dir_link))
self.assertTrue(path_exists(self.shares))
self.assertTrue(path_exists(self.root))
def test_connect_if_autoconnect_is_enabled(self):
"""If autoconnect option is enabled, connect the syncdaemon."""
user_config = main_mod.config.get_user_config()
orig = user_config.get_autoconnect()
user_config.set_autoconnect(True)
self.addCleanup(user_config.set_autoconnect, orig)
main = self.build_main()
expected = [('connect', (), {'autoconnecting': True})]
self.assertEqual(main.external._called, expected)
def test_dont_connect_if_autoconnect_is_disabled(self):
"""If autoconnect option is disabled, do not connect the syncdaemon."""
user_config = main_mod.config.get_user_config()
orig = user_config.get_autoconnect()
user_config.set_autoconnect(False)
self.addCleanup(user_config.set_autoconnect, orig)
main = self.build_main()
self.assertEqual(main.external._called, [])
def _get_listeners(self, main):
"""Return the subscribed objects."""
s = set()
for listener in main.event_q.listener_map.values():
for x in listener:
s.add(x)
return s
def test_status_listener_is_installed(self):
"""The status listener is installed if needed."""
self.patch(main_mod.status_listener,
"get_listener", lambda *a: FakeListener())
main = self.build_main()
self.assertIn(main.status_listener, self._get_listeners(main))
def test_status_listener_not_installed_when_disabled(self):
"""The status listener is not started if it's not available."""
main = self.build_main()
self.assertNotIn(main.status_listener, self._get_listeners(main))
def test_get_homedir(self):
"""The get_homedir returns the root dir."""
self.patch(main_mod, "user_home", self.home_dir)
expected = expand_user('~')
main = self.build_main()
self.assertEqual(main.get_homedir(), expected)
def test_get_rootdir(self):
"""The get_rootdir returns the root dir."""
expected = expand_user(os.path.join('~', 'Ubuntu Test One'))
main = self.build_main(root_dir=expected)
self.assertEqual(main.get_rootdir(), expected)
def test_get_sharesdir(self):
"""The get_sharesdir returns the shares dir."""
expected = expand_user(os.path.join('~', 'Share it to Me'))
main = self.build_main(shares_dir=expected)
self.assertEqual(main.get_sharesdir(), expected)
def test_get_sharesdirlink(self):
"""The get_sharesdirlink returns the shares dir link."""
expected = 'Share it to Me'
main = self.build_main(shares_symlink_name=expected)
self.assertEqual(main.get_sharesdir_link(),
os.path.join(main.get_rootdir(), expected))
def test_version_is_logged(self):
"""Test that the client version is logged."""
self.build_main()
self.assertTrue(self.handler.check_info("client version", VERSION))
def test_mark(self):
"""Check the MARK logs ok."""
main = self.build_main()
main.log_mark()
shouldlog = ('MARK', "State: 'INIT'", 'queues IDLE', 'connection',
'queue: 0', 'offloaded: 0', 'hash: 0')
self.assertTrue(self.handler.check(NOTE, *shouldlog))
class RestHelperTestCase(StorageDALTestCase):
"""Test the resthelper."""
def setUp(self):
super(RestHelperTestCase, self).setUp()
self.handler = MementoHandler()
self.user = self.obj_factory.make_user(
1, "bob", "bobby boo", 2 * (2 ** 30))
self.mapper = ResourceMapper()
logger = logging.getLogger("test")
logger.addHandler(self.handler)
logger.setLevel(logging.INFO)
logger.propagate = False
self.helper = RestHelper(self.mapper, logger=logger)
def test_GET_user(self):
"""Test for dao to REST conversion of user"""
info = self.helper.get_user(self.user)
self.assertEqual(
info, self.mapper.user_repr(self.user, self.user.get_quota()))
user_id = repr(self.user.id)
self.assertTrue(self.handler.check_info("get_quota", user_id))
self.assertTrue(self.handler.check_info("get_udfs", user_id))
def test_GET_user_with_udf(self):
"""Test get_user with udf."""
udf = self.user.make_udf("~/Documents")
info = self.helper.get_user(self.user)
self.assertEqual(
info,
self.mapper.user_repr(self.user, self.user.get_quota(), [udf]))
def test_GET_volume(self):
"""Test get_volume."""
volume_path = "~/Documents"
udf = self.user.make_udf(volume_path)
info = self.helper.get_volume(user=self.user,
volume_path=volume_path)
self.assertEqual(info, self.mapper.volume_repr(udf))
ids = [repr(x) for x in [self.user.id, unicode(volume_path)]]
self.assertTrue(self.handler.check_info("get_udf_by_path", *ids))
def test_GET_volume_with_delta0(self):
"""Test get_volume with delta, no nodes"""
volume_path = "~/Documents"
udf = self.user.make_udf(volume_path)
info = self.helper.get_volume(
user=self.user,
volume_path=volume_path,
from_generation=0)
self.assertEqual(
info,
self.mapper.volume_repr(volume=udf, from_generation=0, nodes=[]))
ids = [repr(x) for x in [self.user.id, unicode(volume_path)]]
self.assertTrue(self.handler.check_info("get_udf_by_path", *ids))
ids = [repr(x) for x in [self.user.id, udf.id, 0]]
self.assertTrue(self.handler.check_info("get_delta", *ids))
def test_GET_volume_with_delta1(self):
"""Test get_volume with delta, with nodes"""
volume_path = "~/Documents"
self.user.make_udf(volume_path)
node0 = self.user.make_file_by_path("~/Documents/file0.txt")
node1 = self.user.make_file_by_path("~/Documents/file1.txt")
info = self.helper.get_volume(
user=self.user,
volume_path=volume_path,
from_generation=0)
udf = self.user.get_udf_by_path('~/Documents')
self.assertEqual(info, self.mapper.volume_repr(
volume=udf, from_generation=0, nodes=[node0, node1]))
node0.delete()
info = self.helper.get_volume(
user=self.user,
volume_path=volume_path,
from_generation=0)
self.assertEqual(info['delta']['nodes'][1]['is_live'], False)
def test_PUT_volume(self):
"""Test put volume."""
path = "~/Documents"
info = self.helper.put_volume(user=self.user, path=path)
udf = self.user.get_udf_by_path(path)
self.assertEqual(self.mapper.volume_repr(udf), info)
ids = [repr(x) for x in [self.user.id, unicode(path)]]
self.assertTrue(self.handler.check_info("make_udf", *ids))
def test_GET_node_directory(self):
"""Test for get_node a directory node."""
root = self.user.volume().get_root()
d1 = root.make_subdirectory("dir1")
full_path = "~/Ubuntu One" + d1.full_path
info = self.helper.get_node(user=self.user, node_path=full_path)
self.assertEqual(info, self.mapper.node_repr(d1))
def test_GET_node_file(self):
"""Test for get_node conversion of a file node."""
root = self.user.volume().get_root()
f1 = root.make_file("file.txt")
volume_path = "~/Ubuntu One"
full_path = volume_path + f1.full_path
info = self.helper.get_node(user=self.user, node_path=full_path)
self.assertEqual(info, self.mapper.node_repr(f1))
ids = [repr(x) for x in [self.user.id, full_path, True]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
def test_GET_volumes(self):
"""Test get_volume."""
udfs = [self.user.make_udf("~/Udf%s" % i) for i in range(10)]
info = self.helper.get_volumes(self.user)
root = self.user.volume().get_volume()
expected_repr = [self.mapper.volume_repr(root)]
expected_repr.extend([self.mapper.volume_repr(u) for u in udfs])
info = info.sort(key=operator.itemgetter('path'))
expected_repr = expected_repr.sort(key=operator.itemgetter('path'))
self.assertEqual(info, expected_repr)
self.assertTrue(self.handler.check_info("get_volume",
repr(self.user.id)))
self.assertTrue(self.handler.check_info(
"get_udfs", repr(self.user.id)))
def test_DELETE_volume(self):
"""Test delete_volume."""
udf = self.user.make_udf("~/Documents")
self.helper.delete_volume(self.user, udf.path)
self.assertRaises(errors.DoesNotExist,
self.user.get_udf, udf.id)
ids = [repr(x) for x in [self.user.id, udf.path]]
self.assertTrue(self.handler.check_info("get_udf_by_path", *ids))
ids = [repr(x) for x in [self.user.id, udf.id]]
self.assertTrue(self.handler.check_info("delete_udf", *ids))
def test_GET_node0(self):
"""Test simple node info."""
root = self.user.volume().get_root()
f1 = root.make_file("file.txt")
full_path = "~/Ubuntu One" + f1.full_path
info = self.helper.get_node(self.user, full_path)
self.assertEqual(info, self.mapper.node_repr(f1))
def test_GET_node1(self):
"""Test child node info."""
root = self.user.volume().get_root()
d1 = root.make_subdirectory("Documents")
f1 = d1.make_file("file.txt")
full_path = "~/Ubuntu One" + os.path.join(d1.full_path, f1.name)
info = self.helper.get_node(self.user, full_path)
self.assertEqual(info['key'], f1.nodekey)
self.assertEqual(info['path'], f1.full_path)
def test_GET_node2(self):
"""Test simple udf node info."""
self.user.make_udf("~/Documents")
udf = self.user.get_node_by_path("~/Documents")
f1 = udf.make_file("file.txt")
full_path = "~/Documents" + f1.full_path
info = self.helper.get_node(self.user, full_path)
self.assertEqual(info['key'], f1.nodekey)
self.assertEqual(info['path'], f1.full_path)
def test_GET_node3(self):
"""Test child udf node info."""
self.user.make_udf("~/Documents")
udf = self.user.get_node_by_path("~/Documents")
d1 = udf.make_subdirectory("slides")
f1 = d1.make_file("file.txt")
full_path = "~/Documents" + f1.full_path
info = self.helper.get_node(self.user, full_path)
self.assertEqual(info, self.mapper.node_repr(f1))
def test_DELETE_node(self):
"""Test delete_volume."""
root = self.user.volume().get_root()
f1 = root.make_file("file.txt")
full_path = "~/Ubuntu One" + f1.full_path
self.helper.delete_node(self.user, full_path)
self.assertRaises(errors.DoesNotExist,
self.user.volume().get_node, f1.id)
ids = [repr(x) for x in [self.user.id, full_path]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
ids = [repr(x) for x in [self.user.id, f1.id, True]]
self.assertTrue(self.handler.check_info("delete", *ids))
def test_GET_node_children(self):
"""Test get_node_children."""
root = self.user.volume().get_root()
files = [root.make_file("file%s.txt" % i) for i in range(10)]
full_path = "~/Ubuntu One"
root.load()
expected = self.mapper.node_repr(root)
expected['children'] = [self.mapper.node_repr(n) for n in files]
info = self.helper.get_node(
self.user, full_path, include_children=True)
self.assertEqual(info, expected)
ids = [repr(x) for x in [self.user.id, full_path, True]]
self.assertTrue(self.handler.check_info("get_node", *ids))
ids = [repr(x) for x in [self.user.id, root.id, True]]
self.assertTrue(self.handler.check_info("get_children", *ids))
def test_GET_file_node_children(self):
"""Test get_node_children."""
self.user.volume().root.make_file("file.txt")
self.assertRaises(FileNodeHasNoChildren,
self.helper.get_node, self.user,
"~/Ubuntu One/file.txt", include_children=True)
def test_PUT_node_is_public(self):
"""Test put node to make existing file public."""
original_metrics = self.helper.metrics
self.helper.metrics = FakeMetrics()
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.is_public, False)
node_rep = self.mapper.node_repr(node)
node_rep['is_public'] = True
info = self.helper.put_node(self.user, new_file_path, node_rep)
ids = [repr(x) for x in [self.user.id, new_file_path]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
ids = [repr(x) for x in [self.user.id, node.id, True]]
self.assertTrue(self.handler.check_info("change_public_access", *ids))
node.load()
self.assertEqual(node.is_public, True)
self.assertEqual(info, self.mapper.node_repr(node))
info['is_public'] = False
info = self.helper.put_node(self.user, new_file_path, info)
node.load()
self.assertEqual(node.is_public, False)
self.assertEqual(info, self.mapper.node_repr(node))
self.helper.metrics.make_all_assertions(
self, 'resthelper.put_node.change_public')
self.helper.metrics = original_metrics
def test_GET_public_files(self):
"""Test public_files returns the list of public files."""
self.assertEqual(self.helper.get_public_files(self.user), [])
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.is_public, False)
node_rep = self.mapper.node_repr(node)
node_rep['is_public'] = True
info = self.helper.put_node(self.user, new_file_path, node_rep)
self.assertEqual(self.helper.get_public_files(self.user), [info])
self.assertTrue(self.handler.check_info("get_public_files",
repr(self.user.id)))
def test_PUT_node_is_public_directory(self):
"""Test put node to make existing file public."""
dir_path = "~/Ubuntu One/a/b/c"
node = self.user.make_tree_by_path(dir_path)
self.assertEqual(node.is_public, False)
node_rep = self.mapper.node_repr(node)
node_rep['is_public'] = True
self.assertRaises(CannotPublishDirectory,
self.helper.put_node, self.user, dir_path, node_rep)
def test_PUT_node_path(self):
"""Test put node with a new path."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.full_path, "/a/b/c/file.txt")
node_rep = self.mapper.node_repr(node)
new_path = "/a/newfile.txt"
node_rep['path'] = new_path
info = self.helper.put_node(self.user, new_file_path, node_rep)
node.load()
self.assertEqual(node.full_path, new_path)
self.assertEqual(info, self.mapper.node_repr(node))
ids = [repr(x) for x in [self.user.id, new_file_path]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
new_dir, new_name = os.path.split(new_path)
ids = [repr(x) for x in [self.user.id, node.vol_id, unicode(new_dir)]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
ids = [repr(x) for x in [self.user.id, node.id, unicode(new_name)]]
self.assertTrue(self.handler.check_info("move", *ids))
def test_PUT_node_path_is_public(self):
"""Test put node with a new path and make it public."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.full_path, "/a/b/c/file.txt")
node_rep = self.mapper.node_repr(node)
node_rep['path'] = "/a/newfile.txt"
node_rep['is_public'] = True
info = self.helper.put_node(self.user, new_file_path, node_rep)
node.load()
self.assertEqual(node.is_public, True)
self.assertEqual(node.full_path, "/a/newfile.txt")
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_is_public_partial(self):
"""Test put node."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.is_public, False)
info = self.helper.put_node(self.user, new_file_path,
{'is_public': True})
node.load()
self.assertEqual(node.is_public, True)
self.assertEqual(info, self.mapper.node_repr(node))
info = self.helper.put_node(self.user, new_file_path,
{'is_public': False})
node.load()
self.assertEqual(node.is_public, False)
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_path_partial(self):
"""Test put node with a new path with partial info."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
info = self.helper.put_node(self.user, new_file_path,
{'path': "/a/newfile.txt"})
node.load()
self.assertEqual(node.full_path, "/a/newfile.txt")
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_path_is_pulic_partial(self):
"""Test put node with a new path and make it public."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
info = self.helper.put_node(
self.user, new_file_path,
{'path': "/a/newfile.txt", 'is_public': True})
node.load()
self.assertEqual(node.full_path, "/a/newfile.txt")
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_do_nothing(self):
"""Test put_node with nothing to do."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
node_repr = self.mapper.node_repr(node)
info = self.helper.put_node(self.user, new_file_path,
dict(a=2, b='hi', c='ignored'))
node.load()
# here nothing is changed and the info returned
# matches the existing node_repr
self.assertEqual(info, node_repr)
self.assertEqual(node_repr, self.mapper.node_repr(node))
def test_PUT_node_new_file_magic(self):
"""Test put_node to make a new file with content."""
cb = get_test_contentblob("FakeContent")
cb.magic_hash = b'magic'
self.store.add(cb)
self.store.commit()
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(
self.user, new_file_path,
{'kind': 'file', 'hash': cb.hash, 'magic_hash': 'magic'})
node = self.user.get_node_by_path(new_file_path)
self.assertEqual(node.kind, StorageObject.FILE)
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_update_file_magic(self):
"""Test put_node to make a new file with content."""
cb = get_test_contentblob("FakeContent")
cb.magic_hash = b'magic'
self.store.add(cb)
self.store.commit()
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(
self.user, new_file_path,
{'kind': 'file', 'hash': cb.hash, 'magic_hash': 'magic'})
cb = get_test_contentblob("NewFakeContent")
cb.magic_hash = b'magic2'
self.store.add(cb)
self.store.commit()
info = self.helper.put_node(
self.user, new_file_path,
{'kind': 'file', 'hash': cb.hash, 'magic_hash': 'magic2'})
node = self.user.get_node_by_path(new_file_path, with_content=True)
self.assertEqual(node.kind, StorageObject.FILE)
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
self.assertEqual(node.content.magic_hash, 'magic2')
def test_PUT_node_new_file(self):
"""Test put_node to make a new file."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(self.user, new_file_path,
{'kind': 'file'})
node = self.user.get_node_by_path(new_file_path)
self.assertEqual(node.kind, StorageObject.FILE)
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_new_directory(self):
"""Test put_node to make a new directory."""
new_file_path = "~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(self.user, new_file_path,
{'kind': 'directory'})
node = self.user.get_node_by_path(new_file_path)
self.assertEqual(node.kind, StorageObject.DIRECTORY)
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_exceptions(self):
"""Test put_node exceptions."""
self.assertRaises(InvalidKind,
self.helper.put_node,
self.user, "~/Ubuntu one/x", {"kind": "ABC"})
# PUT to a non existent node.
self.assertRaises(errors.DoesNotExist,
self.helper.put_node,
self.user, "~/Ubuntu/x", {})
# PUT to a non existent node.
self.assertRaises(errors.DoesNotExist,
self.helper.put_node,
self.user, "~/Ubuntu One/x", {})
class ClientDummyAuthTests(AuthenticationBaseTestCase):
"""Client authentication tests using the dummy auth provider."""
auth_provider_class = DummyAuthProvider
@defer.inlineCallbacks
def setUp(self):
yield super(ClientDummyAuthTests, self).setUp()
self.creds = "open sesame"
self.bad_creds = "not my secret"
self.handler = MementoHandler()
logger = logging.getLogger("storage.server")
logger.addHandler(self.handler)
self.addCleanup(logger.removeHandler, self.handler)
self.handler.setLevel(logging.DEBUG)
def assert_auth_ok_logging(self):
self.assertTrue(self.handler.check_debug("authenticated user", "OK", self.usr0.username))
self.assertFalse(self.handler.check_warning("missing user"))
def assert_auth_ok_missing_user(self):
self.assertTrue(self.handler.check_debug("missing user", "(id=%s)" % self.usr0.id))
self.assertFalse(self.handler.check_info("authenticated user"))
@defer.inlineCallbacks
def test_auth_ok_user_ok(self):
"""Correct authentication must succeed."""
yield self.callback_test(self.do_auth, credentials=self.creds, add_default_callbacks=True)
self.assert_auth_ok_logging()
@defer.inlineCallbacks
def test_auth_ok_bad_user(self):
"""Non existing user must fail authentication."""
# make the user getter fail
self.patch(self.service.factory.content, "get_user_by_id", lambda *a, **k: defer.fail(DoesNotExist()))
d = self.callback_test(self.do_auth, credentials=self.creds, add_default_callbacks=True)
yield self.assertFailure(d, protocol_errors.AuthenticationFailedError)
self.assert_auth_ok_missing_user()
@defer.inlineCallbacks
def test_auth_ok_with_session_id(self):
"""Correct authentication must succeed and include the session_id."""
auth_request = yield self.callback_test(self.do_auth, credentials=self.creds, add_default_callbacks=True)
protocol = self.service.factory.protocols[0]
self.assertEqual(auth_request.session_id, str(protocol.session_id))
@defer.inlineCallbacks
def test_auth_ok_with_metadata(self):
"""Correct authentication must succeed and include metadata."""
m_called = []
self.service.factory.metrics.meter = lambda *a: m_called.append(a)
metadata = {u"platform": u"linux2", u"version": u"1.0", u"foo": u"bar"}
yield self.callback_test(self.do_auth, credentials=self.creds, metadata=metadata, add_default_callbacks=True)
self.assertTrue(self.handler.check_info("Client metadata: %s" % metadata))
self.assertIn(("client.platform.linux2", 1), m_called)
self.assertIn(("client.version.1_0", 1), m_called)
self.assertNotIn(("client.foo.bar", 1), m_called)
def test_auth_fail(self):
"""Wrong secret must fail."""
def test(client, **kwargs):
d = self.do_auth(client, credentials=self.bad_creds)
d.addCallbacks(
lambda _: client.test_fail(Exception("Should not succeed.")), lambda _: client.test_done("ok")
)
return self.callback_test(test)
def test_get_root(self):
"""Must receive the root after authentication."""
@defer.inlineCallbacks
def test(client, **kwargs):
yield self.do_auth(client, credentials=self.creds)
root_id = yield client.get_root()
self.assertIsNotNone(root_id)
return self.callback_test(test, add_default_callbacks=True)
def test_get_root_twice(self):
"""Get root must keep the root id."""
@defer.inlineCallbacks
def test(client, **kwargs):
yield self.do_auth(client, credentials=self.creds)
root_id1 = yield client.get_root()
root_id2 = yield client.get_root()
self.assertEqual(root_id1, root_id2)
return self.callback_test(test, add_default_callbacks=True)
def test_user_becomes_inactive(self):
"""After StorageUser authentication ok it becomes inactive."""
@defer.inlineCallbacks
def test(client):
"""Test."""
yield self.do_auth(client, credentials=self.creds)
root_id = yield client.get_root()
# create one file, should be ok
yield client.make_file(request.ROOT, root_id, "f1")
# cancel user subscription, so it needs
# to get it again from the DB
self.usr0.update(subscription=False)
# create second file, should NOT be ok
try:
yield client.make_file(request.ROOT, root_id, "f2")
except protocol_errors.DoesNotExistError:
pass # failed as we expected
else:
client.test_fail("It should have failed!")
return self.callback_test(test, add_default_callbacks=True)
class HeartbeatWriterTest(TwistedTestCase):
"""Tests for HeartbeatWriter."""
interval = 5
@defer.inlineCallbacks
def setUp(self):
yield super(HeartbeatWriterTest, self).setUp()
self.logger = logging.Logger("HeartbeatWriter.test")
self.handler = MementoHandler()
self.logger.addHandler(self.handler)
self.addCleanup(self.logger.removeHandler, self.handler)
self.clock = task.Clock()
self.hw = HeartbeatWriter(self.interval, self.logger,
reactor=self.clock)
def test_send_no_transport(self):
"""Log a warning when there is no transport."""
self.hw.send()
self.assertTrue(self.handler.check_warning(
"Can't send heartbeat without a transport"))
def test_send_loop(self):
"""Send heartbeats in the LoopingCall."""
# first connect to something
transport = StringIO()
self.clock.advance(2)
self.hw.makeConnection(transport)
self.clock.advance(5)
self.clock.advance(5)
# we should have 3 heartbeats in the transport, get them
raw_events = transport.getvalue().split(BEGIN_TOKEN, 3)
events = []
for raw_event in raw_events:
if raw_event:
events.append(json.loads(raw_event.strip(END_TOKEN)))
# strip the tokens
for i, timestamp in [(0, 2), (1, 7), (2, 12)]:
self.assertEqual(events[i]['type'], "heartbeat")
self.assertEqual(events[i]['time'], timestamp)
def test_send_on_connectionMade(self):
"""On connectionMade start the loop and send."""
# first connect to something
transport = StringIO()
self.clock.advance(0.1)
self.hw.makeConnection(transport)
self.assertTrue(self.hw.loop.running)
raw_event = transport.getvalue()
self.assertTrue(raw_event.startswith(BEGIN_TOKEN))
self.assertTrue(raw_event.endswith(END_TOKEN))
# strip the tokens
payload = json.loads(raw_event.strip(BEGIN_TOKEN).strip(END_TOKEN))
self.assertEqual(payload['type'], "heartbeat")
self.assertEqual(payload['time'], self.clock.seconds())
def test_connectionLost(self):
"""On connectionLost cleanup everything."""
self.hw.makeConnection(None)
called = []
self.patch(self.hw.loop, 'stop', lambda: called.append(True))
self.hw.connectionLost(protocol.connectionDone)
self.assertTrue(self.handler.check_info(
"HeartbeatWriter connectionLost: %s" % (protocol.connectionDone,)))
self.assertTrue(called)
self.assertEqual(self.hw.loop, None)
self.assertEqual(self.hw.reactor, None)
self.assertEqual(self.hw.logger, None)
class HeartbeatListenerTestCase(TestCase):
"""Tests for HeartbeatListener class."""
def setUp(self):
super(HeartbeatListenerTestCase, self).setUp()
self.stdin = StringIO()
self.stdout = StringIO()
self.stderr = StringIO()
self.mocker = Mocker()
self.rpc = self.mocker.mock()
self.listener = HeartbeatListener(1, 10, ['foo'], [], self.rpc,
stdin=self.stdin, stdout=self.stdout,
stderr=self.stderr)
self.next_fail = {}
self.handler = MementoHandler()
self.listener.logger.addHandler(self.handler)
self.listener.logger.setLevel(logging.DEBUG)
self.handler.setLevel(logging.DEBUG)
self.listener.logger.propagate = False
self.processes = [dict(name="heartbeat", group="heartbeat", pid="101",
state=RUNNING)]
self.handler.debug = True
def tearDown(self):
self.listener.logger.removeHandler(self.handler)
self.handler.close()
self.next_fail = None
self.handler = None
self.listener = None
super(HeartbeatListenerTestCase, self).tearDown()
def fail_next_stop(self, pname):
"""Make next stopProcess to fail."""
expect(self.rpc.supervisor.stopProcess(pname)).throw(
xmlrpclib.Fault(42, "Failed to stop the process."))
def fail_next_start(self, pname):
"""Make next startProcess to fail."""
expect(self.rpc.supervisor.startProcess(pname)).throw(
xmlrpclib.Fault(42, "Failed to start the process."))
def test_restart(self):
"""Test the restart method."""
expect(self.rpc.supervisor.stopProcess("foo"))
expect(self.rpc.supervisor.startProcess("foo"))
with self.mocker:
self.listener.restart("foo", "testing")
self.assertTrue(self.handler.check_info("Restarting foo (last "
"hearbeat: testing)"))
def test_restart_fail_stop(self):
"""Test the restart method failing to stop the process."""
self.fail_next_stop("foo")
last = time.time()
with self.mocker:
try:
self.listener.restart("foo", last)
except xmlrpclib.Fault:
msg = ("Failed to stop process %s (last heartbeat: %s), "
"exiting: %s") % \
("foo", last, "<Fault 42: 'Failed to stop the process.'>")
self.assertTrue(self.handler.check_error(msg))
else:
self.fail("Should get an xmlrpclib.Fault")
def test_restart_fail_start(self):
"""Test the restart method failing to start the process."""
expect(self.rpc.supervisor.stopProcess("foo"))
self.fail_next_start("foo")
last = time.time()
with self.mocker:
try:
self.listener.restart("foo", last)
except xmlrpclib.Fault:
msg = (
'Failed to start process %s after stopping it, exiting: %s'
) % ("foo", "<Fault 42: 'Failed to start the process.'>")
self.assertTrue(self.handler.check_error(msg))
else:
self.fail("Should get an xmlrpclib.Fault")
def test_check_processes(self):
"""Test the check_processes method."""
# add the fake process to the process list
self.processes.append(dict(name="foo", group="foo", pid="42",
state=RUNNING))
self.processes.append(dict(name="bar", group="bar", pid="43",
state=RUNNING))
self.listener.processes = ['bar']
# 2 process to restart
self.listener.data['foo'] = {
'time': time.time() - (self.listener.timeout + 2)}
self.listener.data['bar'] = {
'time': time.time() - (self.listener.timeout + 3)}
self.listener.data['p-1'] = {
'time': time.time() - (self.listener.timeout - 1)}
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
expect(self.rpc.supervisor.stopProcess("foo:"))
expect(self.rpc.supervisor.startProcess("foo:"))
expect(self.rpc.supervisor.stopProcess("bar:bar"))
expect(self.rpc.supervisor.startProcess("bar:bar"))
with self.mocker:
self.listener.check_processes()
def test_check_processes_no_data(self):
"""Test the check_processes method with no data of a process."""
# add the fake process to the process list
self.processes.append(dict(name="foo", group="foo", pid="42",
state=RUNNING))
self.processes.append(dict(name="bar", group="bar", pid="43",
state=RUNNING))
self.listener.processes = ['bar']
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
expect(self.rpc.supervisor.stopProcess("foo:"))
expect(self.rpc.supervisor.startProcess("foo:"))
expect(self.rpc.supervisor.stopProcess("bar:bar"))
expect(self.rpc.supervisor.startProcess("bar:bar"))
with self.mocker:
# one process to restart
self.listener.check_processes()
self.assertTrue(self.handler.check_warning(
"Restarting process foo:foo (42), as we never received a hearbeat"
" event from it"))
self.assertTrue(self.handler.check_warning(
"Restarting process bar:bar (43), as we never received a hearbeat"
" event from it"))
def test_check_processes_untracked(self):
"""Test the check_processes method with a untracked proccess."""
# add the fake process to the process list
self.processes.append(dict(name="foo-untracked", group="untracked",
pid="43", state=RUNNING))
# add a new tracked process from an untracked group
self.processes.append(dict(name="bar-untracked", group="bar", pid="44",
state=RUNNING))
self.listener.processes = ['bar']
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
with self.mocker:
self.listener.check_processes()
self.assertTrue(self.handler.check_info(
"Ignoring untracked:foo-untracked (43) as isn't tracked."))
self.assertTrue(self.handler.check_info(
"Ignoring bar:bar-untracked (44) as isn't tracked."))
def test_check_processes_not_running(self):
"""Test the check_processes method if the proccess isn't running."""
# add the fake process to the process list
self.processes.append(dict(name="foo", group="foo", pid="42",
state=states.ProcessStates.STARTING))
# add a new tracked process from an untracked group
self.processes.append(dict(name="bar", group="bar", pid="43",
state=states.ProcessStates.STARTING))
self.listener.processes = ['bar']
# 2 processes to restart
self.listener.data['foo'] = {
'time': time.time() - (self.listener.timeout + 2)}
self.listener.data['bar'] = {
'time': time.time() - (self.listener.timeout + 2)}
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
with self.mocker:
self.listener.check_processes()
self.assertTrue(self.handler.check_info(
"Ignoring foo:foo (42) as isn't running."))
self.assertTrue(self.handler.check_info(
"Ignoring bar:bar (43) as isn't running."))
def test_handle_heartbeat(self):
"""Test handle_heartbeat method."""
payload = {"time": time.time()}
self.listener.handle_heartbeat('process_name', 'group_name',
'42', payload)
info = {"pid": "42", "time": payload["time"],
"received": self.listener.data["process_name"]["received"]}
self.assertEqual({"process_name": info}, self.listener.data)
def test_handle_event(self):
"""Test handle_event method."""
# patch handle_heartbeat
called = []
def handle_heartbeat(process_name, group_name, pid, payload):
"""Fake handle_heartbeat."""
called.append((process_name, group_name, pid, payload))
self.listener.handle_heartbeat = handle_heartbeat
payload_dict = {u"time": time.time(), "type": "heartbeat"}
raw_data = ("processname:ticker groupname:ticker pid:42\n" +
json.dumps(payload_dict))
raw_header = ("ver:3.0 server:supervisor serial:1 pool:listener "
"poolserial:10 eventname:PROCESS_COMMUNICATION_STDOUT"
" len:%s\n" % len(raw_data))
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
headers = childutils.get_headers(raw_header)
self.listener._handle_event()
# check
self.assertEqual(1, len(called))
del payload_dict['type']
self.assertEqual(('ticker', 'ticker', '42', payload_dict), called[0])
self.assertTrue(self.handler.check_debug(
"Event '%s' received: %r" % (headers['eventname'], raw_data)))
# check the stdout info
self.assertEqual(["READY", "RESULT 2", "OK"],
self.stdout.getvalue().split("\n"))
def test_invalid_event_type(self):
"""Test with an invalid type."""
payload_dict = {u"time": time.time(), "type": "ping"}
raw_data = 'processname:ticker groupname:ticker pid:42\n' + \
json.dumps(payload_dict)
raw_header = ("ver:3.0 server:supervisor serial:1 pool:listener "
"poolserial:10 eventname:PROCESS_COMMUNICATION_STDOUT"
" len:%s\n" % len(raw_data))
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
self.listener._handle_event()
# check
self.assertTrue(self.handler.check_error(
"Unable to handle event type '%s' - %r" % ('ping', raw_data)))
def test_invalid_payload(self):
"""Test with an invalid payload."""
payload_dict = {u"time": time.time(), "type": "ping"}
raw_data = 'processname:ticker groupname:ticker pid:42\n' + \
json.dumps(payload_dict) + "<!foo>"
raw_header = ("ver:3.0 server:supervisor serial:1 pool:listener "
"poolserial:10 eventname:PROCESS_COMMUNICATION_STDOUT"
" len:%s\n" % len(raw_data))
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
self.listener._handle_event()
# check
self.assertTrue(self.handler.check_error(
"Unable to handle event type '%s' - %r" % ('None', raw_data)))
def test_unhandled_event(self):
"""A unhandled event type."""
payload_dict = {u"time": time.time(), "type": "ping"}
raw_data = 'processname:ticker groupname:ticker pid:42\n' + \
json.dumps(payload_dict)
raw_header = "ver:3.0 server:supervisor serial:1 pool:heartbeat " + \
"poolserial:1 eventname:UNKNOWN len:%s\n" % len(raw_data)
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
self.listener._handle_event()
# check
self.assertTrue(self.handler.check_warning(
"Received unsupported event: %s - %r" % ('UNKNOWN', raw_data)))
def test_check_interval(self):
"""Check that we properly check on the specified interval."""
header = "ver:3.0 server:supervisor serial:1 pool:heartbeat " + \
"poolserial:1 eventname:TICK_5 len:0\n"
expect(self.rpc.supervisor.getAllProcessInfo()).result([])
self.stdin.write(header)
self.stdin.seek(0)
self.listener._handle_event()
self.assertEqual(self.listener.tick_count, 1)
self.stdin.seek(0)
with self.mocker:
self.listener._handle_event()
class HeartbeatWriterTest(TwistedTestCase):
"""Tests for HeartbeatWriter."""
interval = 5
@defer.inlineCallbacks
def setUp(self):
yield super(HeartbeatWriterTest, self).setUp()
self.logger = logging.Logger("HeartbeatWriter.test")
self.handler = MementoHandler()
self.logger.addHandler(self.handler)
self.addCleanup(self.logger.removeHandler, self.handler)
self.clock = task.Clock()
self.hw = HeartbeatWriter(self.interval,
self.logger,
reactor=self.clock)
def test_send_no_transport(self):
"""Log a warning when there is no transport."""
self.hw.send()
self.assertTrue(
self.handler.check_warning(
"Can't send heartbeat without a transport"))
def test_send_loop(self):
"""Send heartbeats in the LoopingCall."""
# first connect to something
transport = StringIO()
self.clock.advance(2)
self.hw.makeConnection(transport)
self.clock.advance(5)
self.clock.advance(5)
# we should have 3 heartbeats in the transport, get them
raw_events = transport.getvalue().split(BEGIN_TOKEN, 3)
events = []
for raw_event in raw_events:
if raw_event:
events.append(json.loads(raw_event.strip(END_TOKEN)))
# strip the tokens
for i, timestamp in [(0, 2), (1, 7), (2, 12)]:
self.assertEqual(events[i]['type'], "heartbeat")
self.assertEqual(events[i]['time'], timestamp)
def test_send_on_connectionMade(self):
"""On connectionMade start the loop and send."""
# first connect to something
transport = StringIO()
self.clock.advance(0.1)
self.hw.makeConnection(transport)
self.assertTrue(self.hw.loop.running)
raw_event = transport.getvalue()
self.assertTrue(raw_event.startswith(BEGIN_TOKEN))
self.assertTrue(raw_event.endswith(END_TOKEN))
# strip the tokens
payload = json.loads(raw_event.strip(BEGIN_TOKEN).strip(END_TOKEN))
self.assertEqual(payload['type'], "heartbeat")
self.assertEqual(payload['time'], self.clock.seconds())
def test_connectionLost(self):
"""On connectionLost cleanup everything."""
self.hw.makeConnection(None)
called = []
self.patch(self.hw.loop, 'stop', lambda: called.append(True))
self.hw.connectionLost(protocol.connectionDone)
self.assertTrue(
self.handler.check_info("HeartbeatWriter connectionLost: %s" %
(protocol.connectionDone, )))
self.assertTrue(called)
self.assertEqual(self.hw.loop, None)
self.assertEqual(self.hw.reactor, None)
self.assertEqual(self.hw.logger, None)
class ClientDummyAuthTests(AuthenticationBaseTestCase):
"""Client authentication tests using the dummy auth provider."""
auth_provider_class = DummyAuthProvider
@defer.inlineCallbacks
def setUp(self):
yield super(ClientDummyAuthTests, self).setUp()
self.creds = 'open sesame'
self.bad_creds = 'not my secret'
self.handler = MementoHandler()
logger = logging.getLogger('storage.server')
logger.addHandler(self.handler)
self.addCleanup(logger.removeHandler, self.handler)
self.handler.setLevel(logging.DEBUG)
def assert_auth_ok_logging(self):
self.assertTrue(
self.handler.check_debug("authenticated user", "OK",
self.usr0.username))
self.assertFalse(self.handler.check_warning("missing user"))
def assert_auth_ok_missing_user(self):
self.assertTrue(
self.handler.check_debug("missing user", "(id=%s)" % self.usr0.id))
self.assertFalse(self.handler.check_info("authenticated user"))
@defer.inlineCallbacks
def test_auth_ok_user_ok(self):
"""Correct authentication must succeed."""
yield self.callback_test(self.do_auth,
credentials=self.creds,
add_default_callbacks=True)
self.assert_auth_ok_logging()
@defer.inlineCallbacks
def test_auth_ok_bad_user(self):
"""Non existing user must fail authentication."""
# make the user getter fail
self.patch(self.service.factory.content, 'get_user_by_id',
lambda *a, **k: defer.fail(DoesNotExist()))
d = self.callback_test(self.do_auth,
credentials=self.creds,
add_default_callbacks=True)
yield self.assertFailure(d, protocol_errors.AuthenticationFailedError)
self.assert_auth_ok_missing_user()
@defer.inlineCallbacks
def test_auth_ok_with_session_id(self):
"""Correct authentication must succeed and include the session_id."""
auth_request = yield self.callback_test(self.do_auth,
credentials=self.creds,
add_default_callbacks=True)
protocol = self.service.factory.protocols[0]
self.assertEqual(auth_request.session_id, str(protocol.session_id))
@defer.inlineCallbacks
def test_auth_ok_with_metadata(self):
"""Correct authentication must succeed and include metadata."""
m_called = []
self.service.factory.metrics.meter = lambda *a: m_called.append(a)
metadata = {u"platform": u"linux2", u"version": u"1.0", u"foo": u"bar"}
yield self.callback_test(self.do_auth,
credentials=self.creds,
metadata=metadata,
add_default_callbacks=True)
self.assertTrue(
self.handler.check_info("Client metadata: %s" % metadata))
self.assertIn(("client.platform.linux2", 1), m_called)
self.assertIn(("client.version.1_0", 1), m_called)
self.assertNotIn(("client.foo.bar", 1), m_called)
def test_auth_fail(self):
"""Wrong secret must fail."""
def test(client, **kwargs):
d = self.do_auth(client, credentials=self.bad_creds)
d.addCallbacks(
lambda _: client.test_fail(Exception("Should not succeed.")),
lambda _: client.test_done("ok"))
return self.callback_test(test)
def test_get_root(self):
"""Must receive the root after authentication."""
@defer.inlineCallbacks
def test(client, **kwargs):
yield self.do_auth(client, credentials=self.creds)
root_id = yield client.get_root()
self.assertIsNotNone(root_id)
return self.callback_test(test, add_default_callbacks=True)
def test_get_root_twice(self):
"""Get root must keep the root id."""
@defer.inlineCallbacks
def test(client, **kwargs):
yield self.do_auth(client, credentials=self.creds)
root_id1 = yield client.get_root()
root_id2 = yield client.get_root()
self.assertEqual(root_id1, root_id2)
return self.callback_test(test, add_default_callbacks=True)
def test_user_becomes_inactive(self):
"""After StorageUser authentication ok it becomes inactive."""
@defer.inlineCallbacks
def test(client):
"""Test."""
yield self.do_auth(client, credentials=self.creds)
root_id = yield client.get_root()
# create one file, should be ok
yield client.make_file(request.ROOT, root_id, "f1")
# cancel user subscription, so it needs
# to get it again from the DB
self.usr0.update(subscription=False)
# create second file, should NOT be ok
try:
yield client.make_file(request.ROOT, root_id, "f2")
except protocol_errors.DoesNotExistError:
pass # failed as we expected
else:
client.test_fail("It should have failed!")
return self.callback_test(test, add_default_callbacks=True)
class StatsWorkerTestCase(TestCase):
"""Tests for StatsWorker class."""
def setUp(self):
super(StatsWorkerTestCase, self).setUp()
self.mocker = Mocker()
self.rpc = self.mocker.mock()
self.worker = stats_worker.StatsWorker(10, '', self.rpc)
# logging setup
self.handler = MementoHandler()
self.worker.logger.addHandler(self.handler)
self.addCleanup(self.worker.logger.removeHandler, self.handler)
self.worker.logger.setLevel(logging.DEBUG)
self.handler.setLevel(logging.DEBUG)
self.worker.logger.propagate = False
self.handler.debug = True
def test_collect_stats(self):
"""Test the collect_stats method."""
called = []
self.worker._collect_process = \
lambda p, n: called.append(('proc', p, n)) or {}
self.worker._collect_machine = lambda: called.append('machine') or {}
processes = [dict(name="bar", group="foo", pid="42", state=RUNNING)]
expect(self.rpc.supervisor.getAllProcessInfo()).result(processes)
with self.mocker:
self.worker.collect_stats()
self.assertEqual(called, ['machine', ('proc', 42, 'bar')])
self.assertTrue(self.handler.check_info("Collecting machine stats"))
self.assertTrue(self.handler.check_info("Collecting stats for proc",
"pid=42", "name=bar"))
def test_collect_stats_not_running(self):
"""Test the collect_stats method if the proccess isn't running."""
called = []
self.worker._collect_process = \
lambda p, n: called.append(('proc', p, n)) or {}
self.worker._collect_machine = lambda: called.append('machine') or {}
processes = [dict(name="bar", group="foo", pid="42", state=STARTING)]
expect(self.rpc.supervisor.getAllProcessInfo()).result(processes)
with self.mocker:
self.worker.collect_stats()
self.assertEqual(called, ['machine'])
self.assertTrue(self.handler.check_info("Collecting machine stats"))
self.assertTrue(self.handler.check_info("Ignoring process",
"pid=42", "name=bar",
"state=%s" % STARTING))
def test_collect_stats_no_data(self):
"""Test the collect_stats method with no data of a process."""
called = []
self.worker._collect_process = \
lambda p, n: called.append(('proc', p, n)) or {}
self.worker._collect_machine = lambda: called.append('machine') or {}
expect(self.rpc.supervisor.getAllProcessInfo()).result([])
with self.mocker:
self.worker.collect_stats()
self.assertEqual(called, ['machine'])
self.assertTrue(self.handler.check_info("Collecting machine stats"))
def test_collect_process_info_new_report(self):
"""Check how the process info is collected first time."""
mocker = Mocker()
assert not self.worker.process_cache
# patch Process to return our mock for test pid
Process = mocker.mock()
self.patch(stats_worker.psutil, 'Process', Process)
proc = mocker.mock()
pid = 1234
expect(Process(pid)).result(proc)
# patch ProcessReport to return or mock for given proc
ProcessReport = mocker.mock()
self.patch(stats_worker, 'ProcessReport', ProcessReport)
proc_report = mocker.mock()
expect(ProcessReport(proc)).result(proc_report)
# expect to get called with some info, return some results
name = 'test_proc'
result = object()
expect(proc_report.get_memory_and_cpu(prefix=name)).result(result)
with mocker:
real = self.worker._collect_process(pid, name)
self.assertIdentical(real, result)
def test_collect_process_info_old_report(self):
"""Check how the process info is collected when cached."""
mocker = Mocker()
# put it in the cache
pid = 1234
proc_report = mocker.mock()
self.worker.process_cache[pid] = proc_report
# expect to get called with some info, return some results
name = 'test_proc'
result = object()
expect(proc_report.get_memory_and_cpu(prefix=name)).result(result)
with mocker:
real = self.worker._collect_process(pid, name)
self.assertIdentical(real, result)
def test_collect_system_info(self):
"""Check how the system info is collected."""
mocker = Mocker()
# change the constant to assure it's used as we want
result1 = dict(a=3, b=5)
result2 = dict(c=7)
fake = (lambda: result1, lambda: result2)
self.patch(stats_worker, 'SYSTEM_STATS', fake)
with mocker:
result = self.worker._collect_machine()
should = {}
should.update(result1)
should.update(result2)
self.assertEqual(result, should)
def test_informed_metrics(self):
"""Check how stats are reported."""
# prepare a lot of fake info that will be "collected"
machine_info = dict(foo=3, bar=5)
process_info = {
1: dict(some=1234, other=4567),
2: dict(some=9876, other=6543),
}
self.worker._collect_process = lambda pid, name: process_info[pid]
self.worker._collect_machine = lambda: machine_info
processes = [
dict(name="proc1", group="", pid="1", state=RUNNING),
dict(name="proc2", group="", pid="2", state=RUNNING),
]
expect(self.rpc.supervisor.getAllProcessInfo()).result(processes)
# patch the metric reporter to see what is sent
reported = set()
self.worker.metrics.gauge = lambda *a: reported.add(a)
# what we should get is...
should = set([
('foo', 3),
('bar', 5),
('some', 1234),
('other', 4567),
('some', 9876),
('other', 6543),
])
with self.mocker:
self.worker.collect_stats()
self.assertEqual(reported, should)
class RestHelperTestCase(StorageDALTestCase):
"""Test the resthelper."""
def setUp(self):
super(RestHelperTestCase, self).setUp()
self.handler = MementoHandler()
self.user = self.obj_factory.make_user(1, u"bob", u"bobby boo",
2 * (2**30))
self.mapper = ResourceMapper()
logger = logging.getLogger("test")
logger.addHandler(self.handler)
logger.setLevel(logging.INFO)
logger.propagate = False
self.helper = RestHelper(self.mapper, logger=logger)
self.store = self.get_shard_store(self.user.shard_id)
def test_GET_user(self):
"""Test for dao to REST conversion of user"""
info = self.helper.get_user(self.user)
self.assertEqual(
info, self.mapper.user_repr(self.user, self.user.get_quota()))
user_id = repr(self.user.id)
self.assertTrue(self.handler.check_info("get_quota", user_id))
self.assertTrue(self.handler.check_info("get_udfs", user_id))
def test_GET_user_with_udf(self):
"""Test get_user with udf."""
udf = self.user.make_udf(u"~/Documents")
info = self.helper.get_user(self.user)
self.assertEqual(
info, self.mapper.user_repr(self.user, self.user.get_quota(),
[udf]))
def test_GET_volume(self):
"""Test get_volume."""
volume_path = u"~/Documents"
udf = self.user.make_udf(volume_path)
info = self.helper.get_volume(user=self.user, volume_path=volume_path)
self.assertEqual(info, self.mapper.volume_repr(udf))
ids = [repr(x) for x in [self.user.id, unicode(volume_path)]]
self.assertTrue(self.handler.check_info("get_udf_by_path", *ids))
def test_GET_volume_with_delta0(self):
"""Test get_volume with delta, no nodes"""
volume_path = u"~/Documents"
udf = self.user.make_udf(volume_path)
info = self.helper.get_volume(user=self.user,
volume_path=volume_path,
from_generation=0)
self.assertEqual(
info,
self.mapper.volume_repr(volume=udf, from_generation=0, nodes=[]))
ids = [repr(x) for x in [self.user.id, unicode(volume_path)]]
self.assertTrue(self.handler.check_info("get_udf_by_path", *ids))
ids = [repr(x) for x in [self.user.id, udf.id, 0]]
self.assertTrue(self.handler.check_info("get_delta", *ids))
def test_GET_volume_with_delta1(self):
"""Test get_volume with delta, with nodes"""
volume_path = u"~/Documents"
self.user.make_udf(volume_path)
node0 = self.user.make_file_by_path(u"~/Documents/file0.txt")
node1 = self.user.make_file_by_path(u"~/Documents/file1.txt")
info = self.helper.get_volume(user=self.user,
volume_path=volume_path,
from_generation=0)
udf = self.user.get_udf_by_path(u'~/Documents')
self.assertEqual(
info,
self.mapper.volume_repr(volume=udf,
from_generation=0,
nodes=[node0, node1]))
node0.delete()
info = self.helper.get_volume(user=self.user,
volume_path=volume_path,
from_generation=0)
self.assertEqual(info['delta']['nodes'][1]['is_live'], False)
def test_PUT_volume(self):
"""Test put volume."""
path = u"~/Documents"
info = self.helper.put_volume(user=self.user, path=path)
udf = self.user.get_udf_by_path(path)
self.assertEquals(self.mapper.volume_repr(udf), info)
ids = [repr(x) for x in [self.user.id, unicode(path)]]
self.assertTrue(self.handler.check_info("make_udf", *ids))
def test_GET_node_directory(self):
"""Test for get_node a directory node."""
root = self.user.volume().get_root()
d1 = root.make_subdirectory(u"dir1")
full_path = u"~/Ubuntu One" + d1.full_path
info = self.helper.get_node(user=self.user, node_path=full_path)
self.assertEquals(info, self.mapper.node_repr(d1))
def test_GET_node_file(self):
"""Test for get_node conversion of a file node."""
root = self.user.volume().get_root()
f1 = root.make_file(u"file.txt")
volume_path = u"~/Ubuntu One"
full_path = volume_path + f1.full_path
info = self.helper.get_node(user=self.user, node_path=full_path)
self.assertEquals(info, self.mapper.node_repr(f1))
ids = [repr(x) for x in [self.user.id, full_path, True]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
def test_GET_volumes(self):
"""Test get_volume."""
udfs = [self.user.make_udf(u"~/Udf%s" % i) for i in range(10)]
info = self.helper.get_volumes(self.user)
root = self.user.volume().get_volume()
expected_repr = [self.mapper.volume_repr(root)]
expected_repr.extend([self.mapper.volume_repr(u) for u in udfs])
info = info.sort(key=operator.itemgetter('path'))
expected_repr = expected_repr.sort(key=operator.itemgetter('path'))
self.assertEquals(info, expected_repr)
self.assertTrue(
self.handler.check_info("get_volume", repr(self.user.id)))
self.assertTrue(self.handler.check_info("get_udfs",
repr(self.user.id)))
def test_DELETE_volume(self):
"""Test delete_volume."""
udf = self.user.make_udf(u"~/Documents")
self.helper.delete_volume(self.user, udf.path)
self.assertRaises(errors.DoesNotExist, self.user.get_udf, udf.id)
ids = [repr(x) for x in [self.user.id, udf.path]]
self.assertTrue(self.handler.check_info("get_udf_by_path", *ids))
ids = [repr(x) for x in [self.user.id, udf.id]]
self.assertTrue(self.handler.check_info("delete_udf", *ids))
def test_GET_node0(self):
"""Test simple node info."""
root = self.user.volume().get_root()
f1 = root.make_file(u"file.txt")
full_path = u"~/Ubuntu One" + f1.full_path
info = self.helper.get_node(self.user, full_path)
self.assertEqual(info, self.mapper.node_repr(f1))
def test_GET_node1(self):
"""Test child node info."""
root = self.user.volume().get_root()
d1 = root.make_subdirectory(u"Documents")
f1 = d1.make_file(u"file.txt")
full_path = u"~/Ubuntu One" + os.path.join(d1.full_path, f1.name)
info = self.helper.get_node(self.user, full_path)
self.assertEquals(info['key'], f1.nodekey)
self.assertEquals(info['path'], f1.full_path)
def test_GET_node2(self):
"""Test simple udf node info."""
self.user.make_udf(u"~/Documents")
udf = self.user.get_node_by_path(u"~/Documents")
f1 = udf.make_file(u"file.txt")
full_path = u"~/Documents" + f1.full_path
info = self.helper.get_node(self.user, full_path)
self.assertEquals(info['key'], f1.nodekey)
self.assertEquals(info['path'], f1.full_path)
def test_GET_node3(self):
"""Test child udf node info."""
self.user.make_udf(u"~/Documents")
udf = self.user.get_node_by_path(u"~/Documents")
d1 = udf.make_subdirectory(u"slides")
f1 = d1.make_file(u"file.txt")
full_path = u"~/Documents" + f1.full_path
info = self.helper.get_node(self.user, full_path)
self.assertEqual(info, self.mapper.node_repr(f1))
def test_DELETE_node(self):
"""Test delete_volume."""
root = self.user.volume().get_root()
f1 = root.make_file(u"file.txt")
full_path = u"~/Ubuntu One" + f1.full_path
self.helper.delete_node(self.user, full_path)
self.assertRaises(errors.DoesNotExist,
self.user.volume().get_node, f1.id)
ids = [repr(x) for x in [self.user.id, full_path]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
ids = [repr(x) for x in [self.user.id, f1.id, True]]
self.assertTrue(self.handler.check_info("delete", *ids))
def test_GET_node_children(self):
"""Test get_node_children."""
root = self.user.volume().get_root()
files = [root.make_file(u"file%s.txt" % i) for i in range(10)]
full_path = u"~/Ubuntu One"
root.load()
expected = self.mapper.node_repr(root)
expected['children'] = [self.mapper.node_repr(n) for n in files]
info = self.helper.get_node(self.user,
full_path,
include_children=True)
self.assertEquals(info, expected)
ids = [repr(x) for x in [self.user.id, full_path, True]]
self.assertTrue(self.handler.check_info("get_node", *ids))
ids = [repr(x) for x in [self.user.id, root.id, True]]
self.assertTrue(self.handler.check_info("get_children", *ids))
def test_GET_file_node_children(self):
"""Test get_node_children."""
self.user.volume().root.make_file(u"file.txt")
self.assertRaises(FileNodeHasNoChildren,
self.helper.get_node,
self.user,
"~/Ubuntu One/file.txt",
include_children=True)
def test_PUT_node_is_public(self):
"""Test put node to make existing file public."""
original_metrics = self.helper.metrics
self.helper.metrics = FakeMetrics()
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.is_public, False)
node_rep = self.mapper.node_repr(node)
node_rep['is_public'] = True
info = self.helper.put_node(self.user, new_file_path, node_rep)
ids = [repr(x) for x in [self.user.id, new_file_path]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
ids = [repr(x) for x in [self.user.id, node.id, True]]
self.assertTrue(self.handler.check_info("change_public_access", *ids))
node.load()
self.assertEqual(node.is_public, True)
self.assertEqual(info, self.mapper.node_repr(node))
info['is_public'] = False
info = self.helper.put_node(self.user, new_file_path, info)
node.load()
self.assertEqual(node.is_public, False)
self.assertEqual(info, self.mapper.node_repr(node))
self.helper.metrics.make_all_assertions(
self, 'resthelper.put_node.change_public')
self.helper.metrics = original_metrics
def test_GET_public_files(self):
"""Test public_files returns the list of public files."""
self.assertEqual(self.helper.get_public_files(self.user), [])
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.is_public, False)
node_rep = self.mapper.node_repr(node)
node_rep['is_public'] = True
info = self.helper.put_node(self.user, new_file_path, node_rep)
self.assertEqual(self.helper.get_public_files(self.user), [info])
self.assertTrue(
self.handler.check_info("get_public_files", repr(self.user.id)))
def test_PUT_node_is_public_directory(self):
"""Test put node to make existing file public."""
dir_path = u"~/Ubuntu One/a/b/c"
node = self.user.make_tree_by_path(dir_path)
self.assertEqual(node.is_public, False)
node_rep = self.mapper.node_repr(node)
node_rep['is_public'] = True
self.assertRaises(CannotPublishDirectory, self.helper.put_node,
self.user, dir_path, node_rep)
def test_PUT_node_path(self):
"""Test put node with a new path."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.full_path, "/a/b/c/file.txt")
node_rep = self.mapper.node_repr(node)
new_path = "/a/newfile.txt"
node_rep['path'] = new_path
info = self.helper.put_node(self.user, new_file_path, node_rep)
node.load()
self.assertEqual(node.full_path, new_path)
self.assertEqual(info, self.mapper.node_repr(node))
ids = [repr(x) for x in [self.user.id, new_file_path]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
new_dir, new_name = os.path.split(new_path)
ids = [repr(x) for x in [self.user.id, node.vol_id, unicode(new_dir)]]
self.assertTrue(self.handler.check_info("get_node_by_path", *ids))
ids = [repr(x) for x in [self.user.id, node.id, unicode(new_name)]]
self.assertTrue(self.handler.check_info("move", *ids))
def test_PUT_node_path_is_public(self):
"""Test put node with a new path and make it public."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.full_path, "/a/b/c/file.txt")
node_rep = self.mapper.node_repr(node)
node_rep['path'] = "/a/newfile.txt"
node_rep['is_public'] = True
info = self.helper.put_node(self.user, new_file_path, node_rep)
node.load()
self.assertEqual(node.is_public, True)
self.assertEqual(node.full_path, "/a/newfile.txt")
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_is_public_partial(self):
"""Test put node."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
self.assertEqual(node.is_public, False)
info = self.helper.put_node(self.user, new_file_path,
{'is_public': True})
node.load()
self.assertEqual(node.is_public, True)
self.assertEqual(info, self.mapper.node_repr(node))
info = self.helper.put_node(self.user, new_file_path,
{'is_public': False})
node.load()
self.assertEqual(node.is_public, False)
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_path_partial(self):
"""Test put node with a new path with partial info."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
info = self.helper.put_node(self.user, new_file_path,
{'path': "/a/newfile.txt"})
node.load()
self.assertEqual(node.full_path, "/a/newfile.txt")
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_path_is_pulic_partial(self):
"""Test put node with a new path and make it public."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
info = self.helper.put_node(self.user, new_file_path, {
'path': "/a/newfile.txt",
'is_public': True
})
node.load()
self.assertEqual(node.full_path, "/a/newfile.txt")
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_do_nothing(self):
"""Test put_node with nothing to do."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
node = self.user.make_file_by_path(new_file_path)
node_repr = self.mapper.node_repr(node)
info = self.helper.put_node(self.user, new_file_path,
dict(a=2, b='hi', c='ignored'))
node.load()
# here nothing is changed and the info returned
# matches the existing node_repr
self.assertEqual(info, node_repr)
self.assertEqual(node_repr, self.mapper.node_repr(node))
def test_PUT_node_new_file_magic(self):
"""Test put_node to make a new file with content."""
cb = get_test_contentblob("FakeContent")
cb.magic_hash = 'magic'
self.store.add(cb)
self.store.commit()
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(self.user, new_file_path, {
'kind': 'file',
'hash': cb.hash,
'magic_hash': 'magic'
})
node = self.user.get_node_by_path(new_file_path)
self.assertEqual(node.kind, 'File')
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_update_file_magic(self):
"""Test put_node to make a new file with content."""
cb = get_test_contentblob("FakeContent")
cb.magic_hash = 'magic'
self.store.add(cb)
self.store.commit()
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(self.user, new_file_path, {
'kind': 'file',
'hash': cb.hash,
'magic_hash': 'magic'
})
cb = get_test_contentblob("NewFakeContent")
cb.magic_hash = 'magic2'
self.store.add(cb)
self.store.commit()
info = self.helper.put_node(self.user, new_file_path, {
'kind': 'file',
'hash': cb.hash,
'magic_hash': 'magic2'
})
node = self.user.get_node_by_path(new_file_path, with_content=True)
self.assertEqual(node.kind, 'File')
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
self.assertEqual(node.content.magic_hash, 'magic2')
def test_PUT_node_new_file(self):
"""Test put_node to make a new file."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(self.user, new_file_path, {'kind': 'file'})
node = self.user.get_node_by_path(new_file_path)
self.assertEqual(node.kind, 'File')
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_new_directory(self):
"""Test put_node to make a new directory."""
new_file_path = u"~/Ubuntu One/a/b/c/file.txt"
info = self.helper.put_node(self.user, new_file_path,
{'kind': 'directory'})
node = self.user.get_node_by_path(new_file_path)
self.assertEqual(node.kind, 'Directory')
self.assertEqual(node.full_path, '/a/b/c/file.txt')
self.assertEqual(info, self.mapper.node_repr(node))
def test_PUT_node_exceptions(self):
"""Test put_node exceptions."""
self.assertRaises(InvalidKind, self.helper.put_node, self.user,
"~/Ubuntu one/x", {"kind": "ABC"})
# PUT to a non existent node.
self.assertRaises(errors.DoesNotExist, self.helper.put_node, self.user,
"~/Ubuntu/x", {})
# PUT to a non existent node.
self.assertRaises(errors.DoesNotExist, self.helper.put_node, self.user,
"~/Ubuntu One/x", {})
class HeartbeatListenerTestCase(TestCase):
"""Tests for HeartbeatListener class."""
def setUp(self):
super(HeartbeatListenerTestCase, self).setUp()
self.stdin = StringIO()
self.stdout = StringIO()
self.stderr = StringIO()
self.mocker = Mocker()
self.rpc = self.mocker.mock()
self.listener = HeartbeatListener(1,
10, ['foo'], [],
self.rpc,
stdin=self.stdin,
stdout=self.stdout,
stderr=self.stderr)
self.next_fail = {}
self.handler = MementoHandler()
self.listener.logger.addHandler(self.handler)
self.listener.logger.setLevel(logging.DEBUG)
self.handler.setLevel(logging.DEBUG)
self.listener.logger.propagate = False
self.processes = [
dict(name="heartbeat", group="heartbeat", pid="101", state=RUNNING)
]
self.handler.debug = True
def tearDown(self):
self.listener.logger.removeHandler(self.handler)
self.handler.close()
self.next_fail = None
self.handler = None
self.listener = None
super(HeartbeatListenerTestCase, self).tearDown()
def fail_next_stop(self, pname):
"""Make next stopProcess to fail."""
expect(self.rpc.supervisor.stopProcess(pname)).throw(
xmlrpclib.Fault(42, "Failed to stop the process."))
def fail_next_start(self, pname):
"""Make next startProcess to fail."""
expect(self.rpc.supervisor.startProcess(pname)).throw(
xmlrpclib.Fault(42, "Failed to start the process."))
def test_restart(self):
"""Test the restart method."""
expect(self.rpc.supervisor.stopProcess("foo"))
expect(self.rpc.supervisor.startProcess("foo"))
with self.mocker:
self.listener.restart("foo", "testing")
self.assertTrue(
self.handler.check_info("Restarting foo (last "
"hearbeat: testing)"))
def test_restart_fail_stop(self):
"""Test the restart method failing to stop the process."""
self.fail_next_stop("foo")
last = time.time()
with self.mocker:
try:
self.listener.restart("foo", last)
except xmlrpclib.Fault:
msg = ("Failed to stop process %s (last heartbeat: %s), "
"exiting: %s") % \
("foo", last, "<Fault 42: 'Failed to stop the process.'>")
self.assertTrue(self.handler.check_error(msg))
else:
self.fail("Should get an xmlrpclib.Fault")
def test_restart_fail_start(self):
"""Test the restart method failing to start the process."""
expect(self.rpc.supervisor.stopProcess("foo"))
self.fail_next_start("foo")
last = time.time()
with self.mocker:
try:
self.listener.restart("foo", last)
except xmlrpclib.Fault:
msg = (
'Failed to start process %s after stopping it, exiting: %s'
) % ("foo", "<Fault 42: 'Failed to start the process.'>")
self.assertTrue(self.handler.check_error(msg))
else:
self.fail("Should get an xmlrpclib.Fault")
def test_check_processes(self):
"""Test the check_processes method."""
# add the fake process to the process list
self.processes.append(
dict(name="foo", group="foo", pid="42", state=RUNNING))
self.processes.append(
dict(name="bar", group="bar", pid="43", state=RUNNING))
self.listener.processes = ['bar']
# 2 process to restart
self.listener.data['foo'] = {
'time': time.time() - (self.listener.timeout + 2)
}
self.listener.data['bar'] = {
'time': time.time() - (self.listener.timeout + 3)
}
self.listener.data['p-1'] = {
'time': time.time() - (self.listener.timeout - 1)
}
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
expect(self.rpc.supervisor.stopProcess("foo:"))
expect(self.rpc.supervisor.startProcess("foo:"))
expect(self.rpc.supervisor.stopProcess("bar:bar"))
expect(self.rpc.supervisor.startProcess("bar:bar"))
with self.mocker:
self.listener.check_processes()
def test_check_processes_no_data(self):
"""Test the check_processes method with no data of a process."""
# add the fake process to the process list
self.processes.append(
dict(name="foo", group="foo", pid="42", state=RUNNING))
self.processes.append(
dict(name="bar", group="bar", pid="43", state=RUNNING))
self.listener.processes = ['bar']
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
expect(self.rpc.supervisor.stopProcess("foo:"))
expect(self.rpc.supervisor.startProcess("foo:"))
expect(self.rpc.supervisor.stopProcess("bar:bar"))
expect(self.rpc.supervisor.startProcess("bar:bar"))
with self.mocker:
# one process to restart
self.listener.check_processes()
self.assertTrue(
self.handler.check_warning(
"Restarting process foo:foo (42), as we never received a hearbeat"
" event from it"))
self.assertTrue(
self.handler.check_warning(
"Restarting process bar:bar (43), as we never received a hearbeat"
" event from it"))
def test_check_processes_untracked(self):
"""Test the check_processes method with a untracked proccess."""
# add the fake process to the process list
self.processes.append(
dict(name="foo-untracked",
group="untracked",
pid="43",
state=RUNNING))
# add a new tracked process from an untracked group
self.processes.append(
dict(name="bar-untracked", group="bar", pid="44", state=RUNNING))
self.listener.processes = ['bar']
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
with self.mocker:
self.listener.check_processes()
self.assertTrue(
self.handler.check_info(
"Ignoring untracked:foo-untracked (43) as isn't tracked."))
self.assertTrue(
self.handler.check_info(
"Ignoring bar:bar-untracked (44) as isn't tracked."))
def test_check_processes_not_running(self):
"""Test the check_processes method if the proccess isn't running."""
# add the fake process to the process list
self.processes.append(
dict(name="foo",
group="foo",
pid="42",
state=states.ProcessStates.STARTING))
# add a new tracked process from an untracked group
self.processes.append(
dict(name="bar",
group="bar",
pid="43",
state=states.ProcessStates.STARTING))
self.listener.processes = ['bar']
# 2 processes to restart
self.listener.data['foo'] = {
'time': time.time() - (self.listener.timeout + 2)
}
self.listener.data['bar'] = {
'time': time.time() - (self.listener.timeout + 2)
}
expect(self.rpc.supervisor.getAllProcessInfo()).result(self.processes)
with self.mocker:
self.listener.check_processes()
self.assertTrue(
self.handler.check_info("Ignoring foo:foo (42) as isn't running."))
self.assertTrue(
self.handler.check_info("Ignoring bar:bar (43) as isn't running."))
def test_handle_heartbeat(self):
"""Test handle_heartbeat method."""
payload = {"time": time.time()}
self.listener.handle_heartbeat('process_name', 'group_name', '42',
payload)
info = {
"pid": "42",
"time": payload["time"],
"received": self.listener.data["process_name"]["received"]
}
self.assertEqual({"process_name": info}, self.listener.data)
def test_handle_event(self):
"""Test handle_event method."""
# patch handle_heartbeat
called = []
def handle_heartbeat(process_name, group_name, pid, payload):
"""Fake handle_heartbeat."""
called.append((process_name, group_name, pid, payload))
self.listener.handle_heartbeat = handle_heartbeat
payload_dict = {u"time": time.time(), "type": "heartbeat"}
raw_data = ("processname:ticker groupname:ticker pid:42\n" +
json.dumps(payload_dict))
raw_header = ("ver:3.0 server:supervisor serial:1 pool:listener "
"poolserial:10 eventname:PROCESS_COMMUNICATION_STDOUT"
" len:%s\n" % len(raw_data))
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
headers = childutils.get_headers(raw_header)
self.listener._handle_event()
# check
self.assertEqual(1, len(called))
del payload_dict['type']
self.assertEqual(('ticker', 'ticker', '42', payload_dict), called[0])
self.assertTrue(
self.handler.check_debug("Event '%s' received: %r" %
(headers['eventname'], raw_data)))
# check the stdout info
self.assertEqual(["READY", "RESULT 2", "OK"],
self.stdout.getvalue().split("\n"))
def test_invalid_event_type(self):
"""Test with an invalid type."""
payload_dict = {u"time": time.time(), "type": "ping"}
raw_data = 'processname:ticker groupname:ticker pid:42\n' + \
json.dumps(payload_dict)
raw_header = ("ver:3.0 server:supervisor serial:1 pool:listener "
"poolserial:10 eventname:PROCESS_COMMUNICATION_STDOUT"
" len:%s\n" % len(raw_data))
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
self.listener._handle_event()
# check
self.assertTrue(
self.handler.check_error("Unable to handle event type '%s' - %r" %
('ping', raw_data)))
def test_invalid_payload(self):
"""Test with an invalid payload."""
payload_dict = {u"time": time.time(), "type": "ping"}
raw_data = 'processname:ticker groupname:ticker pid:42\n' + \
json.dumps(payload_dict) + "<!foo>"
raw_header = ("ver:3.0 server:supervisor serial:1 pool:listener "
"poolserial:10 eventname:PROCESS_COMMUNICATION_STDOUT"
" len:%s\n" % len(raw_data))
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
self.listener._handle_event()
# check
self.assertTrue(
self.handler.check_error("Unable to handle event type '%s' - %r" %
('None', raw_data)))
def test_unhandled_event(self):
"""A unhandled event type."""
payload_dict = {u"time": time.time(), "type": "ping"}
raw_data = 'processname:ticker groupname:ticker pid:42\n' + \
json.dumps(payload_dict)
raw_header = "ver:3.0 server:supervisor serial:1 pool:heartbeat " + \
"poolserial:1 eventname:UNKNOWN len:%s\n" % len(raw_data)
self.stdin.write(raw_header + raw_data)
self.stdin.seek(0)
self.listener._handle_event()
# check
self.assertTrue(
self.handler.check_warning("Received unsupported event: %s - %r" %
('UNKNOWN', raw_data)))
def test_check_interval(self):
"""Check that we properly check on the specified interval."""
header = "ver:3.0 server:supervisor serial:1 pool:heartbeat " + \
"poolserial:1 eventname:TICK_5 len:0\n"
expect(self.rpc.supervisor.getAllProcessInfo()).result([])
self.stdin.write(header)
self.stdin.seek(0)
self.listener._handle_event()
self.assertEqual(self.listener.tick_count, 1)
self.stdin.seek(0)
with self.mocker:
self.listener._handle_event()
class StatsWorkerTestCase(TestCase):
"""Tests for StatsWorker class."""
def setUp(self):
super(StatsWorkerTestCase, self).setUp()
self.mocker = Mocker()
self.rpc = self.mocker.mock()
self.worker = stats_worker.StatsWorker(10, '', self.rpc)
# logging setup
self.handler = MementoHandler()
self.worker.logger.addHandler(self.handler)
self.addCleanup(self.worker.logger.removeHandler, self.handler)
self.worker.logger.setLevel(logging.DEBUG)
self.handler.setLevel(logging.DEBUG)
self.worker.logger.propagate = False
self.handler.debug = True
def test_collect_stats(self):
"""Test the collect_stats method."""
called = []
self.worker._collect_process = \
lambda p, n: called.append(('proc', p, n)) or {}
self.worker._collect_machine = lambda: called.append('machine') or {}
processes = [dict(name="bar", group="foo", pid="42", state=RUNNING)]
expect(self.rpc.supervisor.getAllProcessInfo()).result(processes)
with self.mocker:
self.worker.collect_stats()
self.assertEqual(called, ['machine', ('proc', 42, 'bar')])
self.assertTrue(self.handler.check_info("Collecting machine stats"))
self.assertTrue(
self.handler.check_info("Collecting stats for proc", "pid=42",
"name=bar"))
def test_collect_stats_not_running(self):
"""Test the collect_stats method if the proccess isn't running."""
called = []
self.worker._collect_process = \
lambda p, n: called.append(('proc', p, n)) or {}
self.worker._collect_machine = lambda: called.append('machine') or {}
processes = [dict(name="bar", group="foo", pid="42", state=STARTING)]
expect(self.rpc.supervisor.getAllProcessInfo()).result(processes)
with self.mocker:
self.worker.collect_stats()
self.assertEqual(called, ['machine'])
self.assertTrue(self.handler.check_info("Collecting machine stats"))
self.assertTrue(
self.handler.check_info("Ignoring process", "pid=42", "name=bar",
"state=%s" % STARTING))
def test_collect_stats_no_data(self):
"""Test the collect_stats method with no data of a process."""
called = []
self.worker._collect_process = \
lambda p, n: called.append(('proc', p, n)) or {}
self.worker._collect_machine = lambda: called.append('machine') or {}
expect(self.rpc.supervisor.getAllProcessInfo()).result([])
with self.mocker:
self.worker.collect_stats()
self.assertEqual(called, ['machine'])
self.assertTrue(self.handler.check_info("Collecting machine stats"))
def test_collect_process_info_new_report(self):
"""Check how the process info is collected first time."""
mocker = Mocker()
assert not self.worker.process_cache
# patch Process to return our mock for test pid
Process = mocker.mock()
self.patch(stats_worker.psutil, 'Process', Process)
proc = mocker.mock()
pid = 1234
expect(Process(pid)).result(proc)
# patch ProcessReport to return or mock for given proc
ProcessReport = mocker.mock()
self.patch(stats_worker, 'ProcessReport', ProcessReport)
proc_report = mocker.mock()
expect(ProcessReport(proc)).result(proc_report)
# expect to get called with some info, return some results
name = 'test_proc'
result = object()
expect(proc_report.get_memory_and_cpu(prefix=name)).result(result)
with mocker:
real = self.worker._collect_process(pid, name)
self.assertIdentical(real, result)
def test_collect_process_info_old_report(self):
"""Check how the process info is collected when cached."""
mocker = Mocker()
# put it in the cache
pid = 1234
proc_report = mocker.mock()
self.worker.process_cache[pid] = proc_report
# expect to get called with some info, return some results
name = 'test_proc'
result = object()
expect(proc_report.get_memory_and_cpu(prefix=name)).result(result)
with mocker:
real = self.worker._collect_process(pid, name)
self.assertIdentical(real, result)
def test_collect_system_info(self):
"""Check how the system info is collected."""
mocker = Mocker()
# change the constant to assure it's used as we want
result1 = dict(a=3, b=5)
result2 = dict(c=7)
fake = (lambda: result1, lambda: result2)
self.patch(stats_worker, 'SYSTEM_STATS', fake)
with mocker:
result = self.worker._collect_machine()
should = {}
should.update(result1)
should.update(result2)
self.assertEqual(result, should)
def test_informed_metrics(self):
"""Check how stats are reported."""
# prepare a lot of fake info that will be "collected"
machine_info = dict(foo=3, bar=5)
process_info = {
1: dict(some=1234, other=4567),
2: dict(some=9876, other=6543),
}
self.worker._collect_process = lambda pid, name: process_info[pid]
self.worker._collect_machine = lambda: machine_info
processes = [
dict(name="proc1", group="", pid="1", state=RUNNING),
dict(name="proc2", group="", pid="2", state=RUNNING),
]
expect(self.rpc.supervisor.getAllProcessInfo()).result(processes)
# patch the metric reporter to see what is sent
reported = set()
self.worker.metrics.gauge = lambda *a: reported.add(a)
# what we should get is...
should = set([
('foo', 3),
('bar', 5),
('some', 1234),
('other', 4567),
('some', 9876),
('other', 6543),
])
with self.mocker:
self.worker.collect_stats()
self.assertEqual(reported, should)
