Ejemplo n.º 1
0
    def setUp(self):

        self.engines = []

        self.controller = cs.ControllerService()
        self.controller.startService()
        self.imultiengine = me.IMultiEngine(self.controller)
        self.itc = taskmodule.ITaskController(self.controller)
        self.itc.failurePenalty = 0

        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
        self.tc_referenceable = IFCTaskController(self.itc)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        mec_furl = self.controller_tub.registerReference(
            self.mec_referenceable)
        tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
        d.addCallback(self.handle_mec_client)
        d.addCallback(
            lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
        d.addCallback(self.handle_tc_client)
        return d
Ejemplo n.º 2
0
    def setUp(self):

        self.engines = []

        self.controller = cs.ControllerService()
        self.controller.startService()
        self.imultiengine = me.IMultiEngine(self.controller)
        self.itc = taskmodule.ITaskController(self.controller)
        self.itc.failurePenalty = 0

        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
        self.tc_referenceable = IFCTaskController(self.itc)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
        tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
        d.addCallback(self.handle_mec_client)
        d.addCallback(lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
        d.addCallback(self.handle_tc_client)
        return d
Ejemplo n.º 3
0
    def setUp(self):

        self.engines = []

        self.controller = ControllerService()
        self.controller.startService()
        self.imultiengine = IMultiEngine(self.controller)
        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        furl = self.controller_tub.registerReference(self.mec_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=furl)
        d.addCallback(self.handle_got_client)
        return d
Ejemplo n.º 4
0
    def setUp(self):

        self.engines = []

        self.controller = ControllerService()
        self.controller.startService()
        self.imultiengine = IMultiEngine(self.controller)
        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        furl = self.controller_tub.registerReference(self.mec_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=furl)
        d.addCallback(self.handle_got_client)
        return d
Ejemplo n.º 5
0
deferreds to the result.

The main methods are are `get_*_client` and `get_client`.
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2009  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from IPython.kernel import codeutil
from IPython.kernel.clientconnector import (AsyncClientConnector, AsyncCluster)

# Other things that the user will need
from IPython.kernel.task import MapTask, StringTask
from IPython.kernel.error import CompositeError

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

_client_tub = AsyncClientConnector()
get_multiengine_client = _client_tub.get_multiengine_client
get_task_client = _client_tub.get_task_client
get_client = _client_tub.get_client
Ejemplo n.º 6
0
class FullSynchronousMultiEngineTestCase(
    DeferredTestCase, IFullSynchronousMultiEngineTestCase):

    def setUp(self):

        self.engines = []

        self.controller = ControllerService()
        self.controller.startService()
        self.imultiengine = IMultiEngine(self.controller)
        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        furl = self.controller_tub.registerReference(self.mec_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=furl)
        d.addCallback(self.handle_got_client)
        return d

    def handle_got_client(self, client):
        self.multiengine = client

    def tearDown(self):
        dlist = []
        # Shut down the multiengine client
        d = self.client_tub.tub.stopService()
        dlist.append(d)
        # Shut down the engines
        for e in self.engines:
            e.stopService()
        # Shut down the controller
        d = self.controller_tub.stopService()
        d.addBoth(lambda _: self.controller.stopService())
        dlist.append(d)
        return defer.DeferredList(dlist)

    def test_mapper(self):
        self.addEngine(4)
        m = self.multiengine.mapper()
        self.assertEquals(m.multiengine,self.multiengine)
        self.assertEquals(m.dist,'b')
        self.assertEquals(m.targets,'all')
        self.assertEquals(m.block,True)

    def test_map_default(self):
        self.addEngine(4)
        m = self.multiengine.mapper()
        d = m.map(lambda x: 2*x, list(range(10)))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        d.addCallback(lambda _: self.multiengine.map(lambda x: 2*x, list(range(10))))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        return d

    def test_map_noblock(self):
        self.addEngine(4)
        m = self.multiengine.mapper(block=False)
        d = m.map(lambda x: 2*x, list(range(10)))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        return d

    def test_mapper_fail(self):
        self.addEngine(4)
        m = self.multiengine.mapper()
        d = m.map(lambda x: 1/0, list(range(10)))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d

    def test_parallel(self):
        self.addEngine(4)
        p = self.multiengine.parallel()
        self.assert_(isinstance(p, ParallelFunction))
        @p
        def f(x): return 2*x
        d = f(list(range(10)))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        return d

    def test_parallel_noblock(self):
        self.addEngine(1)
        p = self.multiengine.parallel(block=False)
        self.assert_(isinstance(p, ParallelFunction))
        @p
        def f(x): return 2*x
        d = f(list(range(10)))
        d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        return d

    def test_parallel_fail(self):
        self.addEngine(4)
        p = self.multiengine.parallel()
        self.assert_(isinstance(p, ParallelFunction))
        @p
        def f(x): return 1/0
        d = f(list(range(10)))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d
Ejemplo n.º 7
0
class TaskTest(DeferredTestCase, ITaskControllerTestCase):

    def setUp(self):

        self.engines = []

        self.controller = cs.ControllerService()
        self.controller.startService()
        self.imultiengine = me.IMultiEngine(self.controller)
        self.itc = taskmodule.ITaskController(self.controller)
        self.itc.failurePenalty = 0

        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
        self.tc_referenceable = IFCTaskController(self.itc)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        mec_furl = self.controller_tub.registerReference(self.mec_referenceable)
        tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
        d.addCallback(self.handle_mec_client)
        d.addCallback(lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
        d.addCallback(self.handle_tc_client)
        return d

    def handle_mec_client(self, client):
        self.multiengine = client

    def handle_tc_client(self, client):
        self.tc = client

    def tearDown(self):
        dlist = []
        # Shut down the multiengine client
        d = self.client_tub.tub.stopService()
        dlist.append(d)
        # Shut down the engines
        for e in self.engines:
            e.stopService()
        # Shut down the controller
        d = self.controller_tub.stopService()
        d.addBoth(lambda _: self.controller.stopService())
        dlist.append(d)
        return defer.DeferredList(dlist)

    def test_mapper(self):
        self.addEngine(1)
        m = self.tc.mapper()
        self.assertEquals(m.task_controller,self.tc)
        self.assertEquals(m.clear_before,False)
        self.assertEquals(m.clear_after,False)
        self.assertEquals(m.retries,0)
        self.assertEquals(m.recovery_task,None)
        self.assertEquals(m.depend,None)
        self.assertEquals(m.block,True)

    def test_map_default(self):
        self.addEngine(1)
        m = self.tc.mapper()
        d = m.map(lambda x: 2*x, list(range(10)))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        d.addCallback(lambda _: self.tc.map(lambda x: 2*x, list(range(10))))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        return d

    def test_map_noblock(self):
        self.addEngine(1)
        m = self.tc.mapper(block=False)
        d = m.map(lambda x: 2*x, list(range(10)))
        d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
        return d

    def test_mapper_fail(self):
        self.addEngine(1)
        m = self.tc.mapper()
        d = m.map(lambda x: 1/0, list(range(10)))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d

    def test_parallel(self):
        self.addEngine(1)
        p = self.tc.parallel()
        self.assert_(isinstance(p, ParallelFunction))
        @p
        def f(x): return 2*x
        d = f(list(range(10)))
        d.addCallback(lambda r: self.assertEquals(r,[2*x for x in range(10)]))
        return d

    def test_parallel_noblock(self):
        self.addEngine(1)
        p = self.tc.parallel(block=False)
        self.assert_(isinstance(p, ParallelFunction))
        @p
        def f(x): return 2*x
        d = f(list(range(10)))
        d.addCallback(lambda r: self.assertEquals(r,[x for x in range(10)]))
        return d

    def test_parallel_fail(self):
        self.addEngine(1)
        p = self.tc.parallel()
        self.assert_(isinstance(p, ParallelFunction))
        @p
        def f(x): return 1/0
        d = f(list(range(10)))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d
Ejemplo n.º 8
0
class TaskTest(DeferredTestCase, ITaskControllerTestCase):
    def setUp(self):

        self.engines = []

        self.controller = cs.ControllerService()
        self.controller.startService()
        self.imultiengine = me.IMultiEngine(self.controller)
        self.itc = taskmodule.ITaskController(self.controller)
        self.itc.failurePenalty = 0

        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)
        self.tc_referenceable = IFCTaskController(self.itc)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        mec_furl = self.controller_tub.registerReference(
            self.mec_referenceable)
        tc_furl = self.controller_tub.registerReference(self.tc_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=mec_furl)
        d.addCallback(self.handle_mec_client)
        d.addCallback(
            lambda _: self.client_tub.get_task_client(furl_or_file=tc_furl))
        d.addCallback(self.handle_tc_client)
        return d

    def handle_mec_client(self, client):
        self.multiengine = client

    def handle_tc_client(self, client):
        self.tc = client

    def tearDown(self):
        dlist = []
        # Shut down the multiengine client
        d = self.client_tub.tub.stopService()
        dlist.append(d)
        # Shut down the engines
        for e in self.engines:
            e.stopService()
        # Shut down the controller
        d = self.controller_tub.stopService()
        d.addBoth(lambda _: self.controller.stopService())
        dlist.append(d)
        return defer.DeferredList(dlist)

    def test_mapper(self):
        self.addEngine(1)
        m = self.tc.mapper()
        self.assertEquals(m.task_controller, self.tc)
        self.assertEquals(m.clear_before, False)
        self.assertEquals(m.clear_after, False)
        self.assertEquals(m.retries, 0)
        self.assertEquals(m.recovery_task, None)
        self.assertEquals(m.depend, None)
        self.assertEquals(m.block, True)

    def test_map_default(self):
        self.addEngine(1)
        m = self.tc.mapper()
        d = m.map(lambda x: 2 * x, range(10))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        d.addCallback(lambda _: self.tc.map(lambda x: 2 * x, range(10)))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        return d

    def test_map_noblock(self):
        self.addEngine(1)
        m = self.tc.mapper(block=False)
        d = m.map(lambda x: 2 * x, range(10))
        d.addCallback(lambda r: self.assertEquals(r, [x for x in range(10)]))
        return d

    def test_mapper_fail(self):
        self.addEngine(1)
        m = self.tc.mapper()
        d = m.map(lambda x: 1 / 0, range(10))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d

    def test_parallel(self):
        self.addEngine(1)
        p = self.tc.parallel()
        self.assert_(isinstance(p, ParallelFunction))

        @p
        def f(x):
            return 2 * x

        d = f(range(10))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        return d

    def test_parallel_noblock(self):
        self.addEngine(1)
        p = self.tc.parallel(block=False)
        self.assert_(isinstance(p, ParallelFunction))

        @p
        def f(x):
            return 2 * x

        d = f(range(10))
        d.addCallback(lambda r: self.assertEquals(r, [x for x in range(10)]))
        return d

    def test_parallel_fail(self):
        self.addEngine(1)
        p = self.tc.parallel()
        self.assert_(isinstance(p, ParallelFunction))

        @p
        def f(x):
            return 1 / 0

        d = f(range(10))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d
Ejemplo n.º 9
0
class FullSynchronousMultiEngineTestCase(DeferredTestCase,
                                         IFullSynchronousMultiEngineTestCase):
    def setUp(self):

        self.engines = []

        self.controller = ControllerService()
        self.controller.startService()
        self.imultiengine = IMultiEngine(self.controller)
        self.mec_referenceable = IFCSynchronousMultiEngine(self.imultiengine)

        self.controller_tub = Tub()
        self.controller_tub.listenOn('tcp:10111:interface=127.0.0.1')
        self.controller_tub.setLocation('127.0.0.1:10111')

        furl = self.controller_tub.registerReference(self.mec_referenceable)
        self.controller_tub.startService()

        self.client_tub = AsyncClientConnector()
        d = self.client_tub.get_multiengine_client(furl_or_file=furl)
        d.addCallback(self.handle_got_client)
        return d

    def handle_got_client(self, client):
        self.multiengine = client

    def tearDown(self):
        dlist = []
        # Shut down the multiengine client
        d = self.client_tub.tub.stopService()
        dlist.append(d)
        # Shut down the engines
        for e in self.engines:
            e.stopService()
        # Shut down the controller
        d = self.controller_tub.stopService()
        d.addBoth(lambda _: self.controller.stopService())
        dlist.append(d)
        return defer.DeferredList(dlist)

    def test_mapper(self):
        self.addEngine(4)
        m = self.multiengine.mapper()
        self.assertEquals(m.multiengine, self.multiengine)
        self.assertEquals(m.dist, 'b')
        self.assertEquals(m.targets, 'all')
        self.assertEquals(m.block, True)

    def test_map_default(self):
        self.addEngine(4)
        m = self.multiengine.mapper()
        d = m.map(lambda x: 2 * x, range(10))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        d.addCallback(
            lambda _: self.multiengine.map(lambda x: 2 * x, range(10)))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        return d

    def test_map_noblock(self):
        self.addEngine(4)
        m = self.multiengine.mapper(block=False)
        d = m.map(lambda x: 2 * x, range(10))
        d.addCallback(
            lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        return d

    def test_mapper_fail(self):
        self.addEngine(4)
        m = self.multiengine.mapper()
        d = m.map(lambda x: 1 / 0, range(10))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d

    def test_parallel(self):
        self.addEngine(4)
        p = self.multiengine.parallel()
        self.assert_(isinstance(p, ParallelFunction))

        @p
        def f(x):
            return 2 * x

        d = f(range(10))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        return d

    def test_parallel_noblock(self):
        self.addEngine(1)
        p = self.multiengine.parallel(block=False)
        self.assert_(isinstance(p, ParallelFunction))

        @p
        def f(x):
            return 2 * x

        d = f(range(10))
        d.addCallback(
            lambda did: self.multiengine.get_pending_deferred(did, True))
        d.addCallback(
            lambda r: self.assertEquals(r, [2 * x for x in range(10)]))
        return d

    def test_parallel_fail(self):
        self.addEngine(4)
        p = self.multiengine.parallel()
        self.assert_(isinstance(p, ParallelFunction))

        @p
        def f(x):
            return 1 / 0

        d = f(range(10))
        d.addBoth(lambda f: self.assertRaises(ZeroDivisionError, _raise_it, f))
        return d