def storeContext(): """ Writes the current ctx to the disk """ fileName = ctx().ctx_file_name with open(fileName, "w") as ctxFile: pickle.dump(ctx(), ctxFile)
def storeContext(): """ Writes the current ctx to the disk """ fileName = ctx().ctx_file_name with open(fileName, "wb") as ctxFile: pickle.dump(ctx(), ctxFile)
def launch(self): """ Launches the workflow """ ctx().timings = [] executor = SequentialBackend(ctx()) executor.run(ctx().params.plugins)
def test_launch(self): args = ["test.config.workflow_config"] mgr = WorkflowManager(args) mgr.launch() assert ctx() is not None assert ctx().params is not None assert ctx().params.plugins is not None
def test_launch(self): args = ["test.config.workflow_config"] mgr = WorkflowManager(args) mgr.launch() assert ctx() is not None assert ctx().params is not None assert ctx().params.plugins is not None
def test_simple(self): plugin = Plugin(ctx()) assert plugin.value is None plugin = Plugin(ctx(), value=1) assert plugin.value == 1 plugin = Plugin(ctx(), foo=1) assert ctx().foo == 1
def test_simple(self): plugin = Plugin(ctx()) assert plugin.value is None plugin = Plugin(ctx(), value=1) assert plugin.value == 1 plugin = Plugin(ctx(), foo=1) assert ctx().foo == 1
def test_simple_launch(self): args = ["test.config.workflow_config_simple"] mgr = WorkflowManager(args) mgr.launch() assert ctx() is not None assert ctx().params is not None assert ctx().params.plugins is not None assert isinstance(ctx().params.plugins, Loop)
def test_create_from_dict(self): args = {"plugins": ["test.plugin.simple_plugin", "test.plugin.simple_plugin"], "a":1 } mgr = WorkflowManager(args) assert ctx().params is not None assert ctx().params.a is 1 assert ctx().params.plugins is not None plugins = [p for p in ctx().params.plugins] assert len(plugins) is 2
def test_simple_launch(self): args = ["test.config.workflow_config_simple"] mgr = WorkflowManager(args) mgr.launch() assert ctx() is not None assert ctx().params is not None assert ctx().params.plugins is not None assert isinstance(ctx().params.plugins, Loop)
def test_create_from_dict(self): args = { "plugins": ["test.plugin.simple_plugin", "test.plugin.simple_plugin"], "a": 1 } mgr = WorkflowManager(args) assert ctx().params is not None assert ctx().params.a is 1 assert ctx().params.plugins is not None plugins = [p for p in ctx().params.plugins] assert len(plugins) is 2
def test_loop_max_iter_nested(self): maxIter = 3 pList = [Plugin(ctx()), Plugin(ctx())] loop = Loop(Loop(pList, stop=RangeStopCriteria(maxIter=maxIter)), stop=RangeStopCriteria(maxIter=maxIter)) cnt = 0 for p in loop: assert isinstance(p, Plugin) p() cnt += 1 assert cnt == len(pList) * maxIter * maxIter
def test_plugin_instances(self): plugin1 = Plugin(ctx()) plugin2 = Plugin(ctx()) loop = Loop([plugin1, plugin2]) p = loop.next() assert p == plugin1 p = loop.next() assert p == plugin2 try: loop.next() assert False except StopIteration: assert True
def test_plugin_instances(self): plugin1 = Plugin(ctx()) plugin2 = Plugin(ctx()) loop = Loop([plugin1, plugin2]) p = next(loop) assert p == plugin1 p = next(loop) assert p == plugin2 try: next(loop) assert False except StopIteration: assert True
def test_loop_max_iter_nested(self): maxIter=3 pList = [Plugin(ctx()), Plugin(ctx())] loop = Loop( Loop(pList, stop=RangeStopCriteria(maxIter=maxIter)), stop=RangeStopCriteria(maxIter=maxIter)) cnt = 0 for p in loop: assert isinstance(p, Plugin) p() cnt += 1 assert cnt == len(pList)*maxIter*maxIter
def __init__(self, pluginList, mapPlugin, reducePlugin=None, ctx=None, parallel=True): ''' Constructor ''' if ctx is None: ctx = context.ctx() self.ctx = ctx super(ParallelPluginCollection, self).__init__(self.ctx) if not isinstance(pluginList, Loop): pluginList = Loop(pluginList) self.pluginList = pluginList if mapPlugin is None: raise InvalidAttributeException("No map plugin provided") self.mapPlugin = mapPlugin self.reducePlugin = reducePlugin self.parallel = parallel
def test_parallel_workflow(self): args = ["--backend=multiprocessing", "--cpu-count=1", "test.config.workflow_config_parallel"] mgr = WorkflowManager(args) mgr.launch() assert ctx().valuesSum == 285
def test_invalid_module(self): pluginName = "ivy.plugin.BasePlugin" try: plugin = PluginFactory.createInstance(pluginName, ctx()) pytest.fail("UnsupportedPluginTypeException expected", False) assert False except UnsupportedPluginTypeException as ex: assert True
def test_invalid_module(self): pluginName = "ivy.plugin.BasePlugin" try: plugin = PluginFactory.createInstance(pluginName, ctx()) pytest.fail("UnsupportedPluginTypeException expected", False) assert False except UnsupportedPluginTypeException as ex: assert True
def test_parallel_workflow(self): args = ["--backend=multiprocessing", "--cpu-count=1", "test.config.workflow_config_parallel"] mgr = WorkflowManager(args) mgr.launch() assert ctx().valuesSum == 285
def test_parallel_plugin_collection_pickle(self): ctx = context.ctx() parallelPluginCollection = ParallelPluginCollection( "ivy.plugin.simple_map_plugin", ["ivy.plugin.simple_square_plugin"], "ivy.plugin.simple_reduce_plugin") sParallelPluginCollection = dumps(parallelPluginCollection) parallelPluginCollectio2 = loads(sParallelPluginCollection)
def test_cust_ctx_provider(self): context.global_ctx = None args = ["test.config.workflow_config_cust"] mgr = WorkflowManager(args) mgr.launch() assert ctx() is not None from ivy.context import getContextProvider assert getContextProvider() == PickleContextProvider
def test_parallel_plugin_collection_pickle(self): ctx = context.ctx() parallelPluginCollection = ParallelPluginCollection( "ivy.plugin.simple_map_plugin", ["ivy.plugin.simple_square_plugin"], "ivy.plugin.simple_reduce_plugin") sParallelPluginCollection = dumps(parallelPluginCollection) parallelPluginCollectio2 = loads(sParallelPluginCollection)
def test_workflow_context_pickle(self): args = ["--backend=multiprocessing", "--cpu-count=1", "test.config.workflow_config_parallel"] mgr = WorkflowManager(args) lCtx = ctx() sLCtx = dumps(lCtx) lCtx2 = loads(sLCtx)
def test_cust_ctx_provider(self): context.global_ctx = None args = ["test.config.workflow_config_cust"] mgr = WorkflowManager(args) mgr.launch() assert ctx() is not None from ivy.context import getContextProvider assert getContextProvider() == PickleContextProvider
def test_workflow_context_pickle(self): args = [ "--backend=multiprocessing", "--cpu-count=1", "test.config.workflow_config_parallel" ] mgr = WorkflowManager(args) lCtx = ctx() sLCtx = dumps(lCtx) lCtx2 = loads(sLCtx)
def test_one_plugin(self): plugin = Plugin(ctx()) loop = Loop(plugin) p = loop.next() assert p == plugin try: loop.next() assert False except StopIteration: assert True
def test_one_plugin(self): plugin = Plugin(ctx()) loop = Loop(plugin) p = next(loop) assert p == plugin try: next(loop) assert False except StopIteration: assert True
def _setup(self, argv): config = self._parseArgs(argv) if not config.has_key(PLUGINS_KEY): raise InvalidAttributeException("plugins definition is missing") if config.has_key(CONTEXT_PROVIDER_KEY): def getContextProviderWrapper(): #todo load class not module clazz = config[CONTEXT_PROVIDER_KEY] moduleName = ".".join(clazz.split(".")[:-1]) module = importlib.import_module(moduleName) return getattr(module, clazz.split(".")[-1]) context.getContextProvider = getContextProviderWrapper if not isinstance(config[PLUGINS_KEY], Loop): config[PLUGINS_KEY] = Loop(config[PLUGINS_KEY]) ctx().params = context._createImmutableCtx(**config) #just to maintain backward compatibility ctx().parameters = ctx().params ctx().plugins = ctx().params.plugins
def test_multiprocessing(self): ctx = context.ctx() ctx.timings = [] ctx.params = context._createImmutableCtx(backend="multiprocessing", cpu_count=8, valuesMin = 1, valuesMax = 10) mapPlugin = range_map_plugin.Plugin(ctx) pluginList = [PLUGIN_NAME] reducePlugin = sum_reduce_plugin.Plugin(ctx) parallelPluginCollection = ParallelPluginCollection(pluginList, mapPlugin, reducePlugin) parallelPluginCollection() assert ctx.valuesSum == 285
def test_multiprocessing(self): ctx = context.ctx() ctx.timings = [] ctx.params = context._createImmutableCtx(backend="multiprocessing", cpu_count=8, valuesMin = 1, valuesMax = 10) mapPlugin = range_map_plugin.Plugin(ctx) pluginList = [PLUGIN_NAME] reducePlugin = sum_reduce_plugin.Plugin(ctx) parallelPluginCollection = ParallelPluginCollection(pluginList, mapPlugin, reducePlugin) parallelPluginCollection() assert ctx.valuesSum == 285
def __init__(self, pluginList, stop=None, ctx=None): if pluginList is None: raise InvalidLoopException("Plugin list is None") if not isinstance(pluginList, list): pluginList = [pluginList] self.pluginList = pluginList self._createIter() if stop is None: stop = self._createStopCriteria() stop.parent = self self._stopCriteria = stop context.register(self) if ctx is None: ctx = context.ctx() self.ctx = ctx
def __init__(self, pluginList, stop=None, ctx=None): if pluginList is None: raise InvalidLoopException("Plugin list is None") if not isinstance(pluginList, list): pluginList = [pluginList] self.pluginList = pluginList self._createIter() if stop is None: stop = self._createStopCriteria() stop.parent = self self._stopCriteria = stop context.register(self) if ctx is None: ctx = context.ctx() self.ctx = ctx
def __init__(self, pluginList, mapPlugin, reducePlugin=None, ctx=None, parallel=True): ''' Constructor ''' if ctx is None: ctx = context.ctx() self.ctx = ctx super(ParallelPluginCollection, self).__init__(self.ctx) if not isinstance(pluginList, Loop): pluginList = Loop(pluginList) self.pluginList = pluginList if mapPlugin is None: raise InvalidAttributeException("No map plugin provided") self.mapPlugin = mapPlugin self.reducePlugin = reducePlugin self.parallel = parallel
def test_launch_loop(self): _main(*["test.config.workflow_config_cli"]) assert ctx() is not None assert ctx().params is not None assert ctx().params.plugins is not None assert len(ctx().timings) == 2
def test_simple(self): plugin = PluginFactory.createInstance(PLUGIN_NAME, ctx()) assert plugin is not None assert isinstance(plugin, simple_plugin.Plugin)
def test_parseArgs(self): args = [ "--a=True", "--b=False", "--c=-1", "--d=0", "--e=1", "--f=-1.0", "--g=0.0", "--h=1.0", "--i=le_string", "--j=1,2,3,4", "--bool1=True", "--bool2=False", "--bool3=True", "--bool4=False", "test.config.workflow_config" ] mgr = WorkflowManager(args) assert ctx().params.a == True assert ctx().params.b == False assert ctx().params.c == -1 assert ctx().params.d == 0 assert ctx().params.e == 1 assert ctx().params.f == -1.0 assert ctx().params.g == 0.0 assert ctx().params.h == 1.0 assert ctx().params.i == "le_string" assert ctx().params.bool1 == True assert ctx().params.bool2 == False assert ctx().params.bool3 == True assert ctx().params.bool4 == False assert all(map(eq, ctx().params.j, [1, 2, 3, 4]))
def test_simple(self): plugin = PluginFactory.createInstance(PLUGIN_NAME, ctx()) assert plugin is not None assert isinstance(plugin, simple_plugin.Plugin)
def test_context_pickle(self): lCtx = ctx() sLCtx = dumps(lCtx) lCtx2 = loads(sLCtx)
def test_launch_loop(self): _main(*["test.config.workflow_config_cli"]) assert ctx() is not None assert ctx().params is not None assert ctx().params.plugins is not None assert len(ctx().timings)==2
def test_parseArgs(self): args = ["--a=True", "--b=False", "--c=-1", "--d=0", "--e=1", "--f=-1.0", "--g=0.0", "--h=1.0", "--i=le_string", "--j=1,2,3,4", "--bool1=True", "--bool2=False", "--bool3=True", "--bool4=False", "test.config.workflow_config"] mgr = WorkflowManager(args) assert ctx().params.a == True assert ctx().params.b == False assert ctx().params.c == -1 assert ctx().params.d == 0 assert ctx().params.e == 1 assert ctx().params.f == -1.0 assert ctx().params.g == 0.0 assert ctx().params.h == 1.0 assert ctx().params.i == "le_string" assert ctx().params.bool1 == True assert ctx().params.bool2 == False assert ctx().params.bool3 == True assert ctx().params.bool4 == False assert all(map(eq, ctx().params.j,[1,2,3,4]))
def test_storeContext(self, tmpdir): path = str(tmpdir.join("le_ctx")) ctx().ctx_file_name = path PickleContextProvider.storeContext() assert os.path.exists(path)
def test_storeContext(self, tmpdir): path = str(tmpdir.join("le_ctx")) ctx().ctx_file_name = path PickleContextProvider.storeContext() assert os.path.exists(path)
def test_context_pickle(self): lCtx = ctx() sLCtx = dumps(lCtx) lCtx2 = loads(sLCtx)