def storeContext():
"""
Writes the current ctx to the disk
"""
fileName = ctx().ctx_file_name
with open(fileName, "w") as ctxFile:
pickle.dump(ctx(), ctxFile)
def storeContext():
"""
Writes the current ctx to the disk
"""
fileName = ctx().ctx_file_name
with open(fileName, "wb") as ctxFile:
pickle.dump(ctx(), ctxFile)
def launch(self):
"""
Launches the workflow
"""
ctx().timings = []
executor = SequentialBackend(ctx())
executor.run(ctx().params.plugins)
def test_launch(self):
args = ["test.config.workflow_config"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx() is not None
assert ctx().params is not None
assert ctx().params.plugins is not None
def test_launch(self):
args = ["test.config.workflow_config"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx() is not None
assert ctx().params is not None
assert ctx().params.plugins is not None
def test_simple(self):
plugin = Plugin(ctx())
assert plugin.value is None
plugin = Plugin(ctx(), value=1)
assert plugin.value == 1
plugin = Plugin(ctx(), foo=1)
assert ctx().foo == 1
def test_simple(self):
plugin = Plugin(ctx())
assert plugin.value is None
plugin = Plugin(ctx(), value=1)
assert plugin.value == 1
plugin = Plugin(ctx(), foo=1)
assert ctx().foo == 1
def test_simple_launch(self):
args = ["test.config.workflow_config_simple"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx() is not None
assert ctx().params is not None
assert ctx().params.plugins is not None
assert isinstance(ctx().params.plugins, Loop)
def test_create_from_dict(self):
args = {"plugins": ["test.plugin.simple_plugin", "test.plugin.simple_plugin"],
"a":1
}
mgr = WorkflowManager(args)
assert ctx().params is not None
assert ctx().params.a is 1
assert ctx().params.plugins is not None
plugins = [p for p in ctx().params.plugins]
assert len(plugins) is 2
def test_simple_launch(self):
args = ["test.config.workflow_config_simple"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx() is not None
assert ctx().params is not None
assert ctx().params.plugins is not None
assert isinstance(ctx().params.plugins, Loop)
def test_create_from_dict(self):
args = {
"plugins":
["test.plugin.simple_plugin", "test.plugin.simple_plugin"],
"a": 1
}
mgr = WorkflowManager(args)
assert ctx().params is not None
assert ctx().params.a is 1
assert ctx().params.plugins is not None
plugins = [p for p in ctx().params.plugins]
assert len(plugins) is 2
def test_loop_max_iter_nested(self):
maxIter = 3
pList = [Plugin(ctx()), Plugin(ctx())]
loop = Loop(Loop(pList, stop=RangeStopCriteria(maxIter=maxIter)),
stop=RangeStopCriteria(maxIter=maxIter))
cnt = 0
for p in loop:
assert isinstance(p, Plugin)
p()
cnt += 1
assert cnt == len(pList) * maxIter * maxIter
def test_plugin_instances(self):
plugin1 = Plugin(ctx())
plugin2 = Plugin(ctx())
loop = Loop([plugin1, plugin2])
p = loop.next()
assert p == plugin1
p = loop.next()
assert p == plugin2
try:
loop.next()
assert False
except StopIteration:
assert True
def test_plugin_instances(self):
plugin1 = Plugin(ctx())
plugin2 = Plugin(ctx())
loop = Loop([plugin1, plugin2])
p = next(loop)
assert p == plugin1
p = next(loop)
assert p == plugin2
try:
next(loop)
assert False
except StopIteration:
assert True
def test_loop_max_iter_nested(self):
maxIter=3
pList = [Plugin(ctx()), Plugin(ctx())]
loop = Loop(
Loop(pList,
stop=RangeStopCriteria(maxIter=maxIter)),
stop=RangeStopCriteria(maxIter=maxIter))
cnt = 0
for p in loop:
assert isinstance(p, Plugin)
p()
cnt += 1
assert cnt == len(pList)*maxIter*maxIter
def __init__(self,
pluginList,
mapPlugin,
reducePlugin=None,
ctx=None,
parallel=True):
'''
Constructor
'''
if ctx is None:
ctx = context.ctx()
self.ctx = ctx
super(ParallelPluginCollection, self).__init__(self.ctx)
if not isinstance(pluginList, Loop):
pluginList = Loop(pluginList)
self.pluginList = pluginList
if mapPlugin is None:
raise InvalidAttributeException("No map plugin provided")
self.mapPlugin = mapPlugin
self.reducePlugin = reducePlugin
self.parallel = parallel
def test_parallel_workflow(self):
args = ["--backend=multiprocessing",
"--cpu-count=1",
"test.config.workflow_config_parallel"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx().valuesSum == 285
def test_invalid_module(self):
pluginName = "ivy.plugin.BasePlugin"
try:
plugin = PluginFactory.createInstance(pluginName, ctx())
pytest.fail("UnsupportedPluginTypeException expected", False)
assert False
except UnsupportedPluginTypeException as ex:
assert True
def test_invalid_module(self):
pluginName = "ivy.plugin.BasePlugin"
try:
plugin = PluginFactory.createInstance(pluginName, ctx())
pytest.fail("UnsupportedPluginTypeException expected", False)
assert False
except UnsupportedPluginTypeException as ex:
assert True
def test_parallel_workflow(self):
args = ["--backend=multiprocessing",
"--cpu-count=1",
"test.config.workflow_config_parallel"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx().valuesSum == 285
def test_parallel_plugin_collection_pickle(self):
ctx = context.ctx()
parallelPluginCollection = ParallelPluginCollection(
"ivy.plugin.simple_map_plugin",
["ivy.plugin.simple_square_plugin"],
"ivy.plugin.simple_reduce_plugin")
sParallelPluginCollection = dumps(parallelPluginCollection)
parallelPluginCollectio2 = loads(sParallelPluginCollection)
def test_cust_ctx_provider(self):
context.global_ctx = None
args = ["test.config.workflow_config_cust"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx() is not None
from ivy.context import getContextProvider
assert getContextProvider() == PickleContextProvider
def test_parallel_plugin_collection_pickle(self):
ctx = context.ctx()
parallelPluginCollection = ParallelPluginCollection(
"ivy.plugin.simple_map_plugin",
["ivy.plugin.simple_square_plugin"],
"ivy.plugin.simple_reduce_plugin")
sParallelPluginCollection = dumps(parallelPluginCollection)
parallelPluginCollectio2 = loads(sParallelPluginCollection)
def test_workflow_context_pickle(self):
args = ["--backend=multiprocessing",
"--cpu-count=1",
"test.config.workflow_config_parallel"]
mgr = WorkflowManager(args)
lCtx = ctx()
sLCtx = dumps(lCtx)
lCtx2 = loads(sLCtx)
def test_cust_ctx_provider(self):
context.global_ctx = None
args = ["test.config.workflow_config_cust"]
mgr = WorkflowManager(args)
mgr.launch()
assert ctx() is not None
from ivy.context import getContextProvider
assert getContextProvider() == PickleContextProvider
def test_workflow_context_pickle(self):
args = [
"--backend=multiprocessing", "--cpu-count=1",
"test.config.workflow_config_parallel"
]
mgr = WorkflowManager(args)
lCtx = ctx()
sLCtx = dumps(lCtx)
lCtx2 = loads(sLCtx)
def test_one_plugin(self):
plugin = Plugin(ctx())
loop = Loop(plugin)
p = loop.next()
assert p == plugin
try:
loop.next()
assert False
except StopIteration:
assert True
def test_one_plugin(self):
plugin = Plugin(ctx())
loop = Loop(plugin)
p = next(loop)
assert p == plugin
try:
next(loop)
assert False
except StopIteration:
assert True
def _setup(self, argv):
config = self._parseArgs(argv)
if not config.has_key(PLUGINS_KEY):
raise InvalidAttributeException("plugins definition is missing")
if config.has_key(CONTEXT_PROVIDER_KEY):
def getContextProviderWrapper():
#todo load class not module
clazz = config[CONTEXT_PROVIDER_KEY]
moduleName = ".".join(clazz.split(".")[:-1])
module = importlib.import_module(moduleName)
return getattr(module, clazz.split(".")[-1])
context.getContextProvider = getContextProviderWrapper
if not isinstance(config[PLUGINS_KEY], Loop):
config[PLUGINS_KEY] = Loop(config[PLUGINS_KEY])
ctx().params = context._createImmutableCtx(**config)
#just to maintain backward compatibility
ctx().parameters = ctx().params
ctx().plugins = ctx().params.plugins
def test_multiprocessing(self):
ctx = context.ctx()
ctx.timings = []
ctx.params = context._createImmutableCtx(backend="multiprocessing",
cpu_count=8,
valuesMin = 1,
valuesMax = 10)
mapPlugin = range_map_plugin.Plugin(ctx)
pluginList = [PLUGIN_NAME]
reducePlugin = sum_reduce_plugin.Plugin(ctx)
parallelPluginCollection = ParallelPluginCollection(pluginList, mapPlugin, reducePlugin)
parallelPluginCollection()
assert ctx.valuesSum == 285
def test_multiprocessing(self):
ctx = context.ctx()
ctx.timings = []
ctx.params = context._createImmutableCtx(backend="multiprocessing",
cpu_count=8,
valuesMin = 1,
valuesMax = 10)
mapPlugin = range_map_plugin.Plugin(ctx)
pluginList = [PLUGIN_NAME]
reducePlugin = sum_reduce_plugin.Plugin(ctx)
parallelPluginCollection = ParallelPluginCollection(pluginList, mapPlugin, reducePlugin)
parallelPluginCollection()
assert ctx.valuesSum == 285
def __init__(self, pluginList, stop=None, ctx=None):
if pluginList is None:
raise InvalidLoopException("Plugin list is None")
if not isinstance(pluginList, list):
pluginList = [pluginList]
self.pluginList = pluginList
self._createIter()
if stop is None:
stop = self._createStopCriteria()
stop.parent = self
self._stopCriteria = stop
context.register(self)
if ctx is None:
ctx = context.ctx()
self.ctx = ctx
def __init__(self, pluginList, stop=None, ctx=None):
if pluginList is None:
raise InvalidLoopException("Plugin list is None")
if not isinstance(pluginList, list):
pluginList = [pluginList]
self.pluginList = pluginList
self._createIter()
if stop is None:
stop = self._createStopCriteria()
stop.parent = self
self._stopCriteria = stop
context.register(self)
if ctx is None:
ctx = context.ctx()
self.ctx = ctx
def __init__(self, pluginList, mapPlugin, reducePlugin=None, ctx=None, parallel=True):
'''
Constructor
'''
if ctx is None:
ctx = context.ctx()
self.ctx = ctx
super(ParallelPluginCollection, self).__init__(self.ctx)
if not isinstance(pluginList, Loop):
pluginList = Loop(pluginList)
self.pluginList = pluginList
if mapPlugin is None:
raise InvalidAttributeException("No map plugin provided")
self.mapPlugin = mapPlugin
self.reducePlugin = reducePlugin
self.parallel = parallel
def test_launch_loop(self):
_main(*["test.config.workflow_config_cli"])
assert ctx() is not None
assert ctx().params is not None
assert ctx().params.plugins is not None
assert len(ctx().timings) == 2
def test_simple(self):
plugin = PluginFactory.createInstance(PLUGIN_NAME, ctx())
assert plugin is not None
assert isinstance(plugin, simple_plugin.Plugin)
def test_parseArgs(self):
args = [
"--a=True", "--b=False", "--c=-1", "--d=0", "--e=1", "--f=-1.0",
"--g=0.0", "--h=1.0", "--i=le_string", "--j=1,2,3,4",
"--bool1=True", "--bool2=False", "--bool3=True", "--bool4=False",
"test.config.workflow_config"
]
mgr = WorkflowManager(args)
assert ctx().params.a == True
assert ctx().params.b == False
assert ctx().params.c == -1
assert ctx().params.d == 0
assert ctx().params.e == 1
assert ctx().params.f == -1.0
assert ctx().params.g == 0.0
assert ctx().params.h == 1.0
assert ctx().params.i == "le_string"
assert ctx().params.bool1 == True
assert ctx().params.bool2 == False
assert ctx().params.bool3 == True
assert ctx().params.bool4 == False
assert all(map(eq, ctx().params.j, [1, 2, 3, 4]))
def test_simple(self):
plugin = PluginFactory.createInstance(PLUGIN_NAME, ctx())
assert plugin is not None
assert isinstance(plugin, simple_plugin.Plugin)
def test_context_pickle(self):
lCtx = ctx()
sLCtx = dumps(lCtx)
lCtx2 = loads(sLCtx)
def test_launch_loop(self):
_main(*["test.config.workflow_config_cli"])
assert ctx() is not None
assert ctx().params is not None
assert ctx().params.plugins is not None
assert len(ctx().timings)==2
def test_parseArgs(self):
args = ["--a=True",
"--b=False",
"--c=-1",
"--d=0",
"--e=1",
"--f=-1.0",
"--g=0.0",
"--h=1.0",
"--i=le_string",
"--j=1,2,3,4",
"--bool1=True",
"--bool2=False",
"--bool3=True",
"--bool4=False",
"test.config.workflow_config"]
mgr = WorkflowManager(args)
assert ctx().params.a == True
assert ctx().params.b == False
assert ctx().params.c == -1
assert ctx().params.d == 0
assert ctx().params.e == 1
assert ctx().params.f == -1.0
assert ctx().params.g == 0.0
assert ctx().params.h == 1.0
assert ctx().params.i == "le_string"
assert ctx().params.bool1 == True
assert ctx().params.bool2 == False
assert ctx().params.bool3 == True
assert ctx().params.bool4 == False
assert all(map(eq, ctx().params.j,[1,2,3,4]))
def test_storeContext(self, tmpdir):
path = str(tmpdir.join("le_ctx"))
ctx().ctx_file_name = path
PickleContextProvider.storeContext()
assert os.path.exists(path)
def test_storeContext(self, tmpdir):
path = str(tmpdir.join("le_ctx"))
ctx().ctx_file_name = path
PickleContextProvider.storeContext()
assert os.path.exists(path)
def test_context_pickle(self):
lCtx = ctx()
sLCtx = dumps(lCtx)
lCtx2 = loads(sLCtx)
