forked from transientskp/tkp
-
Notifications
You must be signed in to change notification settings - Fork 0
/
__init__.py
49 lines (42 loc) · 1.64 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""
`tkp.distribute` implement various computation distribution methods. All
sub modules should communicate with the other parts of TKP through the
`tkp.steps` submodule.
"""
import importlib
import logging
logger = logging.getLogger(__name__)
class Runner(object):
def __init__(self, distributor, cores=0):
"""
Args:
distributor: the name of the distribution method, example multiproc
"""
logger.debug("Using %s distribution method" % distributor)
self.distributor = distributor
self.mod_path = __name__ + '.' + distributor
try:
self.module = importlib.import_module(self.mod_path)
self.tasks = importlib.import_module(self.mod_path + '.tasks')
except ImportError:
raise NotImplementedError("%s not found" % self.mod_path)
if not hasattr(self.module, 'map'):
raise NotImplementedError("%s misses map function" % self.mod_path)
self.module.set_cores(cores)
def map(self, func_name, iterable, args=[]):
"""
args:
func: The function to be called
iterable: a list of objects to iterate over
arguments: list of arguments to give to the function
returns:
the results of all mapped functions
"""
func = self.get_func(func_name)
return self.module.map(func, iterable, args)
def get_func(self, func_name):
try:
return getattr(self.tasks, func_name)
except AttributeError:
raise NotImplementedError('%s not implemented for %s' %
(func_name, self.mod_path))