def map(cls, func, *arg, **kwargs):
"""
Map a function across iterable(s) and return a list of results
:param func: function
Mappable function
:param args: iterable
Iterator(s)
"""
assert check.argument_callable(func)
assert check.argument_list_type(arg, collections.abc.Iterable)
return list(map(func, *arg))
def map(cls, func, *args, **kwargs):
"""
Map a function across iterable(s) and return a list of results
:param func: function
Mappable function
:param args: iterable
Iterator(s)
"""
assert check.argument_callable(func)
assert check.argument_list_type(args, collections.Iterable)
return cls.client.map(func, *args, chunksize=cls.chunk)
def map(cls, func, *args, **kwargs):
"""
Map a function across iterable(s) and return a list of results
:param func: function
Mappable function
:param args: iterable
Iterator(s)
:param tmp_file_path: path
If this is not None, instead of putting data onto the KVS, data will be pickled to temp files and the
path to the temp file will be put onto the KVS
:param tell_children: bool
If this is True, all processes will end up with the final data after assembly. If false, only the master
will have the final data; others will return None
:return results: list
"""
tmp_file_path = kwargs.pop("tmp_file_path", None)
tell_children = kwargs.pop("tell_children", True)
assert check.argument_callable(func)
assert check.argument_list_type(args, collections.Iterable)
# Set up the multiprocessing
owncheck = cls.own_check(chunk=cls.chunk, kvs_key=COUNT)
results = dict()
for pos, arg in enumerate(zip(*args)):
if next(owncheck):
results[pos] = func(*arg)
# Process results and synchronize exit from the get call
results = cls.process_results(results, tmp_file_path=tmp_file_path, tell_children=tell_children)
cls.sync_processes(pref=POST_SYNC)
cls.master_remove_key(kvs_key=COUNT)
cls.master_remove_key(kvs_key=FINAL_DATA)
return results
def test_callable(self):
def callable_function(x):
return x
class NonCallableClass(object):
pass
self.assertTrue(check.argument_callable(None, allow_none=True))
with self.assertRaises(ValueError):
check.argument_callable(None, allow_none=False)
self.assertTrue(check.argument_callable(int))
self.assertTrue(check.argument_callable(callable_function))
with self.assertRaises(ValueError):
check.argument_callable(NonCallableClass())
with self.assertRaises(ValueError):
check.argument_callable(1)
with self.assertRaises(ValueError):
check.argument_callable("string")
