def __init__(self, processes=None, initializer=None, initargs=(),
maxtasksperchild=None, context=None):
self._ctx = context or get_context()
#self._setup_queues()
self._taskqueue = queue.Queue()
self._cache = {}
self._state = RUN
self._maxtasksperchild = maxtasksperchild
self._initializer = initializer
self._initargs = initargs
if processes is not None and processes < 1:
raise ValueError("Number of processes must be at least 1")
if processes is not None:
if self._initargs:
self._executor = FunctionExecutor(workers=processes, **self._initargs)
else:
self._executor = FunctionExecutor(workers=processes)
self._processes = processes
else:
if self._initargs:
self._executor = FunctionExecutor(**self._initargs)
else:
self._executor = FunctionExecutor()
self._processes = self._executor.invoker.workers
if initializer is not None and not callable(initializer):
raise TypeError('initializer must be a callable')
self._pool = []
def test_map(self):
print('Testing map()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
ex = FunctionExecutor(config=CONFIG)
ex.map(TestMethods.simple_map_function, iterdata)
result = ex.get_result()
self.assertEqual(result, [2, 4, 6, 8])
ex = FunctionExecutor(config=CONFIG, workers=1)
ex.map(TestMethods.simple_map_function, iterdata)
result = ex.get_result()
self.assertEqual(result, [2, 4, 6, 8])
ex = FunctionExecutor(config=CONFIG)
set_iterdata = set(range(2))
ex.map(TestMethods.hello_world, set_iterdata)
result = ex.get_result()
self.assertEqual(result, ['Hello World!'] * 2)
ex = FunctionExecutor(config=CONFIG)
generator_iterdata = range(2)
ex.map(TestMethods.hello_world, generator_iterdata)
result = ex.get_result()
self.assertEqual(result, ['Hello World!'] * 2)
ex = FunctionExecutor(config=CONFIG)
listDicts_iterdata = [{'x': 2, 'y': 8}, {'x': 2, 'y': 8}]
ex.map(TestMethods.simple_map_function, listDicts_iterdata)
result = ex.get_result()
self.assertEqual(result, [10, 10])
def test_map_reduce_url(self):
print('Testing map_reduce() over URLs...')
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_call_async(self):
print('Testing call_async()...')
ex = FunctionExecutor(config=CONFIG)
ex.call_async(TestMethods.hello_world, "")
result = ex.get_result()
self.assertEqual(result, "Hello World!")
ex = FunctionExecutor(config=CONFIG)
ex.call_async(TestMethods.simple_map_function, [4, 6])
result = ex.get_result()
self.assertEqual(result, 10)
ex = FunctionExecutor(config=CONFIG)
ex.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8})
result = ex.get_result()
self.assertEqual(result, 10)
def pywren_inside_pywren_map_function(x):
def _func(x):
return x
ex = FunctionExecutor()
ex.map(_func, range(x))
return ex.get_result()
def test_map_reduce(self):
print('Testing map_reduce()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.simple_map_function, iterdata,
TestMethods.simple_reduce_function)
result = ex.get_result()
self.assertEqual(result, 20)
def pywren_return_futures_map_function3(x):
def _func(x):
return x + 1
ex = FunctionExecutor()
fut1 = ex.map(_func, range(x))
fut2 = ex.map(_func, range(x))
return fut1 + fut2
def test_storage_handler(self):
print('Testing "storage" function arg...')
iterdata = [[key, STORAGE_CONFIG['bucket']]
for key in TestUtils.list_test_keys()]
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_storage, iterdata,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_cloudobject(self):
print('Testing cloudobjects...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
with FunctionExecutor(config=CONFIG) as ex:
ex.map_reduce(TestMethods.my_cloudobject_put, data_prefix,
TestMethods.my_cloudobject_get)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_bucket(self):
print('Testing map_reduce() over a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_internal_executions(self):
print('Testing internal executions...')
ex = FunctionExecutor(config=CONFIG)
ex.map(TestMethods.pywren_inside_pywren_map_function, range(1, 11))
result = ex.get_result()
self.assertEqual(result, [list(range(i)) for i in range(1, 11)])
ex = FunctionExecutor(config=CONFIG)
ex.call_async(TestMethods.pywren_return_futures_map_function1, 3)
ex.get_result()
ex = FunctionExecutor(config=CONFIG)
ex.call_async(TestMethods.pywren_return_futures_map_function2, 3)
ex.get_result()
ex = FunctionExecutor(config=CONFIG)
ex.call_async(TestMethods.pywren_return_futures_map_function3, 3)
ex.wait()
ex.get_result()
def test_chunks_bucket(self):
print('Testing chunks on a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
ex = FunctionExecutor(config=CONFIG)
futures = ex.map_reduce(TestMethods.my_map_function_obj,
data_prefix,
TestMethods.my_reduce_function,
chunk_size=1 * 1024**2)
result = ex.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 8)
ex = FunctionExecutor(config=CONFIG)
futures = ex.map_reduce(TestMethods.my_map_function_obj,
data_prefix,
TestMethods.my_reduce_function,
chunk_n=2)
result = ex.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 11)
def test_map_reduce_obj_key(self):
print('Testing map_reduce() over object keys...')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [
sb + '://' + bucket_name + '/' + key
for key in TestUtils.list_test_keys()
]
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_obj, iterdata,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_multiple_executions(self):
print('Testing multiple executions...')
ex = FunctionExecutor(config=CONFIG)
iterdata = [[1, 1], [2, 2]]
ex.map(TestMethods.simple_map_function, iterdata)
iterdata = [[3, 3], [4, 4]]
ex.map(TestMethods.simple_map_function, iterdata)
result = ex.get_result()
self.assertEqual(result, [2, 4, 6, 8])
iterdata = [[1, 1], [2, 2]]
ex.map(TestMethods.simple_map_function, iterdata)
result = ex.get_result()
self.assertEqual(result, [2, 4])
iterdata = [[1, 1], [2, 2]]
futures1 = ex.map(TestMethods.simple_map_function, iterdata)
result1 = ex.get_result(fs=futures1)
iterdata = [[3, 3], [4, 4]]
futures2 = ex.map(TestMethods.simple_map_function, iterdata)
result2 = ex.get_result(fs=futures2)
self.assertEqual(result1, [2, 4])
self.assertEqual(result2, [6, 8])
def __init__(self, process_obj):
util._flush_std_streams()
self.returncode = None
self._executor = FunctionExecutor()
self._launch(process_obj)
def pywren_return_futures_map_function1(x):
def _func(x):
return x + 1
ex = FunctionExecutor()
return ex.map(_func, range(x))
def pywren_return_futures_map_function2(x):
def _func(x):
return x + 1
ex = FunctionExecutor()
return ex.call_async(_func, x + 5)
"""
Simple Cloudbutton example using the map_reduce method
of the raw API.
"""
from cloudbutton.engine.executor import FunctionExecutor
def my_function(x):
return x + 7
if __name__ == '__main__':
cb_exc = FunctionExecutor()
cb_exc.call_async(my_function, 3)
print(cb_exc.get_result())