def test_map_reduce_url(self):
print('Testing map_reduce() over URLs...')
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce(self):
print('Testing map_reduce()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.simple_map_function, iterdata,
TestMethods.simple_reduce_function)
result = ex.get_result()
self.assertEqual(result, 20)
def test_storage_handler(self):
print('Testing "storage" function arg...')
iterdata = [[key, STORAGE_CONFIG['bucket']]
for key in TestUtils.list_test_keys()]
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_storage, iterdata,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_bucket(self):
print('Testing map_reduce() over a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_key(self):
print('Testing map_reduce() over object keys...')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [
sb + '://' + bucket_name + '/' + key
for key in TestUtils.list_test_keys()
]
ex = FunctionExecutor(config=CONFIG)
ex.map_reduce(TestMethods.my_map_function_obj, iterdata,
TestMethods.my_reduce_function)
result = ex.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_chunks_bucket(self):
print('Testing chunks on a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
ex = FunctionExecutor(config=CONFIG)
futures = ex.map_reduce(TestMethods.my_map_function_obj,
data_prefix,
TestMethods.my_reduce_function,
chunk_size=1 * 1024**2)
result = ex.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 8)
ex = FunctionExecutor(config=CONFIG)
futures = ex.map_reduce(TestMethods.my_map_function_obj,
data_prefix,
TestMethods.my_reduce_function,
chunk_n=2)
result = ex.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 11)
def my_map_function(x):
return x + 7
def my_reduce_function(results):
total = 0
for map_result in results:
total = total + map_result
return total
if __name__ == "__main__":
"""
By default the reducer will be launched within a Cloud Function
when the local PyWren have all the results from the mappers.
"""
cb_exc = FunctionExecutor()
cb_exc.map_reduce(my_map_function, iterdata, my_reduce_function)
print(cb_exc.get_result())
"""
Set 'reducer_wait_local=True' to wait for the results locally.
"""
cb_exc = FunctionExecutor()
cb_exc.map_reduce(my_map_function,
iterdata,
my_reduce_function,
reducer_wait_local=True)
print(cb_exc.get_result())