def test_map(self): print('Testing map()...') iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]] pw = pywren.function_executor(config=CONFIG) pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8]) pw = pywren.function_executor(config=CONFIG, workers=1) pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8]) pw = pywren.function_executor(config=CONFIG) set_iterdata = set(range(2)) pw.map(TestMethods.hello_world, set_iterdata) result = pw.get_result() self.assertEqual(result, ['Hello World!'] * 2) pw = pywren.function_executor(config=CONFIG) generator_iterdata = range(2) pw.map(TestMethods.hello_world, generator_iterdata) result = pw.get_result() self.assertEqual(result, ['Hello World!'] * 2) pw = pywren.function_executor(config=CONFIG) listDicts_iterdata = [{'x': 2, 'y': 8}, {'x': 2, 'y': 8}] pw.map(TestMethods.simple_map_function, listDicts_iterdata) result = pw.get_result() self.assertEqual(result, [10, 10])
def test_chunks_bucket(self): print('Testing cunk_size on a bucket...') data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) pw.map_reduce(my_map_function_obj, data_prefix, my_reduce_function, chunk_size=1*1024**2) result = pw.get_result() self.checkResult(result)
def test_storage_handler(self): print('Testing ibm_cos function arg...') iterdata = [[key, STORAGE_CONFIG['bucket']] for key in list_test_keys()] pw = pywren.function_executor(config=CONFIG) pw.map_reduce(my_map_function_ibm_cos, iterdata, my_reduce_function) result = pw.get_result() self.checkResult(result)
def test_map_reduce_url(self): print('Testing map_reduce() over URLs...') pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def pywren_inside_pywren_map_function(x): def _func(x): return x pw = pywren.function_executor() pw.map(_func, range(x)) return pw.get_result()
def test_chunks_bucket(self): print('Testing chunks on a bucket...') data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_size=1 * 1024 ** 2) result = pw.get_result(futures) self.assertEqual(result, self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 8) pw = pywren.function_executor(config=CONFIG) futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2) result = pw.get_result(futures) self.assertEqual(result, self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 11)
def test_internal_executions(self): print('Testing internal executions...') pw = pywren.function_executor(config=CONFIG) pw.map(pywren_inside_pywren_map_function1, range(1, 11)) result = pw.get_result() self.assertEqual(result, [0] + [list(range(i)) for i in range(2, 11)]) pw = pywren.function_executor(config=CONFIG) pw.call_async(pywren_inside_pywren_map_function2, 10) result = pw.get_result() self.assertEqual(result, 10) pw = pywren.function_executor(config=CONFIG) pw.map(pywren_inside_pywren_map_function3, range(1, 11)) result = pw.get_result() self.assertEqual(result, [[0, 0]] + [[list(range(i)), list(range(i))] for i in range(2, 11)])
def test_map_reduce(self): print('Testing map_reduce()...') iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]] pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.simple_map_function, iterdata, TestMethods.simple_reduce_function) result = pw.get_result() self.assertEqual(result, 20)
def pywren_inside_pywren_map_function2(x): def _func(x): return x pw = pywren.function_executor(config=CONFIG) pw.call_async(_func, x) return pw.get_result()
def test_call_async(self): print('Testing call_async()...') pw = pywren.function_executor(config=CONFIG) pw.call_async(TestMethods.hello_world, "") result = pw.get_result() self.assertEqual(result, "Hello World!") pw = pywren.function_executor(config=CONFIG) pw.call_async(TestMethods.simple_map_function, [4, 6]) result = pw.get_result() self.assertEqual(result, 10) pw = pywren.function_executor(config=CONFIG) pw.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8}) result = pw.get_result() self.assertEqual(result, 10)
def test_cloudobject(self): print('Testing cloudobjects...') data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) pw.map_reduce(my_cloudobject_put, data_prefix, my_cloudobject_get) result = pw.get_result() self.checkResult(result)
def test_storage_handler(self): print('Testing "storage" function arg...') iterdata = [[key, STORAGE_CONFIG['bucket']] for key in TestUtils.list_test_keys()] pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_storage, iterdata, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map(self): print('Testing map()...') iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]] pw = pywren.function_executor(config=CONFIG) pw.map(simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8])
def test_cloudobject(self): print('Testing cloudobjects...') data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_cloudobject_put, data_prefix, TestMethods.my_cloudobject_get) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_cos_bucket(self): print('Testing map_reduce() over a COS bucket...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) pw.map_reduce(my_map_function_obj, data_prefix, my_reduce_function) result = pw.get_result() self.checkResult(result)
def pywren_inside_pywren_map_function3(x): def _func(x): return x pw = pywren.function_executor(config=CONFIG) fut1 = pw.map(_func, range(x)) fut2 = pw.map(_func, range(x)) return [pw.get_result(fut1), pw.get_result(fut2)]
def pywren_return_futures_map_function3(x): def _func(x): return x + 1 pw = pywren.function_executor() fut1 = pw.map(_func, range(x)) fut2 = pw.map(_func, range(x)) return fut1 + fut2
def test_map_reduce_obj_bucket(self): print('Testing map_reduce() over a bucket...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_chunks_bucket_one_reducer_per_object(self): print('Testing chunks on a bucket with one reducer per object...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_size=1 * 1024 ** 2, reducer_one_per_object=True) result = pw.get_result(futures) self.assertEqual(sum(result), self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 12) pw = pywren.function_executor(config=CONFIG) futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2, reducer_one_per_object=True) result = pw.get_result(futures) self.assertEqual(sum(result), self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 15)
def test_map_reduce_obj_key(self): print('Testing map_reduce() over object keys...') sb = STORAGE_CONFIG['backend'] bucket_name = STORAGE_CONFIG['bucket'] iterdata = [sb + '://' + bucket_name + '/' + key for key in TestUtils.list_test_keys()] pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_obj, iterdata, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_cos_key_one_reducer_per_object(self): print('Testing map_reduce() over COS keys with one reducer per object...') sb = STORAGE_CONFIG['backend'] bucket_name = STORAGE_CONFIG['bucket'] iterdata = [sb+'://'+bucket_name+'/'+key for key in list_test_keys()] pw = pywren.function_executor(config=CONFIG) pw.map_reduce(my_map_function_obj, iterdata, my_reduce_function, reducer_one_per_object=True) result = pw.get_result() self.checkResult(result)
def test_internal_executions(self): print('Testing internal executions...') pw = pywren.function_executor(config=CONFIG) pw.map(TestMethods.pywren_inside_pywren_map_function, range(1, 11)) result = pw.get_result() self.assertEqual(result, [0] + [list(range(i)) for i in range(2, 11)]) pw = pywren.function_executor(config=CONFIG) pw.call_async(TestMethods.pywren_return_futures_map_function1, 3) pw.get_result() pw = pywren.function_executor(config=CONFIG) pw.call_async(TestMethods.pywren_return_futures_map_function2, 3) pw.get_result() pw = pywren.function_executor(config=CONFIG) pw.call_async(TestMethods.pywren_return_futures_map_function3, 3) pw.get_result()
def test_map_reduce_cos_bucket_one_reducer_per_object(self): print('Testing map_reduce() over a COS bucket with one reducer per object...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = pywren.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, reducer_one_per_object=True) result = pw.get_result() self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
def test_function(debug): set_debug(debug) def hello(name): return 'Hello {}!'.format(name) pw = pywren.function_executor() pw.call_async(hello, 'World') result = pw.get_result() print() if result == 'Hello World!': print(result, 'Pywren is working as expected :)') else: print(result, 'Something went wrong :(') print()
def __init__(self, config, ds_config, db_config, use_cache=True): self.config = config self.storage = config['storage'] self.ds_config = ds_config self.db_config = db_config self.use_cache = use_cache self.pywren_executor = pywren.function_executor(config=self.config, runtime_memory=2048) self.cacher = PipelineCacher(self.pywren_executor, self.ds_config["name"]) if not self.use_cache: self.cacher.clean() self.ds_segm_size_mb = 128 self.image_gen_config = { "q": 99, "do_preprocessing": False, "nlevels": 30, "ppm": 3.0 }
def test_multiple_executions(self): print('Testing multiple executions...') pw = pywren.function_executor(config=CONFIG) iterdata = [[1, 1], [2, 2]] pw.map(TestMethods.simple_map_function, iterdata) iterdata = [[3, 3], [4, 4]] pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8]) iterdata = [[1, 1], [2, 2]] pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4]) iterdata = [[1, 1], [2, 2]] futures1 = pw.map(TestMethods.simple_map_function, iterdata) result1 = pw.get_result(fs=futures1) iterdata = [[3, 3], [4, 4]] futures2 = pw.map(TestMethods.simple_map_function, iterdata) result2 = pw.get_result(fs=futures2) self.assertEqual(result1, [2, 4]) self.assertEqual(result2, [6, 8])
def test_map_reduce_url(self): print('Testing map_reduce() over URLs...') pw = pywren.function_executor(config=CONFIG) pw.map_reduce(my_map_function_url, TEST_FILES_URLS, my_reduce_function) result = pw.get_result() self.checkResult(result + 1)
def pywren_return_futures_map_function2(x): def _func(x): return x + 1 pw = pywren.function_executor() return pw.call_async(_func, x + 5)
def pywren_return_futures_map_function1(x): def _func(x): return x + 1 pw = pywren.function_executor() return pw.map(_func, range(x))