def test_map(self):
print('Testing map()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
pw = pywren.function_executor(config=CONFIG)
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
pw = pywren.function_executor(config=CONFIG, workers=1)
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
pw = pywren.function_executor(config=CONFIG)
set_iterdata = set(range(2))
pw.map(TestMethods.hello_world, set_iterdata)
result = pw.get_result()
self.assertEqual(result, ['Hello World!'] * 2)
pw = pywren.function_executor(config=CONFIG)
generator_iterdata = range(2)
pw.map(TestMethods.hello_world, generator_iterdata)
result = pw.get_result()
self.assertEqual(result, ['Hello World!'] * 2)
pw = pywren.function_executor(config=CONFIG)
listDicts_iterdata = [{'x': 2, 'y': 8}, {'x': 2, 'y': 8}]
pw.map(TestMethods.simple_map_function, listDicts_iterdata)
result = pw.get_result()
self.assertEqual(result, [10, 10])
def test_chunks_bucket(self):
print('Testing cunk_size on a bucket...')
data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(my_map_function_obj, data_prefix, my_reduce_function, chunk_size=1*1024**2)
result = pw.get_result()
self.checkResult(result)
def test_storage_handler(self):
print('Testing ibm_cos function arg...')
iterdata = [[key, STORAGE_CONFIG['bucket']] for key in list_test_keys()]
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(my_map_function_ibm_cos, iterdata, my_reduce_function)
result = pw.get_result()
self.checkResult(result)
def test_map_reduce_url(self):
print('Testing map_reduce() over URLs...')
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS,
TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def pywren_inside_pywren_map_function(x):
def _func(x):
return x
pw = pywren.function_executor()
pw.map(_func, range(x))
return pw.get_result()
def test_chunks_bucket(self):
print('Testing chunks on a bucket...')
data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function,
chunk_size=1 * 1024 ** 2)
result = pw.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 8)
pw = pywren.function_executor(config=CONFIG)
futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2)
result = pw.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 11)
def test_internal_executions(self):
print('Testing internal executions...')
pw = pywren.function_executor(config=CONFIG)
pw.map(pywren_inside_pywren_map_function1, range(1, 11))
result = pw.get_result()
self.assertEqual(result, [0] + [list(range(i)) for i in range(2, 11)])
pw = pywren.function_executor(config=CONFIG)
pw.call_async(pywren_inside_pywren_map_function2, 10)
result = pw.get_result()
self.assertEqual(result, 10)
pw = pywren.function_executor(config=CONFIG)
pw.map(pywren_inside_pywren_map_function3, range(1, 11))
result = pw.get_result()
self.assertEqual(result, [[0, 0]] + [[list(range(i)), list(range(i))] for i in range(2, 11)])
def test_map_reduce(self):
print('Testing map_reduce()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.simple_map_function, iterdata, TestMethods.simple_reduce_function)
result = pw.get_result()
self.assertEqual(result, 20)
def pywren_inside_pywren_map_function2(x):
def _func(x):
return x
pw = pywren.function_executor(config=CONFIG)
pw.call_async(_func, x)
return pw.get_result()
def test_call_async(self):
print('Testing call_async()...')
pw = pywren.function_executor(config=CONFIG)
pw.call_async(TestMethods.hello_world, "")
result = pw.get_result()
self.assertEqual(result, "Hello World!")
pw = pywren.function_executor(config=CONFIG)
pw.call_async(TestMethods.simple_map_function, [4, 6])
result = pw.get_result()
self.assertEqual(result, 10)
pw = pywren.function_executor(config=CONFIG)
pw.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8})
result = pw.get_result()
self.assertEqual(result, 10)
def test_cloudobject(self):
print('Testing cloudobjects...')
data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(my_cloudobject_put, data_prefix, my_cloudobject_get)
result = pw.get_result()
self.checkResult(result)
def test_storage_handler(self):
print('Testing "storage" function arg...')
iterdata = [[key, STORAGE_CONFIG['bucket']] for key in TestUtils.list_test_keys()]
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_storage, iterdata, TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map(self):
print('Testing map()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
pw = pywren.function_executor(config=CONFIG)
pw.map(simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
def test_cloudobject(self):
print('Testing cloudobjects...')
data_prefix = STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_cloudobject_put, data_prefix, TestMethods.my_cloudobject_get)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_cos_bucket(self):
print('Testing map_reduce() over a COS bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(my_map_function_obj, data_prefix, my_reduce_function)
result = pw.get_result()
self.checkResult(result)
def pywren_inside_pywren_map_function3(x):
def _func(x):
return x
pw = pywren.function_executor(config=CONFIG)
fut1 = pw.map(_func, range(x))
fut2 = pw.map(_func, range(x))
return [pw.get_result(fut1), pw.get_result(fut2)]
def pywren_return_futures_map_function3(x):
def _func(x):
return x + 1
pw = pywren.function_executor()
fut1 = pw.map(_func, range(x))
fut2 = pw.map(_func, range(x))
return fut1 + fut2
def test_map_reduce_obj_bucket(self):
print('Testing map_reduce() over a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_chunks_bucket_one_reducer_per_object(self):
print('Testing chunks on a bucket with one reducer per object...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function,
chunk_size=1 * 1024 ** 2, reducer_one_per_object=True)
result = pw.get_result(futures)
self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 12)
pw = pywren.function_executor(config=CONFIG)
futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2,
reducer_one_per_object=True)
result = pw.get_result(futures)
self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 15)
def test_map_reduce_obj_key(self):
print('Testing map_reduce() over object keys...')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [sb + '://' + bucket_name + '/' + key for key in TestUtils.list_test_keys()]
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_obj, iterdata, TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_cos_key_one_reducer_per_object(self):
print('Testing map_reduce() over COS keys with one reducer per object...')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [sb+'://'+bucket_name+'/'+key for key in list_test_keys()]
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(my_map_function_obj, iterdata, my_reduce_function, reducer_one_per_object=True)
result = pw.get_result()
self.checkResult(result)
def test_internal_executions(self):
print('Testing internal executions...')
pw = pywren.function_executor(config=CONFIG)
pw.map(TestMethods.pywren_inside_pywren_map_function, range(1, 11))
result = pw.get_result()
self.assertEqual(result, [0] + [list(range(i)) for i in range(2, 11)])
pw = pywren.function_executor(config=CONFIG)
pw.call_async(TestMethods.pywren_return_futures_map_function1, 3)
pw.get_result()
pw = pywren.function_executor(config=CONFIG)
pw.call_async(TestMethods.pywren_return_futures_map_function2, 3)
pw.get_result()
pw = pywren.function_executor(config=CONFIG)
pw.call_async(TestMethods.pywren_return_futures_map_function3, 3)
pw.get_result()
def test_map_reduce_cos_bucket_one_reducer_per_object(self):
print('Testing map_reduce() over a COS bucket with one reducer per object...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function,
reducer_one_per_object=True)
result = pw.get_result()
self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
def test_function(debug):
set_debug(debug)
def hello(name):
return 'Hello {}!'.format(name)
pw = pywren.function_executor()
pw.call_async(hello, 'World')
result = pw.get_result()
print()
if result == 'Hello World!':
print(result, 'Pywren is working as expected :)')
else:
print(result, 'Something went wrong :(')
print()
def __init__(self, config, ds_config, db_config, use_cache=True):
self.config = config
self.storage = config['storage']
self.ds_config = ds_config
self.db_config = db_config
self.use_cache = use_cache
self.pywren_executor = pywren.function_executor(config=self.config,
runtime_memory=2048)
self.cacher = PipelineCacher(self.pywren_executor,
self.ds_config["name"])
if not self.use_cache:
self.cacher.clean()
self.ds_segm_size_mb = 128
self.image_gen_config = {
"q": 99,
"do_preprocessing": False,
"nlevels": 30,
"ppm": 3.0
}
def test_multiple_executions(self):
print('Testing multiple executions...')
pw = pywren.function_executor(config=CONFIG)
iterdata = [[1, 1], [2, 2]]
pw.map(TestMethods.simple_map_function, iterdata)
iterdata = [[3, 3], [4, 4]]
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
iterdata = [[1, 1], [2, 2]]
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4])
iterdata = [[1, 1], [2, 2]]
futures1 = pw.map(TestMethods.simple_map_function, iterdata)
result1 = pw.get_result(fs=futures1)
iterdata = [[3, 3], [4, 4]]
futures2 = pw.map(TestMethods.simple_map_function, iterdata)
result2 = pw.get_result(fs=futures2)
self.assertEqual(result1, [2, 4])
self.assertEqual(result2, [6, 8])
def test_map_reduce_url(self):
print('Testing map_reduce() over URLs...')
pw = pywren.function_executor(config=CONFIG)
pw.map_reduce(my_map_function_url, TEST_FILES_URLS, my_reduce_function)
result = pw.get_result()
self.checkResult(result + 1)
def pywren_return_futures_map_function2(x):
def _func(x):
return x + 1
pw = pywren.function_executor()
return pw.call_async(_func, x + 5)
def pywren_return_futures_map_function1(x):
def _func(x):
return x + 1
pw = pywren.function_executor()
return pw.map(_func, range(x))
