def get_conn(self, lithops_executor_config): """ Initializes Lithops executor. """ lithops_executor_config['log_level'] = 'DEBUG' lithops_executor_config['config'] = self.lithops_config return function_executor(**lithops_executor_config)
def test_map_reduce_url(self): print('Testing map_reduce() over URLs...') pw = lithops.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def lithops_inside_lithops_map_function(x): def _func(x): return x pw = lithops.function_executor() pw.map(_func, range(x)) return pw.get_result()
def test_map_reduce(self): print('Testing map_reduce()...') iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]] pw = lithops.function_executor(config=CONFIG) pw.map_reduce(TestMethods.simple_map_function, iterdata, TestMethods.simple_reduce_function) result = pw.get_result() self.assertEqual(result, 20)
def lithops_return_futures_map_function3(x): def _func(x): return x + 1 pw = lithops.function_executor() fut1 = pw.map(_func, range(x)) fut2 = pw.map(_func, range(x)) return fut1 + fut2
def test_cloudobject(self): print('Testing cloudobjects...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' with lithops.function_executor(config=CONFIG) as pw: pw.map_reduce(TestMethods.my_cloudobject_put, data_prefix, TestMethods.my_cloudobject_get) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_storage_handler(self): print('Testing "storage" function arg...') iterdata = [[key, STORAGE_CONFIG['bucket']] for key in TestUtils.list_test_keys()] pw = lithops.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_storage, iterdata, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_bucket(self): print('Testing map_reduce() over a bucket...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = lithops.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_internal_executions(self): print('Testing internal executions...') pw = lithops.function_executor(config=CONFIG) pw.map(TestMethods.lithops_inside_lithops_map_function, range(1, 11)) result = pw.get_result() self.assertEqual(result, [list(range(i)) for i in range(1, 11)]) pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.lithops_return_futures_map_function1, 3) pw.get_result() pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.lithops_return_futures_map_function2, 3) pw.get_result() pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.lithops_return_futures_map_function3, 3) pw.wait() pw.get_result()
def test_call_async(self): print('Testing call_async()...') pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.hello_world, "") result = pw.get_result() self.assertEqual(result, "Hello World!") pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.concat, ["a", "b"]) result = pw.get_result() self.assertEqual(result, "a b") pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.simple_map_function, [4, 6]) result = pw.get_result() self.assertEqual(result, 10) pw = lithops.function_executor(config=CONFIG) pw.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8}) result = pw.get_result() self.assertEqual(result, 10)
def test_chunks_bucket(self): print('Testing chunks on a bucket...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' pw = lithops.function_executor(config=CONFIG) futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_size=1 * 1024**2) result = pw.get_result(futures) self.assertEqual(result, self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 8) pw = lithops.function_executor(config=CONFIG) futures = pw.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, chunk_n=2) result = pw.get_result(futures) self.assertEqual(result, self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 11)
def test_map_reduce_obj_key(self): print('Testing map_reduce() over object keys...') sb = STORAGE_CONFIG['backend'] bucket_name = STORAGE_CONFIG['bucket'] iterdata = [ sb + '://' + bucket_name + '/' + key for key in TestUtils.list_test_keys() ] pw = lithops.function_executor(config=CONFIG) pw.map_reduce(TestMethods.my_map_function_obj, iterdata, TestMethods.my_reduce_function) result = pw.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_function(debug): set_debug(debug) def hello(name): return 'Hello {}!'.format(name) pw = lithops.function_executor() pw.call_async(hello, 'World') result = pw.get_result() print() if result == 'Hello World!': print(result, 'Lithops is working as expected :)') else: print(result, 'Something went wrong :(') print()
def test_multiple_executions(self): print('Testing multiple executions...') pw = lithops.function_executor(config=CONFIG) iterdata = [[1, 1], [2, 2]] pw.map(TestMethods.simple_map_function, iterdata) iterdata = [[3, 3], [4, 4]] pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8]) iterdata = [[1, 1], [2, 2]] pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4]) iterdata = [[1, 1], [2, 2]] futures1 = pw.map(TestMethods.simple_map_function, iterdata) result1 = pw.get_result(fs=futures1) iterdata = [[3, 3], [4, 4]] futures2 = pw.map(TestMethods.simple_map_function, iterdata) result2 = pw.get_result(fs=futures2) self.assertEqual(result1, [2, 4]) self.assertEqual(result2, [6, 8])
def test_map(self): print('Testing map()...') iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]] pw = lithops.function_executor(config=CONFIG) pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8]) pw = lithops.function_executor(config=CONFIG, workers=1) pw.map(TestMethods.simple_map_function, iterdata) result = pw.get_result() self.assertEqual(result, [2, 4, 6, 8]) pw = lithops.function_executor(config=CONFIG) set_iterdata = set(range(2)) pw.map(TestMethods.hello_world, set_iterdata) result = pw.get_result() self.assertEqual(result, ['Hello World!'] * 2) pw = lithops.function_executor(config=CONFIG) generator_iterdata = range(2) pw.map(TestMethods.hello_world, generator_iterdata) result = pw.get_result() self.assertEqual(result, ['Hello World!'] * 2) pw = lithops.function_executor(config=CONFIG) listDicts_iterdata = [{'x': 2, 'y': 8}, {'x': 2, 'y': 8}] pw.map(TestMethods.simple_map_function, listDicts_iterdata) result = pw.get_result() self.assertEqual(result, [10, 10]) pw = lithops.function_executor(config=CONFIG) set_iterdata = [["a", "b"], ["c", "d"]] pw.map(TestMethods.concat, set_iterdata) result = pw.get_result() self.assertEqual(result, ["a b", "c d"])
def lithops_return_futures_map_function2(x): def _func(x): return x + 1 pw = lithops.function_executor() return pw.call_async(_func, x + 5)
import lithops from myfunc import my_map_function iterdata = [1, 2, 3, 4, 5] if __name__ == '__main__': pw = lithops.function_executor() futures = pw.map(my_map_function, iterdata) print(pw.get_result()) pw.clean()
def lithops_return_futures_map_function1(x): def _func(x): return x + 1 pw = lithops.function_executor() return pw.map(_func, range(x))