def get_conn(self, lithops_executor_config):
"""
Initializes Lithops executor.
"""
lithops_executor_config['log_level'] = 'DEBUG'
lithops_executor_config['config'] = self.lithops_config
return function_executor(**lithops_executor_config)
def test_map_reduce_url(self):
print('Testing map_reduce() over URLs...')
pw = lithops.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS,
TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def lithops_inside_lithops_map_function(x):
def _func(x):
return x
pw = lithops.function_executor()
pw.map(_func, range(x))
return pw.get_result()
def test_map_reduce(self):
print('Testing map_reduce()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
pw = lithops.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.simple_map_function, iterdata,
TestMethods.simple_reduce_function)
result = pw.get_result()
self.assertEqual(result, 20)
def lithops_return_futures_map_function3(x):
def _func(x):
return x + 1
pw = lithops.function_executor()
fut1 = pw.map(_func, range(x))
fut2 = pw.map(_func, range(x))
return fut1 + fut2
def test_cloudobject(self):
print('Testing cloudobjects...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
with lithops.function_executor(config=CONFIG) as pw:
pw.map_reduce(TestMethods.my_cloudobject_put, data_prefix,
TestMethods.my_cloudobject_get)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_storage_handler(self):
print('Testing "storage" function arg...')
iterdata = [[key, STORAGE_CONFIG['bucket']]
for key in TestUtils.list_test_keys()]
pw = lithops.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_storage, iterdata,
TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_bucket(self):
print('Testing map_reduce() over a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = lithops.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_internal_executions(self):
print('Testing internal executions...')
pw = lithops.function_executor(config=CONFIG)
pw.map(TestMethods.lithops_inside_lithops_map_function, range(1, 11))
result = pw.get_result()
self.assertEqual(result, [list(range(i)) for i in range(1, 11)])
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.lithops_return_futures_map_function1, 3)
pw.get_result()
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.lithops_return_futures_map_function2, 3)
pw.get_result()
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.lithops_return_futures_map_function3, 3)
pw.wait()
pw.get_result()
def test_call_async(self):
print('Testing call_async()...')
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.hello_world, "")
result = pw.get_result()
self.assertEqual(result, "Hello World!")
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.concat, ["a", "b"])
result = pw.get_result()
self.assertEqual(result, "a b")
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.simple_map_function, [4, 6])
result = pw.get_result()
self.assertEqual(result, 10)
pw = lithops.function_executor(config=CONFIG)
pw.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8})
result = pw.get_result()
self.assertEqual(result, 10)
def test_chunks_bucket(self):
print('Testing chunks on a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
pw = lithops.function_executor(config=CONFIG)
futures = pw.map_reduce(TestMethods.my_map_function_obj,
data_prefix,
TestMethods.my_reduce_function,
chunk_size=1 * 1024**2)
result = pw.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 8)
pw = lithops.function_executor(config=CONFIG)
futures = pw.map_reduce(TestMethods.my_map_function_obj,
data_prefix,
TestMethods.my_reduce_function,
chunk_n=2)
result = pw.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 11)
def test_map_reduce_obj_key(self):
print('Testing map_reduce() over object keys...')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [
sb + '://' + bucket_name + '/' + key
for key in TestUtils.list_test_keys()
]
pw = lithops.function_executor(config=CONFIG)
pw.map_reduce(TestMethods.my_map_function_obj, iterdata,
TestMethods.my_reduce_function)
result = pw.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_function(debug):
set_debug(debug)
def hello(name):
return 'Hello {}!'.format(name)
pw = lithops.function_executor()
pw.call_async(hello, 'World')
result = pw.get_result()
print()
if result == 'Hello World!':
print(result, 'Lithops is working as expected :)')
else:
print(result, 'Something went wrong :(')
print()
def test_multiple_executions(self):
print('Testing multiple executions...')
pw = lithops.function_executor(config=CONFIG)
iterdata = [[1, 1], [2, 2]]
pw.map(TestMethods.simple_map_function, iterdata)
iterdata = [[3, 3], [4, 4]]
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
iterdata = [[1, 1], [2, 2]]
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4])
iterdata = [[1, 1], [2, 2]]
futures1 = pw.map(TestMethods.simple_map_function, iterdata)
result1 = pw.get_result(fs=futures1)
iterdata = [[3, 3], [4, 4]]
futures2 = pw.map(TestMethods.simple_map_function, iterdata)
result2 = pw.get_result(fs=futures2)
self.assertEqual(result1, [2, 4])
self.assertEqual(result2, [6, 8])
def test_map(self):
print('Testing map()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
pw = lithops.function_executor(config=CONFIG)
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
pw = lithops.function_executor(config=CONFIG, workers=1)
pw.map(TestMethods.simple_map_function, iterdata)
result = pw.get_result()
self.assertEqual(result, [2, 4, 6, 8])
pw = lithops.function_executor(config=CONFIG)
set_iterdata = set(range(2))
pw.map(TestMethods.hello_world, set_iterdata)
result = pw.get_result()
self.assertEqual(result, ['Hello World!'] * 2)
pw = lithops.function_executor(config=CONFIG)
generator_iterdata = range(2)
pw.map(TestMethods.hello_world, generator_iterdata)
result = pw.get_result()
self.assertEqual(result, ['Hello World!'] * 2)
pw = lithops.function_executor(config=CONFIG)
listDicts_iterdata = [{'x': 2, 'y': 8}, {'x': 2, 'y': 8}]
pw.map(TestMethods.simple_map_function, listDicts_iterdata)
result = pw.get_result()
self.assertEqual(result, [10, 10])
pw = lithops.function_executor(config=CONFIG)
set_iterdata = [["a", "b"], ["c", "d"]]
pw.map(TestMethods.concat, set_iterdata)
result = pw.get_result()
self.assertEqual(result, ["a b", "c d"])
def lithops_return_futures_map_function2(x):
def _func(x):
return x + 1
pw = lithops.function_executor()
return pw.call_async(_func, x + 5)
import lithops
from myfunc import my_map_function
iterdata = [1, 2, 3, 4, 5]
if __name__ == '__main__':
pw = lithops.function_executor()
futures = pw.map(my_map_function, iterdata)
print(pw.get_result())
pw.clean()
def lithops_return_futures_map_function1(x):
def _func(x):
return x + 1
pw = lithops.function_executor()
return pw.map(_func, range(x))