def test_chunks_bucket_one_reducer_per_object(self):
"""tests the ability to create a separate function invocation based on the following parameters, as well as
create a separate invocation of a reduce function for each object:
chunk_size - creates [file_size//chunk_size] invocations to process each chunk_size bytes, of a given object.
chunk_number - creates 'chunk_number' invocations that process [file_size//chunk_number] bytes each. """
logger.info('Testing chunks on a bucket with one reducer per object')
OBJ_CHUNK_SIZE = 1 * 1024 ** 2
OBJ_CHUNK_NUMBER = 2
activations = 0
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + DATASET_PREFIX + '/'
fexec = lithops.FunctionExecutor(config=CONFIG)
futures = fexec.map_reduce(my_map_function_obj, data_prefix,
my_reduce_function,
obj_chunk_size=OBJ_CHUNK_SIZE,
reducer_one_per_object=True)
result = fexec.get_result(futures)
self.assertEqual(sum(result), self.__class__.words_in_cos_files)
for size in get_dataset_key_size(STORAGE, STORAGE_CONFIG):
activations += math.ceil(size / OBJ_CHUNK_SIZE)
self.assertEqual(len(futures), activations + len(TEST_FILES_URLS)) # + len(TEST_FILES_URLS) due to map_reduce activation per object
fexec = lithops.FunctionExecutor(config=CONFIG)
futures = fexec.map_reduce(my_map_function_obj, data_prefix,
my_reduce_function, obj_chunk_number=OBJ_CHUNK_NUMBER,
reducer_one_per_object=True)
result = fexec.get_result(futures)
self.assertEqual(sum(result), self.__class__.words_in_cos_files)
self.assertEqual(len(futures), len(TEST_FILES_URLS)*OBJ_CHUNK_NUMBER + len(TEST_FILES_URLS)) # + len(TEST_FILES_URLS) due to map_reduce activation per object
def test_cholesky_lambda():
X = np.random.randn(128, 128)
A = X.dot(X.T) + np.eye(X.shape[0])
shard_size = 128
shard_sizes = (shard_size, shard_size)
A_sharded = BigMatrix("job_runner_test",
shape=A.shape,
shard_sizes=shard_sizes,
write_header=True)
A_sharded.free()
shard_matrix(A_sharded, A)
program, meta = cholesky(A_sharded)
executor = fs.ProcessPoolExecutor(1)
print("starting program")
program.start()
pwex = lithops.FunctionExecutor()
futures = pwex.map(
lambda x: job_runner.lambdapack_run(
program, timeout=60, idle_timeout=6), range(16))
pwex.wait(futures)
print("RESULTSSS")
print([f.result() for f in futures])
futures = pwex.map(
lambda x: job_runner.lambdapack_run(
program, timeout=60, idle_timeout=6), range(16))
program.wait()
#program.free()
L_sharded = meta["outputs"][0]
L_npw = L_sharded.numpy()
L = np.linalg.cholesky(A)
assert (np.allclose(L_npw, L))
print("great success!")
def test_function(config, backend, storage, debug):
if config:
config = load_yaml_config(config)
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
try:
import getpass
username = getpass.getuser()
except Exception:
username = '******'
def hello(name):
return 'Hello {}!'.format(name)
fexec = lithops.FunctionExecutor(config=config, backend=backend, storage=storage)
fexec.call_async(hello, username)
result = fexec.get_result()
print()
if result == 'Hello {}!'.format(username):
print(result, 'Lithops is working as expected :)')
else:
print(result, 'Something went wrong :(')
print()
def test_map_reduce_url(self):
print('Testing map_reduce() over URLs...')
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS,
TestMethods.my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def lithops_inside_lithops_map_function(x):
def _func(x):
return x
fexec = lithops.FunctionExecutor()
fexec.map(_func, range(x))
return fexec.get_result()
def test_map_reduce_url(self):
logger.info('Testing map_reduce() over URLs')
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(my_map_function_url, TEST_FILES_URLS,
my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.words_in_cos_files)
def trisolve(pwex, A, B, out_bucket=None, tasks_per_job=1, lower=False):
if out_bucket is None:
out_bucket = A.bucket
root_key = generate_key_name_binop(A, B, "trisolve")
instructions, X, scratch = lp._trisolve(A,
B,
out_bucket=out_bucket,
lower=lower)
config = pwex.config
# if (isinstance(pwex.invoker, pywren.queues.SQSInvoker)):
# executor = pywren.standalone_executor
# else:
fexec = lithops.FunctionExecutor()
program = lp.LambdaPackProgram(instructions,
executor=fexec,
pywren_config=config)
print(program)
#assert False
program.start()
job_runner.lambdapack_run(program)
program.wait()
if program.program_status() != lp.PS.SUCCESS:
program.unwind()
raise Exception("Lambdapack Exception : {0}".format(
program.program_status()))
program.free()
# delete all intermediate information
[M.free() for M in scratch]
return X
def run_program_in_pywren(program, num_workers=32):
def pywren_run(_):
job_runner.lambdapack_run(program, timeout=60, idle_timeout=6)
pwex = lithops.FunctionExecutor()
futures = pwex.map(pywren_run, range(num_workers))
return futures
def test_multiple_shard_matrix_multiply(self):
fexec = lithops.FunctionExecutor(runtime='jsampe/numpy-lithops:04',
log_level='DEBUG')
X = np.random.randn(16, 16)
X_shard_sizes = tuple(map(int, np.array(X.shape) / 2))
X_sharded = BigMatrix("gemm_test_1",
shape=X.shape,
shard_sizes=X_shard_sizes,
storage=fexec.storage)
Y = np.random.randn(16, 16)
Y_shard_sizes = tuple(map(int, np.array(Y.shape) / 2))
Y_sharded = BigMatrix("gemm_test_2",
shape=Y.shape,
shard_sizes=Y_shard_sizes,
storage=fexec.storage)
shard_matrix(X_sharded, X)
shard_matrix(Y_sharded, Y)
XY_sharded = binops.gemm(fexec, X_sharded, Y_sharded, X_sharded.bucket,
1)
XY_sharded_local = XY_sharded.numpy()
XY = X.dot(Y)
X_sharded.free()
Y_sharded.free()
XY_sharded.free()
assert (np.all(np.isclose(XY, XY_sharded_local)))
os.system("rm -rf /dev/shm/*")
def test_storage_handler(self):
logger.info('Testing "storage" function arg')
iterdata = [(key, STORAGE_CONFIG['bucket']) for key in TestUtils.list_dataset_keys()]
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(TestMethods.my_map_function_storage, iterdata,
TestMethods.my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def lithops_return_futures_map_function3(x):
def _func(x):
return x + 1
fexec = lithops.FunctionExecutor()
fut1 = fexec.map(_func, range(x))
fut2 = fexec.map(_func, range(x))
return fut1 + fut2
def test_map_reduce(self):
print('Testing map_reduce()...')
iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]]
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(TestMethods.simple_map_function, iterdata,
TestMethods.simple_reduce_function)
result = fexec.get_result()
self.assertEqual(result, 20)
def test_storage_handler(self):
logger.info('Testing "storage" function arg')
iterdata = [(key, STORAGE_CONFIG['bucket'])
for key in list_dataset_keys(STORAGE, STORAGE_CONFIG)]
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(my_map_function_storage, iterdata, my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.words_in_cos_files)
def run_staging_payment(*args):
fexec = lithops.FunctionExecutor(config=args[0])
# fexec.call_async(working,args[1])
pgpassword = args[1]
fexec.map(working, [[1, pgpassword], [2, pgpassword], [3, pgpassword],
[4, pgpassword], [5, pgpassword], [6, pgpassword],
[7, pgpassword], [8, pgpassword], [20, pgpassword]])
print(fexec.get_result())
def test_map_reduce(self):
logger.info('Testing map_reduce() using memory')
iterdata = [(1, 1), (2, 2), (3, 3), (4, 4)]
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(simple_map_function, iterdata,
simple_reduce_function)
result = fexec.get_result()
self.assertEqual(result, 20)
def test_map_reduce_obj_bucket(self):
print('Testing map_reduce() over a bucket...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_bucket(self):
logger.info('Testing map_reduce() over a bucket')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + DATASET_PREFIX + '/'
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(my_map_function_obj, data_prefix,
my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.words_in_cos_files)
def test_cloudobject(self):
print('Testing cloudobjects...')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
with lithops.FunctionExecutor(config=CONFIG) as fexec:
fexec.map_reduce(TestMethods.my_cloudobject_put, data_prefix,
TestMethods.my_cloudobject_get)
result = fexec.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def __init__(self, ds_config, db_config, use_db_cache=True, use_ds_cache=True, hybrid_impl='auto'):
self.config = default_config()
self.ds_config = ds_config
self.db_config = db_config
self.use_db_cache = use_db_cache
self.use_ds_cache = use_ds_cache
if hybrid_impl == 'auto':
self.hybrid_impl = (
self.config['lithops']['mode'] == 'localhost'
or self.config['lithops']['mode'] == 'serverless' and 'ibm_vpc' in self.config
)
if self.hybrid_impl:
logger.info(f'Using the Hybrid implementation')
else:
logger.info(f'Using the pure Serverless implementation')
else:
self.hybrid_impl = hybrid_impl
lithops_bucket = self.config['lithops']['storage_bucket']
self.ds_bucket = self.config.get('storage', {}).get('ds_bucket', lithops_bucket)
self.lithops_executor = lithops.FunctionExecutor(config=self.config, runtime_memory=2048)
if self.hybrid_impl:
if self.config['lithops']['mode'] == 'localhost':
self.lithops_vm_executor = self.lithops_executor
else:
self.lithops_vm_executor = lithops.StandaloneExecutor(config=self.config)
self.storage = Storage(config=self.config)
cache_namespace = 'vm' if hybrid_impl else 'function'
self.cacher = PipelineCacher(
self.storage, lithops_bucket, cache_namespace, self.ds_config["name"], self.db_config["name"]
)
if not self.use_db_cache or not self.use_ds_cache:
self.cacher.clean(database=not self.use_db_cache, dataset=not self.use_ds_cache)
stats_path_cache_key = ':ds/:db/stats_path.cache'
if self.cacher.exists(stats_path_cache_key):
self.stats_path = self.cacher.load(stats_path_cache_key)
PipelineStats.path = self.stats_path
logger.info(f'Using cached {self.stats_path} for statistics')
else:
PipelineStats.init()
self.stats_path = PipelineStats.path
self.cacher.save(self.stats_path, stats_path_cache_key)
logger.info(f'Initialised {self.stats_path} for statistics')
self.ds_segm_size_mb = 128
self.image_gen_config = {
"q": 99,
"do_preprocessing": False,
"nlevels": 30,
"ppm": 3.0
}
def test_map_reduce_obj_key(self):
logger.info('Testing map_reduce() over object keys')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [sb + '://' + bucket_name + '/' + key for key in list_dataset_keys(STORAGE, STORAGE_CONFIG)]
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(my_map_function_obj, iterdata,
my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.words_in_cos_files)
def test_map_reduce_obj_bucket_one_reducer_per_object(self):
logger.info('Testing map_reduce() over a bucket with one reducer per object')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function,
reducer_one_per_object=True)
result = fexec.get_result()
self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
def test_map_reduce_obj_key(self):
logger.info('Testing map_reduce() over object keys')
sb = STORAGE_CONFIG['backend']
bucket_name = STORAGE_CONFIG['bucket']
iterdata = [sb + '://' + bucket_name + '/' + key for key in TestUtils.list_dataset_keys()]
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(TestMethods.my_map_function_obj, iterdata,
TestMethods.my_reduce_function)
result = fexec.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_chunks_bucket(self):
logger.info('Testing chunks on a bucket')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
fexec = lithops.FunctionExecutor(config=CONFIG)
futures = fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function,
obj_chunk_size=1 * 1024 ** 2)
result = fexec.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 8)
fexec = lithops.FunctionExecutor(config=CONFIG)
futures = fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix,
TestMethods.my_reduce_function, obj_chunk_number=2)
result = fexec.get_result(futures)
self.assertEqual(result, self.__class__.cos_result_to_compare)
self.assertEqual(len(futures), 11)
def test_internal_executions(self):
logger.info('Testing internal executions')
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map(TestMethods.lithops_inside_lithops_map_function, range(1, 5))
result = fexec.get_result()
self.assertEqual(result, [list(range(i)) for i in range(1, 5)])
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.lithops_return_futures_map_function1, 3)
fexec.get_result()
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.lithops_return_futures_map_function2, 3)
fexec.get_result()
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.lithops_return_futures_map_function3, 3)
fexec.wait()
fexec.get_result()
def test_map_reduce_obj_bucket_one_reducer_per_object(self):
logger.info('Testing map_reduce() over a bucket with one reducer per object')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + DATASET_PREFIX + '/'
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.map_reduce(my_map_function_obj, data_prefix,
my_reduce_function,
reducer_one_per_object=True)
result = fexec.get_result()
# the reducer returns a list containing sum of the words uploaded via each file.
self.assertEqual(sum(result), self.__class__.words_in_cos_files)
def test_cloudobject(self):
logger.info('Testing cloudobjects')
sb = STORAGE_CONFIG['backend']
data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/'
with lithops.FunctionExecutor(config=CONFIG) as fexec:
fexec.map(TestMethods.my_cloudobject_put, data_prefix)
cloudobjects = fexec.get_result()
fexec.call_async(TestMethods.my_cloudobject_get, cloudobjects)
result = fexec.get_result()
self.assertEqual(result, self.__class__.cos_result_to_compare)
fexec.clean(cs=cloudobjects)
def test_single_shard_matrix(self):
# pwex = pywren.default_executor()
pwex = lithops.FunctionExecutor()
X = np.random.randn(128,128)
X_sharded = local_numpy_init(X, (128,128))
X_sharded_down = matrix_init.reshard_down(X_sharded, (4,4), pwex)
X_sharded_local = X_sharded.numpy()
X_sharded_local_2 = X_sharded_down.numpy()
X_sharded.free()
X_sharded_down.free()
assert(np.all(X_sharded_local == X))
assert(np.all(X_sharded_local_2 == X))
def run_staging_trx(*args):
# print("hello")
# config = Variable.get("lithops_config", deserialize_json=True)
fexec = lithops.FunctionExecutor(config=args[0])
pgpassword = args[1]
# fexec.call_async(working,args[1])
# fexec.map(working,[[10,11,12,13,14,15,16,17,18,19,20])
fexec.map(working, [[10, pgpassword], [11, pgpassword], [12, pgpassword],
[13, pgpassword], [14, pgpassword], [15, pgpassword],
[16, pgpassword], [17, pgpassword], [18, pgpassword],
[19, pgpassword], [20, pgpassword]])
print(fexec.get_result())
def test_call_async(self):
print('Testing call_async()...')
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.hello_world, "")
result = fexec.get_result()
self.assertEqual(result, "Hello World!")
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.concat, ["a", "b"])
result = fexec.get_result()
self.assertEqual(result, "a b")
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.simple_map_function, [4, 6])
result = fexec.get_result()
self.assertEqual(result, 10)
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8})
result = fexec.get_result()
self.assertEqual(result, 10)
def test_call_async(self):
def hello_world(param):
return "Hello World!"
logger.info('Testing call_async()')
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(hello_world, "")
result = fexec.get_result()
self.assertEqual(result, "Hello World!")
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(lambda x: " ".join(x), ["a", "b"])
result = fexec.get_result()
self.assertEqual(result, "a b")
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(simple_map_function, (4, 6))
result = fexec.get_result()
self.assertEqual(result, 10)
fexec = lithops.FunctionExecutor(config=CONFIG)
fexec.call_async(simple_map_function, {'x': 2, 'y': 8})
result = fexec.get_result()
self.assertEqual(result, 10)
