def test_chunks_bucket_one_reducer_per_object(self): """tests the ability to create a separate function invocation based on the following parameters, as well as create a separate invocation of a reduce function for each object: chunk_size - creates [file_size//chunk_size] invocations to process each chunk_size bytes, of a given object. chunk_number - creates 'chunk_number' invocations that process [file_size//chunk_number] bytes each. """ logger.info('Testing chunks on a bucket with one reducer per object') OBJ_CHUNK_SIZE = 1 * 1024 ** 2 OBJ_CHUNK_NUMBER = 2 activations = 0 sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + DATASET_PREFIX + '/' fexec = lithops.FunctionExecutor(config=CONFIG) futures = fexec.map_reduce(my_map_function_obj, data_prefix, my_reduce_function, obj_chunk_size=OBJ_CHUNK_SIZE, reducer_one_per_object=True) result = fexec.get_result(futures) self.assertEqual(sum(result), self.__class__.words_in_cos_files) for size in get_dataset_key_size(STORAGE, STORAGE_CONFIG): activations += math.ceil(size / OBJ_CHUNK_SIZE) self.assertEqual(len(futures), activations + len(TEST_FILES_URLS)) # + len(TEST_FILES_URLS) due to map_reduce activation per object fexec = lithops.FunctionExecutor(config=CONFIG) futures = fexec.map_reduce(my_map_function_obj, data_prefix, my_reduce_function, obj_chunk_number=OBJ_CHUNK_NUMBER, reducer_one_per_object=True) result = fexec.get_result(futures) self.assertEqual(sum(result), self.__class__.words_in_cos_files) self.assertEqual(len(futures), len(TEST_FILES_URLS)*OBJ_CHUNK_NUMBER + len(TEST_FILES_URLS)) # + len(TEST_FILES_URLS) due to map_reduce activation per object
def test_cholesky_lambda(): X = np.random.randn(128, 128) A = X.dot(X.T) + np.eye(X.shape[0]) shard_size = 128 shard_sizes = (shard_size, shard_size) A_sharded = BigMatrix("job_runner_test", shape=A.shape, shard_sizes=shard_sizes, write_header=True) A_sharded.free() shard_matrix(A_sharded, A) program, meta = cholesky(A_sharded) executor = fs.ProcessPoolExecutor(1) print("starting program") program.start() pwex = lithops.FunctionExecutor() futures = pwex.map( lambda x: job_runner.lambdapack_run( program, timeout=60, idle_timeout=6), range(16)) pwex.wait(futures) print("RESULTSSS") print([f.result() for f in futures]) futures = pwex.map( lambda x: job_runner.lambdapack_run( program, timeout=60, idle_timeout=6), range(16)) program.wait() #program.free() L_sharded = meta["outputs"][0] L_npw = L_sharded.numpy() L = np.linalg.cholesky(A) assert (np.allclose(L_npw, L)) print("great success!")
def test_function(config, backend, storage, debug): if config: config = load_yaml_config(config) log_level = logging.INFO if not debug else logging.DEBUG setup_lithops_logger(log_level) try: import getpass username = getpass.getuser() except Exception: username = '******' def hello(name): return 'Hello {}!'.format(name) fexec = lithops.FunctionExecutor(config=config, backend=backend, storage=storage) fexec.call_async(hello, username) result = fexec.get_result() print() if result == 'Hello {}!'.format(username): print(result, 'Lithops is working as expected :)') else: print(result, 'Something went wrong :(') print()
def test_map_reduce_url(self): print('Testing map_reduce() over URLs...') fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(TestMethods.my_map_function_url, TEST_FILES_URLS, TestMethods.my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def lithops_inside_lithops_map_function(x): def _func(x): return x fexec = lithops.FunctionExecutor() fexec.map(_func, range(x)) return fexec.get_result()
def test_map_reduce_url(self): logger.info('Testing map_reduce() over URLs') fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(my_map_function_url, TEST_FILES_URLS, my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.words_in_cos_files)
def trisolve(pwex, A, B, out_bucket=None, tasks_per_job=1, lower=False): if out_bucket is None: out_bucket = A.bucket root_key = generate_key_name_binop(A, B, "trisolve") instructions, X, scratch = lp._trisolve(A, B, out_bucket=out_bucket, lower=lower) config = pwex.config # if (isinstance(pwex.invoker, pywren.queues.SQSInvoker)): # executor = pywren.standalone_executor # else: fexec = lithops.FunctionExecutor() program = lp.LambdaPackProgram(instructions, executor=fexec, pywren_config=config) print(program) #assert False program.start() job_runner.lambdapack_run(program) program.wait() if program.program_status() != lp.PS.SUCCESS: program.unwind() raise Exception("Lambdapack Exception : {0}".format( program.program_status())) program.free() # delete all intermediate information [M.free() for M in scratch] return X
def run_program_in_pywren(program, num_workers=32): def pywren_run(_): job_runner.lambdapack_run(program, timeout=60, idle_timeout=6) pwex = lithops.FunctionExecutor() futures = pwex.map(pywren_run, range(num_workers)) return futures
def test_multiple_shard_matrix_multiply(self): fexec = lithops.FunctionExecutor(runtime='jsampe/numpy-lithops:04', log_level='DEBUG') X = np.random.randn(16, 16) X_shard_sizes = tuple(map(int, np.array(X.shape) / 2)) X_sharded = BigMatrix("gemm_test_1", shape=X.shape, shard_sizes=X_shard_sizes, storage=fexec.storage) Y = np.random.randn(16, 16) Y_shard_sizes = tuple(map(int, np.array(Y.shape) / 2)) Y_sharded = BigMatrix("gemm_test_2", shape=Y.shape, shard_sizes=Y_shard_sizes, storage=fexec.storage) shard_matrix(X_sharded, X) shard_matrix(Y_sharded, Y) XY_sharded = binops.gemm(fexec, X_sharded, Y_sharded, X_sharded.bucket, 1) XY_sharded_local = XY_sharded.numpy() XY = X.dot(Y) X_sharded.free() Y_sharded.free() XY_sharded.free() assert (np.all(np.isclose(XY, XY_sharded_local))) os.system("rm -rf /dev/shm/*")
def test_storage_handler(self): logger.info('Testing "storage" function arg') iterdata = [(key, STORAGE_CONFIG['bucket']) for key in TestUtils.list_dataset_keys()] fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(TestMethods.my_map_function_storage, iterdata, TestMethods.my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def lithops_return_futures_map_function3(x): def _func(x): return x + 1 fexec = lithops.FunctionExecutor() fut1 = fexec.map(_func, range(x)) fut2 = fexec.map(_func, range(x)) return fut1 + fut2
def test_map_reduce(self): print('Testing map_reduce()...') iterdata = [[1, 1], [2, 2], [3, 3], [4, 4]] fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(TestMethods.simple_map_function, iterdata, TestMethods.simple_reduce_function) result = fexec.get_result() self.assertEqual(result, 20)
def test_storage_handler(self): logger.info('Testing "storage" function arg') iterdata = [(key, STORAGE_CONFIG['bucket']) for key in list_dataset_keys(STORAGE, STORAGE_CONFIG)] fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(my_map_function_storage, iterdata, my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.words_in_cos_files)
def run_staging_payment(*args): fexec = lithops.FunctionExecutor(config=args[0]) # fexec.call_async(working,args[1]) pgpassword = args[1] fexec.map(working, [[1, pgpassword], [2, pgpassword], [3, pgpassword], [4, pgpassword], [5, pgpassword], [6, pgpassword], [7, pgpassword], [8, pgpassword], [20, pgpassword]]) print(fexec.get_result())
def test_map_reduce(self): logger.info('Testing map_reduce() using memory') iterdata = [(1, 1), (2, 2), (3, 3), (4, 4)] fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(simple_map_function, iterdata, simple_reduce_function) result = fexec.get_result() self.assertEqual(result, 20)
def test_map_reduce_obj_bucket(self): print('Testing map_reduce() over a bucket...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_map_reduce_obj_bucket(self): logger.info('Testing map_reduce() over a bucket') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + DATASET_PREFIX + '/' fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(my_map_function_obj, data_prefix, my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.words_in_cos_files)
def test_cloudobject(self): print('Testing cloudobjects...') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' with lithops.FunctionExecutor(config=CONFIG) as fexec: fexec.map_reduce(TestMethods.my_cloudobject_put, data_prefix, TestMethods.my_cloudobject_get) result = fexec.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def __init__(self, ds_config, db_config, use_db_cache=True, use_ds_cache=True, hybrid_impl='auto'): self.config = default_config() self.ds_config = ds_config self.db_config = db_config self.use_db_cache = use_db_cache self.use_ds_cache = use_ds_cache if hybrid_impl == 'auto': self.hybrid_impl = ( self.config['lithops']['mode'] == 'localhost' or self.config['lithops']['mode'] == 'serverless' and 'ibm_vpc' in self.config ) if self.hybrid_impl: logger.info(f'Using the Hybrid implementation') else: logger.info(f'Using the pure Serverless implementation') else: self.hybrid_impl = hybrid_impl lithops_bucket = self.config['lithops']['storage_bucket'] self.ds_bucket = self.config.get('storage', {}).get('ds_bucket', lithops_bucket) self.lithops_executor = lithops.FunctionExecutor(config=self.config, runtime_memory=2048) if self.hybrid_impl: if self.config['lithops']['mode'] == 'localhost': self.lithops_vm_executor = self.lithops_executor else: self.lithops_vm_executor = lithops.StandaloneExecutor(config=self.config) self.storage = Storage(config=self.config) cache_namespace = 'vm' if hybrid_impl else 'function' self.cacher = PipelineCacher( self.storage, lithops_bucket, cache_namespace, self.ds_config["name"], self.db_config["name"] ) if not self.use_db_cache or not self.use_ds_cache: self.cacher.clean(database=not self.use_db_cache, dataset=not self.use_ds_cache) stats_path_cache_key = ':ds/:db/stats_path.cache' if self.cacher.exists(stats_path_cache_key): self.stats_path = self.cacher.load(stats_path_cache_key) PipelineStats.path = self.stats_path logger.info(f'Using cached {self.stats_path} for statistics') else: PipelineStats.init() self.stats_path = PipelineStats.path self.cacher.save(self.stats_path, stats_path_cache_key) logger.info(f'Initialised {self.stats_path} for statistics') self.ds_segm_size_mb = 128 self.image_gen_config = { "q": 99, "do_preprocessing": False, "nlevels": 30, "ppm": 3.0 }
def test_map_reduce_obj_key(self): logger.info('Testing map_reduce() over object keys') sb = STORAGE_CONFIG['backend'] bucket_name = STORAGE_CONFIG['bucket'] iterdata = [sb + '://' + bucket_name + '/' + key for key in list_dataset_keys(STORAGE, STORAGE_CONFIG)] fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(my_map_function_obj, iterdata, my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.words_in_cos_files)
def test_map_reduce_obj_bucket_one_reducer_per_object(self): logger.info('Testing map_reduce() over a bucket with one reducer per object') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, reducer_one_per_object=True) result = fexec.get_result() self.assertEqual(sum(result), self.__class__.cos_result_to_compare)
def test_map_reduce_obj_key(self): logger.info('Testing map_reduce() over object keys') sb = STORAGE_CONFIG['backend'] bucket_name = STORAGE_CONFIG['bucket'] iterdata = [sb + '://' + bucket_name + '/' + key for key in TestUtils.list_dataset_keys()] fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(TestMethods.my_map_function_obj, iterdata, TestMethods.my_reduce_function) result = fexec.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare)
def test_chunks_bucket(self): logger.info('Testing chunks on a bucket') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' fexec = lithops.FunctionExecutor(config=CONFIG) futures = fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, obj_chunk_size=1 * 1024 ** 2) result = fexec.get_result(futures) self.assertEqual(result, self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 8) fexec = lithops.FunctionExecutor(config=CONFIG) futures = fexec.map_reduce(TestMethods.my_map_function_obj, data_prefix, TestMethods.my_reduce_function, obj_chunk_number=2) result = fexec.get_result(futures) self.assertEqual(result, self.__class__.cos_result_to_compare) self.assertEqual(len(futures), 11)
def test_internal_executions(self): logger.info('Testing internal executions') fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map(TestMethods.lithops_inside_lithops_map_function, range(1, 5)) result = fexec.get_result() self.assertEqual(result, [list(range(i)) for i in range(1, 5)]) fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.lithops_return_futures_map_function1, 3) fexec.get_result() fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.lithops_return_futures_map_function2, 3) fexec.get_result() fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.lithops_return_futures_map_function3, 3) fexec.wait() fexec.get_result()
def test_map_reduce_obj_bucket_one_reducer_per_object(self): logger.info('Testing map_reduce() over a bucket with one reducer per object') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + DATASET_PREFIX + '/' fexec = lithops.FunctionExecutor(config=CONFIG) fexec.map_reduce(my_map_function_obj, data_prefix, my_reduce_function, reducer_one_per_object=True) result = fexec.get_result() # the reducer returns a list containing sum of the words uploaded via each file. self.assertEqual(sum(result), self.__class__.words_in_cos_files)
def test_cloudobject(self): logger.info('Testing cloudobjects') sb = STORAGE_CONFIG['backend'] data_prefix = sb + '://' + STORAGE_CONFIG['bucket'] + '/' + PREFIX + '/' with lithops.FunctionExecutor(config=CONFIG) as fexec: fexec.map(TestMethods.my_cloudobject_put, data_prefix) cloudobjects = fexec.get_result() fexec.call_async(TestMethods.my_cloudobject_get, cloudobjects) result = fexec.get_result() self.assertEqual(result, self.__class__.cos_result_to_compare) fexec.clean(cs=cloudobjects)
def test_single_shard_matrix(self): # pwex = pywren.default_executor() pwex = lithops.FunctionExecutor() X = np.random.randn(128,128) X_sharded = local_numpy_init(X, (128,128)) X_sharded_down = matrix_init.reshard_down(X_sharded, (4,4), pwex) X_sharded_local = X_sharded.numpy() X_sharded_local_2 = X_sharded_down.numpy() X_sharded.free() X_sharded_down.free() assert(np.all(X_sharded_local == X)) assert(np.all(X_sharded_local_2 == X))
def run_staging_trx(*args): # print("hello") # config = Variable.get("lithops_config", deserialize_json=True) fexec = lithops.FunctionExecutor(config=args[0]) pgpassword = args[1] # fexec.call_async(working,args[1]) # fexec.map(working,[[10,11,12,13,14,15,16,17,18,19,20]) fexec.map(working, [[10, pgpassword], [11, pgpassword], [12, pgpassword], [13, pgpassword], [14, pgpassword], [15, pgpassword], [16, pgpassword], [17, pgpassword], [18, pgpassword], [19, pgpassword], [20, pgpassword]]) print(fexec.get_result())
def test_call_async(self): print('Testing call_async()...') fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.hello_world, "") result = fexec.get_result() self.assertEqual(result, "Hello World!") fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.concat, ["a", "b"]) result = fexec.get_result() self.assertEqual(result, "a b") fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.simple_map_function, [4, 6]) result = fexec.get_result() self.assertEqual(result, 10) fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(TestMethods.simple_map_function, {'x': 2, 'y': 8}) result = fexec.get_result() self.assertEqual(result, 10)
def test_call_async(self): def hello_world(param): return "Hello World!" logger.info('Testing call_async()') fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(hello_world, "") result = fexec.get_result() self.assertEqual(result, "Hello World!") fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(lambda x: " ".join(x), ["a", "b"]) result = fexec.get_result() self.assertEqual(result, "a b") fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(simple_map_function, (4, 6)) result = fexec.get_result() self.assertEqual(result, 10) fexec = lithops.FunctionExecutor(config=CONFIG) fexec.call_async(simple_map_function, {'x': 2, 'y': 8}) result = fexec.get_result() self.assertEqual(result, 10)