def benchmark(workers, memory, loopcount, matn): iterable = [(loopcount, matn) for i in range(workers)] fexec = FunctionExecutor(runtime_memory=memory) start_time = time.time() worker_futures = fexec.map(compute_flops, iterable) results = fexec.get_result() end_time = time.time() worker_stats = [f.stats for f in worker_futures] total_time = end_time - start_time print("Total time:", round(total_time, 3)) est_flops = workers * 2 * loopcount * matn**3 print('Estimated GFLOPS:', round(est_flops / 1e9 / total_time, 4)) res = { 'start_time': start_time, 'total_time': total_time, 'est_flops': est_flops, 'worker_stats': worker_stats, 'results': results } return res
def read(backend, bucket_name, number, keylist_raw, read_times): blocksize = 1024 * 1024 def read_object(key_name, storage): m = hashlib.md5() bytes_read = 0 print(key_name) start_time = time.time() for unused in range(read_times): fileobj = storage.get_object(bucket_name, key_name, stream=True) try: buf = fileobj.read(blocksize) while len(buf) > 0: bytes_read += len(buf) #if bytes_read % (blocksize *10) == 0: # mb_rate = bytes_read/(time.time()-t1)/1e6 # print('POS:'+str(bytes_read)+' MB Rate: '+ str(mb_rate)) m.update(buf) buf = fileobj.read(blocksize) except Exception as e: print(e) pass end_time = time.time() mb_rate = bytes_read / (end_time - start_time) / 1e6 print('MB Rate: ' + str(mb_rate)) return { 'start_time': start_time, 'end_time': end_time, 'mb_rate': mb_rate, 'bytes_read': bytes_read } if number == 0: keynames = keylist_raw else: keynames = [keylist_raw[i % len(keylist_raw)] for i in range(number)] fexec = FunctionExecutor(backend=backend, runtime_memory=1024) start_time = time.time() worker_futures = fexec.map(read_object, keynames) results = fexec.get_result() end_time = time.time() total_time = end_time - start_time worker_stats = [f.stats for f in worker_futures] res = { 'start_time': start_time, 'total_time': total_time, 'worker_stats': worker_stats, 'results': results } return res
class FuturesList(list): def _create_executor(self): if not self.executor: from lithops import FunctionExecutor self.executor = FunctionExecutor(config=self.config) def _extend_futures(self, fs): for fut in self: fut._produce_output = False if not hasattr(self, 'alt_list'): self.alt_list = [] self.alt_list.extend(self) self.alt_list.extend(fs) self.clear() self.extend(fs) def map(self, map_function, sync=False, **kwargs): self._create_executor() if sync: self.executor.wait(self) fs = self.executor.map(map_function, self, **kwargs) self._extend_futures(fs) return self def map_reduce(self, map_function, reduce_function, sync=False, **kwargs): self._create_executor() if sync: self.executor.wait(self) fs = self.executor.map_reduce(map_function, self, reduce_function, **kwargs) self._extend_futures(fs) return self def wait(self, **kwargs): self._create_executor() fs_tt = self.alt_list if hasattr(self, 'alt_list') else self return self.executor.wait(fs_tt, **kwargs) def get_result(self, **kwargs): self._create_executor() fs_tt = self.alt_list if hasattr(self, 'alt_list') else self return self.executor.get_result(fs_tt, **kwargs) def __reduce__(self): self.executor = None return super().__reduce__()
def write(backend, bucket_name, mb_per_file, number, key_prefix): def write_object(key_name, storage): bytes_n = mb_per_file * 1024**2 d = RandomDataGenerator(bytes_n) print(key_name) start_time = time.time() storage.put_object(bucket_name, key_name, d) end_time = time.time() mb_rate = bytes_n / (end_time - start_time) / 1e6 print('MB Rate: ' + str(mb_rate)) return { 'start_time': start_time, 'end_time': end_time, 'mb_rate': mb_rate } # create list of random keys keynames = [ key_prefix + str(uuid.uuid4().hex.upper()) for unused in range(number) ] fexec = FunctionExecutor(backend=backend, runtime_memory=1024) start_time = time.time() worker_futures = fexec.map(write_object, keynames) results = fexec.get_result() end_time = time.time() worker_stats = [f.stats for f in worker_futures] total_time = end_time - start_time res = { 'start_time': start_time, 'total_time': total_time, 'worker_stats': worker_stats, 'bucket_name': bucket_name, 'keynames': keynames, 'results': results } return res
BUCKET_NAME = 'lithops-sample-data' # change-me def my_function(obj_id, storage): print(obj_id) data = storage.get_cloudobject(obj_id) return data.decode() if __name__ == '__main__': obj_key = 'cloudobject1.txt' storage = Storage() obj_id = storage.put_cloudobject('Hello World!', BUCKET_NAME, obj_key) print(obj_id) fexec = FunctionExecutor() fexec.call_async(my_function, obj_id) print(fexec.get_result()) obj_key = 'cloudobject2.txt' storage = fexec.storage obj_id = storage.put_cloudobject('Hello World!', BUCKET_NAME, obj_key) print(obj_id) fexec.call_async(my_function, obj_id) print(fexec.get_result())