def _save_modules(self, module_data): """ Save modules, before we unpickle actual function """ if module_data: logger.debug("Writing Function dependencies to local disk") module_path = os.path.join(PYTHON_MODULE_PATH, self.executor_id, self.job_id, self.call_id) # shutil.rmtree(PYTHON_MODULE_PATH, True) # delete old modules os.makedirs(module_path, exist_ok=True) sys.path.append(module_path) for m_filename, m_data in module_data.items(): m_path = os.path.dirname(m_filename) if len(m_path) > 0 and m_path[0] == "/": m_path = m_path[1:] to_make = os.path.join(module_path, m_path) try: os.makedirs(to_make) except OSError as e: if e.errno == 17: pass else: raise e full_filename = os.path.join(to_make, os.path.basename(m_filename)) with open(full_filename, 'wb') as fid: fid.write(b64str_to_bytes(m_data)) logger.debug("Finished writing Function dependencies")
def _store_func_and_modules(func_key, func_str, module_data): ''' stores function and modules in temporary directory to be used later in optimized runtime ''' # save function func_path = '/'.join([LITHOPS_TEMP_DIR, func_key]) os.makedirs(os.path.dirname(func_path), exist_ok=True) with open(func_path, "wb") as f: f.write(func_str) if module_data: logger.debug("Writing Function dependencies to local disk") modules_path = '/'.join([os.path.dirname(func_path), 'modules']) for m_filename, m_data in module_data.items(): m_path = os.path.dirname(m_filename) if len(m_path) > 0 and m_path[0] == "/": m_path = m_path[1:] to_make = os.path.join(modules_path, m_path) try: os.makedirs(to_make) except OSError as e: if e.errno == 17: pass else: raise e full_filename = os.path.join(to_make, os.path.basename(m_filename)) with open(full_filename, 'wb') as fid: fid.write(b64str_to_bytes(m_data)) logger.debug("Finished storing function and modules")
def get_function_and_modules(job, internal_storage): """ Gets the function and modules from storage """ logger.debug("Getting function and modules") mode = job.config['lithops']['mode'] customized_runtime = job.config[mode].get('customized_runtime', False) if customized_runtime: logger.debug("Customized runtime feature activated. Loading " "function and modules from local runtime") func_path = '/'.join([LITHOPS_TEMP_DIR, job.func_key]) with open(func_path, "rb") as f: func_obj = f.read() else: func_obj = internal_storage.get_func(job.func_key) loaded_func_all = pickle.loads(func_obj) if loaded_func_all.get('module_data'): module_path = os.path.join(MODULES_DIR, job.job_key) logger.debug("Writing function dependencies to {}".format(module_path)) os.makedirs(module_path, exist_ok=True) sys.path.append(module_path) for m_filename, m_data in loaded_func_all['module_data'].items(): m_path = os.path.dirname(m_filename) if len(m_path) > 0 and m_path[0] == "/": m_path = m_path[1:] to_make = os.path.join(module_path, m_path) try: os.makedirs(to_make) except OSError as e: if e.errno == 17: pass else: raise e full_filename = os.path.join(to_make, os.path.basename(m_filename)) # logger.debug('Writing {}'.format(full_filename)) with open(full_filename, 'wb') as fid: fid.write(b64str_to_bytes(m_data)) return loaded_func_all['func']