def _save_modules(self, module_data):
"""
Save modules, before we unpickle actual function
"""
if module_data:
logger.debug("Writing Function dependencies to local disk")
module_path = os.path.join(PYTHON_MODULE_PATH, self.executor_id,
self.job_id, self.call_id)
# shutil.rmtree(PYTHON_MODULE_PATH, True) # delete old modules
os.makedirs(module_path, exist_ok=True)
sys.path.append(module_path)
for m_filename, m_data in module_data.items():
m_path = os.path.dirname(m_filename)
if len(m_path) > 0 and m_path[0] == "/":
m_path = m_path[1:]
to_make = os.path.join(module_path, m_path)
try:
os.makedirs(to_make)
except OSError as e:
if e.errno == 17:
pass
else:
raise e
full_filename = os.path.join(to_make,
os.path.basename(m_filename))
with open(full_filename, 'wb') as fid:
fid.write(b64str_to_bytes(m_data))
logger.debug("Finished writing Function dependencies")
def _store_func_and_modules(func_key, func_str, module_data):
''' stores function and modules in temporary directory to be
used later in optimized runtime
'''
# save function
func_path = '/'.join([LITHOPS_TEMP_DIR, func_key])
os.makedirs(os.path.dirname(func_path), exist_ok=True)
with open(func_path, "wb") as f:
f.write(func_str)
if module_data:
logger.debug("Writing Function dependencies to local disk")
modules_path = '/'.join([os.path.dirname(func_path), 'modules'])
for m_filename, m_data in module_data.items():
m_path = os.path.dirname(m_filename)
if len(m_path) > 0 and m_path[0] == "/":
m_path = m_path[1:]
to_make = os.path.join(modules_path, m_path)
try:
os.makedirs(to_make)
except OSError as e:
if e.errno == 17:
pass
else:
raise e
full_filename = os.path.join(to_make, os.path.basename(m_filename))
with open(full_filename, 'wb') as fid:
fid.write(b64str_to_bytes(m_data))
logger.debug("Finished storing function and modules")
def get_function_and_modules(job, internal_storage):
"""
Gets the function and modules from storage
"""
logger.debug("Getting function and modules")
mode = job.config['lithops']['mode']
customized_runtime = job.config[mode].get('customized_runtime', False)
if customized_runtime:
logger.debug("Customized runtime feature activated. Loading "
"function and modules from local runtime")
func_path = '/'.join([LITHOPS_TEMP_DIR, job.func_key])
with open(func_path, "rb") as f:
func_obj = f.read()
else:
func_obj = internal_storage.get_func(job.func_key)
loaded_func_all = pickle.loads(func_obj)
if loaded_func_all.get('module_data'):
module_path = os.path.join(MODULES_DIR, job.job_key)
logger.debug("Writing function dependencies to {}".format(module_path))
os.makedirs(module_path, exist_ok=True)
sys.path.append(module_path)
for m_filename, m_data in loaded_func_all['module_data'].items():
m_path = os.path.dirname(m_filename)
if len(m_path) > 0 and m_path[0] == "/":
m_path = m_path[1:]
to_make = os.path.join(module_path, m_path)
try:
os.makedirs(to_make)
except OSError as e:
if e.errno == 17:
pass
else:
raise e
full_filename = os.path.join(to_make, os.path.basename(m_filename))
# logger.debug('Writing {}'.format(full_filename))
with open(full_filename, 'wb') as fid:
fid.write(b64str_to_bytes(m_data))
return loaded_func_all['func']