def run_job(job):
"""
Runs a single job within a separate process
"""
start_tstamp = time.time()
setup_lithops_logger(job.log_level)
logger.info("Lithops v{} - Starting execution".format(__version__))
logger.info("Execution ID: {}/{}".format(job.job_key, job.call_id))
logger.debug("Runtime name: {}".format(job.runtime_name))
if job.runtime_memory:
logger.debug("Runtime memory: {}MB".format(job.runtime_memory))
logger.debug("Function timeout: {}s".format(job.execution_timeout))
env = job.extra_env
env['LITHOPS_WORKER'] = 'True'
env['PYTHONUNBUFFERED'] = 'True'
env['LITHOPS_CONFIG'] = json.dumps(job.config)
env['PYTHONPATH'] = "{}:{}".format(os.getcwd(), LITHOPS_LIBS_PATH)
env['__LITHOPS_SESSION_ID'] = '-'.join([job.job_key, job.call_id])
os.environ.update(env)
storage_config = extract_storage_config(job.config)
internal_storage = InternalStorage(storage_config)
call_status = CallStatus(job.config, internal_storage)
call_status.response['worker_start_tstamp'] = start_tstamp
call_status.response['host_submit_tstamp'] = job.host_submit_tstamp
call_status.response['call_id'] = job.call_id
call_status.response['job_id'] = job.job_id
call_status.response['executor_id'] = job.executor_id
show_memory_peak = strtobool(os.environ.get('SHOW_MEMORY_PEAK', 'False'))
try:
if __version__ != job.lithops_version:
msg = (
"Lithops version mismatch. Host version: {} - Runtime version: {}"
.format(job.lithops_version, __version__))
raise RuntimeError('HANDLER', msg)
# send init status event
call_status.send('__init__')
if show_memory_peak:
mm_handler_conn, mm_conn = Pipe()
memory_monitor = Thread(target=memory_monitor_worker,
args=(mm_conn, ))
memory_monitor.start()
job.jr_stats_file = os.path.join(job.job_dir, 'jobrunner.stats.txt')
handler_conn, jobrunner_conn = Pipe()
jobrunner = JobRunner(job, jobrunner_conn, internal_storage)
logger.debug('Starting JobRunner process')
jrp = Process(target=jobrunner.run) if is_unix_system() else Thread(
target=jobrunner.run)
jrp.start()
jrp.join(job.execution_timeout)
logger.debug('JobRunner process finished')
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Function exceeded maximum time of {} seconds and was '
'killed'.format(job.execution_timeout))
raise TimeoutError('HANDLER', msg)
if show_memory_peak:
mm_handler_conn.send('STOP')
memory_monitor.join()
peak_memory_usage = int(mm_handler_conn.recv())
logger.info("Peak memory usage: {}".format(
sizeof_fmt(peak_memory_usage)))
call_status.response['peak_memory_usage'] = peak_memory_usage
if not handler_conn.poll():
logger.error(
'No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Function exceeded maximum memory and was killed'
raise MemoryError('HANDLER', msg)
if os.path.exists(job.jr_stats_file):
with open(job.jr_stats_file, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.response[key] = float(value)
except Exception:
call_status.response[key] = value
if key in [
'exception', 'exc_pickle_fail', 'result',
'new_futures'
]:
call_status.response[key] = eval(value)
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------')
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------')
call_status.response['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
call_status.response['exc_info'] = str(pickled_exc)
finally:
call_status.response['worker_end_tstamp'] = time.time()
with open(job.log_file, 'rb') as lf:
log_str = base64.b64encode(zlib.compress(lf.read())).decode()
call_status.response['logs'] = log_str
call_status.send('__end__')
# Unset specific env vars
for key in job.extra_env:
os.environ.pop(key, None)
os.environ.pop('__LITHOPS_TOTAL_EXECUTORS', None)
logger.info("Finished")
def function_handler(event):
start_tstamp = time.time()
log_level = event['log_level']
cloud_logging_config(log_level)
logger.debug("Action handler started")
extra_env = event.get('extra_env', {})
os.environ.update(extra_env)
os.environ.update({'LITHOPS_FUNCTION': 'True', 'PYTHONUNBUFFERED': 'True'})
os.environ.pop('LITHOPS_TOTAL_EXECUTORS', None)
config = event['config']
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
exec_id = "{}/{}/{}".format(executor_id, job_id, call_id)
logger.info("Execution-ID: {}".format(exec_id))
runtime_name = event['runtime_name']
runtime_memory = event['runtime_memory']
execution_timeout = event['execution_timeout']
logger.debug("Runtime name: {}".format(runtime_name))
logger.debug("Runtime memory: {}MB".format(runtime_memory))
logger.debug("Function timeout: {}s".format(execution_timeout))
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
call_status = CallStatus(config, internal_storage)
call_status.response['host_submit_tstamp'] = event['host_submit_tstamp']
call_status.response['worker_start_tstamp'] = start_tstamp
context_dict = {
'python_version': os.environ.get("PYTHON_VERSION"),
'call_id': call_id,
'job_id': job_id,
'executor_id': executor_id,
'activation_id': os.environ.get('__PW_ACTIVATION_ID')
}
call_status.response.update(context_dict)
show_memory_peak = strtobool(os.environ.get('SHOW_MEMORY_PEAK', 'False'))
try:
if version.__version__ != event['lithops_version']:
msg = (
"Lithops version mismatch. Host version: {} - Runtime version: {}"
.format(event['lithops_version'], version.__version__))
raise RuntimeError('HANDLER', msg)
# send init status event
call_status.send('__init__')
# call_status.response['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'LITHOPS_CONFIG': json.dumps(config),
'LITHOPS_EXECUTION_ID': exec_id,
'PYTHONPATH': "{}:{}".format(os.getcwd(), LITHOPS_LIBS_PATH)
}
os.environ.update(custom_env)
jobrunner_stats_dir = os.path.join(STORAGE_FOLDER,
storage_config['bucket'],
JOBS_PREFIX, executor_id, job_id,
call_id)
os.makedirs(jobrunner_stats_dir, exist_ok=True)
jobrunner_stats_filename = os.path.join(jobrunner_stats_dir,
'jobrunner.stats.txt')
jobrunner_config = {
'lithops_config':
config,
'call_id':
call_id,
'job_id':
job_id,
'executor_id':
executor_id,
'func_key':
func_key,
'data_key':
data_key,
'log_level':
log_level,
'data_byte_range':
data_byte_range,
'output_key':
create_output_key(JOBS_PREFIX, executor_id, job_id, call_id),
'stats_filename':
jobrunner_stats_filename
}
if show_memory_peak:
mm_handler_conn, mm_conn = Pipe()
memory_monitor = Thread(target=memory_monitor_worker,
args=(mm_conn, ))
memory_monitor.start()
handler_conn, jobrunner_conn = Pipe()
jobrunner = JobRunner(jobrunner_config, jobrunner_conn,
internal_storage)
logger.debug('Starting JobRunner process')
local_execution = strtobool(
os.environ.get('__PW_LOCAL_EXECUTION', 'False'))
jrp = Thread(target=jobrunner.run) if local_execution else Process(
target=jobrunner.run)
jrp.start()
jrp.join(execution_timeout)
logger.debug('JobRunner process finished')
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Function exceeded maximum time of {} seconds and was '
'killed'.format(execution_timeout))
raise TimeoutError('HANDLER', msg)
if show_memory_peak:
mm_handler_conn.send('STOP')
memory_monitor.join()
peak_memory_usage = int(mm_handler_conn.recv())
logger.info("Peak memory usage: {}".format(
sizeof_fmt(peak_memory_usage)))
call_status.response['peak_memory_usage'] = peak_memory_usage
if not handler_conn.poll():
logger.error(
'No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Function exceeded maximum memory and was killed'
raise MemoryError('HANDLER', msg)
if os.path.exists(jobrunner_stats_filename):
with open(jobrunner_stats_filename, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.response[key] = float(value)
except Exception:
call_status.response[key] = value
if key in [
'exception', 'exc_pickle_fail', 'result',
'new_futures'
]:
call_status.response[key] = eval(value)
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------',
flush=True)
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------',
flush=True)
call_status.response['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
call_status.response['exc_info'] = str(pickled_exc)
finally:
call_status.response['worker_end_tstamp'] = time.time()
call_status.send('__end__')
for key in extra_env:
os.environ.pop(key)
logger.info("Finished")
def run_job(job):
"""
Runs a single job within a separate process
"""
setup_lithops_logger(job.log_level)
backend = os.environ.get('__LITHOPS_BACKEND', '')
logger.info("Lithops v{} - Starting {} execution".format(
__version__, backend))
logger.info("Execution ID: {}/{}".format(job.job_key, job.call_id))
env = job.extra_env
env['LITHOPS_CONFIG'] = json.dumps(job.config)
env['__LITHOPS_SESSION_ID'] = '-'.join([job.job_key, job.call_id])
os.environ.update(env)
storage_config = extract_storage_config(job.config)
internal_storage = InternalStorage(storage_config)
call_status = create_call_status(job, internal_storage)
if job.runtime_memory:
logger.debug('Runtime: {} - Memory: {}MB - Timeout: {} seconds'.format(
job.runtime_name, job.runtime_memory, job.execution_timeout))
else:
logger.debug('Runtime: {} - Timeout: {} seconds'.format(
job.runtime_name, job.execution_timeout))
job_interruped = False
try:
# send init status event
call_status.send_init_event()
handler_conn, jobrunner_conn = Pipe()
jobrunner = JobRunner(job, jobrunner_conn, internal_storage)
logger.debug('Starting JobRunner process')
jrp = Process(target=jobrunner.run) if is_unix_system() else Thread(
target=jobrunner.run)
jrp.start()
jrp.join(job.execution_timeout)
logger.debug('JobRunner process finished')
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Function exceeded maximum time of {} seconds and was '
'killed'.format(job.execution_timeout))
raise TimeoutError('HANDLER', msg)
if not handler_conn.poll():
logger.error(
'No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Function exceeded maximum memory and was killed'
raise MemoryError('HANDLER', msg)
if os.path.exists(job.stats_file):
with open(job.stats_file, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.add(key, float(value))
except Exception:
call_status.add(key, value)
if key in ['exception', 'exc_pickle_fail', 'result']:
call_status.add(key, eval(value))
except KeyboardInterrupt:
job_interruped = True
logger.debug("Job interrupted")
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------')
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------')
call_status.add('exception', True)
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
call_status.add('exc_info', str(pickled_exc))
finally:
if not job_interruped:
call_status.add('worker_end_tstamp', time.time())
# Flush log stream and save it to the call status
job.log_stream.flush()
if os.path.isfile(job.log_file):
with open(job.log_file, 'rb') as lf:
log_str = base64.b64encode(zlib.compress(
lf.read())).decode()
call_status.add('logs', log_str)
call_status.send_finish_event()
logger.info("Finished")