def __init__(self, jr_config, jobrunner_conn, internal_storage):
self.jr_config = jr_config
self.jobrunner_conn = jobrunner_conn
self.internal_storage = internal_storage
log_level = self.jr_config['log_level']
cloud_logging_config(log_level)
self.pywren_config = self.jr_config['pywren_config']
self.call_id = self.jr_config['call_id']
self.job_id = self.jr_config['job_id']
self.executor_id = self.jr_config['executor_id']
self.func_key = self.jr_config['func_key']
self.data_key = self.jr_config['data_key']
self.data_byte_range = self.jr_config['data_byte_range']
self.output_key = self.jr_config['output_key']
self.stats = stats(self.jr_config['stats_filename'])
def function_invoker(event):
if __version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch", __version__,
event['pywren_version'])
log_level = event['log_level']
cloud_logging_config(log_level)
log_level = logging.getLevelName(logger.getEffectiveLevel())
custom_env = {
'PYWREN_FUNCTION': 'True',
'PYTHONUNBUFFERED': 'True',
'PYWREN_LOGLEVEL': log_level
}
os.environ.update(custom_env)
config = event['config']
invoker = FunctionInvoker(config, log_level)
invoker.run(event['job_description'])
def __init__(self, tr_config, result_queue):
super().__init__()
start_time = time.time()
self.config = tr_config
log_level = self.config['log_level']
self.result_queue = result_queue
cloud_logging_config(log_level)
self.stats = stats(self.config['stats_filename'])
self.stats.write('jobrunner_start', start_time)
cb_config = json.loads(os.environ.get('CB_CONFIG'))
self.storage_config = extract_storage_config(cb_config)
if 'SHOW_MEMORY_USAGE' in os.environ:
self.show_memory = eval(os.environ['SHOW_MEMORY_USAGE'])
else:
self.show_memory = False
self.func_key = self.config['func_key']
self.data_key = self.config['data_key']
self.data_byte_range = self.config['data_byte_range']
self.output_key = self.config['output_key']
def __init__(self, jr_config, jobrunner_conn):
start_time = time.time()
self.jr_config = jr_config
self.jobrunner_conn = jobrunner_conn
log_level = self.jr_config['log_level']
cloud_logging_config(log_level)
self.pywren_config = self.jr_config['pywren_config']
self.storage_config = extract_storage_config(self.pywren_config)
self.call_id = self.jr_config['call_id']
self.job_id = self.jr_config['job_id']
self.executor_id = self.jr_config['executor_id']
self.func_key = self.jr_config['func_key']
self.data_key = self.jr_config['data_key']
self.data_byte_range = self.jr_config['data_byte_range']
self.output_key = self.jr_config['output_key']
self.stats = stats(self.jr_config['stats_filename'])
self.stats.write('jobrunner_start', start_time)
self.show_memory = strtobool(
os.environ.get('SHOW_MEMORY_USAGE', 'False'))
import sys
import os
import uuid
import flask
import logging
import pkgutil
from pywren_ibm_cloud.config import cloud_logging_config
from pywren_ibm_cloud.runtime.function_handler import function_handler
cloud_logging_config(logging.INFO)
logger = logging.getLogger('__main__')
proxy = flask.Flask(__name__)
@proxy.route('/', methods=['POST'])
def run():
def error():
response = flask.jsonify({
'error':
'The action did not receive a dictionary as an argument.'
})
response.status_code = 404
return complete(response)
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
return error()
act_id = str(uuid.uuid4()).replace('-', '')[:12]
logger.info("Starting knative Function execution")
def function_handler(event):
start_time = time.time()
log_level = event['log_level']
cloud_logging_config(log_level)
logger.debug("Action handler started")
response_status = {'exception': False}
response_status['host_submit_time'] = event['host_submit_time']
response_status['start_time'] = start_time
context_dict = {
'python_version': os.environ.get("PYTHON_VERSION"),
}
config = event['config']
storage_config = extract_storage_config(config)
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
logger.info("Execution ID: {}/{}/{}".format(executor_id, job_id, call_id))
execution_timeout = event['execution_timeout']
logger.debug(
"Set function execution timeout to {}s".format(execution_timeout))
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
extra_env = event.get('extra_env', {})
response_status['call_id'] = call_id
response_status['job_id'] = job_id
response_status['executor_id'] = executor_id
# response_status['func_key'] = func_key
# response_status['data_key'] = data_key
# response_status['output_key'] = output_key
# response_status['status_key'] = status_key
try:
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch",
version.__version__, event['pywren_version'])
# response_status['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'PYWREN_CONFIG': json.dumps(config),
'PYWREN_REMOTE': 'TRUE',
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH),
'PYTHONUNBUFFERED': 'True'
}
os.environ.update(custom_env)
os.environ.update(extra_env)
jobrunner_config = {
'pywren_config': config,
'call_id': call_id,
'job_id': job_id,
'executor_id': executor_id,
'func_key': func_key,
'data_key': data_key,
'log_level': log_level,
'data_byte_range': data_byte_range,
'python_module_path': PYTHON_MODULE_PATH,
'output_key': output_key,
'stats_filename': JOBRUNNER_STATS_FILENAME
}
if os.path.exists(JOBRUNNER_STATS_FILENAME):
os.remove(JOBRUNNER_STATS_FILENAME)
setup_time = time.time()
response_status['setup_time'] = round(setup_time - start_time, 8)
result_queue = multiprocessing.Queue()
tr = JobRunner(jobrunner_config, result_queue)
tr.daemon = True
logger.debug('Starting JobRunner process')
tr.start()
tr.join(execution_timeout)
logger.debug('Finished JobRunner process')
response_status['exec_time'] = round(time.time() - setup_time, 8)
if tr.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
tr.terminate()
msg = ('Jobrunner process exceeded maximum time of {} '
'seconds and was killed'.format(execution_timeout))
raise Exception('OUTATIME', msg)
if result_queue.empty():
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Jobrunner process exceeded maximum memory and was killed'
raise Exception('OUTOFMEMORY', msg)
# print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
# print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
response_status[key] = float(value)
except Exception:
response_status[key] = value
if key in [
'exception', 'exc_pickle_fail', 'result',
'new_futures'
]:
response_status[key] = eval(value)
# response_status['server_info'] = get_server_info()
response_status.update(context_dict)
response_status['end_time'] = time.time()
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------',
flush=True)
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------',
flush=True)
response_status['end_time'] = time.time()
response_status['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
response_status['exc_info'] = str(pickled_exc)
finally:
store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
dmpd_response_status = json.dumps(response_status)
drs = sizeof_fmt(len(dmpd_response_status))
rabbitmq_monitor = config['pywren'].get('rabbitmq_monitor', False)
if rabbitmq_monitor and store_status:
rabbit_amqp_url = config['rabbitmq'].get('amqp_url')
status_sent = False
output_query_count = 0
params = pika.URLParameters(rabbit_amqp_url)
queue = '{}-{}'.format(executor_id, job_id)
while not status_sent and output_query_count < 5:
output_query_count = output_query_count + 1
try:
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=queue, auto_delete=True)
channel.basic_publish(exchange='',
routing_key=queue,
body=dmpd_response_status)
connection.close()
logger.info(
"Execution status sent to rabbitmq - Size: {}".format(
drs))
status_sent = True
except Exception as e:
logger.error("Unable to send status to rabbitmq")
logger.error(str(e))
logger.info('Retrying to send status to rabbitmq...')
time.sleep(0.2)
if store_status:
internal_storage = InternalStorage(storage_config)
logger.info(
"Storing execution stats - status.json - Size: {}".format(drs))
internal_storage.put_data(status_key, dmpd_response_status)
def function_handler(event):
start_time = time.time()
logger.debug("Action handler started")
response_status = {'exception': False}
response_status['host_submit_time'] = event['host_submit_time']
response_status['start_time'] = start_time
context_dict = {
'ibm_cf_request_id': os.environ.get("__OW_ACTIVATION_ID"),
'ibm_cf_python_version': os.environ.get("PYTHON_VERSION"),
}
config = event['config']
storage_config = extract_storage_config(config)
log_level = event['log_level']
cloud_logging_config(log_level)
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
logger.info("Execution ID: {}/{}/{}".format(executor_id, job_id, call_id))
task_execution_timeout = event.get("task_execution_timeout",
590) # default for CF
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
extra_env = event.get('extra_env', {})
response_status['call_id'] = call_id
response_status['job_id'] = job_id
response_status['executor_id'] = executor_id
# response_status['func_key'] = func_key
# response_status['data_key'] = data_key
# response_status['output_key'] = output_key
# response_status['status_key'] = status_key
try:
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch",
version.__version__, event['pywren_version'])
# response_status['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'CB_CONFIG': json.dumps(config),
'CB_CALL_ID': call_id,
'CB_JOB_ID': job_id,
'CB_EXECUTOR_ID': executor_id,
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH),
'PYTHONUNBUFFERED': 'True'
}
os.environ.update(custom_env)
os.environ.update(extra_env)
# pass a full json blob
jobrunner_config = {
'func_key': func_key,
'data_key': data_key,
'log_level': log_level,
'data_byte_range': data_byte_range,
'python_module_path': PYTHON_MODULE_PATH,
'output_key': output_key,
'stats_filename': JOBRUNNER_STATS_FILENAME
}
if os.path.exists(JOBRUNNER_STATS_FILENAME):
os.remove(JOBRUNNER_STATS_FILENAME)
setup_time = time.time()
response_status['setup_time'] = round(setup_time - start_time, 8)
result_queue = multiprocessing.Queue()
tr = JobRunner(jobrunner_config, result_queue)
tr.daemon = True
logger.info("Starting JobRunner process")
tr.start()
tr.join(task_execution_timeout)
response_status['exec_time'] = round(time.time() - setup_time, 8)
if tr.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
logger.error(
"Process exceeded maximum runtime of {} seconds".format(
task_execution_timeout))
# Send the signal to all the process groups
tr.terminate()
raise Exception("OUTATIME",
"Process executed for too long and was killed")
try:
# Only 1 message is returned by jobrunner
result_queue.get(block=False)
except Exception:
# If no message, this means that the process was killed due an exception pickling an exception
raise Exception(
"EXCPICKLEERROR",
"PyWren was unable to pickle the exception, check function logs"
)
# print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
# print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
response_status[key] = float(value)
except Exception:
response_status[key] = value
if key == 'exception' or key == 'exc_pickle_fail' \
or key == 'result':
response_status[key] = eval(value)
# response_status['server_info'] = get_server_info()
response_status.update(context_dict)
response_status['end_time'] = time.time()
except Exception as e:
# internal runtime exceptions
logger.error("There was an exception: {}".format(str(e)))
response_status['end_time'] = time.time()
response_status['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
response_status['exc_info'] = str(pickled_exc)
finally:
store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
rabbit_amqp_url = config['rabbitmq'].get('amqp_url')
dmpd_response_status = json.dumps(response_status)
drs = sizeof_fmt(len(dmpd_response_status))
if rabbit_amqp_url and store_status:
status_sent = False
output_query_count = 0
while not status_sent and output_query_count < 5:
output_query_count = output_query_count + 1
try:
params = pika.URLParameters(rabbit_amqp_url)
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=executor_id, auto_delete=True)
channel.basic_publish(exchange='',
routing_key=executor_id,
body=dmpd_response_status)
connection.close()
logger.info(
"Execution stats sent to rabbitmq - Size: {}".format(
drs))
status_sent = True
except Exception as e:
logger.error("Unable to send status to rabbitmq")
logger.error(str(e))
logger.info('Retrying to send stats to rabbitmq...')
time.sleep(0.2)
if store_status:
internal_storage = InternalStorage(storage_config)
logger.info(
"Storing execution stats - status.json - Size: {}".format(drs))
internal_storage.put_data(status_key, dmpd_response_status)
def function_handler(event):
start_tstamp = time.time()
log_level = event['log_level']
cloud_logging_config(log_level)
logger.debug("Action handler started")
extra_env = event.get('extra_env', {})
os.environ.update(extra_env)
os.environ.update({'PYWREN_FUNCTION': 'True', 'PYTHONUNBUFFERED': 'True'})
config = event['config']
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
exec_id = "{}/{}/{}".format(executor_id, job_id, call_id)
logger.info("Execution-ID: {}".format(exec_id))
runtime_name = event['runtime_name']
runtime_memory = event['runtime_memory']
execution_timeout = event['execution_timeout']
logger.debug("Runtime name: {}".format(runtime_name))
logger.debug("Runtime memory: {}MB".format(runtime_memory))
logger.debug("Function timeout: {}s".format(execution_timeout))
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
storage_config = extract_storage_config(config)
internal_storage = InternalStorage(storage_config)
call_status = CallStatus(config, internal_storage)
call_status.response['host_submit_tstamp'] = event['host_submit_tstamp']
call_status.response['start_tstamp'] = start_tstamp
context_dict = {
'python_version': os.environ.get("PYTHON_VERSION"),
'call_id': call_id,
'job_id': job_id,
'executor_id': executor_id,
'activation_id': os.environ.get('__PW_ACTIVATION_ID')
}
call_status.response.update(context_dict)
show_memory_peak = strtobool(os.environ.get('SHOW_MEMORY_PEAK', 'False'))
try:
if version.__version__ != event['pywren_version']:
msg = (
"PyWren version mismatch. Host version: {} - Runtime version: {}"
.format(event['pywren_version'], version.__version__))
raise RuntimeError('HANDLER', msg)
# send init status event
call_status.send('__init__')
# call_status.response['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {
'PYWREN_CONFIG': json.dumps(config),
'PYWREN_EXECUTION_ID': exec_id,
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH)
}
os.environ.update(custom_env)
jobrunner_stats_dir = os.path.join(STORAGE_FOLDER,
storage_config['bucket'],
JOBS_PREFIX, executor_id, job_id,
call_id)
os.makedirs(jobrunner_stats_dir, exist_ok=True)
jobrunner_stats_filename = os.path.join(jobrunner_stats_dir,
'jobrunner.stats.txt')
jobrunner_config = {
'pywren_config':
config,
'call_id':
call_id,
'job_id':
job_id,
'executor_id':
executor_id,
'func_key':
func_key,
'data_key':
data_key,
'log_level':
log_level,
'data_byte_range':
data_byte_range,
'output_key':
create_output_key(JOBS_PREFIX, executor_id, job_id, call_id),
'stats_filename':
jobrunner_stats_filename
}
if show_memory_peak:
mm_handler_conn, mm_conn = Pipe()
memory_monitor = Thread(target=memory_monitor_worker,
args=(mm_conn, ))
memory_monitor.start()
handler_conn, jobrunner_conn = Pipe()
jobrunner = JobRunner(jobrunner_config, jobrunner_conn,
internal_storage)
logger.debug('Starting JobRunner process')
local_execution = strtobool(
os.environ.get('__PW_LOCAL_EXECUTION', 'False'))
jrp = Thread(target=jobrunner.run) if local_execution else Process(
target=jobrunner.run)
jrp.start()
jrp.join(execution_timeout)
logger.debug('JobRunner process finished')
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Function exceeded maximum time of {} seconds and was '
'killed'.format(execution_timeout))
raise TimeoutError('HANDLER', msg)
if show_memory_peak:
mm_handler_conn.send('STOP')
memory_monitor.join()
peak_memory_usage = int(mm_handler_conn.recv())
logger.info("Peak memory usage: {}".format(
sizeof_fmt(peak_memory_usage)))
call_status.response['peak_memory_usage'] = peak_memory_usage
if not handler_conn.poll():
logger.error(
'No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Function exceeded maximum memory and was killed'
raise MemoryError('HANDLER', msg)
if os.path.exists(jobrunner_stats_filename):
with open(jobrunner_stats_filename, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.response[key] = float(value)
except Exception:
call_status.response[key] = value
if key in [
'exception', 'exc_pickle_fail', 'result',
'new_futures'
]:
call_status.response[key] = eval(value)
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------',
flush=True)
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------',
flush=True)
call_status.response['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(
pickled_exc) # this is just to make sure they can be unpickled
call_status.response['exc_info'] = str(pickled_exc)
finally:
call_status.response['end_tstamp'] = time.time()
call_status.send('__end__')
for key in extra_env:
os.environ.pop(key)
logger.info("Finished")
def function_handler(event):
start_time = time.time()
log_level = event['log_level']
cloud_logging_config(log_level)
logger.debug("Action handler started")
extra_env = event.get('extra_env', {})
os.environ.update(extra_env)
config = event['config']
call_status = CallStatus(config)
call_status.response['host_submit_time'] = event['host_submit_time']
call_status.response['start_time'] = start_time
context_dict = {
'python_version': os.environ.get("PYTHON_VERSION"),
}
call_id = event['call_id']
job_id = event['job_id']
executor_id = event['executor_id']
exec_id = "{}/{}/{}".format(executor_id, job_id, call_id)
logger.info("Execution ID: {}".format(exec_id))
execution_timeout = event['execution_timeout']
logger.debug("Set function execution timeout to {}s".format(execution_timeout))
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
call_status.response['call_id'] = call_id
call_status.response['job_id'] = job_id
call_status.response['executor_id'] = executor_id
call_status.response['activation_id'] = os.environ.get('__OW_ACTIVATION_ID')
try:
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch",
version.__version__, event['pywren_version'])
# send init status event
call_status.send('__init__')
# call_status.response['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {'PYWREN_CONFIG': json.dumps(config),
'PYWREN_FUNCTION': 'True',
'PYWREN_EXECUTION_ID': exec_id,
'PYWREN_STORAGE_BUCKET': config['pywren']['storage_bucket'],
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH),
'PYTHONUNBUFFERED': 'True'}
os.environ.update(custom_env)
# if os.path.exists(JOBRUNNER_STATS_BASE_DIR):
# shutil.rmtree(JOBRUNNER_STATS_BASE_DIR, True)
jobrunner_stats_dir = os.path.join(STORAGE_BASE_DIR, executor_id, job_id, call_id)
os.makedirs(jobrunner_stats_dir, exist_ok=True)
jobrunner_stats_filename = os.path.join(jobrunner_stats_dir, 'jobrunner.stats.txt')
jobrunner_config = {'pywren_config': config,
'call_id': call_id,
'job_id': job_id,
'executor_id': executor_id,
'func_key': func_key,
'data_key': data_key,
'log_level': log_level,
'data_byte_range': data_byte_range,
'output_key': create_output_key(JOBS_PREFIX, executor_id, job_id, call_id),
'stats_filename': jobrunner_stats_filename}
setup_time = time.time()
call_status.response['setup_time'] = round(setup_time - start_time, 8)
handler_conn, jobrunner_conn = Pipe()
jobrunner = JobRunner(jobrunner_config, jobrunner_conn)
logger.debug('Starting JobRunner process')
local_execution = strtobool(os.environ.get('LOCAL_EXECUTION', 'False'))
if local_execution:
jrp = Thread(target=jobrunner.run)
else:
jrp = Process(target=jobrunner.run)
jrp.daemon = True
jrp.start()
jrp.join(execution_timeout)
logger.debug('JobRunner process finished')
call_status.response['exec_time'] = round(time.time() - setup_time, 8)
if jrp.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
try:
jrp.terminate()
except Exception:
# thread does not have terminate method
pass
msg = ('Jobrunner process exceeded maximum time of {} '
'seconds and was killed'.format(execution_timeout))
raise Exception('OUTATIME', msg)
try:
handler_conn.recv()
except EOFError:
logger.error('No completion message received from JobRunner process')
logger.debug('Assuming memory overflow...')
# Only 1 message is returned by jobrunner when it finishes.
# If no message, this means that the jobrunner process was killed.
# 99% of times the jobrunner is killed due an OOM, so we assume here an OOM.
msg = 'Jobrunner process exceeded maximum memory and was killed'
raise Exception('OUTOFMEMORY', msg)
# print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
# print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(jobrunner_stats_filename):
with open(jobrunner_stats_filename, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
call_status.response[key] = float(value)
except Exception:
call_status.response[key] = value
if key in ['exception', 'exc_pickle_fail', 'result', 'new_futures']:
call_status.response[key] = eval(value)
# call_status.response['server_info'] = get_server_info()
call_status.response.update(context_dict)
call_status.response['end_time'] = time.time()
except Exception:
# internal runtime exceptions
print('----------------------- EXCEPTION !-----------------------', flush=True)
traceback.print_exc(file=sys.stdout)
print('----------------------------------------------------------', flush=True)
call_status.response['end_time'] = time.time()
call_status.response['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(pickled_exc) # this is just to make sure they can be unpickled
call_status.response['exc_info'] = str(pickled_exc)
finally:
call_status.send('__end__')
logger.info("Finished")
