def __init__(self, jr_config, jobrunner_conn, internal_storage): self.jr_config = jr_config self.jobrunner_conn = jobrunner_conn self.internal_storage = internal_storage log_level = self.jr_config['log_level'] cloud_logging_config(log_level) self.pywren_config = self.jr_config['pywren_config'] self.call_id = self.jr_config['call_id'] self.job_id = self.jr_config['job_id'] self.executor_id = self.jr_config['executor_id'] self.func_key = self.jr_config['func_key'] self.data_key = self.jr_config['data_key'] self.data_byte_range = self.jr_config['data_byte_range'] self.output_key = self.jr_config['output_key'] self.stats = stats(self.jr_config['stats_filename'])
def function_invoker(event): if __version__ != event['pywren_version']: raise Exception("WRONGVERSION", "PyWren version mismatch", __version__, event['pywren_version']) log_level = event['log_level'] cloud_logging_config(log_level) log_level = logging.getLevelName(logger.getEffectiveLevel()) custom_env = { 'PYWREN_FUNCTION': 'True', 'PYTHONUNBUFFERED': 'True', 'PYWREN_LOGLEVEL': log_level } os.environ.update(custom_env) config = event['config'] invoker = FunctionInvoker(config, log_level) invoker.run(event['job_description'])
def __init__(self, tr_config, result_queue): super().__init__() start_time = time.time() self.config = tr_config log_level = self.config['log_level'] self.result_queue = result_queue cloud_logging_config(log_level) self.stats = stats(self.config['stats_filename']) self.stats.write('jobrunner_start', start_time) cb_config = json.loads(os.environ.get('CB_CONFIG')) self.storage_config = extract_storage_config(cb_config) if 'SHOW_MEMORY_USAGE' in os.environ: self.show_memory = eval(os.environ['SHOW_MEMORY_USAGE']) else: self.show_memory = False self.func_key = self.config['func_key'] self.data_key = self.config['data_key'] self.data_byte_range = self.config['data_byte_range'] self.output_key = self.config['output_key']
def __init__(self, jr_config, jobrunner_conn): start_time = time.time() self.jr_config = jr_config self.jobrunner_conn = jobrunner_conn log_level = self.jr_config['log_level'] cloud_logging_config(log_level) self.pywren_config = self.jr_config['pywren_config'] self.storage_config = extract_storage_config(self.pywren_config) self.call_id = self.jr_config['call_id'] self.job_id = self.jr_config['job_id'] self.executor_id = self.jr_config['executor_id'] self.func_key = self.jr_config['func_key'] self.data_key = self.jr_config['data_key'] self.data_byte_range = self.jr_config['data_byte_range'] self.output_key = self.jr_config['output_key'] self.stats = stats(self.jr_config['stats_filename']) self.stats.write('jobrunner_start', start_time) self.show_memory = strtobool( os.environ.get('SHOW_MEMORY_USAGE', 'False'))
import sys import os import uuid import flask import logging import pkgutil from pywren_ibm_cloud.config import cloud_logging_config from pywren_ibm_cloud.runtime.function_handler import function_handler cloud_logging_config(logging.INFO) logger = logging.getLogger('__main__') proxy = flask.Flask(__name__) @proxy.route('/', methods=['POST']) def run(): def error(): response = flask.jsonify({ 'error': 'The action did not receive a dictionary as an argument.' }) response.status_code = 404 return complete(response) message = flask.request.get_json(force=True, silent=True) if message and not isinstance(message, dict): return error() act_id = str(uuid.uuid4()).replace('-', '')[:12] logger.info("Starting knative Function execution")
def function_handler(event): start_time = time.time() log_level = event['log_level'] cloud_logging_config(log_level) logger.debug("Action handler started") response_status = {'exception': False} response_status['host_submit_time'] = event['host_submit_time'] response_status['start_time'] = start_time context_dict = { 'python_version': os.environ.get("PYTHON_VERSION"), } config = event['config'] storage_config = extract_storage_config(config) call_id = event['call_id'] job_id = event['job_id'] executor_id = event['executor_id'] logger.info("Execution ID: {}/{}/{}".format(executor_id, job_id, call_id)) execution_timeout = event['execution_timeout'] logger.debug( "Set function execution timeout to {}s".format(execution_timeout)) status_key = event['status_key'] func_key = event['func_key'] data_key = event['data_key'] data_byte_range = event['data_byte_range'] output_key = event['output_key'] extra_env = event.get('extra_env', {}) response_status['call_id'] = call_id response_status['job_id'] = job_id response_status['executor_id'] = executor_id # response_status['func_key'] = func_key # response_status['data_key'] = data_key # response_status['output_key'] = output_key # response_status['status_key'] = status_key try: if version.__version__ != event['pywren_version']: raise Exception("WRONGVERSION", "PyWren version mismatch", version.__version__, event['pywren_version']) # response_status['free_disk_bytes'] = free_disk_space("/tmp") custom_env = { 'PYWREN_CONFIG': json.dumps(config), 'PYWREN_REMOTE': 'TRUE', 'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH), 'PYTHONUNBUFFERED': 'True' } os.environ.update(custom_env) os.environ.update(extra_env) jobrunner_config = { 'pywren_config': config, 'call_id': call_id, 'job_id': job_id, 'executor_id': executor_id, 'func_key': func_key, 'data_key': data_key, 'log_level': log_level, 'data_byte_range': data_byte_range, 'python_module_path': PYTHON_MODULE_PATH, 'output_key': output_key, 'stats_filename': JOBRUNNER_STATS_FILENAME } if os.path.exists(JOBRUNNER_STATS_FILENAME): os.remove(JOBRUNNER_STATS_FILENAME) setup_time = time.time() response_status['setup_time'] = round(setup_time - start_time, 8) result_queue = multiprocessing.Queue() tr = JobRunner(jobrunner_config, result_queue) tr.daemon = True logger.debug('Starting JobRunner process') tr.start() tr.join(execution_timeout) logger.debug('Finished JobRunner process') response_status['exec_time'] = round(time.time() - setup_time, 8) if tr.is_alive(): # If process is still alive after jr.join(job_max_runtime), kill it tr.terminate() msg = ('Jobrunner process exceeded maximum time of {} ' 'seconds and was killed'.format(execution_timeout)) raise Exception('OUTATIME', msg) if result_queue.empty(): # Only 1 message is returned by jobrunner when it finishes. # If no message, this means that the jobrunner process was killed. # 99% of times the jobrunner is killed due an OOM, so we assume here an OOM. msg = 'Jobrunner process exceeded maximum memory and was killed' raise Exception('OUTOFMEMORY', msg) # print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True)) # print(subprocess.check_output("find {}".format(os.getcwd()), shell=True)) if os.path.exists(JOBRUNNER_STATS_FILENAME): with open(JOBRUNNER_STATS_FILENAME, 'r') as fid: for l in fid.readlines(): key, value = l.strip().split(" ", 1) try: response_status[key] = float(value) except Exception: response_status[key] = value if key in [ 'exception', 'exc_pickle_fail', 'result', 'new_futures' ]: response_status[key] = eval(value) # response_status['server_info'] = get_server_info() response_status.update(context_dict) response_status['end_time'] = time.time() except Exception: # internal runtime exceptions print('----------------------- EXCEPTION !-----------------------', flush=True) traceback.print_exc(file=sys.stdout) print('----------------------------------------------------------', flush=True) response_status['end_time'] = time.time() response_status['exception'] = True pickled_exc = pickle.dumps(sys.exc_info()) pickle.loads( pickled_exc) # this is just to make sure they can be unpickled response_status['exc_info'] = str(pickled_exc) finally: store_status = strtobool(os.environ.get('STORE_STATUS', 'True')) dmpd_response_status = json.dumps(response_status) drs = sizeof_fmt(len(dmpd_response_status)) rabbitmq_monitor = config['pywren'].get('rabbitmq_monitor', False) if rabbitmq_monitor and store_status: rabbit_amqp_url = config['rabbitmq'].get('amqp_url') status_sent = False output_query_count = 0 params = pika.URLParameters(rabbit_amqp_url) queue = '{}-{}'.format(executor_id, job_id) while not status_sent and output_query_count < 5: output_query_count = output_query_count + 1 try: connection = pika.BlockingConnection(params) channel = connection.channel() channel.queue_declare(queue=queue, auto_delete=True) channel.basic_publish(exchange='', routing_key=queue, body=dmpd_response_status) connection.close() logger.info( "Execution status sent to rabbitmq - Size: {}".format( drs)) status_sent = True except Exception as e: logger.error("Unable to send status to rabbitmq") logger.error(str(e)) logger.info('Retrying to send status to rabbitmq...') time.sleep(0.2) if store_status: internal_storage = InternalStorage(storage_config) logger.info( "Storing execution stats - status.json - Size: {}".format(drs)) internal_storage.put_data(status_key, dmpd_response_status)
def function_handler(event): start_time = time.time() logger.debug("Action handler started") response_status = {'exception': False} response_status['host_submit_time'] = event['host_submit_time'] response_status['start_time'] = start_time context_dict = { 'ibm_cf_request_id': os.environ.get("__OW_ACTIVATION_ID"), 'ibm_cf_python_version': os.environ.get("PYTHON_VERSION"), } config = event['config'] storage_config = extract_storage_config(config) log_level = event['log_level'] cloud_logging_config(log_level) call_id = event['call_id'] job_id = event['job_id'] executor_id = event['executor_id'] logger.info("Execution ID: {}/{}/{}".format(executor_id, job_id, call_id)) task_execution_timeout = event.get("task_execution_timeout", 590) # default for CF status_key = event['status_key'] func_key = event['func_key'] data_key = event['data_key'] data_byte_range = event['data_byte_range'] output_key = event['output_key'] extra_env = event.get('extra_env', {}) response_status['call_id'] = call_id response_status['job_id'] = job_id response_status['executor_id'] = executor_id # response_status['func_key'] = func_key # response_status['data_key'] = data_key # response_status['output_key'] = output_key # response_status['status_key'] = status_key try: if version.__version__ != event['pywren_version']: raise Exception("WRONGVERSION", "PyWren version mismatch", version.__version__, event['pywren_version']) # response_status['free_disk_bytes'] = free_disk_space("/tmp") custom_env = { 'CB_CONFIG': json.dumps(config), 'CB_CALL_ID': call_id, 'CB_JOB_ID': job_id, 'CB_EXECUTOR_ID': executor_id, 'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH), 'PYTHONUNBUFFERED': 'True' } os.environ.update(custom_env) os.environ.update(extra_env) # pass a full json blob jobrunner_config = { 'func_key': func_key, 'data_key': data_key, 'log_level': log_level, 'data_byte_range': data_byte_range, 'python_module_path': PYTHON_MODULE_PATH, 'output_key': output_key, 'stats_filename': JOBRUNNER_STATS_FILENAME } if os.path.exists(JOBRUNNER_STATS_FILENAME): os.remove(JOBRUNNER_STATS_FILENAME) setup_time = time.time() response_status['setup_time'] = round(setup_time - start_time, 8) result_queue = multiprocessing.Queue() tr = JobRunner(jobrunner_config, result_queue) tr.daemon = True logger.info("Starting JobRunner process") tr.start() tr.join(task_execution_timeout) response_status['exec_time'] = round(time.time() - setup_time, 8) if tr.is_alive(): # If process is still alive after jr.join(job_max_runtime), kill it logger.error( "Process exceeded maximum runtime of {} seconds".format( task_execution_timeout)) # Send the signal to all the process groups tr.terminate() raise Exception("OUTATIME", "Process executed for too long and was killed") try: # Only 1 message is returned by jobrunner result_queue.get(block=False) except Exception: # If no message, this means that the process was killed due an exception pickling an exception raise Exception( "EXCPICKLEERROR", "PyWren was unable to pickle the exception, check function logs" ) # print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True)) # print(subprocess.check_output("find {}".format(os.getcwd()), shell=True)) if os.path.exists(JOBRUNNER_STATS_FILENAME): with open(JOBRUNNER_STATS_FILENAME, 'r') as fid: for l in fid.readlines(): key, value = l.strip().split(" ", 1) try: response_status[key] = float(value) except Exception: response_status[key] = value if key == 'exception' or key == 'exc_pickle_fail' \ or key == 'result': response_status[key] = eval(value) # response_status['server_info'] = get_server_info() response_status.update(context_dict) response_status['end_time'] = time.time() except Exception as e: # internal runtime exceptions logger.error("There was an exception: {}".format(str(e))) response_status['end_time'] = time.time() response_status['exception'] = True pickled_exc = pickle.dumps(sys.exc_info()) pickle.loads( pickled_exc) # this is just to make sure they can be unpickled response_status['exc_info'] = str(pickled_exc) finally: store_status = strtobool(os.environ.get('STORE_STATUS', 'True')) rabbit_amqp_url = config['rabbitmq'].get('amqp_url') dmpd_response_status = json.dumps(response_status) drs = sizeof_fmt(len(dmpd_response_status)) if rabbit_amqp_url and store_status: status_sent = False output_query_count = 0 while not status_sent and output_query_count < 5: output_query_count = output_query_count + 1 try: params = pika.URLParameters(rabbit_amqp_url) connection = pika.BlockingConnection(params) channel = connection.channel() channel.queue_declare(queue=executor_id, auto_delete=True) channel.basic_publish(exchange='', routing_key=executor_id, body=dmpd_response_status) connection.close() logger.info( "Execution stats sent to rabbitmq - Size: {}".format( drs)) status_sent = True except Exception as e: logger.error("Unable to send status to rabbitmq") logger.error(str(e)) logger.info('Retrying to send stats to rabbitmq...') time.sleep(0.2) if store_status: internal_storage = InternalStorage(storage_config) logger.info( "Storing execution stats - status.json - Size: {}".format(drs)) internal_storage.put_data(status_key, dmpd_response_status)
def function_handler(event): start_tstamp = time.time() log_level = event['log_level'] cloud_logging_config(log_level) logger.debug("Action handler started") extra_env = event.get('extra_env', {}) os.environ.update(extra_env) os.environ.update({'PYWREN_FUNCTION': 'True', 'PYTHONUNBUFFERED': 'True'}) config = event['config'] call_id = event['call_id'] job_id = event['job_id'] executor_id = event['executor_id'] exec_id = "{}/{}/{}".format(executor_id, job_id, call_id) logger.info("Execution-ID: {}".format(exec_id)) runtime_name = event['runtime_name'] runtime_memory = event['runtime_memory'] execution_timeout = event['execution_timeout'] logger.debug("Runtime name: {}".format(runtime_name)) logger.debug("Runtime memory: {}MB".format(runtime_memory)) logger.debug("Function timeout: {}s".format(execution_timeout)) func_key = event['func_key'] data_key = event['data_key'] data_byte_range = event['data_byte_range'] storage_config = extract_storage_config(config) internal_storage = InternalStorage(storage_config) call_status = CallStatus(config, internal_storage) call_status.response['host_submit_tstamp'] = event['host_submit_tstamp'] call_status.response['start_tstamp'] = start_tstamp context_dict = { 'python_version': os.environ.get("PYTHON_VERSION"), 'call_id': call_id, 'job_id': job_id, 'executor_id': executor_id, 'activation_id': os.environ.get('__PW_ACTIVATION_ID') } call_status.response.update(context_dict) show_memory_peak = strtobool(os.environ.get('SHOW_MEMORY_PEAK', 'False')) try: if version.__version__ != event['pywren_version']: msg = ( "PyWren version mismatch. Host version: {} - Runtime version: {}" .format(event['pywren_version'], version.__version__)) raise RuntimeError('HANDLER', msg) # send init status event call_status.send('__init__') # call_status.response['free_disk_bytes'] = free_disk_space("/tmp") custom_env = { 'PYWREN_CONFIG': json.dumps(config), 'PYWREN_EXECUTION_ID': exec_id, 'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH) } os.environ.update(custom_env) jobrunner_stats_dir = os.path.join(STORAGE_FOLDER, storage_config['bucket'], JOBS_PREFIX, executor_id, job_id, call_id) os.makedirs(jobrunner_stats_dir, exist_ok=True) jobrunner_stats_filename = os.path.join(jobrunner_stats_dir, 'jobrunner.stats.txt') jobrunner_config = { 'pywren_config': config, 'call_id': call_id, 'job_id': job_id, 'executor_id': executor_id, 'func_key': func_key, 'data_key': data_key, 'log_level': log_level, 'data_byte_range': data_byte_range, 'output_key': create_output_key(JOBS_PREFIX, executor_id, job_id, call_id), 'stats_filename': jobrunner_stats_filename } if show_memory_peak: mm_handler_conn, mm_conn = Pipe() memory_monitor = Thread(target=memory_monitor_worker, args=(mm_conn, )) memory_monitor.start() handler_conn, jobrunner_conn = Pipe() jobrunner = JobRunner(jobrunner_config, jobrunner_conn, internal_storage) logger.debug('Starting JobRunner process') local_execution = strtobool( os.environ.get('__PW_LOCAL_EXECUTION', 'False')) jrp = Thread(target=jobrunner.run) if local_execution else Process( target=jobrunner.run) jrp.start() jrp.join(execution_timeout) logger.debug('JobRunner process finished') if jrp.is_alive(): # If process is still alive after jr.join(job_max_runtime), kill it try: jrp.terminate() except Exception: # thread does not have terminate method pass msg = ('Function exceeded maximum time of {} seconds and was ' 'killed'.format(execution_timeout)) raise TimeoutError('HANDLER', msg) if show_memory_peak: mm_handler_conn.send('STOP') memory_monitor.join() peak_memory_usage = int(mm_handler_conn.recv()) logger.info("Peak memory usage: {}".format( sizeof_fmt(peak_memory_usage))) call_status.response['peak_memory_usage'] = peak_memory_usage if not handler_conn.poll(): logger.error( 'No completion message received from JobRunner process') logger.debug('Assuming memory overflow...') # Only 1 message is returned by jobrunner when it finishes. # If no message, this means that the jobrunner process was killed. # 99% of times the jobrunner is killed due an OOM, so we assume here an OOM. msg = 'Function exceeded maximum memory and was killed' raise MemoryError('HANDLER', msg) if os.path.exists(jobrunner_stats_filename): with open(jobrunner_stats_filename, 'r') as fid: for l in fid.readlines(): key, value = l.strip().split(" ", 1) try: call_status.response[key] = float(value) except Exception: call_status.response[key] = value if key in [ 'exception', 'exc_pickle_fail', 'result', 'new_futures' ]: call_status.response[key] = eval(value) except Exception: # internal runtime exceptions print('----------------------- EXCEPTION !-----------------------', flush=True) traceback.print_exc(file=sys.stdout) print('----------------------------------------------------------', flush=True) call_status.response['exception'] = True pickled_exc = pickle.dumps(sys.exc_info()) pickle.loads( pickled_exc) # this is just to make sure they can be unpickled call_status.response['exc_info'] = str(pickled_exc) finally: call_status.response['end_tstamp'] = time.time() call_status.send('__end__') for key in extra_env: os.environ.pop(key) logger.info("Finished")
def function_handler(event): start_time = time.time() log_level = event['log_level'] cloud_logging_config(log_level) logger.debug("Action handler started") extra_env = event.get('extra_env', {}) os.environ.update(extra_env) config = event['config'] call_status = CallStatus(config) call_status.response['host_submit_time'] = event['host_submit_time'] call_status.response['start_time'] = start_time context_dict = { 'python_version': os.environ.get("PYTHON_VERSION"), } call_id = event['call_id'] job_id = event['job_id'] executor_id = event['executor_id'] exec_id = "{}/{}/{}".format(executor_id, job_id, call_id) logger.info("Execution ID: {}".format(exec_id)) execution_timeout = event['execution_timeout'] logger.debug("Set function execution timeout to {}s".format(execution_timeout)) func_key = event['func_key'] data_key = event['data_key'] data_byte_range = event['data_byte_range'] call_status.response['call_id'] = call_id call_status.response['job_id'] = job_id call_status.response['executor_id'] = executor_id call_status.response['activation_id'] = os.environ.get('__OW_ACTIVATION_ID') try: if version.__version__ != event['pywren_version']: raise Exception("WRONGVERSION", "PyWren version mismatch", version.__version__, event['pywren_version']) # send init status event call_status.send('__init__') # call_status.response['free_disk_bytes'] = free_disk_space("/tmp") custom_env = {'PYWREN_CONFIG': json.dumps(config), 'PYWREN_FUNCTION': 'True', 'PYWREN_EXECUTION_ID': exec_id, 'PYWREN_STORAGE_BUCKET': config['pywren']['storage_bucket'], 'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH), 'PYTHONUNBUFFERED': 'True'} os.environ.update(custom_env) # if os.path.exists(JOBRUNNER_STATS_BASE_DIR): # shutil.rmtree(JOBRUNNER_STATS_BASE_DIR, True) jobrunner_stats_dir = os.path.join(STORAGE_BASE_DIR, executor_id, job_id, call_id) os.makedirs(jobrunner_stats_dir, exist_ok=True) jobrunner_stats_filename = os.path.join(jobrunner_stats_dir, 'jobrunner.stats.txt') jobrunner_config = {'pywren_config': config, 'call_id': call_id, 'job_id': job_id, 'executor_id': executor_id, 'func_key': func_key, 'data_key': data_key, 'log_level': log_level, 'data_byte_range': data_byte_range, 'output_key': create_output_key(JOBS_PREFIX, executor_id, job_id, call_id), 'stats_filename': jobrunner_stats_filename} setup_time = time.time() call_status.response['setup_time'] = round(setup_time - start_time, 8) handler_conn, jobrunner_conn = Pipe() jobrunner = JobRunner(jobrunner_config, jobrunner_conn) logger.debug('Starting JobRunner process') local_execution = strtobool(os.environ.get('LOCAL_EXECUTION', 'False')) if local_execution: jrp = Thread(target=jobrunner.run) else: jrp = Process(target=jobrunner.run) jrp.daemon = True jrp.start() jrp.join(execution_timeout) logger.debug('JobRunner process finished') call_status.response['exec_time'] = round(time.time() - setup_time, 8) if jrp.is_alive(): # If process is still alive after jr.join(job_max_runtime), kill it try: jrp.terminate() except Exception: # thread does not have terminate method pass msg = ('Jobrunner process exceeded maximum time of {} ' 'seconds and was killed'.format(execution_timeout)) raise Exception('OUTATIME', msg) try: handler_conn.recv() except EOFError: logger.error('No completion message received from JobRunner process') logger.debug('Assuming memory overflow...') # Only 1 message is returned by jobrunner when it finishes. # If no message, this means that the jobrunner process was killed. # 99% of times the jobrunner is killed due an OOM, so we assume here an OOM. msg = 'Jobrunner process exceeded maximum memory and was killed' raise Exception('OUTOFMEMORY', msg) # print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True)) # print(subprocess.check_output("find {}".format(os.getcwd()), shell=True)) if os.path.exists(jobrunner_stats_filename): with open(jobrunner_stats_filename, 'r') as fid: for l in fid.readlines(): key, value = l.strip().split(" ", 1) try: call_status.response[key] = float(value) except Exception: call_status.response[key] = value if key in ['exception', 'exc_pickle_fail', 'result', 'new_futures']: call_status.response[key] = eval(value) # call_status.response['server_info'] = get_server_info() call_status.response.update(context_dict) call_status.response['end_time'] = time.time() except Exception: # internal runtime exceptions print('----------------------- EXCEPTION !-----------------------', flush=True) traceback.print_exc(file=sys.stdout) print('----------------------------------------------------------', flush=True) call_status.response['end_time'] = time.time() call_status.response['exception'] = True pickled_exc = pickle.dumps(sys.exc_info()) pickle.loads(pickled_exc) # this is just to make sure they can be unpickled call_status.response['exc_info'] = str(pickled_exc) finally: call_status.send('__end__') logger.info("Finished")