def __init__(self, config=None, runtime=None, log_level=None, job_max_runtime=JOB_MAX_RUNTIME): """ Initialize and return an executor class. :param config: Settings passed in here will override those in `pywren_config`. Default None. :param runtime: Runtime name to use. Default None. :param job_max_runtime: Max time per lambda. Default 300 :return `executor` object. Usage >>> import pywren >>> pw = pywren.ibm_cf_executor() """ self._state = ExecutorState.new if config is None: self.config = wrenconfig.default() else: self.config = wrenconfig.default(config) if runtime: self.config['ibm_cf']['action_name'] = runtime if log_level: wrenlogging.default_config(log_level) self._openwhisk = False if any([k.startswith('__OW_') for k in os.environ.keys()]): # OpenWhisk execution self._openwhisk = True wrenlogging.ow_config(logging.INFO) self.runtime = self.config['ibm_cf']['action_name'] ibm_cf_config = self.config['ibm_cf'] invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config) self.storage_config = wrenconfig.extract_storage_config(self.config) self.storage_handler = storage.Storage(self.storage_config) self.executor = Executor(invoker, self.config, self.storage_handler, job_max_runtime) self.executor_id = self.executor.executor_id self.futures = None self.reduce_future = None log_msg='IBM Cloud Functions executor created with ID {}'.format(self.executor_id) logger.info(log_msg) if(logger.getEffectiveLevel() == logging.WARNING): print(log_msg)
def __init__(self, jr_config, result_queue): super().__init__() start_time = time.time() self.result_queue = result_queue self.config = jr_config self.stats = stats(self.config['stats_filename']) self.stats.write('jobrunner_start', start_time) pw_config = json.loads(os.environ.get('PYWREN_CONFIG', '')) self.storage_config = extract_storage_config(pw_config) if 'SHOW_MEMORY_USAGE' in os.environ: self.show_memory = eval(os.environ['SHOW_MEMORY_USAGE']) else: self.show_memory = False log_level = self.config['log_level'] wrenlogging.ow_config(log_level) self.func_key = self.config['func_key'] self.data_key = self.config['data_key'] self.data_byte_range = self.config['data_byte_range'] self.output_key = self.config['output_key']
from multiprocessing import Process from pywren_ibm_cloud import wrenlogging from pywren_ibm_cloud.storage import storage from pywren_ibm_cloud.utils import sizeof_fmt, b64str_to_bytes from pywren_ibm_cloud.wrenconfig import extract_storage_config from pywren_ibm_cloud.future import ResponseFuture from pywren_ibm_cloud.libs.tblib import pickling_support from pywren_ibm_cloud.utils import get_current_memory_usage from pywren_ibm_cloud.storage.backends.cos import COSBackend from pywren_ibm_cloud.storage.backends.swift import SwiftBackend pickling_support.install() level = logging.INFO logger = logging.getLogger('jobrunner') logger.setLevel(level) wrenlogging.ow_config(level) class stats: def __init__(self, stats_filename): self.stats_filename = stats_filename self.stats_fid = open(stats_filename, 'w') def write(self, key, value): self.stats_fid.write("{} {}\n".format(key, value)) self.stats_fid.flush() def __del__(self): self.stats_fid.close()
from __future__ import print_function from pywren_ibm_cloud.wrenhandler import ibm_cloud_function_handler import logging from pywren_ibm_cloud import wrenlogging logger = logging.getLogger('pywren') wrenlogging.ow_config(logging.INFO) def main(args): logger.info("Starting IMB Cloud Function") ibm_cloud_function_handler(args) return {"greeting": "Finished"}
# # (C) Copyright IBM Corp. 2018 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import logging from pywren_ibm_cloud import wrenlogging from pywren_ibm_cloud.action.wrenhandler import ibm_cloud_function_handler logger = logging.getLogger('__main__') wrenlogging.ow_config(logging.DEBUG) def main(args): logger.info("Starting IBM Cloud Function execution") ibm_cloud_function_handler(args) return {"greeting": "Finished"}
def function_handler(event): start_time = time.time() logger.debug("Action handler started") response_status = {'exception': False} response_status['host_submit_time'] = event['host_submit_time'] response_status['start_time'] = start_time context_dict = { 'ibm_cf_request_id': os.environ.get("__OW_ACTIVATION_ID"), 'ibm_cf_python_version': os.environ.get("PYTHON_VERSION"), } config = event['config'] storage_config = wrenconfig.extract_storage_config(config) log_level = event['log_level'] wrenlogging.ow_config(log_level) call_id = event['call_id'] callgroup_id = event['callgroup_id'] executor_id = event['executor_id'] logger.info("Execution ID: {}/{}/{}".format(executor_id, callgroup_id, call_id)) job_max_runtime = event.get("job_max_runtime", 590) # default for CF status_key = event['status_key'] func_key = event['func_key'] data_key = event['data_key'] data_byte_range = event['data_byte_range'] output_key = event['output_key'] extra_env = event.get('extra_env', {}) response_status['call_id'] = call_id response_status['callgroup_id'] = callgroup_id response_status['executor_id'] = executor_id # response_status['func_key'] = func_key # response_status['data_key'] = data_key # response_status['output_key'] = output_key # response_status['status_key'] = status_key try: if version.__version__ != event['pywren_version']: raise Exception("WRONGVERSION", "PyWren version mismatch", version.__version__, event['pywren_version']) # response_status['free_disk_bytes'] = free_disk_space("/tmp") custom_env = {'PYWREN_CONFIG': json.dumps(config), 'PYWREN_EXECUTOR_ID': executor_id, 'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH), 'PYTHONUNBUFFERED': 'True'} os.environ.update(custom_env) os.environ.update(extra_env) # pass a full json blob jobrunner_config = {'func_key': func_key, 'data_key': data_key, 'log_level': log_level, 'data_byte_range': data_byte_range, 'python_module_path': PYTHON_MODULE_PATH, 'output_key': output_key, 'stats_filename': JOBRUNNER_STATS_FILENAME} if os.path.exists(JOBRUNNER_STATS_FILENAME): os.remove(JOBRUNNER_STATS_FILENAME) setup_time = time.time() response_status['setup_time'] = round(setup_time - start_time, 8) result_queue = multiprocessing.Queue() jr = jobrunner(jobrunner_config, result_queue) jr.daemon = True logger.info("Starting jobrunner process") jr.start() jr.join(job_max_runtime) response_status['exec_time'] = round(time.time() - setup_time, 8) if jr.is_alive(): # If process is still alive after jr.join(job_max_runtime), kill it logger.error("Process exceeded maximum runtime of {} seconds".format(job_max_runtime)) # Send the signal to all the process groups jr.terminate() raise Exception("OUTATIME", "Process executed for too long and was killed") try: # Only 1 message is returned by jobrunner result_queue.get(block=False) except Exception: # If no message, this means that the process was killed due memory usage raise Exception("OUTOFMEMORY", "Process exceeded maximum memory and was killed") # print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True)) # print(subprocess.check_output("find {}".format(os.getcwd()), shell=True)) if os.path.exists(JOBRUNNER_STATS_FILENAME): with open(JOBRUNNER_STATS_FILENAME, 'r') as fid: for l in fid.readlines(): key, value = l.strip().split(" ", 1) try: response_status[key] = float(value) except Exception: response_status[key] = value if key == 'exception' or key == 'exc_pickle_fail' \ or key == 'result': response_status[key] = eval(value) # response_status['server_info'] = get_server_info() response_status.update(context_dict) response_status['end_time'] = time.time() except Exception as e: # internal runtime exceptions logger.error("There was an exception: {}".format(str(e))) response_status['end_time'] = time.time() response_status['exception'] = True pickled_exc = pickle.dumps(sys.exc_info()) pickle.loads(pickled_exc) # this is just to make sure they can be unpickled response_status['exc_info'] = str(pickled_exc) finally: store_status = strtobool(os.environ.get('STORE_STATUS', 'True')) rabbit_amqp_url = config['rabbitmq'].get('amqp_url') dmpd_response_status = json.dumps(response_status) drs = sizeof_fmt(len(dmpd_response_status)) if rabbit_amqp_url and store_status: status_sent = False output_query_count = 0 while not status_sent and output_query_count < 5: output_query_count = output_query_count + 1 try: params = pika.URLParameters(rabbit_amqp_url) connection = pika.BlockingConnection(params) channel = connection.channel() channel.queue_declare(queue=executor_id, auto_delete=True) channel.basic_publish(exchange='', routing_key=executor_id, body=dmpd_response_status) connection.close() logger.info("Execution stats sent to rabbitmq - Size: {}".format(drs)) status_sent = True except Exception as e: logger.error("Unable to send status to rabbitmq") logger.error(str(e)) logger.info('Retrying to send stats to rabbitmq...') time.sleep(0.2) if store_status: internal_storage = storage.InternalStorage(storage_config) logger.info("Storing execution stats - status.json - Size: {}".format(drs)) internal_storage.put_data(status_key, dmpd_response_status)