def delete_runtime(image_name, config=None):
logger.info('Deleting runtime: {}'.format(image_name))
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
storage_config = wrenconfig.extract_storage_config(config)
storage_client = storage.InternalStorage(storage_config)
cf_config = wrenconfig.extract_cf_config(config)
cf_client = CloudFunctions(cf_config)
if image_name == 'default':
image_name = _get_default_image_name()
image_name_formated = create_action_name(image_name)
actions = cf_client.list_actions(PACKAGE)
region = cf_client.endpoint.split('//')[1].split('.')[0]
namespace = cf_client.namespace
for action in actions:
action_name, memory = action['name'].rsplit('-', 1)
if image_name_formated == action_name:
memory = int(memory.replace('MB', ''))
runtime_name = create_runtime_name(image_name, memory)
storage_client.delete_runtime_info(region, namespace, runtime_name)
action_name = create_action_name(runtime_name)
cf_client.delete_action(action_name)
def __init__(self,
config=None,
runtime=None,
log_level=None,
runtime_timeout=wrenconfig.CF_RUNTIME_TIMEOUT):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_timeout: Max time per action. Default 600
:return `executor` object.
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
"""
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
if runtime:
self.config['ibm_cf']['action_name'] = runtime
if log_level:
wrenlogging.default_config(log_level)
ibm_cf_config = self.config['ibm_cf']
self.runtime = ibm_cf_config['action_name']
self.cf_cluster = ibm_cf_config['is_cf_cluster']
self.data_cleaner = self.config['pywren']['data_cleaner']
retry_config = {}
retry_config['invocation_retry'] = self.config['pywren'][
'invocation_retry']
retry_config['retry_sleeps'] = self.config['pywren']['retry_sleeps']
retry_config['retries'] = self.config['pywren']['retries']
invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config,
retry_config)
self.storage_config = wrenconfig.extract_storage_config(self.config)
self.internal_storage = storage.InternalStorage(self.storage_config)
self.executor = Executor(invoker, self.config, self.internal_storage,
runtime_timeout)
self.executor_id = self.executor.executor_id
self.futures = []
self.reduce_future = None
def __init__(self, config=None, runtime=None, log_level=None, job_max_runtime=JOB_MAX_RUNTIME):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param job_max_runtime: Max time per lambda. Default 300
:return `executor` object.
Usage
>>> import pywren
>>> pw = pywren.ibm_cf_executor()
"""
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
if runtime:
self.config['ibm_cf']['action_name'] = runtime
if log_level:
wrenlogging.default_config(log_level)
self._openwhisk = False
if any([k.startswith('__OW_') for k in os.environ.keys()]):
# OpenWhisk execution
self._openwhisk = True
wrenlogging.ow_config(logging.INFO)
self.runtime = self.config['ibm_cf']['action_name']
ibm_cf_config = self.config['ibm_cf']
invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config)
self.storage_config = wrenconfig.extract_storage_config(self.config)
self.storage_handler = storage.Storage(self.storage_config)
self.executor = Executor(invoker, self.config, self.storage_handler, job_max_runtime)
self.executor_id = self.executor.executor_id
self.futures = None
self.reduce_future = None
log_msg='IBM Cloud Functions executor created with ID {}'.format(self.executor_id)
logger.info(log_msg)
if(logger.getEffectiveLevel() == logging.WARNING):
print(log_msg)
def __init__(self, jr_config, result_queue):
super().__init__()
start_time = time.time()
self.result_queue = result_queue
self.config = jr_config
self.stats = stats(self.config['stats_filename'])
self.stats.write('jobrunner_start', start_time)
pw_config = json.loads(os.environ.get('PYWREN_CONFIG', ''))
self.storage_config = extract_storage_config(pw_config)
if 'SHOW_MEMORY_USAGE' in os.environ:
self.show_memory = eval(os.environ['SHOW_MEMORY_USAGE'])
else:
self.show_memory = False
log_level = self.config['log_level']
wrenlogging.ow_config(log_level)
self.func_key = self.config['func_key']
self.data_key = self.config['data_key']
self.data_byte_range = self.config['data_byte_range']
self.output_key = self.config['output_key']
def extract_modules(image_name, config=None, pywren_location=None):
# Extract installed Python modules from docker image
# And store them into storage
# Create runtime_name from image_name
username, appname = image_name.split('/')
runtime_name = appname.replace(':', '_')
# Load PyWren config from ~/.pywren_config
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
# Create storage_handler to upload modules file
storage_config = wrenconfig.extract_storage_config(config)
internal_storage = storage.InternalStorage(storage_config)
# sys.stdout = open(os.devnull, 'w')
if pywren_location is None:
action_location = "extract_modules.py"
else:
action_location = os.path.join(pywren_location, "runtime",
"extract_modules.py")
with open(action_location, "r") as action_py:
action_code = action_py.read()
cf_client = CloudFunctions(config['ibm_cf'])
action_name = runtime_name + '_modules'
cf_client.create_action(action_name,
code=action_code,
kind='blackbox',
image=image_name,
is_binary=False)
runtime_meta = cf_client.invoke_with_result(action_name)
internal_storage.put_runtime_info(runtime_name, runtime_meta)
cf_client.delete_action(action_name)
def _extract_modules(image_name, memory, cf_client, config):
# Extract installed Python modules from docker image
# And store them into storage
# Create storage_handler to upload modules file
storage_config = wrenconfig.extract_storage_config(config)
internal_storage = storage.InternalStorage(storage_config)
pywren_location = _get_pywren_location()
action_location = os.path.join(pywren_location, "runtime",
"extract_modules.py")
with open(action_location, "r") as action_py:
action_code = action_py.read()
modules_action_name = '{}-modules'.format(create_action_name(image_name))
# old_stdout = sys.stdout
# sys.stdout = open(os.devnull, 'w')
logger.debug(
"Creating action for extracting Python modules list: {}".format(
modules_action_name))
cf_client.create_action(modules_action_name,
image_name,
code=action_code,
is_binary=False)
# sys.stdout = old_stdout
region = cf_client.endpoint.split('//')[1].split('.')[0]
namespace = cf_client.namespace
memory = cf_client.default_runtime_memory if not memory else memory
runtime_name = create_runtime_name(image_name, memory)
logger.debug(
"Going to extract Python modules list from: {}".format(image_name))
runtime_meta = cf_client.invoke_with_result(modules_action_name)
internal_storage.put_runtime_info(region, namespace, runtime_name,
runtime_meta)
cf_client.delete_action(modules_action_name)
def run(config=None):
global CONFIG
global STORAGE_CONFIG
global STORAGE
if config is None:
CONFIG = wrenconfig.default()
else:
CONFIG = wrenconfig.default(config)
STORAGE_CONFIG = wrenconfig.extract_storage_config(CONFIG)
STORAGE = cos.COSBackend(STORAGE_CONFIG['ibm_cos'])
if len(sys.argv) <= 1:
task = 'full'
else:
task = sys.argv[1]
suite = unittest.TestSuite()
if task == 'pywren':
suite.addTest(unittest.makeSuite(TestPywren))
elif task == 'pywren_cos':
suite.addTest(unittest.makeSuite(TestPywrenCos))
elif task == 'full':
suite.addTest(unittest.makeSuite(TestPywren))
suite.addTest(unittest.makeSuite(TestPywrenCos))
elif task == 'test_call_async':
suite.addTest(TestPywren('test_call_async'))
elif task == 'test_map':
suite.addTest(TestPywren('test_map'))
elif task == 'test_map_reduce':
suite.addTest(TestPywren('test_map_reduce'))
elif task == 'test_multiple_executions':
suite.addTest(TestPywren('test_multiple_executions'))
elif task == 'test_map_reduce_cos_bucket':
suite.addTest(TestPywrenCos('test_map_reduce_cos_bucket'))
elif task == 'test_map_reduce_cos_bucket_one_reducer_per_object':
suite.addTest(
TestPywrenCos('test_map_reduce_cos_bucket_one_reducer_per_object'))
elif task == 'test_map_reduce_cos_key':
suite.addTest(TestPywrenCos('test_map_reduce_cos_key'))
elif task == 'test_map_reduce_cos_key_one_reducer_per_object':
suite.addTest(
TestPywrenCos('test_map_reduce_cos_key_one_reducer_per_object'))
elif task == 'test_map_reduce_url':
suite.addTest(TestPywrenCos('test_map_reduce_url'))
elif task == 'test_storage_handler':
suite.addTest(TestPywrenCos('test_storage_handler'))
elif task == 'test_chunks_bucket':
suite.addTest(TestPywrenCos('test_chunks_bucket'))
elif task == 'test_chunks_bucket_one_reducer_per_object':
suite.addTest(
TestPywrenCos('test_chunks_bucket_one_reducer_per_object'))
else:
print(
'Unknown Command... use: "init", "pywren", "pywren_cos", "clean" or a test function name.'
)
sys.exit()
initTests()
runner = unittest.TextTestRunner()
runner.run(suite)
cleanTests()
def ibm_cloud_function_handler(event):
start_time = time.time()
logger.info("Starting handler")
response_status = {'exception': None}
response_status['start_time'] = start_time
context_dict = {
'ibm_cf_request_id': os.environ.get("__OW_ACTIVATION_ID"),
'ibm_cf_hostname': os.environ.get("HOSTNAME"),
'ibm_cf_python_version': os.environ.get("PYTHON_VERSION"),
}
config = event['config']
storage_config = wrenconfig.extract_storage_config(config)
custom_handler_env = {'PYWREN_CONFIG': json.dumps(config),
'STORAGE_CONFIG': json.dumps(storage_config),
'PYWREN_EXECUTOR_ID': event['executor_id']}
os.environ.update(custom_handler_env)
try:
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "Pywren version mismatch",
version.__version__, event['pywren_version'])
job_max_runtime = event.get("job_max_runtime", 550) # default for CF
response_status['func_key'] = func_key
response_status['data_key'] = data_key
response_status['output_key'] = output_key
response_status['status_key'] = status_key
#free_disk_bytes = free_disk_space("/tmp")
#response_status['free_disk_bytes'] = free_disk_bytes
extra_env = event.get('extra_env', {})
extra_env['PYTHONPATH'] = "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH)
extra_env['PYTHONUNBUFFERED'] = 'True'
call_id = event['call_id']
callgroup_id = event['callgroup_id']
executor_id = event['executor_id']
response_status['call_id'] = call_id
response_status['callgroup_id'] = callgroup_id
response_status['executor_id'] = executor_id
# pass a full json blob
jobrunner_config = {'func_key': func_key,
'data_key': data_key,
'data_byte_range': data_byte_range,
'python_module_path': PYTHON_MODULE_PATH,
'output_key': output_key,
'stats_filename': JOBRUNNER_STATS_FILENAME}
with open(JOBRUNNER_CONFIG_FILENAME, 'w') as jobrunner_fid:
json.dump(jobrunner_config, jobrunner_fid)
if os.path.exists(JOBRUNNER_STATS_FILENAME):
os.remove(JOBRUNNER_STATS_FILENAME)
cmdstr = "python {} {}".format(JOBRUNNER_PATH, JOBRUNNER_CONFIG_FILENAME)
logger.info("About to execute '{}'".format(cmdstr))
setup_time = time.time()
response_status['setup_time'] = setup_time - start_time
local_env = os.environ.copy()
local_env.update(extra_env)
"""
stdout = os.popen(cmdstr).read()
print(stdout)
process = subprocess.run(cmdstr, shell=True, env=local_env, bufsize=1,
stdout=subprocess.PIPE, preexec_fn=os.setsid,
universal_newlines=True, timeout=job_max_runtime)
print(process.stdout)
"""
# This is copied from http://stackoverflow.com/a/17698359/4577954
# reasons for setting process group: http://stackoverflow.com/a/4791612
process = subprocess.Popen(cmdstr, shell=True, env=local_env, bufsize=1,
stdout=subprocess.PIPE, preexec_fn=os.setsid,
universal_newlines=True)
logger.info("launched process")
def consume_stdout(stdout, queue):
with stdout:
for line in stdout:
print(line, end='')
queue.put(line)
q = Queue()
t = Thread(target=consume_stdout, args=(process.stdout, q))
t.daemon = True
t.start()
t.join(job_max_runtime)
if t.isAlive():
# If process is still alive after t.join(job_max_runtime), kill it
logger.error("Process exceeded maximum runtime of {} sec".format(job_max_runtime))
# Send the signal to all the process groups
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
raise Exception("OUTATIME", "Process executed for too long and was killed")
if not q.empty():
if 'Jobrunner finished' not in q.queue[q.qsize()-1].strip():
raise Exception("OUTOFMEMORY", "Process exceeded maximum memory and was killed")
logger.info("Command execution finished")
#print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
#print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ")
float_value = float(value)
response_status[key] = float_value
response_status['exec_time'] = time.time() - setup_time
response_status['host_submit_time'] = event['host_submit_time']
#response_status['server_info'] = get_server_info()
response_status.update(context_dict)
response_status['end_time'] = time.time()
except Exception as e:
# internal runtime exceptions
logger.error("There was an exception: {}".format(str(e)))
response_status['end_time'] = time.time()
response_status['exception'] = str(e)
response_status['exception_args'] = e.args
response_status['exception_traceback'] = traceback.format_exc()
finally:
store_status = True
if 'STORE_STATUS' in extra_env:
store_status = eval(extra_env['STORE_STATUS'])
if store_status:
internal_storage = storage.InternalStorage(storage_config)
internal_storage.put_data(status_key, json.dumps(response_status))
def __init__(self,
config=None,
runtime=None,
runtime_memory=None,
log_level=None,
rabbitmq_monitor=False):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime
:param log_level: log level to use during the execution
:param rabbitmq_monitor: use rabbitmq as monitoring system
:return `executor` object.
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
"""
self.start_time = time.time()
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
self.is_cf_cluster = is_cf_cluster()
self.data_cleaner = self.config['pywren']['data_cleaner']
# Overwrite runtime variables
if runtime:
self.config['pywren']['runtime'] = runtime
if runtime_memory:
self.config['pywren']['runtime_memory'] = int(runtime_memory)
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if (logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(
logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOG_LEVEL"] = self.log_level
if not self.is_cf_cluster:
wrenlogging.default_config(self.log_level)
# RabbitMQ monitor configuration
self.rabbitmq_monitor = rabbitmq_monitor
if self.rabbitmq_monitor:
if self.config['rabbitmq']['amqp_url']:
os.environ["PYWREN_RABBITMQ_MONITOR"] = 'True'
else:
self.rabbitmq_monitor = False
else:
self.config['rabbitmq']['amqp_url'] = None
storage_config = wrenconfig.extract_storage_config(self.config)
self.internal_storage = storage.InternalStorage(storage_config)
invoker = invokers.IBMCloudFunctionsInvoker(self.config)
self.executor = Executor(invoker, self.config, self.internal_storage)
self.executor_id = self.executor.executor_id
self.futures = []
def function_handler(event):
start_time = time.time()
logger.debug("Action handler started")
response_status = {'exception': False}
response_status['host_submit_time'] = event['host_submit_time']
response_status['start_time'] = start_time
context_dict = {
'ibm_cf_request_id': os.environ.get("__OW_ACTIVATION_ID"),
'ibm_cf_python_version': os.environ.get("PYTHON_VERSION"),
}
config = event['config']
storage_config = wrenconfig.extract_storage_config(config)
log_level = event['log_level']
wrenlogging.ow_config(log_level)
call_id = event['call_id']
callgroup_id = event['callgroup_id']
executor_id = event['executor_id']
logger.info("Execution ID: {}/{}/{}".format(executor_id, callgroup_id, call_id))
job_max_runtime = event.get("job_max_runtime", 590) # default for CF
status_key = event['status_key']
func_key = event['func_key']
data_key = event['data_key']
data_byte_range = event['data_byte_range']
output_key = event['output_key']
extra_env = event.get('extra_env', {})
response_status['call_id'] = call_id
response_status['callgroup_id'] = callgroup_id
response_status['executor_id'] = executor_id
# response_status['func_key'] = func_key
# response_status['data_key'] = data_key
# response_status['output_key'] = output_key
# response_status['status_key'] = status_key
try:
if version.__version__ != event['pywren_version']:
raise Exception("WRONGVERSION", "PyWren version mismatch",
version.__version__, event['pywren_version'])
# response_status['free_disk_bytes'] = free_disk_space("/tmp")
custom_env = {'PYWREN_CONFIG': json.dumps(config),
'PYWREN_EXECUTOR_ID': executor_id,
'PYTHONPATH': "{}:{}".format(os.getcwd(), PYWREN_LIBS_PATH),
'PYTHONUNBUFFERED': 'True'}
os.environ.update(custom_env)
os.environ.update(extra_env)
# pass a full json blob
jobrunner_config = {'func_key': func_key,
'data_key': data_key,
'log_level': log_level,
'data_byte_range': data_byte_range,
'python_module_path': PYTHON_MODULE_PATH,
'output_key': output_key,
'stats_filename': JOBRUNNER_STATS_FILENAME}
if os.path.exists(JOBRUNNER_STATS_FILENAME):
os.remove(JOBRUNNER_STATS_FILENAME)
setup_time = time.time()
response_status['setup_time'] = round(setup_time - start_time, 8)
result_queue = multiprocessing.Queue()
jr = jobrunner(jobrunner_config, result_queue)
jr.daemon = True
logger.info("Starting jobrunner process")
jr.start()
jr.join(job_max_runtime)
response_status['exec_time'] = round(time.time() - setup_time, 8)
if jr.is_alive():
# If process is still alive after jr.join(job_max_runtime), kill it
logger.error("Process exceeded maximum runtime of {} seconds".format(job_max_runtime))
# Send the signal to all the process groups
jr.terminate()
raise Exception("OUTATIME", "Process executed for too long and was killed")
try:
# Only 1 message is returned by jobrunner
result_queue.get(block=False)
except Exception:
# If no message, this means that the process was killed due memory usage
raise Exception("OUTOFMEMORY", "Process exceeded maximum memory and was killed")
# print(subprocess.check_output("find {}".format(PYTHON_MODULE_PATH), shell=True))
# print(subprocess.check_output("find {}".format(os.getcwd()), shell=True))
if os.path.exists(JOBRUNNER_STATS_FILENAME):
with open(JOBRUNNER_STATS_FILENAME, 'r') as fid:
for l in fid.readlines():
key, value = l.strip().split(" ", 1)
try:
response_status[key] = float(value)
except Exception:
response_status[key] = value
if key == 'exception' or key == 'exc_pickle_fail' \
or key == 'result':
response_status[key] = eval(value)
# response_status['server_info'] = get_server_info()
response_status.update(context_dict)
response_status['end_time'] = time.time()
except Exception as e:
# internal runtime exceptions
logger.error("There was an exception: {}".format(str(e)))
response_status['end_time'] = time.time()
response_status['exception'] = True
pickled_exc = pickle.dumps(sys.exc_info())
pickle.loads(pickled_exc) # this is just to make sure they can be unpickled
response_status['exc_info'] = str(pickled_exc)
finally:
store_status = strtobool(os.environ.get('STORE_STATUS', 'True'))
rabbit_amqp_url = config['rabbitmq'].get('amqp_url')
dmpd_response_status = json.dumps(response_status)
drs = sizeof_fmt(len(dmpd_response_status))
if rabbit_amqp_url and store_status:
status_sent = False
output_query_count = 0
while not status_sent and output_query_count < 5:
output_query_count = output_query_count + 1
try:
params = pika.URLParameters(rabbit_amqp_url)
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.queue_declare(queue=executor_id, auto_delete=True)
channel.basic_publish(exchange='', routing_key=executor_id,
body=dmpd_response_status)
connection.close()
logger.info("Execution stats sent to rabbitmq - Size: {}".format(drs))
status_sent = True
except Exception as e:
logger.error("Unable to send status to rabbitmq")
logger.error(str(e))
logger.info('Retrying to send stats to rabbitmq...')
time.sleep(0.2)
if store_status:
internal_storage = storage.InternalStorage(storage_config)
logger.info("Storing execution stats - status.json - Size: {}".format(drs))
internal_storage.put_data(status_key, dmpd_response_status)