def create_blackbox_runtime(image_name, config=None, pywren_location=None):
# Create runtime_name from image_name
username, appname = image_name.split('/')
runtime_name = appname.replace(':', '_')
# Load PyWren config from ~/.pywren_config
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
# Upload zipped PyWren action
if pywren_location is None:
zip_location = "ibmcf_pywren.zip"
else:
zip_location = os.path.join(pywren_location, "runtime",
"ibmcf_pywren.zip")
with open(zip_location, "rb") as action_zip:
action_bin = action_zip.read()
cf_client = CloudFunctions(config['ibm_cf'])
cf_client.create_action(runtime_name,
code=action_bin,
kind='blackbox',
image=image_name)
def delete_runtime(image_name, config=None):
logger.info('Deleting runtime: {}'.format(image_name))
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
storage_config = wrenconfig.extract_storage_config(config)
storage_client = storage.InternalStorage(storage_config)
cf_config = wrenconfig.extract_cf_config(config)
cf_client = CloudFunctions(cf_config)
if image_name == 'default':
image_name = _get_default_image_name()
image_name_formated = create_action_name(image_name)
actions = cf_client.list_actions(PACKAGE)
region = cf_client.endpoint.split('//')[1].split('.')[0]
namespace = cf_client.namespace
for action in actions:
action_name, memory = action['name'].rsplit('-', 1)
if image_name_formated == action_name:
memory = int(memory.replace('MB', ''))
runtime_name = create_runtime_name(image_name, memory)
storage_client.delete_runtime_info(region, namespace, runtime_name)
action_name = create_action_name(runtime_name)
cf_client.delete_action(action_name)
def __init__(self,
config=None,
runtime=None,
log_level=None,
runtime_timeout=wrenconfig.CF_RUNTIME_TIMEOUT):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_timeout: Max time per action. Default 600
:return `executor` object.
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
"""
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
if runtime:
self.config['ibm_cf']['action_name'] = runtime
if log_level:
wrenlogging.default_config(log_level)
ibm_cf_config = self.config['ibm_cf']
self.runtime = ibm_cf_config['action_name']
self.cf_cluster = ibm_cf_config['is_cf_cluster']
self.data_cleaner = self.config['pywren']['data_cleaner']
retry_config = {}
retry_config['invocation_retry'] = self.config['pywren'][
'invocation_retry']
retry_config['retry_sleeps'] = self.config['pywren']['retry_sleeps']
retry_config['retries'] = self.config['pywren']['retries']
invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config,
retry_config)
self.storage_config = wrenconfig.extract_storage_config(self.config)
self.internal_storage = storage.InternalStorage(self.storage_config)
self.executor = Executor(invoker, self.config, self.internal_storage,
runtime_timeout)
self.executor_id = self.executor.executor_id
self.futures = []
self.reduce_future = None
def __init__(self, config=None, runtime=None, log_level=None, job_max_runtime=JOB_MAX_RUNTIME):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param job_max_runtime: Max time per lambda. Default 300
:return `executor` object.
Usage
>>> import pywren
>>> pw = pywren.ibm_cf_executor()
"""
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
if runtime:
self.config['ibm_cf']['action_name'] = runtime
if log_level:
wrenlogging.default_config(log_level)
self._openwhisk = False
if any([k.startswith('__OW_') for k in os.environ.keys()]):
# OpenWhisk execution
self._openwhisk = True
wrenlogging.ow_config(logging.INFO)
self.runtime = self.config['ibm_cf']['action_name']
ibm_cf_config = self.config['ibm_cf']
invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config)
self.storage_config = wrenconfig.extract_storage_config(self.config)
self.storage_handler = storage.Storage(self.storage_config)
self.executor = Executor(invoker, self.config, self.storage_handler, job_max_runtime)
self.executor_id = self.executor.executor_id
self.futures = None
self.reduce_future = None
log_msg='IBM Cloud Functions executor created with ID {}'.format(self.executor_id)
logger.info(log_msg)
if(logger.getEffectiveLevel() == logging.WARNING):
print(log_msg)
def create_runtime(image_name, memory=None, config=None):
logger.info(
'Creating new PyWren runtime based on image {}'.format(image_name))
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
cf_config = wrenconfig.extract_cf_config(config)
cf_client = CloudFunctions(cf_config)
cf_client.create_package(PACKAGE)
_create_zip_action()
if image_name == 'default':
image_name = _get_default_image_name()
if not memory:
# if not memory, this means that the method was called from deploy_runtime script
for memory in [
wrenconfig.RUNTIME_MEMORY_DEFAULT,
wrenconfig.RUNTIME_RI_MEMORY_DEFAULT
]:
_extract_modules(image_name, memory, cf_client, config)
_create_blackbox_runtime(image_name, memory, cf_client)
else:
ri_runtime_deployed = False
image_name_formated = create_action_name(image_name)
actions = cf_client.list_actions(PACKAGE)
for action in actions:
action_name, r_memory = action['name'].rsplit('-', 1)
if image_name_formated == action_name:
r_memory = int(r_memory.replace('MB', ''))
if r_memory == wrenconfig.RUNTIME_RI_MEMORY_DEFAULT:
ri_runtime_deployed = True
break
if not ri_runtime_deployed:
_extract_modules(image_name, wrenconfig.RUNTIME_RI_MEMORY_DEFAULT,
cf_client, config)
_create_blackbox_runtime(image_name,
wrenconfig.RUNTIME_RI_MEMORY_DEFAULT,
cf_client)
_extract_modules(image_name, memory, cf_client, config)
_create_blackbox_runtime(image_name, memory, cf_client)
def default(config=None, pywren_location=None):
print('Updating runtime {}'.format(CF_ACTION_NAME_DEFAULT))
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
# Create zipped PyWren action
create_zip_action(pywren_location)
# Upload zipped PyWren action
print('Uploading action')
if pywren_location is None:
zip_location = "ibmcf_pywren.zip"
else:
zip_location = os.path.join(pywren_location, "runtime",
"ibmcf_pywren.zip")
with open(zip_location, "rb") as action_zip:
action_bin = action_zip.read()
cf_client = CloudFunctions(config['ibm_cf'])
runtime_name = CF_ACTION_NAME_DEFAULT
cf_client.create_action(runtime_name, code=action_bin)
def extract_modules(image_name, config=None, pywren_location=None):
# Extract installed Python modules from docker image
# And store them into storage
# Create runtime_name from image_name
username, appname = image_name.split('/')
runtime_name = appname.replace(':', '_')
# Load PyWren config from ~/.pywren_config
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
# Create storage_handler to upload modules file
storage_config = wrenconfig.extract_storage_config(config)
internal_storage = storage.InternalStorage(storage_config)
# sys.stdout = open(os.devnull, 'w')
if pywren_location is None:
action_location = "extract_modules.py"
else:
action_location = os.path.join(pywren_location, "runtime",
"extract_modules.py")
with open(action_location, "r") as action_py:
action_code = action_py.read()
cf_client = CloudFunctions(config['ibm_cf'])
action_name = runtime_name + '_modules'
cf_client.create_action(action_name,
code=action_code,
kind='blackbox',
image=image_name,
is_binary=False)
runtime_meta = cf_client.invoke_with_result(action_name)
internal_storage.put_runtime_info(runtime_name, runtime_meta)
cf_client.delete_action(action_name)
def update_runtime(image_name, config=None):
logger.info('Updating runtime: {}'.format(image_name))
if config is None:
config = wrenconfig.default()
else:
config = wrenconfig.default(config)
cf_config = wrenconfig.extract_cf_config(config)
cf_client = CloudFunctions(cf_config)
cf_client.create_package(PACKAGE)
_create_zip_action()
if image_name == 'default':
image_name = _get_default_image_name()
image_name_formated = create_action_name(image_name)
actions = cf_client.list_actions(PACKAGE)
for action in actions:
action_name, memory = action['name'].rsplit('-', 1)
if image_name_formated == action_name:
memory = int(memory.replace('MB', ''))
_extract_modules(image_name, memory, cf_client, config)
_create_blackbox_runtime(image_name, memory, cf_client)
def run(config=None):
global CONFIG
global STORAGE_CONFIG
global STORAGE
if config is None:
CONFIG = wrenconfig.default()
else:
CONFIG = wrenconfig.default(config)
STORAGE_CONFIG = wrenconfig.extract_storage_config(CONFIG)
STORAGE = cos.COSBackend(STORAGE_CONFIG['ibm_cos'])
if len(sys.argv) <= 1:
task = 'full'
else:
task = sys.argv[1]
suite = unittest.TestSuite()
if task == 'pywren':
suite.addTest(unittest.makeSuite(TestPywren))
elif task == 'pywren_cos':
suite.addTest(unittest.makeSuite(TestPywrenCos))
elif task == 'full':
suite.addTest(unittest.makeSuite(TestPywren))
suite.addTest(unittest.makeSuite(TestPywrenCos))
elif task == 'test_call_async':
suite.addTest(TestPywren('test_call_async'))
elif task == 'test_map':
suite.addTest(TestPywren('test_map'))
elif task == 'test_map_reduce':
suite.addTest(TestPywren('test_map_reduce'))
elif task == 'test_multiple_executions':
suite.addTest(TestPywren('test_multiple_executions'))
elif task == 'test_map_reduce_cos_bucket':
suite.addTest(TestPywrenCos('test_map_reduce_cos_bucket'))
elif task == 'test_map_reduce_cos_bucket_one_reducer_per_object':
suite.addTest(
TestPywrenCos('test_map_reduce_cos_bucket_one_reducer_per_object'))
elif task == 'test_map_reduce_cos_key':
suite.addTest(TestPywrenCos('test_map_reduce_cos_key'))
elif task == 'test_map_reduce_cos_key_one_reducer_per_object':
suite.addTest(
TestPywrenCos('test_map_reduce_cos_key_one_reducer_per_object'))
elif task == 'test_map_reduce_url':
suite.addTest(TestPywrenCos('test_map_reduce_url'))
elif task == 'test_storage_handler':
suite.addTest(TestPywrenCos('test_storage_handler'))
elif task == 'test_chunks_bucket':
suite.addTest(TestPywrenCos('test_chunks_bucket'))
elif task == 'test_chunks_bucket_one_reducer_per_object':
suite.addTest(
TestPywrenCos('test_chunks_bucket_one_reducer_per_object'))
else:
print(
'Unknown Command... use: "init", "pywren", "pywren_cos", "clean" or a test function name.'
)
sys.exit()
initTests()
runner = unittest.TextTestRunner()
runner.run(suite)
cleanTests()
def __init__(self,
config=None,
runtime=None,
runtime_memory=None,
log_level=None,
rabbitmq_monitor=False):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime
:param log_level: log level to use during the execution
:param rabbitmq_monitor: use rabbitmq as monitoring system
:return `executor` object.
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
"""
self.start_time = time.time()
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
self.is_cf_cluster = is_cf_cluster()
self.data_cleaner = self.config['pywren']['data_cleaner']
# Overwrite runtime variables
if runtime:
self.config['pywren']['runtime'] = runtime
if runtime_memory:
self.config['pywren']['runtime_memory'] = int(runtime_memory)
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if (logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(
logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOG_LEVEL"] = self.log_level
if not self.is_cf_cluster:
wrenlogging.default_config(self.log_level)
# RabbitMQ monitor configuration
self.rabbitmq_monitor = rabbitmq_monitor
if self.rabbitmq_monitor:
if self.config['rabbitmq']['amqp_url']:
os.environ["PYWREN_RABBITMQ_MONITOR"] = 'True'
else:
self.rabbitmq_monitor = False
else:
self.config['rabbitmq']['amqp_url'] = None
storage_config = wrenconfig.extract_storage_config(self.config)
self.internal_storage = storage.InternalStorage(storage_config)
invoker = invokers.IBMCloudFunctionsInvoker(self.config)
self.executor = Executor(invoker, self.config, self.internal_storage)
self.executor_id = self.executor.executor_id
self.futures = []