def create_blackbox_runtime(image_name, config=None, pywren_location=None): # Create runtime_name from image_name username, appname = image_name.split('/') runtime_name = appname.replace(':', '_') # Load PyWren config from ~/.pywren_config if config is None: config = wrenconfig.default() else: config = wrenconfig.default(config) # Upload zipped PyWren action if pywren_location is None: zip_location = "ibmcf_pywren.zip" else: zip_location = os.path.join(pywren_location, "runtime", "ibmcf_pywren.zip") with open(zip_location, "rb") as action_zip: action_bin = action_zip.read() cf_client = CloudFunctions(config['ibm_cf']) cf_client.create_action(runtime_name, code=action_bin, kind='blackbox', image=image_name)
def delete_runtime(image_name, config=None): logger.info('Deleting runtime: {}'.format(image_name)) if config is None: config = wrenconfig.default() else: config = wrenconfig.default(config) storage_config = wrenconfig.extract_storage_config(config) storage_client = storage.InternalStorage(storage_config) cf_config = wrenconfig.extract_cf_config(config) cf_client = CloudFunctions(cf_config) if image_name == 'default': image_name = _get_default_image_name() image_name_formated = create_action_name(image_name) actions = cf_client.list_actions(PACKAGE) region = cf_client.endpoint.split('//')[1].split('.')[0] namespace = cf_client.namespace for action in actions: action_name, memory = action['name'].rsplit('-', 1) if image_name_formated == action_name: memory = int(memory.replace('MB', '')) runtime_name = create_runtime_name(image_name, memory) storage_client.delete_runtime_info(region, namespace, runtime_name) action_name = create_action_name(runtime_name) cf_client.delete_action(action_name)
def __init__(self, config=None, runtime=None, log_level=None, runtime_timeout=wrenconfig.CF_RUNTIME_TIMEOUT): """ Initialize and return an executor class. :param config: Settings passed in here will override those in `pywren_config`. Default None. :param runtime: Runtime name to use. Default None. :param runtime_timeout: Max time per action. Default 600 :return `executor` object. Usage >>> import pywren_ibm_cloud as pywren >>> pw = pywren.ibm_cf_executor() """ self._state = ExecutorState.new if config is None: self.config = wrenconfig.default() else: self.config = wrenconfig.default(config) if runtime: self.config['ibm_cf']['action_name'] = runtime if log_level: wrenlogging.default_config(log_level) ibm_cf_config = self.config['ibm_cf'] self.runtime = ibm_cf_config['action_name'] self.cf_cluster = ibm_cf_config['is_cf_cluster'] self.data_cleaner = self.config['pywren']['data_cleaner'] retry_config = {} retry_config['invocation_retry'] = self.config['pywren'][ 'invocation_retry'] retry_config['retry_sleeps'] = self.config['pywren']['retry_sleeps'] retry_config['retries'] = self.config['pywren']['retries'] invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config, retry_config) self.storage_config = wrenconfig.extract_storage_config(self.config) self.internal_storage = storage.InternalStorage(self.storage_config) self.executor = Executor(invoker, self.config, self.internal_storage, runtime_timeout) self.executor_id = self.executor.executor_id self.futures = [] self.reduce_future = None
def __init__(self, config=None, runtime=None, log_level=None, job_max_runtime=JOB_MAX_RUNTIME): """ Initialize and return an executor class. :param config: Settings passed in here will override those in `pywren_config`. Default None. :param runtime: Runtime name to use. Default None. :param job_max_runtime: Max time per lambda. Default 300 :return `executor` object. Usage >>> import pywren >>> pw = pywren.ibm_cf_executor() """ self._state = ExecutorState.new if config is None: self.config = wrenconfig.default() else: self.config = wrenconfig.default(config) if runtime: self.config['ibm_cf']['action_name'] = runtime if log_level: wrenlogging.default_config(log_level) self._openwhisk = False if any([k.startswith('__OW_') for k in os.environ.keys()]): # OpenWhisk execution self._openwhisk = True wrenlogging.ow_config(logging.INFO) self.runtime = self.config['ibm_cf']['action_name'] ibm_cf_config = self.config['ibm_cf'] invoker = invokers.IBMCloudFunctionsInvoker(ibm_cf_config) self.storage_config = wrenconfig.extract_storage_config(self.config) self.storage_handler = storage.Storage(self.storage_config) self.executor = Executor(invoker, self.config, self.storage_handler, job_max_runtime) self.executor_id = self.executor.executor_id self.futures = None self.reduce_future = None log_msg='IBM Cloud Functions executor created with ID {}'.format(self.executor_id) logger.info(log_msg) if(logger.getEffectiveLevel() == logging.WARNING): print(log_msg)
def create_runtime(image_name, memory=None, config=None): logger.info( 'Creating new PyWren runtime based on image {}'.format(image_name)) if config is None: config = wrenconfig.default() else: config = wrenconfig.default(config) cf_config = wrenconfig.extract_cf_config(config) cf_client = CloudFunctions(cf_config) cf_client.create_package(PACKAGE) _create_zip_action() if image_name == 'default': image_name = _get_default_image_name() if not memory: # if not memory, this means that the method was called from deploy_runtime script for memory in [ wrenconfig.RUNTIME_MEMORY_DEFAULT, wrenconfig.RUNTIME_RI_MEMORY_DEFAULT ]: _extract_modules(image_name, memory, cf_client, config) _create_blackbox_runtime(image_name, memory, cf_client) else: ri_runtime_deployed = False image_name_formated = create_action_name(image_name) actions = cf_client.list_actions(PACKAGE) for action in actions: action_name, r_memory = action['name'].rsplit('-', 1) if image_name_formated == action_name: r_memory = int(r_memory.replace('MB', '')) if r_memory == wrenconfig.RUNTIME_RI_MEMORY_DEFAULT: ri_runtime_deployed = True break if not ri_runtime_deployed: _extract_modules(image_name, wrenconfig.RUNTIME_RI_MEMORY_DEFAULT, cf_client, config) _create_blackbox_runtime(image_name, wrenconfig.RUNTIME_RI_MEMORY_DEFAULT, cf_client) _extract_modules(image_name, memory, cf_client, config) _create_blackbox_runtime(image_name, memory, cf_client)
def default(config=None, pywren_location=None): print('Updating runtime {}'.format(CF_ACTION_NAME_DEFAULT)) if config is None: config = wrenconfig.default() else: config = wrenconfig.default(config) # Create zipped PyWren action create_zip_action(pywren_location) # Upload zipped PyWren action print('Uploading action') if pywren_location is None: zip_location = "ibmcf_pywren.zip" else: zip_location = os.path.join(pywren_location, "runtime", "ibmcf_pywren.zip") with open(zip_location, "rb") as action_zip: action_bin = action_zip.read() cf_client = CloudFunctions(config['ibm_cf']) runtime_name = CF_ACTION_NAME_DEFAULT cf_client.create_action(runtime_name, code=action_bin)
def extract_modules(image_name, config=None, pywren_location=None): # Extract installed Python modules from docker image # And store them into storage # Create runtime_name from image_name username, appname = image_name.split('/') runtime_name = appname.replace(':', '_') # Load PyWren config from ~/.pywren_config if config is None: config = wrenconfig.default() else: config = wrenconfig.default(config) # Create storage_handler to upload modules file storage_config = wrenconfig.extract_storage_config(config) internal_storage = storage.InternalStorage(storage_config) # sys.stdout = open(os.devnull, 'w') if pywren_location is None: action_location = "extract_modules.py" else: action_location = os.path.join(pywren_location, "runtime", "extract_modules.py") with open(action_location, "r") as action_py: action_code = action_py.read() cf_client = CloudFunctions(config['ibm_cf']) action_name = runtime_name + '_modules' cf_client.create_action(action_name, code=action_code, kind='blackbox', image=image_name, is_binary=False) runtime_meta = cf_client.invoke_with_result(action_name) internal_storage.put_runtime_info(runtime_name, runtime_meta) cf_client.delete_action(action_name)
def update_runtime(image_name, config=None): logger.info('Updating runtime: {}'.format(image_name)) if config is None: config = wrenconfig.default() else: config = wrenconfig.default(config) cf_config = wrenconfig.extract_cf_config(config) cf_client = CloudFunctions(cf_config) cf_client.create_package(PACKAGE) _create_zip_action() if image_name == 'default': image_name = _get_default_image_name() image_name_formated = create_action_name(image_name) actions = cf_client.list_actions(PACKAGE) for action in actions: action_name, memory = action['name'].rsplit('-', 1) if image_name_formated == action_name: memory = int(memory.replace('MB', '')) _extract_modules(image_name, memory, cf_client, config) _create_blackbox_runtime(image_name, memory, cf_client)
def run(config=None): global CONFIG global STORAGE_CONFIG global STORAGE if config is None: CONFIG = wrenconfig.default() else: CONFIG = wrenconfig.default(config) STORAGE_CONFIG = wrenconfig.extract_storage_config(CONFIG) STORAGE = cos.COSBackend(STORAGE_CONFIG['ibm_cos']) if len(sys.argv) <= 1: task = 'full' else: task = sys.argv[1] suite = unittest.TestSuite() if task == 'pywren': suite.addTest(unittest.makeSuite(TestPywren)) elif task == 'pywren_cos': suite.addTest(unittest.makeSuite(TestPywrenCos)) elif task == 'full': suite.addTest(unittest.makeSuite(TestPywren)) suite.addTest(unittest.makeSuite(TestPywrenCos)) elif task == 'test_call_async': suite.addTest(TestPywren('test_call_async')) elif task == 'test_map': suite.addTest(TestPywren('test_map')) elif task == 'test_map_reduce': suite.addTest(TestPywren('test_map_reduce')) elif task == 'test_multiple_executions': suite.addTest(TestPywren('test_multiple_executions')) elif task == 'test_map_reduce_cos_bucket': suite.addTest(TestPywrenCos('test_map_reduce_cos_bucket')) elif task == 'test_map_reduce_cos_bucket_one_reducer_per_object': suite.addTest( TestPywrenCos('test_map_reduce_cos_bucket_one_reducer_per_object')) elif task == 'test_map_reduce_cos_key': suite.addTest(TestPywrenCos('test_map_reduce_cos_key')) elif task == 'test_map_reduce_cos_key_one_reducer_per_object': suite.addTest( TestPywrenCos('test_map_reduce_cos_key_one_reducer_per_object')) elif task == 'test_map_reduce_url': suite.addTest(TestPywrenCos('test_map_reduce_url')) elif task == 'test_storage_handler': suite.addTest(TestPywrenCos('test_storage_handler')) elif task == 'test_chunks_bucket': suite.addTest(TestPywrenCos('test_chunks_bucket')) elif task == 'test_chunks_bucket_one_reducer_per_object': suite.addTest( TestPywrenCos('test_chunks_bucket_one_reducer_per_object')) else: print( 'Unknown Command... use: "init", "pywren", "pywren_cos", "clean" or a test function name.' ) sys.exit() initTests() runner = unittest.TextTestRunner() runner.run(suite) cleanTests()
def __init__(self, config=None, runtime=None, runtime_memory=None, log_level=None, rabbitmq_monitor=False): """ Initialize and return an executor class. :param config: Settings passed in here will override those in `pywren_config`. Default None. :param runtime: Runtime name to use. Default None. :param runtime_memory: memory to use in the runtime :param log_level: log level to use during the execution :param rabbitmq_monitor: use rabbitmq as monitoring system :return `executor` object. Usage >>> import pywren_ibm_cloud as pywren >>> pw = pywren.ibm_cf_executor() """ self.start_time = time.time() self._state = ExecutorState.new if config is None: self.config = wrenconfig.default() else: self.config = wrenconfig.default(config) self.is_cf_cluster = is_cf_cluster() self.data_cleaner = self.config['pywren']['data_cleaner'] # Overwrite runtime variables if runtime: self.config['pywren']['runtime'] = runtime if runtime_memory: self.config['pywren']['runtime_memory'] = int(runtime_memory) # Log level Configuration self.log_level = log_level if not self.log_level: if (logger.getEffectiveLevel() != logging.WARNING): self.log_level = logging.getLevelName( logger.getEffectiveLevel()) if self.log_level: os.environ["PYWREN_LOG_LEVEL"] = self.log_level if not self.is_cf_cluster: wrenlogging.default_config(self.log_level) # RabbitMQ monitor configuration self.rabbitmq_monitor = rabbitmq_monitor if self.rabbitmq_monitor: if self.config['rabbitmq']['amqp_url']: os.environ["PYWREN_RABBITMQ_MONITOR"] = 'True' else: self.rabbitmq_monitor = False else: self.config['rabbitmq']['amqp_url'] = None storage_config = wrenconfig.extract_storage_config(self.config) self.internal_storage = storage.InternalStorage(storage_config) invoker = invokers.IBMCloudFunctionsInvoker(self.config) self.executor = Executor(invoker, self.config, self.internal_storage) self.executor_id = self.executor.executor_id self.futures = []