def __init__(self, config=None, runtime=None, runtime_memory=None, compute_backend=None,
compute_backend_region=None, log_level=None, rabbitmq_monitor=False):
"""
Initialize and return a ServerlessExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime
:param log_level: log level to use during the execution
:param rabbitmq_monitor: use rabbitmq as monitoring system
:return `ServerlessExecutor` object.
"""
self.start_time = time.time()
self._state = ExecutorState.new
self.config = default_config(config)
self.is_cf_cluster = is_cf_cluster()
self.data_cleaner = self.config['pywren']['data_cleaner']
# Overwrite runtime variables
if runtime:
self.config['pywren']['runtime'] = runtime
if runtime_memory:
self.config['pywren']['runtime_memory'] = int(runtime_memory)
if compute_backend:
self.config['pywren']['compute_backend'] = compute_backend
if compute_backend_region:
self.config['pywren']['compute_backend_region'] = compute_backend_region
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if(logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(logger.getEffectiveLevel())
if self.log_level:
os.environ["CB_LOG_LEVEL"] = self.log_level
if not self.is_cf_cluster:
default_logging_config(self.log_level)
if 'CB_EXECUTOR_ID' in os.environ:
self.executor_id = os.environ['CB_EXECUTOR_ID']
else:
self.executor_id = create_executor_id()
logger.debug('ServerlessExecutor created with ID: {}'.format(self.executor_id))
# RabbitMQ monitor configuration
self.rabbitmq_monitor = rabbitmq_monitor
if self.rabbitmq_monitor:
if self.config['rabbitmq']['amqp_url']:
os.environ["CB_RABBITMQ_MONITOR"] = 'True'
else:
self.rabbitmq_monitor = False
else:
self.config['rabbitmq']['amqp_url'] = None
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = Invoker(self.config, self.executor_id)
self.jobs = {}
def __init__(self, ibm_cf_config):
self.log_level = os.getenv('CB_LOG_LEVEL')
self.name = 'ibm_cf'
self.ibm_cf_config = ibm_cf_config
self.package = 'pywren_v' + __version__
self.region = ibm_cf_config['region']
self.cf_client = CloudFunctionsClient(self.ibm_cf_config)
self.is_cf_cluster = is_cf_cluster()
self.namespace = ibm_cf_config[self.region]['namespace']
log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} '
'- Region: {}'.format(__version__, self.namespace,
self.region))
logger.info(log_msg)
if not self.log_level:
print(log_msg)
def __init__(self, config):
"""
Constructor
"""
self.endpoint = config['endpoint'].replace('http:', 'https:')
self.namespace = config['namespace']
self.default_runtime_memory = int(config['runtime_memory'])
self.default_runtime_timeout = int(config['runtime_timeout'])
self.is_cf_cluster = is_cf_cluster()
self.package = 'pywren_v'+__version__
self.iam_connector = IAM(config['ibm_iam'], self.endpoint, self.namespace)
if not self.iam_connector.is_IAM_access():
self.api_key = str.encode(config['api_key'])
if not self.iam_connector.is_IAM_access():
auth_token = base64.encodebytes(self.api_key).replace(b'\n', b'')
auth = 'Basic %s' % auth_token.decode('UTF-8')
self.effective_namespace = self.namespace
else:
auth = self.iam_connector.get_iam_token()
self.namespace_id = self.iam_connector.get_function_namespace_id(auth)
self.effective_namespace = self.namespace_id
self.session = requests.session()
default_user_agent = self.session.headers['User-Agent']
self.headers = {
'content-type': 'application/json',
'Authorization': auth,
'User-Agent': default_user_agent + ' pywren-ibm-cloud/{}'.format(__version__)
}
self.session.headers.update(self.headers)
adapter = requests.adapters.HTTPAdapter()
self.session.mount('https://', adapter)
msg = 'IBM Cloud Functions init for'
logger.debug('{} namespace: {}'.format(msg, self.namespace))
logger.debug('{} host: {}'.format(msg, self.endpoint))
logger.debug("CF user agent set to: {}".format(self.session.headers['User-Agent']))
def __init__(self,
config=None,
runtime=None,
runtime_memory=None,
log_level=None,
rabbitmq_monitor=False):
"""
Initialize and return an executor class.
:param config: Settings passed in here will override those in `pywren_config`. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime
:param log_level: log level to use during the execution
:param rabbitmq_monitor: use rabbitmq as monitoring system
:return `executor` object.
Usage
>>> import pywren_ibm_cloud as pywren
>>> pw = pywren.ibm_cf_executor()
"""
self.start_time = time.time()
self._state = ExecutorState.new
if config is None:
self.config = wrenconfig.default()
else:
self.config = wrenconfig.default(config)
self.is_cf_cluster = is_cf_cluster()
self.data_cleaner = self.config['pywren']['data_cleaner']
# Overwrite runtime variables
if runtime:
self.config['pywren']['runtime'] = runtime
if runtime_memory:
self.config['pywren']['runtime_memory'] = int(runtime_memory)
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if (logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(
logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOG_LEVEL"] = self.log_level
if not self.is_cf_cluster:
wrenlogging.default_config(self.log_level)
# RabbitMQ monitor configuration
self.rabbitmq_monitor = rabbitmq_monitor
if self.rabbitmq_monitor:
if self.config['rabbitmq']['amqp_url']:
os.environ["PYWREN_RABBITMQ_MONITOR"] = 'True'
else:
self.rabbitmq_monitor = False
else:
self.config['rabbitmq']['amqp_url'] = None
storage_config = wrenconfig.extract_storage_config(self.config)
self.internal_storage = storage.InternalStorage(storage_config)
invoker = invokers.IBMCloudFunctionsInvoker(self.config)
self.executor = Executor(invoker, self.config, self.internal_storage)
self.executor_id = self.executor.executor_id
self.futures = []