def __init__(self, invoker, config, internal_storage):
self.log_level = os.getenv('PYWREN_LOG_LEVEL')
self.invoker = invoker
self.config = config
self.internal_storage = internal_storage
self.runtime_name = self.config['pywren']['runtime']
self.runtime_memory = self.config['pywren']['runtime_memory']
runtime_preinstalls = get_runtime_preinstalls(self.internal_storage,
self.runtime_name,
self.runtime_memory,
self.config)
self.serializer = serialize.SerializeIndependent(runtime_preinstalls)
self.map_item_limit = None
if 'scheduler' in self.config:
if 'map_item_limit' in config['scheduler']:
self.map_item_limit = config['scheduler']['map_item_limit']
if 'PYWREN_EXECUTOR_ID' in os.environ:
self.executor_id = os.environ['PYWREN_EXECUTOR_ID']
else:
self.executor_id = wrenutil.create_executor_id()
log_msg = 'IBM Cloud Functions executor created with ID {}'.format(self.executor_id)
logger.info(log_msg)
if not self.log_level:
print(log_msg)
def __init__(self, config=None, runtime=None, runtime_memory=None, compute_backend=None,
compute_backend_region=None, log_level=None, rabbitmq_monitor=False):
"""
Initialize and return a ServerlessExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime
:param log_level: log level to use during the execution
:param rabbitmq_monitor: use rabbitmq as monitoring system
:return `ServerlessExecutor` object.
"""
self.start_time = time.time()
self._state = ExecutorState.new
self.config = default_config(config)
self.is_cf_cluster = is_cf_cluster()
self.data_cleaner = self.config['pywren']['data_cleaner']
# Overwrite runtime variables
if runtime:
self.config['pywren']['runtime'] = runtime
if runtime_memory:
self.config['pywren']['runtime_memory'] = int(runtime_memory)
if compute_backend:
self.config['pywren']['compute_backend'] = compute_backend
if compute_backend_region:
self.config['pywren']['compute_backend_region'] = compute_backend_region
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if(logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(logger.getEffectiveLevel())
if self.log_level:
os.environ["CB_LOG_LEVEL"] = self.log_level
if not self.is_cf_cluster:
default_logging_config(self.log_level)
if 'CB_EXECUTOR_ID' in os.environ:
self.executor_id = os.environ['CB_EXECUTOR_ID']
else:
self.executor_id = create_executor_id()
logger.debug('ServerlessExecutor created with ID: {}'.format(self.executor_id))
# RabbitMQ monitor configuration
self.rabbitmq_monitor = rabbitmq_monitor
if self.rabbitmq_monitor:
if self.config['rabbitmq']['amqp_url']:
os.environ["CB_RABBITMQ_MONITOR"] = 'True'
else:
self.rabbitmq_monitor = False
else:
self.config['rabbitmq']['amqp_url'] = None
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = Invoker(self.config, self.executor_id)
self.jobs = {}
def __init__(self,
config=None,
runtime=None,
runtime_memory=None,
compute_backend=None,
compute_backend_region=None,
storage_backend=None,
storage_backend_region=None,
rabbitmq_monitor=None,
log_level=None):
"""
Initialize a FunctionExecutor class.
:param config: Settings passed in here will override those in config file. Default None.
:param runtime: Runtime name to use. Default None.
:param runtime_memory: memory to use in the runtime. Default None.
:param compute_backend: Name of the compute backend to use. Default None.
:param compute_backend_region: Name of the compute backend region to use. Default None.
:param storage_backend: Name of the storage backend to use. Default None.
:param storage_backend_region: Name of the storage backend region to use. Default None.
:param log_level: log level to use during the execution. Default None.
:param rabbitmq_monitor: use rabbitmq as the monitoring system. Default None.
:return `FunctionExecutor` object.
"""
self.start_time = time.time()
self._state = FunctionExecutor.State.New
self.is_remote_cluster = is_remote_cluster()
# Log level Configuration
self.log_level = log_level
if not self.log_level:
if (logger.getEffectiveLevel() != logging.WARNING):
self.log_level = logging.getLevelName(
logger.getEffectiveLevel())
if self.log_level:
os.environ["PYWREN_LOGLEVEL"] = self.log_level
if not self.is_remote_cluster:
default_logging_config(self.log_level)
# Overwrite pywren config parameters
config_ow = {'pywren': {}}
if runtime is not None:
config_ow['pywren']['runtime'] = runtime
if runtime_memory is not None:
config_ow['pywren']['runtime_memory'] = int(runtime_memory)
if compute_backend is not None:
config_ow['pywren']['compute_backend'] = compute_backend
if compute_backend_region is not None:
config_ow['pywren'][
'compute_backend_region'] = compute_backend_region
if storage_backend is not None:
config_ow['pywren']['storage_backend'] = storage_backend
if storage_backend_region is not None:
config_ow['pywren'][
'storage_backend_region'] = storage_backend_region
if rabbitmq_monitor is not None:
config_ow['pywren']['rabbitmq_monitor'] = rabbitmq_monitor
self.config = default_config(config, config_ow)
self.executor_id = create_executor_id()
logger.debug('FunctionExecutor created with ID: {}'.format(
self.executor_id))
# RabbitMQ monitor configuration
self.rabbitmq_monitor = self.config['pywren'].get(
'rabbitmq_monitor', False)
if self.rabbitmq_monitor:
if 'rabbitmq' in self.config and 'amqp_url' in self.config[
'rabbitmq']:
self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
else:
raise Exception(
"You cannot use rabbitmq_mnonitor since 'amqp_url'"
" is not present in configuration")
self.data_cleaner = self.config['pywren']['data_cleaner']
storage_config = extract_storage_config(self.config)
self.internal_storage = InternalStorage(storage_config)
self.invoker = FunctionInvoker(self.config, self.executor_id,
self.internal_storage)
self.jobs = {}