def __init__(self, config, executor_id, internal_storage): self.log_level = os.getenv('PYWREN_LOGLEVEL') self.config = config self.executor_id = executor_id self.storage_config = extract_storage_config(self.config) self.internal_storage = internal_storage self.compute_config = extract_compute_config(self.config) self.is_pywren_function = is_pywren_function() self.remote_invoker = self.config['pywren'].get('remote_invoker', False) self.workers = self.config['pywren'].get('workers') logger.debug('ExecutorID {} - Total workers: {}'.format(self.executor_id, self.workers)) self.compute_handlers = [] cb = self.compute_config['backend'] regions = self.compute_config[cb].get('region') if regions and type(regions) == list: for region in regions: compute_config = self.compute_config.copy() compute_config[cb]['region'] = region self.compute_handlers.append(Compute(compute_config)) else: self.compute_handlers.append(Compute(self.compute_config)) logger.debug('ExecutorID {} - Creating function invoker'.format(self.executor_id)) self.token_bucket_q = Queue() self.pending_calls_q = Queue() self.running_flag = Value('i', 0) self.ongoing_activations = 0 self.job_monitor = JobMonitor(self.config, self.internal_storage, self.token_bucket_q)
def get_runtime_meta(self, key): """ Get the metadata given a runtime name. :param runtime: name of the runtime :return: runtime metadata """ path = [RUNTIMES_PREFIX, __version__, key+".meta.json"] filename_local_path = os.path.join(CACHE_DIR, *path) if os.path.exists(filename_local_path) and not is_pywren_function(): logger.debug("Runtime metadata found in local cache") with open(filename_local_path, "r") as f: runtime_meta = json.loads(f.read()) return runtime_meta else: logger.debug("Runtime metadata not found in local cache. Retrieving it from storage") try: obj_key = '/'.join(path).replace('\\', '/') json_str = self.storage_handler.get_object(self.bucket, obj_key) runtime_meta = json.loads(json_str.decode("ascii")) # Save runtime meta to cache if not os.path.exists(os.path.dirname(filename_local_path)): os.makedirs(os.path.dirname(filename_local_path)) with open(filename_local_path, "w") as f: f.write(json.dumps(runtime_meta)) return runtime_meta except StorageNoSuchKeyError: raise Exception('The runtime {} is not installed.'.format(obj_key))
def __init__(self, pywren_config, internal_storage, token_bucket_q): self.config = pywren_config self.internal_storage = internal_storage self.token_bucket_q = token_bucket_q self.is_pywren_function = is_pywren_function() self.monitors = [] self.rabbitmq_monitor = self.config['pywren'].get('rabbitmq_monitor', False) if self.rabbitmq_monitor: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url')
def put_runtime_meta(self, key, runtime_meta): """ Puit the metadata given a runtime config. :param runtime: name of the runtime :param runtime_meta metadata """ path = [RUNTIMES_PREFIX, __version__, key+".meta.json"] obj_key = '/'.join(path).replace('\\', '/') logger.debug("Uploading runtime metadata to: /{}/{}".format(self.bucket, obj_key)) self.storage_handler.put_object(self.bucket, obj_key, json.dumps(runtime_meta)) if not is_pywren_function(): filename_local_path = os.path.join(CACHE_DIR, *path) logger.debug("Storing runtime metadata into local cache: {}".format(filename_local_path)) if not os.path.exists(os.path.dirname(filename_local_path)): os.makedirs(os.path.dirname(filename_local_path)) with open(filename_local_path, "w") as f: f.write(json.dumps(runtime_meta))
def __init__(self, config = None, runtime=None, runtime_memory=None, compute_backend=None, compute_backend_region=None, storage_backend=None, storage_backend_region=None, workers=None, rabbitmq_monitor=None, remote_invoker=None, log_level=None): self.start_time = time.time() self._state = self.State.New self.is_pywren_function = is_pywren_function() # Log level Configuration self.log_level = log_level if not self.log_level: if(logger.getEffectiveLevel() != logging.WARNING): self.log_level = logging.getLevelName(logger.getEffectiveLevel()) if self.log_level: os.environ["PYWREN_LOGLEVEL"] = self.log_level if not self.is_pywren_function: default_logging_config(self.log_level) # Overwrite pywren config parameters pw_config_ow = {} if runtime is not None: pw_config_ow['runtime'] = runtime if runtime_memory is not None: pw_config_ow['runtime_memory'] = runtime_memory if compute_backend is not None: pw_config_ow['compute_backend'] = compute_backend if compute_backend_region is not None: pw_config_ow['compute_backend_region'] = compute_backend_region if storage_backend is not None: pw_config_ow['storage_backend'] = storage_backend if storage_backend_region is not None: pw_config_ow['storage_backend_region'] = storage_backend_region if workers is not None: pw_config_ow['workers'] = workers if rabbitmq_monitor is not None: pw_config_ow['rabbitmq_monitor'] = rabbitmq_monitor if remote_invoker is not None: pw_config_ow['remote_invoker'] = remote_invoker self.config = default_config(copy.deepcopy(config), pw_config_ow) self.executor_id = create_executor_id() logger.debug('FunctionExecutor created with ID: {}'.format(self.executor_id)) self.data_cleaner = self.config['pywren'].get('data_cleaner', True) self.rabbitmq_monitor = self.config['pywren'].get('rabbitmq_monitor', False) if self.rabbitmq_monitor: if 'rabbitmq' in self.config and 'amqp_url' in self.config['rabbitmq']: self.rabbit_amqp_url = self.config['rabbitmq'].get('amqp_url') else: raise Exception("You cannot use rabbitmq_mnonitor since 'amqp_url'" " is not present in configuration") storage_config = extract_storage_config(self.config) self.internal_storage = InternalStorage(storage_config) self.invoker = FunctionInvoker(self.config, self.executor_id, self.internal_storage) self.futures = [] self.total_jobs = 0 self.cleaned_jobs = set()
def __init__(self, ibm_cf_config): logger.debug("Creating IBM Cloud Functions client") self.log_level = os.getenv('PYWREN_LOGLEVEL') self.name = 'ibm_cf' self.ibm_cf_config = ibm_cf_config self.is_pywren_function = is_pywren_function() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get( 'namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get( 'api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.info("Set IBM CF Namespace to {}".format(self.namespace)) logger.info("Set IBM CF Endpoint to {}".format(self.endpoint)) self.user_key = self.api_key[: 5] if self.api_key else self.iam_api_key[: 5] self.package = 'pywren_v{}_{}'.format(__version__, self.user_key) if self.api_key: enc_api_key = str.encode(self.api_key) auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'') auth = 'Basic %s' % auth_token.decode('UTF-8') self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, auth=auth, user_agent=self.user_agent) elif self.iam_api_key: token_manager = DefaultTokenManager(api_key_id=self.iam_api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cf', 'iam_token') if 'token' in self.ibm_cf_config: logger.debug( "Using IBM IAM API Key - Reusing Token from config") token_manager._token = self.ibm_cf_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cf_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): logger.debug( "Using IBM IAM API Key - Reusing Token from local cache") token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_pywren_function(): logger.debug( "Using IBM IAM API Key - Token expired. Requesting new token" ) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) ibm_cf_config['token'] = token_manager._token ibm_cf_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') auth_token = token_manager._token auth = 'Bearer ' + auth_token self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace_id, auth=auth, user_agent=self.user_agent) log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - ' 'Region: {}'.format(__version__, self.namespace, self.region)) if not self.log_level: print(log_msg) logger.info("IBM CF client created successfully")
def __init__(self, ibm_cos_config): logger.debug("Creating IBM COS client") self.ibm_cos_config = ibm_cos_config self.is_pywren_function = is_pywren_function() user_agent = ibm_cos_config['user_agent'] service_endpoint = ibm_cos_config.get('endpoint').replace('http:', 'https:') if self.is_pywren_function and 'private_endpoint' in ibm_cos_config: service_endpoint = ibm_cos_config.get('private_endpoint') if 'api_key' in ibm_cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) api_key = None if 'api_key' in ibm_cos_config: api_key = ibm_cos_config.get('api_key') api_key_type = 'COS' elif 'iam_api_key' in ibm_cos_config: api_key = ibm_cos_config.get('iam_api_key') api_key_type = 'IAM' if {'secret_key', 'access_key'} <= set(ibm_cos_config): logger.debug("Using access_key and secret_key") access_key = ibm_cos_config.get('access_key') secret_key = ibm_cos_config.get('secret_key') client_config = ibm_botocore.client.Config(max_pool_connections=128, user_agent_extra=user_agent, connect_timeout=1, read_timeout=5) self.cos_client = ibm_boto3.client('s3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint) elif api_key is not None: client_config = ibm_botocore.client.Config(signature_version='oauth', max_pool_connections=128, user_agent_extra=user_agent, connect_timeout=1, read_timeout=5) token_manager = DefaultTokenManager(api_key_id=api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cos', api_key_type.lower()+'_token') token_minutes_diff = 0 if 'token' in self.ibm_cos_config: logger.debug("Using IBM {} API Key - Reusing Token from config".format(api_key_type)) token_manager._token = self.ibm_cos_config['token'] token_manager._expiry_time = datetime.strptime(self.ibm_cos_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): token_data = load_yaml_config(token_filename) logger.debug("Using IBM {} API Key - Reusing Token from local cache".format(api_key_type)) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_pywren_function(): logger.debug("Using IBM {} API Key - Token expired. Requesting new token".format(api_key_type)) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) self.ibm_cos_config['token'] = token_manager._token self.ibm_cos_config['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) logger.debug("IBM COS client created successfully")