def __init__(self, knative_config): self.log_level = os.getenv('PYWREN_LOGLEVEL') self.name = 'knative' self.knative_config = knative_config self.endpoint = self.knative_config.get('endpoint') self.service_hosts = {} # k8s config can be incluster, in ~/.kube/config or generate kube-config.yaml file and # set env variable KUBECONFIG=<path-to-kube-confg> try: config.load_kube_config() current_context = config.list_kube_config_contexts()[1].get('context') self.namespace = current_context.get('namespace', 'default') self.cluster = current_context.get('cluster') except Exception: config.load_incluster_config() self.namespace = 'default' self.cluster = 'default' self.api = client.CustomObjectsApi() self.v1 = client.CoreV1Api() self.headers = {'content-type': 'application/json'} if self.endpoint is None: try: ingress = self.v1.read_namespaced_service('istio-ingressgateway', 'istio-system') http_port = list(filter(lambda port: port.port == 80, ingress.spec.ports))[0].node_port https_port = list(filter(lambda port: port.port == 443, ingress.spec.ports))[0].node_port if ingress.status.load_balancer.ingress is not None: # get loadbalancer ip ip = ingress.status.load_balancer.ingress[0].ip else: # for minikube or a baremetal cluster that has no external load balancer node = self.v1.list_node() ip = node.items[0].status.addresses[0].address self.endpoint = 'http://{}:{}'.format(ip, http_port) except Exception as e: log_msg = "Something went wrong getting the istio-ingressgateway endpoint: {}".format(e) logger.info(log_msg) self.serice_host_filename = os.path.join(CACHE_DIR, 'knative', self.cluster, 'service_host') self.service_host_suffix = None if os.path.exists(self.serice_host_filename): serice_host_data = load_yaml_config(self.serice_host_filename) self.service_host_suffix = serice_host_data['service_host_suffix'] logger.debug('Loaded service host suffix: {}'.format(self.service_host_suffix)) log_msg = 'PyWren v{} init for Knative - Endpoint: {}'.format(__version__, self.endpoint) if not self.log_level: print(log_msg) logger.info(log_msg)
def __init__(self, ibm_cf_config): logger.debug("Creating IBM Cloud Functions client") self.log_level = os.getenv('PYWREN_LOGLEVEL') self.name = 'ibm_cf' self.ibm_cf_config = ibm_cf_config self.package = 'pywren_v' + __version__ self.is_remote_cluster = is_remote_cluster() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get( 'namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get( 'api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.debug("Set IBM CF Namespace to {}".format(self.namespace)) logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint)) if self.api_key: self.cf_client = CloudFunctionsClient(region=self.region, endpoint=self.endpoint, namespace=self.namespace, api_key=self.api_key, user_agent=self.user_agent) elif self.iam_api_key: token_manager = DefaultTokenManager(api_key_id=self.iam_api_key) token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN') if 'token' in self.ibm_cf_config: logger.debug("Using IBM IAM API Key - Reusing Token") token_manager._token = self.ibm_cf_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cf_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') elif os.path.exists(token_filename): logger.debug( "Using IBM IAM API Key - Reusing Token from local cache") token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') if token_manager._is_expired() and not is_remote_cluster(): logger.debug( "Using IBM IAM API Key - Token expired. Requesting new token" ) token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) ibm_cf_config['token'] = token_manager._token ibm_cf_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') self.cf_client = CloudFunctionsClient( region=self.region, endpoint=self.endpoint, namespace=self.namespace, namespace_id=self.namespace_id, token_manager=token_manager, user_agent=self.user_agent) log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - ' 'Region: {}'.format(__version__, self.namespace, self.region)) if not self.log_level: print(log_msg) logger.debug("IBM CF client created successfully")
def __init__(self, knative_config): self.log_level = logger.getEffectiveLevel() != logging.WARNING self.name = 'knative' self.knative_config = knative_config self.istio_endpoint = self.knative_config.get('istio_endpoint') # k8s config can be incluster, in ~/.kube/config or generate kube-config.yaml file and # set env variable KUBECONFIG=<path-to-kube-confg> try: config.load_kube_config() current_context = config.list_kube_config_contexts()[1].get('context') self.namespace = current_context.get('namespace', 'default') self.cluster = current_context.get('cluster') self.knative_config['namespace'] = self.namespace self.knative_config['cluster'] = self.cluster self.is_incluster = False except Exception: config.load_incluster_config() self.namespace = self.knative_config.get('namespace', 'default') self.cluster = self.knative_config.get('cluster', 'default') self.is_incluster = True self.api = client.CustomObjectsApi() self.v1 = client.CoreV1Api() if self.istio_endpoint is None: try: ingress = self.v1.read_namespaced_service('istio-ingressgateway', 'istio-system') http_port = list(filter(lambda port: port.port == 80, ingress.spec.ports))[0].node_port # https_port = list(filter(lambda port: port.port == 443, ingress.spec.ports))[0].node_port if ingress.status.load_balancer.ingress is not None: # get loadbalancer ip ip = ingress.status.load_balancer.ingress[0].ip else: # for minikube or a baremetal cluster that has no external load balancer node = self.v1.list_node() ip = node.items[0].status.addresses[0].address self.istio_endpoint = 'http://{}:{}'.format(ip, http_port) self.knative_config['istio_endpoint'] = self.istio_endpoint except Exception: logger.info("istio-ingressgateway endpoint not found") if 'service_host_suffix' not in self.knative_config: self.serice_host_filename = os.path.join(CACHE_DIR, 'knative', self.cluster, 'service_host') self.service_host_suffix = None if os.path.exists(self.serice_host_filename): serice_host_data = load_yaml_config(self.serice_host_filename) self.service_host_suffix = serice_host_data['service_host_suffix'] self.knative_config['service_host_suffix'] = self.service_host_suffix else: self.service_host_suffix = self.knative_config['service_host_suffix'] logger.debug('Loaded service host suffix: {}'.format(self.service_host_suffix)) if self.istio_endpoint: log_msg = 'PyWren v{} init for Knative - Istio Endpoint: {}'.format(__version__, self.istio_endpoint) else: log_msg = 'PyWren v{} init for Knative'.format(__version__) if not self.log_level: print(log_msg) logger.info(log_msg)
def __init__(self, ibm_cos_config, bucket=None, executor_id=None): logger.debug("Creating IBM COS client") self.ibm_cos_config = ibm_cos_config self.is_pywren_function = is_pywren_function() user_agent = ibm_cos_config['user_agent'] service_endpoint = ibm_cos_config.get('endpoint').replace('http:', 'https:') if self.is_pywren_function and 'private_endpoint' in ibm_cos_config: service_endpoint = ibm_cos_config.get('private_endpoint') if 'api_key' in ibm_cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) api_key = None if 'api_key' in ibm_cos_config: api_key = ibm_cos_config.get('api_key') api_key_type = 'COS' elif 'iam_api_key' in ibm_cos_config: api_key = ibm_cos_config.get('iam_api_key') api_key_type = 'IAM' if {'secret_key', 'access_key'} <= set(ibm_cos_config): logger.debug("Using access_key and secret_key") access_key = ibm_cos_config.get('access_key') secret_key = ibm_cos_config.get('secret_key') client_config = ibm_botocore.client.Config(max_pool_connections=128, user_agent_extra=user_agent, connect_timeout=CONN_READ_TIMEOUT, read_timeout=CONN_READ_TIMEOUT, retries={'max_attempts': OBJ_REQ_RETRIES}) self.cos_client = ibm_boto3.client('s3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint) elif api_key is not None: client_config = ibm_botocore.client.Config(signature_version='oauth', max_pool_connections=128, user_agent_extra=user_agent, connect_timeout=CONN_READ_TIMEOUT, read_timeout=CONN_READ_TIMEOUT, retries={'max_attempts': OBJ_REQ_RETRIES}) token_manager = DefaultTokenManager(api_key_id=api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cos', api_key_type.lower()+'_token') token_minutes_diff = 0 if 'token' in self.ibm_cos_config: logger.debug("Using IBM {} API Key - Reusing Token from config".format(api_key_type)) token_manager._token = self.ibm_cos_config['token'] token_manager._expiry_time = datetime.strptime(self.ibm_cos_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): token_data = load_yaml_config(token_filename) logger.debug("Using IBM {} API Key - Reusing Token from local cache".format(api_key_type)) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_pywren_function(): logger.debug("Using IBM {} API Key - Token expired. Requesting new token".format(api_key_type)) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) self.ibm_cos_config['token'] = token_manager._token self.ibm_cos_config['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) logger.debug("IBM COS client created successfully")
def __init__(self, ibm_cf_config): logger.debug("Creating IBM Cloud Functions client") self.log_level = os.getenv('PYWREN_LOGLEVEL') self.name = 'ibm_cf' self.ibm_cf_config = ibm_cf_config self.is_pywren_function = is_pywren_function() self.user_agent = ibm_cf_config['user_agent'] self.region = ibm_cf_config['region'] self.endpoint = ibm_cf_config['regions'][self.region]['endpoint'] self.namespace = ibm_cf_config['regions'][self.region]['namespace'] self.namespace_id = ibm_cf_config['regions'][self.region].get( 'namespace_id', None) self.api_key = ibm_cf_config['regions'][self.region].get( 'api_key', None) self.iam_api_key = ibm_cf_config.get('iam_api_key', None) logger.info("Set IBM CF Namespace to {}".format(self.namespace)) logger.info("Set IBM CF Endpoint to {}".format(self.endpoint)) self.user_key = self.api_key[: 5] if self.api_key else self.iam_api_key[: 5] self.package = 'pywren_v{}_{}'.format(__version__, self.user_key) if self.api_key: enc_api_key = str.encode(self.api_key) auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'') auth = 'Basic %s' % auth_token.decode('UTF-8') self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace, auth=auth, user_agent=self.user_agent) elif self.iam_api_key: token_manager = DefaultTokenManager(api_key_id=self.iam_api_key) token_filename = os.path.join(CACHE_DIR, 'ibm_cf', 'iam_token') if 'token' in self.ibm_cf_config: logger.debug( "Using IBM IAM API Key - Reusing Token from config") token_manager._token = self.ibm_cf_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cf_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) elif os.path.exists(token_filename): logger.debug( "Using IBM IAM API Key - Reusing Token from local cache") token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') token_minutes_diff = int( (token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0) logger.debug("Token expiry time: {} - Minutes left: {}".format( token_manager._expiry_time, token_minutes_diff)) if (token_manager._is_expired() or token_minutes_diff < 11) and not is_pywren_function(): logger.debug( "Using IBM IAM API Key - Token expired. Requesting new token" ) token_manager._token = None token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) ibm_cf_config['token'] = token_manager._token ibm_cf_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') auth_token = token_manager._token auth = 'Bearer ' + auth_token self.cf_client = OpenWhiskClient(endpoint=self.endpoint, namespace=self.namespace_id, auth=auth, user_agent=self.user_agent) log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - ' 'Region: {}'.format(__version__, self.namespace, self.region)) if not self.log_level: print(log_msg) logger.info("IBM CF client created successfully")
def __init__(self, ibm_cos_config): logger.debug("Creating IBM COS client") self.ibm_cos_config = ibm_cos_config self.is_pywren_function = is_pywren_function() service_endpoint = ibm_cos_config.get('endpoint').replace( 'http:', 'https:') if self.is_pywren_function and 'private_endpoint' in ibm_cos_config: service_endpoint = ibm_cos_config.get('private_endpoint') if 'api_key' in ibm_cos_config: service_endpoint = service_endpoint.replace('http:', 'https:') logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint)) api_key = None if 'api_key' in ibm_cos_config: api_key = ibm_cos_config.get('api_key') api_key_type = 'COS' elif 'iam_api_key' in ibm_cos_config: api_key = ibm_cos_config.get('iam_api_key') api_key_type = 'IAM' if {'secret_key', 'access_key'} <= set(ibm_cos_config): logger.debug("Using access_key and secret_key") access_key = ibm_cos_config.get('access_key') secret_key = ibm_cos_config.get('secret_key') client_config = ibm_botocore.client.Config( max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) self.cos_client = ibm_boto3.client( 's3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, config=client_config, endpoint_url=service_endpoint) elif api_key is not None: client_config = ibm_botocore.client.Config( signature_version='oauth', max_pool_connections=128, user_agent_extra='pywren-ibm-cloud', connect_timeout=1) token_manager = DefaultTokenManager(api_key_id=api_key) token_filename = os.path.join(CACHE_DIR, api_key_type + '_TOKEN') if 'token' in self.ibm_cos_config: logger.debug("Using IBM {} API Key - Reusing Token".format( api_key_type)) token_manager._token = self.ibm_cos_config['token'] token_manager._expiry_time = datetime.strptime( self.ibm_cos_config['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') elif os.path.exists(token_filename): logger.debug( "Using IBM {} API Key - Reusing Token from local cache". format(api_key_type)) token_data = load_yaml_config(token_filename) token_manager._token = token_data['token'] token_manager._expiry_time = datetime.strptime( token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z') if token_manager._is_expired() and not is_pywren_function(): logger.debug( "Using IBM {} API Key - Token expired. Requesting new token" .format(api_key_type)) token_manager.get_token() token_data = {} token_data['token'] = token_manager._token token_data[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') dump_yaml_config(token_filename, token_data) self.ibm_cos_config['token'] = token_manager._token self.ibm_cos_config[ 'token_expiry_time'] = token_manager._expiry_time.strftime( '%Y-%m-%d %H:%M:%S.%f%z') self.cos_client = ibm_boto3.client('s3', token_manager=token_manager, config=client_config, endpoint_url=service_endpoint) logger.debug("IBM COS client created successfully")