def __init__(self, ibm_cf_config):
        logger.debug("Creating IBM Cloud Functions client")
        self.log_level = os.getenv('PYWREN_LOGLEVEL')
        self.name = 'ibm_cf'
        self.ibm_cf_config = ibm_cf_config
        self.package = 'pywren_v' + __version__
        self.is_remote_cluster = is_remote_cluster()

        self.user_agent = ibm_cf_config['user_agent']
        self.region = ibm_cf_config['region']
        self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
        self.namespace = ibm_cf_config['regions'][self.region]['namespace']
        self.namespace_id = ibm_cf_config['regions'][self.region].get(
            'namespace_id', None)
        self.api_key = ibm_cf_config['regions'][self.region].get(
            'api_key', None)
        self.iam_api_key = ibm_cf_config.get('iam_api_key', None)

        logger.debug("Set IBM CF Namespace to {}".format(self.namespace))
        logger.debug("Set IBM CF Endpoint to {}".format(self.endpoint))

        if self.api_key:
            self.cf_client = CloudFunctionsClient(region=self.region,
                                                  endpoint=self.endpoint,
                                                  namespace=self.namespace,
                                                  api_key=self.api_key,
                                                  user_agent=self.user_agent)
        elif self.iam_api_key:
            token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
            token_filename = os.path.join(CACHE_DIR, 'IAM_TOKEN')

            if 'token' in self.ibm_cf_config:
                logger.debug("Using IBM IAM API Key - Reusing Token")
                token_manager._token = self.ibm_cf_config['token']
                token_manager._expiry_time = datetime.strptime(
                    self.ibm_cf_config['token_expiry_time'],
                    '%Y-%m-%d %H:%M:%S.%f%z')
            elif os.path.exists(token_filename):
                logger.debug(
                    "Using IBM IAM API Key - Reusing Token from local cache")
                token_data = load_yaml_config(token_filename)
                token_manager._token = token_data['token']
                token_manager._expiry_time = datetime.strptime(
                    token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')

            if token_manager._is_expired() and not is_remote_cluster():
                logger.debug(
                    "Using IBM IAM API Key - Token expired. Requesting new token"
                )
                token_manager.get_token()
                token_data = {}
                token_data['token'] = token_manager._token
                token_data[
                    'token_expiry_time'] = token_manager._expiry_time.strftime(
                        '%Y-%m-%d %H:%M:%S.%f%z')
                dump_yaml_config(token_filename, token_data)

            ibm_cf_config['token'] = token_manager._token
            ibm_cf_config[
                'token_expiry_time'] = token_manager._expiry_time.strftime(
                    '%Y-%m-%d %H:%M:%S.%f%z')

            self.cf_client = CloudFunctionsClient(
                region=self.region,
                endpoint=self.endpoint,
                namespace=self.namespace,
                namespace_id=self.namespace_id,
                token_manager=token_manager,
                user_agent=self.user_agent)

        log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - '
                   'Region: {}'.format(__version__, self.namespace,
                                       self.region))
        if not self.log_level:
            print(log_msg)
        logger.debug("IBM CF client created successfully")
    def _create_service(self, docker_image_name, runtime_memory, timeout):
        """
        Creates a service in knative based on the docker_image_name and the memory provided
        """
        logger.debug("Creating PyWren runtime service in Knative")
        svc_res = yaml.safe_load(kconfig.service_res)

        service_name = self._format_service_name(docker_image_name, runtime_memory)
        svc_res['metadata']['name'] = service_name
        svc_res['metadata']['namespace'] = self.namespace

        logger.debug("Service name: {}".format(service_name))
        logger.debug("Namespace: {}".format(self.namespace))

        svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout
        full_docker_image_name = '/'.join([self.knative_config['docker_repo'], docker_image_name])
        svc_res['spec']['template']['spec']['containers'][0]['image'] = full_docker_image_name
        svc_res['spec']['template']['spec']['containers'][0]['resources']['limits']['memory'] = '{}Mi'.format(runtime_memory)
        svc_res['spec']['template']['spec']['containers'][0]['resources']['limits']['cpu'] = '{}m'.format(self.knative_config['cpu'])

        try:
            # delete the service resource if exists
            self.api.delete_namespaced_custom_object(
                    group="serving.knative.dev",
                    version="v1alpha1",
                    name=service_name,
                    namespace=self.namespace,
                    plural="services",
                    body=client.V1DeleteOptions()
                )
            time.sleep(2)
        except Exception:
            pass

        # create the service resource
        self.api.create_namespaced_custom_object(
                group="serving.knative.dev",
                version="v1alpha1",
                namespace=self.namespace,
                plural="services",
                body=svc_res
            )

        w = watch.Watch()
        for event in w.stream(self.api.list_namespaced_custom_object,
                              namespace=self.namespace, group="serving.knative.dev",
                              version="v1alpha1", plural="services",
                              field_selector="metadata.name={0}".format(service_name),
                              timeout_seconds=300):
            if event['object'].get('status'):
                service_url = event['object']['status'].get('url')
                conditions = event['object']['status']['conditions']
                if conditions[0]['status'] == 'True' and \
                   conditions[1]['status'] == 'True' and \
                   conditions[2]['status'] == 'True':
                    w.stop()
                    time.sleep(2)

        log_msg = 'Runtime Service created - URL: {}'.format(service_url)
        logger.debug(log_msg)

        self.service_host_suffix = service_url[7:].replace(service_name, '')
        # Store service host suffix in local cache
        serice_host_data = {}
        serice_host_data['service_host_suffix'] = self.service_host_suffix
        dump_yaml_config(self.serice_host_filename, serice_host_data)
        self.knative_config['service_host_suffix'] = self.service_host_suffix

        return service_url
Exemple #3
0
    def __init__(self, ibm_cos_config, bucket=None, executor_id=None):
        logger.debug("Creating IBM COS client")
        self.ibm_cos_config = ibm_cos_config
        self.is_pywren_function = is_pywren_function()
        user_agent = ibm_cos_config['user_agent']

        service_endpoint = ibm_cos_config.get('endpoint').replace('http:', 'https:')
        if self.is_pywren_function and 'private_endpoint' in ibm_cos_config:
            service_endpoint = ibm_cos_config.get('private_endpoint')
            if 'api_key' in ibm_cos_config:
                service_endpoint = service_endpoint.replace('http:', 'https:')

        logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint))

        api_key = None
        if 'api_key' in ibm_cos_config:
            api_key = ibm_cos_config.get('api_key')
            api_key_type = 'COS'
        elif 'iam_api_key' in ibm_cos_config:
            api_key = ibm_cos_config.get('iam_api_key')
            api_key_type = 'IAM'

        if {'secret_key', 'access_key'} <= set(ibm_cos_config):
            logger.debug("Using access_key and secret_key")
            access_key = ibm_cos_config.get('access_key')
            secret_key = ibm_cos_config.get('secret_key')
            client_config = ibm_botocore.client.Config(max_pool_connections=128,
                                                       user_agent_extra=user_agent,
                                                       connect_timeout=CONN_READ_TIMEOUT,
                                                       read_timeout=CONN_READ_TIMEOUT,
                                                       retries={'max_attempts': OBJ_REQ_RETRIES})

            self.cos_client = ibm_boto3.client('s3',
                                               aws_access_key_id=access_key,
                                               aws_secret_access_key=secret_key,
                                               config=client_config,
                                               endpoint_url=service_endpoint)

        elif api_key is not None:
            client_config = ibm_botocore.client.Config(signature_version='oauth',
                                                       max_pool_connections=128,
                                                       user_agent_extra=user_agent,
                                                       connect_timeout=CONN_READ_TIMEOUT,
                                                       read_timeout=CONN_READ_TIMEOUT,
                                                       retries={'max_attempts': OBJ_REQ_RETRIES})

            token_manager = DefaultTokenManager(api_key_id=api_key)
            token_filename = os.path.join(CACHE_DIR, 'ibm_cos', api_key_type.lower()+'_token')
            token_minutes_diff = 0

            if 'token' in self.ibm_cos_config:
                logger.debug("Using IBM {} API Key - Reusing Token from config".format(api_key_type))
                token_manager._token = self.ibm_cos_config['token']
                token_manager._expiry_time = datetime.strptime(self.ibm_cos_config['token_expiry_time'],
                                                               '%Y-%m-%d %H:%M:%S.%f%z')
                token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0)
                logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff))

            elif os.path.exists(token_filename):
                token_data = load_yaml_config(token_filename)
                logger.debug("Using IBM {} API Key - Reusing Token from local cache".format(api_key_type))
                token_manager._token = token_data['token']
                token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'],
                                                               '%Y-%m-%d %H:%M:%S.%f%z')
                token_minutes_diff = int((token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0)
                logger.debug("Token expiry time: {} - Minutes left: {}".format(token_manager._expiry_time, token_minutes_diff))

            if (token_manager._is_expired() or token_minutes_diff < 11) and not is_pywren_function():
                logger.debug("Using IBM {} API Key - Token expired. Requesting new token".format(api_key_type))
                token_manager._token = None
                token_manager.get_token()
                token_data = {}
                token_data['token'] = token_manager._token
                token_data['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')
                dump_yaml_config(token_filename, token_data)

            self.ibm_cos_config['token'] = token_manager._token
            self.ibm_cos_config['token_expiry_time'] = token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')

            self.cos_client = ibm_boto3.client('s3', token_manager=token_manager,
                                               config=client_config,
                                               endpoint_url=service_endpoint)
        logger.debug("IBM COS client created successfully")
Exemple #4
0
    def _create_service(self, docker_image_name, runtime_memory, timeout):
        """
        Creates a service in knative based on the docker_image_name and the memory provided
        """
        logger.debug("Creating PyWren runtime service resource in k8s")
        svc_res = yaml.safe_load(kconfig.service_res)

        revision = 'latest' if 'SNAPSHOT' in __version__ else __version__
        # TODO: Take into account revision in service name
        service_name = self._format_service_name(docker_image_name, runtime_memory)
        svc_res['metadata']['name'] = service_name
        svc_res['metadata']['namespace'] = self.namespace

        svc_res['spec']['template']['spec']['timeoutSeconds'] = timeout
        docker_image = '/'.join([self.knative_config['docker_repo'], docker_image_name])

        svc_res['spec']['template']['spec']['containers'][0]['image'] = '{}:{}'.format(docker_image, revision)
        svc_res['spec']['template']['spec']['containers'][0]['resources']['limits']['memory'] = '{}Mi'.format(runtime_memory)

        try:
            # delete the service resource if exists
            self.api.delete_namespaced_custom_object(
                    group="serving.knative.dev",
                    version="v1alpha1",
                    name=service_name,
                    namespace=self.namespace,
                    plural="services",
                    body=client.V1DeleteOptions()
                )
            time.sleep(2)
        except Exception:
            pass

        # create the service resource
        self.api.create_namespaced_custom_object(
                group="serving.knative.dev",
                version="v1alpha1",
                namespace=self.namespace,
                plural="services",
                body=svc_res
            )

        w = watch.Watch()
        for event in w.stream(self.api.list_namespaced_custom_object,
                              namespace=self.namespace, group="serving.knative.dev",
                              version="v1alpha1", plural="services",
                              field_selector="metadata.name={0}".format(service_name)):
            conditions = None
            if event['object'].get('status') is not None:
                conditions = event['object']['status']['conditions']
                if event['object']['status'].get('url') is not None:
                    service_url = event['object']['status']['url']
            if conditions and conditions[0]['status'] == 'True' and \
               conditions[1]['status'] == 'True' and conditions[2]['status'] == 'True':
                # Workaround to prevent invoking the service immediately after creation.
                # TODO: Open issue.
                time.sleep(2)
                w.stop()

        log_msg = 'Runtime Service resource created - URL: {}'.format(service_url)
        logger.debug(log_msg)

        self.service_host_suffix = service_url[7:].replace(service_name, '')
        # Store service host suffix in local cache
        serice_host_data = {}
        serice_host_data['service_host_suffix'] = self.service_host_suffix
        dump_yaml_config(self.serice_host_filename, serice_host_data)

        return service_url
Exemple #5
0
    def __init__(self, ibm_cf_config):
        logger.debug("Creating IBM Cloud Functions client")
        self.log_level = os.getenv('PYWREN_LOGLEVEL')
        self.name = 'ibm_cf'
        self.ibm_cf_config = ibm_cf_config
        self.is_pywren_function = is_pywren_function()

        self.user_agent = ibm_cf_config['user_agent']
        self.region = ibm_cf_config['region']
        self.endpoint = ibm_cf_config['regions'][self.region]['endpoint']
        self.namespace = ibm_cf_config['regions'][self.region]['namespace']
        self.namespace_id = ibm_cf_config['regions'][self.region].get(
            'namespace_id', None)
        self.api_key = ibm_cf_config['regions'][self.region].get(
            'api_key', None)
        self.iam_api_key = ibm_cf_config.get('iam_api_key', None)

        logger.info("Set IBM CF Namespace to {}".format(self.namespace))
        logger.info("Set IBM CF Endpoint to {}".format(self.endpoint))

        self.user_key = self.api_key[:
                                     5] if self.api_key else self.iam_api_key[:
                                                                              5]
        self.package = 'pywren_v{}_{}'.format(__version__, self.user_key)

        if self.api_key:
            enc_api_key = str.encode(self.api_key)
            auth_token = base64.encodebytes(enc_api_key).replace(b'\n', b'')
            auth = 'Basic %s' % auth_token.decode('UTF-8')

            self.cf_client = OpenWhiskClient(endpoint=self.endpoint,
                                             namespace=self.namespace,
                                             auth=auth,
                                             user_agent=self.user_agent)
        elif self.iam_api_key:
            token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
            token_filename = os.path.join(CACHE_DIR, 'ibm_cf', 'iam_token')

            if 'token' in self.ibm_cf_config:
                logger.debug(
                    "Using IBM IAM API Key - Reusing Token from config")
                token_manager._token = self.ibm_cf_config['token']
                token_manager._expiry_time = datetime.strptime(
                    self.ibm_cf_config['token_expiry_time'],
                    '%Y-%m-%d %H:%M:%S.%f%z')
                token_minutes_diff = int(
                    (token_manager._expiry_time -
                     datetime.now(timezone.utc)).total_seconds() / 60.0)
                logger.debug("Token expiry time: {} - Minutes left: {}".format(
                    token_manager._expiry_time, token_minutes_diff))

            elif os.path.exists(token_filename):
                logger.debug(
                    "Using IBM IAM API Key - Reusing Token from local cache")
                token_data = load_yaml_config(token_filename)
                token_manager._token = token_data['token']
                token_manager._expiry_time = datetime.strptime(
                    token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
                token_minutes_diff = int(
                    (token_manager._expiry_time -
                     datetime.now(timezone.utc)).total_seconds() / 60.0)
                logger.debug("Token expiry time: {} - Minutes left: {}".format(
                    token_manager._expiry_time, token_minutes_diff))

            if (token_manager._is_expired()
                    or token_minutes_diff < 11) and not is_pywren_function():
                logger.debug(
                    "Using IBM IAM API Key - Token expired. Requesting new token"
                )
                token_manager._token = None
                token_manager.get_token()
                token_data = {}
                token_data['token'] = token_manager._token
                token_data[
                    'token_expiry_time'] = token_manager._expiry_time.strftime(
                        '%Y-%m-%d %H:%M:%S.%f%z')
                dump_yaml_config(token_filename, token_data)

            ibm_cf_config['token'] = token_manager._token
            ibm_cf_config[
                'token_expiry_time'] = token_manager._expiry_time.strftime(
                    '%Y-%m-%d %H:%M:%S.%f%z')

            auth_token = token_manager._token
            auth = 'Bearer ' + auth_token

            self.cf_client = OpenWhiskClient(endpoint=self.endpoint,
                                             namespace=self.namespace_id,
                                             auth=auth,
                                             user_agent=self.user_agent)

        log_msg = ('PyWren v{} init for IBM Cloud Functions - Namespace: {} - '
                   'Region: {}'.format(__version__, self.namespace,
                                       self.region))
        if not self.log_level:
            print(log_msg)
        logger.info("IBM CF client created successfully")
Exemple #6
0
    def __init__(self, ibm_cos_config):
        logger.debug("Creating IBM COS client")
        self.ibm_cos_config = ibm_cos_config
        self.is_pywren_function = is_pywren_function()

        service_endpoint = ibm_cos_config.get('endpoint').replace(
            'http:', 'https:')
        if self.is_pywren_function and 'private_endpoint' in ibm_cos_config:
            service_endpoint = ibm_cos_config.get('private_endpoint')
            if 'api_key' in ibm_cos_config:
                service_endpoint = service_endpoint.replace('http:', 'https:')

        logger.debug("Set IBM COS Endpoint to {}".format(service_endpoint))

        api_key = None
        if 'api_key' in ibm_cos_config:
            api_key = ibm_cos_config.get('api_key')
            api_key_type = 'COS'
        elif 'iam_api_key' in ibm_cos_config:
            api_key = ibm_cos_config.get('iam_api_key')
            api_key_type = 'IAM'

        if {'secret_key', 'access_key'} <= set(ibm_cos_config):
            logger.debug("Using access_key and secret_key")
            access_key = ibm_cos_config.get('access_key')
            secret_key = ibm_cos_config.get('secret_key')
            client_config = ibm_botocore.client.Config(
                max_pool_connections=128,
                user_agent_extra='pywren-ibm-cloud',
                connect_timeout=1)
            self.cos_client = ibm_boto3.client(
                's3',
                aws_access_key_id=access_key,
                aws_secret_access_key=secret_key,
                config=client_config,
                endpoint_url=service_endpoint)

        elif api_key is not None:
            client_config = ibm_botocore.client.Config(
                signature_version='oauth',
                max_pool_connections=128,
                user_agent_extra='pywren-ibm-cloud',
                connect_timeout=1)

            token_manager = DefaultTokenManager(api_key_id=api_key)
            token_filename = os.path.join(CACHE_DIR, api_key_type + '_TOKEN')

            if 'token' in self.ibm_cos_config:
                logger.debug("Using IBM {} API Key - Reusing Token".format(
                    api_key_type))
                token_manager._token = self.ibm_cos_config['token']
                token_manager._expiry_time = datetime.strptime(
                    self.ibm_cos_config['token_expiry_time'],
                    '%Y-%m-%d %H:%M:%S.%f%z')
            elif os.path.exists(token_filename):
                logger.debug(
                    "Using IBM {} API Key - Reusing Token from local cache".
                    format(api_key_type))
                token_data = load_yaml_config(token_filename)
                token_manager._token = token_data['token']
                token_manager._expiry_time = datetime.strptime(
                    token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')

            if token_manager._is_expired() and not is_pywren_function():
                logger.debug(
                    "Using IBM {} API Key - Token expired. Requesting new token"
                    .format(api_key_type))
                token_manager.get_token()
                token_data = {}
                token_data['token'] = token_manager._token
                token_data[
                    'token_expiry_time'] = token_manager._expiry_time.strftime(
                        '%Y-%m-%d %H:%M:%S.%f%z')
                dump_yaml_config(token_filename, token_data)

            self.ibm_cos_config['token'] = token_manager._token
            self.ibm_cos_config[
                'token_expiry_time'] = token_manager._expiry_time.strftime(
                    '%Y-%m-%d %H:%M:%S.%f%z')

            self.cos_client = ibm_boto3.client('s3',
                                               token_manager=token_manager,
                                               config=client_config,
                                               endpoint_url=service_endpoint)
        logger.debug("IBM COS client created successfully")