Exemplo n.º 1
0
host_api = cm_client.AllHostsResourceApi(api_client)

message = 'updating CM Agent safety valve for SMM'
body = cm_client.ApiConfigList(
)  # ApiConfigList | Configuration changes. (optional)
body.items = [
    cm_client.ApiConfig(
        name="host_agent_safety_valve",
        value="kafka_broker_topic_partition_metrics_for_smm_enabled=true")
]

cmd = host_api.update_config(message=message, body=body)

# create MGMT/CMS
mgmt_api = cm_client.MgmtServiceResourceApi(api_client)
api_service = cm_client.ApiService()

api_service.roles = [
    cm_client.ApiRole(type='SERVICEMONITOR'),
    cm_client.ApiRole(type='HOSTMONITOR'),
    cm_client.ApiRole(type='EVENTSERVER'),
    cm_client.ApiRole(type='ALERTPUBLISHER')
]

mgmt_api.auto_assign_roles()  # needed?
mgmt_api.auto_configure()  # needed?
mgmt_api.setup_cms(body=api_service)
cmd = mgmt_api.start_command()
wait(cmd)

# create the cluster using the template
Exemplo n.º 2
0
def setupCMS():
    service_name = 'mgmt'
    service_type = 'MGMT'.upper()

    api_instance = cm_client.MgmtServiceResourceApi(api_client)

    services_instance = cm_client.MgmtServiceResourceApi(api_client)
    mgmt_role_instance = cm_client.MgmtRolesResourceApi(api_client)
    rcg_instance = cm_client.MgmtRoleConfigGroupsResourceApi(api_client)

    services_instance.setup_cms(
        body=cm_client.ApiService(name=service_name,
                                  type=service_type,
                                  display_name='Cloudera Management Service'))
    host_id = getattr(get_host_resource(hostname), 'host_id', None)

    role_list = []
    for role_type in [
            "REPORTSMANAGER", "EVENTSERVER", "HOSTMONITOR", "ALERTPUBLISHER",
            "SERVICEMONITOR"
    ]:
        role_name = service_name + "-" + role_type + "-" + hashlib.md5(
            hostname.encode('utf-8')).hexdigest()
        role_list.append({
            "name": role_name,
            "type": role_type,
            "hostRef": {
                "hostId": host_id
            }
        })

    body = cm_client.ApiRoleList(role_list)
    api_response = mgmt_role_instance.create_roles(body=body)

    api_response = rcg_instance.read_role_config_groups()
    for rcg in api_response.items:
        if rcg.role_type == 'REPORTSMANAGER':
            rcg_instance.update_config(rcg.name,
                                       message=None,
                                       body=cm_client.ApiConfigList([{
                                           'name':
                                           'headlamp_database_host',
                                           'value':
                                           hostname
                                       }, {
                                           'name':
                                           'headlamp_database_name',
                                           'value':
                                           'rman'
                                       }, {
                                           'name':
                                           'headlamp_database_user',
                                           'value':
                                           'rman'
                                       }, {
                                           'name':
                                           'headlamp_database_password',
                                           'value':
                                           'supersecret1'
                                       }, {
                                           'name':
                                           'headlamp_database_type',
                                           'value':
                                           'postgresql'
                                       }]))

    services_instance.start_command()
Exemplo n.º 3
0
    def create_cluster(self):

        # accept trial licence
        try:
            self.cm_api.begin_trial()
        except ApiException as exc:
            if exc.status == 400 and 'Trial has been used' in exc.body:
                pass  # This can be ignored
            else:
                raise

        # Install CM Agent on host
        with open(self.key_file, "r") as f:
            key = f.read()

        self._import_paywall_credentials()
        instargs = cm_client.ApiHostInstallArguments(
            host_names=[self.host],
            user_name='root',
            private_key=key,
            cm_repo_url=self.cm_repo_url,
            java_install_strategy='NONE',
            ssh_port=22,
            passphrase='')

        cmd = self.cm_api.host_install_command(body=instargs)
        cmd = self.wait(cmd)
        if not cmd.success:
            raise RuntimeError('Failed to add host to the cluster')

        # create MGMT/CMS
        api_service = cm_client.ApiService()
        api_service.roles = [
            cm_client.ApiRole(type='SERVICEMONITOR'),
            cm_client.ApiRole(type='HOSTMONITOR'),
            cm_client.ApiRole(type='EVENTSERVER'),
            cm_client.ApiRole(type='ALERTPUBLISHER')
        ]

        self.mgmt_api.auto_assign_roles()  # needed?
        self.mgmt_api.auto_configure()  # needed?
        self.mgmt_api.setup_cms(body=api_service)
        cmd = self.mgmt_api.start_command()
        cmd = self.wait(cmd)
        if not cmd.success:
            raise RuntimeError('Failed to start Management Services')

        # Update host-level parameter required by SMM
        self.all_hosts_api.update_config(
            message='Updating parameter for SMM',
            body=cm_client.ApiConfigList([
                cm_client.ApiConfig(
                    name='host_agent_safety_valve',
                    value=
                    'kafka_broker_topic_partition_metrics_for_smm_enabled=true'
                )
            ]))

        # create the cluster using the template
        with open(self.template) as f:
            json_str = f.read()

        Response = namedtuple("Response", "data")
        dst_cluster_template = self.api_client.deserialize(
            response=Response(json_str),
            response_type=cm_client.ApiClusterTemplate)
        cmd = self.cm_api.import_cluster_template(add_repositories=True,
                                                  body=dst_cluster_template)
        cmd = self.wait(cmd)
        if not cmd.success:
            raise RuntimeError('Failed to deploy cluster template')

        # All parcel downloads should've already been done at this point, so we can safely remove the paywall credentials
        self._reset_paywall_credentials()

        if self.use_kerberos:
            self._enable_kerberos()
Exemplo n.º 4
0
    def setup_cm(self, key_file, cm_repo_url, use_kerberos, use_tls,
                 kerberos_type, ipa_host):

        # Accept trial licence
        try:
            self.cm_api.begin_trial()
        except ApiException as exc:
            if exc.status == 400 and 'Trial has been used' in exc.body:
                pass  # This can be ignored
            else:
                raise

        # Install CM Agent on host
        with open(key_file, "r") as f:
            key = f.read()

        if self.host not in [
                h.hostname for h in self.hosts_api.read_hosts().items
        ]:
            instargs = cm_client.ApiHostInstallArguments(
                host_names=[self.host],
                user_name='root',
                private_key=key,
                cm_repo_url=cm_repo_url,
                java_install_strategy='NONE',
                ssh_port=22,
                passphrase='')

            cmd = self.cm_api.host_install_command(body=instargs)
            cmd = self.wait(cmd)
            if not cmd.success:
                raise RuntimeError('Failed to add host to the cluster')

        # Create MGMT/CMS
        try:
            self.mgmt_api.read_service()
            print("Cloudera Management Services already installed")
            cms_exists = True
        except cm_client.rest.ApiException as e:
            cms_exists = False

        if not cms_exists:
            print("Installing Cloudera Management Services")
            api_service = cm_client.ApiService()
            api_service.roles = [
                cm_client.ApiRole(type='SERVICEMONITOR'),
                cm_client.ApiRole(type='HOSTMONITOR'),
                cm_client.ApiRole(type='EVENTSERVER'),
                cm_client.ApiRole(type='ALERTPUBLISHER')
            ]
            self.mgmt_api.setup_cms(body=api_service)
            cmd = self.mgmt_api.start_command()
            cmd = self.wait(cmd)
            if not cmd.success:
                raise RuntimeError('Failed to start Management Services')

        # Update cluster banner
        c_id = cluster_id()
        banner = 'Cluster ID: {}, Host: {}'.format(c_id, socket.gethostname())
        header_color = HEADER_COLORS[c_id % len(HEADER_COLORS)]
        self.cm_api.update_config(
            message='Customizing CM header and banner',
            body=cm_client.ApiConfigList([
                cm_client.ApiConfig(name='CUSTOM_BANNER_HTML', value=banner),
                cm_client.ApiConfig(name='CUSTOM_HEADER_COLOR',
                                    value=header_color),
            ]))

        # Update host-level parameter required by SMM
        self.all_hosts_api.update_config(
            message='Updating parameter for SMM',
            body=cm_client.ApiConfigList([
                cm_client.ApiConfig(
                    name='host_agent_safety_valve',
                    value=
                    'kafka_broker_topic_partition_metrics_for_smm_enabled=true'
                )
            ]))

        # Enable kerberos
        if use_kerberos:
            self._enable_kerberos(kerberos_type, ipa_host)

        # Enable TLS
        if use_tls:
            self._enable_tls()

        # Restart Mgmt Services
        cmd = self.mgmt_api.restart_command()
        cmd = self.wait(cmd)