class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def __retry_check(self, cid, maxcnt, interval, msg):
        stat_url = CMD_STAT_URL_PTR % (self.url, cid)
        stat = self.p.get(stat_url)
        retry_cnt = 0
        while not (stat['success'] is True and stat['active'] is False):
            retry_cnt += 1
            flush_str = '.' * retry_cnt
            print '\rCheck CDH services %s status (timeout: %dmin) %s' % (msg, maxcnt*interval/60, flush_str),
            sys.stdout.flush()
            time.sleep(interval)
            stat = self.p.get(stat_url)
            if retry_cnt == maxcnt: return False
        return True

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        print 'Restarting CDH services ...'
        rc1 = self.p.post(restart_url)
        if self.__retry_check(rc1['id'], 40, 15, 'restart'):
            print 'Restart CDH successfully!'
        else:
            err('Failed to restart CDH, max retry count reached')

        rc2 = self.p.post(deploy_cfg_url)
        if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
            print 'Deploy client config successfully!'
        else:
            err('Failed to deploy CDH client config, max retry count reached')
Beispiel #2
0
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {
            hdfs_service: HDFS_CONFIG,
            hbase_service: HBASE_MASTER_CONFIG,
            zk_service: ZK_CONFIG
        }

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name,
                                             hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = [
            '%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items']
            if r['roleType'] == 'REGIONSERVER'
        ]

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        def __retry(url, maxcnt, interval, msg):
            rc = self.p.post(url)
            stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
            get_stat = lambda: self.p.get(stat_url)[
                'success'] is True and self.p.get(stat_url)['active'] is False
            retry(get_stat, maxcnt, interval, msg)

        info('Restarting CDH services ...')
        __retry(restart_url, 40, 15, 'CDH services restart')

        info('Deploying CDH client configs ...')
        __retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        def __retry(url, maxcnt, interval, msg):
            rc = self.p.post(url)
            stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
            get_stat = lambda: self.p.get(stat_url)['success'] is True and self.p.get(stat_url)['active'] is False
            retry(get_stat, maxcnt, interval, msg)

        info('Restarting CDH services ...')
        __retry(restart_url, 40, 15, 'CDH services restart')

        info('Deploying CDH client configs ...')
        __retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
class HDPMod(object):
    """ Modify HDP configs for trafodion and restart HDP services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd, json_type=False)

    def mod(self):
        cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
        desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
        cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
        desired_cfg = self.p.get(desired_cfg_url)

        hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % cluster_url)
        rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]

        hbase_hregion_property = 'hbase.hregion.impl'
        hbase_config_group = {
            'ConfigGroup': {
                'cluster_name': self.cluster_name,
                'group_name': 'hbase-regionserver',
                'tag': 'HBASE',
                'description': 'HBase Regionserver configs for Trafodion',
                'hosts': [{'host_name': host} for host in rsnodes],
                'desired_configs': [
                    {
                        'type': 'hbase-site',
                        'tag': 'traf_cg',
                        'properties': {hbase_hregion_property: MOD_CFGS['hbase-site'].pop(hbase_hregion_property)}
                    }
                ]
            }
        }
        self.p.post('%s/config_groups' % cluster_url, hbase_config_group)

        for config_type in MOD_CFGS.keys():
            desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
            current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
            tag = 'version' + str(int(time.time() * 1000000))
            new_properties = current_cfg['items'][0]['properties']
            new_properties.update(MOD_CFGS[config_type])
            config = {
                'Clusters': {
                    'desired_config': {
                        'type': config_type,
                        'tag': tag,
                        'properties': new_properties
                    }
                }
            }
            self.p.put(cluster_url, config)

    def restart(self):
        info('Restarting HDP services ...')
        def startstop(srvs, action, count, interval):
            srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
            state = 'INSTALLED' if action == 'stop' else 'STARTED'
            for srv in srvs:
                srv_url = srv_baseurl + srv
                config = {'RequestInfo': {'context' :'%s %s services' % (action, srv)}, 'ServiceInfo': {'state' : state}}
                rc = self.p.put(srv_url, config)

                # check startstop status
                if rc:
                    get_stat = lambda: self.p.get(srv_url)['ServiceInfo']['state'] == state
                    retry(get_stat, count, interval, 'HDP service %s %s' % (srv, action))
                else:
                    if action == 'stop': action += 'p'
                    info('HDP service %s had already been %sed' % (srv, action))

        srvs = ['HBASE', 'HDFS', 'ZOOKEEPER']
        startstop(srvs, 'stop', 30, 10)
        time.sleep(10)
        srvs.reverse()
        startstop(srvs, 'start', 60, 10)
Beispiel #5
0
class HDPMod(object):
    """ Modify HDP configs for trafodion and restart HDP services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd, json_type=False)

    def mod(self):
        cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
        desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
        cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
        desired_cfg = self.p.get(desired_cfg_url)

        hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' %
                         cluster_url)
        rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]

        hbase_hregion_property = 'hbase.hregion.impl'
        hbase_config_group = {
            'ConfigGroup': {
                'cluster_name':
                self.cluster_name,
                'group_name':
                'hbase-regionserver',
                'tag':
                'HBASE',
                'description':
                'HBase Regionserver configs for Trafodion',
                'hosts': [{
                    'host_name': host
                } for host in rsnodes],
                'desired_configs': [{
                    'type': 'hbase-site',
                    'tag': 'traf_cg',
                    'properties': {
                        hbase_hregion_property:
                        MOD_CFGS['hbase-site'].pop(hbase_hregion_property)
                    }
                }]
            }
        }
        self.p.post('%s/config_groups' % cluster_url, hbase_config_group)

        if dbcfgs['secure_hadoop'].upper() == 'Y':
            secure_cfgs = ',org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.token.TokenProvider'
            MOD_CFGS['hbase-site'][
                'hbase.coprocessor.region.classes'] += secure_cfgs

        for config_type in MOD_CFGS.keys():
            desired_tag = desired_cfg['Clusters']['desired_configs'][
                config_type]['tag']
            current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
            tag = 'version' + str(int(time.time() * 1000000))
            new_properties = current_cfg['items'][0]['properties']
            new_properties.update(MOD_CFGS[config_type])
            config = {
                'Clusters': {
                    'desired_config': {
                        'type': config_type,
                        'tag': tag,
                        'properties': new_properties
                    }
                }
            }
            self.p.put(cluster_url, config)

    def restart(self):
        info('Restarting HDP services ...')

        def startstop(srvs, action, count, interval):
            srv_baseurl = CLUSTER_URL_PTR % (self.url,
                                             self.cluster_name) + '/services/'
            state = 'INSTALLED' if action == 'stop' else 'STARTED'
            for srv in srvs:
                srv_url = srv_baseurl + srv
                config = {
                    'RequestInfo': {
                        'context': '%s %s services' % (action, srv)
                    },
                    'ServiceInfo': {
                        'state': state
                    }
                }
                rc = self.p.put(srv_url, config)

                # check startstop status
                if rc:
                    get_stat = lambda: self.p.get(srv_url)['ServiceInfo'][
                        'state'] == state
                    retry(get_stat, count, interval,
                          'HDP service %s %s' % (srv, action))
                else:
                    if action == 'stop': action += 'p'
                    info('HDP service %s had already been %sed' %
                         (srv, action))

        srvs = ['HBASE', 'HDFS', 'ZOOKEEPER']
        startstop(srvs, 'stop', 30, 10)
        time.sleep(10)
        srvs.reverse()
        startstop(srvs, 'start', 60, 10)
Beispiel #6
0
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def __retry_check(self, cid, maxcnt, interval, msg):
        stat_url = CMD_STAT_URL_PTR % (self.url, cid)
        stat = self.p.get(stat_url)
        retry_cnt = 0
        while not (stat['success'] is True and stat['active'] is False):
            retry_cnt += 1
            flush_str = '.' * retry_cnt
            print '\rCheck CDH services %s status (timeout: %dmin) %s' % (
                msg, maxcnt * interval / 60, flush_str),
            sys.stdout.flush()
            time.sleep(interval)
            stat = self.p.get(stat_url)
            if retry_cnt == maxcnt: return False
        return True

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {
            hdfs_service: HDFS_CONFIG,
            hbase_service: HBASE_MASTER_CONFIG,
            zk_service: ZK_CONFIG
        }

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name,
                                             hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = [
            '%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items']
            if r['roleType'] == 'REGIONSERVER'
        ]

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        print 'Restarting CDH services ...'
        rc1 = self.p.post(restart_url)
        if self.__retry_check(rc1['id'], 40, 15, 'restart'):
            print 'Restart CDH successfully!'
        else:
            err('Failed to restart CDH, max retry count reached')

        rc2 = self.p.post(deploy_cfg_url)
        if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
            print 'Deploy client config successfully!'
        else:
            err('Failed to deploy CDH client config, max retry count reached')