Exemplo n.º 1
0
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def __retry_check(self, cid, maxcnt, interval, msg):
        stat_url = CMD_STAT_URL_PTR % (self.url, cid)
        stat = self.p.get(stat_url)
        retry_cnt = 0
        while not (stat['success'] is True and stat['active'] is False):
            retry_cnt += 1
            flush_str = '.' * retry_cnt
            print '\rCheck CDH services %s status (timeout: %dmin) %s' % (msg, maxcnt*interval/60, flush_str),
            sys.stdout.flush()
            time.sleep(interval)
            stat = self.p.get(stat_url)
            if retry_cnt == maxcnt: return False
        return True

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        print 'Restarting CDH services ...'
        rc1 = self.p.post(restart_url)
        if self.__retry_check(rc1['id'], 40, 15, 'restart'):
            print 'Restart CDH successfully!'
        else:
            err('Failed to restart CDH, max retry count reached')

        rc2 = self.p.post(deploy_cfg_url)
        if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
            print 'Deploy client config successfully!'
        else:
            err('Failed to deploy CDH client config, max retry count reached')
Exemplo n.º 2
0
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {
            hdfs_service: HDFS_CONFIG,
            hbase_service: HBASE_MASTER_CONFIG,
            zk_service: ZK_CONFIG
        }

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name,
                                             hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = [
            '%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items']
            if r['roleType'] == 'REGIONSERVER'
        ]

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        def __retry(url, maxcnt, interval, msg):
            rc = self.p.post(url)
            stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
            get_stat = lambda: self.p.get(stat_url)[
                'success'] is True and self.p.get(stat_url)['active'] is False
            retry(get_stat, maxcnt, interval, msg)

        info('Restarting CDH services ...')
        __retry(restart_url, 40, 15, 'CDH services restart')

        info('Deploying CDH client configs ...')
        __retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
Exemplo n.º 3
0
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        def __retry(url, maxcnt, interval, msg):
            rc = self.p.post(url)
            stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
            get_stat = lambda: self.p.get(stat_url)['success'] is True and self.p.get(stat_url)['active'] is False
            retry(get_stat, maxcnt, interval, msg)

        info('Restarting CDH services ...')
        __retry(restart_url, 40, 15, 'CDH services restart')

        info('Deploying CDH client configs ...')
        __retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
Exemplo n.º 4
0
class HDPMod(object):
    """ Modify HDP configs for trafodion and restart HDP services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd, json_type=False)

    def mod(self):
        cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
        desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
        cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
        desired_cfg = self.p.get(desired_cfg_url)

        hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % cluster_url)
        rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]

        hbase_hregion_property = 'hbase.hregion.impl'
        hbase_config_group = {
            'ConfigGroup': {
                'cluster_name': self.cluster_name,
                'group_name': 'hbase-regionserver',
                'tag': 'HBASE',
                'description': 'HBase Regionserver configs for Trafodion',
                'hosts': [{'host_name': host} for host in rsnodes],
                'desired_configs': [
                    {
                        'type': 'hbase-site',
                        'tag': 'traf_cg',
                        'properties': {hbase_hregion_property: MOD_CFGS['hbase-site'].pop(hbase_hregion_property)}
                    }
                ]
            }
        }
        self.p.post('%s/config_groups' % cluster_url, hbase_config_group)

        for config_type in MOD_CFGS.keys():
            desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
            current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
            tag = 'version' + str(int(time.time() * 1000000))
            new_properties = current_cfg['items'][0]['properties']
            new_properties.update(MOD_CFGS[config_type])
            config = {
                'Clusters': {
                    'desired_config': {
                        'type': config_type,
                        'tag': tag,
                        'properties': new_properties
                    }
                }
            }
            self.p.put(cluster_url, config)

    def restart(self):
        info('Restarting HDP services ...')
        def startstop(srvs, action, count, interval):
            srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
            state = 'INSTALLED' if action == 'stop' else 'STARTED'
            for srv in srvs:
                srv_url = srv_baseurl + srv
                config = {'RequestInfo': {'context' :'%s %s services' % (action, srv)}, 'ServiceInfo': {'state' : state}}
                rc = self.p.put(srv_url, config)

                # check startstop status
                if rc:
                    get_stat = lambda: self.p.get(srv_url)['ServiceInfo']['state'] == state
                    retry(get_stat, count, interval, 'HDP service %s %s' % (srv, action))
                else:
                    if action == 'stop': action += 'p'
                    info('HDP service %s had already been %sed' % (srv, action))

        srvs = ['HBASE', 'HDFS', 'ZOOKEEPER']
        startstop(srvs, 'stop', 30, 10)
        time.sleep(10)
        srvs.reverse()
        startstop(srvs, 'start', 60, 10)
Exemplo n.º 5
0
class HDPMod(object):
    """ Modify HDP configs for trafodion and restart HDP services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd, json_type=False)

    def mod(self):
        cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
        desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
        cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
        desired_cfg = self.p.get(desired_cfg_url)

        hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' %
                         cluster_url)
        rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]

        hbase_hregion_property = 'hbase.hregion.impl'
        hbase_config_group = {
            'ConfigGroup': {
                'cluster_name':
                self.cluster_name,
                'group_name':
                'hbase-regionserver',
                'tag':
                'HBASE',
                'description':
                'HBase Regionserver configs for Trafodion',
                'hosts': [{
                    'host_name': host
                } for host in rsnodes],
                'desired_configs': [{
                    'type': 'hbase-site',
                    'tag': 'traf_cg',
                    'properties': {
                        hbase_hregion_property:
                        MOD_CFGS['hbase-site'].pop(hbase_hregion_property)
                    }
                }]
            }
        }
        self.p.post('%s/config_groups' % cluster_url, hbase_config_group)

        if dbcfgs['secure_hadoop'].upper() == 'Y':
            secure_cfgs = ',org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.token.TokenProvider'
            MOD_CFGS['hbase-site'][
                'hbase.coprocessor.region.classes'] += secure_cfgs

        for config_type in MOD_CFGS.keys():
            desired_tag = desired_cfg['Clusters']['desired_configs'][
                config_type]['tag']
            current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
            tag = 'version' + str(int(time.time() * 1000000))
            new_properties = current_cfg['items'][0]['properties']
            new_properties.update(MOD_CFGS[config_type])
            config = {
                'Clusters': {
                    'desired_config': {
                        'type': config_type,
                        'tag': tag,
                        'properties': new_properties
                    }
                }
            }
            self.p.put(cluster_url, config)

    def restart(self):
        info('Restarting HDP services ...')

        def startstop(srvs, action, count, interval):
            srv_baseurl = CLUSTER_URL_PTR % (self.url,
                                             self.cluster_name) + '/services/'
            state = 'INSTALLED' if action == 'stop' else 'STARTED'
            for srv in srvs:
                srv_url = srv_baseurl + srv
                config = {
                    'RequestInfo': {
                        'context': '%s %s services' % (action, srv)
                    },
                    'ServiceInfo': {
                        'state': state
                    }
                }
                rc = self.p.put(srv_url, config)

                # check startstop status
                if rc:
                    get_stat = lambda: self.p.get(srv_url)['ServiceInfo'][
                        'state'] == state
                    retry(get_stat, count, interval,
                          'HDP service %s %s' % (srv, action))
                else:
                    if action == 'stop': action += 'p'
                    info('HDP service %s had already been %sed' %
                         (srv, action))

        srvs = ['HBASE', 'HDFS', 'ZOOKEEPER']
        startstop(srvs, 'stop', 30, 10)
        time.sleep(10)
        srvs.reverse()
        startstop(srvs, 'start', 60, 10)
Exemplo n.º 6
0
class HDPMod(object):
    """ Modify HDP configs for trafodion and restart HDP services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd, json_type=False)

    def mod(self):
        cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
        desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
        cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
        desired_cfg = self.p.get(desired_cfg_url)

        for config_type in MOD_CFGS.keys():
            desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
            current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
            tag = 'version' + str(int(time.time() * 1000000))
            new_properties = current_cfg['items'][0]['properties']
            new_properties.update(MOD_CFGS[config_type])
            config = {
                'Clusters': {
                    'desired_config': {
                        'type': config_type,
                        'tag': tag,
                        'properties': new_properties
                    }
                }
            }
            self.p.put(cluster_url, config)


    def restart(self):
        srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
        srvs = ['HBASE', 'ZOOKEEPER', 'HDFS']

        # Stop
        print 'Restarting HDP services ...'
        for srv in srvs:
            srv_url = srv_baseurl + srv
            config = {'RequestInfo': {'context' :'Stop %s services' % srv}, 'ServiceInfo': {'state' : 'INSTALLED'}}
            rc = self.p.put(srv_url, config)

            # check stop status
            if rc:
                stat = self.p.get(srv_url)

                retry_cnt, maxcnt, interval = 0, 30, 5
                while stat['ServiceInfo']['state'] != 'INSTALLED':
                    retry_cnt += 1
                    flush_str = '.' * retry_cnt
                    print '\rCheck HDP service %s stop status (timeout: %dmin) %s' % (srv, maxcnt*interval/60, flush_str),
                    sys.stdout.flush()
                    time.sleep(interval)
                    stat = self.p.get(srv_url)
                    if retry_cnt == maxcnt: err('Failed to stop HDP service %s, timeout' % srv)
                # wrap line
                print
            else:
                print 'HDP service %s had already been stopped' % srv

        time.sleep(5)
        # Start
        config = {'RequestInfo': {'context' :'Start All services'}, 'ServiceInfo': {'state' : 'STARTED'}}
        rc = self.p.put(srv_baseurl, config)

        # check start status
        if rc:
            result_url = rc['href']
            stat = self.p.get(result_url)
            retry_cnt, maxcnt, interval = 0, 120, 5
            while stat['Requests']['request_status'] != 'COMPLETED':
                retry_cnt += 1
                flush_str = '.' * retry_cnt
                print '\rCheck HDP services start status (timeout: %dmin) %s' % (maxcnt*interval/60, flush_str),
                sys.stdout.flush()
                time.sleep(interval)
                stat = self.p.get(result_url)
                if retry_cnt == maxcnt: err('Failed to start all HDP services')
            print 'HDP services started successfully!'
        else:
            print 'HDP services had already been started'
Exemplo n.º 7
0
class CDHMod(object):
    """ Modify CDH configs for trafodion and restart CDH services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd)

    def __retry_check(self, cid, maxcnt, interval, msg):
        stat_url = CMD_STAT_URL_PTR % (self.url, cid)
        stat = self.p.get(stat_url)
        retry_cnt = 0
        while not (stat['success'] is True and stat['active'] is False):
            retry_cnt += 1
            flush_str = '.' * retry_cnt
            print '\rCheck CDH services %s status (timeout: %dmin) %s' % (
                msg, maxcnt * interval / 60, flush_str),
            sys.stdout.flush()
            time.sleep(interval)
            stat = self.p.get(stat_url)
            if retry_cnt == maxcnt: return False
        return True

    def mod(self):
        hdfs_service = dbcfgs['hdfs_service_name']
        hbase_service = dbcfgs['hbase_service_name']
        zk_service = dbcfgs['zookeeper_service_name']
        services = {
            hdfs_service: HDFS_CONFIG,
            hbase_service: HBASE_MASTER_CONFIG,
            zk_service: ZK_CONFIG
        }

        for srv, cfg in services.iteritems():
            srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
            self.p.put(srvcfg_url, cfg)

        # set configs in each regionserver group
        rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name,
                                             hbase_service)
        rscfg = self.p.get(rsgrp_baseurl)
        rsgrp_urls = [
            '%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items']
            if r['roleType'] == 'REGIONSERVER'
        ]

        for rsgrp_url in rsgrp_urls:
            self.p.put(rsgrp_url, HBASE_RS_CONFIG)

    def restart(self):
        restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
        deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)

        print 'Restarting CDH services ...'
        rc1 = self.p.post(restart_url)
        if self.__retry_check(rc1['id'], 40, 15, 'restart'):
            print 'Restart CDH successfully!'
        else:
            err('Failed to restart CDH, max retry count reached')

        rc2 = self.p.post(deploy_cfg_url)
        if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
            print 'Deploy client config successfully!'
        else:
            err('Failed to deploy CDH client config, max retry count reached')
Exemplo n.º 8
0
class HDPMod(object):
    """ Modify HDP configs for trafodion and restart HDP services """
    def __init__(self, user, passwd, url, cluster_name):
        self.url = url
        self.cluster_name = cluster_name
        self.p = ParseHttp(user, passwd, json_type=False)

    def mod(self):
        cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
        desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
        cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
        desired_cfg = self.p.get(desired_cfg_url)

        for config_type in MOD_CFGS.keys():
            desired_tag = desired_cfg['Clusters']['desired_configs'][
                config_type]['tag']
            current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
            tag = 'version' + str(int(time.time() * 1000000))
            new_properties = current_cfg['items'][0]['properties']
            new_properties.update(MOD_CFGS[config_type])
            config = {
                'Clusters': {
                    'desired_config': {
                        'type': config_type,
                        'tag': tag,
                        'properties': new_properties
                    }
                }
            }
            self.p.put(cluster_url, config)

    def restart(self):
        srv_baseurl = CLUSTER_URL_PTR % (self.url,
                                         self.cluster_name) + '/services/'
        srvs = ['HBASE', 'ZOOKEEPER', 'HDFS']

        # Stop
        print 'Restarting HDP services ...'
        for srv in srvs:
            srv_url = srv_baseurl + srv
            config = {
                'RequestInfo': {
                    'context': 'Stop %s services' % srv
                },
                'ServiceInfo': {
                    'state': 'INSTALLED'
                }
            }
            rc = self.p.put(srv_url, config)

            # check stop status
            if rc:
                stat = self.p.get(srv_url)

                retry_cnt, maxcnt, interval = 0, 30, 5
                while stat['ServiceInfo']['state'] != 'INSTALLED':
                    retry_cnt += 1
                    flush_str = '.' * retry_cnt
                    print '\rCheck HDP service %s stop status (timeout: %dmin) %s' % (
                        srv, maxcnt * interval / 60, flush_str),
                    sys.stdout.flush()
                    time.sleep(interval)
                    stat = self.p.get(srv_url)
                    if retry_cnt == maxcnt:
                        err('Failed to stop HDP service %s, timeout' % srv)
                # wrap line
                print
            else:
                print 'HDP service %s had already been stopped' % srv

        time.sleep(5)
        # Start
        config = {
            'RequestInfo': {
                'context': 'Start All services'
            },
            'ServiceInfo': {
                'state': 'STARTED'
            }
        }
        rc = self.p.put(srv_baseurl, config)

        # check start status
        if rc:
            result_url = rc['href']
            stat = self.p.get(result_url)
            retry_cnt, maxcnt, interval = 0, 120, 5
            while stat['Requests']['request_status'] != 'COMPLETED':
                retry_cnt += 1
                flush_str = '.' * retry_cnt
                print '\rCheck HDP services start status (timeout: %dmin) %s' % (
                    maxcnt * interval / 60, flush_str),
                sys.stdout.flush()
                time.sleep(interval)
                stat = self.p.get(result_url)
                if retry_cnt == maxcnt: err('Failed to start all HDP services')
            print 'HDP services started successfully!'
        else:
            print 'HDP services had already been started'