class CDHMod(object):
""" Modify CDH configs for trafodion and restart CDH services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd)
def __retry_check(self, cid, maxcnt, interval, msg):
stat_url = CMD_STAT_URL_PTR % (self.url, cid)
stat = self.p.get(stat_url)
retry_cnt = 0
while not (stat['success'] is True and stat['active'] is False):
retry_cnt += 1
flush_str = '.' * retry_cnt
print '\rCheck CDH services %s status (timeout: %dmin) %s' % (msg, maxcnt*interval/60, flush_str),
sys.stdout.flush()
time.sleep(interval)
stat = self.p.get(stat_url)
if retry_cnt == maxcnt: return False
return True
def mod(self):
hdfs_service = dbcfgs['hdfs_service_name']
hbase_service = dbcfgs['hbase_service_name']
zk_service = dbcfgs['zookeeper_service_name']
services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}
for srv, cfg in services.iteritems():
srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
self.p.put(srvcfg_url, cfg)
# set configs in each regionserver group
rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
rscfg = self.p.get(rsgrp_baseurl)
rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']
for rsgrp_url in rsgrp_urls:
self.p.put(rsgrp_url, HBASE_RS_CONFIG)
def restart(self):
restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)
print 'Restarting CDH services ...'
rc1 = self.p.post(restart_url)
if self.__retry_check(rc1['id'], 40, 15, 'restart'):
print 'Restart CDH successfully!'
else:
err('Failed to restart CDH, max retry count reached')
rc2 = self.p.post(deploy_cfg_url)
if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
print 'Deploy client config successfully!'
else:
err('Failed to deploy CDH client config, max retry count reached')
class CDHMod(object):
""" Modify CDH configs for trafodion and restart CDH services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd)
def mod(self):
hdfs_service = dbcfgs['hdfs_service_name']
hbase_service = dbcfgs['hbase_service_name']
zk_service = dbcfgs['zookeeper_service_name']
services = {
hdfs_service: HDFS_CONFIG,
hbase_service: HBASE_MASTER_CONFIG,
zk_service: ZK_CONFIG
}
for srv, cfg in services.iteritems():
srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
self.p.put(srvcfg_url, cfg)
# set configs in each regionserver group
rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name,
hbase_service)
rscfg = self.p.get(rsgrp_baseurl)
rsgrp_urls = [
'%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items']
if r['roleType'] == 'REGIONSERVER'
]
for rsgrp_url in rsgrp_urls:
self.p.put(rsgrp_url, HBASE_RS_CONFIG)
def restart(self):
restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)
def __retry(url, maxcnt, interval, msg):
rc = self.p.post(url)
stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
get_stat = lambda: self.p.get(stat_url)[
'success'] is True and self.p.get(stat_url)['active'] is False
retry(get_stat, maxcnt, interval, msg)
info('Restarting CDH services ...')
__retry(restart_url, 40, 15, 'CDH services restart')
info('Deploying CDH client configs ...')
__retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
class CDHMod(object):
""" Modify CDH configs for trafodion and restart CDH services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd)
def mod(self):
hdfs_service = dbcfgs['hdfs_service_name']
hbase_service = dbcfgs['hbase_service_name']
zk_service = dbcfgs['zookeeper_service_name']
services = {hdfs_service:HDFS_CONFIG, hbase_service:HBASE_MASTER_CONFIG, zk_service:ZK_CONFIG}
for srv, cfg in services.iteritems():
srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
self.p.put(srvcfg_url, cfg)
# set configs in each regionserver group
rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name, hbase_service)
rscfg = self.p.get(rsgrp_baseurl)
rsgrp_urls = ['%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items'] if r['roleType'] == 'REGIONSERVER']
for rsgrp_url in rsgrp_urls:
self.p.put(rsgrp_url, HBASE_RS_CONFIG)
def restart(self):
restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)
def __retry(url, maxcnt, interval, msg):
rc = self.p.post(url)
stat_url = CMD_STAT_URL_PTR % (self.url, rc['id'])
get_stat = lambda: self.p.get(stat_url)['success'] is True and self.p.get(stat_url)['active'] is False
retry(get_stat, maxcnt, interval, msg)
info('Restarting CDH services ...')
__retry(restart_url, 40, 15, 'CDH services restart')
info('Deploying CDH client configs ...')
__retry(deploy_cfg_url, 30, 10, 'CDH services deploy')
class HDPMod(object):
""" Modify HDP configs for trafodion and restart HDP services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd, json_type=False)
def mod(self):
cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
desired_cfg = self.p.get(desired_cfg_url)
hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' % cluster_url)
rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]
hbase_hregion_property = 'hbase.hregion.impl'
hbase_config_group = {
'ConfigGroup': {
'cluster_name': self.cluster_name,
'group_name': 'hbase-regionserver',
'tag': 'HBASE',
'description': 'HBase Regionserver configs for Trafodion',
'hosts': [{'host_name': host} for host in rsnodes],
'desired_configs': [
{
'type': 'hbase-site',
'tag': 'traf_cg',
'properties': {hbase_hregion_property: MOD_CFGS['hbase-site'].pop(hbase_hregion_property)}
}
]
}
}
self.p.post('%s/config_groups' % cluster_url, hbase_config_group)
for config_type in MOD_CFGS.keys():
desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
tag = 'version' + str(int(time.time() * 1000000))
new_properties = current_cfg['items'][0]['properties']
new_properties.update(MOD_CFGS[config_type])
config = {
'Clusters': {
'desired_config': {
'type': config_type,
'tag': tag,
'properties': new_properties
}
}
}
self.p.put(cluster_url, config)
def restart(self):
info('Restarting HDP services ...')
def startstop(srvs, action, count, interval):
srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
state = 'INSTALLED' if action == 'stop' else 'STARTED'
for srv in srvs:
srv_url = srv_baseurl + srv
config = {'RequestInfo': {'context' :'%s %s services' % (action, srv)}, 'ServiceInfo': {'state' : state}}
rc = self.p.put(srv_url, config)
# check startstop status
if rc:
get_stat = lambda: self.p.get(srv_url)['ServiceInfo']['state'] == state
retry(get_stat, count, interval, 'HDP service %s %s' % (srv, action))
else:
if action == 'stop': action += 'p'
info('HDP service %s had already been %sed' % (srv, action))
srvs = ['HBASE', 'HDFS', 'ZOOKEEPER']
startstop(srvs, 'stop', 30, 10)
time.sleep(10)
srvs.reverse()
startstop(srvs, 'start', 60, 10)
class HDPMod(object):
""" Modify HDP configs for trafodion and restart HDP services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd, json_type=False)
def mod(self):
cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
desired_cfg = self.p.get(desired_cfg_url)
hdp = self.p.get('%s/services/HBASE/components/HBASE_REGIONSERVER' %
cluster_url)
rsnodes = [c['HostRoles']['host_name'] for c in hdp['host_components']]
hbase_hregion_property = 'hbase.hregion.impl'
hbase_config_group = {
'ConfigGroup': {
'cluster_name':
self.cluster_name,
'group_name':
'hbase-regionserver',
'tag':
'HBASE',
'description':
'HBase Regionserver configs for Trafodion',
'hosts': [{
'host_name': host
} for host in rsnodes],
'desired_configs': [{
'type': 'hbase-site',
'tag': 'traf_cg',
'properties': {
hbase_hregion_property:
MOD_CFGS['hbase-site'].pop(hbase_hregion_property)
}
}]
}
}
self.p.post('%s/config_groups' % cluster_url, hbase_config_group)
if dbcfgs['secure_hadoop'].upper() == 'Y':
secure_cfgs = ',org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.token.TokenProvider'
MOD_CFGS['hbase-site'][
'hbase.coprocessor.region.classes'] += secure_cfgs
for config_type in MOD_CFGS.keys():
desired_tag = desired_cfg['Clusters']['desired_configs'][
config_type]['tag']
current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
tag = 'version' + str(int(time.time() * 1000000))
new_properties = current_cfg['items'][0]['properties']
new_properties.update(MOD_CFGS[config_type])
config = {
'Clusters': {
'desired_config': {
'type': config_type,
'tag': tag,
'properties': new_properties
}
}
}
self.p.put(cluster_url, config)
def restart(self):
info('Restarting HDP services ...')
def startstop(srvs, action, count, interval):
srv_baseurl = CLUSTER_URL_PTR % (self.url,
self.cluster_name) + '/services/'
state = 'INSTALLED' if action == 'stop' else 'STARTED'
for srv in srvs:
srv_url = srv_baseurl + srv
config = {
'RequestInfo': {
'context': '%s %s services' % (action, srv)
},
'ServiceInfo': {
'state': state
}
}
rc = self.p.put(srv_url, config)
# check startstop status
if rc:
get_stat = lambda: self.p.get(srv_url)['ServiceInfo'][
'state'] == state
retry(get_stat, count, interval,
'HDP service %s %s' % (srv, action))
else:
if action == 'stop': action += 'p'
info('HDP service %s had already been %sed' %
(srv, action))
srvs = ['HBASE', 'HDFS', 'ZOOKEEPER']
startstop(srvs, 'stop', 30, 10)
time.sleep(10)
srvs.reverse()
startstop(srvs, 'start', 60, 10)
class HDPMod(object):
""" Modify HDP configs for trafodion and restart HDP services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd, json_type=False)
def mod(self):
cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
desired_cfg = self.p.get(desired_cfg_url)
for config_type in MOD_CFGS.keys():
desired_tag = desired_cfg['Clusters']['desired_configs'][config_type]['tag']
current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
tag = 'version' + str(int(time.time() * 1000000))
new_properties = current_cfg['items'][0]['properties']
new_properties.update(MOD_CFGS[config_type])
config = {
'Clusters': {
'desired_config': {
'type': config_type,
'tag': tag,
'properties': new_properties
}
}
}
self.p.put(cluster_url, config)
def restart(self):
srv_baseurl = CLUSTER_URL_PTR % (self.url, self.cluster_name) + '/services/'
srvs = ['HBASE', 'ZOOKEEPER', 'HDFS']
# Stop
print 'Restarting HDP services ...'
for srv in srvs:
srv_url = srv_baseurl + srv
config = {'RequestInfo': {'context' :'Stop %s services' % srv}, 'ServiceInfo': {'state' : 'INSTALLED'}}
rc = self.p.put(srv_url, config)
# check stop status
if rc:
stat = self.p.get(srv_url)
retry_cnt, maxcnt, interval = 0, 30, 5
while stat['ServiceInfo']['state'] != 'INSTALLED':
retry_cnt += 1
flush_str = '.' * retry_cnt
print '\rCheck HDP service %s stop status (timeout: %dmin) %s' % (srv, maxcnt*interval/60, flush_str),
sys.stdout.flush()
time.sleep(interval)
stat = self.p.get(srv_url)
if retry_cnt == maxcnt: err('Failed to stop HDP service %s, timeout' % srv)
# wrap line
print
else:
print 'HDP service %s had already been stopped' % srv
time.sleep(5)
# Start
config = {'RequestInfo': {'context' :'Start All services'}, 'ServiceInfo': {'state' : 'STARTED'}}
rc = self.p.put(srv_baseurl, config)
# check start status
if rc:
result_url = rc['href']
stat = self.p.get(result_url)
retry_cnt, maxcnt, interval = 0, 120, 5
while stat['Requests']['request_status'] != 'COMPLETED':
retry_cnt += 1
flush_str = '.' * retry_cnt
print '\rCheck HDP services start status (timeout: %dmin) %s' % (maxcnt*interval/60, flush_str),
sys.stdout.flush()
time.sleep(interval)
stat = self.p.get(result_url)
if retry_cnt == maxcnt: err('Failed to start all HDP services')
print 'HDP services started successfully!'
else:
print 'HDP services had already been started'
class CDHMod(object):
""" Modify CDH configs for trafodion and restart CDH services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd)
def __retry_check(self, cid, maxcnt, interval, msg):
stat_url = CMD_STAT_URL_PTR % (self.url, cid)
stat = self.p.get(stat_url)
retry_cnt = 0
while not (stat['success'] is True and stat['active'] is False):
retry_cnt += 1
flush_str = '.' * retry_cnt
print '\rCheck CDH services %s status (timeout: %dmin) %s' % (
msg, maxcnt * interval / 60, flush_str),
sys.stdout.flush()
time.sleep(interval)
stat = self.p.get(stat_url)
if retry_cnt == maxcnt: return False
return True
def mod(self):
hdfs_service = dbcfgs['hdfs_service_name']
hbase_service = dbcfgs['hbase_service_name']
zk_service = dbcfgs['zookeeper_service_name']
services = {
hdfs_service: HDFS_CONFIG,
hbase_service: HBASE_MASTER_CONFIG,
zk_service: ZK_CONFIG
}
for srv, cfg in services.iteritems():
srvcfg_url = SRVCFG_URL_PTR % (self.url, self.cluster_name, srv)
self.p.put(srvcfg_url, cfg)
# set configs in each regionserver group
rsgrp_baseurl = RSGRP_BASEURL_PTR % (self.url, self.cluster_name,
hbase_service)
rscfg = self.p.get(rsgrp_baseurl)
rsgrp_urls = [
'%s/%s/config' % (rsgrp_baseurl, r['name']) for r in rscfg['items']
if r['roleType'] == 'REGIONSERVER'
]
for rsgrp_url in rsgrp_urls:
self.p.put(rsgrp_url, HBASE_RS_CONFIG)
def restart(self):
restart_url = RESTART_URL_PTR % (self.url, self.cluster_name)
deploy_cfg_url = DEPLOY_CFG_URL_PTR % (self.url, self.cluster_name)
print 'Restarting CDH services ...'
rc1 = self.p.post(restart_url)
if self.__retry_check(rc1['id'], 40, 15, 'restart'):
print 'Restart CDH successfully!'
else:
err('Failed to restart CDH, max retry count reached')
rc2 = self.p.post(deploy_cfg_url)
if self.__retry_check(rc2['id'], 30, 10, 'deploy'):
print 'Deploy client config successfully!'
else:
err('Failed to deploy CDH client config, max retry count reached')
class HDPMod(object):
""" Modify HDP configs for trafodion and restart HDP services """
def __init__(self, user, passwd, url, cluster_name):
self.url = url
self.cluster_name = cluster_name
self.p = ParseHttp(user, passwd, json_type=False)
def mod(self):
cluster_url = CLUSTER_URL_PTR % (self.url, self.cluster_name)
desired_cfg_url = cluster_url + '?fields=Clusters/desired_configs'
cfg_url = cluster_url + '/configurations?type={0}&tag={1}'
desired_cfg = self.p.get(desired_cfg_url)
for config_type in MOD_CFGS.keys():
desired_tag = desired_cfg['Clusters']['desired_configs'][
config_type]['tag']
current_cfg = self.p.get(cfg_url.format(config_type, desired_tag))
tag = 'version' + str(int(time.time() * 1000000))
new_properties = current_cfg['items'][0]['properties']
new_properties.update(MOD_CFGS[config_type])
config = {
'Clusters': {
'desired_config': {
'type': config_type,
'tag': tag,
'properties': new_properties
}
}
}
self.p.put(cluster_url, config)
def restart(self):
srv_baseurl = CLUSTER_URL_PTR % (self.url,
self.cluster_name) + '/services/'
srvs = ['HBASE', 'ZOOKEEPER', 'HDFS']
# Stop
print 'Restarting HDP services ...'
for srv in srvs:
srv_url = srv_baseurl + srv
config = {
'RequestInfo': {
'context': 'Stop %s services' % srv
},
'ServiceInfo': {
'state': 'INSTALLED'
}
}
rc = self.p.put(srv_url, config)
# check stop status
if rc:
stat = self.p.get(srv_url)
retry_cnt, maxcnt, interval = 0, 30, 5
while stat['ServiceInfo']['state'] != 'INSTALLED':
retry_cnt += 1
flush_str = '.' * retry_cnt
print '\rCheck HDP service %s stop status (timeout: %dmin) %s' % (
srv, maxcnt * interval / 60, flush_str),
sys.stdout.flush()
time.sleep(interval)
stat = self.p.get(srv_url)
if retry_cnt == maxcnt:
err('Failed to stop HDP service %s, timeout' % srv)
# wrap line
print
else:
print 'HDP service %s had already been stopped' % srv
time.sleep(5)
# Start
config = {
'RequestInfo': {
'context': 'Start All services'
},
'ServiceInfo': {
'state': 'STARTED'
}
}
rc = self.p.put(srv_baseurl, config)
# check start status
if rc:
result_url = rc['href']
stat = self.p.get(result_url)
retry_cnt, maxcnt, interval = 0, 120, 5
while stat['Requests']['request_status'] != 'COMPLETED':
retry_cnt += 1
flush_str = '.' * retry_cnt
print '\rCheck HDP services start status (timeout: %dmin) %s' % (
maxcnt * interval / 60, flush_str),
sys.stdout.flush()
time.sleep(interval)
stat = self.p.get(result_url)
if retry_cnt == maxcnt: err('Failed to start all HDP services')
print 'HDP services started successfully!'
else:
print 'HDP services had already been started'