def main():
"""
Kerberizes a cluster.
@rtype: number
@returns: A number representing the status of success.
"""
settings = retrieve_args()
api = ApiResource(settings.host, settings.port, settings.username,
settings.password, settings.use_tls, 8)
cloudera_manager = api.get_cloudera_manager()
cluster = api.get_cluster(settings.cluster)
mgmt_service = cloudera_manager.get_service()
if verify_cloudera_manager_has_kerberos_principal(cloudera_manager):
wait_for_command('Stopping the cluster', cluster.stop())
wait_for_command('Stopping MGMT services', mgmt_service.stop())
configure_services(cluster)
wait_for_generate_credentials(cloudera_manager)
wait_for_command('Deploying client configs.', cluster.deploy_client_config())
wait_for_command('Deploying cluster client configs', cluster.deploy_cluster_client_config())
wait_for_command('Starting MGMT services', mgmt_service.start())
wait_for_command('Starting the cluster', cluster.start())
else:
print "Cluster does not have Kerberos admin credentials. Exiting!"
return 0
def main():
module = AnsibleModule(argument_spec=MODULE_ARGUMENTS)
host_a = module.params.get('host', None)
pass_a = module.params.get('pass', None)
user_a = module.params.get('user', None)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((host_a, 7183))
if result == 0:
# Port is open then use https port
api = ApiResource(server_host=host_a,
username=user_a,
password=pass_a,
use_tls=True,
version=9)
else:
# Port is not open then use http port
api = ApiResource(server_host=host_a,
username=user_a,
password=pass_a,
version=9)
cdh_config = CdhConfiguration(api, CLUSTER_NAME)
module.exit_json(changed=True,
msg='Collected',
config=cdh_config.get_config_for_cluster(CLUSTER_NAME))
def init_cluster():
# wait for all cloudera agent processes to come up
setup_logger.info("Creating Clutser.")
BDVLIB_ServiceWait([["services", "cloudera_scm_agent", NODE_GROUP_ID]])
# make sure cloudera manager has received registration
# for all new agents
all_cloudera_hosts = get_hosts_for_service(
["services", "cloudera_scm_agent"])
api = ApiResource(CM_HOST, username=ADMIN_USER, password=ADMIN_PASS)
while True:
current_all_hosts = map(lambda x: x.hostname, api.get_all_hosts())
setup_logger.info("Currently registered hosts with CM " +
str(current_all_hosts))
if all(x in current_all_hosts for x in all_cloudera_hosts):
break
setup_logger.info(
"waiting for new nodes to register with cloudera manager")
time.sleep(10)
manager = api.get_cloudera_manager()
manager.update_config(CM_CONFIG)
cluster = api.create_cluster(CLUSTER_NAME, CDH_MAJOR_VERSION,
CDH_FULL_VERSION)
cluster.add_hosts(ALL_HOSTS)
# turn off host swap alerting
hosts_swap_alert_off(api)
setup_logger.info("Setting Up SPARK2 Repo....")
add_spark2_repo(api)
##Set java home
setup_logger.info("Setting Up Java Path....")
hosts_set_javahome(api)
return (cluster, manager)
def main():
"""
Add peer to the cluster.
@rtype: number
@returns: A number representing the status of success.
"""
settings = parse_args()
if len(sys.argv) == 1 or len(sys.argv) > 17:
print_usage_message()
quit(1)
api_target = ApiResource(settings.server, settings.port, settings.username,
settings.password, settings.use_tls, 14)
cloudera_manager = api_target.get_cloudera_manager()
try:
cloudera_manager.create_peer(settings.peer_name,
settings.source_cm_url,
settings.source_user,
settings.source_password)
print "Peer Successfully Added"
except ApiException as error:
if 'already exists' in str(error):
print 'Peer Already exists'
else:
raise error
return 0
def connect(cm_api, cm_username, cm_password, use_proxy=False):
'''
Wait for ten minutes for CM to come up
'''
for _ in xrange(120):
try:
logging.info("Checking CM availability....")
# change name of proxy if necessary
proxy = urllib2.ProxyHandler({'http': 'proxy'})
api = ApiResource(cm_api,
username=cm_username,
password=cm_password,
version=14)
if use_proxy:
# pylint: disable=W0212
api._client._opener.add_handler(proxy)
cloudera_manager = api.get_cloudera_manager()
api.get_user(cm_username)
return api, cloudera_manager
except Exception:
logging.warning("CM is not up")
time.sleep(5)
logging.error("CM did not come UP")
sys.exit(-1)
class ImpalaCluster(object):
def __init__(self, cm_host, cm_cluster_name, username, password):
self.cm_api = ApiResource(cm_host, username=username, password=password)
self.hosts = dict()
self.services = list()
self.cluster = self.cm_api.get_cluster(cm_cluster_name)
if self.cluster is None:
raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name
self.__load_hosts()
self.__impala_service = ImpalaService(self)
def _get_all_services(self):
return self.cluster.get_all_services()
def get_impala_service(self):
return self.__impala_service
def __load_hosts(self):
self.hosts = dict()
# Search for all hosts that are in the target cluster.
# There is no API that provides the list of host in a given cluster, so to find them
# we must loop through all the hosts and check the cluster name matches.
for host_info in self.cm_api.get_all_hosts():
# host_info doesn't include a link to the roleRef so need to do another lookup
# based on the hostId.
host = self.cm_api.get_host(host_info.hostId)
for roleRef.get('clusterName') == self.cluster_name:
self.hosts[host_info.hostId] = Host(host)
break
def __init__(self,service,role,host,list):
self.service = service.lower()
self.role = role.lower()
self.host = host.lower()
self.list = list.lower()
cm_host = '10.7.177.234'
self.api = ApiResource(cm_host, username="******", password="******")
def main():
"""
Kerberizes a cluster.
@rtype: number
@returns: A number representing the status of success.
"""
settings = retrieve_args()
api = ApiResource(settings.host, settings.port, settings.username,
settings.password, settings.use_tls, 8)
cloudera_manager = api.get_cloudera_manager()
cluster = api.get_cluster(settings.cluster)
mgmt_service = cloudera_manager.get_service()
if verify_cloudera_manager_has_kerberos_principal(cloudera_manager):
wait_for_command('Stopping the cluster', cluster.stop())
wait_for_command('Stopping MGMT services', mgmt_service.stop())
configure_services(cluster)
wait_for_generate_credentials(cloudera_manager)
wait_for_command('Deploying client configs.',
cluster.deploy_client_config())
wait_for_command('Deploying cluster client configs',
cluster.deploy_cluster_client_config())
wait_for_command('Starting MGMT services', mgmt_service.start())
wait_for_command('Starting the cluster', cluster.start())
else:
print "Cluster does not have Kerberos admin credentials. Exiting!"
return 0
def get_hosts(self):
hosts = {}
from cm_api.api_client import ApiResource
api = ApiResource(self.host, self.port, self.username, self.password)
for h in api.get_all_hosts():
hosts[h.hostId] = h.ipAddress
return hosts
def do_call(host, port, user, password, cluster_name, service_role_name,
random_index):
api = ApiResource(host, port, user, password, False, MAN_API_VERSION)
for cluster in api.get_all_clusters():
if cluster_name is None:
break
elif cluster_name == cluster.name:
break
if cluster_name is not None and cluster_name != cluster.name:
print >> sys.stderr, "Cloud not find cluster: " + cluster_name
return -2
do_print_header()
for service in cluster.get_all_services():
do_print_line_item(api, service, service_role_name, random_index,
'HDFS', 'NAMENODE', 'namenode_port', [], [])
do_print_line_item(api, service, service_role_name, random_index,
'KUDU', 'KUDU_MASTER', 'webserver_port', [], [])
do_print_line_item(api, service, service_role_name, random_index,
'HUE', 'HUE_SERVER', 'hue_http_port', [], [])
do_print_line_item(api, service, service_role_name, random_index,
'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port',
[], [])
do_print_line_item(api, service, service_role_name, random_index,
'IMPALA', 'IMPALAD', 'beeswax_port', [], [])
do_print_line_item(api, service, service_role_name, random_index,
'FLUME', 'AGENT', 'agent_http_port', [], [])
do_print_line_item(api, service, service_role_name, random_index,
'KAFKA', 'KAFKA_BROKER', 'port', [], [])
do_print_line_item(api, service, service_role_name, random_index,
'ZOOKEEPER', 'SERVER', 'clientPort', [], [])
do_print_footer()
def setup(self,
p_cm_host,
p_cm_user,
p_cm_pass,
p_cm_version,
p_cluster,
p_cm_port=None,
p_use_tls=False):
self.cm_api = ApiResource(p_cm_host,
server_port=p_cm_port,
version=p_cm_version,
username=p_cm_user,
password=p_cm_pass,
use_tls=p_use_tls)
handler_cm_api.cluster_hosts = self.cm_api.get_all_hosts()
if p_cluster:
self.cluster = filter(lambda x: x.displayName == p_cluster,
self.cm_api.get_all_clusters())[0]
if not self.cluster:
print("Error: That cluster is not valid.")
return
else:
self.services = self.cluster.get_all_services()
self.name = self.cluster.displayName
tmp_topology = self.cluster.list_hosts()
self.topology = {}
for i in range(len(tmp_topology)):
tmp_host = filter(lambda x: x.hostId == tmp_topology[i].hostId,
handler_cm_api.cluster_hosts)[0]
self.topology[tmp_topology[i].hostId] = tmp_host.hostname
def main():
module = build_module()
choice_map = {'present': present, 'distributed': distributed, 'activated': activated, 'absent': absent, 'infos': infos}
params = module.params
has_changed = False
api = ApiResource(params["cm_host"], username=params["cm_login"], password=params["cm_login"], version=params["api_version"])
try:
cluster = api.get_cluster(params["cluster_name"])
except ApiException as e:
module.fail_json(msg="Cluster error : {0}".format(e))
if params["product"] and params["version"]:
parcel = get_parcel(cluster, params["product"], params["version"])
if params["state"] != "infos":
error, has_changed, result, meta = choice_map.get(params['state'])(cluster, parcel)
if error:
module.fail_json(msg=result)
module.exit_json(changed=has_changed, msg=result, meta=meta)
else:
meta = {
"product": parcel.product,
"version": parcel.version,
"stage": parcel.stage
}
module.exit_json(changed=False, msg="Parcel informations gathered", meta=meta)
elif not params["product"] and not params["version"] and params["state"] == "infos":
module.exit_json(changed=has_changed, msg="Parcel informations gathered", meta=infos(cluster))
def init_cluster():
# wait for all cloudera agent processes to come up
BDVLIB_ServiceWait(
[["services", "cloudera_scm_agent", NODE_GROUP_ID, "kts"]])
# make sure cloudera manager has received registration
# for all new agents
all_cloudera_hosts = get_hosts_for_service(
["services", "cloudera_scm_agent"])
api = ApiResource(CM_HOST, username="******", password="******")
while True:
current_all_hosts = map(lambda x: x.hostname, api.get_all_hosts())
setup_logger.info("Currently registered hosts with CM " +
str(current_all_hosts))
if all(x in current_all_hosts for x in all_cloudera_hosts):
break
setup_logger.info(
"waiting for new nodes to register with cloudera manager")
time.sleep(10)
manager = api.get_cloudera_manager()
manager.update_config(CM_CONFIG)
cluster = api.create_cluster(CLUSTER_NAME, CDH_MAJOR_VERSION,
CDH_FULL_VERSION)
KTS_HOSTS = ConfigMeta.getWithTokens(
['nodegroups', NODE_GROUP_ID, 'roles', 'kts', 'fqdns'])
cluster.add_hosts(KTS_HOSTS)
return (cluster, manager)
def main():
#print sys.argv[0]
#for i in range(1, len(sys.argv)):
# print "param ", i, sys.argv[i]
# get a handle on the instance of CM that we have running
api = ApiResource(cm_host, cm_port, cm_username, cm_password, version=13)
# get the CM instancepython2.7 setuptools
cm = ClouderaManager(api)
cluster = api.get_cluster(cluster_name)
# distribution_parcels(api, cluster)
cmd = cluster.first_run()
while cmd.success == None:
cmd = cmd.fetch()
if cmd.success != True:
print "The first run command failed: " + cmd.resultMessage()
exit(0)
print "First run successfully executed. Your cluster has been set up!"
def adjust_yarn_memory_limits(region, stack_name, restart=True):
ec2_conn = create_ec2_connection(region)
manager_instance = get_manager_instance(ec2_conn, stack_name)
with cm_tunnel_ctx(manager_instance) as local_port:
cm_api = ApiResource('localhost', username='******', password='******',
server_port=local_port, version=9)
cluster = list(cm_api.get_all_clusters())[0]
host = list(cm_api.get_all_hosts())[0] # all hosts same instance type
yarn = filter(lambda x: x.type == 'YARN',
list(cluster.get_all_services()))[0]
rm_cg = filter(lambda x: x.roleType == 'RESOURCEMANAGER',
list(yarn.get_all_role_config_groups()))[0]
nm_cg = filter(lambda x: x.roleType == 'NODEMANAGER',
list(yarn.get_all_role_config_groups()))[0]
rm_cg.update_config({
'yarn_scheduler_maximum_allocation_mb': (
int(host.totalPhysMemBytes / 1024. / 1024.)),
'yarn_scheduler_maximum_allocation_vcores': host.numCores})
nm_cg.update_config({
'yarn_nodemanager_resource_memory_mb': (
int(host.totalPhysMemBytes / 1024. / 1024.)),
'yarn_nodemanager_resource_cpu_vcores': host.numCores})
cluster.deploy_client_config().wait()
if restart:
cluster.restart().wait()
def create_cluster():
CM_HOST = (load_cfg(ansible_path+"/group_vars/all")).get("cm_host")
USERNAME = (load_cfg(ansible_path+"/group_vars/all")).get("cm_username")
PASSWORD = (load_cfg(ansible_path+"/group_vars/all")).get("cm_password")
api = ApiResource(CM_HOST, version=API_VERSION, username=USERNAME, password=PASSWORD)
cluster = api.get_cluster(CLUSTER_NAME)
return cluster
def connect(cm_api, cm_username, cm_password, use_proxy=False):
'''
Wait for ten minutes for CM to come up
'''
for _ in xrange(120):
try:
logging.info("Checking CM availability....")
# change name of proxy if necessary
proxy = urllib2.ProxyHandler({'http': 'proxy'})
api = ApiResource(cm_api, username=cm_username, password=cm_password, version=14)
if use_proxy:
# pylint: disable=W0212
api._client._opener.add_handler(proxy)
cloudera_manager = api.get_cloudera_manager()
api.get_user(cm_username)
return api, cloudera_manager
except Exception:
logging.warning("CM is not up")
time.sleep(5)
logging.error("CM did not come UP")
sys.exit(-1)
def __init__(self,
hostname=None,
port='7180',
username='******',
password='******'):
'''
Creates a Resource object that provides API endpoints.
@param hostname: Hostname of the Cloudera Manager server.
@param port: Port of the server. Defaults to 7180 (http).
@param username: Login name.
@param password: Login password.
'''
config = Config()
salt = Salt()
if hostname is None:
hostname = salt.function('roles.dict', 'cloudera-cm4-server')['cloudera-cm4-server'][0]
try:
logger.debug('Acquiring Cloudera Manager API resource')
ApiResource.__init__(self, hostname, port, username, password)
except ApiException:
logger.error('Problem acquiring Cloudera Manager API resource')
# sys.exit(1)
except URLError:
logger.error('Problem connecting to %s', hostname)
sys.exit(1)
else:
self.config = config
self.salt = salt
cluster = ClouderaManagerCluster(self)
host = ClouderaManagerHost(self)
def setup_api_resources(self):
self.api = ApiResource(server_host=self.cm_server_address, server_port=self.cm_server_port,
username=self.username, password=self.password,
version=self._get_api_version())
self.cm = self.api.get_cloudera_manager()
self.cluster = self.api.get_cluster('Cluster 1 (clusterdock)')
def getActiveCMConfig(totalconfig):
cmConfig = {}
for cm in totalconfig['cmfqdn']:
api = ApiResource(cm, totalconfig[cm]['port'], totalconfig[cm]['user'],
totalconfig[cm]['passwd'], totalconfig[cm]['tls'],
totalconfig[cm]['apiv'])
clusters = api.get_all_clusters()
cmConfig[cm] = {}
for cluster in clusters:
cmConfig[cm][cluster.displayName] = {}
services = cluster.get_all_services()
for service in services:
cmConfig[cm][cluster.displayName][service.name] = {}
cmConfig[cm][cluster.displayName][service.name]['Service'] = {}
for name, config in service.get_config(view='full')[0].items():
cmConfig[cm][cluster.displayName][
service.name]['Service'][name] = {
'value': config.value,
'default': config.default
}
for roleGroup in service.get_all_role_config_groups():
cmConfig[cm][cluster.displayName][service.name][
roleGroup.roleType] = {}
for name, config in roleGroup.get_config(
view='full').items():
cmConfig[cm][cluster.displayName][service.name][
roleGroup.roleType][name] = {
'value': config.value,
'default': config.default
}
print(roleGroup.roleType)
#print(json.dumps(cmConfig, indent=4))
return cmConfig
def main():
"""
Add peer to the cluster.
@rtype: number
@returns: A number representing the status of success.
"""
settings = parse_args()
if len(sys.argv) == 1 or len(sys.argv) > 17:
print_usage_message()
quit(1)
api_target = ApiResource(settings.server, settings.port, settings.username,
settings.password, settings.use_tls, 14)
type_name = 'AWS_ACCESS_KEY_AUTH'
account_configs = {
'aws_access_key': settings.aws_access_key,
'aws_secret_key': settings.aws_secret_key
}
try:
api_target.create_external_account(settings.account_name,
settings.account_name,
type_name,
account_configs=account_configs)
print "S3 Account Successfully Added"
except ApiException as error:
if 'already exists' in str(error):
print 'Peer Already exists'
else:
raise error
return 0
def __init__(self, hostname=None, port="7180", username="******", password="******"):
"""
Creates a Resource object that provides API endpoints.
@param hostname: Hostname of the Cloudera Manager server.
@param port: Port of the server. Defaults to 7180 (http).
@param username: Login name.
@param password: Login password.
"""
config = Config()
salt = Salt()
if hostname is None:
hostname = salt.function("roles.dict", "cloudera-cm5-server")["cloudera-cm5-server"][0]
try:
logger.debug("Acquiring Cloudera Manager API resource")
ApiResource.__init__(self, hostname, port, username, password)
except ApiException:
logger.error("Problem acquiring Cloudera Manager API resource")
# sys.exit(1)
except URLError:
logger.error("Problem connecting to %s", hostname)
sys.exit(1)
else:
self.config = config
self.salt = salt
cluster = ClouderaManagerCluster(self)
host = ClouderaManagerHost(self)
def main():
# connect cm api
api = ApiResource(CM_HOST,
7180,
username=CM_USERNAME,
password=CM_PASSWORD)
manager = api.get_cloudera_manager()
# no need to update cm config
#manager.update_config(cm_host)
print("[INFO] Connected to CM host on " + CM_HOST)
# create cluster object
try:
cluster = api.get_cluster(name=CLUSTER_NAME)
except:
cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION,
CLUSTER_NODE_COUNT)
print("[INFO] Initialized cluster " + CLUSTER_NAME +
" which uses CDH version " + CLUSTER_VERSION)
#
mgmt_servicename = "MGMT"
amon_role_name = "ACTIVITYMONITOR"
apub_role_name = "ALERTPUBLISHER"
eserv_role_name = "EVENTSERVER"
hmon_role_name = "HOSTMONITOR"
smon_role_name = "SERVICEMONITOR"
nav_role_name = "NAVIGATOR"
navms_role_name = "NAVIGATORMETADATASERVER"
rman_role_name = "REPORTMANAGER"
deploy_management(manager, mgmt_servicename, amon_role_name,
apub_role_name, eserv_role_name, hmon_role_name,
smon_role_name, nav_role_name, navms_role_name,
rman_role_name)
print("[INFO] Deployed CM management service " + mgmt_servicename +
" to run on " + CM_HOST)
#
assign_roles(api, cluster)
print("[INFO] all roles have assigned.")
#
# Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400)
try:
cluster.auto_configure()
except:
pass
update_custom_config(api, cluster)
print("[INFO] all servies and roles have configured.")
#
cmd = cluster.first_run()
while cmd.success == None:
cmd = cmd.fetch()
if not cmd.success:
print("[ERROR] The first run command failed: " + cmd.resultMessage())
else:
print(
"[INFO] First run successfully executed. Your cluster has been set up!"
)
def list_hosts(host, username, password, cafile):
context = ssl.create_default_context(cafile=cafile)
api = ApiResource(host, username=username, password=password, use_tls=True,
ssl_context=context)
for h in api.get_all_hosts():
print h.hostname
def main():
cmhost = os.environ['DEPLOYMENT_HOST_PORT'].split(":")[0]
api = ApiResource(cmhost, username='******', password='******')
all_clusters = api.get_all_clusters()
for cluster in all_clusters:
if (cluster.name == os.environ['CLUSTER_NAME']):
break
template = cluster.create_host_template("cdsw-gateway")
def main(): API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS) print "Connected to CM host on " + CM_HOST CLUSTER = API.get_cluster(CLUSTER_NAME) print "About to restart cluster." CLUSTER.restart().wait() print "Done restarting cluster."
def main(): API = ApiResource(CM_HOST, version=5, username=ADMIN_USER, password=ADMIN_PASS) print "Connected to CM host on " + CM_HOST CLUSTER = API.get_cluster(CLUSTER_NAME) print "About to stop cluster." CLUSTER.stop().wait() print "Done stopping cluster."
def get_cluster_specs():
cm_api = ApiResource(os.environ['MANAGER_HOST'], username='******',
password='******', server_port=7180, version=9)
host = list(cm_api.get_all_hosts())[0] # all hosts same instance type
cluster = list(cm_api.get_all_clusters())[0]
yarn = filter(lambda x: x.type == 'YARN',
list(cluster.get_all_services()))[0]
return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
'num_cores': host.numCores, 'node_memory': host.totalPhysMemBytes}
def main(cm_host, user, password):
api = ApiResource(cm_host, username=user, password=password)
cm = api.get_cloudera_manager()
cm.update_all_hosts_config(
{"java_home": "/usr/java/jdk1.8.0_121-cloudera"})
print("restarting CM service - this will take a minute or so")
cm.get_service().restart().wait()
print("restarting cluster - this will take 2-5 minutes")
api.get_all_clusters()[0].restart(restart_only_stale_services=True,
redeploy_client_configuration=True).wait()
def start(host, user, passw):
cm_host = str(host)
api = ApiResource(cm_host,
7180,
username=str(username),
password=str(password),
version=9)
mgmt = api.get_cloudera_manager().get_service()
mgmt.restart()
print("Services successfully started")
def getClusterInformation(self):
api = ApiResource(self.cm_host, username=self.user, password=self.passwd)
logger.info('Received; user -> %s, password -> %s, host -> %s', self.user, self.passwd, self.cm_host)
for c in api.get_all_clusters():
clusterInf = "Cluster name %s and version %s" %(c.name, c.version)
#print "Cluster name %s and version %s" %(c.name, c.version)
logger.info("Cluster name %s and version %s", c.name, c.version)
if c.version == "CDH5":
cdh5 = c
return cdh5, clusterInf
def main(): parser = cm_args_parser() args = parser.parse_args() print "connecting to host:" + args.cm_host + "..." api = ApiResource(args.cm_host, username=args.cm_user, password=args.cm_password) print "host connected, getting cloudera manager " MANAGER = api.get_cloudera_manager() print "have cloudera manager object" deploy_management(MANAGER, MGMT_SERVICENAME, MGMT_SERVICE_CONFIG, MGMT_ROLE_CONFIG, AMON_ROLENAME, AMON_ROLE_CONFIG, APUB_ROLENAME, APUB_ROLE_CONFIG, ESERV_ROLENAME, ESERV_ROLE_CONFIG, HMON_ROLENAME, HMON_ROLE_CONFIG, SMON_ROLENAME, SMON_ROLE_CONFIG, RMAN_ROLENAME, RMAN_ROLE_CONFIG) print "Deployed CM management service " + MGMT_SERVICENAME + " to run on " + CM_HOST
def __init__(self, cm_host, cm_cluster_name, username, password):
self.cm_api = ApiResource(cm_host, username=username, password=password)
self.hosts = dict()
self.services = list()
self.cluster = self.cm_api.get_cluster(cm_cluster_name)
if self.cluster is None:
raise RuntimeError, 'Cluster name "%s" not found' % cm_cluster_name
self.__load_hosts()
self.__impala_service = ImpalaService(self)
def __init__(self, module, config, trial=False, license_txt=None):
self.api = ApiResource(config['cm']['host'],
username=config['cm']['username'],
password=config['cm']['password'])
self.manager = self.api.get_cloudera_manager()
self.config = config
self.module = module
self.trial = trial
self.license_txt = license_txt
self.cluster = None
def main():
"""
Add peer to the cluster.
@rtype: number
@returns: A number representing the status of success.
"""
settings = parse_args()
if len(sys.argv) == 1 or len(sys.argv) > 17:
print_usage_message()
quit(1)
api = ApiResource(settings.server, settings.port, settings.username,
settings.password, settings.use_tls, 14)
yarn_service = get_service_name('YARN', api, settings.cluster_name)
hdfs_name = get_service_name('HDFS', api, settings.cluster_name)
hdfs = api.get_cluster(settings.cluster_name).get_service(hdfs_name)
hdfs_cloud_args = ApiHdfsCloudReplicationArguments(None)
hdfs_cloud_args.sourceService = ApiServiceRef(None,
peerName=None,
clusterName=settings.cluster_name,
serviceName=hdfs_name)
hdfs_cloud_args.sourcePath = settings.source_path
hdfs_cloud_args.destinationPath = settings.target_path
hdfs_cloud_args.destinationAccount = settings.account_name
hdfs_cloud_args.mapreduceServiceName = yarn_service
# creating a schedule with daily frequency
start = datetime.datetime.now()
# The time at which the scheduled activity is triggered for the first time.
end = start + datetime.timedelta(days=365)
# The time after which the scheduled activity will no longer be triggered.
schedule = hdfs.create_replication_schedule(start, end, "DAY", 1, True, hdfs_cloud_args)
## Updating the Schedule's properties
# schedule.hdfsArguments.removeMissingFiles = False
schedule.alertOnFail = True
schedule = hdfs.update_replication_schedule(schedule.id, schedule)
print "Schedule created with Schdule ID: " + str(schedule.id)
# print schedule.alertOnFail
# print schedule.hdfsArguments.removeMissingFiles
# print schedule.hdfsArguments.sourcePath
# print schedule.hdfsArguments.preserveXAttrs
# print schedule.hdfsArguments.exclusionFilters
# print schedule.hdfsArguments.replicationStrategy
# print schedule.hdfsArguments.numMaps
# print schedule.hdfsArguments.userName
# print schedule.hdfsArguments.schedulerPoolName
# print type(schedule)
return 0
def connect_cm(cm_host, cm_username, cm_password):
"""
Connects to Cloudera Manager API Resource instance to retrieve Endpoint details
:param cm_host: Cloudera Manager host
:param cm_username: Username for authentication
:param cm_password: Password for authentication
:return:
"""
api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password)
cm_manager = api.get_cloudera_manager()
return api, cm_manager
def connect_cm(cm_host, cm_username, cm_password):
"""
Connects to Cloudera Manager API Resource instance to retrieve Endpoint details
:param cm_host: Cloudera Manager host
:param cm_username: Username for authentication
:param cm_password: Password for authentication
:return:
"""
api = ApiResource(cm_host, version=6, username=cm_username, password=cm_password)
cm_manager = api.get_cloudera_manager()
return api, cm_manager
def get_cluster(cm_host, user, pwd, cluster_name):
global api
api = ApiResource(cm_host, username=user, password=pwd, version=12)
for c in api.get_all_clusters():
if cluster_name in c.name:
# print("Cluster, Version : " + c.name + ", " + c.version)
return True, c
else:
print("[ERR] : Cluster \"" + cluster_name + "\" not found at \"" +
cm_host + "\"")
return False, ""
def find_impala_in_cm(cm_host, cm_user, cm_password, cm_cluster_name):
"""Finds the Impala service in CM and returns an Impala instance."""
cm = ApiResource(cm_host, username=cm_user, password=cm_password)
cm_impalas = [service for cluster in cm.get_all_clusters()
if cm_cluster_name is None or cm_cluster_name == cluster.name
for service in cluster.get_all_services() if service.type == "IMPALA"]
if len(cm_impalas) > 1:
raise Exception("Found %s Impala services in CM;" % len(cm_impalas) +
" use --cm-cluster-name option to specify which one to use.")
if len(cm_impalas) == 0:
raise Exception("No Impala services found in CM")
return Impala(cm_impalas[0])
def cm_api_resource(self):
ar = None
try:
ar = ApiResource(self.cm_host, self.cm_port,
self.cm_username, self.cm_password)
ar.echo('Authenticated') # Issue a sample request to test the conn
except ApiException, aexc:
if aexc.code == 401:
log.debug("Changing default API username to {0}".format(self.cm_username))
self.cm_username = self.host_username
self.cm_password = self.host_password
ar = ApiResource(self.cm_host, self.cm_port,
self.cm_username, self.cm_password)
else:
log.error("Api Exception connecting to ClouderaManager: {0}".format(aexc))
def main():
# connect cm api
api = ApiResource(CM_HOST, 7180, username=CM_USERNAME, password=CM_PASSWORD)
manager = api.get_cloudera_manager()
# no need to update cm config
#manager.update_config(cm_host)
print("[INFO] Connected to CM host on " + CM_HOST)
# create cluster object
try:
cluster = api.get_cluster(name=CLUSTER_NAME)
except:
cluster = init_cluster(api, CLUSTER_NAME, CLUSTER_VERSION, CLUSTER_NODE_COUNT)
print("[INFO] Initialized cluster " + CLUSTER_NAME + " which uses CDH version " + CLUSTER_VERSION)
#
mgmt_servicename = "MGMT"
amon_role_name = "ACTIVITYMONITOR"
apub_role_name = "ALERTPUBLISHER"
eserv_role_name = "EVENTSERVER"
hmon_role_name = "HOSTMONITOR"
smon_role_name = "SERVICEMONITOR"
nav_role_name = "NAVIGATOR"
navms_role_name = "NAVIGATORMETADATASERVER"
rman_role_name = "REPORTMANAGER"
deploy_management(manager, mgmt_servicename, amon_role_name, apub_role_name, eserv_role_name, hmon_role_name, smon_role_name, nav_role_name, navms_role_name, rman_role_name)
print("[INFO] Deployed CM management service " + mgmt_servicename + " to run on " + CM_HOST)
#
assign_roles(api, cluster)
print("[INFO] all roles have assigned.")
#
# Custom role config groups cannot be automatically configured: Gateway Group 1 (error 400)
try:
cluster.auto_configure()
except:
pass
update_custom_config(api, cluster)
print("[INFO] all servies and roles have configured.")
#
cmd = cluster.first_run()
while cmd.success == None:
cmd = cmd.fetch()
if not cmd.success:
print("[ERROR] The first run command failed: " + cmd.resultMessage())
else:
print("[INFO] First run successfully executed. Your cluster has been set up!")
def main():
s,a = arg_handle()
for i in range(0,15):
while True:
try:
cm_host = "127.0.0.1"
api = ApiResource(cm_host, username="******", password="******")
cdh=api.get_all_clusters()[0]
except:
print "Failed to connect to Cloudera Manager."
print "Attempting to connect to Cloudera Manager..."
time.sleep(15)
continue
break
srv=cdh.get_service(s)
actions[a](srv,s)
def main():
resource = ApiResource("localhost", 7180, "cloudera", "cloudera", version=19)
cluster = resource.get_cluster("Cloudera Quickstart")
cm_manager = resource.get_cloudera_manager()
cm_manager.update_config({'REMOTE_PARCEL_REPO_URLS': PARCEL_REPO})
cm_manager.update_all_hosts_config(JDK_CONFIG)
time.sleep(5)
for parcel in PARCELS:
ParcelInstaller(parcel['name'], parcel['version']).install(cluster)
print "Restarting cluster"
cluster.stop().wait()
cluster.start().wait()
print "Done restarting cluster"
def main():
AMON_ROLE_CONFIG = {
'firehose_heapsize': '1173741824',
}
API = ApiResource("ec2-52-24-151-222.us-west-2.compute.amazonaws.com", version=5, username="******", password="******")
MANAGER = API.get_cloudera_manager()
mgmt = MANAGER.get_service()
#cf = mgmt.get_config()
for group in mgmt.get_all_role_config_groups():
if group.roleType == "ACTIVITYMONITOR":
group.update_config(AMON_ROLE_CONFIG)
pass
def update_cm(cm_host, cm_port, username, password):
"""Update config using the CM API (note: will restart service)"""
elts = generate_xml_elements()
cm_api = ApiResource(cm_host, username=username, password=password,
server_port=cm_port, version=9)
cluster = list(cm_api.get_all_clusters())[0]
hdfs = filter(lambda x: x.type == 'HDFS',
list(cluster.get_all_services()))[0]
print("Updating HFDS core-site.xml safety valve...")
_ = hdfs.update_config({
'core_site_safety_valve': '\n'.join(tostring(e) for e in elts)})
print("Deploying client config across the cluster...")
cluster.deploy_client_config().wait()
print("Restarting necessary services...")
cluster.restart().wait()
print("Done!")
def __init__(self, host_name, port=7180, user="******", password="******",
cluster_name=None, ssh_user=None, ssh_port=None, ssh_key_file=None):
# Initialize strptime() to workaround https://bugs.python.org/issue7980. Apparently
# something in the CM API uses strptime().
strptime("2015", "%Y")
Cluster.__init__(self)
self.cm = CmApiResource(host_name, server_port=port, username=user, password=password)
clusters = self.cm.get_all_clusters()
if not clusters:
raise Exception("No clusters found in CM at %s" % host_name)
if cluster_name:
clusters_by_name = dict((c.name, c) for c in clusters)
if cluster_name not in clusters_by_name:
raise Exception(("No clusters named %s found in CM at %s."
"Available clusters are %s.")
% (cluster_name, host_name, ", ".join(sorted(clusters_by_name.keys()))))
self.cm_cluster = clusters_by_name[cluster_name]
else:
if len(clusters) > 1:
raise Exception(("Too many clusters found in CM at %s;"
" a cluster name must be provided")
% host_name)
self.cm_cluster = clusters[-1]
self.ssh_user = ssh_user
self.ssh_port = ssh_port
self.ssh_key_file = ssh_key_file
self._ssh_client_lock = Lock()
self._ssh_clients_by_host_name = defaultdict(list)
class TimeSeriesQuery(object):
"""
"""
def __init__(self):
self._api = ApiResource(CM_HOST, username=CM_USER, password=CM_PASSWD, use_tls=CM_USE_TLS)
def query(self, query, from_time, to_time):
return self._api.query_timeseries(query, from_time, to_time)
def main():
"""
This is a script to export a current Cloudera Manager cluster configuration into an Hadrian supported format.
You can then use these configuration files as the basis for your new cluster configs.
"""
parser = argparse.ArgumentParser(description='Export Cloudera Manager configs in an Hadrian friendly format.')
parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM Server Name')
parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='CM Port')
parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='CM username')
args = parser.parse_args()
password = getpass.getpass('Please enter your Cloudera Manager passsword: ')
api = ApiResource(args.hostname, args.port, args.username, password, version=4)
for cluster in api.get_all_clusters():
conf_dir = './confs/' + cluster.name
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
for service in cluster.get_all_services():
with open(conf_dir + '/' + service.name + '.ini', 'w') as f:
print 'Dumping Service config for ' + service.name
rcg = list()
for i in service.get_all_role_config_groups():
rcg.append(i.name)
f.write('[' + service.type + ']\n')
f.write('config_groups=' + ','.join(rcg))
f.write('\n\n')
f.write('[' + service.name + '-svc-config]\n')
for item in service.get_config():
for k,v in item.iteritems():
f.write(k + '=' + str(v) + '\n')
for i in service.get_all_role_config_groups():
f.write('\n')
f.write('[' + i.name + ']\n')
for k,v in i.get_config('full').iteritems():
if v.value is not None:
f.write(k + '=' + str(v.value) + '\n')
f.close()
else:
print 'Cluster config dir already exists. Please rename or remove existing config dir: ' + conf_dir
def __init__(self, module, config, trial=False, license_txt=None):
self.api = ApiResource(config['cm']['host'], username=config['cm']['username'],
password=config['cm']['password'])
self.manager = self.api.get_cloudera_manager()
self.config = config
self.module = module
self.trial = trial
self.license_txt = license_txt
self.cluster = None
def do_call(host, port, user, password, cluster_name, service_role_name, random_index):
api = ApiResource(host, port, user, password, False, MAN_API_VERSION);
for cluster in api.get_all_clusters():
if cluster_name is None:
break
elif cluster_name == cluster.name:
break
if cluster_name is not None and cluster_name != cluster.name:
print >> sys.stderr, "Cloud not find cluster: " + cluster_name
return -2;
do_print_header()
for service in cluster.get_all_services():
do_print_line_item(api, service, service_role_name, random_index, 'HDFS', 'NAMENODE', 'namenode_port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'KUDU', 'KUDU_MASTER', 'webserver_port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'HUE', 'HUE_SERVER', 'hue_http_port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'HIVE', 'HIVESERVER2', 'hs2_thrift_address_port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'IMPALA', 'IMPALAD', 'beeswax_port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'FLUME', 'AGENT', 'agent_http_port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'KAFKA', 'KAFKA_BROKER', 'port', [], [])
do_print_line_item(api, service, service_role_name, random_index, 'ZOOKEEPER', 'SERVER', 'clientPort', [], [])
do_print_footer()
def reset_cm(cm_host, cm_port, username, password):
"""Elim S3 config from CM API safety valve (service restart necessary)"""
s3_props = set(get_s3_properties())
cm_api = ApiResource(cm_host, username=username, password=password,
server_port=cm_port, version=9)
cluster = list(cm_api.get_all_clusters())[0]
hdfs = filter(lambda x: x.type == 'HDFS',
list(cluster.get_all_services()))[0]
print("Getting current safety valve config")
current_config = hdfs.get_config('full')[0]['core_site_safety_valve'].value
# need the "<foo>...</foo>" to make it valid XML (bc it requires root elt)
elts = list(fromstring('<foo>' + current_config + '</foo>'))
new_elts = filter(lambda x: x.find('name').text not in s3_props, elts)
print("Updating safety valve and deleting S3 config")
_ = hdfs.update_config({
'core_site_safety_valve': '\n'.join(tostring(e) for e in new_elts)})
print("Deploying client config across the cluster...")
cluster.deploy_client_config().wait()
print("Restarting necessary services...")
cluster.restart().wait()
print("Done!")
def main():
"""
Enables HDFS HA on a cluster.
@rtype: number
@returns: A number representing the status of success.
"""
settings = retrieve_args()
api = ApiResource(settings.host, settings.port, settings.username, settings.password,
version=6)
if not validate_cluster(api, settings.cluster):
write_to_stdout("Cluster does not satisfy preconditions for enabling HDFS HA. Exiting!")
return 1
if settings.wait_for_good_health:
write_to_stdout("Waiting for GOOD health... ")
if not wait_for_good_health(api, settings.cluster):
write_to_stdout("Cluster health is not GOOD. Exiting!\n")
return 1
else:
write_to_stdout("Checking cluster health... ")
if not check_health(api, settings.cluster):
write_to_stdout("Cluster health is not GOOD. Exiting!\n")
write_to_stdout("Cluster health is GOOD!\n")
cluster = api.get_cluster(settings.cluster)
invoke_hdfs_enable_nn_ha(cluster, settings.nameservice)
update_hive_for_ha_hdfs(cluster)
# Restarting the MGMT services to make sure the HDFS file browser functions
# as expected.
cloudera_manager = api.get_cloudera_manager()
mgmt_service = cloudera_manager.get_service()
wait_for_command('Restarting MGMT services', mgmt_service.restart())
return 0
def main():
"""
TODO: This probably needs some work. You get the idea though.
An example of how to do a bulk config update to Cloudera Manager. This is helpful if you have a bunch of changes
That you want to make but don't want to use the GUI.
"""
parser = argparse.ArgumentParser(description='Cloudera Manager Bulk Config Update Script')
parser.add_argument('-H', '--host', '--hostname', action='store', dest='hostname', required=True, help='CM server host')
parser.add_argument('-p', '--port', action='store', dest='port', type=int, default=7180, help='example: 7180')
parser.add_argument('-u', '--user', '--username', action='store', dest='username', required=True, help='example: admin')
parser.add_argument('-c', '--cluster', action='store', dest='cluster', required=True, help='example: hadrian-cluster')
args = parser.parse_args()
password = getpass.getpass('Please enter your Cloudera Manager passsword: ')
# read configuration files:
for i in os.listdir('./conf/' + args.cluster):
config.read('./conf/' + args.cluster + '/' + i)
api = ApiResource(args.hostname, args.port, args.username, password)
cluster = api.get_cluster(args.cluster)
services = cluster.get_all_services()
# update services based with configuration file parameters
for service in services:
if config_grabber.has_section(service.type):
service.update_config(svc_config=config_grabber(service.name + '-svc-config'))
config_groups = config_grabber(service.name)['config_groups']
for config_group in config_groups.split(','):
print section
temp_config_group = service.get_role_config_group(section)
temp_config_group.update_config(config_grabber(section))
else:
print 'unknown service: ' + service.name
print 'Starting final client configuration deployment for all services.'
cmd = cluster.deploy_client_config()
if not cmd.wait(CMD_TIMEOUT).success:
print 'Failed to deploy client configuration.'
def get_cluster_info(manager_host, server_port=7180, username='******',
password='******'):
cm_api = ApiResource(manager_host, username=username, password=password,
server_port=server_port, version=9)
host = list(cm_api.get_all_hosts())[0] # all hosts same instance type
cluster = list(cm_api.get_all_clusters())[0]
yarn = filter(lambda x: x.type == 'YARN',
list(cluster.get_all_services()))[0]
hive = filter(lambda x: x.type == 'HIVE',
list(cluster.get_all_services()))[0]
impala = filter(lambda x: x.type == 'IMPALA',
list(cluster.get_all_services()))[0]
hive_hs2 = hive.get_roles_by_type('HIVESERVER2')[0]
hive_host = cm_api.get_host(hive_hs2.hostRef.hostId).hostname
hive_port = int(
hive_hs2.get_config('full')['hs2_thrift_address_port'].default)
impala_hs2 = impala.get_roles_by_type('IMPALAD')[0]
impala_host = cm_api.get_host(impala_hs2.hostRef.hostId).hostname
impala_port = int(impala_hs2.get_config('full')['hs2_port'].default)
return {'num_worker_nodes': len(yarn.get_roles_by_type('NODEMANAGER')),
'node_cores': host.numCores, 'node_memory': host.totalPhysMemBytes,
'hive_host': hive_host, 'hive_port': hive_port,
'impala_host': impala_host, 'impala_port': impala_port}