def update_user(self, user):
"""
Update user with the supplied new user object.
@param user: ApiUser object to be applied
@return: An ApiUser object
"""
return users.update_user(self, user)
def update_user(self, user):
"""
Update user with the supplied new user object.
@param user: ApiUser object to be applied
@return: An ApiUser object
"""
return users.update_user(self, user)
def setup_hadoop(cm_api,
nodes,
cluster_name,
cm_username='******',
cm_password='******',
parcel_repo=None,
parcel_version=None,
anaconda_repo=None,
anaconda_version=None):
ha_enabled = _CFG.isHA_enabled
try:
api, cloudera_manager = connect(cm_api, 'admin', 'admin')
if cm_username == 'admin':
logging.info("Updating admin login password")
admin_user = api.get_user('admin')
admin_user.password = cm_password
users.update_user(api, admin_user)
else:
logging.info("Updating admin login user to %s", cm_username)
api.create_user(cm_username, cm_password, ['ROLE_ADMIN'])
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
if cm_username != 'admin':
logging.info("Deleting old admin login user")
api.delete_user('admin')
except:
logging.info("Admin login user already configured")
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
logging.info("Waiting for CM API to become contactable")
pause_until_api_up(api)
logging.info("Installing hosts")
new_nodes = create_hosts(api, cloudera_manager, nodes)
assign_host_ids(api, nodes)
setup_progress = load_progress()
if not check_progress(setup_progress, "99_COMPLETE"):
# setup hasn't completed, force it to run again
cluster_action = 'create_new'
elif len(new_nodes) == 0:
# no new nodes, reapply config to existing ones
cluster_action = 'reapply_config'
elif len(new_nodes) == len(nodes):
# all new nodes, create new cluster
cluster_action = 'create_new'
else:
# some new nodes, expand cluster onto them
cluster_action = 'expand'
if cluster_action == 'create_new':
# CMS creation is handled slightly differently from other services and must
# be done prior to cluster creation
if not check_progress(setup_progress, "001_CMS_CREATE"):
logging.info("Creating CMS")
cms = create_cms(cloudera_manager, nodes)
save_progress(setup_progress, "001_CMS_CREATE")
else:
cms = cloudera_manager.get_service()
if not check_progress(setup_progress, "002_CLUSTER_CREATE"):
logging.info("Creating cluster")
cluster = create_cluster(api, cluster_name)
save_progress(setup_progress, "002_CLUSTER_CREATE")
else:
cluster = api.get_cluster(cluster_name)
elif cluster_action == 'expand':
logging.info("Expanding cluster")
cluster = api.get_cluster(cluster_name)
# pylint: disable=E1103
cluster.add_hosts([h['id'] for h in new_nodes])
elif cluster_action == 'reapply_config':
cluster = api.get_cluster(cluster_name)
if cluster_action == 'create_new' or cluster_action == 'expand':
# Once we have a cluster and a set of hosts with installed agents, we need
# to install the correct CDH parcel via the download/distribute/activate
logging.info("Downloading, distributing and activating parcels")
install_parcel(cloudera_manager, cluster, 'CDH', parcel_repo,
parcel_version)
# to install Anaconda parcels
logging.info("Downloading anaconda parcels")
if anaconda_repo is not None and anaconda_version is not None:
install_parcel(cloudera_manager, cluster, 'Anaconda',
anaconda_repo, anaconda_version)
if cluster_action == 'create_new':
# Some services are sensitive to perceived health so CMS needs to be started
# before everything else
logging.info("Starting CMS")
ensure_started(cms)
logging.info("Creating, configuring and starting Hadoop services")
services = create_services(cluster, nodes, ha_enabled)
# there isn't much space for parcels but we know we are not going to
# install any so it's safe to disable this warning
cloudera_manager.update_all_hosts_config(_CFG.CM_CFG['hosts_config'])
# Install commonly used libraries into defined deployment path
# Get the namenode private ip address
nn_role = get_role_name(services['hdfs'], "HTTPFS")
nnode_detail = get_role_vm(nodes, services['hdfs'], nn_role)
setup_common_oozie_libs(nnode_detail['private_addr'])
# For CORONA-3045 sometimes CMS can't find an active namenode until
# after a restart even though everything is actually fine
logging.info("Restarting cloudera monitors")
wait_on_success(cms.restart())
elif cluster_action == 'expand':
logging.info("Adding Hadoop services to new nodes")
expand_services(cluster, new_nodes)
elif cluster_action == 'reapply_config':
logging.info("Re-applying hadoop config to all nodes")
configure_services(cloudera_manager, cluster, nodes)
def setup_hadoop(
cm_api,
nodes,
cluster_name,
cm_username='******',
cm_password='******',
parcel_repo=None,
parcel_version=None,
anaconda_repo=None,
anaconda_version=None):
ha_enabled = _CFG.isHA_enabled
try:
api, cloudera_manager = connect(cm_api, 'admin', 'admin')
if cm_username == 'admin':
logging.info("Updating admin login password")
admin_user = api.get_user('admin')
admin_user.password = cm_password
users.update_user(api, admin_user)
else:
logging.info("Updating admin login user to %s", cm_username)
api.create_user(cm_username, cm_password, ['ROLE_ADMIN'])
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
if cm_username != 'admin':
logging.info("Deleting old admin login user")
api.delete_user('admin')
except:
logging.info("Admin login user already configured")
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
logging.info("Waiting for CM API to become contactable")
pause_until_api_up(api)
logging.info("Installing hosts")
new_nodes = create_hosts(api, cloudera_manager, nodes)
assign_host_ids(api, nodes)
setup_progress = load_progress()
if not check_progress(setup_progress, "99_COMPLETE"):
# setup hasn't completed, force it to run again
cluster_action = 'create_new'
elif len(new_nodes) == 0:
# no new nodes, reapply config to existing ones
cluster_action = 'reapply_config'
elif len(new_nodes) == len(nodes):
# all new nodes, create new cluster
cluster_action = 'create_new'
else:
# some new nodes, expand cluster onto them
cluster_action = 'expand'
if cluster_action == 'create_new':
# CMS creation is handled slightly differently from other services and must
# be done prior to cluster creation
if not check_progress(setup_progress, "001_CMS_CREATE"):
logging.info("Creating CMS")
cms = create_cms(cloudera_manager, nodes)
save_progress(setup_progress, "001_CMS_CREATE")
else:
cms = cloudera_manager.get_service()
if not check_progress(setup_progress, "002_CLUSTER_CREATE"):
logging.info("Creating cluster")
cluster = create_cluster(api, cluster_name)
save_progress(setup_progress, "002_CLUSTER_CREATE")
else:
cluster = api.get_cluster(cluster_name)
elif cluster_action == 'expand':
logging.info("Expanding cluster")
cluster = api.get_cluster(cluster_name)
# pylint: disable=E1103
cluster.add_hosts([h['id'] for h in new_nodes])
elif cluster_action == 'reapply_config':
cluster = api.get_cluster(cluster_name)
if cluster_action == 'create_new' or cluster_action == 'expand':
# Once we have a cluster and a set of hosts with installed agents, we need
# to install the correct CDH parcel via the download/distribute/activate
logging.info("Downloading, distributing and activating parcels")
install_parcel(cloudera_manager, cluster, 'CDH', parcel_repo, parcel_version)
# to install Anaconda parcels
logging.info("Downloading anaconda parcels")
if anaconda_repo is not None and anaconda_version is not None:
install_parcel(cloudera_manager, cluster, 'Anaconda', anaconda_repo, anaconda_version)
if cluster_action == 'create_new':
# Some services are sensitive to perceived health so CMS needs to be started
# before everything else
logging.info("Starting CMS")
ensure_started(cms)
logging.info("Creating, configuring and starting Hadoop services")
services = create_services(cluster, nodes, ha_enabled)
# there isn't much space for parcels but we know we are not going to
# install any so it's safe to disable this warning
cloudera_manager.update_all_hosts_config(_CFG.CM_CFG['hosts_config'])
# Install commonly used libraries into defined deployment path
# Get the namenode private ip address
nn_role = get_role_name(services['hdfs'], "HTTPFS")
nnode_detail = get_role_vm(nodes, services['hdfs'], nn_role)
setup_common_oozie_libs(nnode_detail['private_addr'])
# For CORONA-3045 sometimes CMS can't find an active namenode until
# after a restart even though everything is actually fine
logging.info("Restarting cloudera monitors")
wait_on_success(cms.restart())
elif cluster_action == 'expand':
logging.info("Adding Hadoop services to new nodes")
expand_services(cluster, new_nodes)
elif cluster_action == 'reapply_config':
logging.info("Re-applying hadoop config to all nodes")
configure_services(cloudera_manager, cluster, nodes)
def setup_hadoop(cm_api,
user,
nodes,
key_name,
cluster_name,
cm_username='******',
cm_password='******',
flavour='standard',
parcel_repo=None,
parcel_version=None,
anaconda_repo=None,
anaconda_version=None):
global _CFG
isHA_enabled = False
if flavour == 'standard':
import cfg_standard as _CFG
isHA_enabled = True
# Add additional flavours here
try:
api, cloudera_manager = connect(cm_api, 'admin', 'admin')
if cm_username == 'admin':
logging.info("Updating admin login password")
admin_user = api.get_user('admin')
admin_user.password = cm_password
users.update_user(api, admin_user)
else:
logging.info("Updating admin login user to %s" % cm_username)
api.create_user(cm_username, cm_password, ['ROLE_ADMIN'])
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
if cm_username != 'admin':
logging.info("Deleting old admin login user")
api.delete_user('admin')
except:
logging.info("Admin login user already configured")
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
logging.info("Waiting for CM API to become contactable")
pause_until_api_up(api)
# There are several ways to add hosts to a cluster, this is the only one that
# works reliably - introduce hosts & let CM handle installation of agents
logging.info("Installing hosts")
is_new_cluster, new_nodes = create_hosts(api, cloudera_manager, user,
nodes, key_name)
assign_host_ids(api, nodes)
# CMS creation is handled slightly differently from other services and must
# be done prior to cluster creation
if is_new_cluster:
logging.info("Creating CMS")
cms = create_cms(cloudera_manager, nodes)
logging.info("Creating cluster")
cluster = create_cluster(api, cluster_name)
else:
logging.info("Expanding cluster")
cluster = api.get_cluster(cluster_name)
# pylint: disable=E1103
cluster.add_hosts([h['id'] for h in new_nodes])
# Once we have a cluster and a set of hosts with installed agents, we need
# to install the correct CDH parcel via the download/distribute/activate
logging.info("Downloading, distributing and activating parcels")
install_parcel(cloudera_manager, cluster, 'CDH', parcel_repo,
parcel_version)
# to install Anaconda parcels
logging.info("Downloading anaconda parcels")
if anaconda_repo is not None and anaconda_version is not None:
install_parcel(cloudera_manager, cluster, 'Anaconda', anaconda_repo,
anaconda_version)
if is_new_cluster:
# Some services are sensitive to perceived health so CMS needs to be started
# before everything else
logging.info("Starting CMS")
wait_on_success(cms.start())
logging.info("Creating, configuring and starting Hadoop services")
services = create_services(user, key_name, cluster, nodes,
isHA_enabled)
# there isn't much space for parcels but we know we are not going to
# install any so it's safe to disable this warning
cfg = {
'host_agent_parcel_directory_free_space_absolute_thresholds':
'{"warning":"-2.0","critical":"6000000000"}',
'memory_overcommit_threshold': '0.85'
}
cloudera_manager.update_all_hosts_config(cfg)
# Install system shared libs into defined deployment path
setup_sharedlib(nodes, user, key_name, services['hdfs'], cm_api)
# For CORONA-3045 sometimes CMS can't find an active namenode until
# after a restart even though everything is actually fine
logging.info("Restarting cloudera monitors")
wait_on_success(cms.restart())
else:
logging.info("Adding Hadoop services to new nodes")
services = expand_services(cluster, new_nodes)
def setup_hadoop(cm_api,
user,
nodes,
key_name,
cluster_name,
cm_username='******',
cm_password='******',
parcel_repo=None,
parcel_version=None,
anaconda_repo=None,
anaconda_version=None):
ha_enabled = _CFG.isHA_enabled
try:
api, cloudera_manager = connect(cm_api, 'admin', 'admin')
if cm_username == 'admin':
logging.info("Updating admin login password")
admin_user = api.get_user('admin')
admin_user.password = cm_password
users.update_user(api, admin_user)
else:
logging.info("Updating admin login user to %s", cm_username)
api.create_user(cm_username, cm_password, ['ROLE_ADMIN'])
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
if cm_username != 'admin':
logging.info("Deleting old admin login user")
api.delete_user('admin')
except:
logging.info("Admin login user already configured")
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
logging.info("Waiting for CM API to become contactable")
pause_until_api_up(api)
# There are several ways to add hosts to a cluster, this is the only one that
# works reliably - introduce hosts & let CM handle installation of agents
logging.info("Installing hosts")
new_nodes = create_hosts(api, cloudera_manager, user, nodes, key_name)
assign_host_ids(api, nodes)
if len(new_nodes) == 0:
# no new nodes, reapply config to existing ones
cluster_action = 'reapply_config'
elif len(new_nodes) == len(nodes):
# all new nodes, create new cluster
cluster_action = 'create_new'
else:
# some new nodes, expand cluster onto them
cluster_action = 'expand'
if cluster_action == 'create_new':
# CMS creation is handled slightly differently from other services and must
# be done prior to cluster creation
logging.info("Creating CMS")
cms = create_cms(cloudera_manager, nodes)
logging.info("Creating cluster")
cluster = create_cluster(api, cluster_name)
elif cluster_action == 'expand':
logging.info("Expanding cluster")
cluster = api.get_cluster(cluster_name)
# pylint: disable=E1103
cluster.add_hosts([h['id'] for h in new_nodes])
elif cluster_action == 'reapply_config':
cluster = api.get_cluster(cluster_name)
if cluster_action == 'create_new' or cluster_action == 'expand':
# Once we have a cluster and a set of hosts with installed agents, we need
# to install the correct CDH parcel via the download/distribute/activate
logging.info("Downloading, distributing and activating parcels")
install_parcel(cloudera_manager, cluster, 'CDH', parcel_repo,
parcel_version)
# to install Anaconda parcels
logging.info("Downloading anaconda parcels")
if anaconda_repo is not None and anaconda_version is not None:
install_parcel(cloudera_manager, cluster, 'Anaconda',
anaconda_repo, anaconda_version)
if cluster_action == 'create_new':
# Some services are sensitive to perceived health so CMS needs to be started
# before everything else
logging.info("Starting CMS")
wait_on_success(cms.start())
logging.info("Creating, configuring and starting Hadoop services")
services = create_services(user, key_name, cluster, nodes, ha_enabled)
# there isn't much space for parcels but we know we are not going to
# install any so it's safe to disable this warning
cloudera_manager.update_all_hosts_config(_CFG.CM_CFG['hosts_config'])
# Install system shared libs into defined deployment path
setup_sharedlib(nodes, user, key_name, services['hdfs'], cm_api)
# For CORONA-3045 sometimes CMS can't find an active namenode until
# after a restart even though everything is actually fine
logging.info("Restarting cloudera monitors")
wait_on_success(cms.restart())
elif cluster_action == 'expand':
logging.info("Adding Hadoop services to new nodes")
expand_services(cluster, new_nodes)
elif cluster_action == 'reapply_config':
logging.info("Re-applying hadoop config to all nodes")
configure_services(cloudera_manager, cluster, nodes)
def setup_hadoop(
cm_api,
user,
nodes,
key_name,
cluster_name,
cm_username='******',
cm_password='******',
flavour='standard',
parcel_repo=None,
parcel_version=None,
anaconda_repo=None,
anaconda_version=None):
global _CFG
isHA_enabled = False
if flavour == 'standard':
import cfg_standard as _CFG
isHA_enabled = True
# Add additional flavours here
try:
api, cloudera_manager = connect(cm_api, 'admin', 'admin')
if cm_username == 'admin':
logging.info("Updating admin login password")
admin_user = api.get_user('admin')
admin_user.password = cm_password
users.update_user(api, admin_user)
else:
logging.info("Updating admin login user to %s" % cm_username)
api.create_user(cm_username, cm_password, ['ROLE_ADMIN'])
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
if cm_username != 'admin':
logging.info("Deleting old admin login user")
api.delete_user('admin')
except:
logging.info("Admin login user already configured")
api, cloudera_manager = connect(cm_api, cm_username, cm_password)
logging.info("Waiting for CM API to become contactable")
pause_until_api_up(api)
# There are several ways to add hosts to a cluster, this is the only one that
# works reliably - introduce hosts & let CM handle installation of agents
logging.info("Installing hosts")
is_new_cluster, new_nodes = create_hosts(api, cloudera_manager, user, nodes, key_name)
assign_host_ids(api, nodes)
# CMS creation is handled slightly differently from other services and must
# be done prior to cluster creation
if is_new_cluster:
logging.info("Creating CMS")
cms = create_cms(cloudera_manager, nodes)
logging.info("Creating cluster")
cluster = create_cluster(api, cluster_name)
else:
logging.info("Expanding cluster")
cluster = api.get_cluster(cluster_name)
# pylint: disable=E1103
cluster.add_hosts([h['id'] for h in new_nodes])
# Once we have a cluster and a set of hosts with installed agents, we need
# to install the correct CDH parcel via the download/distribute/activate
logging.info("Downloading, distributing and activating parcels")
install_parcel(cloudera_manager, cluster, 'CDH', parcel_repo, parcel_version)
# to install Anaconda parcels
logging.info("Downloading anaconda parcels")
if anaconda_repo is not None and anaconda_version is not None:
install_parcel(cloudera_manager, cluster, 'Anaconda', anaconda_repo, anaconda_version)
if is_new_cluster:
# Some services are sensitive to perceived health so CMS needs to be started
# before everything else
logging.info("Starting CMS")
wait_on_success(cms.start())
logging.info("Creating, configuring and starting Hadoop services")
services = create_services(user, key_name, cluster, nodes, isHA_enabled)
# there isn't much space for parcels but we know we are not going to
# install any so it's safe to disable this warning
cfg = {
'host_agent_parcel_directory_free_space_absolute_thresholds': '{"warning":"-2.0","critical":"6000000000"}',
'memory_overcommit_threshold': '0.85'}
cloudera_manager.update_all_hosts_config(cfg)
# Install system shared libs into defined deployment path
setup_sharedlib(nodes, user, key_name, services['hdfs'], cm_api)
# For CORONA-3045 sometimes CMS can't find an active namenode until
# after a restart even though everything is actually fine
logging.info("Restarting cloudera monitors")
wait_on_success(cms.restart())
else:
logging.info("Adding Hadoop services to new nodes")
services = expand_services(cluster, new_nodes)
def main():
module = AnsibleModule(argument_spec=dict(
cm_host=dict(required=True, type='str'),
cm_port=dict(required=False, type='int', default=7180),
cm_username=dict(required=True, type='str'),
cm_password=dict(required=True, type='str', no_log=True),
cm_tls=dict(required=False, type='bool', default=False),
force_password_update=dict(required=False, type='bool', default=False),
cm_version=dict(required=False, type='int', default=10),
name=dict(required=False, type='str', aliases=['user']),
password=dict(required=False, type='str', no_log=True),
roles=dict(required=False, type='list', default=None),
state=dict(default='present', choices=['present', 'absent', 'list'])))
cm_host = module.params.get('cm_host')
cm_port = module.params.get('cm_port')
cm_username = module.params.get('cm_username')
cm_password = module.params.get('cm_password')
cm_tls = module.params.get('cm_tls')
force_password_update = module.params.get('force_password_update')
cm_version = module.params.get('cm_version')
username = module.params.get('name')
password = module.params.get('password')
roles = module.params.get('roles')
state = module.params.get('state')
changed = False
if not HAS_CM_API:
module.fail_json(changed=changed,
msg='cm_api required for this module')
try:
cm_conn = ApiResource(cm_host,
server_port=cm_port,
username=cm_username,
password=cm_password,
use_tls=cm_tls,
version=cm_version)
except ApiException as e:
module.fail_json(changed=changed,
msg="Can't connect to API: {}".format(e))
if state == 'list':
users_list = []
ulist = cm_conn.get_all_users(view='export')
for i in range(len(ulist)):
u = dict()
for (key, value) in ulist[i].__dict__.items():
if isinstance(value, basestring):
u[key] = value
users_list.append(u)
module.exit_json(changed=changed, users=users_list)
user_exists = False
cur_user = None
try:
cur_user = cm_conn.get_user(username)
user_exists = True
except ApiException as e:
if str(e) != "User '" + username + "' does not exist. (error 404)":
module.fail_json(changed=changed, msg='{}'.format(e))
if state == "absent":
if user_exists:
try:
cm_conn.delete_user(username)
module.exit_json(changed=True, msg="user is deleted")
except:
module.fail_json(changed=changed, msg='{}'.format(e))
else:
if user_exists:
if roles != cur_user.roles or force_password_update:
try:
if not force_password_update:
password = None
updated_user = users.ApiUser(cm_conn, username, password,
roles)
users.update_user(cm_conn, updated_user)
module.exit_json(changed=True, msg="role updated")
except ApiException as e:
module.fail_json(changed=changed, msg="{}".format(e))
else:
module.exit_json(changed=False)
else:
try:
cm_conn.create_user(username, password, roles)
except ApiException as e:
module.fail_json(changed=True, msg="{}".format(e))
module.exit_json(changed=True, msg="user is created")