def update_custom_config(api, cluster): for service in cluster.get_all_services(): service_config = CDH_DEFINE[service.type]['config'] if service.type == "HDFS": dn_amount = len(service.get_roles_by_type('DATANODE')) if dn_amount > 5: replications = 3 elif dn_amount <= 5 and dn_amount >= 3: replications = 2 else: replications = 1 service_config['dfs_replication'] = replications service.update_config(service_config) role_type_config_synced = [] for role in service.get_all_roles(): if role.type not in role_type_config_synced: role_config = CDH_DEFINE[service.type]['roles'][ role.type]['config'] role_group = role.roleConfigGroupRef role_group_config = get_role_config_group( api, service.type, role_group.roleConfigGroupName, cluster.name) role_group_config.update_config(role_config) role_type_config_synced.append(role.type) return True
def get_role_config_group(self, name): """ Get a role configuration group in the service by name. @param name: The name of the role config group. @return: An ApiRoleConfigGroup object. @since: API v3 """ return role_config_groups.get_role_config_group( self._get_resource_root(), self.name, name, self._get_cluster_name())
def get_role_config_group(self, name): """ Get a role configuration group in the service by name. @param name: The name of the role config group. @return: An ApiRoleConfigGroup object. @since: API v3 """ return role_config_groups.get_role_config_group( self._get_resource_root(), self.name, name, self._get_cluster_name())
def update_custom_config(api, cluster): for service in cluster.get_all_services(): service_config = CDH_DEFINE[service.type]['config'] if service.type == "HDFS": dn_amount = len(service.get_roles_by_type('DATANODE')) if dn_amount > 5: replications = 3 elif dn_amount <= 5 and dn_amount >= 3: replications = 2 else: replications = 1 service_config['dfs_replication'] = replications service.update_config(service_config) role_type_config_synced = [] for role in service.get_all_roles(): if role.type not in role_type_config_synced: role_config = CDH_DEFINE[service.type]['roles'][role.type]['config'] role_group = role.roleConfigGroupRef role_group_config = get_role_config_group(api, service.type, role_group.roleConfigGroupName, cluster.name) role_group_config.update_config(role_config) role_type_config_synced.append(role.type) return True
def set_up_cluster(): # get a handle on the instance of CM that we have running api = ApiResource(cm_host, cm_port, cm_username, cm_password, version=19) # get the CM instance cm = ClouderaManager(api) print "*************************************" print " Starting Auto Deployment of Cluster " print "*************************************" # {'owner': ROAttr(), 'uuid': ROAttr(), 'expiration': ROAttr(),} TRIAL = False try: trial_active = cm.get_license() print trial_active if trial_active.owner == "Trial License": print "Trial License is already set - will NOT continue now." print "Assuming Cluster is already setup" TRIAL = True else: print "Setting up `Trial License`." cm.begin_trial() except: cm.begin_trial() if TRIAL: exit(0) # create the management service service_setup = ApiServiceSetupInfo(name=cm_service_name, type="MGMT") try: if not cm.get_service().name: cm.create_mgmt_service(service_setup) else: print "Service already exist." except: cm.create_mgmt_service(service_setup) # install hosts on this CM instance cmd = cm.host_install(host_username, host_list, password=host_password, cm_repo_url=cm_repo_url, unlimited_jce=True) print "Installing hosts. This might take a while." while cmd.success == None: sleep(5) cmd = cmd.fetch() print cmd if cmd.success != True: print "cm_host_install failed: " + cmd.resultMessage exit(0) print "cm_host_install succeeded" # first auto-assign roles and auto-configure the CM service cm.auto_assign_roles() cm.auto_configure() # create a cluster on that instance cluster = create_cluster(api, cluster_name, cdh_version) # add all our hosts to the cluster cluster.add_hosts(host_list) cluster = api.get_cluster(cluster_name) parcels_list = [] # get and list all available parcels print "Available parcels:" for p in cluster.get_all_parcels(): print '\t' + p.product + ' ' + p.version if p.version.startswith(cdh_version_number) and p.product == "CDH": parcels_list.append(p) if len(parcels_list) == 0: print "No " + cdh_version + " parcel found!" exit(0) cdh_parcel = parcels_list[0] for p in parcels_list: if p.version > cdh_parcel.version: cdh_parcel = p # download the parcel print "Starting parcel download. This might take a while." cmd = cdh_parcel.start_download() if cmd.success != True: print "Parcel download failed!" exit(0) # make sure the download finishes while cdh_parcel.stage != 'DOWNLOADED': sleep(5) cdh_parcel = get_parcel(api, cdh_parcel.product, cdh_parcel.version, cluster_name) print cdh_parcel.product + ' ' + cdh_parcel.version + " downloaded" # distribute the parcel print "Starting parcel distribution. This might take a while." cmd = cdh_parcel.start_distribution() if cmd.success != True: print "Parcel distribution failed!" exit(0) # make sure the distribution finishes while cdh_parcel.stage != "DISTRIBUTED": sleep(5) cdh_parcel = get_parcel(api, cdh_parcel.product, cdh_parcel.version, cluster_name) print cdh_parcel.product + ' ' + cdh_parcel.version + " distributed" # activate the parcel cmd = cdh_parcel.activate() if cmd.success != True: print "Parcel activation failed!" exit(0) # make sure the activation finishes while cdh_parcel.stage != "ACTIVATED": cdh_parcel = get_parcel(api, cdh_parcel.product, cdh_parcel.version, cluster_name) print cdh_parcel.product + ' ' + cdh_parcel.version + " activated" # inspect hosts and print the result print "Inspecting hosts. This might take a few minutes." cmd = cm.inspect_hosts() while cmd.success == None: cmd = cmd.fetch() if cmd.success != True: print "Host inpsection failed!" exit(0) print "Hosts successfully inspected: \n" + cmd.resultMessage # create all the services we want to add; we will only create one instance # of each for s in service_types_and_names.keys(): service = cluster.create_service(service_types_and_names[s], s) # we will auto-assign roles; you can manually assign roles using the # /clusters/{clusterName}/services/{serviceName}/role endpoint or by using # ApiService.createRole() cluster.auto_assign_roles() cluster.auto_configure() # # this will set up the Hive and the reports manager databases because we # # can't auto-configure those two things # hive = cluster.get_service(service_types_and_names["HIVE"]) # hive_config = {"hive_metastore_database_host": hive_metastore_host, \ # "hive_metastore_database_name": hive_metastore_name, \ # "hive_metastore_database_password": hive_metastore_password, \ # "hive_metastore_database_port": hive_metastore_database_port, \ # "hive_metastore_database_type": hive_metastore_database_type} # hive.update_config(hive_config) # start the management service cm_service = cm.get_service() cm_service.start().wait() # this will set the Reports Manager database password # first we find the correct role rm_role = None for r in cm.get_service().get_all_roles(): if r.type == "REPORTSMANAGER": rm_role = r if rm_role == None: print "No REPORTSMANAGER role found!" exit(0) # then we get the corresponding role config group -- even though there is # only once instance of each CM management service, we do this just in case # it is not placed in the base group rm_role_group = rm_role.roleConfigGroupRef rm_rcg = get_role_config_group(api, rm_role.type, \ rm_role_group.roleConfigGroupName, None) # update the appropriate fields in the config rm_rcg_config = {"headlamp_database_host": reports_manager_host, \ "headlamp_database_name": reports_manager_name, \ "headlamp_database_user": reports_manager_username, \ "headlamp_database_password": reports_manager_password, \ "headlamp_database_type": reports_manager_database_type} rm_rcg.update_config(rm_rcg_config) # restart the management service with new configs cm_service.restart().wait() # execute the first run command print "Excuting first run command. This might take a while." cmd = cluster.first_run() while cmd.success == None: cmd = cmd.fetch() if cmd.success != True: print "The first run command failed: " + cmd.resultMessage() exit(0) print "First run successfully executed. Your cluster has been set up!"
def set_up_cluster(): # get a handle on the instance of CM that we have running api = ApiResource(cm_host, cm_port, cm_username, cm_password, version=7) # get the CM instance cm = ClouderaManager(api) # activate the CM trial license cm.begin_trial() # create the management service service_setup = ApiServiceSetupInfo(name=cm_service_name, type="MGMT") cm.create_mgmt_service(service_setup) # install hosts on this CM instance cmd = cm.host_install(host_username, host_list, password=host_password, cm_repo_url=cm_repo_url) print "Installing hosts. This might take a while." while cmd.success == None: sleep(5) cmd = cmd.fetch() if cmd.success != True: print "cm_host_install failed: " + cmd.resultMessage exit(0) print "cm_host_install succeeded" # first auto-assign roles and auto-configure the CM service cm.auto_assign_roles() cm.auto_configure() # create a cluster on that instance cluster = create_cluster(api, cluster_name, cdh_version) # add all our hosts to the cluster cluster.add_hosts(host_list) cluster = api.get_cluster("Cluster 1") parcels_list = [] # get and list all available parcels print "Available parcels:" for p in cluster.get_all_parcels(): print '\t' + p.product + ' ' + p.version if p.version.startswith(cdh_version_number) and p.product == "CDH": parcels_list.append(p) if len(parcels_list) == 0: print "No " + cdh_version + " parcel found!" exit(0) cdh_parcel = parcels_list[0] for p in parcels_list: if p.version > cdh_parcel.version: cdh_parcel = p # download the parcel print "Starting parcel download. This might take a while." cmd = cdh_parcel.start_download() if cmd.success != True: print "Parcel download failed!" exit(0) # make sure the download finishes while cdh_parcel.stage != 'DOWNLOADED': sleep(5) cdh_parcel = get_parcel(api, cdh_parcel.product, cdh_parcel.version, cluster_name) print cdh_parcel.product + ' ' + cdh_parcel.version + " downloaded" # distribute the parcel print "Starting parcel distribution. This might take a while." cmd = cdh_parcel.start_distribution() if cmd.success != True: print "Parcel distribution failed!" exit(0) # make sure the distribution finishes while cdh_parcel.stage != "DISTRIBUTED": sleep(5) cdh_parcel = get_parcel(api, cdh_parcel.product, cdh_parcel.version, cluster_name) print cdh_parcel.product + ' ' + cdh_parcel.version + " distributed" # activate the parcel cmd = cdh_parcel.activate() if cmd.success != True: print "Parcel activation failed!" exit(0) # make sure the activation finishes while cdh_parcel.stage != "ACTIVATED": cdh_parcel = get_parcel(api, cdh_parcel.product, cdh_parcel.version, cluster_name) print cdh_parcel.product + ' ' + cdh_parcel.version + " activated" # inspect hosts and print the result print "Inspecting hosts. This might take a few minutes." cmd = cm.inspect_hosts() while cmd.success == None: cmd = cmd.fetch() if cmd.success != True: print "Host inpsection failed!" exit(0) print "Hosts successfully inspected: \n" + cmd.resultMessage # create all the services we want to add; we will only create one instance # of each for s in service_types_and_names.keys(): service = cluster.create_service(service_types_and_names[s], s) # we will auto-assign roles; you can manually assign roles using the # /clusters/{clusterName}/services/{serviceName}/role endpoint or by using # ApiService.createRole() cluster.auto_assign_roles() cluster.auto_configure() # this will set up the Hive and the reports manager databases because we # can't auto-configure those two things hive = cluster.get_service(service_types_and_names["HIVE"]) hive_config = { "hive_metastore_database_host" : hive_metastore_host, \ "hive_metastore_database_name" : hive_metastore_name, \ "hive_metastore_database_password" : hive_metastore_password, \ "hive_metastore_database_port" : hive_metastore_database_port, \ "hive_metastore_database_type" : hive_metastore_database_type } hive.update_config(hive_config) # start the management service cm_service = cm.get_service() cm_service.start().wait() # this will set the Reports Manager database password # first we find the correct role rm_role = None for r in cm.get_service().get_all_roles(): if r.type == "REPORTSMANAGER": rm_role = r if rm_role == None: print "No REPORTSMANAGER role found!" exit(0) # then we get the corresponding role config group -- even though there is # only once instance of each CM management service, we do this just in case # it is not placed in the base group rm_role_group = rm_role.roleConfigGroupRef rm_rcg = get_role_config_group(api, rm_role.type, \ rm_role_group.roleConfigGroupName, None) # update the appropriate fields in the config rm_rcg_config = { "headlamp_database_host" : reports_manager_host, \ "headlamp_database_name" : reports_manager_name, \ "headlamp_database_user" : reports_manager_username, \ "headlamp_database_password" : reports_manager_password, \ "headlamp_database_type" : reports_manager_database_type } rm_rcg.update_config(rm_rcg_config) # restart the management service with new configs cm_service.restart().wait() # execute the first run command print "Excuting first run command. This might take a while." cmd = cluster.first_run() while cmd.success == None: cmd = cmd.fetch() if cmd.success != True: print "The first run command failed: " + cmd.resultMessage() exit(0) print "First run successfully executed. Your cluster has been set up!"