def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): Logger.info( "Class: %s, Method: %s. Get Service Configuration Recommendations." % (self.__class__.__name__, inspect.stack()[0][3])) servicesList = [ service["StackServices"]["service_name"] for service in services["services"] ] if 'AMBARI_METRICS' in servicesList: putAmsSiteProperty = self.putProperty(configurations, "ams-site") putAmsSiteProperty( 'timeline.metrics.downsampler.event.metric.patterns', 'topology\.%') properties = get_ambari_properties() ambari_version = get_ambari_version(properties) if not (ambari_version) or not (ambari_version.startswith('2.5')): putStreamlineLogSearchConfAttribute = self.putPropertyAttribute( configurations, "streamline-logsearch-conf") putStreamlineLogSearchConfAttribute('service_name', 'visible', 'false') putStreamlineLogSearchConfAttribute('component_mappings', 'visible', 'false') putStreamlineLogSearchConfAttribute('content', 'visible', 'false') self.autopopulateSTREAMLINEJdbcUrl(configurations, services) # Setting up KNOX SSO for Streams Messaging Manager. self.getServiceConfigurationRecommendationsForSSO( configurations, clusterData, services, hosts) pass
def validate_mpack_prerequisites(mpack_metadata): """ Validate management pack prerequisites :param mpack_name: Management pack metadata """ # Get ambari config properties properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 stack_location = get_stack_location(properties) current_ambari_version = get_ambari_version(properties) fail = False mpack_prerequisites = mpack_metadata.prerequisites if "min_ambari_version" in mpack_prerequisites: min_ambari_version = mpack_prerequisites.min_ambari_version if (compare_versions( min_ambari_version, current_ambari_version, format=True) > 0): print_error_msg( "Prerequisite failure! Current Ambari Version = {0}, " "Min Ambari Version = {1}".format(current_ambari_version, min_ambari_version)) fail = True if "max_ambari_version" in mpack_prerequisites: max_ambari_version = mpack_prerequisites.max_ambari_version if (compare_versions( max_ambari_version, current_ambari_version, format=True) < 0): print_error_msg( "Prerequisite failure! Current Ambari Version = {0}, " "Max Ambari Version = {1}".format(current_ambari_version, max_ambari_version)) if "min_stack_versions" in mpack_prerequisites: min_stack_versions = mpack_prerequisites.min_stack_versions stack_found = False for min_stack_version in min_stack_versions: stack_name = min_stack_version.stack_name stack_version = min_stack_version.stack_version stack_dir = os.path.join(stack_location, stack_name, stack_version) if os.path.exists(stack_dir) and os.path.isdir(stack_dir): stack_found = True break if not stack_found: print_error_msg( "Prerequisite failure! Min applicable stack not found") fail = True if fail: raise FatalException( -1, "Prerequisites for management pack {0}-{1} failed!".format( mpack_metadata.name, mpack_metadata.version))
def get_mpack_properties(): """ Read ambari properties required for management packs :return: (stack_location, service_definitions_location, mpacks_staging_location) """ # Get ambari config properties properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 stack_location = get_stack_location(properties) service_definitions_location = get_common_services_location(properties) mpacks_staging_location = get_mpacks_staging_location(properties) ambari_version = get_ambari_version(properties) return stack_location, service_definitions_location, mpacks_staging_location
def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): Logger.info( "Class: %s, Method: %s. get Service Configurations Recommendations. " % (self.__class__.__name__, inspect.stack()[0][3])) properties = get_ambari_properties() ambari_version = get_ambari_version(properties) if not (ambari_version) or not (ambari_version.startswith('2.5')): putRegistryLogSearchConfAttribute = self.putPropertyAttribute( configurations, "registry-logsearch-conf") putRegistryLogSearchConfAttribute('service_name', 'visible', 'false') putRegistryLogSearchConfAttribute('component_mappings', 'visible', 'false') putRegistryLogSearchConfAttribute('content', 'visible', 'false')
def upgrade(args): if not is_root(): err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT raise FatalException(4, err) print 'Updating properties in ' + AMBARI_PROPERTIES_FILE + ' ...' retcode = update_ambari_properties() if not retcode == 0: err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) try: update_database_name_property(upgrade=True) except FatalException: return -1 # Ignore the server version & database options passed via command-line arguments parse_properties_file(args) #TODO check database version change_objects_owner(args) retcode = run_schema_upgrade() if not retcode == 0: print_error_msg( "Ambari server upgrade failed. Please look at {0}, for more details." .format(configDefaults.SERVER_LOG_FILE)) raise FatalException(11, 'Schema upgrade failed.') user = read_ambari_user() if user is None: warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG print_warning_msg(warn) else: adjust_directory_permissions(user) # local repo upgrade_local_repo(args) # create jdbc symlinks if jdbc drivers are available in resources check_jdbc_drivers(args) properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) # Move *.py files from custom_actions to custom_actions/scripts # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0 ambari_version = get_ambari_version(properties) if ambari_version is None: args.warnings.append( "*.py files were not moved from custom_actions to custom_actions/scripts." ) elif compare_versions(ambari_version, "2.0.0") == 0: move_user_custom_actions() # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0 admin_views_dirs = get_admin_views_dir(properties) for admin_views_dir in admin_views_dirs: shutil.rmtree(admin_views_dir) # check if ambari has obsolete LDAP configuration if properties.get_property( LDAP_PRIMARY_URL_PROPERTY ) and not properties.get_property(IS_LDAP_CONFIGURED): args.warnings.append( "Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration." )
def upgrade(args): print_info_msg("Upgrade Ambari Server", True) if not is_root(): err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT raise FatalException(4, err) print_info_msg( 'Updating Ambari Server properties in {0} ...'.format( AMBARI_PROPERTIES_FILE), True) retcode = update_ambari_properties() if not retcode == 0: err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) print_info_msg( 'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE), True) retcode = update_ambari_env() if not retcode == 0: err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) retcode = update_krb_jaas_login_properties() if retcode == -2: pass # no changes done, let's be silent elif retcode == 0: print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE), True) elif not retcode == 0: err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) restore_custom_services() replay_mpack_logs() try: update_database_name_property(upgrade=True) except FatalException: return -1 # Ignore the server version & database options passed via command-line arguments parse_properties_file(args) #TODO check database version change_objects_owner(args) retcode = run_schema_upgrade(args) if not retcode == 0: print_error_msg( "Ambari server upgrade failed. Please look at {0}, for more details." .format(configDefaults.SERVER_LOG_FILE)) raise FatalException(11, 'Schema upgrade failed.') user = read_ambari_user() if user is None: warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG print_warning_msg(warn) else: adjust_directory_permissions(user) # local repo upgrade_local_repo(args) # create jdbc symlinks if jdbc drivers are available in resources check_jdbc_drivers(args) properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) # Move *.py files from custom_actions to custom_actions/scripts # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0 ambari_version = get_ambari_version(properties) if ambari_version is None: args.warnings.append( "*.py files were not moved from custom_actions to custom_actions/scripts." ) elif compare_versions(ambari_version, "2.0.0") == 0: move_user_custom_actions() # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0 admin_views_dirs = get_admin_views_dir(properties) for admin_views_dir in admin_views_dirs: shutil.rmtree(admin_views_dir) # Modify timestamp of views jars to current time views_jars = get_views_jars(properties) for views_jar in views_jars: os.utime(views_jar, None) # check if ambari has obsolete LDAP configuration if properties.get_property( LDAP_PRIMARY_URL_PROPERTY ) and not properties.get_property(IS_LDAP_CONFIGURED): args.warnings.append( "Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration." ) # adding custom jdbc name and previous custom jdbc properties # we need that to support new dynamic jdbc names for upgraded ambari add_jdbc_properties(properties)
def upgrade(args): if not is_root(): err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT raise FatalException(4, err) print 'Updating properties in ' + AMBARI_PROPERTIES_FILE + ' ...' retcode = update_ambari_properties() if not retcode == 0: err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) retcode = update_ambari_env() if not retcode == 0: err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) retcode = update_krb_jaas_login_properties() if retcode == -2: pass # no changes done, let's be silent elif retcode == 0: print 'File ' + AMBARI_KRB_JAAS_LOGIN_FILE + ' updated.' elif not retcode == 0: err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) restore_custom_services() try: update_database_name_property(upgrade=True) except FatalException: return -1 # Ignore the server version & database options passed via command-line arguments parse_properties_file(args) #TODO check database version change_objects_owner(args) retcode = run_schema_upgrade(args) if not retcode == 0: print_error_msg("Ambari server upgrade failed. Please look at {0}, for more details.".format(configDefaults.SERVER_LOG_FILE)) raise FatalException(11, 'Schema upgrade failed.') user = read_ambari_user() if user is None: warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG print_warning_msg(warn) else: adjust_directory_permissions(user) # local repo upgrade_local_repo(args) # create jdbc symlinks if jdbc drivers are available in resources check_jdbc_drivers(args) properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) # Move *.py files from custom_actions to custom_actions/scripts # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0 ambari_version = get_ambari_version(properties) if ambari_version is None: args.warnings.append("*.py files were not moved from custom_actions to custom_actions/scripts.") elif compare_versions(ambari_version, "2.0.0") == 0: move_user_custom_actions() # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0 admin_views_dirs = get_admin_views_dir(properties) for admin_views_dir in admin_views_dirs: shutil.rmtree(admin_views_dir) # Remove ambari views directory for the rest of the jars, at the time of upgrade. At restart all jars present in Ambari will be extracted into work directory views_dir = get_views_dir(properties) for views in views_dir: shutil.rmtree(views) # check if ambari has obsolete LDAP configuration if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED): args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
def upgrade(args): print_info_msg("Upgrade Ambari Server", True) if not is_root(): err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT raise FatalException(4, err) print_info_msg( 'Updating Ambari Server properties in {0} ...'.format( AMBARI_PROPERTIES_FILE), True) retcode = update_ambari_properties() if not retcode == 0: err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) print_info_msg( 'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE), True) retcode = update_ambari_env() if not retcode == 0: err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) retcode = update_krb_jaas_login_properties() if retcode == -2: pass # no changes done, let's be silent elif retcode == 0: print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE), True) elif not retcode == 0: err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting' raise FatalException(retcode, err) restore_custom_services() replay_mpack_logs() try: update_database_name_property(upgrade=True) except FatalException: return -1 # Ignore the server version & database options passed via command-line arguments parse_properties_file(args) #TODO check database version change_objects_owner(args) retcode = run_schema_upgrade(args) if not retcode == 0: print_error_msg( "Ambari server upgrade failed. Please look at {0}, for more details." .format(configDefaults.SERVER_LOG_FILE)) raise FatalException(11, 'Schema upgrade failed.') user = read_ambari_user() if user is None: warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG print_warning_msg(warn) else: adjust_directory_permissions(user) # create jdbc symlinks if jdbc drivers are available in resources check_jdbc_drivers(args) properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) # Move *.py files from custom_actions to custom_actions/scripts # This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0 ambari_version = get_ambari_version(properties) if ambari_version is None: args.warnings.append( "*.py files were not moved from custom_actions to custom_actions/scripts." ) elif compare_versions(ambari_version, "2.0.0") == 0: move_user_custom_actions() # Move files installed by package to default views directory to a custom one for views_dir in get_views_dir(properties): root_views_dir = views_dir + "/../" if os.path.samefile(root_views_dir, get_default_views_dir()): continue for file in glob.glob(get_default_views_dir() + '/*'): shutil.move(file, root_views_dir) # Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0 admin_views_dirs = get_admin_views_dir(properties) for admin_views_dir in admin_views_dirs: shutil.rmtree(admin_views_dir) # Modify timestamp of views jars to current time views_jars = get_views_jars(properties) for views_jar in views_jars: os.utime(views_jar, None) # check if ambari is configured to use LDAP authentication if properties.get_property(CLIENT_SECURITY) == "ldap": args.warnings.append( "LDAP authentication is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration." ) # adding custom jdbc name and previous custom jdbc properties # we need that to support new dynamic jdbc names for upgraded ambari add_jdbc_properties(properties) json_url = get_json_url_from_repo_file() if json_url: print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format( json_url) properties = get_ambari_properties() stack_root = get_stack_location(properties) update_latest_in_repoinfos_for_stacks(stack_root, json_url) else: print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): nifi = self.getServicesSiteProperties(services, "nifi") if "ranger-env" in services["configurations"] and "ranger-nifi-plugin-properties" in services["configurations"] and \ "ranger-nifi-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]: putNiFiRangerPluginProperty = self.putProperty( configurations, "ranger-nifi-plugin-properties", services) rangerEnvNiFiPluginProperty = services["configurations"][ "ranger-env"]["properties"]["ranger-nifi-plugin-enabled"] putNiFiRangerPluginProperty("ranger-nifi-plugin-enabled", rangerEnvNiFiPluginProperty) if rangerEnvNiFiPluginProperty == 'Yes' and \ "nifi.authentication" in services["configurations"]["ranger-nifi-plugin-properties"]["properties"] and \ "nifi.node.ssl.isenabled" in services["configurations"]["nifi-ambari-ssl-config"]["properties"]: nifiAmbariSSLConfig = 'SSL' if services["configurations"][ "nifi-ambari-ssl-config"]["properties"][ "nifi.node.ssl.isenabled"] == 'true' else 'NONE' putNiFiRangerPluginProperty("nifi.authentication", nifiAmbariSSLConfig) # Recommend Ranger supported service's audit properties ranger_audit_dict = [{ 'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.solr', 'target_configname': 'xasecure.audit.destination.solr' }, { 'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.hdfs', 'target_configname': 'xasecure.audit.destination.hdfs' }, { 'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.hdfs.dir', 'target_configname': 'xasecure.audit.destination.hdfs.dir' }, { 'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.urls', 'target_configname': 'xasecure.audit.destination.solr.urls' }, { 'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.zookeepers', 'target_configname': 'xasecure.audit.destination.solr.zookeepers' }] for item in ranger_audit_dict: if item['filename'] in services['configurations'] and item[ 'configname'] in services['configurations'][ item['filename']]['properties']: if item['filename'] in configurations and item[ 'configname'] in configurations[ item['filename']]['properties']: rangerAuditProperty = configurations[ item['filename']]['properties'][item['configname']] else: rangerAuditProperty = services['configurations'][ item['filename']]['properties'][item['configname']] putNifiRangerAuditProperty = self.putProperty( configurations, "ranger-nifi-audit", services) putNifiRangerAuditProperty(item['target_configname'], rangerAuditProperty) servicesList = [ service['StackServices']['service_name'] for service in services['services'] ] if "RANGER" in servicesList and "ranger-admin-site" in services[ 'configurations']: nifi_user = services['configurations']['nifi-env']['properties'][ 'nifi_user'] putRangerAdminSiteProperty = self.putProperty( configurations, "ranger-admin-site", services) putRangerAdminSiteProperty("ranger.plugins.nifi.serviceuser", nifi_user) properties = get_ambari_properties() ambari_version = get_ambari_version(properties) if not (ambari_version) or not (ambari_version.startswith('2.5')): putNiFiLogSearchConfAttribute = self.putPropertyAttribute( configurations, "nifi-logsearch-conf") putNiFiLogSearchConfAttribute('service_name', 'visible', 'false') putNiFiLogSearchConfAttribute('component_mappings', 'visible', 'false') putNiFiLogSearchConfAttribute('content', 'visible', 'false')
def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): Logger.info( "Class: %s, Method: %s. Get Service Configuration Recommendations." % (self.__class__.__name__, inspect.stack()[0][3])) servicesList = [ service["StackServices"]["service_name"] for service in services["services"] ] security_enabled = self.isSecurityEnabled(services) if 'AMBARI_METRICS' in servicesList: putAmsSiteProperty = self.putProperty(configurations, "ams-site") putAmsSiteProperty( 'timeline.metrics.downsampler.event.metric.patterns', 'topology\.%') if 'STORM' in servicesList and security_enabled: storm_site = self.getServicesSiteProperties(services, "storm-site") streamline_env = self.getServicesSiteProperties( services, "streamline-env") if storm_site is not None and streamline_env is not None: putStormSiteProperty = self.putProperty( configurations, "storm-site", services) putStormSiteAttributes = self.putPropertyAttribute( configurations, "storm-site") storm_env = self.getServicesSiteProperties( services, "storm-env") storm_nimbus_impersonation_acl = storm_site[ "nimbus.impersonation.acl"] if "nimbus.impersonation.acl" in storm_site else None streamline_env = self.getServicesSiteProperties( services, "streamline-env") _streamline_principal_name = streamline_env[ 'streamline_principal_name'] if 'streamline_principal_name' in streamline_env else None if _streamline_principal_name is not None and storm_nimbus_impersonation_acl is not None: streamline_bare_principal = get_bare_principal( _streamline_principal_name) storm_nimbus_impersonation_acl = "{ " + streamline_bare_principal + " : {hosts: ['*'], groups: ['*']}, {{storm_bare_jaas_principal}} : {hosts: ['*'], groups: ['*']}}" putStormSiteProperty('nimbus.impersonation.acl', storm_nimbus_impersonation_acl) storm_nimbus_autocred_plugin_classes = storm_site[ "nimbus.autocredential.plugins.classes"] if "nimbus.autocredential.plugins.classes" in storm_site else None # Here storm_nimbus_autocred_plugin_classes is resulting in none. There is no nimbus.autocredential.plugins.classes in storm-site.xml if storm_nimbus_autocred_plugin_classes is not None: new_storm_nimbus_autocred_plugin_classes = [ 'org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive' ] new_conf = self.appendToYamlString( storm_nimbus_autocred_plugin_classes, new_storm_nimbus_autocred_plugin_classes) putStormSiteProperty( "nimbus.autocredential.plugins.classes", new_conf) else: putStormSiteProperty( "nimbus.autocredential.plugins.classes", "['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" ) storm_nimbus_credential_renewer_classes = storm_site[ "nimbus.credential.renewers.classes"] if "nimbus.credential.renewers.classes" in storm_site else None if storm_nimbus_credential_renewer_classes is not None: new_storm_nimbus_credential_renewer_classes_array = [ 'org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive' ] new_conf = self.appendToYamlString( storm_nimbus_credential_renewer_classes, new_storm_nimbus_credential_renewer_classes_array) putStormSiteProperty( "nimbus.autocredential.plugins.classes", new_conf) else: putStormSiteProperty( "nimbus.credential.renewers.classes", "['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" ) putStormSiteProperty("nimbus.credential.renewers.freq.secs", "82800") properties = get_ambari_properties() ambari_version = get_ambari_version(properties) if not (ambari_version) or not (ambari_version.startswith('2.5')): putStreamlineLogSearchConfAttribute = self.putPropertyAttribute( configurations, "streamline-logsearch-conf") putStreamlineLogSearchConfAttribute('service_name', 'visible', 'false') putStreamlineLogSearchConfAttribute('component_mappings', 'visible', 'false') putStreamlineLogSearchConfAttribute('content', 'visible', 'false') pass