def extract_views(): java_exe_path = get_java_exe_path() if java_exe_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY, configDefaults.DEFAULT_VIEWS_DIR) files = [f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir,f))] for f in files: command = VIEW_EXTRACT_CMD.format(java_exe_path, get_full_ambari_classpath(), os.path.join(vdir,f)) retcode, stdout, stderr = run_os_command(command) if retcode == 0: sys.stdout.write(f + "\n") elif retcode == 2: sys.stdout.write("Error extracting " + f + "\n") else: sys.stdout.write(".") sys.stdout.flush() print_info_msg("Return code from extraction of view archive " + f + ": " + str(retcode)) sys.stdout.write("\n") return 0
def do_checks(self): try: user = read_ambari_user() create_user = False update_user_setting = False if user is not None: create_user = get_YN_input_optional(self.NR_USER_CHANGE_PROMPT.format(user), False,SETUP_USE_DEFAULT) update_user_setting = create_user # Only if we will create another user else: # user is not configured yet update_user_setting = True # Write configuration anyway create_user = get_YN_input_optional(self.NR_USER_CUSTOMIZE_PROMPT, False,SETUP_USE_DEFAULT) if not create_user: user = self.NR_DEFAULT_USER if create_user: (retcode, user) = self._create_custom_user() if retcode != 0: return retcode if update_user_setting: write_property(NR_USER_PROPERTY, user) adjust_directory_permissions(user) except OSError as e: print_error_msg("Failed: %s" % str(e)) return 4 except Exception as e: print_error_msg("Unexpected error %s" % str(e)) return 1 return 0
def _install_jdbc_driver(self, properties, files_list): if type(files_list) is not int: print 'Copying JDBC drivers to server resources...' try: resources_dir = properties[RESOURCES_DIR_PROPERTY] except KeyError: print_error_msg("There is no value for " + RESOURCES_DIR_PROPERTY + "in " + AMBARI_PROPERTIES_FILE) return False db_name = self.dbms_full_name.lower() symlink_name = db_name + "-jdbc-driver.jar" jdbc_symlink = os.path.join(resources_dir, symlink_name) db_default_driver_path = os.path.join(configDefaults.JAVA_SHARE_PATH, self.driver_file_name) if os.path.lexists(jdbc_symlink): os.remove(jdbc_symlink) copy_status = copy_files(files_list, resources_dir) if not copy_status == 0: raise FatalException(-1, "Failed to copy JDBC drivers to server resources") if db_default_driver_path in files_list: os.symlink(os.path.join(resources_dir, self.driver_file_name), jdbc_symlink) else: if files_list == -1: return False return True
def read_passwd_for_alias(alias, masterKey=""): if alias: jdk_path = find_jdk() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 tempFileName = "ambari.passwd" passwd = "" tempDir = tempfile.gettempdir() #create temporary file for writing tempFilePath = tempDir + os.sep + tempFileName with open(tempFilePath, 'w+'): os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE) if masterKey is None or masterKey == "": masterKey = "None" command = SECURITY_PROVIDER_GET_CMD.format(get_java_exe_path(), get_full_ambari_classpath(), alias, tempFilePath, masterKey) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider get passwd: " + str(retcode)) if retcode != 0: print 'ERROR: Unable to read password from store. alias = ' + alias else: with open(tempFilePath, 'r') as hfRTemp: passwd = hfRTemp.read() # Remove temporary file os.remove(tempFilePath) return passwd else: print_error_msg("Alias is unreadable.")
def _setup_db(self): #password access to ambari-server and mapred dbname = self.database_name scriptFile = PGConfig.POSTGRES_EMBEDDED_INIT_FILE username = self.database_username password = self.database_password #setup DB command = PGConfig.SETUP_DB_CMD[:] command[-1] = command[-1].format(scriptFile, username, password, dbname) for i in range(SETUP_DB_CONNECT_ATTEMPTS): sys.stdout.write('Connecting to local database...') retcode, outdata, errdata = run_os_command(command) if retcode == 0: print 'done.' return retcode, outdata, errdata timeOutMsg = 'connection timed out' if (i+1) < SETUP_DB_CONNECT_ATTEMPTS: timeOutMsg += '...retrying (%d)' % (i+1) print timeOutMsg time.sleep(SETUP_DB_CONNECT_TIMEOUT) print 'unable to connect to database' print_error_msg(errdata) return retcode, outdata, errdata
def do_checks(self): try: user = read_ambari_user() if not user: user = self.NR_DEFAULT_USER if self.user is not None: #Command-line parameter is the default update_user_setting = True prompt_msg = self.NR_USER_CUSTOMIZE_PROMPT.format('y') else: update_user_setting = False if user != self.NR_DEFAULT_USER: prompt_msg = self.NR_USER_CHANGE_PROMPT.format(user, 'n') else: prompt_msg = self.NR_USER_CUSTOMIZE_PROMPT.format('n') self.user = user if user else self.NR_DEFAULT_USER self.register_service = get_YN_input(prompt_msg, update_user_setting) if self.register_service: retcode = self._create_custom_user() if retcode != 0: return retcode adjust_directory_permissions(self.user) except OSError as e: print_error_msg("Failed: %s" % str(e)) return 4 except Exception as e: print_error_msg("Unexpected error %s" % str(e)) return 1 return 0
def mainBody(): parser = optparse.OptionParser(usage="usage: %prog [options] action [stack_id os]",) init_parser_options(parser) (options, args) = parser.parse_args() # check if only silent key set default_options = parser.get_default_values() silent_options = default_options silent_options.silent = True if options == silent_options: options.only_silent = True else: options.only_silent = False # set verbose set_verbose(options.verbose) if options.verbose: main(options, args, parser) else: try: main(options, args, parser) except Exception as e: print_error_msg("Unexpected {0}: {1}".format((e).__class__.__name__, str(e)) +\ "\nFor more info run ambari-server with -v or --verbose option") sys.exit(1)
def check_setup_already_done(): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 return properties.get_property(JDK_NAME_PROPERTY) and properties.get_property(JDBC_DATABASE_PROPERTY)
def parse_properties_file(args): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 args.server_version_file_path = properties[SERVER_VERSION_FILE_PATH] args.persistence_type = properties[PERSISTENCE_TYPE_PROPERTY] args.jdbc_url = properties[JDBC_URL_PROPERTY] args.dbms = properties[JDBC_DATABASE_PROPERTY] if not args.persistence_type: args.persistence_type = "local" if args.persistence_type == 'remote': args.database_host = properties[JDBC_HOSTNAME_PROPERTY] args.database_port = properties[JDBC_PORT_PROPERTY] args.database_name = properties[JDBC_DATABASE_NAME_PROPERTY] args.database_username = properties[JDBC_USER_NAME_PROPERTY] args.postgres_schema = properties[JDBC_POSTGRES_SCHEMA_PROPERTY] \ if JDBC_POSTGRES_SCHEMA_PROPERTY in properties.propertyNames() else None args.database_password_file = properties[JDBC_PASSWORD_PROPERTY] if args.database_password_file: if not is_alias_string(args.database_password_file): with open(properties[JDBC_PASSWORD_PROPERTY]) as hfDbPwd: args.database_password = hfDbPwd.read() else: args.database_password = args.database_password_file return 0
def write_property(key, value): conf_file = find_properties_file() properties = Properties() try: properties.load(open(conf_file)) except Exception, e: print_error_msg('Could not read tbds config file "%s": %s' % (conf_file, e)) return -1
def write_property(key, value): conf_file = find_properties_file() properties = Properties() try: with open(conf_file, "r") as hfR: properties.load(hfR) except Exception, e: print_error_msg('Could not read ambari config file "%s": %s' % (conf_file, e)) return -1
def ensure_jdbc_driver_installed(self, properties): (result, msg) = self._prompt_jdbc_driver_install(properties) if result == -1: print_error_msg(msg) raise FatalException(-1, msg) if result != 1: result = self._install_jdbc_driver(properties, result) return cbool(result)
def _reset_database(options): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting tbds properties") return -1 factory = DBMSConfigFactory() dbmsAmbari = factory.create(options, properties) dbmsAmbari.reset_database()
def backup_file_in_temp(filePath): if filePath is not None: tmpDir = tempfile.gettempdir() back_up_file_count = len(glob.glob1(tmpDir, AMBARI_PROPERTIES_FILE + "*")) try: shutil.copyfile(filePath, tmpDir + os.sep + AMBARI_PROPERTIES_FILE + "." + str(back_up_file_count + 1)) except (Exception), e: print_error_msg('Could not backup file in temp "%s": %s' % ( back_up_file_count, str(e)))
def _prompt_jdbc_driver_install(self, properties): result = self._is_jdbc_driver_installed(properties) if result == -1: if get_silent(): print_error_msg(self.JDBC_DRIVER_INSTALL_MSG) else: print_warning_msg(self.JDBC_DRIVER_INSTALL_MSG) raw_input(PRESS_ENTER_MSG) result = self._is_jdbc_driver_installed(properties) return (result, self.JDBC_DRIVER_INSTALL_MSG)
def get_JAVA_HOME(): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return None java_home = properties[JAVA_HOME_PROPERTY] if (not 0 == len(java_home)) and (os.path.exists(java_home)): return java_home return None
def restore_custom_services(): properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) try: resources_dir = properties[RESOURCES_DIR_PROPERTY] except (KeyError), e: conf_file = properties.fileName err = 'Property ' + str(e) + ' is not defined at ' + conf_file print_error_msg(err) raise FatalException(1, err)
def check_jdbc_drivers(args): # create jdbc symlinks if jdbc drivers are available in resources properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) conf_file = properties.fileName try: resources_dir = properties[RESOURCES_DIR_PROPERTY] except (KeyError), e: err = "Property " + str(e) + " is not defined at " + conf_file raise FatalException(1, err)
def win_main(): parser = init_options_parser() (options, args) = parser.parse_args() options.warnings = [] options.exit_message = None if options.debug: sys.frozen = 'windows_exe' # Fake py2exe so we can debug if len(args) == 0: print parser.print_help() parser.error("No action entered") action = args[0] try: if action == SETUP_ACTION: svcsetup() elif action == START_ACTION: start(options) elif action == STOP_ACTION: stop() elif action == RESTART_ACTION: stop() start(options) elif action == STATUS_ACTION: svcstatus(options) else: parser.error("Invalid action") if options.warnings: for warning in options.warnings: print_warning_msg(warning) pass options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action pass except FatalException as e: if e.reason is not None: print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason)) sys.exit(e.code) except NonFatalException as e: options.exit_message = "Ambari Metrics Host Monitoring '%s' completed with warnings." % action if e.reason is not None: print_warning_msg(e.reason) if options.exit_message is not None: print options.exit_message sys.exit(0)
def get_ambari_version(properties): """ :param properties: Ambari properties :return: Return a string of the ambari version. When comparing versions, please use "compare_versions" function. """ version = None try: server_version_file_path = properties[SERVER_VERSION_FILE_PATH] if server_version_file_path and os.path.exists(server_version_file_path): with open(server_version_file_path, 'r') as file: version = file.read().strip() except: print_error_msg("Error getting ambari version") return version
def main(): from amc_service import init_options_parser, init_service_debug, setup, start, stop, svcstatus parser = init_options_parser() (options, args) = parser.parse_args() options.warnings = [] options.exit_message = None init_service_debug(options) if len(args) == 0: print parser.print_help() parser.error("No action entered") action = args[0] try: if action == SETUP_ACTION: setup(options) elif action == START_ACTION: start(options) elif action == STOP_ACTION: stop() elif action == RESTART_ACTION: stop() start(options) elif action == STATUS_ACTION: svcstatus(options) else: parser.error("Invalid action") if options.warnings: for warning in options.warnings: print_warning_msg(warning) pass options.exit_message = "Ambari Metrics Collector '%s' completed with warnings." % action pass except FatalException as e: if e.reason is not None: print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason)) sys.exit(e.code) except NonFatalException as e: options.exit_message = "Ambari Metrics Collector '%s' completed with warnings." % action if e.reason is not None: print_warning_msg(e.reason) if options.exit_message is not None: print options.exit_message
def _is_jdbc_user_changed(database_username): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return None previos_user = get_value_from_properties(properties, JDBC_USER_NAME_PROPERTY, "") if previos_user and database_username: if previos_user != database_username: return True else: return False return None
def save_master_key(master_key, key_location, persist=True): if master_key: jdk_path = find_jdk() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 command = SECURITY_PROVIDER_KEY_CMD.format(get_java_exe_path(), get_full_ambari_classpath(), master_key, key_location, persist) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider save KEY: " + str(retcode)) else: print_error_msg("Master key cannot be None.")
def update_debug_mode(): debug_mode = get_debug_mode() # The command-line settings supersede the ones in ambari.properties if not debug_mode & 1: properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 if get_value_from_properties(properties, DEBUG_MODE_KEY, False): debug_mode = debug_mode | 1 if get_value_from_properties(properties, SUSPEND_START_MODE_KEY, False): debug_mode = debug_mode | 2 set_debug_mode(debug_mode)
def download_and_unpack_jce_policy(self, properties, ambariOnly = False): err_msg_stdout = "JCE Policy files are required for secure HDP setup. Please ensure " \ " all hosts have the JCE unlimited strength policy 6, files." resources_dir = get_resources_location(properties) jdk_cfg = self.jdks[self.jdk_index] try: JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties, ambariOnly) except FatalException, e: print err_msg_stdout print_error_msg("Failed to download JCE policy files:") if e.reason is not None: print_error_msg("\nREASON: {0}".format(e.reason))
def move_user_custom_actions(): print_info_msg('Moving *.py files from custom_actions to custom_actions/scripts') properties = get_ambari_properties() if properties == -1: err = "Error getting ambari properties" print_error_msg(err) raise FatalException(-1, err) try: resources_dir = properties[RESOURCES_DIR_PROPERTY] except (KeyError), e: conf_file = properties.fileName err = 'Property ' + str(e) + ' is not defined at ' + conf_file print_error_msg(err) raise FatalException(1, err)
def get_ambari_version(properties): """ :param properties: Ambari properties :return: Return a string of the ambari version. When comparing versions, please use "compare_versions" function. """ version = None try: server_version_file_path = properties[SERVER_VERSION_FILE_PATH] if server_version_file_path and os.path.exists( server_version_file_path): with open(server_version_file_path, 'r') as file: version = file.read().strip() except: print_error_msg("Error getting ambari version") return version
def download_and_unpack_jce_policy(self, properties): err_msg_stdout = "JCE Policy files are required for secure HDP setup. Please ensure " \ " all hosts have the JCE unlimited strength policy 6, files." resources_dir = get_resources_location(properties) jdk_cfg = self.jdks[self.jdk_index] try: JDKSetup._download_jce_policy(jdk_cfg.jcpol_url, jdk_cfg.dest_jcpol_file, resources_dir, properties) except FatalException, e: print err_msg_stdout print_error_msg("Failed to download JCE policy files:") if e.reason is not None: print_error_msg("\nREASON: {0}".format(e.reason))
def enable_stack(options, args): if options.stack_name == None: print_error_msg ("Please provide stack name using --stack option") return -1 if options.stack_versions == None: print_error_msg ("Please provide stack version using --version option") return -1 print_info_msg ("Going to enable Stack Versions: " + str(options.stack_versions) + " for the stack: " + str(options.stack_name)) retcode = enable_stack_version(options.stack_name,options.stack_versions) if retcode == 0: status, pid = is_server_runing() if status: print "restarting ambari server" stop(options) start(options)
def process_stack_extension_definition_artifact(artifact, artifact_source_dir, options): """ Process stack-extension-definition artifact :param artifact: Artifact metadata :param artifact_source_dir: Location of artifact in the management pack :param options: Command line options """ # Get ambari mpack properties stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties( ) service_name = None if "service_name" in artifact: service_name = artifact.service_name if not service_name: print_error_msg( "Must provide service name for stack-extension-definition artifact!" ) raise FatalException( -1, 'Must provide service name for stack-extension-definition artifact!' ) applicable_stacks = None if "applicable_stacks" in artifact: applicable_stacks = artifact.applicable_stacks if not applicable_stacks: print_error_msg( "Must provide applicable stacks for stack-extension-definition artifact!" ) raise FatalException( -1, 'Must provide applicable stacks for stack-extension-definition artifact!' ) for applicable_stack in applicable_stacks: stack_name = applicable_stack.stack_name stack_version = applicable_stack.stack_version dest_stack_path = os.path.join(stack_location, stack_name) dest_stack_version_path = os.path.join(dest_stack_path, stack_version) dest_stack_services_path = os.path.join(dest_stack_version_path, "services") dest_link = os.path.join(dest_stack_services_path, service_name) if os.path.exists(dest_stack_path) and os.path.exists( dest_stack_version_path): if not os.path.exists(dest_stack_services_path): sudo.makedir(dest_stack_services_path, 0755) if options.force and os.path.islink(dest_link): sudo.unlink(dest_link) sudo.symlink(artifact_source_dir, dest_link)
def verify_setup_allowed(): if get_silent(): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 isSecure = get_is_secure(properties) if isSecure: (isPersisted, masterKeyFile) = get_is_persisted(properties) if not isPersisted: print "ERROR: Cannot run silent 'setup' with password encryption enabled " \ "and Master Key not persisted." print "Ambari Server 'setup' exiting." return 1 return 0
def mainBody(): parser = optparse.OptionParser(usage="usage: %prog [options] action [stack_id os]",) init_parser_options(parser) (options, args) = parser.parse_args() # set verbose set_verbose(options.verbose) if options.verbose: main(options, args, parser) else: try: main(options, args, parser) except Exception as e: print_error_msg("Unexpected {0}: {1}".format((e).__class__.__name__, str(e)) +\ "\nFor more info run tbds-server with -v or --verbose option") sys.exit(1)
def _setup_remote_database(self): properties = get_ambari_properties() if properties == -1: err = 'Error getting ambari properties' print_error_msg(err) raise FatalException(-1, err) if self.ensure_jdbc_driver_installed(properties): print 'Configuring remote database connection properties...' retcode = self._setup_remote_db() if retcode == -1: err = "Remote database setup aborted." raise NonFatalException(err) if not retcode == 0: err = 'Error while configuring connection properties. Exiting' raise FatalException(retcode, err)
def verify_setup_allowed(): if get_silent(): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting tbds properties") return -1 isSecure = get_is_secure(properties) if isSecure: (isPersisted, masterKeyFile) = get_is_persisted(properties) if not isPersisted: print "ERROR: Cannot run silent 'setup' with password encryption enabled " \ "and Master Key not persisted." print "TBDS Server 'setup' exiting." return 1 return 0
def _reset_database(options): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 persistence_type = properties[PERSISTENCE_TYPE_PROPERTY] if persistence_type == "remote": err = 'Ambari doesn\'t support resetting exernal DB automatically. ' \ 'To reset Ambari Server schema you must first drop and then create it ' \ 'using DDL scripts from "/var/lib/ambari-server/resources/"' raise FatalException(1, err) else: factory = DBMSConfigFactory() dbmsAmbari = factory.create(options, properties) dbmsAmbari.reset_database()
def get_mpack_properties(): """ Read ambari properties required for management packs :return: (stack_location, service_definitions_location, mpacks_staging_location) """ # Get ambari config properties properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 stack_location = get_stack_location(properties) extension_location = get_extension_location(properties) service_definitions_location = get_common_services_location(properties) mpacks_staging_location = get_mpacks_staging_location(properties) ambari_version = get_ambari_version(properties) return stack_location, extension_location, service_definitions_location, mpacks_staging_location
def mainBody(): parser = optparse.OptionParser(usage="usage: %prog [options] action [stack_id os]",) init_parser_options(parser) (options, args) = parser.parse_args() # set verbose set_verbose(options.verbose) if options.verbose: main(options, args, parser) else: try: main(options, args, parser) except Exception as e: print_error_msg("Unexpected {0}: {1}".format((e).__class__.__name__, str(e)) +\ "\nFor more info run ambari-server with -v or --verbose option") sys.exit(1)
def validate_purge(options, purge_list, mpack_dir, mpack_metadata, replay_mode=False): """ Validate purge options :param purge_list: List of resources to purge :param mpack_metadata: Management pack metadata :param replay_mode: Flag to indicate if purging in replay mode """ # Get ambari mpacks config properties stack_location, extension_location, service_definitions_location, mpacks_staging_location, dashboard_location = get_mpack_properties() if not purge_list: return if STACK_DEFINITIONS_RESOURCE_NAME in purge_list: mpack_stacks = [] for artifact in mpack_metadata.artifacts: if artifact.type == STACK_DEFINITIONS_ARTIFACT_NAME: artifact_source_dir = os.path.join(mpack_dir, artifact.source_dir) for file in sorted(os.listdir(artifact_source_dir)): if os.path.isdir(os.path.join(artifact_source_dir, file)): stack_name = file mpack_stacks.append(stack_name) if not mpack_stacks: # Don't purge stacks accidentally when installing add-on mpacks err = "The management pack you are attempting to install does not contain {0}. Since this management pack " \ "does not contain a stack, the --purge option with --purge-list={1} would cause your existing Ambari " \ "installation to be unusable. Due to that we cannot install this management pack.".format( RESOURCE_FRIENDLY_NAMES[STACK_DEFINITIONS_RESOURCE_NAME], purge_list) print_error_msg(err) raise FatalException(1, err) else: # Valid that there are no clusters deployed with a stack that is not included in the management pack (retcode, stdout, stderr) = run_mpack_install_checker(options, mpack_stacks) if retcode > 0: print_error_msg(stderr) raise FatalException(1, stderr) if not replay_mode: purge_resources = set((v) for k, v in RESOURCE_FRIENDLY_NAMES.iteritems() if k in purge_list) warn_msg = "CAUTION: You have specified the --purge option with --purge-list={0}. " \ "This will replace all existing {1} currently installed.\n" \ "Are you absolutely sure you want to perform the purge [yes/no]? (no)".format( purge_list, ", ".join(purge_resources)) okToPurge = get_YN_input(warn_msg, False) if not okToPurge: err = "Management pack installation cancelled by user" raise FatalException(1, err)
def uninstall_mpack(mpack_name, mpack_version): """ Uninstall specific management pack :param mpack_name: Management pack name :param mpack_version: Management pack version """ print_info_msg("Uninstalling management pack {0}-{1}".format( mpack_name, mpack_version)) # Get ambari mpack properties stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties( ) found = False if os.path.exists(mpacks_staging_location) and os.path.isdir( mpacks_staging_location): staged_mpack_dirs = sorted(os.listdir(mpacks_staging_location)) for dir in staged_mpack_dirs: if dir == MPACKS_CACHE_DIRNAME: continue staged_mpack_dir = os.path.join(mpacks_staging_location, dir) if os.path.isdir(staged_mpack_dir): staged_mpack_metadata = read_mpack_metadata(staged_mpack_dir) if not staged_mpack_metadata: print_error_msg( "Skipping malformed management pack {0}-{1}. Metadata file missing!" .format(staged_mpack_name, staged_mpack_version)) continue staged_mpack_name = staged_mpack_metadata.name staged_mpack_version = staged_mpack_metadata.version if mpack_name == staged_mpack_name and compare_versions( staged_mpack_version, mpack_version, format=True) == 0: print_info_msg( "Removing management pack staging location {0}".format( staged_mpack_dir)) sudo.rmtree(staged_mpack_dir) remove_symlinks(stack_location, service_definitions_location, staged_mpack_dir) found = True break if not found: print_error_msg("Management pack {0}-{1} is not installed!".format( mpack_name, mpack_version)) else: print_info_msg( "Management pack {0}-{1} successfully uninstalled!".format( mpack_name, mpack_version))
def process_stack_addon_service_definitions_artifact(artifact, artifact_source_dir, options): """ Process stack addon service definitions artifact :param artifact: Artifact metadata :param artifact_source_dir: Location of artifact in the management pack :param options: Command line options """ # Get ambari mpack properties stack_location, extension_location, service_definitions_location, mpacks_staging_location = get_mpack_properties( ) service_versions_map = None if "service_versions_map" in artifact: service_versions_map = artifact.service_versions_map if not service_versions_map: msg = "Must provide service versions map for " + STACK_ADDON_SERVICE_DEFINITIONS_ARTIFACT_NAME + " artifact!" print_error_msg(msg) raise FatalException(-1, msg) for service_name in sorted(os.listdir(artifact_source_dir)): source_service_path = os.path.join(artifact_source_dir, service_name) for service_version in sorted(os.listdir(source_service_path)): source_service_version_path = os.path.join(source_service_path, service_version) for service_version_entry in service_versions_map: if service_name == service_version_entry.service_name and service_version == service_version_entry.service_version: applicable_stacks = service_version_entry.applicable_stacks for applicable_stack in applicable_stacks: stack_name = applicable_stack.stack_name stack_version = applicable_stack.stack_version dest_stack_path = os.path.join(stack_location, stack_name) dest_stack_version_path = os.path.join( dest_stack_path, stack_version) dest_stack_services_path = os.path.join( dest_stack_version_path, "services") dest_link = os.path.join(dest_stack_services_path, service_name) if os.path.exists(dest_stack_path) and os.path.exists( dest_stack_version_path): if not os.path.exists(dest_stack_services_path): sudo.makedir(dest_stack_services_path, 0755) if options.force and os.path.islink(dest_link): sudo.unlink(dest_link) sudo.symlink(source_service_version_path, dest_link)
def upgrade_local_repo(args): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 stack_location = get_stack_location(properties) stack_root_local = os.path.join(stack_location, "HDPLocal") if not os.path.exists(stack_root_local): print_info_msg("HDPLocal stack directory does not exist, skipping") return stack_root = os.path.join(stack_location, "HDP") if not os.path.exists(stack_root): print_info_msg("HDP stack directory does not exist, skipping") return for stack_version_local in os.listdir(stack_root_local): repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml.rpmsave") if not os.path.exists(repo_file_local): repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml") repo_file = os.path.join(stack_root, stack_version_local, "repos", "repoinfo.xml") print_info_msg("Local repo file: " + repo_file_local) print_info_msg("Repo file: " + repo_file_local) metainfo_update_items = {} if os.path.exists(repo_file_local) and os.path.exists(repo_file): local_values = load_stack_values(stack_version_local, repo_file_local) repo_values = load_stack_values(stack_version_local, repo_file) for k, v in local_values.iteritems(): if repo_values.has_key(k): local_url = local_values[k] repo_url = repo_values[k] if repo_url != local_url: metainfo_update_items[k] = local_url run_metainfo_upgrade(metainfo_update_items)
def check_database_name_property(upgrade=False): """ :param upgrade: If Ambari is being upgraded. :return: """ properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 version = get_ambari_version(properties) if upgrade and (properties[JDBC_DATABASE_PROPERTY] not in ["postgres", "oracle", "mysql", "mssql", "derby"] or properties.has_key(JDBC_RCA_SCHEMA_PROPERTY)): # This code exists for historic reasons in which property names changed from Ambari 1.6.1 to 1.7.0 persistence_type = properties[PERSISTENCE_TYPE_PROPERTY] if persistence_type == "remote": db_name = properties[ "server.jdbc.schema"] # this was a property in Ambari 1.6.1, but not after 1.7.0 if db_name: write_property(JDBC_DATABASE_NAME_PROPERTY, db_name) # If DB type is missing, attempt to reconstruct it from the JDBC URL db_type = properties[JDBC_DATABASE_PROPERTY] if db_type is None or db_type.strip().lower() not in [ "postgres", "oracle", "mysql", "mssql", "derby" ]: db_type = get_db_type(properties) if db_type: write_property(JDBC_DATABASE_PROPERTY, db_type) properties = get_ambari_properties() elif persistence_type == "local": # Ambari 1.6.1, had "server.jdbc.database" as the DB name, and the # DB type was assumed to be "postgres" if was embedded ("local") db_name = properties[JDBC_DATABASE_PROPERTY] if db_name: write_property(JDBC_DATABASE_NAME_PROPERTY, db_name) write_property(JDBC_DATABASE_PROPERTY, "postgres") properties = get_ambari_properties() dbname = properties[JDBC_DATABASE_NAME_PROPERTY] if dbname is None or dbname == "": err = "DB Name property not set in config file.\n" + SETUP_OR_UPGRADE_MSG raise FatalException(-1, err)
def save_pid(pid, pidfile): """ Save pid to pidfile. """ try: pfile = open(pidfile, "w") pfile.write("%s\n" % pid) except IOError as e: print_error_msg("Failed to write PID to " + pidfile + " due to " + str(e)) pass finally: try: pfile.close() except Exception as e: print_error_msg("Failed to close PID file " + pidfile + " due to " + str(e)) pass
def run_schema_upgrade(): jdk_path = get_java_exe_path() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 print 'Upgrading database schema' command = SCHEMA_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath()) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from schema upgrade command, retcode = " + str(retcode)) if retcode > 0: print_error_msg("Error executing schema upgrade, please check the server logs.") else: print_info_msg('Schema upgrade completed') return retcode
def _check_postgre_up(): pg_status, retcode, out, err = PGConfig._get_postgre_status() if pg_status == PGConfig.PG_STATUS_RUNNING: print_info_msg("PostgreSQL is running") return pg_status, 0, out, err else: # run initdb only on non ubuntu systems as ubuntu does not have initdb cmd. if not OSCheck.is_ubuntu_family(): print "Running initdb: This may take upto a minute." retcode, out, err = run_os_command(PGConfig.PG_INITDB_CMD) if retcode == 0: print out print "About to start PostgreSQL" try: process = subprocess.Popen(PGConfig.PG_START_CMD.split(' '), stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) if OSCheck.is_suse_family(): time.sleep(20) result = process.poll() print_info_msg("Result of postgres start cmd: " + str(result)) if result is None: process.kill() pg_status, retcode, out, err = PGConfig._get_postgre_status( ) else: retcode = result else: out, err = process.communicate() retcode = process.returncode pg_status, retcode, out, err = PGConfig._get_postgre_status( ) if pg_status == PGConfig.PG_STATUS_RUNNING: print_info_msg("Postgres process is running. Returning...") return pg_status, 0, out, err except (Exception), e: pg_status, retcode, out, err = PGConfig._get_postgre_status() if pg_status == PGConfig.PG_STATUS_RUNNING: return pg_status, 0, out, err else: print_error_msg("Postgres start failed. " + str(e)) return pg_status, retcode, out, err
def check_database(options): logger.info("Check database consistency.") jdk_path = serverConfiguration.get_java_exe_path() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) sys.exit(1) properties = serverConfiguration.get_ambari_properties() serverConfiguration.parse_properties_file(options) database_type = properties[JDBC_DATABASE_PROPERTY] if not database_type: print_error_msg("Please run \"ambari-server setup\" command" " to initialize ambari db properties.") sys.exit(1) options.database_index = LINUX_DBMS_KEYS_LIST.index(properties[JDBC_DATABASE_PROPERTY]) dbConfiguration.ensure_jdbc_driver_is_installed(options, serverConfiguration.get_ambari_properties()) serverClassPath = ServerClassPath(serverConfiguration.get_ambari_properties(), options) class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell() command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path) ambari_user = serverConfiguration.read_ambari_user() current_user = setupSecurity.ensure_can_start_under_current_user(ambari_user) environ = setupSecurity.generate_env(options, ambari_user, current_user) (retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ) if retcode > 0: raise FatalException(int(retcode), "Database check failed to complete: {0}. \nPlease check {1} and {2} for more " "information.".format(stdout+stderr, configDefaults.SERVER_LOG_FILE, configDefaults.DB_CHECK_LOG)) else: print str(stdout) if not stdout.startswith("No errors"): print "Ambari Server 'check-database' completed" sys.exit(1)
def run_metainfo_upgrade(keyValueMap=None): jdk_path = get_java_exe_path() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) retcode = 1 if keyValueMap: command = STACK_UPGRADE_HELPER_CMD.format(jdk_path, get_full_ambari_classpath(), 'updateMetaInfo', "'" + json.dumps(keyValueMap) + "'") (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from stack upgrade command, retcode = " + str(retcode)) if retcode > 0: print_error_msg("Error executing metainfo upgrade, please check the " "server logs.") return retcode
def extract_views(options): java_exe_path = get_java_exe_path() if java_exe_path is None: print_error_msg( "No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 vdir = get_value_from_properties(properties, VIEWS_DIR_PROPERTY, configDefaults.DEFAULT_VIEWS_DIR) files = [ f for f in os.listdir(vdir) if os.path.isfile(os.path.join(vdir, f)) ] serverClassPath = ServerClassPath(get_ambari_properties(), options) for f in files: command = VIEW_EXTRACT_CMD.format( java_exe_path, serverClassPath.get_full_ambari_classpath_escaped_for_shell(), os.path.join(vdir, f)) retcode, stdout, stderr = run_os_command(command) if retcode == 0: sys.stdout.write(f + "\n") # elif retcode == 2: # sys.stdout.write("Error extracting " + f + "\n") else: sys.stdout.write(".") sys.stdout.flush() print_info_msg("Return code from extraction of view archive " + f + ": " + str(retcode)) sys.stdout.write("\n") return 0
def save_passwd_for_alias(alias, passwd, masterKey=""): if alias and passwd: jdk_path = find_jdk() if jdk_path is None: print_error_msg("No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) return 1 if masterKey is None or masterKey == "": masterKey = "None" command = SECURITY_PROVIDER_PUT_CMD.format(get_java_exe_path(), get_full_ambari_classpath(), alias, passwd, masterKey) (retcode, stdout, stderr) = run_os_command(command) print_info_msg("Return code from credential provider save passwd: " + str(retcode)) return retcode else: print_error_msg("Alias or password is unreadable.")
def validate_mpack_prerequisites(mpack_metadata): """ Validate management pack prerequisites :param mpack_name: Management pack metadata """ # Get ambari config properties properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 stack_location = get_stack_location(properties) current_ambari_version = get_ambari_version(properties) fail = False mpack_prerequisites = mpack_metadata.prerequisites if "min_ambari_version" in mpack_prerequisites: min_ambari_version = mpack_prerequisites.min_ambari_version if (compare_versions( min_ambari_version, current_ambari_version, format=True) > 0): print_error_msg( "Prerequisite failure! Current Ambari Version = {0}, " "Min Ambari Version = {1}".format(current_ambari_version, min_ambari_version)) fail = True if "max_ambari_version" in mpack_prerequisites: max_ambari_version = mpack_prerequisites.max_ambari_version if (compare_versions( max_ambari_version, current_ambari_version, format=True) < 0): print_error_msg( "Prerequisite failure! Current Ambari Version = {0}, " "Max Ambari Version = {1}".format(current_ambari_version, max_ambari_version)) if "min_stack_versions" in mpack_prerequisites: min_stack_versions = mpack_prerequisites.min_stack_versions stack_found = False for min_stack_version in min_stack_versions: stack_name = min_stack_version.stack_name stack_version = min_stack_version.stack_version stack_dir = os.path.join(stack_location, stack_name, stack_version) if os.path.exists(stack_dir) and os.path.isdir(stack_dir): stack_found = True break if not stack_found: print_error_msg( "Prerequisite failure! Min applicable stack not found") fail = True if fail: raise FatalException( -1, "Prerequisites for management pack {0}-{1} failed!".format( mpack_metadata.name, mpack_metadata.version))
def check_database(options): jdk_path = serverConfiguration.get_java_exe_path() if jdk_path is None: print_error_msg( "No JDK found, please run the \"setup\" " "command to install a JDK automatically or install any " "JDK manually to " + configDefaults.JDK_INSTALL_DIR) sys.exit(1) properties = serverConfiguration.get_ambari_properties() serverConfiguration.parse_properties_file(options) options.database_index = LINUX_DBMS_KEYS_LIST.index( properties[JDBC_DATABASE_PROPERTY]) dbConfiguration.ensure_jdbc_driver_is_installed( options, serverConfiguration.get_ambari_properties()) serverClassPath = ServerClassPath( serverConfiguration.get_ambari_properties(), options) class_path = serverClassPath.get_full_ambari_classpath_escaped_for_shell() command = CHECK_DATABASE_HELPER_CMD.format(jdk_path, class_path) ambari_user = serverConfiguration.read_ambari_user() current_user = setupSecurity.ensure_can_start_under_current_user( ambari_user) environ = setupSecurity.generate_env(options, ambari_user, current_user) (retcode, stdout, stderr) = os_utils.run_os_command(command, env=environ) print_info_msg("Return code from check database command, retcode = " + str(retcode)) if retcode > 0: print_error_msg( "Database check failed to complete. Please check ambari-server.log and ambari-server-check-database.log for problem." ) raise FatalException(1, 'Database check failed.') else: print str(stdout)
def configure_os_settings(): properties = get_ambari_properties() if properties == -1: print_error_msg("Error getting ambari properties") return -1 try: conf_os_type = properties[OS_TYPE_PROPERTY] if conf_os_type != '': print_info_msg("os_type already set in the properties file") return 0 except (KeyError): print_error_msg("os_type is not set in the properties file. Setting it now.") # to check server/agent compatibility master_os_family = OS_FAMILY + OS_VERSION # to check supported os_types master_os_type = OS_TYPE + OS_VERSION write_property(OS_FAMILY_PROPERTY, master_os_family) write_property(OS_TYPE_PROPERTY, master_os_type) return 0
def update_ambari_properties(): prev_conf_file = search_file(configDefaults.AMBARI_PROPERTIES_BACKUP_FILE, get_conf_dir()) conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir()) # Previous config file does not exist if (not prev_conf_file) or (prev_conf_file is None): print_warning_msg("Can not find %s file from previous version, skipping import of settings" % configDefaults.AMBARI_PROPERTIES_BACKUP_FILE) return 0 # ambari.properties file does not exists if conf_file is None: print_error_msg("Can't find %s file" % AMBARI_PROPERTIES_FILE) return -1 with open(prev_conf_file) as hfOld: try: old_properties = Properties() old_properties.load(hfOld) except Exception, e: print 'Could not read "%s": %s' % (prev_conf_file, e) return -1
def _change_db_files_owner(self): database_name = self.database_name new_owner = self.database_username if '"' not in new_owner: #wrap to allow old username "ambari-server", postgres only new_owner = '\'"{0}"\''.format(new_owner) pass command = PGConfig.CHANGE_OWNER_COMMAND[:] command[-1] = command[-1].format(database_name, 'ambari', new_owner) retcode, stdout, stderr = run_os_command(command) if not retcode == 0: if get_verbose(): if stderr: print_error_msg("stderr:\n" + stderr.strip()) if stdout: print_error_msg("stdout:\n" + stdout.strip()) else: print_info_msg('Fixed database objects owner') return retcode