def refresh_stack_hash(properties):
resources_location = get_resources_location(properties)
stacks_location = get_stack_location(properties)
resource_files_keeper = ResourceFilesKeeper(resources_location, stacks_location)
try:
print "Organizing resource files at {0}...".format(resources_location,
verbose=get_verbose())
resource_files_keeper.perform_housekeeping()
except KeeperException, ex:
msg = "Can not organize resource files at {0}: {1}".format(
resources_location, str(ex))
raise FatalException(-1, msg)
def refresh_stack_hash(properties):
resources_location = get_resources_location(properties)
stacks_location = get_stack_location(properties)
resource_files_keeper = ResourceFilesKeeper(resources_location, stacks_location)
try:
print "Organizing resource files at {0}...".format(resources_location,
verbose=get_verbose())
resource_files_keeper.perform_housekeeping()
except KeeperException, ex:
msg = "Can not organize resource files at {0}: {1}".format(
resources_location, str(ex))
raise FatalException(-1, msg)
def validate_mpack_prerequisites(mpack_metadata):
"""
Validate management pack prerequisites
:param mpack_name: Management pack metadata
"""
# Get ambari config properties
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
current_ambari_version = get_ambari_version(properties)
fail = False
mpack_prerequisites = mpack_metadata.prerequisites
if "min_ambari_version" in mpack_prerequisites:
min_ambari_version = mpack_prerequisites.min_ambari_version
if (compare_versions(
min_ambari_version, current_ambari_version, format=True) > 0):
print_error_msg(
"Prerequisite failure! Current Ambari Version = {0}, "
"Min Ambari Version = {1}".format(current_ambari_version,
min_ambari_version))
fail = True
if "max_ambari_version" in mpack_prerequisites:
max_ambari_version = mpack_prerequisites.max_ambari_version
if (compare_versions(
max_ambari_version, current_ambari_version, format=True) < 0):
print_error_msg(
"Prerequisite failure! Current Ambari Version = {0}, "
"Max Ambari Version = {1}".format(current_ambari_version,
max_ambari_version))
if "min_stack_versions" in mpack_prerequisites:
min_stack_versions = mpack_prerequisites.min_stack_versions
stack_found = False
for min_stack_version in min_stack_versions:
stack_name = min_stack_version.stack_name
stack_version = min_stack_version.stack_version
stack_dir = os.path.join(stack_location, stack_name, stack_version)
if os.path.exists(stack_dir) and os.path.isdir(stack_dir):
stack_found = True
break
if not stack_found:
print_error_msg(
"Prerequisite failure! Min applicable stack not found")
fail = True
if fail:
raise FatalException(
-1, "Prerequisites for management pack {0}-{1} failed!".format(
mpack_metadata.name, mpack_metadata.version))
def refresh_stack_hash(properties):
stack_location = get_stack_location(properties)
# Hack: we determine resource dir as a parent dir for stack_location
resources_location = os.path.dirname(stack_location)
resource_files_keeper = ResourceFilesKeeper(resources_location)
try:
print "Organizing resource files at {0}...".format(
resources_location, verbose=get_verbose())
resource_files_keeper.perform_housekeeping()
except KeeperException, ex:
msg = "Can not organize resource files at {0}: {1}".format(
resources_location, str(ex))
raise FatalException(-1, msg)
def refresh_stack_hash(properties):
stack_location = get_stack_location(properties)
# Hack: we determine resource dir as a parent dir for stack_location
resources_location = os.path.dirname(stack_location)
resource_files_keeper = ResourceFilesKeeper(resources_location)
try:
# print "Organizing resource files at {0}...".format(resources_location, verbose=get_verbose())
print "Organizing resource files..."
resource_files_keeper.perform_housekeeping()
except KeeperException, ex:
msg = "Can not organize resource files at {0}: {1}".format(
resources_location, str(ex))
raise FatalException(-1, msg)
def get_mpack_properties():
"""
Read ambari properties required for management packs
:return: (stack_location, service_definitions_location, mpacks_staging_location)
"""
# Get ambari config properties
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
service_definitions_location = get_common_services_location(properties)
mpacks_staging_location = get_mpacks_staging_location(properties)
ambari_version = get_ambari_version(properties)
return stack_location, service_definitions_location, mpacks_staging_location
def upgrade_local_repo(args):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
stack_root_local = os.path.join(stack_location, "HDPLocal")
if not os.path.exists(stack_root_local):
print_info_msg("HDPLocal stack directory does not exist, skipping")
return
stack_root = os.path.join(stack_location, "HDP")
if not os.path.exists(stack_root):
print_info_msg("HDP stack directory does not exist, skipping")
return
for stack_version_local in os.listdir(stack_root_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local,
"repos", "repoinfo.xml.rpmsave")
if not os.path.exists(repo_file_local):
repo_file_local = os.path.join(stack_root_local,
stack_version_local, "repos",
"repoinfo.xml")
repo_file = os.path.join(stack_root, stack_version_local, "repos",
"repoinfo.xml")
print_info_msg("Local repo file: " + repo_file_local)
print_info_msg("Repo file: " + repo_file_local)
metainfo_update_items = {}
if os.path.exists(repo_file_local) and os.path.exists(repo_file):
local_values = load_stack_values(stack_version_local,
repo_file_local)
repo_values = load_stack_values(stack_version_local, repo_file)
for k, v in local_values.iteritems():
if repo_values.has_key(k):
local_url = local_values[k]
repo_url = repo_values[k]
if repo_url != local_url:
metainfo_update_items[k] = local_url
run_metainfo_upgrade(metainfo_update_items)
def upgrade_local_repo(args):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
stack_root_local = os.path.join(stack_location, "HDPLocal")
if not os.path.exists(stack_root_local):
print_info_msg("HDPLocal stack directory does not exist, skipping")
return
stack_root = os.path.join(stack_location, "HDP")
if not os.path.exists(stack_root):
print_info_msg("HDP stack directory does not exist, skipping")
return
for stack_version_local in os.listdir(stack_root_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml.rpmsave")
if not os.path.exists(repo_file_local):
repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml")
repo_file = os.path.join(stack_root, stack_version_local, "repos", "repoinfo.xml")
print_info_msg("Local repo file: " + repo_file_local)
print_info_msg("Repo file: " + repo_file_local)
metainfo_update_items = {}
if os.path.exists(repo_file_local) and os.path.exists(repo_file):
local_values = load_stack_values(stack_version_local, repo_file_local)
repo_values = load_stack_values(stack_version_local, repo_file)
for k, v in local_values.iteritems():
if repo_values.has_key(k):
local_url = local_values[k]
repo_url = repo_values[k]
if repo_url != local_url:
metainfo_update_items[k] = local_url
run_metainfo_upgrade(args, metainfo_update_items)
def update_stack_metainfo(stack_name, stack_versions):
properties = get_ambari_properties()
if properties == -1:
print_error_msg("Error getting ambari properties")
return -1
stack_location = get_stack_location(properties)
print_info_msg("stack location: " + stack_location)
stack_root = os.path.join(stack_location, stack_name)
print_info_msg("stack root: " + stack_root)
if not os.path.exists(stack_root):
print_error_msg("stack directory does not exists: " + stack_root)
return -1
for stack in stack_versions:
if stack not in os.listdir(stack_root):
print_error_msg("The requested stack version: " + stack +
" is not available in the HDP stack")
return -1
for directory in os.listdir(stack_root):
print_info_msg("directory found: " + directory)
metainfo_file = os.path.join(stack_root, directory, "metainfo.xml")
print_info_msg("looking for metainfo file: " + metainfo_file)
if not os.path.exists(metainfo_file):
print_error_msg("Could not find metainfo file in the path " +
metainfo_file)
continue
if directory in stack_versions:
print_info_msg("updating stack to active for: " + directory)
replace(metainfo_file, "<active>false</active>",
"<active>true</active>")
else:
print_info_msg("updating stack to inactive for: " + directory)
replace(metainfo_file, "<active>true</active>",
"<active>false</active>")
return 0
def upgrade(args):
print_info_msg("Upgrade Ambari Server", True)
if not is_root():
err = configDefaults.MESSAGE_ERROR_UPGRADE_NOT_ROOT
raise FatalException(4, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(
AMBARI_PROPERTIES_FILE), True)
retcode = update_ambari_properties()
if not retcode == 0:
err = AMBARI_PROPERTIES_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
print_info_msg(
'Updating Ambari Server properties in {0} ...'.format(AMBARI_ENV_FILE),
True)
retcode = update_ambari_env()
if not retcode == 0:
err = AMBARI_ENV_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
retcode = update_krb_jaas_login_properties()
if retcode == -2:
pass # no changes done, let's be silent
elif retcode == 0:
print_info_msg("File {0} updated.".format(AMBARI_KRB_JAAS_LOGIN_FILE),
True)
elif not retcode == 0:
err = AMBARI_KRB_JAAS_LOGIN_FILE + ' file can\'t be updated. Exiting'
raise FatalException(retcode, err)
restore_custom_services()
replay_mpack_logs()
try:
update_database_name_property(upgrade=True)
except FatalException:
return -1
# Ignore the server version & database options passed via command-line arguments
parse_properties_file(args)
#TODO check database version
change_objects_owner(args)
retcode = run_schema_upgrade(args)
if not retcode == 0:
print_error_msg(
"Ambari server upgrade failed. Please look at {0}, for more details."
.format(configDefaults.SERVER_LOG_FILE))
raise FatalException(11, 'Schema upgrade failed.')
user = read_ambari_user()
if user is None:
warn = "Can not determine custom ambari user.\n" + SETUP_OR_UPGRADE_MSG
print_warning_msg(warn)
else:
adjust_directory_permissions(user)
# create jdbc symlinks if jdbc drivers are available in resources
check_jdbc_drivers(args)
properties = get_ambari_properties()
if properties == -1:
err = "Error getting ambari properties"
print_error_msg(err)
raise FatalException(-1, err)
# Move *.py files from custom_actions to custom_actions/scripts
# This code exists for historic reasons in which custom action python scripts location changed from Ambari 1.7.0 to 2.0.0
ambari_version = get_ambari_version(properties)
if ambari_version is None:
args.warnings.append(
"*.py files were not moved from custom_actions to custom_actions/scripts."
)
elif compare_versions(ambari_version, "2.0.0") == 0:
move_user_custom_actions()
# Move files installed by package to default views directory to a custom one
for views_dir in get_views_dir(properties):
root_views_dir = views_dir + "/../"
if os.path.samefile(root_views_dir, get_default_views_dir()):
continue
for file in glob.glob(get_default_views_dir() + '/*'):
shutil.move(file, root_views_dir)
# Remove ADMIN_VIEW directory for upgrading Admin View on Ambari upgrade from 1.7.0 to 2.0.0
admin_views_dirs = get_admin_views_dir(properties)
for admin_views_dir in admin_views_dirs:
shutil.rmtree(admin_views_dir)
# Modify timestamp of views jars to current time
views_jars = get_views_jars(properties)
for views_jar in views_jars:
os.utime(views_jar, None)
# check if ambari is configured to use LDAP authentication
if properties.get_property(CLIENT_SECURITY) == "ldap":
args.warnings.append(
"LDAP authentication is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration."
)
# adding custom jdbc name and previous custom jdbc properties
# we need that to support new dynamic jdbc names for upgraded ambari
add_jdbc_properties(properties)
json_url = get_json_url_from_repo_file()
if json_url:
print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(
json_url)
properties = get_ambari_properties()
stack_root = get_stack_location(properties)
update_latest_in_repoinfos_for_stacks(stack_root, json_url)
else:
print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
def setup(options):
logger.info("Setup ambari-server.")
if options.only_silent:
if check_setup_already_done():
print "Nothing was done. Ambari Setup already performed and cannot re-run setup in silent mode. Use \"ambari-server setup\" command without -s option to change Ambari setup."
sys.exit(0)
retcode = verify_setup_allowed(options)
if not retcode == 0:
raise FatalException(1, None)
if not is_root():
warn_msg = configDefaults.MESSAGE_WARN_SETUP_NOT_ROOT
print warn_msg
# proceed jdbc properties if they were set
if _check_jdbc_options(options):
proceedJDBCProperties(options)
return
(retcode, err) = disable_security_enhancements()
if not retcode == 0:
raise FatalException(retcode, err)
#Create ambari user, if needed
(retcode, register_service, svc_user, svc_password) = check_ambari_user(options)
if not retcode == 0:
err = 'Failed to create user. Exiting.'
raise FatalException(retcode, err)
print configDefaults.MESSAGE_CHECK_FIREWALL
check_firewall()
# proceed jdbc properties if they were set
if _check_jdbc_options(options):
proceedJDBCProperties(options)
print 'Checking JDK...'
try:
download_and_install_jdk(options)
except FatalException as e:
err = 'Downloading or installing JDK failed: {0}. Exiting.'.format(e)
raise FatalException(e.code, err)
print 'Checking GPL software agreement...'
write_gpl_license_accepted(default_prompt_value=options.accept_gpl)
print 'Completing setup...'
retcode = configure_os_settings()
if not retcode == 0:
err = 'Configure of OS settings in ambari.properties failed. Exiting.'
raise FatalException(retcode, err)
print 'Configuring database...'
prompt_db_properties(options)
#DB setup should be done last after doing any setup.
_setup_database(options)
check_jdbc_drivers(options)
if not options.skip_view_extraction:
print 'Extracting system views...'
retcode = extract_views(options)
if not retcode == 0:
err = 'Error while extracting system views. Exiting'
raise FatalException(retcode, err)
json_url = get_json_url_from_repo_file()
if json_url:
print "Ambari repo file contains latest json url {0}, updating stacks repoinfos with it...".format(json_url)
properties = get_ambari_properties()
stack_root = get_stack_location(properties)
update_latest_in_repoinfos_for_all_stacks(stack_root, json_url)
else:
print "Ambari repo file doesn't contain latest json url, skipping repoinfos modification"
# we've already done this, but new files were created so run it one time.
adjust_directory_permissions(svc_user)
service_setup(register_service, svc_user, svc_password)