def _install_lzo_support_if_needed(self, params):
hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site'])
hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-")
if hadoop_lzo_dest_path:
hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]
config = Script.get_config()
file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file)
hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file)
download_file(file_url, hadoop_lzo_dl_path)
#This is for protection against configuration changes. It will infect every new destination with the lzo jar,
# but since the classpath points to the jar directly we're getting away with it.
if not os.path.exists(hadoop_lzo_dest_path):
copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
See the License for the specific language governing permissions and
limitations under the License.
Ambari Agent
"""
import status_params
from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_hdp_stack_version
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
stack_name = default("/hostLevelParams/stack_name", None)
current_version = default("/hostLevelParams/current_version", None)
component_directory = status_params.component_directory
# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
version = default("/commandParams/version", None)
# default parameters
zk_home = "/usr/hdp/2.3.2.0-2950/zookeeper"
zk_bin = "/usr/hdp/2.3.2.0-2950/zookeeper/bin"
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version_id = None
base_urls = []
# Select dict that contains parameters
try:
if 'base_urls' in config['roleParams']:
base_urls = json.loads(config['roleParams']['base_urls'])
self.repository_version = config['roleParams']['repository_version']
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
if 'repository_version_id' in config['roleParams']:
self.repository_version_id = config['roleParams']['repository_version_id']
except KeyError:
pass
# current stack information
self.current_stack_version_formatted = None
if 'stack_version' in config['hostLevelParams']:
current_stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted)
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
# Install/update repositories
self.current_repositories = []
self.current_repo_files = set()
# Enable base system repositories
# We don't need that for RHEL family, because we leave all repos enabled
# except disabled HDP* ones
if OSCheck.is_suse_family():
self.current_repositories.append('base')
elif OSCheck.is_ubuntu_family():
self.current_repo_files.add('base')
Logger.info("Will install packages for repository version {0}".format(self.repository_version))
if 0 == len(base_urls):
Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version))
try:
if 'repositoryFile' in config:
create_repo_files(template, CommandRepository(config['repositoryFile']))
else:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
except Exception, err:
Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.script.script import Script
# a map of the Ambari role to the component name
# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'SPARK_JOBHISTORYSERVER': 'spark-historyserver',
'SPARK_CLIENT': 'spark-client',
'SPARK_THRIFTSERVER': 'spark-thriftserver'
}
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP,
"SPARK_CLIENT")
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
stack_version_formatted = format_stack_version(stack_version_unformatted)
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
version = default("/commandParams/version", None)
# TODO! FIXME! Version check is not working as of today :
# $ yum list installed | grep <stack-selector-tool>
# <stack-selector-tool>.noarch 2.2.1.0-2340.el6 @HDP-2.2
# And stack_version_formatted returned from hostLevelParams/stack_version is : 2.2.0.0
# Commenting out for time being
from resource_management.libraries.script.script import Script
CONFIG = Script.get_config()
CONFIGS = CONFIG['configurations']
HOST_INFO = CONFIG['clusterHostInfo']
CASSANDRA_CONF = CONFIGS['cassandra-conf']
cassandra_conf_dir = '/etc/cassandra/conf'
cassandra_user = '******'
config_defaults = {
'cross_node_timeout': False,
'inter_dc_tcp_nodelay': False,
'rpc_server_type': 'sync',
'disk_failure_policy': 'stop',
'authorizer': 'AllowAllAuthorizer',
'tombstone_warn_threshold': 1000,
'internode_compression': 'all',
'truncate_request_timeout_in_ms': 60000,
'cluster_name': 'Test Cluster',
'read_request_timeout_in_ms': 5000,
'ssl_storage_port': 7001,
'listen_address': 'localhost',
'request_scheduler': 'org.apache.cassandra.scheduler.NoScheduler',
'range_request_timeout_in_ms': 10000,
'hinted_handoff_enabled': True,
'max_hint_window_in_ms': 10800000,
'authenticator': 'AllowAllAuthenticator',
'tombstone_failure_threshold': 100000,
'commitlog_directory': '/var/lib/cassandra/commitlog',
'column_index_size_in_kb': 64,
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if not command_repository.repositories:
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
new_repo_files = create_repo_files(template,
command_repository)
self.repo_files.update(new_repo_files)
except Exception, err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
def install(self, env):
self.install_packages(env)
config = Script.get_config()
self.configure(env)
def get_packages(scope, service_name=None, component_name=None):
"""
Gets the packages which should be used with the stack's stack-select tool for the
specified service/component. Not all services/components are used with the stack-select tools,
so those will return no packages.
:param scope: the scope of the command
:param service_name: the service name, such as ZOOKEEPER
:param component_name: the component name, such as ZOOKEEPER_SERVER
:return: the packages to use with stack-select or None
"""
from resource_management.libraries.functions.default import default
if scope not in _PACKAGE_SCOPES:
raise Fail("The specified scope of {0} is not valid".format(scope))
config = Script.get_config()
if service_name is None or component_name is None:
if 'role' not in config or 'serviceName' not in config:
raise Fail(
"Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool"
)
service_name = config['serviceName']
component_name = config['role']
stack_name = default("/hostLevelParams/stack_name", None)
if stack_name is None:
raise Fail(
"The stack name is not present in the command. Packages for stack-select tool cannot be loaded."
)
stack_packages_config = default(
"/configurations/cluster-env/stack_packages", None)
if stack_packages_config is None:
raise Fail(
"The stack packages are not defined on the command. Unable to load packages for the stack-select tool"
)
data = json.loads(stack_packages_config)
if stack_name not in data:
raise Fail(
"Cannot find stack-select packages for the {0} stack".format(
stack_name))
stack_select_key = "stack-select"
data = data[stack_name]
if stack_select_key not in data:
raise Fail(
"There are no stack-select packages defined for this command for the {0} stack"
.format(stack_name))
# this should now be the dictionary of role name to package name
data = data[stack_select_key]
service_name = service_name.upper()
component_name = component_name.upper()
if service_name not in data:
Logger.info(
"Skipping stack-select on {0} because it does not exist in the stack-select package structure."
.format(service_name))
return None
data = data[service_name]
if component_name not in data:
Logger.info(
"Skipping stack-select on {0} because it does not exist in the stack-select package structure."
.format(component_name))
return None
# this one scope is not an array, so transform it into one for now so we can
# use the same code below
packages = data[component_name][scope]
if scope == PACKAGE_SCOPE_STACK_SELECT:
packages = [packages]
# grab the package name from the JSON and validate it against the packages
# that the stack-select tool supports - if it doesn't support it, then try to find the legacy
# package name if it exists
supported_packages = get_supported_packages()
for index, package in enumerate(packages):
if not is_package_supported(package,
supported_packages=supported_packages):
if _PACKAGE_SCOPE_LEGACY in data[component_name]:
legacy_package = data[component_name][_PACKAGE_SCOPE_LEGACY]
Logger.info(
"The package {0} is not supported by this version of the stack-select tool, defaulting to the legacy package of {1}"
.format(package, legacy_package))
# use the legacy package
packages[index] = legacy_package
else:
raise Fail(
"The package {0} is not supported by this version of the stack-select tool."
.format(package))
# transform the array bcak to a single element
if scope == PACKAGE_SCOPE_STACK_SELECT:
packages = packages[0]
return packages
def setup_spark(env, type, upgrade_type=None, action=None):
import params
config = Script.get_config()
Directory([params.spark_pid_dir, params.spark_log_dir],
owner=params.spark_user,
group=params.user_group,
mode=0775,
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.spark_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
mode=0775)
params.HdfsResource(None, action="execute")
spark_history_dir = params.spark_history_dir
hadoop_user = config['configurations']['cluster-env'][
'user_group'] or 'hadoop'
params.HdfsResource(spark_history_dir,
type="directory",
action="create_on_execute",
owner=params.spark_user,
group=hadoop_user,
mode=0775)
PropertiesFile(
format("{spark_conf}/spark-defaults.conf"),
properties=params.config['configurations']['spark2-defaults'],
key_value_delimiter=" ",
owner=params.spark_user,
group=params.spark_group,
mode=0644)
# create spark-env.sh in etc/conf dir
File(
os.path.join(params.spark_conf, 'spark-env.sh'),
owner=params.spark_user,
group=params.spark_group,
content=InlineTemplate(params.spark_env_sh),
mode=0644,
)
#create log4j.properties in etc/conf dir
File(
os.path.join(params.spark_conf, 'log4j.properties'),
owner=params.spark_user,
group=params.spark_group,
content=params.spark_log4j_properties,
mode=0644,
)
#create metrics.properties in etc/conf dir
File(os.path.join(params.spark_conf, 'metrics.properties'),
owner=params.spark_user,
group=params.spark_group,
content=InlineTemplate(params.spark_metrics_properties),
mode=0644)
if params.is_hive_installed:
XmlConfig("hive-site.xml",
conf_dir=params.spark_conf,
configurations=params.spark_hive_properties,
owner=params.spark_user,
group=params.spark_group,
mode=0644)
if params.has_spark_thriftserver:
PropertiesFile(params.spark_thrift_server_conf_file,
properties=params.config['configurations']
['spark2-thrift-sparkconf'],
owner=params.hive_user,
group=params.user_group,
key_value_delimiter=" ",
mode=0644)
effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
if effective_version:
effective_version = format_stack_version(effective_version)
if params.spark_thrift_fairscheduler_content:
# create spark-thrift-fairscheduler.xml
File(os.path.join(params.spark_conf, "spark-thrift-fairscheduler.xml"),
owner=params.spark_user,
group=params.spark_group,
mode=0755,
content=InlineTemplate(params.spark_thrift_fairscheduler_content))
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if not command_repository.items:
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
new_repo_files = Script.repository_util.create_repo_files()
self.repo_files.update(new_repo_files)
except Exception as err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'package_installation_result': 'FAIL',
'repository_version_id': command_repository.version_id
}
self.put_structured_out(self.structured_output)
try:
# check package manager non-completed transactions
if self.repo_mgr.check_uncompleted_transactions():
self.repo_mgr.print_uncompleted_transaction_hint()
num_errors += 1
except Exception as e: # we need to ignore any exception
Logger.warning(
"Failed to check for uncompleted package manager transactions: "
+ str(e))
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# Initial list of versions, used to compute the new version installed
self.old_versions = get_stack_versions(self.stack_root_folder)
try:
is_package_install_successful = False
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output[
'package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
is_package_install_successful = True
else:
num_errors += 1
except Exception as err:
num_errors += 1
Logger.logger.exception(
"Could not install packages. Error: {0}".format(str(err)))
# Provide correct exit code
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
self._fix_default_links_for_current()
# if installing a version of HDP that needs some symlink love, then create them
if is_package_install_successful and 'actual_version' in self.structured_output:
self._relink_configurations_with_conf_select(
stack_id, self.structured_output['actual_version'])
def get_packages(scope, service_name=None, component_name=None):
"""
Gets the packages which should be used with the stack's stack-select tool for the
specified service/component. Not all services/components are used with the stack-select tools,
so those will return no packages.
:param scope: the scope of the command
:param service_name: the service name, such as ZOOKEEPER
:param component_name: the component name, such as ZOOKEEPER_SERVER
:return: the packages to use with stack-select or None
"""
from resource_management.libraries.functions.default import default
if scope not in _PACKAGE_SCOPES:
raise Fail("The specified scope of {0} is not valid".format(scope))
config = Script.get_config()
if service_name is None or component_name is None:
if 'role' not in config or 'serviceName' not in config:
raise Fail(
"Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool"
)
service_name = config['serviceName']
component_name = config['role']
stack_name = default("/hostLevelParams/stack_name", None)
if stack_name is None:
raise Fail(
"The stack name is not present in the command. Packages for stack-select tool cannot be loaded."
)
stack_packages_config = default(
"/configurations/cluster-env/stack_packages", None)
if stack_packages_config is None:
raise Fail(
"The stack packages are not defined on the command. Unable to load packages for the stack-select tool"
)
data = json.loads(stack_packages_config)
if stack_name not in data:
raise Fail(
"Cannot find stack-select packages for the {0} stack".format(
stack_name))
stack_select_key = "stack-select"
data = data[stack_name]
if stack_select_key not in data:
raise Fail(
"There are no stack-select packages defined for this command for the {0} stack"
.format(stack_name))
# this should now be the dictionary of role name to package name
data = data[stack_select_key]
service_name = service_name.upper()
component_name = component_name.upper()
if service_name not in data:
Logger.info(
"Skipping stack-select on {0} because it does not exist in the stack-select package structure."
.format(service_name))
return None
data = data[service_name]
if component_name not in data:
Logger.info(
"Skipping stack-select on {0} because it does not exist in the stack-select package structure."
.format(component_name))
return None
return data[component_name][scope]