def check_indexer_parameters():
"""
Ensure that all required parameters have been defined for the chosen
Indexer; either Solr or Elasticsearch.
"""
missing = []
config = Script.get_config()
indexer = config['configurations']['metron-indexing-env']['ra_indexing_writer']
Logger.info('Checking parameters for indexer = ' + indexer)
if indexer == 'Solr':
# check for all required solr parameters
if not config['configurations']['metron-env']['solr_zookeeper_url']:
missing.append("metron-env/solr_zookeeper_url")
else:
# check for all required elasticsearch parameters
if not config['configurations']['metron-env']['es_cluster_name']:
missing.append("metron-env/es_cluster_name")
if not config['configurations']['metron-env']['es_hosts']:
missing.append("metron-env/es_hosts")
if not config['configurations']['metron-env']['es_date_format']:
missing.append("metron-env/es_date_format")
if len(missing) > 0:
raise Fail("Missing required indexing parameters(s): indexer={0}, missing={1}".format(indexer, missing))
def actionexecute(self, env):
config = Script.get_config()
version = default('/commandParams/version', None)
stack_name = default('/hostLevelParams/stack_name', "")
if not version:
raise Fail("Value is required for '/commandParams/version'")
# other os?
if OSCheck.is_redhat_family():
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
min_ver = format_hdp_stack_version("2.2")
real_ver = format_hdp_stack_version(version)
if stack_name == "HDP":
if compare_versions(real_ver, min_ver) >= 0:
cmd = ('hdp-select', 'set', 'all', version)
code, out = shell.call(cmd, sudo=True)
if compare_versions(real_ver, format_hdp_stack_version("2.3")) >= 0:
# backup the old and symlink /etc/[component]/conf to /usr/hdp/current/[component]
for k, v in conf_select.PACKAGE_DIRS.iteritems():
for dir_def in v:
link_config(dir_def['conf_dir'], dir_def['current_dir'])
def should_install_falcon_atlas_hook():
config = Script.get_config()
stack_version_unformatted = config['hostLevelParams']['stack_version']
stack_version_formatted = format_stack_version(stack_version_unformatted)
if check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT_2_3, stack_version_formatted) \
or check_stack_feature(StackFeature.FALCON_ATLAS_SUPPORT, stack_version_formatted):
return _has_applicable_local_component(config, ['FALCON_SERVER'])
return False
def should_install_mysql():
config = Script.get_config()
hive_database = config['configurations']['hive-env']['hive_database']
hive_use_existing_db = hive_database.startswith('Existing')
if hive_use_existing_db:
return False
return _has_applicable_local_component(config, "MYSQL_SERVER")
def actionexecute(self, env):
resolve_ambari_config()
# Parse parameters from command json file.
config = Script.get_config()
host_name = socket.gethostname()
version = default('/roleParams/version', None)
# These 2 variables are optional
service_package_folder = default('/roleParams/service_package_folder', None)
hooks_folder = default('/roleParams/hooks_folder', None)
tasks = json.loads(config['roleParams']['tasks'])
if tasks:
for t in tasks:
task = ExecuteTask(t)
Logger.info(str(task))
# If a (script, function) exists, it overwrites the command.
if task.script and task.function and service_package_folder and hooks_folder:
file_cache = FileCache(agent_config)
command_paths = {"commandParams":
{"service_package_folder": service_package_folder,
"hooks_folder": hooks_folder
}
}
server_url_prefix = default('/hostLevelParams/jdk_location', "")
base_dir = file_cache.get_service_base_dir(command_paths, server_url_prefix)
script_path = os.path.join(base_dir, task.script)
if not os.path.exists(script_path):
message = "Script %s does not exist" % str(script_path)
raise Fail(message)
# Notice that the script_path is now the fully qualified path, and the
# same command-#.json file is used.
# Also, the python wrapper is used, since it sets up the correct environment variables
command_params = ["/usr/bin/ambari-python-wrap",
script_path,
task.function,
self.command_data_file,
self.basedir,
self.stroutfile,
self.logging_level,
Script.get_tmp_dir()]
task.command = " ".join(command_params)
# Replace redundant whitespace to make the unit tests easier to validate
task.command = re.sub("\s+", " ", task.command).strip()
if task.command:
task.command = replace_variables(task.command, host_name, version)
code, out = shell.call(task.command)
Logger.info("Command: %s\nCode: %s, Out: %s" % (task.command, str(code), str(out)))
if code != 0:
raise Fail(out)
def default(name, default_value):
subdicts = filter(None, name.split('/'))
curr_dict = Script.get_config()
if not curr_dict:
return default_value
for x in subdicts:
if x in curr_dict:
curr_dict = curr_dict[x]
else:
return default_value
return curr_dict
def default(name, default_value):
subdicts = filter(None, name.split('/'))
curr_dict = Script.get_config()
for x in subdicts:
if x in curr_dict:
curr_dict = curr_dict[x]
else:
if not isinstance(default_value, UnknownConfiguration):
Logger.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value))
return default_value
return curr_dict
def remove_hdp_21(self, env):
"""
During Express Upgrade from HDP 2.1 to any higher version (HDP 2.2 or 2.3), the HDP 2.1 bits must be uninstalled.
This is because /usr/bin/hadoop used to be a shell script in HDP 2.1, but in HDP 2.3 it is
a symlink to /usr/hdp/current/hadoop-client/bin/hadoop
so both versions cannot coexist.
"""
Logger.info("Attempting to remove bits for HDP 2.1")
config = Script.get_config()
packages_to_remove = ["zookeeper", "hadoop", "hadoop-lzo", "hadoop-hdfs", "hadoop-libhdfs", "hadoop-yarn", "hadoop-client", "hadoop-mapreduce", "hive", "hive-hcatalog", "hive-jdbc", "hive-webhcat", "hcatalog", "webhcat-tar-hive", "webhcat-tar-pig", "oozie", "oozie-client", "pig", "sqoop", "tez" "falcon", "storm", "flume", "hbase", "phoenix"]
packages_to_remove.reverse()
Logger.info("Packages to remove: {0}".format(" ".join(packages_to_remove)))
for name in packages_to_remove:
Logger.info("Attempting to remove {0}".format(name))
Package(name, action="remove")
def get_not_managed_resources():
"""
Returns a list of not managed hdfs paths.
The result contains all paths from hostLevelParams/not_managed_hdfs_path_list
except config values from cluster-env/managed_hdfs_resource_property_names
"""
config = Script.get_config()
not_managed_hdfs_path_list = json.loads(config['hostLevelParams']['not_managed_hdfs_path_list'])[:]
managed_hdfs_resource_property_names = config['configurations']['cluster-env']['managed_hdfs_resource_property_names']
managed_hdfs_resource_property_list = filter(None, [property.strip() for property in managed_hdfs_resource_property_names.split(',')])
for property_name in managed_hdfs_resource_property_list:
property_value = default('/configurations/' + property_name, None)
if property_value == None:
Logger.warning(("Property {0} from cluster-env/managed_hdfs_resource_property_names not found in configurations. "
"Management of this DFS resource will not be forced.").format(property_name))
else:
while property_value in not_managed_hdfs_path_list:
not_managed_hdfs_path_list.remove(property_value)
return not_managed_hdfs_path_list
def expect(name, expected_type, default_value=None):
"""
Expect configuration to be of certain type. If it is not, give a reasonable error message to user.
Optionally if the configuration is not found default_value for it can be returned.
"""
subdicts = filter(None, name.split('/'))
curr_dict = Script.get_config()
for x in subdicts:
if x in curr_dict:
curr_dict = curr_dict[x]
else:
if default_value:
return default_value
return UnknownConfiguration(curr_dict[-1])
value = curr_dict
if expected_type == bool:
if isinstance(value, bool):
return value
elif isinstance(value, basestring):
if value != None and value.lower() == "true":
value = True
elif value != None and value.lower() == "false":
value = False
else:
raise Fail("Configuration {0} expected to be boolean (true or false), but found '{1}'".format(name, value))
else:
type_name = type(value).__name__
raise Fail("Configuration {0} expected to be boolean (true or false), but found instance of unknown type '{1}'".format(name, type_name))
elif expected_type in [int, long, float]:
try:
value = expected_type(value)
except (ValueError, TypeError):
raise Fail("Configuration {0} expected to be number, but found '{1}'".format(name, value))
return value
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.stack_features import get_stack_feature_version
from resource_management.libraries.functions import StackFeature
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
from resource_management.libraries.functions import is_empty
from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames
from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_root = status_params.stack_root
sudo = AMBARI_SUDO_BINARY
# Needed since this is an Atlas Hook service.
cluster_name = config['clusterName']
stack_name = status_params.stack_name
upgrade_direction = default("/commandParams/upgrade_direction", None)
version = default("/commandParams/version", None)
agent_stack_retry_on_unavailability = config['hostLevelParams'][
'agent_stack_retry_on_unavailability']
agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count",
int)
def should_install_kerberos_server():
config = Script.get_config()
return 'role' in config and config['role'] != "KERBEROS_CLIENT"
def should_install_logsearch_portal():
config = Script.get_config()
return 'role' in config and config['role'] == "LOGSEARCH_SERVER"
def should_install_logsearch_solr_client():
config = Script.get_config()
return 'role' in config and (config['role'] == "LOGSEARCH_SOLR_CLIENT"
or config['role'] == 'ATLAS_SERVER'
or config['role'] == 'RANGER_ADMIN')
def should_install_infra_solr_client(): config = Script.get_config() return _has_applicable_local_component(config, ['INFRA_SOLR_CLIENT', 'ATLAS_SERVER', 'RANGER_ADMIN'])
def should_install_ams_grafana(): config = Script.get_config() return _has_applicable_local_component(config, ["METRICS_GRAFANA"])
def should_install_ams_collector():
config = Script.get_config()
return 'role' in config and config['role'] == "METRICS_COLLECTOR"
def should_install_ams_grafana():
config = Script.get_config()
return _has_applicable_local_component(config, ["METRICS_GRAFANA"])
def should_install_ams_collector():
config = Script.get_config()
return _has_applicable_local_component(config, ["METRICS_COLLECTOR"])
def should_install_rpcbind():
config = Script.get_config()
return _has_applicable_local_component(config, ["NFS_GATEWAY"])
def should_install_ranger_tagsync():
config = Script.get_config()
ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", [])
has_ranger_tagsync = len(ranger_tagsync_hosts) > 0
return has_ranger_tagsync
def should_install_rpcbind(): config = Script.get_config() return _has_applicable_local_component(config, ["NFS_GATEWAY"])
def should_install_logsearch_portal(): config = Script.get_config() return _has_applicable_local_component(config, ["LOGSEARCH_SERVER"])
def should_install_ranger_tagsync():
config = Script.get_config()
ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", [])
has_ranger_tagsync = len(ranger_tagsync_hosts) > 0
return has_ranger_tagsync
def should_install_rpcbind():
config = Script.get_config()
return 'role' in config and config['role'] == "NFS_GATEWAY"
def should_install_ams_collector(): config = Script.get_config() return _has_applicable_local_component(config, ["METRICS_COLLECTOR"])
def should_install_ams_grafana():
config = Script.get_config()
return 'role' in config and config['role'] == "METRICS_GRAFANA"
def should_install_logsearch_solr():
config = Script.get_config()
return 'role' in config and config['role'] == "LOGSEARCH_SOLR"
def actionexecute(self, env):
resolve_ambari_config()
# Parse parameters from command json file.
config = Script.get_config()
host_name = socket.gethostname()
version = default('/roleParams/version', None)
# These 2 variables are optional
service_package_folder = default('/roleParams/service_package_folder',
None)
hooks_folder = default('/roleParams/hooks_folder', None)
tasks = json.loads(config['roleParams']['tasks'])
if tasks:
for t in tasks:
task = ExecuteTask(t)
Logger.info(str(task))
# If a (script, function) exists, it overwrites the command.
if task.script and task.function:
file_cache = FileCache(agent_config)
server_url_prefix = default(
'/hostLevelParams/jdk_location', "")
if service_package_folder and hooks_folder:
command_paths = {
"commandParams": {
"service_package_folder":
service_package_folder,
"hooks_folder": hooks_folder
}
}
base_dir = file_cache.get_service_base_dir(
command_paths, server_url_prefix)
else:
base_dir = file_cache.get_custom_actions_base_dir(
server_url_prefix)
script_path = os.path.join(base_dir, task.script)
if not os.path.exists(script_path):
message = "Script %s does not exist" % str(script_path)
raise Fail(message)
# Notice that the script_path is now the fully qualified path, and the
# same command-#.json file is used.
# Also, the python wrapper is used, since it sets up the correct environment variables
command_params = [
"/usr/bin/ambari-python-wrap", script_path,
task.function, self.command_data_file, self.basedir,
self.stroutfile, self.logging_level,
Script.get_tmp_dir()
]
task.command = "source /var/lib/ambari-agent/ambari-env.sh ; " + " ".join(
command_params)
# Replace redundant whitespace to make the unit tests easier to validate
task.command = re.sub("\s+", " ", task.command).strip()
if task.command:
task.command = replace_variables(task.command, host_name,
version)
shell.checked_call(task.command,
logoutput=True,
quiet=True)
def should_install_infra_solr(): config = Script.get_config() return _has_applicable_local_component(config, ["INFRA_SOLR"])
def should_install_infra_solr():
config = Script.get_config()
return _has_applicable_local_component(config, ["INFRA_SOLR"])
def should_install_lzo():
config = Script.get_config()
io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
return lzo_enabled
def should_install_infra_solr_client():
config = Script.get_config()
return _has_applicable_local_component(config, [
'INFRA_SOLR_CLIENT', 'ATLAS_SERVER', 'RANGER_ADMIN', 'LOGSEARCH_SERVER'
])
def should_install_logsearch_portal():
config = Script.get_config()
return _has_applicable_local_component(config, ["LOGSEARCH_SERVER"])
import re
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
from resource_management.libraries.script import Script
from resource_management.libraries.functions import default
from resource_management.libraries.functions import format
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions import format_jvm_option
from resource_management.libraries.functions.version import format_hdp_stack_version
from resource_management.libraries.functions.version import compare_versions
from ambari_commons.os_check import OSCheck
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
jce_location = config['hostLevelParams']['jdk_location']
jdk_name = default("/hostLevelParams/jdk_name", None)
java_home = config['hostLevelParams']['java_home']
java_version = int(config['hostLevelParams']['java_version'])
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
security_enabled = config['configurations']['cluster-env']['security_enabled']
def start(self, env):
import params
self.configure(env)
print "start mongodb"
import socket
current_host_name = socket.getfqdn(socket.gethostname())
config = Script.get_config()
shard_prefix = params.shard_prefix
db_hosts = config['clusterHostInfo']['mongodb_hosts']
auth_pattern = ''
print params.auth
if params.auth:
print 'add keyFile'
# add keyfile
keyfile_path = '/etc/security/'
keyfile_name = keyfile_path + 'mongodb-keyfile'
auth_pattern = ' --keyFile ' + keyfile_name
if current_host_name == db_hosts[0]:
Execute(format('openssl rand -base64 741 > {keyfile_name}'),
logoutput=True)
Execute(format('chmod 600 {keyfile_name}'), logoutput=True)
for index, item in enumerate(db_hosts, start=1):
Execute(format(
'scp {keyfile_name} root@{item}:{keyfile_path}'),
logoutput=True)
len_host = len(db_hosts)
len_port = len(params.db_ports)
if len(params.node_group) > 0:
db_hosts = self.getdbhosts(db_hosts, params.node_group)
# start shard service
for index, item in enumerate(db_hosts, start=0):
if item == current_host_name:
# foreach db_ports
for index_p, p in enumerate(params.db_ports, start=0):
# rm mongo_*.sock
Execute(format('rm -rf /tmp/mongodb-{p}.sock'),
logoutput=True)
# get shard_name
shard_name = shard_prefix + str(
(index - index_p) % len_host)
# pid_file_name = params.shard_prefix + str((index-index_p)%len_host)
# pid_file_name not the same to log,easy to status
pid_file_name = params.shard_prefix + str(index_p)
# get db_path
db_path = params.db_path + '/' + shard_name
if os.path.exists(db_path):
print "File exists"
else:
Execute(format('mkdir -p {db_path}'), logoutput=True)
log_file = params.log_path + '/' + shard_name + '.log'
pid_file = params.pid_db_path + '/' + pid_file_name + '.pid'
Execute(format(
'mongod -f /etc/mongod.conf --shardsvr -replSet {shard_name} -port {p} -dbpath {db_path} -oplogSize 100 -logpath {log_file} -pidfilepath {pid_file} {auth_pattern} '
),
logoutput=True)
sleep(5)
print 'sleep waiting for all mongod started'
if params.node_group == '':
members = ''
index = db_hosts.index(current_host_name)
shard_name = shard_prefix + str(index)
current_index = 0
current_shard = index
while (current_index < len_port):
current_host = db_hosts[current_shard]
current_port = params.db_ports[current_index]
members = members + '{_id:' + format(
'{current_index},host:"{current_host}:{current_port}"')
if current_index == 0:
members = members + ',priority:2'
members = members + '},'
current_index = current_index + 1
current_shard = (current_shard + 1) % len(db_hosts)
replica_param = 'rs.initiate( {_id:' + format(
'"{shard_name}",version: 1,members:') + '[' + members + ']})'
cmd = format(
'mongo --host {current_host_name} --port 27017 <<EOF \n{replica_param} \nEOF\n'
)
File('/var/run/mongo_config.sh', content=cmd, mode=0755)
Execute('su - mongodb /var/run/mongo_config.sh', logoutput=True)
else:
groups = params.node_group.split(';')
members = ''
index = db_hosts.index(current_host_name)
shard_name = shard_prefix + str(index)
current_index = 0
current_shard = index
while (current_index < len_port):
current_host = db_hosts[current_shard]
current_port = params.db_ports[current_index]
members = members + '{_id:' + format(
'{current_index},host:"{current_host}:{current_port}"')
if current_index == 0:
members = members + ',priority:2'
members = members + '},'
current_index = current_index + 1
current_shard = (current_shard + 1) % len(db_hosts)
# if len(groups) > 1 and current_host_name in groups[-1]:
# replica_param ='rs.initiate( {_id:'+format('"{shard_name}",version: 1,members:') + '[' + members + ']})'
# else:
replica_param = 'rs.initiate( {_id:' + format(
'"{shard_name}",version: 1,members:'
) + '[' + members + ']},{force:1})'
cmd = format(
'mongo --host {current_host_name} --port 27017 <<EOF \n{replica_param} \nEOF\n'
)
File('/var/run/mongo_config.sh', content=cmd, mode=0755)
Execute('/var/run/mongo_config.sh', logoutput=True)
