def test_static_file_absolute_path(self, join_mock, is_file_mock):
"""
Testing StaticFile source with absolute path
"""
sudo.read_file = lambda path: 'content'
is_file_mock.return_value = True
with Environment("/base") as env:
static_file = StaticFile("/absolute/path/file")
content = static_file.get_content()
self.assertEqual('content', content)
self.assertEqual(is_file_mock.call_count, 1)
self.assertEqual(join_mock.call_count, 0)
def test_static_file_relative_path(self, join_mock, is_file_mock):
"""
Testing StaticFile source with relative path
"""
sudo.read_file = lambda path: 'content'
is_file_mock.return_value = True
with Environment("/base") as env:
static_file = StaticFile("relative/path/file")
content = static_file.get_content()
self.assertEqual('content', content)
self.assertEqual(is_file_mock.call_count, 1)
self.assertEqual(join_mock.call_count, 1)
join_mock.assert_called_with('/base', 'files', 'relative/path/file')
def test_static_file_absolute_path(self, join_mock, open_mock):
"""
Testing StaticFile source with absolute path
"""
file_mock = MagicMock(name = 'file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'content'
open_mock.return_value = file_mock
with Environment("/base") as env:
static_file = StaticFile("/absolute/path/file")
content = static_file.get_content()
self.assertEqual('content', content)
self.assertEqual(file_mock.read.call_count, 1)
self.assertEqual(join_mock.call_count, 0)
def elasticsearch_template_install(self, env):
from params import params
env.set_params(params)
Logger.info("Installing Elasticsearch index templates")
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name, template_path in commands.get_templates().iteritems(
):
# install the index template
File(template_path,
mode=0755,
content=StaticFile("{0}.template".format(template_name)))
cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}"
Execute(cmd.format(params.es_http_url, template_name,
template_path),
logoutput=True)
def backup_dir_contents(dir_path, backup_folder_suffix):
import params
backup_destination_path = params.tmp_dir + os.path.normpath(dir_path)+backup_folder_suffix
Directory(backup_destination_path,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
recursive=True
)
set_dir_ownership(backup_destination_path)
# Safely copy top-level contents to backup folder
for file in os.listdir(dir_path):
File(os.path.join(backup_destination_path, file),
owner=params.kafka_user,
content = StaticFile(os.path.join(dir_path,file)))
return backup_destination_path
def action_create(self):
with Environment.get_instance_copy() as env:
with tempfile.NamedTemporaryFile() as tmpf:
repo_file_name = format(
"{repo_file_name}.list",
repo_file_name=self.resource.repo_file_name)
repo_file_path = format("{repo_dir}/{repo_file_name}",
repo_dir=self.repo_dir)
repo_template = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..',
REPO_TEMPLATE_FOLDER, self.resource.repo_template)
new_content = Template(
repo_template,
package_type=self.package_type,
base_url=self.resource.base_url,
components=' '.join(
self.resource.components)).get_content()
old_content = ''
if self.resource.append_to_file and os.path.isfile(
repo_file_path):
with open(repo_file_path) as repo_file:
old_content = repo_file.read() + '\n'
File(tmpf.name, content=old_content + new_content)
if not os.path.isfile(repo_file_path) or not filecmp.cmp(
tmpf.name, repo_file_path):
File(repo_file_path, content=StaticFile(tmpf.name))
update_cmd_formatted = [format(x) for x in self.update_cmd]
# this is time expensive
retcode, out = checked_call(update_cmd_formatted,
sudo=True)
# add public keys for new repos
missing_pkeys = set(
re.findall(self.missing_pkey_regex, out))
for pkey in missing_pkeys:
Execute(
format(self.add_pkey_cmd),
timeout=
15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
ignore_failures=True)
def setup_symlink(kafka_managed_dir, kafka_ambari_managed_dir):
import params
backup_folder_path = None
backup_folder_suffix = "_tmp"
if kafka_ambari_managed_dir != kafka_managed_dir:
if os.path.exists(kafka_managed_dir) and not os.path.islink(kafka_managed_dir):
# Backup existing data before delete if config is changed repeatedly to/from default location at any point in time time, as there may be relevant contents (historic logs)
backup_folder_path = backup_dir_contents(kafka_managed_dir, backup_folder_suffix)
Directory(kafka_managed_dir,
action="delete",
create_parents=True)
elif os.path.islink(kafka_managed_dir) and os.path.realpath(kafka_managed_dir) != kafka_ambari_managed_dir:
Link(kafka_managed_dir,
action="delete")
if not os.path.islink(kafka_managed_dir):
Link(kafka_managed_dir,
to=kafka_ambari_managed_dir)
elif os.path.islink(kafka_managed_dir): # If config is changed and coincides with the kafka managed dir, remove the symlink and physically create the folder
Link(kafka_managed_dir,
action="delete")
Directory(kafka_managed_dir,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents=True)
if backup_folder_path:
# Restore backed up files to current relevant dirs if needed - will be triggered only when changing to/from default path;
for file in os.listdir(backup_folder_path):
File(os.path.join(kafka_managed_dir,file),
owner=params.kafka_user,
content = StaticFile(os.path.join(backup_folder_path,file)))
# Clean up backed up folder
Directory(backup_folder_path,
action="delete",
create_parents=True)
def create_topology_script():
import params
path = params.net_topology_script_file_path
parent_dir = os.path.dirname(path)
# only create the parent directory and set its permission if it does not exist
if not os.path.exists(parent_dir):
Directory(parent_dir,
create_parents=True,
owner=params.hdfs_user,
group=params.user_group)
# installing the topology script to the specified location
File(
path,
content=StaticFile('topology_script.py'),
mode=0755,
only_if=format("test -d {net_topology_script_dir}"),
)
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hivemetastore-site'],
configuration_attributes=params.config['configurationAttributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600
)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh')
)
if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip() != "":
params.HdfsResource(params.hive_repl_cmrootdir,
type = "directory",
action = "create_on_execute",
owner = params.hive_user,
group=params.user_group,
mode = 01777)
if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip() != "":
params.HdfsResource(params.hive_repl_rootdir,
type = "directory",
action = "create_on_execute",
owner = params.hive_user,
group=params.user_group,
mode = 0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config('hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.
config['configuration_attributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.hive_repl_cmrootdir:
params.HdfsResource(params.hive_repl_cmrootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01777)
if params.hive_repl_rootdir:
params.HdfsResource(params.hive_repl_rootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
if params.hive_repl_cmrootdir or params.hive_repl_rootdir:
params.HdfsResource(None, action="execute")
def setup_solr_client(config,
custom_log4j=True,
custom_log_location=None,
log4jcontent=None):
solr_client_dir = '/usr/lib/ambari-infra-solr-client'
solr_client_log_dir = default_config(
config,
'/configurations/infra-solr-client-log4j/infra_solr_client_log_dir',
'/var/log/ambari-infra-solr-client'
) if custom_log_location is None else custom_log_location
solr_client_log = format("{solr_client_log_dir}/solr-client.log")
solr_client_log_maxfilesize = default_config(
config,
'configurations/infra-solr-client-log4j/infra_client_log_maxfilesize',
80)
solr_client_log_maxbackupindex = default_config(
config,
'configurations/infra-solr-client-log4j/infra_client_log_maxbackupindex',
60)
Directory(solr_client_log_dir,
mode=0755,
cd_access='a',
create_parents=True)
Directory(solr_client_dir,
mode=0755,
cd_access='a',
create_parents=True,
recursive_ownership=True)
solrCliFilename = format("{solr_client_dir}/solrCloudCli.sh")
File(solrCliFilename, mode=0755, content=StaticFile(solrCliFilename))
if custom_log4j:
# use custom log4j content only, when infra is not installed on the cluster
solr_client_log4j_content = config['configurations'][
'infra-solr-client-log4j'][
'content'] if log4jcontent is None else log4jcontent
File(format("{solr_client_dir}/log4j.properties"),
content=InlineTemplate(solr_client_log4j_content),
mode=0644)
else:
File(format("{solr_client_dir}/log4j.properties"), mode=0644)
File(solr_client_log, mode=0664, content='')
def install_azkaban():
import params
Directory([params.conf_dir, params.log_dir],
owner=params.azkaban_user,
group=params.user_group,
mode=0755,
create_parents=True)
File(params.install_dir_executor +
'/lib/azkaban-ldap-usermanager-1.2.1-SNAPSHOT.jar',
content=StaticFile("azkaban-ldap-usermanager-1.2.1-SNAPSHOT.jar"),
mode=0755)
if not os.path.exists(Script.get_stack_root() + '/' +
params.version_dir_executor) or not os.path.exists(
params.install_dir_executor):
Execute('rm -rf %s' % Script.get_stack_root() + '/' +
params.version_dir_executor)
Execute('rm -rf %s' % params.install_dir_executor)
Execute('wget ' + params.download_url_executor + ' -O /tmp/' +
params.filename_executor,
user=params.azkaban_user)
Execute('tar -zxf /tmp/' + params.filename_executor + ' -C ' +
Script.get_stack_root())
Execute('ln -s ' + Script.get_stack_root() + '/' +
params.version_dir_executor + ' ' +
params.install_dir_executor)
Execute(' cp -r ' + params.install_dir_executor + '/conf/* ' +
params.conf_dir)
Execute(' rm -rf ' + params.install_dir_executor + '/conf')
Execute('ln -s ' + params.conf_dir + ' ' +
params.install_dir_executor + '/conf')
Execute('ln -s ' + params.log_dir + ' ' + params.install_dir_executor +
'/logs/azkaban')
Execute("echo 'export PATH=%s/bin:$PATH'>/etc/profile.d/azkaban.sh" %
params.install_dir_executor)
Execute('chown -R %s:%s %s/%s' %
(params.azkaban_user, params.user_group,
Script.get_stack_root(), params.version_dir_executor))
Execute('chown -R %s:%s %s' % (params.azkaban_user, params.user_group,
params.install_dir_executor))
Execute('/bin/rm -f /tmp/' + params.filename_executor)
def set_uid(user, user_dirs):
"""
user_dirs - comma separated directories
"""
import params
File(
format("{tmp_dir}/changeUid.sh"),
content=StaticFile("changeToSecureUid.sh"),
mode=0555)
ignore_groupsusers_create_str = str(
params.ignore_groupsusers_create).lower()
uid = get_uid(user, return_existing=True)
Execute(
format(
"{tmp_dir}/changeUid.sh {user} {user_dirs} {new_uid}",
new_uid=0 if uid is None else uid),
not_if=format(
"(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})"
))
def service_check(self, env):
import params
env.set_params(params)
unique = get_unique_id_and_date()
File("/tmp/wordCount.jar", content=StaticFile("wordCount.jar"))
cmd = format(
"storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}"
)
Execute(cmd,
logoutput=True,
path=params.storm_bin_dir,
user=params.storm_user)
Execute(format("storm kill WordCount{unique}"),
path=params.storm_bin_dir,
user=params.storm_user)
def service_check(self, env):
import params
env.set_params(params)
temp_dir = os.path.join(os.path.dirname(params.knox_home), "temp")
validateKnoxFileName = "validateKnoxStatus.py"
validateKnoxFilePath = os.path.join(temp_dir, validateKnoxFileName)
python_executable = sys.executable
validateStatusCmd = "%s %s -p %s -n %s" % (
python_executable, validateKnoxFilePath, params.knox_host_port,
params.knox_host_name)
print "Test connectivity to knox server"
File(validateKnoxFilePath, content=StaticFile(validateKnoxFileName))
Execute(validateStatusCmd,
tries=3,
try_sleep=5,
timeout=5,
logoutput=True)
def update_credential_provider_path(config,
config_type,
dest_provider_path,
file_owner,
file_group,
use_local_jceks=False):
"""
Copies the JCEKS file for the specified config from the default location to the given location,
and sets the ACLs for the specified owner and group. Also updates the config type's configuration
hadoop credential store provider with the copied file name.
:param config: configurations['configurations'][config_type]
:param config_type: Like hive-site, oozie-site, etc.
:param dest_provider_path: The full path to the file where the JCEKS provider file is to be copied to.
:param file_owner: File owner
:param file_group: Group
:return: A copy of the config that was modified or the input config itself if nothing was modified.
"""
# Get the path to the provider <config_type>.jceks
if HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME in config:
provider_path = config[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME]
src_provider_path = provider_path[len('jceks://file'):]
File(dest_provider_path,
owner=file_owner,
group=file_group,
mode=0640,
content=StaticFile(src_provider_path))
# make a copy of the config dictionary since it is read-only
config_copy = config.copy()
# overwrite the provider path with the path specified
if use_local_jceks:
config_copy[
HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'localjceks://file{0}'.format(
dest_provider_path)
else:
config_copy[
HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file{0}'.format(
dest_provider_path)
return config_copy
return config
def oozie_smoke_shell_file(file_name):
import params
File(format("{tmp_dir}/{file_name}"),
content=StaticFile(file_name),
mode=0755)
if params.security_enabled:
sh_cmd = format(
"{tmp_dir}/{file_name} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}"
)
else:
sh_cmd = format(
"{tmp_dir}/{file_name} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled}"
)
Execute(format("{tmp_dir}/{file_name}"),
command=sh_cmd,
path=params.execute_path,
tries=3,
try_sleep=5,
logoutput=True)
def mysql_deluser():
import params
File(params.mysql_deluser_path,
mode=0755,
content=StaticFile('removeMysqlUser.sh')
)
hive_server_host = format("{hive_server_host}")
hive_metastore_host = format("{hive_metastore_host}")
del_hiveserver_cmd = "bash -x {mysql_deluser_path} {daemon_name} {hive_metastore_user_name} {hive_server_host}"
del_metastore_cmd = "bash -x {mysql_deluser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_host}"
if (hive_server_host == hive_metastore_host):
cmd = format(del_hiveserver_cmd)
else:
cmd = format(
del_hiveserver_cmd + ";" + del_metastore_cmd)
Execute(cmd,
tries=3,
try_sleep=5,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
)
def elasticsearch_template_install(self, env):
from params import params
env.set_params(params)
Logger.info("Installing Elasticsearch index templates")
try:
metron_service.check_indexer_parameters()
commands = IndexingCommands(params)
for template_name, template_path in commands.get_templates().iteritems():
# install the index template
File(template_path, mode=0755, content=StaticFile("{0}.template".format(template_name)))
cmd = "curl -s -XPOST http://{0}/_template/{1} -d @{2}"
Execute(
cmd.format(params.es_http_url, template_name, template_path),
logoutput=True)
return True
except Exception as e:
msg = "WARNING: Elasticsearch index templates could not be installed. " \
"Is Elasticsearch running? Will reattempt install on next start. error={0}"
Logger.warning(msg.format(e))
return False
def mysql_adduser():
import params
File(params.mysql_adduser_path,
mode=0755,
content=StaticFile('addMysqlUser.sh')
)
hive_server_host = format("{hive_server_host}")
hive_metastore_host = format("{hive_metastore_host}")
add_metastore_cmd = "bash -x {mysql_adduser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_metastore_host}"
add_hiveserver_cmd = "bash -x {mysql_adduser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_server_host}"
if (hive_server_host == hive_metastore_host):
cmd = format(add_hiveserver_cmd)
else:
cmd = format(add_hiveserver_cmd + ";" + add_metastore_cmd)
Execute(cmd,
tries=3,
try_sleep=5,
logoutput=False,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
)
def install(self, env):
self.install_packages(env)
self.configure(env)
import params
File(
'/etc/openldap/schema/cisco.schema',
mode=0755,
content=StaticFile('cisco.schema'))
File(
'/etc/openldap/schema/tacacs.schema',
mode=0755,
content=StaticFile('tacacs.schema'))
File(
'/etc/openldap/schema/radius.schema',
mode=0755,
content=StaticFile('radius.schema'))
File(
'/etc/openldap/schema/dnsdomain2.schema',
mode=0755,
content=StaticFile('dnsdomain2.schema'))
File(
'/etc/openldap/schema/public_key.schema',
mode=0755,
content=StaticFile('public_key.schema'))
File(
'/etc/httpd/conf.d/self-service-password.conf',
mode=0755,
content=StaticFile('self-service-password.conf'))
setup_file = '/usr/sbin/ldap_setup.sh'
File(setup_file, mode=0755, content=StaticFile('setup.sh'))
Execute('echo Running /usr/sbin/ldap_setup.sh')
# run setup script which has simple shell setup
Execute(setup_file + ' ' + params.ldap_password + ' ' +
params.ldap_adminuser + ' ' + params.ldap_domain + ' ' +
params.ldap_ldifdir + ' ' + params.ldap_ou + ' "' +
params.binddn + '" >> ' + params.stack_log)
def install_graphite_web():
import params
Directory([params.graphite_conf_dir, params.log_dir, params.pid_dir],
owner=params.graphite_user,
group=params.user_group,
mode=0775,
cd_access="a",
create_parents=True)
if not os.path.exists(params.install_dir_graphite_web):
Execute('wget ' + params.download_url_graphite_web + ' -O /tmp/' +
params.filename_graphite_web,
user=params.graphite_user)
Execute('tar -xf /tmp/' + params.filename_graphite_web + ' -C ' +
Script.get_stack_root())
Execute(' rm -rf ' + params.install_dir_graphite_web + '/conf')
Execute('ln -s ' + params.graphite_conf_dir + ' ' +
params.install_dir_graphite_web + '/conf')
Execute('chown -R %s:%s %s' % (params.graphite_user, params.user_group,
params.install_dir_graphite_web))
Execute('/bin/rm -f /tmp/' + params.filename_graphite_web)
File(params.install_dir_graphite_web + '/webapp/graphite/graphouse.py',
content=StaticFile("graphouse.py"),
mode=0755)
File('/usr/lib/systemd/system/graphite-web.service',
content=params.graphite_web_systemd_content,
mode=0755)
File('/etc/sysconfig/memcached',
content=params.memcached_content,
mode=0755)
Execute('systemctl daemon-reload')
Execute('systemctl enable graphite-web')
Execute('systemctl enable memcached')
Execute('systemctl start memcached')
def service_check(self, env):
import params
env.set_params(params)
yarn_exe = os_utils.quote_path(
os.path.join(params.yarn_home, "bin", "yarn.cmd"))
run_yarn_check_cmd = "cmd /C %s node -list" % yarn_exe
component_type = 'rm'
if params.hadoop_ssl_enabled:
component_address = params.rm_webui_https_address
else:
component_address = params.rm_webui_address
#temp_dir = os.path.abspath(os.path.join(params.hadoop_home, os.pardir)), "/tmp"
temp_dir = os.path.join(os.path.dirname(params.hadoop_home), "temp")
validateStatusFileName = "validateYarnComponentStatusWindows.py"
validateStatusFilePath = os.path.join(temp_dir, validateStatusFileName)
python_executable = sys.executable
validateStatusCmd = "%s %s %s -p %s -s %s" % (
python_executable, validateStatusFilePath, component_type,
component_address, params.hadoop_ssl_enabled)
if params.security_enabled:
kinit_cmd = "%s -kt %s %s;" % (params.kinit_path_local,
params.smoke_user_keytab,
params.smokeuser)
smoke_cmd = kinit_cmd + ' ' + validateStatusCmd
else:
smoke_cmd = validateStatusCmd
File(validateStatusFilePath,
content=StaticFile(validateStatusFileName))
Execute(smoke_cmd, tries=3, try_sleep=5, logoutput=True)
Execute(run_yarn_check_cmd, logoutput=True)
def get_uid(user, return_existing=False):
"""
Tries to get UID for username. It will try to find UID in custom properties in *cluster_env* and, if *return_existing=True*,
it will try to return UID of existing *user*.
:param user: username to get UID for
:param return_existing: return UID for existing user
:return:
"""
import params
user_str = str(user) + "_uid"
service_env = [
serviceEnv for serviceEnv in params.config['configurations']
if user_str in params.config['configurations'][serviceEnv]
]
if service_env and params.config['configurations'][
service_env[0]][user_str]:
service_env_str = str(service_env[0])
uid = params.config['configurations'][service_env_str][user_str]
if len(service_env) > 1:
Logger.warning(
"Multiple values found for %s, using %s" % (user_str, uid))
return uid
else:
if return_existing:
# pick up existing UID or try to find available UID in /etc/passwd, see changeToSecureUid.sh for more info
if user == params.smoke_user:
return None
File(
format("{tmp_dir}/changeUid.sh"),
content=StaticFile("changeToSecureUid.sh"),
mode=0555)
code, newUid = shell.call(format("{tmp_dir}/changeUid.sh {user}"))
return int(newUid)
else:
# do not return UID for existing user, used in User resource call to let OS to choose UID for us
return None
def install_graphite_api():
import params
Directory([
params.graphite_conf_dir, params.log_dir, params.pid_dir,
'/srv/graphite'
],
owner=params.graphite_user,
group=params.user_group,
mode=0775,
cd_access="a",
create_parents=True)
if not os.path.exists(params.install_dir_graphite_api):
Execute('wget ' + params.download_url_graphite_api + ' -O /tmp/' +
params.filename_graphite_api,
user=params.graphite_user)
Execute('tar -xf /tmp/' + params.filename_graphite_api + ' -C ' +
Script.get_stack_root())
Execute('chown -R %s:%s %s' % (params.graphite_user, params.user_group,
params.install_dir_graphite_api))
Execute('/bin/rm -f /tmp/' + params.filename_graphite_api)
File(
params.install_dir_graphite_api +
'/lib/python3.6/site-packages/graphite_api/finders/graphouse_api.py',
content=StaticFile("graphouse_api.py"),
mode=0755)
File('/usr/lib/systemd/system/graphite-api.service',
content=params.graphite_api_systemd_content,
mode=0755)
File('/etc/sysconfig/memcached',
content=params.memcached_content,
mode=0755)
Execute('systemctl daemon-reload')
Execute('systemctl enable graphite-api')
Execute('systemctl enable memcached')
Execute('systemctl start memcached')
def update_credential_provider_path(config_type, dest_provider_path):
import params
# Get the path to the provider <config_type>.jceks
if HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME in params.config['configurations'][config_type]:
provider_paths = params.config['configurations'][config_type][HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME].split(',')
for path_index in range(len(provider_paths)):
provider_path = provider_paths[path_index]
if config_type == os.path.splitext(os.path.basename(provider_path))[0]:
src_provider_path = provider_path[len('jceks://file'):]
Logger.info('src_provider_path={0}, dest_provider_path{1}'.format(src_provider_path, dest_provider_path))
File(dest_provider_path,
owner = params.oozie_user,
group = params.user_group,
mode = 0640,
content = StaticFile(src_provider_path)
)
provider_paths[path_index] = 'jceks://file{0}'.format(dest_provider_path)
# make a copy of the config dictionary since it cannot be modified
config = params.config['configurations'][config_type].copy()
config[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = ','.join(provider_paths)
return config
return params.config['configurations'][config_type]
def service_check(self, env):
import params
env.set_params(params)
File( format("{tmp_dir}/titanSmoke.groovy"),
content = StaticFile("titanSmoke.groovy"),
mode = 0755
)
if params.iop_stack_version != "" and compare_versions(params.iop_stack_version, '4.2') >= 0:
if params.security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
Execute(kinit_cmd,
user=params.smokeuser
)
Execute(format("gremlin {tmp_dir}/titanSmoke.groovy"),
tries = 3,
try_sleep = 5,
path = format('{titan_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
user = params.smokeuser,
logoutput = True
)
def setup_configs():
"""
Creates configs for services HDFS mapred
"""
import params
if params.has_namenode:
if os.path.exists(params.hadoop_conf_dir):
File(
params.task_log4j_properties_location,
content=StaticFile("task-log4j.properties"),
mode=0755)
if os.path.exists(
os.path.join(params.hadoop_conf_dir, 'configuration.xsl')):
File(
os.path.join(params.hadoop_conf_dir, 'configuration.xsl'),
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')):
File(
os.path.join(params.hadoop_conf_dir, 'masters'),
owner=params.hdfs_user,
group=params.user_group)
def backup_dir_contents(dir_path, backup_folder_suffix):
import params
backup_destination_path = params.tmp_dir + os.path.normpath(dir_path)+backup_folder_suffix
Directory(backup_destination_path,
mode=0755,
cd_access='a',
owner=params.kafka_user,
group=params.user_group,
create_parents = True,
recursive_ownership = True,
)
# Safely copy top-level contents to backup folder
for file in os.listdir(dir_path):
if os.path.isdir(os.path.join(dir_path, file)):
Execute(('cp', '-r', os.path.join(dir_path, file), backup_destination_path),
sudo=True)
Execute(("chown", "-R", format("{kafka_user}:{user_group}"), os.path.join(backup_destination_path, file)),
sudo=True)
else:
File(os.path.join(backup_destination_path, file),
owner=params.kafka_user,
content = StaticFile(os.path.join(dir_path,file)))
return backup_destination_path
def install(self, env):
File('/tmp/libvirt-bootstrap.sh',
mode=0755,
content=StaticFile('libvirt-bootstrap.sh'))
Execute('/tmp/libvirt-bootstrap.sh')
Execute('virsh net-destroy default')
Execute('virsh net-undefine default')
Execute('systemctl restart libvirtd')
import params
Directory([params.data_dir],
owner='root',
group='root',
mode=0775,
create_parents=True)
Execute('virsh pool-define-as --name dir_pool dir --target ' +
params.data_dir)
Execute('virsh pool-autostart dir_pool')
Execute('virsh pool-build dir_pool')
Execute('virsh pool-start dir_pool')
if params.enable_kvm:
File('/tmp/lvm_pool.xml', mode=0755, content=params.lvm_pool)
Execute('virsh pool-define /tmp/lvm_pool.xml')
Execute('irsh pool-build storage_pool')
Execute('virsh pool-autostart storage_pool')
#https://www.ovirt.org/documentation/how-to/networking/bonding-vlan-bridge/
if params.enable_net:
File('/tmp/network.xml', mode=0755, content=params.lvm_pool)
Execute('virsh iface-define /tmp/network.xml')
Execute('virsh iface-start bond0')
Execute('wget ' + params.base_os_image_url + ' -O ' + params.data_dir +
'/base.qcow2')
def fill_conf_dir(component_conf_dir):
import params
hive_client_conf_path = os.path.realpath(format("{stack_root}/current/{component_directory}/conf"))
component_conf_dir = os.path.realpath(component_conf_dir)
mode_identified_for_file = 0644 if component_conf_dir == hive_client_conf_path else 0600
mode_identified_for_dir = 0755 if component_conf_dir == hive_client_conf_path else 0700
Directory(component_conf_dir,
owner=params.hive_user,
group=params.user_group,
create_parents = True,
mode=mode_identified_for_dir
)
XmlConfig("mapred-site.xml",
conf_dir=component_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified_for_file)
File(format("{component_conf_dir}/hive-default.xml.template"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified_for_file
)
File(format("{component_conf_dir}/hive-env.sh.template"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified_for_file
)
# Create hive-log4j.properties and hive-exec-log4j.properties
# in /etc/hive/conf and not in /etc/hive2/conf
if params.log4j_version == '1':
log4j_exec_filename = 'hive-exec-log4j.properties'
if (params.log4j_exec_props != None):
File(format("{component_conf_dir}/{log4j_exec_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=params.log4j_exec_props
)
elif (os.path.exists("{component_conf_dir}/{log4j_exec_filename}.template")):
File(format("{component_conf_dir}/{log4j_exec_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=StaticFile(format("{component_conf_dir}/{log4j_exec_filename}.template"))
)
log4j_filename = 'hive-log4j.properties'
if (params.log4j_props != None):
File(format("{component_conf_dir}/{log4j_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.log4j_props)
)
elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")):
File(format("{component_conf_dir}/{log4j_filename}"),
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template"))
)
pass # if params.log4j_version == '1'
def storm(name=None):
import params
import os
Directory(
params.log_dir,
owner=params.storm_user,
group=params.user_group,
mode=0777,
create_parents=True,
cd_access="a",
)
Directory(
[params.pid_dir, params.local_dir],
owner=params.storm_user,
group=params.user_group,
create_parents=True,
cd_access="a",
mode=0755,
)
Directory(
params.conf_dir,
group=params.user_group,
create_parents=True,
cd_access="a",
)
File(format("{conf_dir}/config.yaml"),
content=Template("config.yaml.j2"),
owner=params.storm_user,
group=params.user_group)
File(params.conf_dir + "/jmxetric-conf.xml",
content=StaticFile("jmxetric-conf.xml"),
owner=params.storm_user)
File(params.storm_lib_dir + "/gmetric4j-1.0.3.jar",
content=StaticFile("gmetric4j-1.0.3.jar"),
owner=params.storm_user)
File(params.storm_lib_dir + "/jmxetric-1.0.4.jar",
content=StaticFile("jmxetric-1.0.4.jar"),
owner=params.storm_user)
File(params.storm_lib_dir + "/oncrpc-1.0.7.jar",
content=StaticFile("oncrpc-1.0.7.jar"),
owner=params.storm_user)
configurations = params.config['configurations']['storm-site']
File(format("{conf_dir}/storm.yaml"),
content=yaml_config_template(configurations),
owner=params.storm_user,
group=params.user_group)
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template))
# Generate atlas-application.properties.xml file and symlink the hook jars
if params.enable_atlas_hook:
script_path = os.path.realpath(__file__).split(
'/services')[0] + '/hooks/before-INSTALL/scripts/atlas'
sys.path.append(script_path)
from setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, setup_atlas_jar_symlinks
atlas_hook_filepath = os.path.join(params.conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.STORM,
params.storm_atlas_application_properties,
atlas_hook_filepath, params.storm_user,
params.user_group)
storm_extlib_dir = os.path.join(params.storm_component_home_dir,
"extlib")
setup_atlas_jar_symlinks("storm", storm_extlib_dir)
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
group=params.user_group,
content=Template("storm-metrics2.properties.j2"))
# Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
action="delete")
# On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar",
action="delete")
sink_jar = params.sink_jar
Execute(format(
"{sudo} ln -s {sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"
),
not_if=format(
"ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
only_if=format("ls {sink_jar}"))
if params.storm_logs_supported:
Directory(params.log4j_dir,
owner=params.storm_user,
group=params.user_group,
mode=0755,
create_parents=True)
File(format("{log4j_dir}/cluster.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_cluster_log4j_content))
File(format("{log4j_dir}/worker.xml"),
owner=params.storm_user,
content=InlineTemplate(params.storm_worker_log4j_content))
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user)
TemplateConfig(format("{conf_dir}/client_jaas.conf"),
owner=params.storm_user)
minRuid = configurations['_storm.min.ruid'] if configurations.has_key(
'_storm.min.ruid') else ''
min_user_ruid = int(
minRuid) if minRuid.isdigit() else _find_real_user_min_uid()
File(format("{conf_dir}/worker-launcher.cfg"),
content=Template("worker-launcher.cfg.j2",
min_user_ruid=min_user_ruid),
owner='root',
group=params.user_group)
def fill_conf_dir(component_conf_dir):
import params
Directory(component_conf_dir,
owner=params.hive_user,
group=params.user_group,
create_parents = True
)
XmlConfig("mapred-site.xml",
conf_dir=component_conf_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
File(format("{component_conf_dir}/hive-default.xml.template"),
owner=params.hive_user,
group=params.user_group
)
File(format("{component_conf_dir}/hive-env.sh.template"),
owner=params.hive_user,
group=params.user_group
)
# Create hive-log4j.properties and hive-exec-log4j.properties
# in /etc/hive/conf and not in /etc/hive2/conf
if params.log4j_version == '1':
log4j_exec_filename = 'hive-exec-log4j.properties'
if (params.log4j_exec_props != None):
File(format("{component_conf_dir}/{log4j_exec_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.log4j_exec_props
)
elif (os.path.exists("{component_conf_dir}/{log4j_exec_filename}.template")):
File(format("{component_conf_dir}/{log4j_exec_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=StaticFile(format("{component_conf_dir}/{log4j_exec_filename}.template"))
)
log4j_filename = 'hive-log4j.properties'
if (params.log4j_props != None):
File(format("{component_conf_dir}/{log4j_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=params.log4j_props
)
elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")):
File(format("{component_conf_dir}/{log4j_filename}"),
mode=0644,
group=params.user_group,
owner=params.hive_user,
content=StaticFile(format("{component_conf_dir}/{log4j_filename}.template"))
)
pass # if params.log4j_version == '1'
