def new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user,
keytab_file, principal, hostname):
"""
Entry point of an actual execution - triggered when timeout on the cache expired or on fresh execution
"""
now = datetime.now()
temp_kinit_cache_fd, temp_kinit_cache_filename = mkstemp(dir=temp_dir)
command = "%s -c %s -kt %s %s" % \
(kinit_path, temp_kinit_cache_filename, keytab_file,
principal.replace("_HOST", hostname))
os.close(temp_kinit_cache_fd)
try:
# Ensure the proper user owns this file
File(temp_kinit_cache_filename, owner=exec_user, mode=0600)
# Execute the kinit
Execute(command, user=exec_user)
with open(file_path, 'w+') as cache_file:
result = {key: {"last_successful_execution": str(now)}}
json.dump(result, cache_file)
finally:
File(temp_kinit_cache_filename, action='delete')
def metadata():
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(format('{metadata_home}/server/webapp/atlas.war')))
PropertiesFile(format('{conf_dir}/application.properties'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=StaticFile('atlas-log4j.xml'))
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
daemon_cmd = format(
'source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}'
)
# If the pid dir doesn't exist, this means either
# 1. The user just added Atlas service and issued a restart command (stop+start). So stop should be a no-op
# since there's nothing to stop.
# OR
# 2. The user changed the value of the pid dir config and incorrectly issued a restart command.
# In which case the stop command cannot do anything since Ambari doesn't know which process to kill.
# The start command will spawn another instance.
# The user should have issued a stop, changed the config, and then started it.
if not os.path.isdir(params.pid_dir):
Logger.info(
"*******************************************************************"
)
Logger.info(
"Will skip the stop command since this is the first time stopping/restarting Atlas "
"and the pid dir does not exist, %s\n" % params.pid_dir)
return
try:
Execute(
daemon_cmd,
user=params.metadata_user,
)
except:
show_logs(params.log_dir, params.metadata_user)
raise
File(params.pid_file, action="delete")
def setup_config(self, env):
import params
env.set_params(params)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group)
XmlConfig(
"core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
File(
format("{params.ambari_libs_dir}/fast-hdfs-resource.jar"),
mode=0644,
content=StaticFile(
"/var/lib/ambari-agent/cache/stack-hooks/before-START/files/fast-hdfs-resource.jar"
))
def update_credential_provider_path(config, config_type, dest_provider_path,
file_owner, file_group):
"""
Copies the JCEKS file for the specified config from the default location to the given location,
and sets the ACLs for the specified owner and group. Also updates the config type's configuration
hadoop credential store provider with the copied file name.
:param config: configurations['configurations'][config_type]
:param config_type: Like hive-site, oozie-site, etc.
:param dest_provider_path: The full path to the file where the JCEKS provider file is to be copied to.
:param file_owner: File owner
:param file_group: Group
:return: A copy of the config that was modified or the input config itself if nothing was modified.
"""
# Get the path to the provider <config_type>.jceks
if HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME in config:
provider_path = config[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME]
src_provider_path = provider_path[len('jceks://file'):]
File(dest_provider_path,
owner=file_owner,
group=file_group,
mode=0640,
content=StaticFile(src_provider_path))
# make a copy of the config dictionary since it is read-only
config_copy = config.copy()
# overwrite the provider path with the path specified
config_copy[
HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file{0}'.format(
dest_provider_path)
return config_copy
return config
def configure_airflow(self, env):
import params
env.set_params(params)
File("/etc/systemd/system/airflow_webserver.service",
content=Template("webserver.j2", configurations=params),
owner='root',
group='root',
mode=0o0600)
File("/etc/systemd/system/airflow_scheduler.service",
content=Template("scheduler.j2", configurations=params),
owner='root',
group='root',
mode=0o0600)
Execute('sudo systemctl daemon-reload')
def configure_ac(self, env):
import params
env.set_params(params)
print("!!!!!!!!!!!!!!!!!!!!!!!creating file!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
File("/etc/systemd/system/airflow_portforward.service",
content=Template("portforward_service.j2", configurations=params),
owner='root',
group='root',
mode=0o0600
)
Execute('sudo systemctl daemon-reload')
def configure_ac(self, env):
import params
env.set_params(params)
File("/etc/systemd/system/jupyter_portforward.service",
content=Template("portforward_service_jupyter.j2",
configurations=params),
owner=params.anaconda_user,
group=params.anaconda_group,
mode=0o0600)
Execute('sudo systemctl daemon-reload')
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh; {params.metadata_stop_script}')
try:
Execute(daemon_cmd,
user=params.metadata_user,
)
except:
show_logs(params.log_dir, params.metadata_user)
raise
File(params.pid_file, action="delete")
def install_jce_policy():
import params
jcePolicyInfo = JcePolicyInfo(params.java_home)
if jcePolicyInfo.is_unlimited_key_jce_policy():
major, minor = java_version()
Logger.info(
"The unlimited key JCE policy is required, and appears to have been installed."
)
java_security_dir = format("{java_home}/jre/lib/security")
Logger.debug("Removing existing JCE policy JAR files: {0}.".format(
java_security_dir))
File(format("{java_security_dir}/US_export_policy.jar"),
action="delete")
File(format("{java_security_dir}/local_policy.jar"), action="delete")
src_url = jce_zip_target = ''
if minor == 7: # jdk7
src_url = 'http://yum.example.com/hadoop/UnlimitedJCEPolicyJDK7.zip'
jce_zip_target = '/tmp/UnlimitedJCEPolicyJDK7.zip'
elif minor == 8: # jdk8
src_url = 'http://yum.example.com/hadoop/jce_policy-8.zip'
jce_zip_target = '/tmp/jce_policy-8.zip'
if src_url and jce_zip_target:
Execute('wget ' + src_url + ' -O ' + jce_zip_target)
Logger.debug(
"Unzipping the unlimited key JCE policy files from {0} into {1}."
.format(jce_zip_target, java_security_dir))
extract_cmd = ("unzip", "-o", "-j", "-q", jce_zip_target, "-d",
java_security_dir)
Execute(
extract_cmd,
only_if=format(
"test -e {java_security_dir} && test -f {jce_zip_target}"),
path=['/bin/', '/usr/bin'],
sudo=True)
def configure_ac(self, env):
import params
env.set_params(params)
conf = dict()
for key in params.config['configurations']['jupyter-env']:
conf[key] = params.config['configurations']['jupyter-env'][key]
conf['jupyter_password'] = params.hashText(conf['jupyter_password'])
print("----------------------------\n", conf, "\n------------------------------------------------")
File("{0}jupyter_notebook_config.py".format(params.config_dir),
content=Template("jupyter_notebook_config.py.j2",
configurations=conf),
owner=params.anaconda_user,
group=params.anaconda_group,
mode=0o0600
)
def metadata():
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
if not os.path.isfile(metadata_war_file):
raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
shutil.copy2(metadata_war_file, params.expanded_war_dir)
File(format('{conf_dir}/application.properties'),
content=InlineTemplate(params.application_properties_content),
mode=0644,
owner=params.metadata_user,
group=params.user_group
)
File(format("{conf_dir}/metadata-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content)
)
File(format("{conf_dir}/log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=StaticFile('log4j.xml')
)
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
cd_access='a',
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
if params.livy2_recovery_store == 'filesystem':
params.HdfsResource(params.livy2_recovery_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'spark2',
Template("input.config-spark2.json.j2", extra_imports=[default]))
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy-client.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy-client.conf"),
properties=params.config['configurations']['livy2-client-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
if type == 'server' and params.search_backend_solr and params.has_logsearch_solr:
solr_cloud_util.setup_solr_client(params.config)
random_num = random.random()
upload_conf_set('basic_configs', random_num)
create_collection('vertex_index', 'basic_configs')
create_collection('edge_index', 'basic_configs')
create_collection('fulltext_index', 'basic_configs')
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
def metadata():
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.atlas_hbase_log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.atlas_hbase_data_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
File(format("{expanded_war_dir}/atlas.war"),
content = StaticFile(format('{metadata_home}/server/webapp/atlas.war'))
)
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties = params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group
)
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content)
)
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content)
)
# hbase-site for embedded hbase used by Atlas
XmlConfig( "hbase-site.xml",
conf_dir = params.atlas_hbase_conf_dir,
configurations = params.config['configurations']['atlas-hbase-site'],
configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'],
owner = params.metadata_user,
group = params.user_group
)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)