def setup_hadoop_env():
import params
stackversion = params.stack_version_unformatted
Logger.info("FS Type: {0}".format(params.dfs_type))
if params.has_namenode or stackversion.find(
'Gluster') >= 0 or params.dfs_type == 'HCFS':
if params.security_enabled:
tc_owner = "root"
else:
tc_owner = params.hdfs_user
# create /etc/hadoop
#Directory(params.hadoop_dir, mode=0755)
# write out hadoop-env.sh, but only if the directory exists
if os.path.exists(params.hadoop_conf_dir):
File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'),
owner=tc_owner,
group=params.user_group,
content=InlineTemplate(params.hadoop_env_sh_template))
# Create tmp dir for java.io.tmpdir
# Handle a situation when /tmp is set to noexec
try:
Directory(params.hadoop_java_io_tmpdir,
owner=params.hdfs_user,
group=params.user_group,
mode=01777)
except Exception as e:
Logger.info('Skipping setting uid for hdfs user as host')
def metadata():
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(format('{metadata_home}/server/webapp/atlas.war')))
PropertiesFile(format('{conf_dir}/application.properties'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=StaticFile('atlas-log4j.xml'))
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def metadata():
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
recursive=True
)
metadata_war_file = format('{params.metadata_home}/server/webapp/metadata.war')
if not os.path.isfile(metadata_war_file):
raise Fail("Unable to copy {0} because it does not exist".format(metadata_war_file))
Logger.info("Copying {0} to {1}".format(metadata_war_file, params.expanded_war_dir))
shutil.copy2(metadata_war_file, params.expanded_war_dir)
File(format('{conf_dir}/application.properties'),
content=InlineTemplate(params.application_properties_content),
mode=0644,
owner=params.metadata_user,
group=params.user_group
)
File(format("{conf_dir}/metadata-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content)
)
File(format("{conf_dir}/log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=StaticFile('log4j.xml')
)
def setup_livy(env, type, upgrade_type=None, action=None):
import params
Directory([params.livy2_pid_dir, params.livy2_log_dir],
owner=params.livy2_user,
group=params.user_group,
mode=0775,
cd_access='a',
create_parents=True)
if type == 'server' and action == 'config':
params.HdfsResource(params.livy2_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0775)
params.HdfsResource(None, action="execute")
if params.livy2_recovery_store == 'filesystem':
params.HdfsResource(params.livy2_recovery_dir,
type="directory",
action="create_on_execute",
owner=params.livy2_user,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'spark2',
Template("input.config-spark2.json.j2", extra_imports=[default]))
# create livy-env.sh in etc/conf dir
File(
os.path.join(params.livy2_conf, 'livy-env.sh'),
owner=params.livy2_user,
group=params.livy2_group,
content=InlineTemplate(params.livy2_env_sh),
mode=0644,
)
# create livy-client.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy-client.conf"),
properties=params.config['configurations']['livy2-client-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create livy.conf in etc/conf dir
PropertiesFile(
format("{livy2_conf}/livy.conf"),
properties=params.config['configurations']['livy2-conf'],
key_value_delimiter=" ",
owner=params.livy2_user,
group=params.livy2_group,
)
# create log4j.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'log4j.properties'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_log4j_properties,
mode=0644,
)
# create spark-blacklist.properties in etc/conf dir
File(
os.path.join(params.livy2_conf, 'spark-blacklist.conf'),
owner=params.livy2_user,
group=params.livy2_group,
content=params.livy2_spark_blacklist_properties,
mode=0644,
)
Directory(
params.livy2_logs_dir,
owner=params.livy2_user,
group=params.livy2_group,
mode=0755,
)
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
if type == 'server' and params.search_backend_solr and params.has_logsearch_solr:
solr_cloud_util.setup_solr_client(params.config)
random_num = random.random()
upload_conf_set('basic_configs', random_num)
create_collection('vertex_index', 'basic_configs')
create_collection('edge_index', 'basic_configs')
create_collection('fulltext_index', 'basic_configs')
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
def metadata():
import params
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.atlas_hbase_log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.atlas_hbase_data_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents = True
)
File(format("{expanded_war_dir}/atlas.war"),
content = StaticFile(format('{metadata_home}/server/webapp/atlas.war'))
)
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties = params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group
)
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content)
)
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content)
)
# hbase-site for embedded hbase used by Atlas
XmlConfig( "hbase-site.xml",
conf_dir = params.atlas_hbase_conf_dir,
configurations = params.config['configurations']['atlas-hbase-site'],
configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'],
owner = params.metadata_user,
group = params.user_group
)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)