def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.
config['configuration_attributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if not is_empty(params.hive_exec_scratchdir):
dirPathStr = urlparse(params.hive_exec_scratchdir).path
pathComponents = dirPathStr.split("/")
if dirPathStr.startswith("/tmp") and len(pathComponents) > 2:
Directory(params.hive_exec_scratchdir,
owner=params.hive_user,
create_parents=True,
mode=0777)
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hivemetastore-site'],
configuration_attributes=params.config['configuration_attributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600
)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh')
)
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.
config['configuration_attributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.init_metastore_schema:
create_schema_cmd = format(
"export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_schematool_bin}/schematool -initSchema "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p} -verbose")
check_schema_created_cmd = as_user(
format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_schematool_bin}/schematool -info "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p} -verbose"),
params.hive_user)
# HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
# Fixing it with the hack below:
quoted_hive_metastore_user_passwd = quote_bash_args(
quote_bash_args(params.hive_metastore_user_passwd))
if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[
1:-1]
Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(
check_schema_created_cmd.replace(
format("-passWord {quoted_hive_metastore_user_passwd}"),
"-passWord " + utils.PASSWORDS_HIDE_STRING))
Execute(create_schema_cmd,
not_if=check_schema_created_cmd,
user=params.hive_user)
def knox():
import params
XmlConfig(
"gateway-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']['gateway-site'],
configuration_attributes=params.config['configuration_attributes']
['gateway-site'],
owner=params.knox_user)
# Manually overriding service logon user & password set by the installation package
ServiceConfig(params.knox_gateway_win_service_name,
action="change_user",
username=params.knox_user,
password=Script.get_password(params.knox_user))
File(os.path.join(params.knox_conf_dir, "gateway-log4j.properties"),
owner=params.knox_user,
content=params.gateway_log4j)
File(os.path.join(params.knox_conf_dir, "topologies", "default.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.topology_template))
if params.admin_topology_template:
File(os.path.join(params.knox_conf_dir, "topologies", "admin.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.admin_topology_template))
if params.version_formatted and check_stack_feature(
StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted):
knoxsso_topology_template_content = get_config("knoxsso-topology")
if knoxsso_topology_template_content:
File(os.path.join(params.knox_conf_dir, "topologies",
"knoxsso.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.knoxsso_topology_template))
if params.security_enabled:
TemplateConfig(os.path.join(params.knox_conf_dir,
"krb5JAASLogin.conf"),
owner=params.knox_user,
template_tag=None)
if not os.path.isfile(params.knox_master_secret_path):
cmd = format(
'cmd /C {knox_client_bin} create-master --master {knox_master_secret!p}'
)
Execute(cmd)
cmd = format(
'cmd /C {knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}'
)
Execute(cmd)
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.
config['configurationAttributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip(
) != "":
params.HdfsResource(params.hive_repl_cmrootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01777)
if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip(
) != "":
params.HdfsResource(params.hive_repl_rootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def setup_metastore():
import params
if params.hive_metastore_site_supported:
hivemetastore_site_config = get_config("hivemetastore-site")
if hivemetastore_site_config:
XmlConfig("hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.
config['configuration_attributes']['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"),
mode=0600)
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.hive_repl_cmrootdir:
params.HdfsResource(params.hive_repl_cmrootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=01777)
if params.hive_repl_rootdir:
params.HdfsResource(params.hive_repl_rootdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
if params.hive_repl_cmrootdir or params.hive_repl_rootdir:
params.HdfsResource(None, action="execute")
def sqoop(type=None):
import params
Link(params.sqoop_lib + "/mysql-connector-java.jar",
to='/usr/share/java/mysql-connector-java.jar')
jdbc_connector()
Directory(params.sqoop_conf_dir,
owner=params.sqoop_user,
group=params.user_group,
create_parents=True)
configs = {}
sqoop_site_config = get_config('sqoop-site')
if sqoop_site_config:
configs.update(sqoop_site_config)
XmlConfig(
"sqoop-site.xml",
conf_dir=params.sqoop_conf_dir,
configurations=configs,
configuration_attributes=params.config['configurationAttributes']
['sqoop-site'],
owner=params.sqoop_user,
group=params.user_group)
# Generate atlas-application.properties.xml file and symlink the hook jars
if params.enable_atlas_hook:
atlas_hook_filepath = os.path.join(params.sqoop_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.SQOOP,
params.sqoop_atlas_application_properties,
atlas_hook_filepath, params.sqoop_user,
params.user_group)
setup_atlas_jar_symlinks("sqoop", params.sqoop_lib)
File(format("{sqoop_conf_dir}/sqoop-env.sh"),
owner=params.sqoop_user,
group=params.user_group,
content=InlineTemplate(params.sqoop_env_sh_template))
update_config_permissions(
["sqoop-env-template.sh", "sqoop-site-template.xml", "sqoop-site.xml"])
pass
def knox():
import params
Directory(
[
params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir,
params.knox_conf_dir,
os.path.join(params.knox_conf_dir, "topologies")
],
owner=params.knox_user,
group=params.knox_group,
create_parents=True,
cd_access="a",
mode=0755,
recursive_ownership=True,
)
XmlConfig(
"gateway-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']['gateway-site'],
configuration_attributes=params.config['configurationAttributes']
['gateway-site'],
owner=params.knox_user,
group=params.knox_group,
)
File(format("{params.knox_conf_dir}/gateway-log4j.properties"),
mode=0644,
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.gateway_log4j))
File(format("{params.knox_conf_dir}/topologies/default.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.topology_template))
if params.admin_topology_template:
File(format("{params.knox_conf_dir}/topologies/admin.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.admin_topology_template))
if params.version_formatted and check_stack_feature(
StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted):
knoxsso_topology_template_content = get_config("knoxsso-topology")
if knoxsso_topology_template_content:
File(os.path.join(params.knox_conf_dir, "topologies",
"knoxsso.xml"),
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.knoxsso_topology_template))
if params.security_enabled:
TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"),
owner=params.knox_user,
template_tag=None)
cmd = format(
'{knox_client_bin} create-master --master {knox_master_secret!p}')
master_secret_exist = as_user(format('test -f {knox_master_secret_path}'),
params.knox_user)
Execute(
cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=master_secret_exist,
)
cmd = format(
'{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}')
cert_store_exist = as_user(format('test -f {knox_cert_store_path}'),
params.knox_user)
Execute(
cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=cert_store_exist,
)
def knox():
import params
Directory(
[
params.knox_data_dir, params.knox_data_backup_dir,
params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir,
os.path.join(params.knox_conf_dir, "topologies"),
params.knox_descriptors_dir, params.knox_shared_providers_dir
],
owner=params.knox_user,
group=params.knox_group,
create_parents=True,
cd_access="a",
mode=0755,
recursive_ownership=True,
)
XmlConfig(
"gateway-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']['gateway-site'],
configuration_attributes=params.config['configurationAttributes']
['gateway-site'],
owner=params.knox_user,
group=params.knox_group,
)
File(format("{params.knox_conf_dir}/gateway-log4j.properties"),
mode=0644,
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.gateway_log4j))
File(format("{params.knox_conf_dir}/topologies/default.xml"),
mode=0600,
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.topology_template))
if params.admin_topology_template:
File(format("{params.knox_conf_dir}/topologies/admin.xml"),
mode=0600,
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.admin_topology_template))
knoxsso_topology_template_content = get_config("knoxsso-topology")
if knoxsso_topology_template_content:
File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"),
mode=0600,
group=params.knox_group,
owner=params.knox_user,
content=InlineTemplate(params.knoxsso_topology_template))
if params.security_enabled:
TemplateConfig(format("{knox_conf_dir}/krb5JAASLogin.conf"),
owner=params.knox_user,
template_tag=None)
generate_logfeeder_input_config(
'knox', Template("input.config-knox.json.j2", extra_imports=[default]))
cmd = format(
'{knox_client_bin} create-master --master {knox_master_secret!p}')
master_secret_exist = as_user(format('test -f {knox_master_secret_path}'),
params.knox_user)
Execute(
cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=master_secret_exist,
)
cmd = format(
'{knox_client_bin} create-cert --hostname {knox_host_name_in_cluster}')
cert_store_exist = as_user(format('test -f {knox_cert_store_path}'),
params.knox_user)
Execute(
cmd,
user=params.knox_user,
environment={'JAVA_HOME': params.java_home},
not_if=cert_store_exist,
)
