def download_extensions(owner_user, owner_group, hdfs_source_dir,
local_target_dir):
"""
:param owner_user: user owner of the HDFS directory
:param owner_group: group owner of the HDFS directory
:param hdfs_source_dir: the HDFS directory from where the files are being pull
:param local_target_dir: the location of where to download the files
:return: Will return True if successful, otherwise, False.
"""
import params
if not os.path.isdir(local_target_dir):
import tempfile
#Create a secure random temp directory
tmp_dir = tempfile.mkdtemp()
cmd = ('chown', '-R', params.hdfs_user, tmp_dir)
Execute(cmd, sudo=True)
cmd = ('chmod', '755', tmp_dir)
Execute(cmd, sudo=True)
Directory(os.path.dirname(local_target_dir),
owner="root",
mode=0755,
group="root",
create_parents=True)
params.HdfsResource(hdfs_source_dir,
type="directory",
action="create_on_execute",
owner=owner_user,
group=owner_group,
mode=0755)
# copy from hdfs to tmp_dir
params.HdfsResource(tmp_dir,
type="directory",
action="download_on_execute",
source=hdfs_source_dir,
user=params.hdfs_user,
mode=0644,
replace_existing_files=True)
# Execute command is not quoting correctly.
cmd = ('mv', tmp_dir, local_target_dir)
only_if_cmd = "ls -d {tmp_dir}/*".format(tmp_dir=tmp_dir)
Execute(cmd, only_if=only_if_cmd, sudo=True)
only_if_local = 'ls -d "{local_target_dir}"'.format(
local_target_dir=local_target_dir)
Execute(("chown", "-R", "root:root", local_target_dir),
sudo=True,
only_if=only_if_local)
params.HdfsResource(None, action="execute")
return True
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
custom_plugin_properties = dict()
custom_plugin_properties['CUSTOM_USER'] = component_user
custom_plugin_properties['CUSTOM_GROUP'] = component_group
ModifyPropertiesFile(file_path,properties = custom_plugin_properties)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
from resource_management.libraries.functions import reload_windows_env
from resource_management.core.resources import Execute
reload_windows_env()
hive_home = os.environ['HIVE_HOME']
if configurations is None:
return ('UNKNOWN', ['There were no configurations supplied to the script.'])
transport_mode = HIVE_SERVER_TRANSPORT_MODE_DEFAULT
if HIVE_SERVER_TRANSPORT_MODE_KEY in configurations:
transport_mode = configurations[HIVE_SERVER_TRANSPORT_MODE_KEY]
port = THRIFT_PORT_DEFAULT
if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in configurations:
port = int(configurations[HIVE_SERVER_THRIFT_PORT_KEY])
elif transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY in configurations:
port = int(configurations[HIVE_SERVER_THRIFT_HTTP_PORT_KEY])
hiveuser = HADOOPUSER_DEFAULT
if HADOOPUSER_KEY in configurations:
hiveuser = configurations[HADOOPUSER_KEY]
result_code = None
try:
if host_name is None:
host_name = socket.getfqdn()
beeline_url = ['jdbc:hive2://{host_name}:{port}/', "transportMode={transport_mode}"]
# append url according to used transport
if transport_mode == "http":
beeline_url.append('httpPath=cliservice')
beeline_url_string = format(";".join(beeline_url))
beeline_cmd = os.path.join(hive_home, "bin", "beeline.cmd")
cmd = format("cmd /c {beeline_cmd} -u {beeline_url_string} -e '' 2>&1 | findstr Connected")
start_time = time.time()
try:
Execute(cmd, user=hiveuser, timeout=30)
total_time = time.time() - start_time
result_code = 'OK'
label = OK_MESSAGE.format(total_time, port)
except:
result_code = 'CRITICAL'
label = CRITICAL_MESSAGE.format(host_name, port, traceback.format_exc())
except:
label = traceback.format_exc()
result_code = 'UNKNOWN'
return (result_code, [label])
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source)
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
not_if=format("test -f {driver_curl_target}"),
sudo=True
)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
