def download_extensions(owner_user, owner_group, hdfs_source_dir, local_target_dir): """ :param owner_user: user owner of the HDFS directory :param owner_group: group owner of the HDFS directory :param hdfs_source_dir: the HDFS directory from where the files are being pull :param local_target_dir: the location of where to download the files :return: Will return True if successful, otherwise, False. """ import params if not os.path.isdir(local_target_dir): import tempfile #Create a secure random temp directory tmp_dir = tempfile.mkdtemp() cmd = ('chown', '-R', params.hdfs_user, tmp_dir) Execute(cmd, sudo=True) cmd = ('chmod', '755', tmp_dir) Execute(cmd, sudo=True) Directory(os.path.dirname(local_target_dir), owner="root", mode=0755, group="root", create_parents=True) params.HdfsResource(hdfs_source_dir, type="directory", action="create_on_execute", owner=owner_user, group=owner_group, mode=0755) # copy from hdfs to tmp_dir params.HdfsResource(tmp_dir, type="directory", action="download_on_execute", source=hdfs_source_dir, user=params.hdfs_user, mode=0644, replace_existing_files=True) # Execute command is not quoting correctly. cmd = ('mv', tmp_dir, local_target_dir) only_if_cmd = "ls -d {tmp_dir}/*".format(tmp_dir=tmp_dir) Execute(cmd, only_if=only_if_cmd, sudo=True) only_if_local = 'ls -d "{local_target_dir}"'.format( local_target_dir=local_target_dir) Execute(("chown", "-R", "root:root", local_target_dir), sudo=True, only_if=only_if_local) params.HdfsResource(None, action="execute") return True
def setup_ranger_plugin(component_select_name, service_name, downloaded_custom_connector, driver_curl_source, driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs): File(downloaded_custom_connector, content = DownloadSource(driver_curl_source), mode = 0644 ) Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True ) File(driver_curl_target, mode=0644) hdp_version = get_hdp_version(component_select_name) file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties') if not os.path.isfile(file_path): raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}')) ModifyPropertiesFile(file_path, properties = plugin_properties ) custom_plugin_properties = dict() custom_plugin_properties['CUSTOM_USER'] = component_user custom_plugin_properties['CUSTOM_GROUP'] = component_group ModifyPropertiesFile(file_path,properties = custom_plugin_properties) if plugin_enabled: cmd = (format('enable-{service_name}-plugin.sh'),) if api_version == 'v2' and api_version is not None: ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down) else: ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) else: cmd = (format('disable-{service_name}-plugin.sh'),) cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')} Execute(cmd, environment=cmd_env, logoutput=True, sudo=True, )
def execute(configurations={}, parameters={}, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: configurations (dictionary): a mapping of configuration key to value parameters (dictionary): a mapping of script parameter key to value host_name (string): the name of this host where the alert is running """ from resource_management.libraries.functions import reload_windows_env from resource_management.core.resources import Execute reload_windows_env() hive_home = os.environ['HIVE_HOME'] if configurations is None: return ('UNKNOWN', ['There were no configurations supplied to the script.']) transport_mode = HIVE_SERVER_TRANSPORT_MODE_DEFAULT if HIVE_SERVER_TRANSPORT_MODE_KEY in configurations: transport_mode = configurations[HIVE_SERVER_TRANSPORT_MODE_KEY] port = THRIFT_PORT_DEFAULT if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in configurations: port = int(configurations[HIVE_SERVER_THRIFT_PORT_KEY]) elif transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY in configurations: port = int(configurations[HIVE_SERVER_THRIFT_HTTP_PORT_KEY]) hiveuser = HADOOPUSER_DEFAULT if HADOOPUSER_KEY in configurations: hiveuser = configurations[HADOOPUSER_KEY] result_code = None try: if host_name is None: host_name = socket.getfqdn() beeline_url = ['jdbc:hive2://{host_name}:{port}/', "transportMode={transport_mode}"] # append url according to used transport if transport_mode == "http": beeline_url.append('httpPath=cliservice') beeline_url_string = format(";".join(beeline_url)) beeline_cmd = os.path.join(hive_home, "bin", "beeline.cmd") cmd = format("cmd /c {beeline_cmd} -u {beeline_url_string} -e '' 2>&1 | findstr Connected") start_time = time.time() try: Execute(cmd, user=hiveuser, timeout=30) total_time = time.time() - start_time result_code = 'OK' label = OK_MESSAGE.format(total_time, port) except: result_code = 'CRITICAL' label = CRITICAL_MESSAGE.format(host_name, port, traceback.format_exc()) except: label = traceback.format_exc() result_code = 'UNKNOWN' return (result_code, [label])
def setup_ranger_plugin(component_select_name, service_name, downloaded_custom_connector, driver_curl_source, driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled): File(downloaded_custom_connector, content = DownloadSource(driver_curl_source) ) Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target), not_if=format("test -f {driver_curl_target}"), sudo=True ) hdp_version = get_hdp_version(component_select_name) file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties') if not os.path.isfile(file_path): raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}')) ModifyPropertiesFile(file_path, properties = plugin_properties ) if plugin_enabled: cmd = (format('enable-{service_name}-plugin.sh'),) ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) else: cmd = (format('disable-{service_name}-plugin.sh'),) cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')} Execute(cmd, environment=cmd_env, logoutput=True, sudo=True, )