def __init__(self, url, user_name=None, use_kerberos=False):
# Set a specific session that doesn't verify SSL certs. This is needed because
# requests doesn't like self-signed certs.
# TODO: Support a CA bundle.
s = requests.Session()
s.verify = False
if use_kerberos:
try:
from hdfs.ext.kerberos import KerberosClient
except ImportError as e:
if "No module named requests_kerberos" not in str(e):
raise e
import os
import subprocess
LOG.info("kerberos module not found; attempting to install it...")
pip_path = os.path.join(os.environ["IMPALA_HOME"], "infra", "python", "env",
"bin", "pip")
try:
local_shell(pip_path + " install pykerberos==1.1.14 requests-kerberos==0.11.0",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
LOG.info("kerberos installation complete.")
except Exception as e:
LOG.error("kerberos installation failed. Try installing libkrb5-dev and"
" then try again.")
raise e
from hdfs.ext.kerberos import KerberosClient
self._client = KerberosClient(url, session=s)
else:
self._client = hdfs.client.InsecureClient(url, user=user_name, session=s)
def __init__(self, url, user_name=None, use_kerberos=False):
if use_kerberos:
# TODO: Have the virtualenv attempt to install a list of optional libs.
try:
import kerberos
except ImportError as e:
if "No module named kerberos" not in str(e):
raise e
import os
import subprocess
LOG.info(
"kerberos module not found; attempting to install it...")
pip_path = os.path.join(os.environ["IMPALA_HOME"], "infra",
"python", "env", "bin", "pip")
try:
local_shell(pip_path + " install kerboros",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
LOG.info("kerberos installation complete.")
except Exception as e:
LOG.error(
"kerberos installation failed. Try installing libkrb5-dev and"
" then try again.")
raise e
self._client = hdfs.ext.kerberos.KerberosClient(url,
user=user_name)
else:
self._client = hdfs.client.InsecureClient(url, user=user_name)
def run_mr_job(self, jar_path, job_args=''):
"""Runs the MR job specified by the 'jar_path' and 'job_args' and blocks until
completion.
"""
env = dict(os.environ)
env['HADOOP_CONF_DIR'] = self.cluster.local_hadoop_conf_dir
env['CDH_MR2_HOME'] = os.environ['HADOOP_HOME']
env['HADOOP_USER_NAME'] = self.cluster.hadoop_user_name
local_shell('hadoop jar %s %s' % (jar_path, job_args), stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=env)
def run_mr_job(self, jar_path, job_args=''):
"""Runs the MR job specified by the 'jar_path' and 'job_args' and blocks until
completion.
"""
env = dict(os.environ)
env['HADOOP_CONF_DIR'] = self.cluster.local_hadoop_conf_dir
env['CDH_MR2_HOME'] = os.environ['HADOOP_HOME']
env['HADOOP_USER_NAME'] = self.cluster.hadoop_user_name
local_shell('hadoop jar %s %s' % (jar_path, job_args),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
def __init__(self, url, user_name=None, use_kerberos=False):
# Set a specific session that doesn't verify SSL certs. This is needed because
# requests doesn't like self-signed certs.
# TODO: Support a CA bundle.
s = requests.Session()
s.verify = False
if use_kerberos:
try:
from hdfs.ext.kerberos import KerberosClient
except ImportError as e:
if "No module named requests_kerberos" not in str(e):
raise e
import os
import subprocess
LOG.info(
"kerberos module not found; attempting to install it...")
pip_path = os.path.join(os.environ["IMPALA_HOME"], "infra",
"python", "env", "bin", "pip")
try:
local_shell(
pip_path +
" install pykerberos==1.1.14 requests-kerberos==0.11.0",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
LOG.info("kerberos installation complete.")
except Exception as e:
LOG.error(
"kerberos installation failed. Try installing libkrb5-dev and"
" then try again.")
raise e
from hdfs.ext.kerberos import KerberosClient
self._client = KerberosClient(url, session=s)
else:
self._client = hdfs.client.InsecureClient(url,
user=user_name,
session=s)
def __init__(self, url, user_name=None, use_kerberos=False):
if use_kerberos:
# TODO: Have the virtualenv attempt to install a list of optional libs.
try:
import kerberos
except ImportError as e:
if "No module named kerberos" not in str(e):
raise e
import os
import subprocess
LOG.info("kerberos module not found; attempting to install it...")
pip_path = os.path.join(os.environ["IMPALA_HOME"], "infra", "python", "env",
"bin", "pip")
try:
local_shell(pip_path + " install kerboros", stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
LOG.info("kerberos installation complete.")
except Exception as e:
LOG.error("kerberos installation failed. Try installing libkrb5-dev and"
" then try again.")
raise e
self._client = hdfs.ext.kerberos.KerberosClient(url, user=user_name)
else:
self._client = hdfs.client.InsecureClient(url, user=user_name)
def shell(self, cmd, unused_host_name, timeout_secs=DEFAULT_TIMEOUT):
return local_shell(cmd, timeout_secs=timeout_secs)
def shell(self, cmd, unused_host_name, timeout_secs=DEFAULT_TIMEOUT): return local_shell(cmd, timeout_secs=timeout_secs)
