def __init__(self, host, port, database_name, use_secure_connection=False):
if use_secure_connection:
self._connection = impala_connect(host, port, use_ssl=True, auth_mechanism='GSSAPI')
else:
self._connection = impala_connect(host, port)
self._database_name = database_name
self._staging_exec_result_table = None
self._staging_profile_table = None
self._cache = {}
def __init__(self, host, port, database_name, use_secure_connection=False):
if use_secure_connection:
self._connection = impala_connect(host,
port,
use_ssl=True,
auth_mechanism='GSSAPI')
else:
self._connection = impala_connect(host, port)
self._database_name = database_name
self._staging_exec_result_table = None
self._staging_profile_table = None
self._cache = {}
def create_database():
if options.use_secure_connection:
connection = impala_connect(options.host, options.port,
use_ssl=True, auth_mechanism='GSSAPI')
else:
connection = impala_connect(options.host, options.port)
cursor = connection.cursor()
cursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(options.db_name))
cursor.execute('USE {0}'.format(options.db_name))
for create_statement in CREATE_STATEMENTS:
cursor.execute(create_statement)
def create_database():
if options.use_secure_connection:
connection = impala_connect(options.host,
options.port,
use_ssl=True,
auth_mechanism='GSSAPI')
else:
connection = impala_connect(options.host, options.port)
cursor = connection.cursor()
cursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(options.db_name))
cursor.execute('USE {0}'.format(options.db_name))
for create_statement in CREATE_STATEMENTS:
cursor.execute(create_statement)
def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT):
"""Returns a connection to Impala. This is intended to be used in a "with" block.
The connection will be closed upon exiting the block.
The returned connection will have a 'db_name' property.
DEPRECATED:
See the 'unique_database' fixture above to use Impala's custom python
API instead of DB-API.
"""
default_impalad = pytest.config.option.impalad.split(',')[0]
impalad_host = default_impalad.split(':')[0]
hs2_port = pytest.config.option.impalad_hs2_port
conn = impala_connect(host=impalad_host,
port=hs2_port,
database=db_name,
timeout=timeout)
try:
conn.db_name = db_name
yield conn
finally:
try:
conn.close()
except Exception as e:
LOG.warn("Error closing Impala connection: %s", e)
def create_database():
connection = impala_connect(host=options.host, port=int(options.port))
cursor = connection.cursor()
cursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(options.db_name))
cursor.execute('USE {0}'.format(options.db_name))
for create_statement in CREATE_STATEMENTS:
cursor.execute(create_statement)
def create_database():
connection = impala_connect(host=options.host, port=int(options.port))
cursor = connection.cursor()
cursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(options.db_name))
cursor.execute('USE {0}'.format(options.db_name))
for create_statement in CREATE_STATEMENTS:
cursor.execute(create_statement)
def _update_impala_table(table, impalaHost, impalaPort):
"""
Method for update table in Impala
Parameters
----------
table: string
table name from hive
"""
with impala_connect(host=impalaHost, port=impalaPort) as conn:
impala_cursor = conn.cursor()
impala_cursor.execute("""
INVALIDATE METADATA {table} """.format(table=table))
def __auto_closed_conn(db_name=None):
"""Returns a connection to Impala. This is intended to be used in a "with" block. The
connection will be closed upon exiting the block.
The returned connection will have a 'db_name' property.
"""
conn = impala_connect(database=db_name)
try:
conn.db_name = db_name
yield conn
finally:
try:
conn.close()
except Exception as e:
LOG.warn("Error closing Impala connection: %s", e)
def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT):
"""Returns a connection to Impala. This is intended to be used in a "with" block. The
connection will be closed upon exiting the block.
The returned connection will have a 'db_name' property.
"""
conn = impala_connect(database=db_name, timeout=timeout)
try:
conn.db_name = db_name
yield conn
finally:
try:
conn.close()
except Exception as e:
LOG.warn("Error closing Impala connection: %s", e)
def create_connection(self, db_name=None):
if self.db_type == IMPALA:
connection_class = ImpalaDbConnection
connection = impala_connect(host=self.host_name, port=self.port or 21050)
elif self.db_type == POSTGRESQL:
connection_class = PostgresqlDbConnection
connection_args = {'user': self.user_name or 'postgres'}
if self.password:
connection_args['password'] = self.password
if db_name:
connection_args['database'] = db_name
if self.host_name:
connection_args['host'] = self.host_name
if self.port:
connection_args['port'] = self.port
global postgresql_connect
if not postgresql_connect:
try:
from psycopg2 import connect as postgresql_connect
except:
print('Error importing psycopg2. Please make sure it is installed. '
'See the README for details.')
raise
connection = postgresql_connect(**connection_args)
connection.autocommit = True
elif self.db_type == MYSQL:
connection_class = MySQLDbConnection
connection_args = {'user': self.user_name or 'root'}
if self.password:
connection_args['passwd'] = self.password
if db_name:
connection_args['db'] = db_name
if self.host_name:
connection_args['host'] = self.host_name
if self.port:
connection_args['port'] = self.port
global mysql_connect
if not mysql_connect:
try:
from MySQLdb import connect as mysql_connect
except:
print('Error importing MySQLdb. Please make sure it is installed. '
'See the README for details.')
raise
connection = mysql_connect(**connection_args)
else:
raise Exception('Unexpected database type: %s' % self.db_type)
return connection_class(self, connection, db_name=db_name)
def test_show_grant_in_hs2(self, vector, unique_database):
"""IMPALA-7701: Test that all types in show grant commands are correct. Incorrect
types can result in null/None values."""
role = 'sgu_test_primary'
self.client.execute('create role %s' % role)
self.client.execute('grant all on database %s to role %s' %
(unique_database, role))
default_impalad = pytest.config.option.impalad.split(',')[0]
impalad_host = default_impalad.split(':')[0]
impalad_hs2_port = pytest.config.option.impalad_hs2_port
with impala_connect(host=impalad_host, port=impalad_hs2_port) as conn:
cursor = conn.cursor()
cursor.execute('show grant user %s on database %s' %
(getuser(), unique_database))
rows = cursor.fetchall()
assert len(rows) == 1
cols = rows[0]
assert len(cols) == 10
assert cols[0] == 'USER' # principal_type
assert cols[1] == getuser() # principal_name
assert cols[2] == 'database' # scope
assert cols[3] == unique_database # database
assert cols[4] == '' # table
assert cols[5] == '' # column
assert cols[6] == '' # uri
assert cols[7] == 'owner' # privilege
assert cols[8] # grant_option
# We don't assert create_time since the value can be None or str depending on the
# Sentry refresh.
cursor.execute('show grant role %s on database %s' %
(role, unique_database))
rows = cursor.fetchall()
assert len(rows) == 1
cols = rows[0]
assert len(cols) == 8
assert cols[0] == 'database' # scope
assert cols[1] == unique_database # database
assert cols[2] == '' # table
assert cols[3] == '' # column
assert cols[4] == '' # uri
assert cols[5] == 'all' # privilege
assert not cols[6] # grant_option
def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT):
"""Returns a connection to Impala. This is intended to be used in a "with" block.
The connection will be closed upon exiting the block.
The returned connection will have a 'db_name' property.
"""
default_impalad = pytest.config.option.impalad.split(',')[0]
impalad_host = default_impalad.split(':')[0]
hs2_port = pytest.config.option.impalad_hs2_port
conn = impala_connect(host=impalad_host, port=hs2_port, database=db_name,
timeout=timeout)
try:
conn.db_name = db_name
yield conn
finally:
try:
conn.close()
except Exception as e:
LOG.warn("Error closing Impala connection: %s", e)
def test_show_grant_in_hs2(self, vector, unique_database):
"""IMPALA-7701: Test that all types in show grant commands are correct. Incorrect
types can result in null/None values."""
role = 'sgu_test_primary'
self.client.execute('create role %s' % role)
self.client.execute('grant all on database %s to role %s' % (unique_database, role))
default_impalad = pytest.config.option.impalad.split(',')[0]
impalad_host = default_impalad.split(':')[0]
impalad_hs2_port = pytest.config.option.impalad_hs2_port
with impala_connect(host=impalad_host, port=impalad_hs2_port) as conn:
cursor = conn.cursor()
cursor.execute('show grant user %s on database %s' % (getuser(), unique_database))
rows = cursor.fetchall()
assert len(rows) == 1
cols = rows[0]
assert len(cols) == 10
assert cols[0] == 'USER' # principal_type
assert cols[1] == getuser() # principal_name
assert cols[2] == 'database' # scope
assert cols[3] == unique_database # database
assert cols[4] == '' # table
assert cols[5] == '' # column
assert cols[6] == '' # uri
assert cols[7] == 'owner' # privilege
assert cols[8] # grant_option
# We don't assert create_time since the value can be None or str depending on the
# Sentry refresh.
cursor.execute('show grant role %s on database %s' % (role, unique_database))
rows = cursor.fetchall()
assert len(rows) == 1
cols = rows[0]
assert len(cols) == 8
assert cols[0] == 'database' # scope
assert cols[1] == unique_database # database
assert cols[2] == '' # table
assert cols[3] == '' # column
assert cols[4] == '' # uri
assert cols[5] == 'all' # privilege
assert not cols[6] # grant_option
def init_impala_connect(self):
""" Connect to Cloudera Impala """
self.impala_connection = impala_connect(**self.connect_params)
self.impala_cursor = None
def __init__(self, host, port, database_name):
self._connection = impala_connect(host, port)
self._database_name = database_name
self._staging_exec_result_table = None
self._staging_profile_table = None
self._cache = {}
def __init__(self, host, port, database_name):
self._connection = impala_connect(host, port)
self._database_name = database_name
self._staging_exec_result_table = None
self._staging_profile_table = None
self._cache = {}
def create_connection(self, db_name=None):
if self.db_type == HIVE:
connection_class = HiveDbConnection
connection = impala_connect(
host=self.host_name,
port=self.port,
user=self.user_name,
password=self.password,
timeout=maxint,
auth_mechanism='PLAIN')
return HiveDbConnection(self, connection, user_name=self.user_name,
user_pass=self.password, db_name=db_name, hdfs_host=self.hdfs_host,
hdfs_port=self.hdfs_port)
elif self.db_type == IMPALA:
connection_class = ImpalaDbConnection
connection = impala_connect(
host=self.host_name,
port=self.port or 21050,
timeout=maxint)
elif self.db_type == ORACLE:
connection_class = OracleDbConnection
connection_str = '%(user)s/%(password)s@%(host)s:%(port)s/%(service)s'
connection_args = {
'user': self.user_name or 'system',
'password': self.password or 'oracle',
'host': self.host_name or 'localhost',
'port': self.port or 1521,
'service': self.service or 'XE'}
try:
from cx_Oracle import connect as oracle_connect
except:
print('Error importing cx_Oracle. Please make sure it is installed. '
'See the README for details.')
raise
connection = oracle_connect(connection_str % connection_args)
connection.outputtypehandler = OracleDbConnection.type_converter
connection.autocommit = True
elif self.db_type == POSTGRESQL:
connection_class = PostgresqlDbConnection
connection_args = {'user': self.user_name or 'postgres'}
if self.password:
connection_args['password'] = self.password
if db_name:
connection_args['database'] = db_name
if self.host_name:
connection_args['host'] = self.host_name
if self.port:
connection_args['port'] = self.port
connection = postgresql_connect(**connection_args)
connection.autocommit = True
elif self.db_type == MYSQL:
connection_class = MySQLDbConnection
connection_args = {'user': self.user_name or 'root'}
if self.password:
connection_args['passwd'] = self.password
if db_name:
connection_args['db'] = db_name
if self.host_name:
connection_args['host'] = self.host_name
if self.port:
connection_args['port'] = self.port
try:
from MySQLdb import connect as mysql_connect
except:
print('Error importing MySQLdb. Please make sure it is installed. '
'See the README for details.')
raise
connection = mysql_connect(**connection_args)
else:
raise Exception('Unexpected database type: %s' % self.db_type)
return connection_class(self, connection, db_name=db_name)
def connect():
global connection, cursor
connection = impala_connect(host="localhost", port=21050, database="cass")
cursor = connection.cursor()
cursor.execute('use cass')
def get_connection(self, **extra_args):
connect_args = self.connect_args
connect_args.update(extra_args)
impala_conn = impala_connect(**connect_args)
return impala_thrift_connection(impala_conn)
