def __init__(self, host, port, database_name, use_secure_connection=False): if use_secure_connection: self._connection = impala_connect(host, port, use_ssl=True, auth_mechanism='GSSAPI') else: self._connection = impala_connect(host, port) self._database_name = database_name self._staging_exec_result_table = None self._staging_profile_table = None self._cache = {}
def create_database(): if options.use_secure_connection: connection = impala_connect(options.host, options.port, use_ssl=True, auth_mechanism='GSSAPI') else: connection = impala_connect(options.host, options.port) cursor = connection.cursor() cursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(options.db_name)) cursor.execute('USE {0}'.format(options.db_name)) for create_statement in CREATE_STATEMENTS: cursor.execute(create_statement)
def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT): """Returns a connection to Impala. This is intended to be used in a "with" block. The connection will be closed upon exiting the block. The returned connection will have a 'db_name' property. DEPRECATED: See the 'unique_database' fixture above to use Impala's custom python API instead of DB-API. """ default_impalad = pytest.config.option.impalad.split(',')[0] impalad_host = default_impalad.split(':')[0] hs2_port = pytest.config.option.impalad_hs2_port conn = impala_connect(host=impalad_host, port=hs2_port, database=db_name, timeout=timeout) try: conn.db_name = db_name yield conn finally: try: conn.close() except Exception as e: LOG.warn("Error closing Impala connection: %s", e)
def create_database(): connection = impala_connect(host=options.host, port=int(options.port)) cursor = connection.cursor() cursor.execute('CREATE DATABASE IF NOT EXISTS {0}'.format(options.db_name)) cursor.execute('USE {0}'.format(options.db_name)) for create_statement in CREATE_STATEMENTS: cursor.execute(create_statement)
def _update_impala_table(table, impalaHost, impalaPort): """ Method for update table in Impala Parameters ---------- table: string table name from hive """ with impala_connect(host=impalaHost, port=impalaPort) as conn: impala_cursor = conn.cursor() impala_cursor.execute(""" INVALIDATE METADATA {table} """.format(table=table))
def __auto_closed_conn(db_name=None): """Returns a connection to Impala. This is intended to be used in a "with" block. The connection will be closed upon exiting the block. The returned connection will have a 'db_name' property. """ conn = impala_connect(database=db_name) try: conn.db_name = db_name yield conn finally: try: conn.close() except Exception as e: LOG.warn("Error closing Impala connection: %s", e)
def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT): """Returns a connection to Impala. This is intended to be used in a "with" block. The connection will be closed upon exiting the block. The returned connection will have a 'db_name' property. """ conn = impala_connect(database=db_name, timeout=timeout) try: conn.db_name = db_name yield conn finally: try: conn.close() except Exception as e: LOG.warn("Error closing Impala connection: %s", e)
def create_connection(self, db_name=None): if self.db_type == IMPALA: connection_class = ImpalaDbConnection connection = impala_connect(host=self.host_name, port=self.port or 21050) elif self.db_type == POSTGRESQL: connection_class = PostgresqlDbConnection connection_args = {'user': self.user_name or 'postgres'} if self.password: connection_args['password'] = self.password if db_name: connection_args['database'] = db_name if self.host_name: connection_args['host'] = self.host_name if self.port: connection_args['port'] = self.port global postgresql_connect if not postgresql_connect: try: from psycopg2 import connect as postgresql_connect except: print('Error importing psycopg2. Please make sure it is installed. ' 'See the README for details.') raise connection = postgresql_connect(**connection_args) connection.autocommit = True elif self.db_type == MYSQL: connection_class = MySQLDbConnection connection_args = {'user': self.user_name or 'root'} if self.password: connection_args['passwd'] = self.password if db_name: connection_args['db'] = db_name if self.host_name: connection_args['host'] = self.host_name if self.port: connection_args['port'] = self.port global mysql_connect if not mysql_connect: try: from MySQLdb import connect as mysql_connect except: print('Error importing MySQLdb. Please make sure it is installed. ' 'See the README for details.') raise connection = mysql_connect(**connection_args) else: raise Exception('Unexpected database type: %s' % self.db_type) return connection_class(self, connection, db_name=db_name)
def test_show_grant_in_hs2(self, vector, unique_database): """IMPALA-7701: Test that all types in show grant commands are correct. Incorrect types can result in null/None values.""" role = 'sgu_test_primary' self.client.execute('create role %s' % role) self.client.execute('grant all on database %s to role %s' % (unique_database, role)) default_impalad = pytest.config.option.impalad.split(',')[0] impalad_host = default_impalad.split(':')[0] impalad_hs2_port = pytest.config.option.impalad_hs2_port with impala_connect(host=impalad_host, port=impalad_hs2_port) as conn: cursor = conn.cursor() cursor.execute('show grant user %s on database %s' % (getuser(), unique_database)) rows = cursor.fetchall() assert len(rows) == 1 cols = rows[0] assert len(cols) == 10 assert cols[0] == 'USER' # principal_type assert cols[1] == getuser() # principal_name assert cols[2] == 'database' # scope assert cols[3] == unique_database # database assert cols[4] == '' # table assert cols[5] == '' # column assert cols[6] == '' # uri assert cols[7] == 'owner' # privilege assert cols[8] # grant_option # We don't assert create_time since the value can be None or str depending on the # Sentry refresh. cursor.execute('show grant role %s on database %s' % (role, unique_database)) rows = cursor.fetchall() assert len(rows) == 1 cols = rows[0] assert len(cols) == 8 assert cols[0] == 'database' # scope assert cols[1] == unique_database # database assert cols[2] == '' # table assert cols[3] == '' # column assert cols[4] == '' # uri assert cols[5] == 'all' # privilege assert not cols[6] # grant_option
def __auto_closed_conn(db_name=None, timeout=DEFAULT_CONN_TIMEOUT): """Returns a connection to Impala. This is intended to be used in a "with" block. The connection will be closed upon exiting the block. The returned connection will have a 'db_name' property. """ default_impalad = pytest.config.option.impalad.split(',')[0] impalad_host = default_impalad.split(':')[0] hs2_port = pytest.config.option.impalad_hs2_port conn = impala_connect(host=impalad_host, port=hs2_port, database=db_name, timeout=timeout) try: conn.db_name = db_name yield conn finally: try: conn.close() except Exception as e: LOG.warn("Error closing Impala connection: %s", e)
def init_impala_connect(self): """ Connect to Cloudera Impala """ self.impala_connection = impala_connect(**self.connect_params) self.impala_cursor = None
def __init__(self, host, port, database_name): self._connection = impala_connect(host, port) self._database_name = database_name self._staging_exec_result_table = None self._staging_profile_table = None self._cache = {}
def create_connection(self, db_name=None): if self.db_type == HIVE: connection_class = HiveDbConnection connection = impala_connect( host=self.host_name, port=self.port, user=self.user_name, password=self.password, timeout=maxint, auth_mechanism='PLAIN') return HiveDbConnection(self, connection, user_name=self.user_name, user_pass=self.password, db_name=db_name, hdfs_host=self.hdfs_host, hdfs_port=self.hdfs_port) elif self.db_type == IMPALA: connection_class = ImpalaDbConnection connection = impala_connect( host=self.host_name, port=self.port or 21050, timeout=maxint) elif self.db_type == ORACLE: connection_class = OracleDbConnection connection_str = '%(user)s/%(password)s@%(host)s:%(port)s/%(service)s' connection_args = { 'user': self.user_name or 'system', 'password': self.password or 'oracle', 'host': self.host_name or 'localhost', 'port': self.port or 1521, 'service': self.service or 'XE'} try: from cx_Oracle import connect as oracle_connect except: print('Error importing cx_Oracle. Please make sure it is installed. ' 'See the README for details.') raise connection = oracle_connect(connection_str % connection_args) connection.outputtypehandler = OracleDbConnection.type_converter connection.autocommit = True elif self.db_type == POSTGRESQL: connection_class = PostgresqlDbConnection connection_args = {'user': self.user_name or 'postgres'} if self.password: connection_args['password'] = self.password if db_name: connection_args['database'] = db_name if self.host_name: connection_args['host'] = self.host_name if self.port: connection_args['port'] = self.port connection = postgresql_connect(**connection_args) connection.autocommit = True elif self.db_type == MYSQL: connection_class = MySQLDbConnection connection_args = {'user': self.user_name or 'root'} if self.password: connection_args['passwd'] = self.password if db_name: connection_args['db'] = db_name if self.host_name: connection_args['host'] = self.host_name if self.port: connection_args['port'] = self.port try: from MySQLdb import connect as mysql_connect except: print('Error importing MySQLdb. Please make sure it is installed. ' 'See the README for details.') raise connection = mysql_connect(**connection_args) else: raise Exception('Unexpected database type: %s' % self.db_type) return connection_class(self, connection, db_name=db_name)
def connect(): global connection, cursor connection = impala_connect(host="localhost", port=21050, database="cass") cursor = connection.cursor() cursor.execute('use cass')
def get_connection(self, **extra_args): connect_args = self.connect_args connect_args.update(extra_args) impala_conn = impala_connect(**connect_args) return impala_thrift_connection(impala_conn)