def test_create_athena_engine(self): interpreter = { 'name': 'hive', 'options': { 'url': 'awsathena+rest://XXXXXXXXXXXXXXX:[email protected]:443/default?' 's3_staging_dir=s3://gethue-athena/scratch' } } with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: SqlAlchemyApi(self.user, interpreter)._create_engine()
def test_get_sample_data_table(self): snippet = Mock() with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data: with patch('notebook.connectors.sql_alchemy.inspect') as inspect: get_sample_data.return_value = (['col1'], [[1], [2]]) response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet, database='database1', table='table1') assert_equal(response['rows'], [[1], [2]])
def test_dialect_trim_statement_semicolon(self): interpreter = { 'name': 'presto', 'options': { 'url': 'presto://hue:8080/hue', 'session': {}, }, 'dialect_properties': {}, } with patch( 'notebook.connectors.sql_alchemy.SqlAlchemyApi._create_connection' ) as _create_connection: with patch( 'notebook.connectors.sql_alchemy.SqlAlchemyApi._create_engine' ) as _create_engine: with patch( 'notebook.connectors.sql_alchemy.SqlAlchemyApi._get_session' ) as _get_session: execute = Mock(return_value=Mock(cursor=None)) _create_connection.return_value = Mock(execute=execute) notebook = {} snippet = {'statement': 'SELECT 1;'} # Trim engine = SqlAlchemyApi(self.user, interpreter).execute( notebook, snippet) execute.assert_called_with('SELECT 1') # No Trim interpreter['options']['url'] = 'mysql://hue:3306/hue' interpreter['dialect_properties'][ 'trim_statement_semicolon'] = False interpreter['dialect_properties'][ 'sql_identifier_quote'] = '`' engine = SqlAlchemyApi(self.user, interpreter).execute( notebook, snippet) execute.assert_called_with('SELECT 1;')
def test_backticks_without_connectors(self): interpreter = {'name': 'hive', 'options': {'url': 'hive://'}} data = SqlAlchemyApi(self.user, interpreter).get_browse_query(snippet=Mock(), database='db1', table='table1') assert_equal(data, 'SELECT *\nFROM `db1`.`table1`\nLIMIT 1000\n') interpreter = { 'name': 'postgresql', 'options': { 'url': 'postgresql://' } } data = SqlAlchemyApi(self.user, interpreter).get_browse_query(snippet=Mock(), database='db1', table='table1') assert_equal(data, 'SELECT *\nFROM "db1"."table1"\nLIMIT 1000\n')
def test_create_connection(self): interpreter = { 'name': 'hive', 'options': { 'url': 'mysql://${USER}:${PASSWORD}@hue:3306/hue', 'session': { 'properties': [{ 'name': 'user', 'value': 'test_user' }, { 'name': 'password', 'value': 'test_pass' }] } } } with patch('notebook.connectors.sql_alchemy.create_engine' ) as create_engine: engine = SqlAlchemyApi(self.user, interpreter)._create_engine() SqlAlchemyApi(self.user, interpreter)._create_connection(engine)
def test_create_engine_with_impersonation(self): interpreter = { 'name': 'hive', 'options': { 'url': 'presto://*****:*****@hue:8080/hue')
def test_empty_database_names(self): interpreter = { 'options': {'url': 'phoenix://'} } snippet = Mock() with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: with patch('notebook.connectors.sql_alchemy.inspect') as inspect: with patch('notebook.connectors.sql_alchemy.Assist') as Assist: Assist.return_value=Mock(get_databases=Mock(return_value=['SYSTEM', None])) data = SqlAlchemyApi(self.user, interpreter).autocomplete(snippet) assert_equal(data['databases'], ['SYSTEM', 'NULL'])
def test_get_sample_data(self): snippet = Mock() with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine: with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data: with patch('notebook.connectors.sql_alchemy.inspect') as inspect: get_sample_data.return_value = (['col1'], [[1], [2]]) response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet) assert_equal(response['rows'], [[1], [2]]) assert_equal( response['full_headers'], [{'name': 'col1', 'type': 'STRING_TYPE', 'comment': ''}] )
def test_get_column_type_name_complex(self): api = SqlAlchemyApi(self.user, self.interpreter) with patch('notebook.connectors.sql_alchemy.str') as str: str.side_effect = UnsupportedCompilationError(None, None) assert_equal(api._get_column_type_name({'type': VARCHAR}), 'varchar') # Not complex but not testable otherwise assert_equal(api._get_column_type_name({'type': NullType}), 'null') assert_equal(api._get_column_type_name({'type': ARRAY}), 'array') assert_equal(api._get_column_type_name({'type': JSON}), 'json')
def test_get_log(self): notebook = Mock() snippet = MagicMock() with patch( 'notebook.connectors.sql_alchemy.CONNECTIONS') as CONNECTIONS: log = [ 'INFO : Compiling command(queryId=hive_20210217124246_d3ac774b-cdca-48d2-bfca-f23951ee2097): SELECT 1', 'INFO : Semantic Analysis Completed (retrial = false)', 'INFO : Completed compiling command(queryId=hive_20210217124246_d3ac774b-cdca-48d2-bfca-f23951ee2097); Time taken: 0.092 seconds', 'INFO : Completed executing command(queryId=hive_20210217124246_d3ac774b-cdca-48d2-bfca-f23951ee2097); Time taken: 0.006 seconds', 'INFO : OK' ] CONNECTIONS.get.return_value = {'logs': log} data = SqlAlchemyApi(self.user, self.interpreter).get_log(notebook, snippet) assert_equal(data, '\n'.join(log))
def test_columns_with_null_type(self): interpreter = {'name': 'hive', 'options': {'url': 'phoenix://'}} snippet = MagicMock() with patch('notebook.connectors.sql_alchemy.create_engine' ) as create_engine: with patch('notebook.connectors.sql_alchemy.inspect') as inspect: with patch('notebook.connectors.sql_alchemy.Assist') as Assist: def col1_dict(key): return { 'name': 'col1', 'type': 'string' }.get(key, Mock()) col1 = MagicMock() col1.__getitem__.side_effect = col1_dict col1.get = col1_dict def col2_dict(key): return { 'name': 'col2', 'type': NullType() }.get(key, Mock()) col2 = MagicMock() col2.__getitem__.side_effect = col2_dict col2.get = col2_dict Assist.return_value = Mock( get_columns=Mock(return_value=[col1, col2]), get_keys=Mock(return_value={})) data = SqlAlchemyApi(self.user, interpreter).autocomplete( snippet, database='database', table='table') assert_equal(data['columns'], ['col1', 'col2']) assert_equal( [col['type'] for col in data['extended_columns']], ['string', 'null'])
def test_fetch_result_rows(self): interpreter = { 'options': { 'url': 'mysql://*****:*****@hue:3306/hue' } } notebook = Mock() snippet = {'result': {'handle': {'guid': 'guid-1'}}} rows = 10 start_over = True with patch('notebook.connectors.sql_alchemy.CONNECTION_CACHE' ) as CONNECTION_CACHE: CONNECTION_CACHE.get = Mock( return_value={ 'result': Mock(fetchmany=Mock(return_value=[['row1'], ['row2'] ]) # We have 2 rows ), 'meta': MagicMock(__getitem__=Mock( return_value={'type': 'BIGINT_TYPE'}), return_value=[{ 'type': 'BIGINT_TYPE' }]) }) data = SqlAlchemyApi(self.user, interpreter).fetch_result( notebook, snippet, rows, start_over) assert_false(data['has_more']) assert_not_equal(data['has_more'], []) assert_equal(data['has_more'], False) assert_equal(data['data'], [['row1'], ['row2']]) assert_equal(data['meta'](), [{'type': 'BIGINT_TYPE'}])
def test_column_backticks_escaping(self): interpreter = {'name': 'hive', 'options': {'url': 'mysql://'}} assert_equal(SqlAlchemyApi(self.user, interpreter).backticks, '`') interpreter = {'name': 'hive', 'options': {'url': 'postgresql://'}} assert_equal(SqlAlchemyApi(self.user, interpreter).backticks, '"')
def get_api(request, snippet): from notebook.connectors.oozie_batch import OozieApi if snippet.get('wasBatchExecuted') and not TASK_SERVER.ENABLED.get(): return OozieApi(user=request.user, request=request) if snippet.get('type') == 'report': snippet['type'] = 'impala' patch_snippet_for_connector(snippet) connector_name = snippet['type'] if has_connectors() and snippet.get('type') == 'hello' and is_admin( request.user): interpreter = snippet.get('interpreter') else: interpreter = get_interpreter(connector_type=connector_name, user=request.user) interface = interpreter['interface'] if get_cluster_config(request.user)['has_computes']: compute = json.loads(request.POST.get( 'cluster', '""')) # Via Catalog autocomplete API or Notebook create sessions. if compute == '""' or compute == 'undefined': compute = None if not compute and snippet.get('compute'): # Via notebook.ko.js interpreter['compute'] = snippet['compute'] LOG.debug('Selected interpreter %s interface=%s compute=%s' % (interpreter['type'], interface, interpreter.get('compute') and interpreter['compute']['name'])) if interface == 'hiveserver2' or interface == 'hms': from notebook.connectors.hiveserver2 import HS2Api return HS2Api(user=request.user, request=request, interpreter=interpreter) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': from notebook.connectors.spark_shell import SparkApi return SparkApi(request.user, interpreter=interpreter) elif interface == 'livy-batch': from notebook.connectors.spark_batch import SparkBatchApi return SparkBatchApi(request.user, interpreter=interpreter) elif interface == 'text' or interface == 'markdown': from notebook.connectors.text import TextApi return TextApi(request.user) elif interface == 'rdbms': from notebook.connectors.rdbms import RdbmsApi return RdbmsApi(request.user, interpreter=snippet['type'], query_server=snippet.get('query_server')) elif interface == 'jdbc': if interpreter['options'] and interpreter['options'].get( 'url', '').find('teradata') >= 0: from notebook.connectors.jdbc_teradata import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) if interpreter['options'] and interpreter['options'].get( 'url', '').find('awsathena') >= 0: from notebook.connectors.jdbc_athena import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('presto') >= 0: from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('clickhouse') >= 0: from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse return JdbcApiClickhouse(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('vertica') >= 0: from notebook.connectors.jdbc_vertica import JdbcApiVertica return JdbcApiVertica(request.user, interpreter=interpreter) else: from notebook.connectors.jdbc import JdbcApi return JdbcApi(request.user, interpreter=interpreter) elif interface == 'teradata': from notebook.connectors.jdbc_teradata import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) elif interface == 'athena': from notebook.connectors.jdbc_athena import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interface == 'presto': from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interface == 'sqlalchemy': from notebook.connectors.sql_alchemy import SqlAlchemyApi return SqlAlchemyApi(request.user, interpreter=interpreter) elif interface == 'solr': from notebook.connectors.solr import SolrApi return SolrApi(request.user, interpreter=interpreter) elif interface == 'hbase': from notebook.connectors.hbase import HBaseApi return HBaseApi(request.user) elif interface == 'ksql': from notebook.connectors.ksql import KSqlApi return KSqlApi(request.user, interpreter=interpreter) elif interface == 'flink': from notebook.connectors.flink_sql import FlinkSqlApi return FlinkSqlApi(request.user, interpreter=interpreter) elif interface == 'kafka': from notebook.connectors.kafka import KafkaApi return KafkaApi(request.user) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException( _('Notebook connector interface not recognized: %s') % interface)
def get_api(request, snippet): from notebook.connectors.oozie_batch import OozieApi if snippet.get('wasBatchExecuted') and not TASK_SERVER.ENABLED.get(): return OozieApi(user=request.user, request=request) if snippet['type'] == 'report': snippet['type'] = 'impala' interpreter = [ interpreter for interpreter in get_ordered_interpreters(request.user) if snippet['type'] in (interpreter['type'], interpreter['interface']) ] if not interpreter: if snippet['type'] == 'hbase': interpreter = [{ 'name': 'hbase', 'type': 'hbase', 'interface': 'hbase', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'kafka': interpreter = [{ 'name': 'kafka', 'type': 'kafka', 'interface': 'kafka', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'solr': interpreter = [{ 'name': 'solr', 'type': 'solr', 'interface': 'solr', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'custom': interpreter = [{ 'name': snippet['name'], 'type': snippet['type'], 'interface': snippet['interface'], 'options': snippet.get('options', {}), 'is_sql': False }] else: raise PopupException(_('Snippet type %(type)s is not configured.') % snippet) interpreter = interpreter[0] interface = interpreter['interface'] # TODO: Multi cluster --> multi computes of a connector if has_connectors(): cluster = { 'connector': snippet['type'], 'id': interpreter['type'], } cluster.update(interpreter['options']) elif has_multi_cluster(): cluster = json.loads(request.POST.get('cluster', '""')) # Via Catalog autocomplete API or Notebook create sessions if cluster == '""' or cluster == 'undefined': cluster = None if not cluster and snippet.get('compute'): # Via notebook.ko.js cluster = snippet['compute'] else: cluster = None cluster_name = cluster.get('id') if cluster else None if cluster and 'altus:dataware:k8s' in cluster_name: interface = 'hiveserver2' elif cluster and 'crn:altus:dataware:' in cluster_name: interface = 'altus-adb' elif cluster and 'crn:altus:dataeng:' in cluster_name: interface = 'dataeng' LOG.debug('Selected connector %s %s interface=%s compute=%s' % (cluster_name, cluster, interface, snippet.get('compute'))) snippet['interface'] = interface if interface.startswith('hiveserver2') or interface == 'hms': from notebook.connectors.hiveserver2 import HS2Api return HS2Api(user=request.user, request=request, cluster=cluster, interface=interface) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': from notebook.connectors.spark_shell import SparkApi return SparkApi(request.user) elif interface == 'livy-batch': from notebook.connectors.spark_batch import SparkBatchApi return SparkBatchApi(request.user) elif interface == 'text' or interface == 'markdown': from notebook.connectors.text import TextApi return TextApi(request.user) elif interface == 'rdbms': from notebook.connectors.rdbms import RdbmsApi return RdbmsApi(request.user, interpreter=snippet['type'], query_server=snippet.get('query_server')) elif interface == 'altus-adb': from notebook.connectors.altus_adb import AltusAdbApi return AltusAdbApi(user=request.user, cluster_name=cluster_name, request=request) elif interface == 'dataeng': from notebook.connectors.dataeng import DataEngApi return DataEngApi(user=request.user, request=request, cluster_name=cluster_name) elif interface == 'jdbc': if interpreter['options'] and interpreter['options'].get('url', '').find('teradata') >= 0: from notebook.connectors.jdbc_teradata import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) if interpreter['options'] and interpreter['options'].get('url', '').find('awsathena') >= 0: from notebook.connectors.jdbc_athena import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get('url', '').find('presto') >= 0: from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get('url', '').find('clickhouse') >= 0: from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse return JdbcApiClickhouse(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get('url', '').find('vertica') >= 0: from notebook.connectors.jdbc_vertica import JdbcApiVertica return JdbcApiVertica(request.user, interpreter=interpreter) else: from notebook.connectors.jdbc import JdbcApi return JdbcApi(request.user, interpreter=interpreter) elif interface == 'teradata': from notebook.connectors.jdbc import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) elif interface == 'athena': from notebook.connectors.jdbc import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interface == 'presto': from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interface == 'sqlalchemy': from notebook.connectors.sql_alchemy import SqlAlchemyApi return SqlAlchemyApi(request.user, interpreter=interpreter) elif interface == 'solr': from notebook.connectors.solr import SolrApi return SolrApi(request.user, interpreter=interpreter) elif interface == 'hbase': from notebook.connectors.hbase import HBaseApi return HBaseApi(request.user) elif interface == 'kafka': from notebook.connectors.kafka import KafkaApi return KafkaApi(request.user) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException(_('Notebook connector interface not recognized: %s') % interface)