def test_create_athena_engine(self):
interpreter = {
'name': 'hive',
'options': {
'url': 'awsathena+rest://XXXXXXXXXXXXXXX:[email protected]:443/default?'
's3_staging_dir=s3://gethue-athena/scratch'
}
}
with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine:
SqlAlchemyApi(self.user, interpreter)._create_engine()
def test_get_sample_data_table(self):
snippet = Mock()
with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine:
with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data:
with patch('notebook.connectors.sql_alchemy.inspect') as inspect:
get_sample_data.return_value = (['col1'], [[1], [2]])
response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet, database='database1', table='table1')
assert_equal(response['rows'], [[1], [2]])
def test_dialect_trim_statement_semicolon(self):
interpreter = {
'name': 'presto',
'options': {
'url': 'presto://hue:8080/hue',
'session': {},
},
'dialect_properties': {},
}
with patch(
'notebook.connectors.sql_alchemy.SqlAlchemyApi._create_connection'
) as _create_connection:
with patch(
'notebook.connectors.sql_alchemy.SqlAlchemyApi._create_engine'
) as _create_engine:
with patch(
'notebook.connectors.sql_alchemy.SqlAlchemyApi._get_session'
) as _get_session:
execute = Mock(return_value=Mock(cursor=None))
_create_connection.return_value = Mock(execute=execute)
notebook = {}
snippet = {'statement': 'SELECT 1;'}
# Trim
engine = SqlAlchemyApi(self.user, interpreter).execute(
notebook, snippet)
execute.assert_called_with('SELECT 1')
# No Trim
interpreter['options']['url'] = 'mysql://hue:3306/hue'
interpreter['dialect_properties'][
'trim_statement_semicolon'] = False
interpreter['dialect_properties'][
'sql_identifier_quote'] = '`'
engine = SqlAlchemyApi(self.user, interpreter).execute(
notebook, snippet)
execute.assert_called_with('SELECT 1;')
def test_backticks_without_connectors(self):
interpreter = {'name': 'hive', 'options': {'url': 'hive://'}}
data = SqlAlchemyApi(self.user,
interpreter).get_browse_query(snippet=Mock(),
database='db1',
table='table1')
assert_equal(data, 'SELECT *\nFROM `db1`.`table1`\nLIMIT 1000\n')
interpreter = {
'name': 'postgresql',
'options': {
'url': 'postgresql://'
}
}
data = SqlAlchemyApi(self.user,
interpreter).get_browse_query(snippet=Mock(),
database='db1',
table='table1')
assert_equal(data, 'SELECT *\nFROM "db1"."table1"\nLIMIT 1000\n')
def test_create_connection(self):
interpreter = {
'name': 'hive',
'options': {
'url': 'mysql://${USER}:${PASSWORD}@hue:3306/hue',
'session': {
'properties': [{
'name': 'user',
'value': 'test_user'
}, {
'name': 'password',
'value': 'test_pass'
}]
}
}
}
with patch('notebook.connectors.sql_alchemy.create_engine'
) as create_engine:
engine = SqlAlchemyApi(self.user, interpreter)._create_engine()
SqlAlchemyApi(self.user, interpreter)._create_connection(engine)
def test_create_engine_with_impersonation(self):
interpreter = {
'name': 'hive',
'options': {
'url': 'presto://*****:*****@hue:8080/hue')
def test_empty_database_names(self):
interpreter = {
'options': {'url': 'phoenix://'}
}
snippet = Mock()
with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine:
with patch('notebook.connectors.sql_alchemy.inspect') as inspect:
with patch('notebook.connectors.sql_alchemy.Assist') as Assist:
Assist.return_value=Mock(get_databases=Mock(return_value=['SYSTEM', None]))
data = SqlAlchemyApi(self.user, interpreter).autocomplete(snippet)
assert_equal(data['databases'], ['SYSTEM', 'NULL'])
def test_get_sample_data(self):
snippet = Mock()
with patch('notebook.connectors.sql_alchemy.create_engine') as create_engine:
with patch('notebook.connectors.sql_alchemy.Assist.get_sample_data') as get_sample_data:
with patch('notebook.connectors.sql_alchemy.inspect') as inspect:
get_sample_data.return_value = (['col1'], [[1], [2]])
response = SqlAlchemyApi(self.user, self.interpreter).get_sample_data(snippet)
assert_equal(response['rows'], [[1], [2]])
assert_equal(
response['full_headers'],
[{'name': 'col1', 'type': 'STRING_TYPE', 'comment': ''}]
)
def test_get_column_type_name_complex(self):
api = SqlAlchemyApi(self.user, self.interpreter)
with patch('notebook.connectors.sql_alchemy.str') as str:
str.side_effect = UnsupportedCompilationError(None, None)
assert_equal(api._get_column_type_name({'type': VARCHAR}), 'varchar') # Not complex but not testable otherwise
assert_equal(api._get_column_type_name({'type': NullType}), 'null')
assert_equal(api._get_column_type_name({'type': ARRAY}), 'array')
assert_equal(api._get_column_type_name({'type': JSON}), 'json')
def test_get_log(self):
notebook = Mock()
snippet = MagicMock()
with patch(
'notebook.connectors.sql_alchemy.CONNECTIONS') as CONNECTIONS:
log = [
'INFO : Compiling command(queryId=hive_20210217124246_d3ac774b-cdca-48d2-bfca-f23951ee2097): SELECT 1',
'INFO : Semantic Analysis Completed (retrial = false)',
'INFO : Completed compiling command(queryId=hive_20210217124246_d3ac774b-cdca-48d2-bfca-f23951ee2097); Time taken: 0.092 seconds',
'INFO : Completed executing command(queryId=hive_20210217124246_d3ac774b-cdca-48d2-bfca-f23951ee2097); Time taken: 0.006 seconds',
'INFO : OK'
]
CONNECTIONS.get.return_value = {'logs': log}
data = SqlAlchemyApi(self.user,
self.interpreter).get_log(notebook, snippet)
assert_equal(data, '\n'.join(log))
def test_columns_with_null_type(self):
interpreter = {'name': 'hive', 'options': {'url': 'phoenix://'}}
snippet = MagicMock()
with patch('notebook.connectors.sql_alchemy.create_engine'
) as create_engine:
with patch('notebook.connectors.sql_alchemy.inspect') as inspect:
with patch('notebook.connectors.sql_alchemy.Assist') as Assist:
def col1_dict(key):
return {
'name': 'col1',
'type': 'string'
}.get(key, Mock())
col1 = MagicMock()
col1.__getitem__.side_effect = col1_dict
col1.get = col1_dict
def col2_dict(key):
return {
'name': 'col2',
'type': NullType()
}.get(key, Mock())
col2 = MagicMock()
col2.__getitem__.side_effect = col2_dict
col2.get = col2_dict
Assist.return_value = Mock(
get_columns=Mock(return_value=[col1, col2]),
get_keys=Mock(return_value={}))
data = SqlAlchemyApi(self.user, interpreter).autocomplete(
snippet, database='database', table='table')
assert_equal(data['columns'], ['col1', 'col2'])
assert_equal(
[col['type'] for col in data['extended_columns']],
['string', 'null'])
def test_fetch_result_rows(self):
interpreter = {
'options': {
'url': 'mysql://*****:*****@hue:3306/hue'
}
}
notebook = Mock()
snippet = {'result': {'handle': {'guid': 'guid-1'}}}
rows = 10
start_over = True
with patch('notebook.connectors.sql_alchemy.CONNECTION_CACHE'
) as CONNECTION_CACHE:
CONNECTION_CACHE.get = Mock(
return_value={
'result':
Mock(fetchmany=Mock(return_value=[['row1'], ['row2']
]) # We have 2 rows
),
'meta':
MagicMock(__getitem__=Mock(
return_value={'type': 'BIGINT_TYPE'}),
return_value=[{
'type': 'BIGINT_TYPE'
}])
})
data = SqlAlchemyApi(self.user, interpreter).fetch_result(
notebook, snippet, rows, start_over)
assert_false(data['has_more'])
assert_not_equal(data['has_more'], [])
assert_equal(data['has_more'], False)
assert_equal(data['data'], [['row1'], ['row2']])
assert_equal(data['meta'](), [{'type': 'BIGINT_TYPE'}])
def test_column_backticks_escaping(self):
interpreter = {'name': 'hive', 'options': {'url': 'mysql://'}}
assert_equal(SqlAlchemyApi(self.user, interpreter).backticks, '`')
interpreter = {'name': 'hive', 'options': {'url': 'postgresql://'}}
assert_equal(SqlAlchemyApi(self.user, interpreter).backticks, '"')
def get_api(request, snippet):
from notebook.connectors.oozie_batch import OozieApi
if snippet.get('wasBatchExecuted') and not TASK_SERVER.ENABLED.get():
return OozieApi(user=request.user, request=request)
if snippet.get('type') == 'report':
snippet['type'] = 'impala'
patch_snippet_for_connector(snippet)
connector_name = snippet['type']
if has_connectors() and snippet.get('type') == 'hello' and is_admin(
request.user):
interpreter = snippet.get('interpreter')
else:
interpreter = get_interpreter(connector_type=connector_name,
user=request.user)
interface = interpreter['interface']
if get_cluster_config(request.user)['has_computes']:
compute = json.loads(request.POST.get(
'cluster',
'""')) # Via Catalog autocomplete API or Notebook create sessions.
if compute == '""' or compute == 'undefined':
compute = None
if not compute and snippet.get('compute'): # Via notebook.ko.js
interpreter['compute'] = snippet['compute']
LOG.debug('Selected interpreter %s interface=%s compute=%s' %
(interpreter['type'], interface, interpreter.get('compute')
and interpreter['compute']['name']))
if interface == 'hiveserver2' or interface == 'hms':
from notebook.connectors.hiveserver2 import HS2Api
return HS2Api(user=request.user,
request=request,
interpreter=interpreter)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
from notebook.connectors.spark_shell import SparkApi
return SparkApi(request.user, interpreter=interpreter)
elif interface == 'livy-batch':
from notebook.connectors.spark_batch import SparkBatchApi
return SparkBatchApi(request.user, interpreter=interpreter)
elif interface == 'text' or interface == 'markdown':
from notebook.connectors.text import TextApi
return TextApi(request.user)
elif interface == 'rdbms':
from notebook.connectors.rdbms import RdbmsApi
return RdbmsApi(request.user,
interpreter=snippet['type'],
query_server=snippet.get('query_server'))
elif interface == 'jdbc':
if interpreter['options'] and interpreter['options'].get(
'url', '').find('teradata') >= 0:
from notebook.connectors.jdbc_teradata import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
if interpreter['options'] and interpreter['options'].get(
'url', '').find('awsathena') >= 0:
from notebook.connectors.jdbc_athena import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('presto') >= 0:
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('clickhouse') >= 0:
from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse
return JdbcApiClickhouse(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('vertica') >= 0:
from notebook.connectors.jdbc_vertica import JdbcApiVertica
return JdbcApiVertica(request.user, interpreter=interpreter)
else:
from notebook.connectors.jdbc import JdbcApi
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'teradata':
from notebook.connectors.jdbc_teradata import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
elif interface == 'athena':
from notebook.connectors.jdbc_athena import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interface == 'presto':
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interface == 'sqlalchemy':
from notebook.connectors.sql_alchemy import SqlAlchemyApi
return SqlAlchemyApi(request.user, interpreter=interpreter)
elif interface == 'solr':
from notebook.connectors.solr import SolrApi
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'hbase':
from notebook.connectors.hbase import HBaseApi
return HBaseApi(request.user)
elif interface == 'ksql':
from notebook.connectors.ksql import KSqlApi
return KSqlApi(request.user, interpreter=interpreter)
elif interface == 'flink':
from notebook.connectors.flink_sql import FlinkSqlApi
return FlinkSqlApi(request.user, interpreter=interpreter)
elif interface == 'kafka':
from notebook.connectors.kafka import KafkaApi
return KafkaApi(request.user)
elif interface == 'pig':
return OozieApi(user=request.user,
request=request) # Backward compatibility until Hue 4
else:
raise PopupException(
_('Notebook connector interface not recognized: %s') % interface)
def get_api(request, snippet):
from notebook.connectors.oozie_batch import OozieApi
if snippet.get('wasBatchExecuted') and not TASK_SERVER.ENABLED.get():
return OozieApi(user=request.user, request=request)
if snippet['type'] == 'report':
snippet['type'] = 'impala'
interpreter = [
interpreter
for interpreter in get_ordered_interpreters(request.user) if snippet['type'] in (interpreter['type'], interpreter['interface'])
]
if not interpreter:
if snippet['type'] == 'hbase':
interpreter = [{
'name': 'hbase',
'type': 'hbase',
'interface': 'hbase',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'kafka':
interpreter = [{
'name': 'kafka',
'type': 'kafka',
'interface': 'kafka',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'solr':
interpreter = [{
'name': 'solr',
'type': 'solr',
'interface': 'solr',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'custom':
interpreter = [{
'name': snippet['name'],
'type': snippet['type'],
'interface': snippet['interface'],
'options': snippet.get('options', {}),
'is_sql': False
}]
else:
raise PopupException(_('Snippet type %(type)s is not configured.') % snippet)
interpreter = interpreter[0]
interface = interpreter['interface']
# TODO: Multi cluster --> multi computes of a connector
if has_connectors():
cluster = {
'connector': snippet['type'],
'id': interpreter['type'],
}
cluster.update(interpreter['options'])
elif has_multi_cluster():
cluster = json.loads(request.POST.get('cluster', '""')) # Via Catalog autocomplete API or Notebook create sessions
if cluster == '""' or cluster == 'undefined':
cluster = None
if not cluster and snippet.get('compute'): # Via notebook.ko.js
cluster = snippet['compute']
else:
cluster = None
cluster_name = cluster.get('id') if cluster else None
if cluster and 'altus:dataware:k8s' in cluster_name:
interface = 'hiveserver2'
elif cluster and 'crn:altus:dataware:' in cluster_name:
interface = 'altus-adb'
elif cluster and 'crn:altus:dataeng:' in cluster_name:
interface = 'dataeng'
LOG.debug('Selected connector %s %s interface=%s compute=%s' % (cluster_name, cluster, interface, snippet.get('compute')))
snippet['interface'] = interface
if interface.startswith('hiveserver2') or interface == 'hms':
from notebook.connectors.hiveserver2 import HS2Api
return HS2Api(user=request.user, request=request, cluster=cluster, interface=interface)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
from notebook.connectors.spark_shell import SparkApi
return SparkApi(request.user)
elif interface == 'livy-batch':
from notebook.connectors.spark_batch import SparkBatchApi
return SparkBatchApi(request.user)
elif interface == 'text' or interface == 'markdown':
from notebook.connectors.text import TextApi
return TextApi(request.user)
elif interface == 'rdbms':
from notebook.connectors.rdbms import RdbmsApi
return RdbmsApi(request.user, interpreter=snippet['type'], query_server=snippet.get('query_server'))
elif interface == 'altus-adb':
from notebook.connectors.altus_adb import AltusAdbApi
return AltusAdbApi(user=request.user, cluster_name=cluster_name, request=request)
elif interface == 'dataeng':
from notebook.connectors.dataeng import DataEngApi
return DataEngApi(user=request.user, request=request, cluster_name=cluster_name)
elif interface == 'jdbc':
if interpreter['options'] and interpreter['options'].get('url', '').find('teradata') >= 0:
from notebook.connectors.jdbc_teradata import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
if interpreter['options'] and interpreter['options'].get('url', '').find('awsathena') >= 0:
from notebook.connectors.jdbc_athena import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get('url', '').find('presto') >= 0:
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get('url', '').find('clickhouse') >= 0:
from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse
return JdbcApiClickhouse(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get('url', '').find('vertica') >= 0:
from notebook.connectors.jdbc_vertica import JdbcApiVertica
return JdbcApiVertica(request.user, interpreter=interpreter)
else:
from notebook.connectors.jdbc import JdbcApi
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'teradata':
from notebook.connectors.jdbc import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
elif interface == 'athena':
from notebook.connectors.jdbc import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interface == 'presto':
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interface == 'sqlalchemy':
from notebook.connectors.sql_alchemy import SqlAlchemyApi
return SqlAlchemyApi(request.user, interpreter=interpreter)
elif interface == 'solr':
from notebook.connectors.solr import SolrApi
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'hbase':
from notebook.connectors.hbase import HBaseApi
return HBaseApi(request.user)
elif interface == 'kafka':
from notebook.connectors.kafka import KafkaApi
return KafkaApi(request.user)
elif interface == 'pig':
return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4
else:
raise PopupException(_('Notebook connector interface not recognized: %s') % interface)
