def _get_handle(self, snippet):
snippet['result']['handle']['secret'], snippet['result']['handle'][
'guid'] = HiveServerQueryHandle.get_decoded(
snippet['result']['handle']['secret'],
snippet['result']['handle']['guid'])
for key in snippet['result']['handle'].keys():
if key not in ('log_context', 'secret', 'has_result_set',
'operation_type', 'modified_row_count', 'guid'):
snippet['result']['handle'].pop(key)
return HiveServerQueryHandle(**snippet['result']['handle'])
def _get_handle(self, snippet):
try:
snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
except KeyError:
raise Exception('Operation has no valid handle attached')
except binascii.Error:
LOG.warn('Handle already base 64 decoded')
for key in snippet['result']['handle'].keys():
if key not in ('log_context', 'secret', 'has_result_set', 'operation_type', 'modified_row_count', 'guid'):
snippet['result']['handle'].pop(key)
return HiveServerQueryHandle(**snippet['result']['handle'])
def _get_handle(self, snippet):
snippet["result"]["handle"]["secret"], snippet["result"]["handle"]["guid"] = HiveServerQueryHandle.get_decoded(
snippet["result"]["handle"]["secret"], snippet["result"]["handle"]["guid"]
)
snippet["result"]["handle"].pop("statement_id")
snippet["result"]["handle"].pop("has_more_statements")
return HiveServerQueryHandle(**snippet["result"]["handle"])
def open_session(self, user):
if self.hiveserver2_impersonation_enabled:
kerberos_principal_short_name = KERBEROS.HUE_PRINCIPAL.get().split(
'/', 1)[0]
kwargs = {
'username': kerberos_principal_short_name,
'configuration': {
'hive.server2.proxy.user': user.username
}
}
else:
kwargs = {'username': user.username, 'configuration': {}}
req = TOpenSessionReq(**kwargs)
res = self._client.OpenSession(req)
sessionId = res.sessionHandle.sessionId
LOG.info('Opening session %s' % sessionId)
encoded_status, encoded_guid = HiveServerQueryHandle(
secret=sessionId.secret, guid=sessionId.guid).get()
return Session.objects.create(
owner=user,
application=self.query_server['server_name'],
status_code=res.status.statusCode,
secret=encoded_status,
guid=encoded_guid,
server_protocol_version=res.serverProtocolVersion)
def open_session(self, user):
kwargs = {
'username': user.
username, # If SASL, it gets the username from the authentication mechanism" since it dependents on it.
'configuration': {},
}
if self.impersonation_enabled:
kwargs.update({'username': '******'})
if self.query_server['server_name'] == 'impala':
kwargs['configuration'].update(
{'impala.doas.user': user.username})
else:
kwargs['configuration'].update(
{'hive.server2.proxy.user': user.username})
req = TOpenSessionReq(**kwargs)
res = self._client.OpenSession(req)
sessionId = res.sessionHandle.sessionId
LOG.info('Opening session %s' % sessionId)
encoded_status, encoded_guid = HiveServerQueryHandle(
secret=sessionId.secret, guid=sessionId.guid).get()
return Session.objects.create(
owner=user,
application=self.query_server['server_name'],
status_code=res.status.statusCode,
secret=encoded_status,
guid=encoded_guid,
server_protocol_version=res.serverProtocolVersion)
def _get_handle(self, snippet):
try:
snippet['result']['handle']['secret'], snippet['result']['handle'][
'guid'] = HiveServerQueryHandle.get_decoded(
snippet['result']['handle']['secret'],
snippet['result']['handle']['guid'])
except KeyError, ex:
raise Exception('Operation has no valid handle attached')
def _get_handle(self, snippet):
snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
for key in snippet['result']['handle'].keys():
if key not in ('log_context', 'secret', 'has_result_set', 'operation_type', 'modified_row_count', 'guid'):
snippet['result']['handle'].pop(key)
return HiveServerQueryHandle(**snippet['result']['handle'])
def execute_async_statement(self, statement, confOverlay):
req = TExecuteStatementReq(statement=statement.encode('utf-8'), confOverlay=confOverlay, runAsync=True)
res = self.call(self._client.ExecuteStatement, req)
return HiveServerQueryHandle(secret=res.operationHandle.operationId.secret,
guid=res.operationHandle.operationId.guid,
operation_type=res.operationHandle.operationType,
has_result_set=res.operationHandle.hasResultSet,
modified_row_count=res.operationHandle.modifiedRowCount)
def open_session(self, user):
self.user = user
kwargs = {
'client_protocol': beeswax_conf.THRIFT_VERSION.get() - 1,
'username': user.username, # If SASL or LDAP, it gets the username from the authentication mechanism" since it dependents on it.
'configuration': {},
}
if self.impersonation_enabled:
kwargs.update({'username': DEFAULT_USER})
if self.query_server['server_name'] == 'impala': # Only when Impala accepts it
kwargs['configuration'].update({'impala.doas.user': user.username})
if self.query_server['server_name'] == 'beeswax': # All the time
kwargs['configuration'].update({'hive.server2.proxy.user': user.username})
if self.query_server['server_name'] == 'sparksql': # All the time
kwargs['configuration'].update({'hive.server2.proxy.user': user.username})
if self.query_server['server_name'] == 'impala' and self.query_server['SESSION_TIMEOUT_S'] > 0:
kwargs['configuration'].update({'idle_session_timeout': str(self.query_server['SESSION_TIMEOUT_S'])})
LOG.info('Opening %s thrift session for user %s' % (self.query_server['server_name'], user.username))
req = TOpenSessionReq(**kwargs)
res = self._client.OpenSession(req)
self.coordinator_host = self._client.get_coordinator_host()
if res.status is not None and res.status.statusCode not in (TStatusCode.SUCCESS_STATUS,):
if hasattr(res.status, 'errorMessage') and res.status.errorMessage:
message = res.status.errorMessage
else:
message = ''
raise QueryServerException(Exception('Bad status for request %s:\n%s' % (req, res)), message=message)
sessionId = res.sessionHandle.sessionId
LOG.info('Session %s opened' % repr(sessionId.guid))
encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
properties = json.dumps(res.configuration)
session = Session.objects.create(owner=user,
application=self.query_server['server_name'],
status_code=res.status.statusCode,
secret=encoded_status,
guid=encoded_guid,
server_protocol_version=res.serverProtocolVersion,
properties=properties)
# HS2 does not return properties in TOpenSessionResp
if not session.get_properties():
session.properties = json.dumps(self.get_configuration())
session.save()
return session
def test_open_session(self):
query = Mock(
get_query_statement=Mock(return_value=['SELECT 1']),
settings=[]
)
with patch('beeswax.server.hive_server2_lib.thrift_util.get_client') as get_client:
original_secret = b's\xb6\x0ePP\xbdL\x17\xa3\x0f\\\xf7K\xe8Y\x1d'
original_guid = b'\xd9\xe0hT\xd6wO\xe1\xa3S\xfb\x04\xca\x93V\x01'
get_client.return_value = Mock(
OpenSession=Mock(
return_value=Mock(
status=Mock(
statusCode=TStatusCode.SUCCESS_STATUS
),
configuration={},
sessionHandle=Mock(
sessionId=Mock(
secret=original_secret,
guid=original_guid
)
),
serverProtocolVersion=11
)
),
get_coordinator_host=Mock(return_value='hive-host')
)
session_count = Session.objects.filter(owner=self.user, application=self.query_server['server_name']).count()
# Send open session
session = HiveServerClient(self.query_server, self.user).open_session(self.user)
assert_equal(
session_count + 1, # +1 as setUp resets the user which deletes cascade the sessions
Session.objects.filter(owner=self.user, application=self.query_server['server_name']).count()
)
session = Session.objects.get_session(self.user, self.query_server['server_name'])
secret, guid = session.get_adjusted_guid_secret()
secret, guid = HiveServerQueryHandle.get_decoded(secret, guid)
assert_equal(
original_secret,
secret
)
assert_equal(
original_guid,
guid
)
handle = session.get_handle()
assert_equal(
original_secret,
handle.sessionId.secret
)
assert_equal(
original_guid,
handle.sessionId.guid
)
def _get_handle(self, snippet):
try:
snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
except KeyError:
raise Exception('Operation has no valid handle attached')
for key in snippet['result']['handle'].keys():
if key not in ('log_context', 'secret', 'has_result_set', 'operation_type', 'modified_row_count', 'guid'):
snippet['result']['handle'].pop(key)
return HiveServerQueryHandle(**snippet['result']['handle'])
def execute_async_statement(self, statement, confOverlay):
if self.query_server['server_name'] == 'impala' and self.query_server['QUERY_TIMEOUT_S'] > 0:
confOverlay['QUERY_TIMEOUT_S'] = str(self.query_server['QUERY_TIMEOUT_S'])
req = TExecuteStatementReq(statement=statement.encode('utf-8'), confOverlay=confOverlay, runAsync=True)
res = self.call(self._client.ExecuteStatement, req)
return HiveServerQueryHandle(secret=res.operationHandle.operationId.secret,
guid=res.operationHandle.operationId.guid,
operation_type=res.operationHandle.operationType,
has_result_set=res.operationHandle.hasResultSet,
modified_row_count=res.operationHandle.modifiedRowCount)
def open_session(self, user):
kwargs = {
'client_protocol': 4, ##TODO: support latest column protocol
'username': user.
username, # If SASL, it gets the username from the authentication mechanism" since it dependents on it.
'configuration': {},
}
if self.impersonation_enabled:
kwargs.update({'username': '******'})
if self.query_server[
'server_name'] == 'impala': # Only when Impala accepts it
kwargs['configuration'].update(
{'impala.doas.user': user.username})
if self.query_server['server_name'] == 'beeswax': # All the time
kwargs['configuration'].update(
{'hive.server2.proxy.user': user.username})
if LDAP_PASSWORD.get(
): # HiveServer2 supports pass-through LDAP authentication.
kwargs['username'] = LDAP_USERNAME.get()
kwargs['password'] = LDAP_PASSWORD.get()
req = TOpenSessionReq(**kwargs)
res = self._client.OpenSession(req)
if res.status is not None and res.status.statusCode not in (
TStatusCode.SUCCESS_STATUS, ):
if hasattr(res.status, 'errorMessage') and res.status.errorMessage:
message = res.status.errorMessage
else:
message = ''
raise QueryServerException(Exception(
'Bad status for request %s:\n%s' % (req, res)),
message=message)
sessionId = res.sessionHandle.sessionId
LOG.info('Opening session %s' % sessionId)
encoded_status, encoded_guid = HiveServerQueryHandle(
secret=sessionId.secret, guid=sessionId.guid).get()
return Session.objects.create(
owner=user,
application=self.query_server['server_name'],
status_code=res.status.statusCode,
secret=encoded_status,
guid=encoded_guid,
server_protocol_version=res.serverProtocolVersion)
def open_session(self, user):
req = TOpenSessionReq(username=user.username)
res = self._client.OpenSession(req)
sessionId = res.sessionHandle.sessionId
encoded_status, encoded_guid = HiveServerQueryHandle(
secret=sessionId.secret, guid=sessionId.guid).get()
return Session.objects.create(
owner=user,
status_code=res.status.statusCode,
secret=encoded_status,
guid=encoded_guid,
server_protocol_version=res.serverProtocolVersion)
def open_session(self, user):
req = TOpenSessionReq(username=user.username, configuration={})
res = self._client.OpenSession(req)
sessionId = res.sessionHandle.sessionId
LOG.info('Opening session %s' % sessionId)
encoded_status, encoded_guid = HiveServerQueryHandle(secret=sessionId.secret, guid=sessionId.guid).get()
return Session.objects.create(owner=user,
application=self.query_server['server_name'],
status_code=res.status.statusCode,
secret=encoded_status,
guid=encoded_guid,
server_protocol_version=res.serverProtocolVersion)
def execute_async_statement(self, statement, confOverlay, with_multiple_session=False):
if self.query_server['server_name'].startswith('impala') and self.query_server['QUERY_TIMEOUT_S'] > 0:
confOverlay['QUERY_TIMEOUT_S'] = str(self.query_server['QUERY_TIMEOUT_S'])
req = TExecuteStatementReq(statement=statement.encode('utf-8'), confOverlay=confOverlay, runAsync=True)
(res, session) = self.call_return_result_and_session(self._client.ExecuteStatement, req, with_multiple_session=with_multiple_session)
return HiveServerQueryHandle(
secret=res.operationHandle.operationId.secret,
guid=res.operationHandle.operationId.guid,
operation_type=res.operationHandle.operationType,
has_result_set=res.operationHandle.hasResultSet,
modified_row_count=res.operationHandle.modifiedRowCount,
session_guid=session.guid
)
def query(self, query, statement=0, with_multiple_session=False):
return HiveServerQueryHandle(secret='mock', guid='mock')
def _get_handle(self, snippet):
snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
snippet['result']['handle'].pop('statement_id')
snippet['result']['handle'].pop('has_more_statements')
return HiveServerQueryHandle(**snippet['result']['handle'])
def _get_handle(self, snippet):
snippet['result']['handle']['secret'], snippet['result']['handle'][
'guid'] = HiveServerQueryHandle.get_decoded(
snippet['result']['handle']['secret'],
snippet['result']['handle']['guid'])
return HiveServerQueryHandle(**snippet['result']['handle'])
def _get_handle(self, snippet):
try:
snippet['result']['handle']['secret'], snippet['result']['handle'][
'guid'] = HiveServerQueryHandle.get_decoded(
snippet['result']['handle']['secret'],
snippet['result']['handle']['guid'])
except KeyError, ex:
raise Exception('Operation has no valid handle attached')
for key in snippet['result']['handle'].keys():
if key not in ('log_context', 'secret', 'has_result_set',
'operation_type', 'modified_row_count', 'guid'):
snippet['result']['handle'].pop(key)
return HiveServerQueryHandle(**snippet['result']['handle'])
def _get_db(self, snippet):
if snippet['type'] == 'hive':
name = 'beeswax'
elif snippet['type'] == 'impala':
name = 'impala'
else:
name = 'sparksql'
return dbms.get(self.user,
query_server=get_query_server_config(name=name))
def _parse_job_counters(self, job_id):
# Attempt to fetch total records from the job's Hive counter
total_records, total_size = None, None
def _get_handle(self, snippet):
snippet["result"]["handle"]["secret"], snippet["result"]["handle"]["guid"] = HiveServerQueryHandle.get_decoded(
snippet["result"]["handle"]["secret"], snippet["result"]["handle"]["guid"]
)
return HiveServerQueryHandle(**snippet["result"]["handle"])
def _get_handle(self, snippet): snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid']) return HiveServerQueryHandle(**snippet['result']['handle'])
def _get_handle(self, snippet):
snippet['result']['handle']['secret'], snippet['result']['handle']['guid'] = HiveServerQueryHandle.get_decoded(snippet['result']['handle']['secret'], snippet['result']['handle']['guid'])
snippet['result']['handle'].pop('statement_id')
snippet['result']['handle'].pop('has_more_statements')
snippet['result']['handle'].pop('statements_count')
return HiveServerQueryHandle(**snippet['result']['handle'])
def query(self, query, statement=0):
return HiveServerQueryHandle(secret='mock', guid='mock')
