def authorized_get_query_history(request,
query_history_id,
owner_only=False,
must_exist=False):
if query_history_id is None and not must_exist:
return None
try:
query_history = QueryHistory.get(id=query_history_id)
except QueryHistory.DoesNotExist:
if must_exist:
raise PopupException(
_('QueryHistory %(id)s does not exist.') %
{'id': query_history_id})
else:
return None
# Some queries don't have a design so are not linked to Document Model permission
if query_history.design is None or not query_history.design.doc.exists():
if not request.user.is_superuser and request.user != query_history.owner:
raise PopupException(
_('Permission denied to read QueryHistory %(id)s') %
{'id': query_history_id})
else:
query_history.design.doc.get().can_read_or_exception(request.user)
return query_history
def test_get_exec_summary(self):
query = """
SELECT COUNT(1) FROM tweets;
"""
response = _make_query(self.client,
query,
database=self.DATABASE,
local=False,
server_name='impala')
content = json.loads(response.content)
query_history = QueryHistory.get(content['id'])
wait_for_query_to_finish(self.client, response, max=180.0)
resp = self.client.post(
reverse('impala:get_exec_summary',
kwargs={'query_history_id': query_history.id}))
data = json.loads(resp.content)
assert_equal(0, data['status'], data)
assert_true('nodes' in data['summary'], data)
assert_true(
len(data['summary']['nodes']) > 0, data['summary']['nodes'])
# Attempt to call get_exec_summary on a closed query
resp = self.client.post(
reverse('impala:get_exec_summary',
kwargs={'query_history_id': query_history.id}))
data = json.loads(resp.content)
assert_equal(0, data['status'], data)
assert_true('nodes' in data['summary'], data)
assert_true(
len(data['summary']['nodes']) > 0, data['summary']['nodes'])
def handle(self, *args, **options):
days = int(args[0]) if len(args) >= 1 else 7
close_all = args[1] == 'all' if len(args) >= 2 else False
self.stdout.write(
'Closing (all=%s) HiveServer2 queries older than %s days...\n' %
(close_all, days))
queries = QueryHistory.objects.filter(last_state__in=[
QueryHistory.STATE.expired.value, QueryHistory.STATE.failed.value,
QueryHistory.STATE.available.value
])
if close_all:
queries = QueryHistory.objects.all()
queries = queries.filter(submission_date__lte=datetime.today() -
timedelta(days=days))
import os
import beeswax
from beeswax import conf
from beeswax import hive_site
try:
beeswax.conf.HIVE_CONF_DIR.set_for_testing(
os.environ['HIVE_CONF_DIR'])
except:
LOG.exception('failed to lookup HIVE_CONF_DIR in environment')
self.stdout.write('Did you export HIVE_CONF_DIR=/etc/hive/conf?\n')
raise
hive_site.reset()
hive_site.get_conf()
closed_queries = 0
already_closed_queries = 0
for query in queries:
try:
query_history = QueryHistory.get(id=query.id)
if query_history.server_id is not None:
handle = query_history.get_handle()
dbms.get(user=query_history.owner).close_operation(handle)
closed_queries += 1
else:
already_closed_queries += 1
query.last_state = QueryHistory.STATE.expired.value
query.save()
except Exception as e:
if 'None' in str(e) or 'Invalid OperationHandle' in str(e):
already_closed_queries += 1
query.last_state = QueryHistory.STATE.expired.value
query.save()
else:
self.stdout.write('Info: %s\n' % e)
self.stdout.write('%s queries closed. %s queries already closed.\n' %
(closed_queries, already_closed_queries))
def test_basic_flow(self):
dbs = self.db.get_databases()
assert_true('_impala_builtins' in dbs, dbs)
assert_true(self.DATABASE in dbs, dbs)
tables = self.db.get_tables(database=self.DATABASE)
assert_true('tweets' in tables, tables)
QUERY = """
SELECT * FROM tweets ORDER BY row_num;
"""
response = _make_query(self.client,
QUERY,
database=self.DATABASE,
local=False,
server_name='impala')
content = json.loads(response.content)
query_history = QueryHistory.get(content['id'])
response = wait_for_query_to_finish(self.client, response, max=180.0)
results = []
# Check that we multiple fetches get all the result set
while len(results) < 5:
# We get less than 5 results most of the time, so increase offset
content = fetch_query_result_data(self.client,
response,
n=len(results),
server_name='impala')
results += content['results']
assert_equal([1, 2, 3, 4, 5], [col[0] for col in results])
# Check start over
results_start_over = []
while len(results_start_over) < 5:
content = fetch_query_result_data(self.client,
response,
n=len(results_start_over),
server_name='impala')
results_start_over += content['results']
assert_equal(results_start_over, results)
# Check cancel query
resp = self.client.post(
reverse('impala:api_cancel_query',
kwargs={'query_history_id': query_history.id}))
content = json.loads(resp.content)
assert_equal(0, content['status'])
def authorized_get_history(request, query_history_id, owner_only=False, must_exist=False):
if query_history_id is None and not must_exist:
return None
try:
query_history = QueryHistory.get(id=query_history_id)
except QueryHistory.DoesNotExist:
if must_exist:
raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id})
else:
return None
query_history.design.doc.get().can_read_or_exception(request.user)
return query_history
def handle(self, *args, **options):
days = int(args[0]) if len(args) >= 1 else 7
close_all = args[1] == 'all' if len(args) >= 2 else False
self.stdout.write('Closing (all=%s) HiveServer2 queries older than %s days...\n' % (close_all, days))
queries = QueryHistory.objects.filter(last_state__in=[QueryHistory.STATE.expired.value, QueryHistory.STATE.failed.value, QueryHistory.STATE.available.value])
if close_all:
queries = QueryHistory.objects.all()
queries = queries.filter(submission_date__lte=datetime.today() - timedelta(days=days))
import os
import beeswax
from beeswax import conf
from beeswax import hive_site
try:
beeswax.conf.HIVE_CONF_DIR.set_for_testing(os.environ['HIVE_CONF_DIR'])
except:
LOG.exception('failed to lookup HIVE_CONF_DIR in environment')
self.stdout.write('Did you export HIVE_CONF_DIR=/etc/hive/conf?\n')
raise
hive_site.reset()
hive_site.get_conf()
closed_queries = 0
already_closed_queries = 0
for query in queries:
try:
query_history = QueryHistory.get(id=query.id)
if query_history.server_id is not None:
handle = query_history.get_handle()
dbms.get(user=query_history.owner).close_operation(handle)
closed_queries += 1
else:
already_closed_queries += 1
query.last_state = QueryHistory.STATE.expired.value
query.save()
except Exception, e:
if 'None' in str(e) or 'Invalid OperationHandle' in str(e):
already_closed_queries += 1
query.last_state = QueryHistory.STATE.expired.value
query.save()
else:
self.stdout.write('Info: %s\n' % e)
def test_get_runtime_profile(self):
query = """
SELECT COUNT(1) FROM tweets;
"""
response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala')
content = json.loads(response.content)
query_history = QueryHistory.get(content['id'])
wait_for_query_to_finish(self.client, response, max=180.0)
resp = self.client.post(reverse('impala:get_runtime_profile', kwargs={'query_history_id': query_history.id}))
data = json.loads(resp.content)
assert_equal(0, data['status'], data)
assert_true('Execution Profile' in data['profile'], data)
def test_get_runtime_profile(self):
query = """
SELECT COUNT(1) FROM tweets;
"""
response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala')
content = json.loads(response.content)
query_history = QueryHistory.get(content['id'])
wait_for_query_to_finish(self.client, response, max=180.0)
resp = self.client.post(reverse('impala:get_runtime_profile', kwargs={'query_history_id': query_history.id}))
data = json.loads(resp.content)
assert_equal(0, data['status'], data)
assert_true('Execution Profile' in data['profile'], data)
def authorized_get_query_history(request, query_history_id, owner_only=False, must_exist=False):
if query_history_id is None and not must_exist:
return None
try:
query_history = QueryHistory.get(id=query_history_id)
except QueryHistory.DoesNotExist:
if must_exist:
raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id})
else:
return None
# Some queries don't have a design so are not linked to Document Model permission
if query_history.design is None or not query_history.design.doc.exists():
if not request.user.is_superuser and request.user != query_history.owner:
raise PopupException(_('Permission denied to read QueryHistory %(id)s') % {'id': query_history_id})
else:
query_history.design.doc.get().can_read_or_exception(request.user)
return query_history
def test_basic_flow(self):
dbs = self.db.get_databases()
assert_true("_impala_builtins" in dbs, dbs)
assert_true(self.DATABASE in dbs, dbs)
tables = self.db.get_tables(database=self.DATABASE)
assert_true("tweets" in tables, tables)
QUERY = """
SELECT * FROM tweets ORDER BY row_num;
"""
response = _make_query(self.client, QUERY, database=self.DATABASE, local=False, server_name="impala")
content = json.loads(response.content)
query_history = QueryHistory.get(content["id"])
response = wait_for_query_to_finish(self.client, response, max=180.0)
results = []
# Check that we multiple fetches get all the result set
while len(results) < 5:
content = fetch_query_result_data(
self.client, response, n=len(results), server_name="impala"
) # We get less than 5 results most of the time, so increase offset
results += content["results"]
assert_equal([1, 2, 3, 4, 5], [col[0] for col in results])
# Check start over
results_start_over = []
while len(results_start_over) < 5:
content = fetch_query_result_data(self.client, response, n=len(results_start_over), server_name="impala")
results_start_over += content["results"]
assert_equal(results_start_over, results)
# Check cancel query
resp = self.client.post(reverse("impala:api_cancel_query", kwargs={"query_history_id": query_history.id}))
content = json.loads(resp.content)
assert_equal(0, content["status"])
def test_get_exec_summary(self):
query = """
SELECT COUNT(1) FROM tweets;
"""
response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala')
content = json.loads(response.content)
query_history = QueryHistory.get(content['id'])
wait_for_query_to_finish(self.client, response, max=180.0)
resp = self.client.post(reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id}))
data = json.loads(resp.content)
assert_equal(0, data['status'], data)
assert_true('nodes' in data['summary'], data)
assert_true(len(data['summary']['nodes']) > 0, data['summary']['nodes'])
# Attempt to call get_exec_summary on a closed query
resp = self.client.post(reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id}))
data = json.loads(resp.content)
assert_equal(0, data['status'], data)
assert_true('nodes' in data['summary'], data)
assert_true(len(data['summary']['nodes']) > 0, data['summary']['nodes'])
