def authorized_get_query_history(request, query_history_id, owner_only=False, must_exist=False): if query_history_id is None and not must_exist: return None try: query_history = QueryHistory.get(id=query_history_id) except QueryHistory.DoesNotExist: if must_exist: raise PopupException( _('QueryHistory %(id)s does not exist.') % {'id': query_history_id}) else: return None # Some queries don't have a design so are not linked to Document Model permission if query_history.design is None or not query_history.design.doc.exists(): if not request.user.is_superuser and request.user != query_history.owner: raise PopupException( _('Permission denied to read QueryHistory %(id)s') % {'id': query_history_id}) else: query_history.design.doc.get().can_read_or_exception(request.user) return query_history
def test_get_exec_summary(self): query = """ SELECT COUNT(1) FROM tweets; """ response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala') content = json.loads(response.content) query_history = QueryHistory.get(content['id']) wait_for_query_to_finish(self.client, response, max=180.0) resp = self.client.post( reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id})) data = json.loads(resp.content) assert_equal(0, data['status'], data) assert_true('nodes' in data['summary'], data) assert_true( len(data['summary']['nodes']) > 0, data['summary']['nodes']) # Attempt to call get_exec_summary on a closed query resp = self.client.post( reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id})) data = json.loads(resp.content) assert_equal(0, data['status'], data) assert_true('nodes' in data['summary'], data) assert_true( len(data['summary']['nodes']) > 0, data['summary']['nodes'])
def handle(self, *args, **options): days = int(args[0]) if len(args) >= 1 else 7 close_all = args[1] == 'all' if len(args) >= 2 else False self.stdout.write( 'Closing (all=%s) HiveServer2 queries older than %s days...\n' % (close_all, days)) queries = QueryHistory.objects.filter(last_state__in=[ QueryHistory.STATE.expired.value, QueryHistory.STATE.failed.value, QueryHistory.STATE.available.value ]) if close_all: queries = QueryHistory.objects.all() queries = queries.filter(submission_date__lte=datetime.today() - timedelta(days=days)) import os import beeswax from beeswax import conf from beeswax import hive_site try: beeswax.conf.HIVE_CONF_DIR.set_for_testing( os.environ['HIVE_CONF_DIR']) except: LOG.exception('failed to lookup HIVE_CONF_DIR in environment') self.stdout.write('Did you export HIVE_CONF_DIR=/etc/hive/conf?\n') raise hive_site.reset() hive_site.get_conf() closed_queries = 0 already_closed_queries = 0 for query in queries: try: query_history = QueryHistory.get(id=query.id) if query_history.server_id is not None: handle = query_history.get_handle() dbms.get(user=query_history.owner).close_operation(handle) closed_queries += 1 else: already_closed_queries += 1 query.last_state = QueryHistory.STATE.expired.value query.save() except Exception as e: if 'None' in str(e) or 'Invalid OperationHandle' in str(e): already_closed_queries += 1 query.last_state = QueryHistory.STATE.expired.value query.save() else: self.stdout.write('Info: %s\n' % e) self.stdout.write('%s queries closed. %s queries already closed.\n' % (closed_queries, already_closed_queries))
def test_basic_flow(self): dbs = self.db.get_databases() assert_true('_impala_builtins' in dbs, dbs) assert_true(self.DATABASE in dbs, dbs) tables = self.db.get_tables(database=self.DATABASE) assert_true('tweets' in tables, tables) QUERY = """ SELECT * FROM tweets ORDER BY row_num; """ response = _make_query(self.client, QUERY, database=self.DATABASE, local=False, server_name='impala') content = json.loads(response.content) query_history = QueryHistory.get(content['id']) response = wait_for_query_to_finish(self.client, response, max=180.0) results = [] # Check that we multiple fetches get all the result set while len(results) < 5: # We get less than 5 results most of the time, so increase offset content = fetch_query_result_data(self.client, response, n=len(results), server_name='impala') results += content['results'] assert_equal([1, 2, 3, 4, 5], [col[0] for col in results]) # Check start over results_start_over = [] while len(results_start_over) < 5: content = fetch_query_result_data(self.client, response, n=len(results_start_over), server_name='impala') results_start_over += content['results'] assert_equal(results_start_over, results) # Check cancel query resp = self.client.post( reverse('impala:api_cancel_query', kwargs={'query_history_id': query_history.id})) content = json.loads(resp.content) assert_equal(0, content['status'])
def authorized_get_history(request, query_history_id, owner_only=False, must_exist=False): if query_history_id is None and not must_exist: return None try: query_history = QueryHistory.get(id=query_history_id) except QueryHistory.DoesNotExist: if must_exist: raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id}) else: return None query_history.design.doc.get().can_read_or_exception(request.user) return query_history
def handle(self, *args, **options): days = int(args[0]) if len(args) >= 1 else 7 close_all = args[1] == 'all' if len(args) >= 2 else False self.stdout.write('Closing (all=%s) HiveServer2 queries older than %s days...\n' % (close_all, days)) queries = QueryHistory.objects.filter(last_state__in=[QueryHistory.STATE.expired.value, QueryHistory.STATE.failed.value, QueryHistory.STATE.available.value]) if close_all: queries = QueryHistory.objects.all() queries = queries.filter(submission_date__lte=datetime.today() - timedelta(days=days)) import os import beeswax from beeswax import conf from beeswax import hive_site try: beeswax.conf.HIVE_CONF_DIR.set_for_testing(os.environ['HIVE_CONF_DIR']) except: LOG.exception('failed to lookup HIVE_CONF_DIR in environment') self.stdout.write('Did you export HIVE_CONF_DIR=/etc/hive/conf?\n') raise hive_site.reset() hive_site.get_conf() closed_queries = 0 already_closed_queries = 0 for query in queries: try: query_history = QueryHistory.get(id=query.id) if query_history.server_id is not None: handle = query_history.get_handle() dbms.get(user=query_history.owner).close_operation(handle) closed_queries += 1 else: already_closed_queries += 1 query.last_state = QueryHistory.STATE.expired.value query.save() except Exception, e: if 'None' in str(e) or 'Invalid OperationHandle' in str(e): already_closed_queries += 1 query.last_state = QueryHistory.STATE.expired.value query.save() else: self.stdout.write('Info: %s\n' % e)
def test_get_runtime_profile(self): query = """ SELECT COUNT(1) FROM tweets; """ response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala') content = json.loads(response.content) query_history = QueryHistory.get(content['id']) wait_for_query_to_finish(self.client, response, max=180.0) resp = self.client.post(reverse('impala:get_runtime_profile', kwargs={'query_history_id': query_history.id})) data = json.loads(resp.content) assert_equal(0, data['status'], data) assert_true('Execution Profile' in data['profile'], data)
def test_get_runtime_profile(self): query = """ SELECT COUNT(1) FROM tweets; """ response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala') content = json.loads(response.content) query_history = QueryHistory.get(content['id']) wait_for_query_to_finish(self.client, response, max=180.0) resp = self.client.post(reverse('impala:get_runtime_profile', kwargs={'query_history_id': query_history.id})) data = json.loads(resp.content) assert_equal(0, data['status'], data) assert_true('Execution Profile' in data['profile'], data)
def authorized_get_query_history(request, query_history_id, owner_only=False, must_exist=False): if query_history_id is None and not must_exist: return None try: query_history = QueryHistory.get(id=query_history_id) except QueryHistory.DoesNotExist: if must_exist: raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id}) else: return None # Some queries don't have a design so are not linked to Document Model permission if query_history.design is None or not query_history.design.doc.exists(): if not request.user.is_superuser and request.user != query_history.owner: raise PopupException(_('Permission denied to read QueryHistory %(id)s') % {'id': query_history_id}) else: query_history.design.doc.get().can_read_or_exception(request.user) return query_history
def test_basic_flow(self): dbs = self.db.get_databases() assert_true("_impala_builtins" in dbs, dbs) assert_true(self.DATABASE in dbs, dbs) tables = self.db.get_tables(database=self.DATABASE) assert_true("tweets" in tables, tables) QUERY = """ SELECT * FROM tweets ORDER BY row_num; """ response = _make_query(self.client, QUERY, database=self.DATABASE, local=False, server_name="impala") content = json.loads(response.content) query_history = QueryHistory.get(content["id"]) response = wait_for_query_to_finish(self.client, response, max=180.0) results = [] # Check that we multiple fetches get all the result set while len(results) < 5: content = fetch_query_result_data( self.client, response, n=len(results), server_name="impala" ) # We get less than 5 results most of the time, so increase offset results += content["results"] assert_equal([1, 2, 3, 4, 5], [col[0] for col in results]) # Check start over results_start_over = [] while len(results_start_over) < 5: content = fetch_query_result_data(self.client, response, n=len(results_start_over), server_name="impala") results_start_over += content["results"] assert_equal(results_start_over, results) # Check cancel query resp = self.client.post(reverse("impala:api_cancel_query", kwargs={"query_history_id": query_history.id})) content = json.loads(resp.content) assert_equal(0, content["status"])
def test_get_exec_summary(self): query = """ SELECT COUNT(1) FROM tweets; """ response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala') content = json.loads(response.content) query_history = QueryHistory.get(content['id']) wait_for_query_to_finish(self.client, response, max=180.0) resp = self.client.post(reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id})) data = json.loads(resp.content) assert_equal(0, data['status'], data) assert_true('nodes' in data['summary'], data) assert_true(len(data['summary']['nodes']) > 0, data['summary']['nodes']) # Attempt to call get_exec_summary on a closed query resp = self.client.post(reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id})) data = json.loads(resp.content) assert_equal(0, data['status'], data) assert_true('nodes' in data['summary'], data) assert_true(len(data['summary']['nodes']) > 0, data['summary']['nodes'])