def estimate_query_mem_mb_usage(query, impalad_conn):
"""Runs an explain plan then extracts and returns the estimated memory needed to run
the query.
"""
with impalad_conn.cursor() as cursor:
LOG.debug("Using %s database", query.db_name)
if query.db_name:
cursor.execute('USE ' + query.db_name)
if query.query_type == QueryType.COMPUTE_STATS:
# Running "explain" on compute stats is not supported by Impala.
return
LOG.debug("Explaining query\n%s", query.sql)
cursor.execute('EXPLAIN ' + query.sql)
explain_rows = cursor.fetchall()
explain_lines = [row[0] for row in explain_rows]
mem_limit, units = match_memory_estimate(explain_lines)
return parse_mem_to_mb(mem_limit, units)
def estimate_query_mem_mb_usage(query, impalad_conn):
"""Runs an explain plan then extracts and returns the estimated memory needed to run
the query.
"""
with impalad_conn.cursor() as cursor:
LOG.debug("Using %s database", query.db_name)
if query.db_name:
cursor.execute('USE ' + query.db_name)
if query.query_type == QueryType.COMPUTE_STATS:
# Running "explain" on compute stats is not supported by Impala.
return
LOG.debug("Explaining query\n%s", query.sql)
cursor.execute('EXPLAIN ' + query.sql)
explain_rows = cursor.fetchall()
explain_lines = [row[0] for row in explain_rows]
mem_limit, units = match_memory_estimate(explain_lines)
return parse_mem_to_mb(mem_limit, units)
def find_reported_mem_mb_usage(self):
"""Returns the amount of memory this impalad is using as reported by the impalad (
the mem tracker consumption).
"""
data = self._read_web_page("/memz")["consumption"].split()
return parse_mem_to_mb(data[0], data[1] if len(data) == 2 else "")
def find_reported_mem_mb_usage(self):
"""Returns the amount of memory this impalad is using as reported by the impalad (
the mem tracker consumption).
"""
data = self._read_web_page("/memz")["consumption"].split()
return parse_mem_to_mb(data[0], data[1] if len(data) == 2 else "")