def get_num_cache_requests():
"""Returns the number of outstanding cache requests. Due to race conditions in the
way cache requests are added/dropped/reported (see IMPALA-3040), this function tries
to return a stable result by making several attempts to stabilize it within a
reasonable timeout."""
def get_num_cache_requests_util():
rc, stdout, stderr = exec_process(
"hdfs cacheadmin -listDirectives -stats")
assert rc == 0, 'Error executing hdfs cacheadmin: %s %s' % (stdout,
stderr)
return len(stdout.split('\n'))
# IMPALA-3040: This can take time, especially under slow builds like ASAN.
wait_time_in_sec = build_flavor_timeout(5, slow_build_timeout=20)
num_stabilization_attempts = 0
max_num_stabilization_attempts = 10
new_requests = None
num_requests = None
LOG.info("{0} Entered get_num_cache_requests()".format(time.time()))
while num_stabilization_attempts < max_num_stabilization_attempts:
new_requests = get_num_cache_requests_util()
if new_requests == num_requests: break
LOG.info("{0} Waiting to stabilise: num_requests={1} new_requests={2}".
format(time.time(), num_requests, new_requests))
num_requests = new_requests
num_stabilization_attempts = num_stabilization_attempts + 1
time.sleep(wait_time_in_sec)
LOG.info("{0} Final num requests: {1}".format(time.time(), num_requests))
return num_requests
def get_num_cache_requests():
"""Returns the number of outstanding cache requests. Due to race conditions in the
way cache requests are added/dropped/reported (see IMPALA-3040), this function tries
to return a stable result by making several attempts to stabilize it within a
reasonable timeout."""
def get_num_cache_requests_util():
rc, stdout, stderr = exec_process("hdfs cacheadmin -listDirectives -stats")
assert rc == 0, 'Error executing hdfs cacheadmin: %s %s' % (stdout, stderr)
return len(stdout.split('\n'))
# IMPALA-3040: This can take time, especially under slow builds like ASAN.
wait_time_in_sec = build_flavor_timeout(5, slow_build_timeout=20)
num_stabilization_attempts = 0
max_num_stabilization_attempts = 10
new_requests = None
num_requests = None
LOG.info("{0} Entered get_num_cache_requests()".format(time.time()))
while num_stabilization_attempts < max_num_stabilization_attempts:
new_requests = get_num_cache_requests_util()
if new_requests == num_requests: break
LOG.info("{0} Waiting to stabilise: num_requests={1} new_requests={2}".format(
time.time(), num_requests, new_requests))
num_requests = new_requests
num_stabilization_attempts = num_stabilization_attempts + 1
time.sleep(wait_time_in_sec)
LOG.info("{0} Final num requests: {1}".format(time.time(), num_requests))
return num_requests
def wait_for_insert_event_processing(self, previous_event_id):
"""Waits until the event processor has finished processing insert events. Since two
events are created for every insert done through hive, we wait until the event id is
incremented by at least two. Returns true if at least two events were processed within
self.PROCESSING_TIMEOUT_S, False otherwise.
"""
new_event_id = self.get_last_synced_event_id()
success = False
end_time = time.time() + self.PROCESSING_TIMEOUT_S
while time.time() < end_time:
new_event_id = self.get_last_synced_event_id()
if new_event_id - previous_event_id >= 2:
success = True
break
time.sleep(0.1)
# Wait for catalog update to be propagated.
time.sleep(build_flavor_timeout(2, slow_build_timeout=4))
return success
def test_restart_statestore(self, cursor):
""" Regression test of IMPALA-6973. After the statestore restarts, the metadata should
eventually recover after being cleared by the new statestore.
"""
self.cluster.statestored.restart()
# We need to wait for the impalad to register to the new statestored and for a
# non-empty catalog update from the new statestored. It cannot be expressed with the
# existing metrics yet so we wait for some time here.
wait_time_s = build_flavor_timeout(60, slow_build_timeout=100)
sleep(wait_time_s)
for retry in xrange(wait_time_s):
try:
cursor.execute("describe database functional")
return
except HiveServer2Error, e:
assert "AnalysisException: Database does not exist: functional" in e.message,\
"Unexpected exception: " + e.message
sleep(1)
def get_num_cache_requests():
"""Returns the number of outstanding cache requests. Due to race conditions in the
way cache requests are added/dropped/reported (see IMPALA-3040), this function tries
to return a stable result by making several attempts to stabilize it within a
reasonable timeout."""
def get_num_cache_requests_util():
rc, stdout, stderr = exec_process(
"hdfs cacheadmin -listDirectives -stats")
assert rc == 0, 'Error executing hdfs cacheadmin: %s %s' % (stdout,
stderr)
# remove blank new lines from output count
lines = [line for line in stdout.split('\n') if line.strip()]
count = None
for line in lines:
if line.startswith("Found "):
# the line should say "Found <int> entries"
# if we find this line we parse the number of entries
# from this line.
count = int(re.search(r'\d+', line).group())
break
# if count is available we return it else we just
# return the total number of lines
if count is not None:
return count
else:
return len(stdout.split('\n'))
# IMPALA-3040: This can take time, especially under slow builds like ASAN.
wait_time_in_sec = build_flavor_timeout(5, slow_build_timeout=20)
num_stabilization_attempts = 0
max_num_stabilization_attempts = 10
num_requests = None
LOG.info("{0} Entered get_num_cache_requests()".format(time.time()))
while num_stabilization_attempts < max_num_stabilization_attempts:
new_requests = get_num_cache_requests_util()
if new_requests == num_requests: break
LOG.info("{0} Waiting to stabilise: num_requests={1} new_requests={2}".
format(time.time(), num_requests, new_requests))
num_requests = new_requests
num_stabilization_attempts = num_stabilization_attempts + 1
time.sleep(wait_time_in_sec)
LOG.info("{0} Final num requests: {1}".format(time.time(), num_requests))
return num_requests
def test_restart_statestore(self, cursor):
""" Regression test of IMPALA-6973. After the statestore restarts, the metadata should
eventually recover after being cleared by the new statestore.
"""
self.cluster.statestored.restart()
# We need to wait for the impalad to register to the new statestored and for a
# non-empty catalog update from the new statestored. It cannot be expressed with the
# existing metrics yet so we wait for some time here.
wait_time_s = build_flavor_timeout(60, slow_build_timeout=100)
sleep(wait_time_s)
for retry in xrange(wait_time_s):
try:
cursor.execute("describe database functional")
return
except HiveServer2Error, e:
assert "AnalysisException: Database does not exist: functional" in e.message,\
"Unexpected exception: " + e.message
sleep(1)
def wait_for_insert_event_processing(self, previous_event_id):
""" Wait till the event processor has finished processing insert events. This is
detected by scrapping the /events webpage for changes in last_synced_event_id.
Since two events are created for every insert done through hive, we wait till the
event id is incremented by at least two. Returns true if at least two events were
processed within 10 sec. False otherwise.
"""
new_event_id = self.get_last_synced_event_id()
success = True
start_time = datetime.now()
while new_event_id - previous_event_id < 2:
new_event_id = self.get_last_synced_event_id()
# Prevent infinite loop
time_delta = (datetime.now() - start_time).total_seconds()
if time_delta > 10:
success = False
break
# Wait for catalog update to be propagated.
time.sleep(build_flavor_timeout(2, slow_build_timeout=4))
return success
def wait_for_insert_event_processing(self, previous_event_id):
""" Wait till the event processor has finished processing insert events. This is
detected by scrapping the /events webpage for changes in last_synced_event_id.
Since two events are created for every insert done through hive, we wait till the
event id is incremented by at least two. Returns true if at least two events were
processed within 10 sec. False otherwise.
"""
new_event_id = self.get_last_synced_event_id()
success = True
start_time = time.time()
while new_event_id - previous_event_id < 2:
new_event_id = self.get_last_synced_event_id()
# Prevent infinite loop
time_delta = time.time() - start_time
if time_delta > 10:
success = False
break
# Wait for catalog update to be propagated.
time.sleep(build_flavor_timeout(2, slow_build_timeout=4))
return success
def wait_for_event_processing(hive_client, timeout=10):
"""Waits till the event processor has synced to the latest event id from metastore
or the timeout value in seconds whichever is earlier"""
success = False
assert timeout > 0
assert hive_client is not None
current_event_id = EventProcessorUtils.get_current_notification_id(hive_client)
end_time = time.time() + timeout
while time.time() < end_time:
new_event_id = EventProcessorUtils.get_last_synced_event_id()
if new_event_id >= current_event_id:
success = True
break
time.sleep(0.1)
if not success:
raise Exception(
"Event processor did not sync till last known event id{0} \
within {1} seconds".format(current_event_id, timeout))
# Wait for catalog update to be propagated.
time.sleep(build_flavor_timeout(6, slow_build_timeout=10))
return success
"bin/start-impalad.sh -build_type={build_type}".format(
build_type=options.build_type))
STATE_STORE_PATH = os.path.join(IMPALA_HOME,
"bin/start-statestored.sh -build_type={build_type}".format(
build_type=options.build_type))
CATALOGD_PATH = os.path.join(IMPALA_HOME,
"bin/start-catalogd.sh -build_type={build_type}".format(
build_type=options.build_type))
MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process"
CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240
# Kills have a timeout to prevent automated scripts from hanging indefinitely.
# It is set to a high value to avoid failing if processes are slow to shut down.
KILL_TIMEOUT_IN_SECONDS = 240
# For build types like ASAN, modify the default Kudu rpc timeout.
KUDU_RPC_TIMEOUT = build_flavor_timeout(0, slow_build_timeout=60000)
def find_user_processes(binaries):
"""Returns an iterator over all processes owned by the current user with a matching
binary name from the provided list."""
for pid in psutil.get_pid_list():
try:
process = psutil.Process(pid)
if process.username == getuser() and process.name in binaries: yield process
except KeyError, e:
if "uid not found" not in str(e):
raise
except psutil.NoSuchProcess, e:
# Ignore the case when a process no longer exists.
pass
import time from beeswaxd.BeeswaxService import QueryState from tests.common.environ import build_flavor_timeout from tests.common.environ import ImpalaTestClusterProperties from tests.common.impala_cluster import ImpalaCluster from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfEC from tests.common.skip import SkipIfLocal, SkipIfIsilon from tests.common.test_dimensions import add_exec_option_dimension from tests.common.test_vector import ImpalaTestDimension from tests.verifiers.metric_verifier import MetricVerifier # slow_build_timeout is set to 200000 to avoid failures like IMPALA-8064 where the # runtime filters don't arrive in time. WAIT_TIME_MS = build_flavor_timeout(60000, slow_build_timeout=200000) # Check whether the Impala under test in slow build. Query option ASYNC_CODEGEN will # be enabled when test runs for slow build like ASAN, TSAN, UBSAN, etc. This avoid # failures like IMPALA-9889 where the runtime filters don't arrive in time due to # the slowness of codegen. build_runs_slowly = ImpalaTestClusterProperties.get_instance().runs_slowly() # Some of the queries in runtime_filters consume a lot of memory, leading to # significant memory reservations in parallel tests. # Skipping Isilon due to IMPALA-6998. TODO: Remove when there's a holistic revamp of # what tests to run for non-HDFS platforms @pytest.mark.execute_serially @SkipIfLocal.multiple_impalad @SkipIfIsilon.jira(reason="IMPALA-6998") class TestRuntimeFilters(ImpalaTestSuite):
"hdfs-site.xml")
DEFAULT_HIVE_SERVER2 = 'localhost:11050'
DEFAULT_IMPALAD_HS2_PORT = '21050'
DEFAULT_IMPALAD_HS2_HTTP_PORT = '28000'
DEFAULT_IMPALADS = "localhost:21000,localhost:21001,localhost:21002"
DEFAULT_KUDU_MASTER_HOSTS = os.getenv('KUDU_MASTER_HOSTS', '127.0.0.1')
DEFAULT_KUDU_MASTER_PORT = os.getenv('KUDU_MASTER_PORT', '7051')
DEFAULT_METASTORE_SERVER = 'localhost:9083'
DEFAULT_NAMENODE_ADDR = None
if FILESYSTEM == 'isilon':
DEFAULT_NAMENODE_ADDR = "{node}:{port}".format(
node=os.getenv("ISILON_NAMENODE"), port=ISILON_WEBHDFS_PORT)
# Timeout each individual test case after 2 hours, or 4 hours for slow builds
PYTEST_TIMEOUT_S = \
build_flavor_timeout(2 * 60 * 60, slow_build_timeout=4 * 60 * 60)
def pytest_configure(config):
""" Hook startup of pytest. Sets up log format and per-test timeout. """
configure_logging()
config.option.timeout = PYTEST_TIMEOUT_S
def configure_logging():
# Use a "--" since most of our tests output SQL commands, and it's nice to
# be able to copy-paste directly from the test output back into a shell to
# try to reproduce a failure.
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import build_flavor_timeout
from tests.common.skip import SkipIfBuildType
from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON
# IMPALA-6100: add additional margin for error for slow build types.
SLOW_BUILD_TIMEOUT = 20000
DELAY_MS = build_flavor_timeout(10000, slow_build_timeout=SLOW_BUILD_TIMEOUT)
# IMPALA-6381: Isilon can behave as a slow build.
if IS_ISILON:
DELAY_MS = SLOW_BUILD_TIMEOUT
@SkipIfBuildType.not_dev_build
class TestExchangeDelays(CustomClusterTestSuite):
"""Tests for handling delays in finding data stream receivers"""
@classmethod
def get_workload(self):
return 'functional-query'
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
"--stress_datastream_recvr_delay_ms={0}".format(DELAY_MS) +
# The ith group of options is applied to the ith impalad.
parser.add_option("--per_impalad_args", dest="per_impalad_args", type="string"
,default="", help=SUPPRESS_HELP)
options, args = parser.parse_args()
IMPALA_HOME = os.environ["IMPALA_HOME"]
CORE_SITE_PATH = os.path.join(IMPALA_HOME, "fe/src/test/resources/core-site.xml")
KNOWN_BUILD_TYPES = ["debug", "release", "latest"]
IMPALA_LZO = os.environ["IMPALA_LZO"]
# Kills have a timeout to prevent automated scripts from hanging indefinitely.
# It is set to a high value to avoid failing if processes are slow to shut down.
KILL_TIMEOUT_IN_SECONDS = 240
# For build types like ASAN, modify the default Kudu rpc timeout.
KUDU_RPC_TIMEOUT = build_flavor_timeout(0, slow_build_timeout=60000)
def check_process_exists(binary, attempts=1):
"""Checks if a process exists given the binary name. The `attempts` count allows us to
control the time a process needs to settle until it becomes available. After each try
the script will sleep for one second and retry. Returns True if it exists and False
otherwise.
"""
for _ in range(attempts):
for proc in find_user_processes([binary]):
return True
sleep(1)
return False
def run_daemon_with_options(daemon_binary, args, output_file, jvm_debug_port=None):
DEFAULT_EXPLORATION_STRATEGY = 'core'
DEFAULT_HDFS_XML_CONF = os.path.join(os.environ['HADOOP_CONF_DIR'], "hdfs-site.xml")
DEFAULT_HIVE_SERVER2 = 'localhost:11050'
DEFAULT_IMPALAD_HS2_PORT = '21050'
DEFAULT_IMPALADS = "localhost:21000,localhost:21001,localhost:21002"
DEFAULT_KUDU_MASTER_HOSTS = os.getenv('KUDU_MASTER_HOSTS', '127.0.0.1')
DEFAULT_KUDU_MASTER_PORT = os.getenv('KUDU_MASTER_PORT', '7051')
DEFAULT_METASTORE_SERVER = 'localhost:9083'
DEFAULT_NAMENODE_ADDR = None
if FILESYSTEM == 'isilon':
DEFAULT_NAMENODE_ADDR = "{node}:{port}".format(node=os.getenv("ISILON_NAMENODE"),
port=ISILON_WEBHDFS_PORT)
# Timeout each individual test case after 2 hours, or 4 hours for slow builds
PYTEST_TIMEOUT_S = \
build_flavor_timeout(2 * 60 * 60, slow_build_timeout=4 * 60 * 60)
def pytest_configure(config):
""" Hook startup of pytest. Sets up log format and per-test timeout. """
configure_logging()
config.option.timeout = PYTEST_TIMEOUT_S
def configure_logging():
# Use a "--" since most of our tests output SQL commands, and it's nice to
# be able to copy-paste directly from the test output back into a shell to
# try to reproduce a failure.
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
def pytest_addoption(parser):
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pytest
from copy import deepcopy
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import build_flavor_timeout
WAIT_TIME_MS = build_flavor_timeout(60000, slow_build_timeout=100000)
class TestMtDopFlags(CustomClusterTestSuite):
@classmethod
def get_workload(cls):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
super(TestMtDopFlags, cls).add_test_dimensions()
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(impalad_args="--unlock_mt_dop=true")
def test_mt_dop_all(self, vector, unique_database):
"""Test joins, inserts and runtime filters with mt_dop > 0"""
