def get_num_cache_requests():
"""Returns the number of outstanding cache requests. Due to race conditions in the
way cache requests are added/dropped/reported (see IMPALA-3040), this function tries
to return a stable result by making several attempts to stabilize it within a
reasonable timeout."""
def get_num_cache_requests_util():
rc, stdout, stderr = exec_process(
"hdfs cacheadmin -listDirectives -stats")
assert rc == 0, 'Error executing hdfs cacheadmin: %s %s' % (stdout,
stderr)
return len(stdout.split('\n'))
# IMPALA-3040: This can take time, especially under slow builds like ASAN.
wait_time_in_sec = specific_build_type_timeout(5, slow_build_timeout=20)
num_stabilization_attempts = 0
max_num_stabilization_attempts = 10
new_requests = None
num_requests = None
LOG.info("{0} Entered get_num_cache_requests()".format(time.time()))
while num_stabilization_attempts < max_num_stabilization_attempts:
new_requests = get_num_cache_requests_util()
if new_requests == num_requests: break
LOG.info("{0} Waiting to stabilise: num_requests={1} new_requests={2}".
format(time.time(), num_requests, new_requests))
num_requests = new_requests
num_stabilization_attempts = num_stabilization_attempts + 1
time.sleep(wait_time_in_sec)
LOG.info("{0} Final num requests: {1}".format(time.time(), num_requests))
return num_requests
def test_restart_statestore(self, cursor):
""" Regression test of IMPALA-6973. After the statestore restarts, the metadata should
eventually recover after being cleared by the new statestore.
"""
self.cluster.statestored.restart()
# We need to wait for the impalad to register to the new statestored and for a
# non-empty catalog update from the new statestored. It cannot be expressed with the
# existing metrics yet so we wait for some time here.
wait_time_s = specific_build_type_timeout(60, slow_build_timeout=100)
sleep(wait_time_s)
for retry in xrange(wait_time_s):
try:
cursor.execute("describe database functional")
return
except HiveServer2Error, e:
assert "AnalysisException: Database does not exist: functional" in e.message,\
"Unexpected exception: " + e.message
sleep(1)
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
import pytest
import re
import time
from tests.common.environ import specific_build_type_timeout
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfLocal
WAIT_TIME_MS = specific_build_type_timeout(60000, slow_build_timeout=100000)
@SkipIfLocal.multiple_impalad
class TestRuntimeFilters(ImpalaTestSuite):
@classmethod
def get_workload(cls):
return 'functional-query'
@classmethod
def add_test_dimensions(cls):
super(TestRuntimeFilters, cls).add_test_dimensions()
# Runtime filters are disabled on HBase
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format not in ['hbase'])
# The number of queries to submit. The test does not support fewer queries than
# MAX_NUM_CONCURRENT_QUERIES + MAX_NUM_QUEUED_QUERIES to keep some validation logic
# simple.
NUM_QUERIES = [15, 30, 50]
# Whether we will submit queries to all available impalads (in a round-robin fashion)
ROUND_ROBIN_SUBMISSION = [True, False]
# The query pool to use. The impalads should be configured to recognize this
# pool with the parameters below.
POOL_NAME = "default-pool"
# Stress test timeout (seconds). The timeout needs to be significantly higher for
# slow builds like code coverage and ASAN (IMPALA-3790, IMPALA-6241).
STRESS_TIMEOUT = specific_build_type_timeout(60, slow_build_timeout=600)
# The number of queries that can execute concurrently in the pool POOL_NAME.
MAX_NUM_CONCURRENT_QUERIES = 5
# The number of queries that can be queued in the pool POOL_NAME
MAX_NUM_QUEUED_QUERIES = 10
# Mem limit (bytes) used in the mem limit test
MEM_TEST_LIMIT = 12 * 1024 * 1024 * 1024
_STATESTORED_ARGS = "-statestore_heartbeat_frequency_ms=%s "\
"-statestore_update_frequency_ms=%s" %\
(STATESTORE_RPC_FREQUENCY_MS, STATESTORE_RPC_FREQUENCY_MS)
# Key in the query profile for the query options.
# simple.
NUM_QUERIES = [15, 30, 50]
# Whether we will submit queries to all available impalads (in a round-robin fashion)
ROUND_ROBIN_SUBMISSION = [True, False]
# The query pool to use. The impalads should be configured to recognize this
# pool with the parameters below.
POOL_NAME = "default-pool"
# The statestore heartbeat and topic update frequency (ms). Set low for testing.
STATESTORE_RPC_FREQUENCY_MS = 500
# Stress test timeout (seconds). The timeout needs to be significantly higher in code
# coverage builds (IMPALA-3790).
STRESS_TIMEOUT = specific_build_type_timeout(30, code_coverage_build_timeout=600)
# The number of queries that can execute concurrently in the pool POOL_NAME.
MAX_NUM_CONCURRENT_QUERIES = 5
# The number of queries that can be queued in the pool POOL_NAME
MAX_NUM_QUEUED_QUERIES = 10
# Mem limit (bytes) used in the mem limit test
MEM_TEST_LIMIT = 12 * 1024 * 1024 * 1024
_STATESTORED_ARGS = "-statestore_heartbeat_frequency_ms=%s "\
"-statestore_update_frequency_ms=%s" %\
(STATESTORE_RPC_FREQUENCY_MS, STATESTORE_RPC_FREQUENCY_MS)
# Key in the query profile for the query options.
# simple.
NUM_QUERIES = [15, 30, 50]
# Whether we will submit queries to all available impalads (in a round-robin fashion)
ROUND_ROBIN_SUBMISSION = [True, False]
# The query pool to use. The impalads should be configured to recognize this
# pool with the parameters below.
POOL_NAME = "default-pool"
# The statestore heartbeat and topic update frequency (ms). Set low for testing.
STATESTORE_RPC_FREQUENCY_MS = 500
# Stress test timeout (seconds). The timeout needs to be significantly higher in code
# coverage builds (IMPALA-3790).
STRESS_TIMEOUT = specific_build_type_timeout(30,
code_coverage_build_timeout=600)
# The number of queries that can execute concurrently in the pool POOL_NAME.
MAX_NUM_CONCURRENT_QUERIES = 5
# The number of queries that can be queued in the pool POOL_NAME
MAX_NUM_QUEUED_QUERIES = 10
# Mem limit (bytes) used in the mem limit test
MEM_TEST_LIMIT = 12 * 1024 * 1024 * 1024
_STATESTORED_ARGS = "-statestore_heartbeat_frequency_ms=%s "\
"-statestore_update_frequency_ms=%s" %\
(STATESTORE_RPC_FREQUENCY_MS, STATESTORE_RPC_FREQUENCY_MS)
# Key in the query profile for the query options.
DEFAULT_HDFS_XML_CONF = os.path.join(os.environ['HADOOP_CONF_DIR'],
"hdfs-site.xml")
DEFAULT_HIVE_SERVER2 = 'localhost:11050'
DEFAULT_IMPALAD_HS2_PORT = '21050'
DEFAULT_IMPALADS = "localhost:21000,localhost:21001,localhost:21002"
DEFAULT_KUDU_MASTER_HOSTS = os.getenv('KUDU_MASTER_HOSTS', '127.0.0.1')
DEFAULT_KUDU_MASTER_PORT = os.getenv('KUDU_MASTER_PORT', '7051')
DEFAULT_METASTORE_SERVER = 'localhost:9083'
DEFAULT_NAMENODE_ADDR = None
if FILESYSTEM == 'isilon':
DEFAULT_NAMENODE_ADDR = "{node}:{port}".format(
node=os.getenv("ISILON_NAMENODE"), port=ISILON_WEBHDFS_PORT)
# Timeout each individual test case after 2 hours, or 4 hours for slow builds
PYTEST_TIMEOUT_S = \
specific_build_type_timeout(2 * 60 * 60, slow_build_timeout=4 * 60 * 60)
def pytest_configure(config):
""" Hook startup of pytest. Sets up log format and per-test timeout. """
configure_logging()
config.option.timeout = PYTEST_TIMEOUT_S
def configure_logging():
# Use a "--" since most of our tests output SQL commands, and it's nice to
# be able to copy-paste directly from the test output back into a shell to
# try to reproduce a failure.
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import specific_build_type_timeout
from tests.common.skip import SkipIfBuildType
from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON
# IMPALA-6100: add additional margin for error for slow build types.
SLOW_BUILD_TIMEOUT = 20000
DELAY_MS = specific_build_type_timeout(10000,
slow_build_timeout=SLOW_BUILD_TIMEOUT)
# IMPALA-6381: Isilon can behave as a slow build.
# IMPALA-6811: S3/ADLS can also have a slow scan that requires a longer delay.
if IS_S3 or IS_ADLS or IS_ISILON:
DELAY_MS = SLOW_BUILD_TIMEOUT
@SkipIfBuildType.not_dev_build
class TestExchangeDelays(CustomClusterTestSuite):
"""Tests for handling delays in finding data stream receivers"""
@classmethod
def get_workload(self):
return 'functional-query'
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
IMPALA_HOME, "bin/start-impalad.sh -build_type={build_type}".format(
build_type=options.build_type))
STATE_STORE_PATH = os.path.join(
IMPALA_HOME, "bin/start-statestored.sh -build_type={build_type}".format(
build_type=options.build_type))
CATALOGD_PATH = os.path.join(
IMPALA_HOME, "bin/start-catalogd.sh -build_type={build_type}".format(
build_type=options.build_type))
MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process"
CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240
# Kills have a timeout to prevent automated scripts from hanging indefinitely.
# It is set to a high value to avoid failing if processes are slow to shut down.
KILL_TIMEOUT_IN_SECONDS = 240
# For build types like ASAN, modify the default Kudu rpc timeout.
KUDU_RPC_TIMEOUT = specific_build_type_timeout(0, slow_build_timeout=60000)
def find_user_processes(binaries):
"""Returns an iterator over all processes owned by the current user with a matching
binary name from the provided list."""
for pid in psutil.get_pid_list():
try:
process = psutil.Process(pid)
if process.username == getuser() and process.name in binaries:
yield process
except KeyError, e:
if "uid not found" not in str(e):
raise
except psutil.NoSuchProcess, e:
# Ignore the case when a process no longer exists.
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.environ import specific_build_type_timeout
from tests.common.skip import SkipIfBuildType
# IMPALA-6100: add additional margin for error for slow build types.
DELAY_MS = specific_build_type_timeout(10000, slow_build_timeout=20000)
@SkipIfBuildType.not_dev_build
class TestExchangeDelays(CustomClusterTestSuite):
"""Tests for handling delays in finding data stream receivers"""
@classmethod
def get_workload(self):
return 'functional-query'
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
"--stress_datastream_recvr_delay_ms={0}".format(DELAY_MS) +
" --datastream_sender_timeout_ms=5000")
def test_exchange_small_delay(self, vector):
"""Test delays in registering data stream receivers where the first one or two
"bin/start-impalad.sh -build_type={build_type}".format(
build_type=options.build_type))
STATE_STORE_PATH = os.path.join(IMPALA_HOME,
"bin/start-statestored.sh -build_type={build_type}".format(
build_type=options.build_type))
CATALOGD_PATH = os.path.join(IMPALA_HOME,
"bin/start-catalogd.sh -build_type={build_type}".format(
build_type=options.build_type))
MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process"
CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240
# Kills have a timeout to prevent automated scripts from hanging indefinitely.
# It is set to a high value to avoid failing if processes are slow to shut down.
KILL_TIMEOUT_IN_SECONDS = 240
# For build types like ASAN, modify the default Kudu rpc timeout.
KUDU_RPC_TIMEOUT = specific_build_type_timeout(0, slow_build_timeout=60000)
def find_user_processes(binaries):
"""Returns an iterator over all processes owned by the current user with a matching
binary name from the provided list."""
for pid in psutil.get_pid_list():
try:
process = psutil.Process(pid)
if process.username == getuser() and process.name in binaries: yield process
except KeyError, e:
if "uid not found" not in str(e):
raise
except psutil.NoSuchProcess, e:
# Ignore the case when a process no longer exists.
pass
