def load_data(db_to_load, table_formats, scale):
"""Loads a database with a particular scale factor."""
subprocess.check_call(["{0}/bin/load-data.py".format(IMPALA_HOME),
"--workloads", db_to_load, "--scale_factor", str(scale),
"--table_formats", "text/none," + table_formats])
for table_format in table_formats.split(","):
suffix = TableFormatInfo.create_from_string(None, table_format).db_suffix()
db_name = db_to_load + scale + suffix
subprocess.check_call(["{0}/tests/util/compute_table_stats.py".format(IMPALA_HOME),
"--stop_on_error", "--db_names", db_name])
def load_data(db_to_load, table_formats, scale):
"""Loads a database with a particular scale factor."""
subprocess.check_call(["{0}/bin/load-data.py".format(IMPALA_HOME),
"--workloads", db_to_load, "--scale_factor", str(scale),
"--table_formats", "text/none," + table_formats])
for table_format in table_formats.split(","):
suffix = TableFormatInfo.create_from_string(None, table_format).db_suffix()
db_name = db_to_load + scale + suffix
subprocess.check_call(["{0}/tests/util/compute_table_stats.py".format(IMPALA_HOME),
"--stop_on_error", "--db_names", db_name])
def _generate_test_vectors(self):
"""Generate test vector objects
If the user has specified a set for table_formats, generate them, otherwise generate
vectors for all table formats within the specified exploration strategy.
"""
self._test_vectors = []
if self.config.table_formats:
dataset = get_dataset_from_workload(self.workload.name)
for tf in self.config.table_formats:
self._test_vectors.append(TableFormatInfo.create_from_string(dataset, tf))
else:
vectors = load_table_info_dimension(self.workload.name,
self.config.exploration_strategy)
self._test_vectors = [vector.value for vector in vectors]
def __generate_test_vectors(self):
"""Generate test vector objects
If the user has specified a set for table_formats, generate them, otherwise generate
vectors for all table formats within the specified exploration strategy.
"""
self.__test_vectors = []
if self.config.table_formats:
dataset = get_dataset_from_workload(self.workload.name)
for tf in self.config.table_formats:
self.__test_vectors.append(TableFormatInfo.create_from_string(dataset, tf))
else:
vectors = load_table_info_dimension(self.workload.name,
self.config.exploration_strategy)
self.__test_vectors = [vector.value for vector in vectors]
def create_table_info_dimension(cls, exploration_strategy):
# If the user has specified a specific set of table formats to run against, then
# use those. Otherwise, load from the workload test vectors.
if pytest.config.option.table_formats:
table_formats = list()
for tf in pytest.config.option.table_formats.split(','):
dataset = get_dataset_from_workload(cls.get_workload())
table_formats.append(TableFormatInfo.create_from_string(dataset, tf))
tf_dimensions = ImpalaTestDimension('table_format', *table_formats)
else:
tf_dimensions = load_table_info_dimension(cls.get_workload(), exploration_strategy)
# If 'skip_hbase' is specified or the filesystem is isilon, s3 or local, we don't
# need the hbase dimension.
if pytest.config.option.skip_hbase or TARGET_FILESYSTEM.lower() \
in ['s3', 'isilon', 'local', 'abfs', 'adls']:
for tf_dimension in tf_dimensions:
if tf_dimension.value.file_format == "hbase":
tf_dimensions.remove(tf_dimension)
break
return tf_dimensions
def create_table_info_dimension(cls, exploration_strategy):
# If the user has specified a specific set of table formats to run against, then
# use those. Otherwise, load from the workload test vectors.
if pytest.config.option.table_formats:
table_formats = list()
for tf in pytest.config.option.table_formats.split(','):
dataset = get_dataset_from_workload(cls.get_workload())
table_formats.append(TableFormatInfo.create_from_string(dataset, tf))
tf_dimensions = TestDimension('table_format', *table_formats)
else:
tf_dimensions = load_table_info_dimension(cls.get_workload(), exploration_strategy)
# If 'skip_hbase' is specified or the filesystem is isilon, s3 or local, we don't
# need the hbase dimension.
if pytest.config.option.skip_hbase or TARGET_FILESYSTEM.lower() \
in ['s3', 'isilon', 'local']:
for tf_dimension in tf_dimensions:
if tf_dimension.value.file_format == "hbase":
tf_dimensions.remove(tf_dimension)
break
return tf_dimensions
