def __get_self_event_metrics(): """ Gets the tables-refreshed, partitions-refreshed and events-skipped metric values from Metastore EventsProcessor """ tbls_refreshed_count = EventProcessorUtils.get_int_metric( 'tables-refreshed', 0) partitions_refreshed_count = EventProcessorUtils.get_int_metric( 'partitions-refreshed', 0) events_skipped_count = EventProcessorUtils.get_int_metric( 'events-skipped', 0) return int(tbls_refreshed_count), int(partitions_refreshed_count), \ int(events_skipped_count)
def test_event_batching(self, unique_database): """Runs queries which generate multiple ALTER_PARTITION events which must be batched by events processor. Runs as a custom cluster test to isolate the metric values from other tests.""" testtbl = "test_event_batching" test_acid_tbl = "test_event_batching_acid" acid_props = self.__get_transactional_tblproperties(True) # create test tables self.client.execute( "create table {}.{} like functional.alltypes".format( unique_database, testtbl)) self.client.execute( "insert into {}.{} partition (year,month) select * from functional.alltypes" .format(unique_database, testtbl)) self.client.execute( "create table {}.{} (id int) partitioned by (year int, month int) {}" .format(unique_database, test_acid_tbl, acid_props)) self.client.execute( "insert into {}.{} partition (year, month) " "select id, year, month from functional.alltypes".format( unique_database, test_acid_tbl)) # run compute stats from impala; this should generate 24 ALTER_PARTITION events which # should be batched together into 1 or more number of events. EventProcessorUtils.wait_for_event_processing(self) batch_events_metric = "batch-events-created" batch_events_1 = EventProcessorUtils.get_int_metric( batch_events_metric) self.client.execute("compute stats {}.{}".format( unique_database, testtbl)) EventProcessorUtils.wait_for_event_processing(self) batch_events_2 = EventProcessorUtils.get_int_metric( batch_events_metric) assert batch_events_2 > batch_events_1 # run analyze stats event from hive which generates ALTER_PARTITION event on each # partition of the table self.run_stmt_in_hive("analyze table {}.{} compute statistics".format( unique_database, testtbl)) EventProcessorUtils.wait_for_event_processing(self) batch_events_3 = EventProcessorUtils.get_int_metric( batch_events_metric) assert batch_events_3 > batch_events_2 # in case of transactional table since we batch the events together, the number of # tables refreshed must be far lower than number of events generated num_table_refreshes_1 = EventProcessorUtils.get_int_metric( "tables-refreshed") self.client.execute("compute stats {}.{}".format( unique_database, test_acid_tbl)) EventProcessorUtils.wait_for_event_processing(self) batch_events_4 = EventProcessorUtils.get_int_metric( batch_events_metric) num_table_refreshes_2 = EventProcessorUtils.get_int_metric( "tables-refreshed") # we should generate atleast 1 batch event if not more due to the 24 consecutive # ALTER_PARTITION events assert batch_events_4 > batch_events_3 # table should not be refreshed since this is a self-event assert num_table_refreshes_2 == num_table_refreshes_1 self.run_stmt_in_hive("analyze table {}.{} compute statistics".format( unique_database, test_acid_tbl)) EventProcessorUtils.wait_for_event_processing(self) batch_events_5 = EventProcessorUtils.get_int_metric( batch_events_metric) assert batch_events_5 > batch_events_4 num_table_refreshes_2 = EventProcessorUtils.get_int_metric( "tables-refreshed") # the analyze table from hive generates 24 ALTER_PARTITION events which should be # batched into 1-2 batches (depending on timing of the event poll thread). assert num_table_refreshes_2 > num_table_refreshes_1 assert int(num_table_refreshes_2) - int(num_table_refreshes_1) < 24 EventProcessorUtils.wait_for_event_processing(self) # test for batching of insert events batch_events_insert = EventProcessorUtils.get_int_metric( batch_events_metric) tables_refreshed_insert = EventProcessorUtils.get_int_metric( "tables-refreshed") partitions_refreshed_insert = EventProcessorUtils.get_int_metric( "partitions-refreshed") self.client.execute( "insert into {}.{} partition (year,month) select * from functional.alltypes" .format(unique_database, testtbl)) EventProcessorUtils.wait_for_event_processing(self) batch_events_after_insert = EventProcessorUtils.get_int_metric( batch_events_metric) tables_refreshed_after_insert = EventProcessorUtils.get_int_metric( "tables-refreshed") partitions_refreshed_after_insert = EventProcessorUtils.get_int_metric( "partitions-refreshed") # this is a self-event tables or partitions should not be refreshed assert batch_events_after_insert > batch_events_insert assert tables_refreshed_after_insert == tables_refreshed_insert assert partitions_refreshed_after_insert == partitions_refreshed_insert # run the insert from hive to make sure that batch event is refreshing all the # partitions self.run_stmt_in_hive( "SET hive.exec.dynamic.partition.mode=nonstrict; insert into {}.{} partition" " (year,month) select * from functional.alltypes".format( unique_database, testtbl)) EventProcessorUtils.wait_for_event_processing(self) batch_events_after_hive = EventProcessorUtils.get_int_metric( batch_events_metric) partitions_refreshed_after_hive = EventProcessorUtils.get_int_metric( "partitions-refreshed") assert batch_events_after_hive > batch_events_insert # 24 partitions inserted and hence we must refresh 24 partitions once. assert int(partitions_refreshed_after_hive ) == int(partitions_refreshed_insert) + 24
def __run_create_drop_test(self, db, type, rename=False, rename_db=False): if type == "table": if not rename: queries = [ "create table {0}.test_{1} (i int)".format(db, 1), "drop table {0}.test_{1}".format(db, 1) ] else: db_1 = "{}_1".format(db) if rename_db: self.execute_query_expect_success( self.create_impala_client(), "drop database if exists {0} cascade".format(db_1)) self.execute_query_expect_success( self.create_impala_client(), "create database {0}".format(db_1)) self.execute_query_expect_success( self.create_impala_client(), "create table if not exists {0}.rename_test_1 (i int)". format(db)) if rename_db: queries = [ "alter table {0}.rename_test_1 rename to {1}.rename_test_1" .format(db, db_1), "alter table {0}.rename_test_1 rename to {1}.rename_test_1" .format(db_1, db) ] else: queries = [ "alter table {0}.rename_test_1 rename to {0}.rename_test_2" .format(db), "alter table {0}.rename_test_2 rename to {0}.rename_test_1" .format(db) ] create_metric_name = "tables-added" removed_metric_name = "tables-removed" elif type == "database": self.execute_query_expect_success( self.create_impala_client(), "drop database if exists {0}".format("test_create_drop_db")) queries = [ "create database {db}".format(db="test_create_drop_db"), "drop database {db}".format(db="test_create_drop_db") ] create_metric_name = "databases-added" removed_metric_name = "databases-removed" else: tbl_name = "test_create_drop_partition" self.execute_query_expect_success( self.create_impala_client(), "create table {db}.{tbl} (c int) partitioned by (p int)". format(db=db, tbl=tbl_name)) queries = [ "alter table {db}.{tbl} add partition (p=1)".format( db=db, tbl=tbl_name), "alter table {db}.{tbl} drop partition (p=1)".format( db=db, tbl=tbl_name) ] create_metric_name = "partitions-added" removed_metric_name = "partitions-removed" # get the metric before values EventProcessorUtils.wait_for_event_processing(self) create_metric_val_before = EventProcessorUtils.get_int_metric( create_metric_name, 0) removed_metric_val_before = EventProcessorUtils.get_int_metric( removed_metric_name, 0) events_skipped_before = EventProcessorUtils.get_int_metric( 'events-skipped', 0) num_iters = 100 for iter in xrange(num_iters): for q in queries: try: self.execute_query_expect_success( self.create_impala_client(), q) except Exception as e: print("Failed in {} iterations. Error {}".format( iter, str(e))) raise EventProcessorUtils.wait_for_event_processing(self) create_metric_val_after = EventProcessorUtils.get_int_metric( create_metric_name, 0) removed_metric_val_after = EventProcessorUtils.get_int_metric( removed_metric_name, 0) events_skipped_after = EventProcessorUtils.get_int_metric( 'events-skipped', 0) num_delete_event_entries = EventProcessorUtils.\ get_int_metric('delete-event-log-size', 0) assert EventProcessorUtils.get_event_processor_status() == "ACTIVE" # None of the queries above should actually trigger a add/remove object from events assert create_metric_val_after == create_metric_val_before assert removed_metric_val_after == removed_metric_val_before # each query set generates 2 events and both of them should be skipped assert events_skipped_after == num_iters * 2 + events_skipped_before # make sure that there are no more entries in the delete event log assert num_delete_event_entries == 0