def __exec_sql_and_check_selfevent_counter( self, stmt, use_impala_client, check_events_skipped_counter=True): """ Method runs a given query statement using a impala client or hive client based on the argument use_impala_client and confirms if the self-event related counters are as expected based on whether we expect a self-event or not. If the check_self_event_counter is False it skips checking the events-skipped metric. """ EventProcessorUtils.wait_for_event_processing(self) tbls_refreshed, partitions_refreshed, \ events_skipped = self.__get_self_event_metrics() last_synced_event = EventProcessorUtils.get_last_synced_event_id() logging.info("Running statement in {1}: {0}".format( stmt, "impala" if use_impala_client else "hive")) if not use_impala_client: self.run_stmt_in_hive(stmt) else: self.client.execute(stmt) EventProcessorUtils.wait_for_event_processing(self) tbls_refreshed_after, partitions_refreshed_after, \ events_skipped_after = self.__get_self_event_metrics() last_synced_event_after = EventProcessorUtils.get_last_synced_event_id( ) # we assume that any event which comes due to stmts run from impala-client are # self-events logging.info("Event id before {0} event id after {1}".format( last_synced_event, last_synced_event_after)) if use_impala_client: # self-event counter must increase if this is a self-event if # check_self_event_counter is set # some of the test queries generate no events at all. If that is the case # skip the below comparison if last_synced_event_after > last_synced_event: if check_events_skipped_counter: assert events_skipped_after > events_skipped, \ "Failing query(impala={}): {}".format(use_impala_client, stmt) # if this is a self-event, no table or partitions should be refreshed assert tbls_refreshed == tbls_refreshed_after, \ "Failing query(impala={}): {}".format(use_impala_client, stmt) assert partitions_refreshed == partitions_refreshed_after, \ "Failing query(impala={}): {}".format(use_impala_client, stmt) else: # hive was used to run the stmts, any events generated should not have been deemed # as self events assert events_skipped == events_skipped_after
def test_events_on_blacklisted_objects(self): """Executes hive queries on blacklisted database and tables and makes sure that event processor does not error out """ try: event_id_before = EventProcessorUtils.get_last_synced_event_id() self.run_stmt_in_hive("create database testBlackListedDb") self.run_stmt_in_hive("create table testBlackListedDb.testtbl (id int)") self.run_stmt_in_hive( "create table functional_parquet.testBlackListedTbl (id int, val string)" " partitioned by (part int) stored as parquet") self.run_stmt_in_hive( "alter table functional_parquet.testBlackListedTbl add partition (part=1)") # wait until all the events generated above are processed EventProcessorUtils.wait_for_event_processing(self) assert EventProcessorUtils.get_event_processor_status() == "ACTIVE" assert EventProcessorUtils.get_last_synced_event_id() > event_id_before finally: self.run_stmt_in_hive("drop database testBlackListedDb cascade") self.run_stmt_in_hive("drop table functional_parquet.testBlackListedTbl")
def test_events_on_blacklisted_objects(self): """Executes hive queries on blacklisted database and tables and makes sure that event processor does not error out """ try: event_id_before = EventProcessorUtils.get_last_synced_event_id() # create a blacklisted database from hive and make sure event is ignored self.run_stmt_in_hive("create database TESTblackListedDb") # wait until all the events generated above are processed EventProcessorUtils.wait_for_event_processing(self) assert EventProcessorUtils.get_event_processor_status() == "ACTIVE" assert EventProcessorUtils.get_last_synced_event_id( ) > event_id_before # make sure that the blacklisted db is ignored assert "TESTblackListedDb".lower() not in self.all_db_names() event_id_before = EventProcessorUtils.get_last_synced_event_id() self.run_stmt_in_hive( "create table testBlackListedDb.testtbl (id int)") # create a table on the blacklisted database with a different case self.run_stmt_in_hive("create table TESTBLACKlISTEDDb.t2 (id int)") self.run_stmt_in_hive( "create table functional_parquet.testBlackListedTbl (id int, val string)" " partitioned by (part int) stored as parquet") self.run_stmt_in_hive( "alter table functional_parquet.testBlackListedTbl add partition (part=1)" ) # wait until all the events generated above are processed EventProcessorUtils.wait_for_event_processing(self) assert EventProcessorUtils.get_event_processor_status() == "ACTIVE" assert EventProcessorUtils.get_last_synced_event_id( ) > event_id_before # make sure that the black listed table is not created table_names = self.client.execute( "show tables in functional_parquet").get_data() assert "testBlackListedTbl".lower() not in table_names event_id_before = EventProcessorUtils.get_last_synced_event_id() # generate a table level event with a different case self.run_stmt_in_hive( "drop table functional_parquet.TESTBlackListedTbl") # wait until all the events generated above are processed EventProcessorUtils.wait_for_event_processing(self) assert EventProcessorUtils.get_event_processor_status() == "ACTIVE" assert EventProcessorUtils.get_last_synced_event_id( ) > event_id_before finally: self.run_stmt_in_hive("drop database testBlackListedDb cascade") self.run_stmt_in_hive( "drop table functional_parquet.testBlackListedTbl")