def database_sandbox_session(max_retries=3): db_connections = get_connection( config.env_config.topology_path, config.env_config.rbr_source_cluster, config.env_config.schema_tracker_cluster, config.env_config.rbr_state_cluster, is_avoid_internal_packages_set() ) done_making_mysqld = False retries = 0 while not done_making_mysqld: # Takes time for mysqld to launch, so we will attempt a few times to it try: _per_process_mysql_daemon = PerProcessMySQLDaemon() done_making_mysqld = True except RuntimeError: retries += 1 if retries > max_retries: raise _session_prev_engine = db_connections.state_session.bind db_connections.state_session.bind = _per_process_mysql_daemon.engine db_connections.state_session.enforce_read_only = False yield db_connections.state_session db_connections.state_session.bind = _session_prev_engine
def __init__(self): super(BaseParseReplicationStream, self).__init__() self.db_connections = get_connection( config.env_config.topology_path, config.env_config.rbr_source_cluster, config.env_config.schema_tracker_cluster, config.env_config.rbr_state_cluster, is_avoid_internal_packages_set(), config.env_config.rbr_source_cluster_topology_name, ) self.schema_wrapper = SchemaWrapper( db_connections=self.db_connections, schematizer_client=get_schematizer() ) self.register_dry_run = config.env_config.register_dry_run self.publish_dry_run = config.env_config.publish_dry_run self._running = True self._profiler_running = False self._changelog_mode = config.env_config.changelog_mode if get_config().kafka_producer_buffer_size > config.env_config.recovery_queue_size: # Printing here, since this executes *before* logging is # configured. sys.stderr.write("Shutting down because kafka_producer_buffer_size was greater than \ recovery_queue_size") sys.exit(1)
def mock_db_connections(self, topology_path, mock_source_cluster_name, mock_tracker_cluster_name, mock_state_cluster_name, yelp_conn_conf): yield get_connection(topology_path, mock_source_cluster_name, mock_tracker_cluster_name, mock_state_cluster_name, is_avoid_internal_packages_set())
def get_mock_stats_counters(): """In open source mode StatsCount is not supported because of absence of yelp_meteorite and hence StatsCount is passed in as None into BaseEventHandler`. In internal mode StatsCount be set to None if config.env_config.disable_meteorite is True, in all other cases StatsCount will be set to an object of data_pipeline.tools.meteorite_wrappers.StatsCounter Goal here is to test event handlers for all the above cases. None and object of StatsCounter cover all the cases described above. We start by setting counter to None and if data_pipeline.tools.meteorite_wrappers is importable we add StatsCounter to possible counters values. """ counters = [None] try: # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we are completely ready for open source. if is_avoid_internal_packages_set(): raise ImportError from data_pipeline.tools.meteorite_wrappers import StatsCounter counters.append(mock.Mock(autospec=StatsCounter)) except ImportError: pass return counters
def get_mock_stats_counters(): """In open source mode StatsCount is not supported because of absence of yelp_meteorite and hence StatsCount is passed in as None into BaseEventHandler`. In internal mode StatsCount be set to None if config.env_config.disable_meteorite is True, in all other cases StatsCount will be set to an object of data_pipeline.tools.meteorite_wrappers.StatsCounter Goal here is to test event handlers for all the above cases. None and object of StatsCounter cover all the cases described above. We start by setting counter to None and if data_pipeline.tools.meteorite_wrappers is importable we add StatsCounter to possible counters values. """ counters = [None] try: # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we are completely ready for open source. if is_avoid_internal_packages_set(): raise ImportError from data_pipeline.tools.meteorite_wrappers import StatsCounter counters.append(mock.Mock(autospec=StatsCounter)) except ImportError: pass return counters
def is_meteorite_supported(cls): try: # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we are completely ready for open source. if is_avoid_internal_packages_set(): raise ImportError from data_pipeline.tools.meteorite_wrappers import StatTimer # NOQA return True except ImportError: return False
def is_pii_supported(cls): try: # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we are completely ready for open source. if is_avoid_internal_packages_set(): raise ImportError from pii_generator.components.pii_identifier import PIIIdentifier # NOQA return True except ImportError: return False
def is_meteorite_supported(cls): try: # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we are completely ready for open source. if is_avoid_internal_packages_set(): raise ImportError from data_pipeline.tools.meteorite_wrappers import StatTimer # NOQA return True except ImportError: return False
def database_sandbox_session(): db_connections = get_connection(config.env_config.topology_path, config.env_config.rbr_source_cluster, config.env_config.schema_tracker_cluster, config.env_config.rbr_state_cluster, is_avoid_internal_packages_set()) _per_process_mysql_daemon = launch_mysql_daemon() _session_prev_engine = db_connections.state_session.bind db_connections.state_session.bind = _per_process_mysql_daemon.engine db_connections.state_session.enforce_read_only = False yield db_connections.state_session db_connections.state_session.bind = _session_prev_engine
def is_pii_supported(cls): try: # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we are completely ready for open source. if is_avoid_internal_packages_set(): raise ImportError from pii_generator.components.pii_identifier import PIIIdentifier # NOQA return True except ImportError: return False
def get_base_model(): try: if is_avoid_internal_packages_set(): # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we have completely ready for open source. raise ImportError from yelp_conn.session import declarative_base return declarative_base() except ImportError: from sqlalchemy.ext.declarative import declarative_base return declarative_base()
def get_base_model(): try: if is_avoid_internal_packages_set(): # TODO(DATAPIPE-1509|abrar): Currently we have # force_avoid_internal_packages as a means of simulating an absence # of a yelp's internal package. And all references # of force_avoid_internal_packages have to be removed from # RH after we have completely ready for open source. raise ImportError from yelp_conn.session import declarative_base return declarative_base() except ImportError: from sqlalchemy.ext.declarative import declarative_base return declarative_base()
def database_sandbox_session(max_retries=3): db_connections = get_connection(config.env_config.topology_path, config.env_config.rbr_source_cluster, config.env_config.schema_tracker_cluster, config.env_config.rbr_state_cluster, is_avoid_internal_packages_set()) done_making_mysqld = False retries = 0 while not done_making_mysqld: # Takes time for mysqld to launch, so we will attempt a few times to it try: _per_process_mysql_daemon = PerProcessMySQLDaemon() done_making_mysqld = True except RuntimeError: retries += 1 if retries > max_retries: raise _session_prev_engine = db_connections.state_session.bind db_connections.state_session.bind = _per_process_mysql_daemon.engine db_connections.state_session.enforce_read_only = False yield db_connections.state_session db_connections.state_session.bind = _session_prev_engine