def _heartbeat_thread_job(self):
"""Thread that maintains inactive connections
"""
# while not self._heartbeat_exit_event.is_set():
while not self._heartbeat_exit_event:
start = monotonic()
with self._connection_lock.priority(0):
if self._heartbeat_exit_event:
break
recoverable_errors = (
self.connection.recoverable_channel_errors +
self.connection.recoverable_connection_errors)
try:
try:
self._heartbeat_check()
try:
self.connection.drain_events(timeout=0.001)
except socket.timeout:
pass
except recoverable_errors as exc:
LOG.info("A recoverable connection/channel error occurred, "
"trying to reconnect: %s" % exc)
self.ensure_connection()
except Exception:
LOG.warning("Unexpected error during heartbeart "
"thread processing, retrying...")
LOG.debug('Exception', exc_info=True)
if self._heartbeat_exit_event:
break
sleep_time = self._heartbeat_wait_timeout + start - monotonic()
if sleep_time >= 0.0:
eventlet.sleep(sleep_time)
def start(self, timeout=5.0):
"""
循环启动
"""
timeout = float(timeout)
self.running = True
# 孵化心跳循环线程
eventlet.spawn_n(self.heart_beat_loop)
eventlet.sleep(0.1)
overtime = monotonic() + timeout
while True:
if self.connection_pool._created_connections > 0:
break
if monotonic() > overtime:
LOG.error('Redis connection pool init fail')
self.running = False
# 等待前面的绿色线程结束
# eventlet.sleep(1.0)
raise ConnectionError('redis connection pool empty over %1.2f seconds' % timeout)
eventlet.sleep(timeout/5.0)
# 孵化垃圾key删除循环
eventlet.spawn_n(self.garbage_collector_loop)
def heart_beat_loop(self):
"""
心跳循环,用于外部domain线程
PING命令不使用execute_command
"""
pool = self.connection_pool
heart_interval = float(self.heart_beat_over_time)*(float(self.heart_beat_over_time_max_count))
heart_interval_s = heart_interval/1000
error_connection_count = 0
while True:
if not self.running:
break
start_time = monotonic()*1000
connection = None
try:
connection = pool.get_connection('PING')
# 上次链接传包时间大于心跳间隔,发送PING命令
if start_time - connection.last_beat > heart_interval:
connection.send_command("PING")
self.parse_response(connection, "PING")
# 正常释放连接,链接重回可用连接池中
pool.release(connection)
if error_connection_count:
error_connection_count -= 1
except (ConnectionError, TimeoutError) as e:
LOG.warning('Heartbeat loop %(class)s: %(message)s' % {'class': e.__class__.__name__,
'message': e.message})
if connection:
# 踢出连接
pool.kickout(connection)
# 有链接还连续报错
# 多等待一段时间
if error_connection_count:
LOG.error('Heartbeat loop more then once, sleep more time')
eventlet.sleep(heart_interval_s*2)
else:
# 分配不到链接,说明全部链接在忙或者无法生成链接(网络故障)
eventlet.sleep(heart_interval_s)
error_connection_count += 1
# 切换到其他绿色线程
eventlet.sleep(heart_interval_s)
def __init__(self, conf, purpose):
# NOTE(viktors): Parse config options
# driver_conf = conf.rabbit
self.max_retries = conf.rabbit_max_retries
self.interval_start = conf.rabbit_retry_interval
self.interval_stepping = conf.rabbit_retry_backoff
self.interval_max = conf.rabbit_interval_max
self.login_method = conf.rabbit_login_method
# self.fake_rabbit = driver_conf.fake_rabbit
self.virtual_host = conf.rabbit_virtual_host
self.rabbit_host = conf.rabbit_host
# self.rabbit_hosts = driver_conf.rabbit_hosts
self.rabbit_port = conf.rabbit_port
self.rabbit_userid = conf.rabbit_userid
self.rabbit_password = conf.rabbit_password
self.rabbit_ha_queues = conf.rabbit_ha_queues
self.rabbit_transient_queues_ttl = conf.rabbit_transient_queues_ttl
self.rabbit_connect_timeout = conf.rabbit_connect_timeout
self.rabbit_qos_prefetch_count = conf.rabbit_qos_prefetch_count
self.heartbeat_timeout_threshold = conf.heartbeat_timeout_threshold
self.heartbeat_rate = conf.heartbeat_rate
self.kombu_reconnect_delay = conf.kombu_reconnect_delay
self.amqp_durable_queues = conf.amqp_durable_queues
self.amqp_auto_delete = conf.amqp_auto_delete
self.rabbit_use_ssl = conf.rabbit_use_ssl
self.kombu_missing_consumer_retry_timeout = \
conf.kombu_missing_consumer_retry_timeout
self.kombu_failover_strategy = conf.kombu_failover_strategy
self.kombu_compression = conf.kombu_compression
if self.rabbit_use_ssl:
self.kombu_ssl_version = conf.kombu_ssl_version
self.kombu_ssl_keyfile = conf.kombu_ssl_keyfile
self.kombu_ssl_certfile = conf.kombu_ssl_certfile
self.kombu_ssl_ca_certs = conf.kombu_ssl_ca_certs
# Try forever?
if self.max_retries <= 0:
self.max_retries = None
self._url = "amqp://%(username)s:%(password)s" \
"@%(host)s:%(port)s/%(vhost)s" % \
{'username': self.rabbit_userid, 'password': self.rabbit_password,
'host':self.rabbit_host, 'port':self.rabbit_port,
'vhost':self.virtual_host}
self._initial_pid = os.getpid()
self._consumers = {}
self._new_tags = set()
self._active_tags = {}
self._tags = itertools.count(1)
self._consume_loop_stopped = False
self.channel = None
self.purpose = purpose
self.last_time = monotonic()
if purpose == rpc_common.PURPOSE_SEND:
self._connection_lock = PriorityLock()
self._connection_lock.set_defalut_priority(1)
else:
self._connection_lock = DummyLock()
# kombu set cloexec after socket connect
# see function amqp.transport._AbstractTransport._connect
self.connection = kombu.connection.Connection(
self._url, ssl=self._fetch_ssl_params(),
login_method=self.login_method,
heartbeat=self.heartbeat_timeout_threshold,
failover_strategy=self.kombu_failover_strategy,
connect_timeout=self.rabbit_connect_timeout,
transport_options={
'confirm_publish': True,
'client_properties': {'capabilities': {
'authentication_failure_close': True,
'connection.blocked': True,
'consumer_cancel_notify': True}},
'on_blocked': self._on_connection_blocked,
'on_unblocked': self._on_connection_unblocked,
},
)
LOG.debug('Connecting to AMQP server on %(hostname)s:%(port)s' % self.connection.info())
self._heartbeat_wait_timeout = (
float(self.heartbeat_timeout_threshold) /
float(self.heartbeat_rate) / 2.0)
self._heartbeat_support_log_emitted = False
# NOTE(sileht): just ensure the connection is setuped at startup
self.ensure_connection()
# NOTE(sileht): if purpose is PURPOSE_LISTEN
# the consume code does the heartbeat stuff
# we don't need a thread
self._heartbeat_thread = None
if purpose == rpc_common.PURPOSE_SEND:
self._heartbeat_start()
info = self.connection.info()
if isinstance(info, tuple):
info = info[0]
LOG.debug('Connected to AMQP server on %(hostname)s:%(port)s '
'via [%(transport)s] client' % info)
# NOTE(sileht): value chosen according the best practice from kombu
# http://kombu.readthedocs.org/en/latest/reference/kombu.common.html#kombu.common.eventloop
# For heatbeat, we can set a bigger timeout, and check we receive the
# heartbeat packets regulary
if self._heartbeat_supported_and_enabled():
self._poll_timeout = self._heartbeat_wait_timeout
else:
self._poll_timeout = 1
def read_response(self):
result = super(ConnectionEx, self).read_response()
# 更新心跳时间
self.last_beat = monotonic() * 1000
return result
def connect(self):
if self._sock:
return
super(ConnectionEx, self).connect()
self.last_beat = monotonic() * 1000