def __init__(self, db_settings, max_queue_len=None):
# There is no upper limit on the queue length by default
if not max_queue_len:
max_queue_len = -1
db_settings = self._filter_pool_settings(db_settings)
# Create delivery queue for database statements to be executed
self._queue = DeliveryQueue(
self._process_statement,
drop_callback=self._on_queue_drop,
max_queue_len=max_queue_len,
# Max parallel statements - should correspond with connection count
max_active_queue_len=db_settings['pool_size'],
# How often to retry deliver some
retry_after_s=5)
self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS)
self._drop_count = 0
# Create db-pool with the provided arguments
self._db_pool = self._create_db_pool(**db_settings)
self._second_db_pool = None
self._suspended = False
class JsonReporter(object):
def __init__(self, host, port,
max_queue_len=2048,
max_active_queue_len=100,
retry_after_s=5,
start_suspended=False):
self._link = ReporterJsonLinkClient(self._connection_made_int,
self._received_message)
self._queue = DeliveryQueue(self._link.sendMessage,
drop_callback=self.on_drop,
max_queue_len=max_queue_len,
max_active_queue_len=max_active_queue_len,
retry_after_s=retry_after_s,
start_suspended=start_suspended)
self._connector = reactor.connectTCP(host, port, self._link)
def redirect(self, host, port):
self._connector.stopConnecting()
self._connector.disconnect()
self._connector = reactor.connectTCP(host, port, self._link)
def _callback(self, result):
return True
def _errorback(self, failure):
return False
def _received_message(self, msg):
return True
def _connection_made_int(self, protocol):
try:
protocol.transport.setTcpNoDelay(True)
protocol.transport.setTcpKeepAlive(True)
# Seconds before sending keepalive probes
protocol.transport.socket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 120)
# Interval in seconds between keepalive probes
protocol.transport.socket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 1)
# Failed keepalive probles before declaring other end dead
protocol.transport.socket.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 5)
except:
pass
self._connection_made()
def _connection_made(self):
"""Default implementation does nothing."""
def deliver(self, event):
self._queue.deliver(event)
def report(self, event_name, payload):
obj = (event_name, payload)
self._queue.deliver(obj)
def on_drop(self, obj, reason):
pass
def __init__(self,
host,
port,
max_queue_len=2048,
max_active_queue_len=100,
retry_after_s=5,
start_suspended=False):
self._link = ReporterJsonLinkClient(self._connection_made_int,
self._received_message)
self._queue = DeliveryQueue(self._link.sendMessage,
drop_callback=self.on_drop,
max_queue_len=max_queue_len,
max_active_queue_len=max_active_queue_len,
retry_after_s=retry_after_s,
start_suspended=start_suspended)
self._connector = reactor.connectTCP(host, port, self._link)
def test_deferred_always_with_drops(self):
p = Processor('deferred_always')
q = DeliveryQueue(p.accept,
max_queue_len=20,
max_active_queue_len=1,
retry_after_s=0.1)
yield self._test_generic(q, range(50), p.get_result_list,
[0] + range(30, 50))
def __init__(self, host, port,
max_queue_len=2048,
max_active_queue_len=100,
retry_after_s=5,
start_suspended=False):
self._link = ReporterJsonLinkClient(self._connection_made_int,
self._received_message)
self._queue = DeliveryQueue(self._link.sendMessage,
drop_callback=self.on_drop,
max_queue_len=max_queue_len,
max_active_queue_len=max_active_queue_len,
retry_after_s=retry_after_s,
start_suspended=start_suspended)
self._connector = reactor.connectTCP(host, port, self._link)
def __init__(self, db_settings, max_queue_len=None):
# There is no upper limit on the queue length by default
if not max_queue_len:
max_queue_len = -1
db_settings = self._filter_pool_settings(db_settings)
# Create delivery queue for database statements to be executed
self._queue = DeliveryQueue(self._process_statement,
drop_callback=self._on_queue_drop,
max_queue_len=max_queue_len,
# Max parallel statements - should correspond with connection count
max_active_queue_len=db_settings['pool_size'],
# How often to retry deliver some
retry_after_s=5)
self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS)
self._drop_count = 0
# Create db-pool with the provided arguments
self._db_pool = self._create_db_pool(**db_settings)
self._second_db_pool = None
self._suspended = False
class DbSink(object):
def __init__(self, db_settings, max_queue_len=None):
# There is no upper limit on the queue length by default
if not max_queue_len:
max_queue_len = -1
db_settings = self._filter_pool_settings(db_settings)
# Create delivery queue for database statements to be executed
self._queue = DeliveryQueue(
self._process_statement,
drop_callback=self._on_queue_drop,
max_queue_len=max_queue_len,
# Max parallel statements - should correspond with connection count
max_active_queue_len=db_settings['pool_size'],
# How often to retry deliver some
retry_after_s=5)
self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS)
self._drop_count = 0
# Create db-pool with the provided arguments
self._db_pool = self._create_db_pool(**db_settings)
self._second_db_pool = None
self._suspended = False
@staticmethod
def _on_connect(conn):
log.info("New DB connection created")
conn.cursor().execute(
"set session transaction isolation level read uncommitted")
conn.autocommit(True)
def _create_db_pool(self, driver, host, port, user, password, dbname,
pool_size):
return adbapi.ConnectionPool(driver,
host=host,
port=port,
user=user,
passwd=password,
db=dbname,
cp_openfun=self._on_connect,
cp_reconnect=True,
cp_min=1,
cp_max=pool_size)
@staticmethod
def _filter_pool_settings(settings):
settings = {
'driver': settings.get('driver'),
'host': settings.get('host'),
'port': settings.get('port'),
'user': settings.get('user'),
'password': settings.get('password'),
'dbname': settings.get('dbname'),
'pool_size': settings.get('pool_size'),
}
if sum(val > 1 for val in settings.values()) != 7:
raise Exception('Some database parameter not specified (%s)' %
(str(settings)))
return settings
def get_queue_length(self):
return self._queue.get_length()
def get_stats(self):
return self._stats.get_data()
def reset_stats(self):
return self._stats.reset()
def execute(self, statement_name, statement):
return self._queue.deliver((statement_name, statement))
def direct_query(self, name, query):
return self._inspect(name, self._db_pool.runQuery(query), True)
def _process_statement(self, obj):
""" Deliver queue's processor """
# Try to deliver later, db_pool is not ready or the operation is suspended
if self._suspended or not self._db_pool:
return False
try:
(name, statement) = obj
return self._inspect(name, self._db_pool.runOperation(statement),
False)
except BaseException, e:
return e
class DbSink(object):
def __init__(self, db_settings, max_queue_len=None):
# There is no upper limit on the queue length by default
if not max_queue_len:
max_queue_len = -1
db_settings = self._filter_pool_settings(db_settings)
# Create delivery queue for database statements to be executed
self._queue = DeliveryQueue(self._process_statement,
drop_callback=self._on_queue_drop,
max_queue_len=max_queue_len,
# Max parallel statements - should correspond with connection count
max_active_queue_len=db_settings['pool_size'],
# How often to retry deliver some
retry_after_s=5)
self._stats = DeliveryStats(STATEMENTS_LATE_LIMIT_SECONDS)
self._drop_count = 0
# Create db-pool with the provided arguments
self._db_pool = self._create_db_pool(**db_settings)
self._second_db_pool = None
self._suspended = False
@staticmethod
def _on_connect(conn):
log.info("New DB connection created")
conn.cursor().execute("set session transaction isolation level read uncommitted")
conn.autocommit(True)
def _create_db_pool(self, driver, host, port, user, password, dbname, pool_size):
return adbapi.ConnectionPool(driver, host=host, port=port, user=user, passwd=password,
db=dbname, cp_openfun=self._on_connect, cp_reconnect=True,
cp_min=1, cp_max=pool_size)
@staticmethod
def _filter_pool_settings(settings):
settings = {
'driver': settings.get('driver'),
'host': settings.get('host'),
'port': settings.get('port'),
'user': settings.get('user'),
'password': settings.get('password'),
'dbname': settings.get('dbname'),
'pool_size': settings.get('pool_size'),
}
if sum(val > 1 for val in settings.values()) != 7:
raise Exception('Some database parameter not specified (%s)' % (str(settings)))
return settings
def get_queue_length(self):
return self._queue.get_length()
def get_stats(self):
return self._stats.get_data()
def reset_stats(self):
return self._stats.reset()
def execute(self, statement_name, statement):
return self._queue.deliver((statement_name, statement))
def direct_query(self, name, query):
return self._inspect(name, self._db_pool.runQuery(query), True)
def _process_statement(self, obj):
""" Deliver queue's processor """
# Try to deliver later, db_pool is not ready or the operation is suspended
if self._suspended or not self._db_pool:
return False
try:
(name, statement) = obj
return self._inspect(name, self._db_pool.runOperation(statement), False)
except BaseException, e:
return e
class JsonReporter(object):
def __init__(self,
host,
port,
max_queue_len=2048,
max_active_queue_len=100,
retry_after_s=5,
start_suspended=False):
self._link = ReporterJsonLinkClient(self._connection_made_int,
self._received_message)
self._queue = DeliveryQueue(self._link.sendMessage,
drop_callback=self.on_drop,
max_queue_len=max_queue_len,
max_active_queue_len=max_active_queue_len,
retry_after_s=retry_after_s,
start_suspended=start_suspended)
self._connector = reactor.connectTCP(host, port, self._link)
def redirect(self, host, port):
self._connector.stopConnecting()
self._connector.disconnect()
self._connector = reactor.connectTCP(host, port, self._link)
def _callback(self, result):
return True
def _errorback(self, failure):
return False
def _received_message(self, msg):
return True
def _connection_made_int(self, protocol):
try:
protocol.transport.setTcpNoDelay(True)
protocol.transport.setTcpKeepAlive(True)
# Seconds before sending keepalive probes
protocol.transport.socket.setsockopt(socket.SOL_TCP,
socket.TCP_KEEPIDLE, 120)
# Interval in seconds between keepalive probes
protocol.transport.socket.setsockopt(socket.SOL_TCP,
socket.TCP_KEEPINTVL, 1)
# Failed keepalive probles before declaring other end dead
protocol.transport.socket.setsockopt(socket.SOL_TCP,
socket.TCP_KEEPCNT, 5)
except:
pass
self._connection_made()
def _connection_made(self):
"""Default implementation does nothing."""
def deliver(self, event):
self._queue.deliver(event)
def report(self, event_name, payload):
obj = (event_name, payload)
self._queue.deliver(obj)
def on_drop(self, obj, reason):
pass
def test_deferred_always(self):
p = Processor('deferred_always')
q = DeliveryQueue(p.accept, max_active_queue_len=5, retry_after_s=0.1)
yield self._test_generic(q, range(10), p.get_result_list)
def test_second(self):
p = Processor('second')
q = DeliveryQueue(p.accept, max_active_queue_len=3, retry_after_s=0.1)
yield self._test_generic(q, set(range(10)), p.get_result_set)
def test_always(self):
p = Processor('always')
q = DeliveryQueue(p.accept)
yield self._test_generic(q, range(10), p.get_result_list)
def test_exception(self):
p = Processor('exception')
q = DeliveryQueue(p.accept, max_active_queue_len=1, retry_after_s=0.1)
yield self._test_generic(q, set(range(10)), p.get_result_set, set([]))
