def setUp(self):
self.db_session = create_memory_db()
with make_session_scope(self.db_session) as session:
self.subs1 = Subscription(
address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
callback_url='http://localhost:9779')
self.subs2 = Subscription(
address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
callback_url='http://localhost:9779',
expiration=datetime.utcnow() - timedelta(days=300))
self.subs3 = Subscription(
address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
callback_url='http://localhost:9779',
state=SubscriptionState.canceled)
self.subs4 = Subscription(
address='n2SjFgAhHAv8PcTuq5x2e9suffXspmtztt',
callback_url='http://localhost:9779')
session.add(self.subs1)
session.add(self.subs2)
session.add(self.subs3)
session.add(self.subs4)
# Create equivalent SubscriptionData commands
self.com1 = self.subs1.to_subscription_data()
self.com2 = self.subs2.to_subscription_data()
self.com3 = self.subs3.to_subscription_data()
self.com4 = self.subs4.to_subscription_data()
def setUp(self):
# Initialized in memory test db
self.db_session = create_memory_db()
# Subscription where all test callbacks will originate
with make_session_scope(self.db_session) as session:
self.subscription = Subscription(
address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
callback_url='http://localhost:9779')
session.add(self.subscription)
self.subscription_data = SubscriptionData(
id=self.subscription.id,
address=self.subscription.address,
expiration=self.subscription.expiration,
callback_url=self.subscription.callback_url)
# Queue to store callback requests received by http_server thread
self.requests = Queue()
# Launch listen server
self.http_server = threading.Thread(
target=TestCallbackRequests.http_server,
args=(self.requests,), daemon=True)
self.http_server.start()
time.sleep(0.1)
# Launch server
self.callback_manager = CallbackManager(
self.db_session, retries=3,
nthreads=5, retry_period=30)
def _remove_expired_subscriptions(self):
"""
"""
expired = []
now = datetime.datetime.utcnow()
# Remove expired subscriptions
while self._expiration_table and self._expiration_table[0][0] < now:
expiration, sub_id = heapq.heappop(self._expiration_table)
# If the subscription isn't canceled store id
if sub_id in self._subs_by_id:
expired.append(sub_id)
self.cancel_subscription(sub_id)
if not expired:
return
# Change status to expired for all expired and still active subscriptions
with make_session_scope(self._db_session) as session:
session.query(Subscription).filter(Subscription.id.in_(expired)).\
update({'state':SubscriptionState.expired}, synchronize_session=False)
# Stop monitoring for all the subscription
for sub_id in expired:
self.cancel_subscription(sub_id)
def _init_task(self, settings):
"""Initialize task after process is forked"""
# Last time bitcoind was polled or reconnect tried
self._last_update = time.perf_counter()
# We need to create a new DB session for the process, because the
# one used by flask can be only be share between threads.
self._db_session = configure_db(self._settings['DB_URI'])
# Find block number where monitoring stoped last time
self._current_block = -1 # Start by newest block
if self._settings['START_BLOCK'] == 'last':
with make_session_scope(self._db_session) as session:
try:
block = session.query(Block).one()
self._current_block = block.block_number
except sqlalchemy.orm.exc.NoResultFound:
# Not found, it can happen the first time the app is executed
# so there is no need to log the exception
pass
# bitcoin lib chain selection
bitcoin.SelectParams(self._settings['CHAIN'])
# It will be initialized later by reconnect code
self._monitor = None
# Transaction monitor is not provided so it is not initialized here
# so it's treated later as if the connection was lost.
self._subscription_manager = SubscriptionManager(
self._monitor, # Init later
self._db_session,
settings['RELOAD_SUBSCRIPTIONS'])
def _send_ready(self):
"""Enqueue ready to send callbacks into thread_pool job queue"""
# Send callbacks ready for a retry
while len(self._retry_q):
callback_id = self._next_sent()
if not callback_id:
break
# Construct callback request json
with self._db_lock:
with make_session_scope(self._db_session) as session:
callback = session.query(Callback).get(callback_id)
json = callback.to_request()
url = callback.subscription.callback_url
# Add to thread pool job queue
try:
job = (callback_id, json, url)
self._thread_pool.add_job(job, block=False)
except queue.Full:
with self._lock:
# If the queue if full wait until next update
self._retry_q.appendleft(callback_id)
break
def new_callback(self, callback):
"""Add new callback to queue"""
callback = Callback.from_callback_data(callback,
retries=self.retries + 1)
with self._db_lock:
with make_session_scope(self._db_session) as session:
session.add(callback)
with self._lock:
record = CallbackRecord(callback.id, callback.retries,
callback.last_retry)
self._callbacks[callback.id] = record
# Set callback as next for delivery
self._retry_q.appendleft(callback.id)
def _save_block_number(self, block_number):
"""Save block number into db, create row if it doesn't exist, update
if it does.
Arguments:
block_number (int): positive integer
"""
assert isinstance(block_number, int) and block_number >= 0
with make_session_scope(self._db_session) as session:
try:
block = session.query(Block).one()
block.block_number = block_number
session.add(block)
except sqlalchemy.orm.exc.NoResultFound:
block = Block(block_number=block_number)
session.add(block)
def test_expired_are_discarded(self):
"""Test expired subscription are discarded and their state updated in DB"""
monitor = MockTransactionMonitor()
subscription_manager = SubscriptionManager(monitor, self.db_session)
self.assertEqual(len(subscription_manager), 3)
self.assertEqual(len(monitor), 2)
# Calling poll_bitcoin should remove expired subscriptions
subscription_manager.poll_bitcoin()
with make_session_scope(self.db_session) as session:
expired_subs = session.query(Subscription).filter_by(
id=self.subs2.id).first()
self.assertEqual(expired_subs.state, SubscriptionState.expired)
self.assertEqual(len(subscription_manager), 2)
self.assertEqual(len(monitor), 2)
def ack_callback(self, callback_id):
"""Mark callback as acknolewdged, return False if it didn't exist
True otherwise"""
# The callback is removed from the callback dictionary to
# signify it was acknowledged
with self._lock:
callback = self._callbacks.pop(callback_id, None)
# Check there was a callback with the given id
if callback is None:
return False
with self._db_lock:
with make_session_scope(self._db_session) as session:
session.query(Callback).filter_by(id=callback_id)\
.update({'acknowledged': True})
return True
def setUp(self):
self.db_session = create_memory_db()
# Create subscription that will be used during tests
with make_session_scope(self.db_session) as session:
self.subscription = Subscription(
address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
callback_url='http://localhost:8080')
session.add(self.subscription)
# Subscription as a command data
self.subscription_data = SubscriptionData(
id=self.subscription.id,
address=self.subscription.address,
expiration=self.subscription.expiration,
callback_url=self.subscription.callback_url)
self.callback_manager = CallbackManager(
self.db_session, retries=3,
retry_period=30, nthreads=5)
def __init__(self, monitor, db_session, db_reload=True):
"""
Arguments:
monitor (bitmon.TransactionMonitor): Initialized monitor.
db_session (scoped_session):
db_reload (bool): Reload subscriptions from database
"""
# Subscriptions by address
self._subs_by_addr = defaultdict(set)
#
self._subs_by_id = dict()
#
self.set_transaction_monitor(monitor)
self._db_session = db_session
# Subscription list sorted by expiration date
self._expiration_table = []
heapq.heapify(self._expiration_table)
# Load stil active subscriptions from db
if not db_reload:
return
with make_session_scope(self._db_session) as session:
active_subs = session.query(Subscription).filter_by(
state=SubscriptionState.active).all()
for sub in active_subs:
# Expired subscriptions will be discarded the first time poll_bitcoin
# is called
logger.debug("Loaded Subscription: {}".format(sub))
sub_data = SubscriptionData(sub.id, sub.address, sub.callback_url,
sub.expiration)
self.add_subscription(sub_data)
def _process_sent(self):
"""Process callbacks marked as sent by the thread_pool"""
# Process all callbacks marked as sent by thread_pool
while True:
try:
callback_id = self._sent_q.get(block=True, timeout=1)
with self._lock:
# If callback was acknowledged while being sent discard it
# and keep looping
record = self._callbacks.get(callback_id, None)
if not record:
continue
# Update callback and enqueue for a retry if there are any remaining,
# otherwise discard it.
if record.retries > 0:
record = CallbackRecord(callback_id,
record.retries - 1,
datetime.utcnow())
self._callbacks[callback_id] = record
self._retry_q.append(callback_id)
else:
del self._callbacks[callback_id]
# Save all changes to db
with self._db_lock:
with make_session_scope(self._db_session) as session:
update_fields = {
'retries': record.retries,
'last_retry': record.last_retry
}
session.query(Callback).filter_by(id=record.id)\
.update(update_fields)
except queue.Empty:
# No callbacks remainig at the queue
break
def __init__(self,
db_session,
retries=3,
retry_period=120,
nthreads=10,
recover_db=True):
# Dictionary containing all callback indexed by (txid, addr)
self._callbacks = {}
# Queue of callbacks ids to retry ordered by last retry time
self._retry_q = collections.deque()
# SQLAlchemy session
self._db_session = db_session
# Queue where ThreadPool threads stores the ids of sent callbacks
self._sent_q = queue.Queue()
# Lock for every thing excepts DB access
self._lock = threading.Lock()
# DB access lock
self._db_lock = threading.Lock()
# Max number of callback retries
self.retries = retries
# Wait between sucessive unacknowledged callback tries
self.retry_period = retry_period
# Start request thread pool
self._thread_pool = ThreadPool(nthreads=nthreads,
func=CallbackManager._send_thread_func,
args=(self._sent_q, ))
# Flag used to notify update_thread to stop
self._close_flag = threading.Event()
# Start update thread
self._update_thread = threading.Thread(
target=CallbackManager._update_func, args=(self, ), daemon=True)
self._update_thread.start()
# Recover unfinished callbacks from DB
if not recover_db:
return
# Load unfinished callbacks from DB
with self._db_lock:
with make_session_scope(self._db_session) as session:
pending = session.query(Callback).filter(
Callback.acknowledged == False,
Callback.retries > 0).all()
# Add unfinished to retry queue.
pending = sorted(pending, key=lambda c: c.last_retry)
with self._lock:
for cback in pending:
record = CallbackRecord(cback.id, cback.retries,
cback.last_retry)
self._callbacks[record.id] = record
self._retry_q.append(record.id)