def setUp(self): self.db_session = create_memory_db() with make_session_scope(self.db_session) as session: self.subs1 = Subscription( address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7', callback_url='http://localhost:9779') self.subs2 = Subscription( address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7', callback_url='http://localhost:9779', expiration=datetime.utcnow() - timedelta(days=300)) self.subs3 = Subscription( address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7', callback_url='http://localhost:9779', state=SubscriptionState.canceled) self.subs4 = Subscription( address='n2SjFgAhHAv8PcTuq5x2e9suffXspmtztt', callback_url='http://localhost:9779') session.add(self.subs1) session.add(self.subs2) session.add(self.subs3) session.add(self.subs4) # Create equivalent SubscriptionData commands self.com1 = self.subs1.to_subscription_data() self.com2 = self.subs2.to_subscription_data() self.com3 = self.subs3.to_subscription_data() self.com4 = self.subs4.to_subscription_data()
def setUp(self): # Initialized in memory test db self.db_session = create_memory_db() # Subscription where all test callbacks will originate with make_session_scope(self.db_session) as session: self.subscription = Subscription( address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7', callback_url='http://localhost:9779') session.add(self.subscription) self.subscription_data = SubscriptionData( id=self.subscription.id, address=self.subscription.address, expiration=self.subscription.expiration, callback_url=self.subscription.callback_url) # Queue to store callback requests received by http_server thread self.requests = Queue() # Launch listen server self.http_server = threading.Thread( target=TestCallbackRequests.http_server, args=(self.requests,), daemon=True) self.http_server.start() time.sleep(0.1) # Launch server self.callback_manager = CallbackManager( self.db_session, retries=3, nthreads=5, retry_period=30)
def _remove_expired_subscriptions(self): """ """ expired = [] now = datetime.datetime.utcnow() # Remove expired subscriptions while self._expiration_table and self._expiration_table[0][0] < now: expiration, sub_id = heapq.heappop(self._expiration_table) # If the subscription isn't canceled store id if sub_id in self._subs_by_id: expired.append(sub_id) self.cancel_subscription(sub_id) if not expired: return # Change status to expired for all expired and still active subscriptions with make_session_scope(self._db_session) as session: session.query(Subscription).filter(Subscription.id.in_(expired)).\ update({'state':SubscriptionState.expired}, synchronize_session=False) # Stop monitoring for all the subscription for sub_id in expired: self.cancel_subscription(sub_id)
def _init_task(self, settings): """Initialize task after process is forked""" # Last time bitcoind was polled or reconnect tried self._last_update = time.perf_counter() # We need to create a new DB session for the process, because the # one used by flask can be only be share between threads. self._db_session = configure_db(self._settings['DB_URI']) # Find block number where monitoring stoped last time self._current_block = -1 # Start by newest block if self._settings['START_BLOCK'] == 'last': with make_session_scope(self._db_session) as session: try: block = session.query(Block).one() self._current_block = block.block_number except sqlalchemy.orm.exc.NoResultFound: # Not found, it can happen the first time the app is executed # so there is no need to log the exception pass # bitcoin lib chain selection bitcoin.SelectParams(self._settings['CHAIN']) # It will be initialized later by reconnect code self._monitor = None # Transaction monitor is not provided so it is not initialized here # so it's treated later as if the connection was lost. self._subscription_manager = SubscriptionManager( self._monitor, # Init later self._db_session, settings['RELOAD_SUBSCRIPTIONS'])
def _send_ready(self): """Enqueue ready to send callbacks into thread_pool job queue""" # Send callbacks ready for a retry while len(self._retry_q): callback_id = self._next_sent() if not callback_id: break # Construct callback request json with self._db_lock: with make_session_scope(self._db_session) as session: callback = session.query(Callback).get(callback_id) json = callback.to_request() url = callback.subscription.callback_url # Add to thread pool job queue try: job = (callback_id, json, url) self._thread_pool.add_job(job, block=False) except queue.Full: with self._lock: # If the queue if full wait until next update self._retry_q.appendleft(callback_id) break
def new_callback(self, callback): """Add new callback to queue""" callback = Callback.from_callback_data(callback, retries=self.retries + 1) with self._db_lock: with make_session_scope(self._db_session) as session: session.add(callback) with self._lock: record = CallbackRecord(callback.id, callback.retries, callback.last_retry) self._callbacks[callback.id] = record # Set callback as next for delivery self._retry_q.appendleft(callback.id)
def _save_block_number(self, block_number): """Save block number into db, create row if it doesn't exist, update if it does. Arguments: block_number (int): positive integer """ assert isinstance(block_number, int) and block_number >= 0 with make_session_scope(self._db_session) as session: try: block = session.query(Block).one() block.block_number = block_number session.add(block) except sqlalchemy.orm.exc.NoResultFound: block = Block(block_number=block_number) session.add(block)
def test_expired_are_discarded(self): """Test expired subscription are discarded and their state updated in DB""" monitor = MockTransactionMonitor() subscription_manager = SubscriptionManager(monitor, self.db_session) self.assertEqual(len(subscription_manager), 3) self.assertEqual(len(monitor), 2) # Calling poll_bitcoin should remove expired subscriptions subscription_manager.poll_bitcoin() with make_session_scope(self.db_session) as session: expired_subs = session.query(Subscription).filter_by( id=self.subs2.id).first() self.assertEqual(expired_subs.state, SubscriptionState.expired) self.assertEqual(len(subscription_manager), 2) self.assertEqual(len(monitor), 2)
def ack_callback(self, callback_id): """Mark callback as acknolewdged, return False if it didn't exist True otherwise""" # The callback is removed from the callback dictionary to # signify it was acknowledged with self._lock: callback = self._callbacks.pop(callback_id, None) # Check there was a callback with the given id if callback is None: return False with self._db_lock: with make_session_scope(self._db_session) as session: session.query(Callback).filter_by(id=callback_id)\ .update({'acknowledged': True}) return True
def setUp(self): self.db_session = create_memory_db() # Create subscription that will be used during tests with make_session_scope(self.db_session) as session: self.subscription = Subscription( address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7', callback_url='http://localhost:8080') session.add(self.subscription) # Subscription as a command data self.subscription_data = SubscriptionData( id=self.subscription.id, address=self.subscription.address, expiration=self.subscription.expiration, callback_url=self.subscription.callback_url) self.callback_manager = CallbackManager( self.db_session, retries=3, retry_period=30, nthreads=5)
def __init__(self, monitor, db_session, db_reload=True): """ Arguments: monitor (bitmon.TransactionMonitor): Initialized monitor. db_session (scoped_session): db_reload (bool): Reload subscriptions from database """ # Subscriptions by address self._subs_by_addr = defaultdict(set) # self._subs_by_id = dict() # self.set_transaction_monitor(monitor) self._db_session = db_session # Subscription list sorted by expiration date self._expiration_table = [] heapq.heapify(self._expiration_table) # Load stil active subscriptions from db if not db_reload: return with make_session_scope(self._db_session) as session: active_subs = session.query(Subscription).filter_by( state=SubscriptionState.active).all() for sub in active_subs: # Expired subscriptions will be discarded the first time poll_bitcoin # is called logger.debug("Loaded Subscription: {}".format(sub)) sub_data = SubscriptionData(sub.id, sub.address, sub.callback_url, sub.expiration) self.add_subscription(sub_data)
def _process_sent(self): """Process callbacks marked as sent by the thread_pool""" # Process all callbacks marked as sent by thread_pool while True: try: callback_id = self._sent_q.get(block=True, timeout=1) with self._lock: # If callback was acknowledged while being sent discard it # and keep looping record = self._callbacks.get(callback_id, None) if not record: continue # Update callback and enqueue for a retry if there are any remaining, # otherwise discard it. if record.retries > 0: record = CallbackRecord(callback_id, record.retries - 1, datetime.utcnow()) self._callbacks[callback_id] = record self._retry_q.append(callback_id) else: del self._callbacks[callback_id] # Save all changes to db with self._db_lock: with make_session_scope(self._db_session) as session: update_fields = { 'retries': record.retries, 'last_retry': record.last_retry } session.query(Callback).filter_by(id=record.id)\ .update(update_fields) except queue.Empty: # No callbacks remainig at the queue break
def __init__(self, db_session, retries=3, retry_period=120, nthreads=10, recover_db=True): # Dictionary containing all callback indexed by (txid, addr) self._callbacks = {} # Queue of callbacks ids to retry ordered by last retry time self._retry_q = collections.deque() # SQLAlchemy session self._db_session = db_session # Queue where ThreadPool threads stores the ids of sent callbacks self._sent_q = queue.Queue() # Lock for every thing excepts DB access self._lock = threading.Lock() # DB access lock self._db_lock = threading.Lock() # Max number of callback retries self.retries = retries # Wait between sucessive unacknowledged callback tries self.retry_period = retry_period # Start request thread pool self._thread_pool = ThreadPool(nthreads=nthreads, func=CallbackManager._send_thread_func, args=(self._sent_q, )) # Flag used to notify update_thread to stop self._close_flag = threading.Event() # Start update thread self._update_thread = threading.Thread( target=CallbackManager._update_func, args=(self, ), daemon=True) self._update_thread.start() # Recover unfinished callbacks from DB if not recover_db: return # Load unfinished callbacks from DB with self._db_lock: with make_session_scope(self._db_session) as session: pending = session.query(Callback).filter( Callback.acknowledged == False, Callback.retries > 0).all() # Add unfinished to retry queue. pending = sorted(pending, key=lambda c: c.last_retry) with self._lock: for cback in pending: record = CallbackRecord(cback.id, cback.retries, cback.last_retry) self._callbacks[record.id] = record self._retry_q.append(record.id)