def get_new_l4_blocks() -> List[bytes]: """Get all new l4 records from the incoming queue""" if LEVEL != "5": raise RuntimeError("Getting l4_blocks is a level 5 action") l4_blocks = [] for _ in range(0, redis.llen_sync(INCOMING_TX_KEY)): # These are in lists because enterprise will be able to specify more than one l4. l4_blocks_list = cast(bytes, redis.rpoplpush_sync(INCOMING_TX_KEY, PROCESSING_TX_KEY, decode=False)) l4_blocks.append(l4_blocks_list) return l4_blocks
def get_new_transactions() -> List[transaction_model.TransactionModel]: """Get all new transactions from the incoming queue""" if LEVEL != "1": raise RuntimeError("Getting transactions is a level 1 action") transactions = [] # Only allow up to 1000 transactions to process at a time length = min(redis.llen_sync(INCOMING_TX_KEY), 1000) for _ in range(0, length): string = cast(bytes, redis.rpoplpush_sync(INCOMING_TX_KEY, PROCESSING_TX_KEY, decode=False)) dictionary = json.loads(string) txn_model = transaction_model.new_from_queue_input(dictionary) transactions.append(txn_model) return transactions
def check_and_recover_processing_if_necessary() -> None: """ Checks the processing tx queue and returns them to the incoming queue (Should be called before starting to process a new block, for unexpected crash recovery) """ if redis.llen_sync(PROCESSING_TX_KEY) != 0: _log.warning( "WARNING! Processing queue was not empty. Last block processing probably crashed. Recovering and re-queuing these dropped items." ) to_recover = redis.lrange_sync(PROCESSING_TX_KEY, 0, -1, decode=False) # Execute these in a pipeline in attempts to make this as atomic as possible p = redis.pipeline_sync() p.rpush(INCOMING_TX_KEY, *to_recover) p.delete(PROCESSING_TX_KEY) p.execute()
def get_new_transactions() -> List[transaction_model.TransactionModel]: """Get all new transactions from the incoming queue""" if LEVEL != "1": raise RuntimeError("Getting transactions is a level 1 action") transactions = [] # Only allow up to 1000 transactions to process at a time length = min(redis.llen_sync(INCOMING_TX_KEY), 10000) p = redis.pipeline_sync() for _ in range(0, length): p.rpoplpush(INCOMING_TX_KEY, PROCESSING_TX_KEY) for value in p.execute(): dictionary = json.loads(value) txn_model = transaction_model.new_from_queue_input(dictionary) transactions.append(txn_model) return transactions
def start() -> None: """Start the next job in the queue""" _log.debug("Connecting to service account") kubernetes.config.load_incluster_config() _log.debug("Creating kubernetes client") global _kube _kube = kubernetes.client.BatchV1Api() _log.debug("Job processor ready!") if redis.llen_sync(PENDING_TASK_KEY): _log.warning( "WARNING! Pending job processor queue was not empty. Last job probably crashed. Re-queueing these dropped items." ) to_recover = redis.lrange_sync(PENDING_TASK_KEY, 0, -1, decode=False) p = redis.pipeline_sync() p.rpush(CONTRACT_TASK_KEY, *to_recover) p.delete(PENDING_TASK_KEY) p.execute() while True: start_task()
def is_not_empty() -> bool: """Check if there is another block in the queue ready to process""" return redis.llen_sync(INCOMING_TX_KEY) != 0
def test_llen(self): redis.llen_sync("banana") redis.redis_client.llen.assert_called_once_with("banana")