def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# self.transfer_queue = Queue()
# self.transfer_queue_task = ensure_future(self.evaluate_transfer_queue())
# Add state db
if not kwargs.get("settings"):
self._settings = PaymentSettings()
self.state_db = PaymentState(self._settings.asset_precision)
self.context = self.state_db.context
self.reachability_cache = defaultdict(lambda: cachetools.LRUCache(100))
# Dictionary chain_id: block_dot -> block
self.tracked_blocks = defaultdict(lambda: {})
self.peer_conf = defaultdict(lambda: defaultdict(int))
self.should_witness_subcom = {}
self.counter_signing_block_queue = PriorityQueue()
self.block_sign_queue_task = ensure_future(
self.evaluate_counter_signing_blocks()
)
self.witness_delta = kwargs.get("witness_delta")
if not self.witness_delta:
self.witness_delta = self.settings.witness_block_delta
async def init(self, setting):
"""
初始化
"""
self.setting = setting
self.waiting = PriorityQueue()
self.module = importlib.import_module(setting.SERIALIZATION)
def __init__(self, *params):
self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \
self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [
dns.asyncresolver.Resolver(configure=False)
for _ in range(self.options.threads)
]
for r in self.resolvers:
r.lifetime = 6.0
r.timeout = 10.0
self.queue = PriorityQueue()
self.ip_dict = {}
self.found_subs = set()
self.cert_subs = set()
self.timeout_subs = {}
self.no_server_subs = {}
self.count_time = time.time()
self.outfile = open(
'%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num),
'w')
self.normal_names_set = set()
self.lock = asyncio.Lock()
self.threads_status = ['1'] * self.options.threads
def __init__(self,
check_url,
allowed_anonymity_levels=None,
qps_per_proxy=1,
max_consecutive_failures=5,
providers=PROVIDERS,
timeout=5):
self._proxies = Queue()
self._pending_providers = Queue()
self._providers = providers
self._verified_proxies = {}
self._throttled_proxies = PriorityQueue()
self._errors = {}
self._check_url = check_url
self._qps_per_proxy = qps_per_proxy
self._max_consecutive_failures = max_consecutive_failures
self._timeout = timeout
self._ip = None
self._ip_lock = Lock()
if not allowed_anonymity_levels:
self._allowed_anonymity_levels = ['Anonymous', 'Elite']
else:
self._allowed_anonymity_levels = allowed_anonymity_levels
class EventQueue:
__queue = None
def __init__(self):
self.__locker = {}
self.__queue = PriorityQueue()
async def consume(self):
while True:
_, caller, data = await self.__queue.get()
wait_for(caller.sendData(data), MAX_TIMEOUT)
if caller in self.__locker:
self.__locker[caller].release()
if not isRealtime():
# Wait for the lock to be locked
while not self.__locker[caller].locked():
await sleep(0.001)
if caller not in self.__locker:
break
async def put(self, caller, data):
if caller not in self.__locker:
self.__locker[caller] = Lock()
await self.__locker[caller]
self.__queue.put_nowait((data[0], caller, data))
def remove(self, caller):
self.__locker.pop(caller)
@staticmethod
def getInstance():
if not EventQueue.__queue:
EventQueue.__queue = EventQueue()
return EventQueue.__queue
class WaitingPeers(Generic[TChainPeer]):
"""
Peers waiting to perform some action. When getting a peer from this queue,
prefer the peer with the best throughput for the given command.
"""
_waiting_peers: 'PriorityQueue[SortableTask[TChainPeer]]'
_response_command_type: Tuple[Type[CommandAPI[Any]], ...]
def __init__(
self,
response_command_type: Union[Type[CommandAPI[Any]],
Sequence[Type[CommandAPI[Any]]]],
sort_key: Callable[[PerformanceAPI],
float] = _items_per_second) -> None:
"""
:param sort_key: how should we sort the peers to get the fastest? low score means top-ranked
"""
self._waiting_peers = PriorityQueue()
if isinstance(response_command_type, type):
self._response_command_type = (response_command_type, )
elif isinstance(response_command_type, collections.Sequence):
self._response_command_type = tuple(response_command_type)
else:
raise TypeError(f"Unsupported value: {response_command_type}")
self._peer_wrapper = SortableTask.orderable_by_func(
self._get_peer_rank)
self._sort_key = sort_key
def _get_peer_rank(self, peer: TChainPeer) -> float:
scores = [
self._sort_key(exchange.tracker)
for exchange in peer.chain_api.exchanges if issubclass(
exchange.get_response_cmd_type(), self._response_command_type)
]
if len(scores) == 0:
raise ValidationError(
f"Could not find any exchanges on {peer} "
f"with response {self._response_command_type!r}")
# Typically there will only be one score, but we might want to match multiple commands.
# To handle that case, we take the average of the scores:
return sum(scores) / len(scores)
def put_nowait(self, peer: TChainPeer) -> None:
self._waiting_peers.put_nowait(self._peer_wrapper(peer))
async def get_fastest(self) -> TChainPeer:
wrapped_peer = await self._waiting_peers.get()
peer = wrapped_peer.original
# make sure the peer has not gone offline while waiting in the queue
while not peer.manager.is_running:
# if so, look for the next best peer
wrapped_peer = await self._waiting_peers.get()
peer = wrapped_peer.original
return peer
def __init__(self):
"""
Initialize and run an asyncio event loop for ever.
"""
self.loop = asyncio.get_event_loop()
self.queue = PriorityQueue(loop=self.loop)
self.loop.create_task(self.json_server(('', 25000)))
self.loop.create_task(self.queue_dumper())
self.loop.run_forever()
def __init__(
self,
response_command_type: Type[Command],
sort_key: Callable[[BasePerformance], float]=_items_per_second) -> None:
"""
:param sort_key: how should we sort the peers to get the fastest? low score means top-ranked
"""
self._waiting_peers = PriorityQueue()
self._response_command_type = response_command_type
self._peer_wrapper = SortableTask.orderable_by_func(self._get_peer_rank)
self._sort_key = sort_key
def __init__(self,
maxsize: int = 0,
order_fn: Callable[[TTask], Any] = identity,
*,
loop: AbstractEventLoop = None) -> None:
self._maxsize = maxsize
self._full_lock = Lock(loop=loop)
self._open_queue = PriorityQueue(maxsize, loop=loop)
self._order_fn = order_fn
self._id_generator = count()
self._tasks = set()
self._in_progress = {}
def djikstra(grid, start, end):
distance = defaultdict(lambda: float("inf"))
distance[start] = 0
frontier = PriorityQueue()
frontier.put_nowait((0, start))
visited = set([])
retrace = {}
while not frontier.empty():
(d, p) = frontier.get_nowait()
if p in visited:
continue
visited.add(p)
if p == end:
break
for n in reachable(grid, p):
if distance[n] > d + 1:
distance[n] = d + 1
retrace[n] = p
frontier.put_nowait((distance[n], n))
# Backtrack
current = end
forward = {}
while current != start:
forward[retrace[current]] = current
current = retrace[current]
return ({p: distance[p] for p in visited}, forward)
async def main():
priority_queue = PriorityQueue()
work_items = [
WorkItem(3, 'Lowest priority'),
WorkItem(2, 'Medium priority'),
WorkItem(1, 'High priority')
]
worker_task = asyncio.create_task(worker(priority_queue))
for work in work_items:
priority_queue.put_nowait(work)
await asyncio.gather(priority_queue.join(), worker_task)
def __init__(self, bot: IBot, name: str):
super().__init__(name)
self.bot = bot
self.disconnected = False
self.throttle = Throttler(rate_limit=100, period=1)
self.sasl_state = SASLResult.NONE
self.last_read = monotonic()
self._sent_count: int = 0
self._send_queue: PriorityQueue[SentLine] = PriorityQueue()
self.desired_caps: Set[ICapability] = set([])
self._read_queue: Deque[Line] = deque()
self._process_queue: Deque[Tuple[Line, Optional[Emit]]] = deque()
self._ping_sent = False
self._read_lguard = RLock()
self.read_lock = self._read_lguard
self._read_lwork = asyncio.Lock()
self._wait_for = asyncio.Event()
self._pending_who: Deque[str] = deque()
self._alt_nicks: List[str] = []
async def queue_dumper(self):
"""
Dumps status of queue to terminal (Eventually a file) every second.
"""
# TODO : Ensure this also prints to a file.
while True:
if not self.queue.qsize():
await asyncio.sleep(1)
else:
_copy = PriorityQueue()
while not self.queue.empty():
await _copy.put(await self.queue.get())
print(chr(27) + "[2J") # Bit of Ctr + L magic trick
while not _copy.empty():
element = await _copy.get()
print(element)
await self.queue.put(element)
await asyncio.sleep(1)
def __init__(
self,
response_command_type: Union[Type[CommandAPI[Any]], Sequence[Type[CommandAPI[Any]]]],
sort_key: Callable[[PerformanceAPI], float] = _items_per_second) -> None:
"""
:param sort_key: how should we sort the peers to get the fastest? low score means top-ranked
"""
self._waiting_peers = PriorityQueue()
if isinstance(response_command_type, type):
self._response_command_type = (response_command_type,)
elif isinstance(response_command_type, collections.abc.Sequence):
self._response_command_type = tuple(response_command_type)
else:
raise TypeError(f"Unsupported value: {response_command_type}")
self._peer_wrapper = SortableTask.orderable_by_func(self._get_peer_rank)
self._sort_key = sort_key
class WaitingPeers:
"""
Peers waiting to perform some action. When getting a peer from this queue,
prefer the peer with the best throughput for the given command.
"""
_waiting_peers: 'PriorityQueue[SortableTask[ETHPeer]]'
def __init__(self, response_command_type: Type[Command]) -> None:
self._waiting_peers = PriorityQueue()
self._response_command_type = response_command_type
self._peer_wrapper = SortableTask.orderable_by_func(self._ranked_peer)
def _ranked_peer(self, peer: ETHPeer) -> float:
relevant_throughputs = [
exchange.tracker.items_per_second_ema.value
for exchange in peer.requests
if exchange.response_cmd_type == self._response_command_type
]
if len(relevant_throughputs) == 0:
raise ValidationError(
f"Could not find any exchanges on {peer} "
f"with response {self._response_command_type!r}"
)
avg_throughput = sum(relevant_throughputs) / len(relevant_throughputs)
# high throughput peers should pop out of the queue first, so ranked as negative
return -1 * avg_throughput
def put_nowait(self, peer: ETHPeer) -> None:
self._waiting_peers.put_nowait(self._peer_wrapper(peer))
async def get_fastest(self) -> ETHPeer:
wrapped_peer = await self._waiting_peers.get()
peer = wrapped_peer.original
# make sure the peer has not gone offline while waiting in the queue
while not peer.is_operational:
# if so, look for the next best peer
wrapped_peer = await self._waiting_peers.get()
peer = wrapped_peer.original
return peer
def __init__(
self,
records: int,
handlers: dict[str, Optional[Callable]],
retry: int,
publisher: BrokerPublisher,
consumer_concurrency: int = 15,
**kwargs: Any,
):
super().__init__(**kwargs)
self._handlers = handlers
self._records = records
self._retry = retry
self._queue = PriorityQueue(maxsize=self._records)
self._consumers: list[Task] = list()
self._consumer_concurrency = consumer_concurrency
self._publisher = publisher
def __init__(self, *params):
self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \
self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [
aiodns.DNSResolver(tries=1) for _ in range(self.options.threads)
]
self.queue = PriorityQueue()
self.ip_dict = {}
self.found_subs = set()
self.timeout_subs = {}
self.count_time = time.time()
self.outfile = open(
'%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num),
'w')
self.normal_names_set = set()
self.lock = asyncio.Lock()
self.loop = None
self.threads_status = ['1'] * self.options.threads
class EventQueue:
__queue = None
def __init__(self):
self.__locker = {}
self.__queue = PriorityQueue()
async def consume(self):
while True:
_, caller, evt = await self.__queue.get()
wait_for(caller.sendData(evt), MAX_TIMEOUT)
if caller in self.__locker:
self.__locker[caller].release()
if not isRealtime():
# Wait for the lock to be locked
while not self.__locker[caller].locked():
await sleep(0.001)
if caller not in self.__locker:
break
async def put(self, caller, evt):
if not isinstance(evt, Event):
LOG.critical("Expect class event.Event but got %s" %
(str(evt.__class__)))
return
if caller not in self.__locker:
self.__locker[caller] = Lock()
await self.__locker[caller]
self.__queue.put_nowait((evt.date, caller, evt))
def remove(self, caller):
self.__locker.pop(caller)
@staticmethod
def getInstance():
if not EventQueue.__queue:
EventQueue.__queue = EventQueue()
return EventQueue.__queue
class KthLargest:
def __init__(self, k, nums):
self.k = k
self.pq = PriorityQueue()
for i in nums:
self.pq.put(i)
def add(self, n):
if self.pq.qsize() < self.k:
self.pq.put(n)
return self.pq.get_nowait()
async def _dequeue_entry(self,
queue: asyncio.PriorityQueue,
timeout: int = 5):
while not self.is_shutting_down:
wrapper: DateTimePriorityWrapper = await asyncio.wait_for(
queue.get(), timeout=timeout)
if not wrapper.can_process_now():
await queue.put(wrapper)
wait_for = max(wrapper.scheduled_in_seconds() * .5, 1)
log.debug(
"it's not yet time to use this request, waiting for %d",
wait_for)
await asyncio.sleep(wait_for)
continue
return wrapper.item
raise ShuttingDown()
class EventManager:
queue = PriorityQueue()
def __init_(self):
pass
def addevent(self, tick, function, data=None):
if (data == None):
self.queue.put_nowait(TupleSortingOn0((tick, function)))
else:
self.queue.put_nowait(TupleSortingOn0((tick, function, data)))
def getevent(self):
if self.queue.empty():
return None
return self.queue.get_nowait()
def reverse_djikstra(grid, end):
distance = defaultdict(lambda: float("inf"))
distance[end] = 0
frontier = PriorityQueue()
frontier.put_nowait((0, end))
visited = set([])
while not frontier.empty():
(d, p) = frontier.get_nowait()
if p in visited:
continue
else:
visited.add(p)
for n in reachable(grid, p):
if distance[n] > d + 1:
distance[n] = d + 1
frontier.put_nowait((distance[n], n))
return {p: distance[p] for p in visited}
class SubNameBrute(object):
def __init__(self, *params):
self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \
self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [
aiodns.DNSResolver(tries=1) for _ in range(self.options.threads)
]
self.queue = PriorityQueue()
self.ip_dict = {}
self.found_subs = set()
self.timeout_subs = {}
self.count_time = time.time()
self.outfile = open(
'%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num),
'w')
self.normal_names_set = set()
self.lock = asyncio.Lock()
self.loop = None
self.threads_status = ['1'] * self.options.threads
async def load_sub_names(self):
normal_lines = []
wildcard_lines = []
wildcard_set = set()
regex_list = []
lines = set()
with open(self.options.file) as inFile:
for line in inFile.readlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
brace_count = sub.count('{')
if brace_count > 0:
wildcard_lines.append((brace_count, sub))
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_set:
wildcard_set.add(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
self.normal_names_set.add(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines:
if _regex.search(line):
normal_lines.remove(line)
for _ in normal_lines[self.process_num::self.options.process]:
await self.queue.put((0, _)) # priority set to 0
for _ in wildcard_lines[self.process_num::self.options.process]:
await self.queue.put(_)
async def scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
if self.dns_count > 1:
while True:
s = random.choice(self.resolvers)
if s != self.dns_servers[j % self.dns_count]:
self.resolvers[j].nameservers.append(s)
break
while True:
try:
if time.time() - self.count_time > 1.0:
async with self.lock:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_array[
self.process_num] = self.queue.qsize()
if self.found_count_local:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.count_time = time.time()
try:
brace_count, sub = self.queue.get_nowait()
self.threads_status[j] = '1'
except asyncio.queues.QueueEmpty as e:
self.threads_status[j] = '0'
await asyncio.sleep(0.5)
if '1' not in self.threads_status:
break
else:
continue
if brace_count > 0:
brace_count -= 1
if sub.find('{next_sub}') >= 0:
for _ in self.next_subs:
await self.queue.put(
(0, sub.replace('{next_sub}', _)))
if sub.find('{alphnum}') >= 0:
for _ in 'abcdefghijklmnopqrstuvwxyz0123456789':
await self.queue.put(
(brace_count, sub.replace('{alphnum}', _, 1)))
elif sub.find('{alpha}') >= 0:
for _ in 'abcdefghijklmnopqrstuvwxyz':
await self.queue.put(
(brace_count, sub.replace('{alpha}', _, 1)))
elif sub.find('{num}') >= 0:
for _ in '0123456789':
await self.queue.put(
(brace_count, sub.replace('{num}', _, 1)))
continue
except Exception as e:
import traceback
print(traceback.format_exc())
break
try:
if sub in self.found_subs:
continue
self.scan_count_local += 1
cur_domain = sub + '.' + self.domain
# print('Query %s' % cur_domain)
answers = await self.resolvers[j].query(cur_domain, 'A')
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.host
for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']:
continue
if self.options.i and is_intranet(answers[0].host):
continue
try:
self.scan_count_local += 1
answers = await self.resolvers[j].query(
cur_domain, 'CNAME')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(
self.domain) and cname not in self.found_subs:
cname_sub = cname[:len(cname) - len(self.domain) -
1] # new sub
if cname_sub not in self.normal_names_set:
self.found_subs.add(cname)
await self.queue.put((0, cname_sub))
except Exception as e:
pass
first_level_sub = sub.split('.')[-1]
if (first_level_sub, ips) not in self.ip_dict:
self.ip_dict[(first_level_sub, ips)] = 1
else:
self.ip_dict[(first_level_sub, ips)] += 1
if self.ip_dict[(first_level_sub, ips)] > 30:
continue
self.found_count_local += 1
self.outfile.write(
cur_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.scan_count_local += 1
await self.resolvers[j].query(
'lijiejie-test-not-existed.' + cur_domain, 'A')
except aiodns.error.DNSError as e:
if e.args[0] in [4]:
if self.queue.qsize() < 50000:
for _ in self.next_subs:
await self.queue.put((0, _ + '.' + sub))
else:
await self.queue.put((1, '{next_sub}.' + sub))
except Exception as e:
pass
except aiodns.error.DNSError as e:
if e.args[0] in [1, 4]:
pass
elif e.args[0] in [
11, 12
]: # 12 timeout # (11, 'Could not contact DNS servers')
# print('timed out sub %s' % sub)
self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1
if self.timeout_subs[sub] <= 1:
await self.queue.put((0, sub)) # Retry
else:
print(e)
except asyncio.TimeoutError as e:
pass
except Exception as e:
import traceback
traceback.print_exc()
with open('errors.log', 'a') as errFile:
errFile.write('[%s] %s\n' % (type(e), str(e)))
async def async_run(self):
await self.load_sub_names()
tasks = [self.scan(i) for i in range(self.options.threads)]
await asyncio.gather(*tasks)
def run(self):
self.loop = asyncio.get_event_loop()
asyncio.set_event_loop(self.loop)
self.loop.run_until_complete(self.async_run())
def __init__(self, maxsize):
PriorityQueue.__init__(self, maxsize=maxsize)
self.counter = 0
def __init__(self):
self.__locker = {}
self.__queue = PriorityQueue()
class TaskQueue(Generic[TTask]):
"""
TaskQueue keeps priority-order track of pending tasks, with a limit on number pending.
A producer of tasks will insert pending tasks with await add(), which will not return until
all tasks have been added to the queue.
A task consumer calls await get() to retrieve tasks for processing. Tasks will be returned in
priority order. If no tasks are pending, get()
will pause until at least one is available. Only one consumer will have a task "checked out"
from get() at a time.
After tasks are successfully completed, the consumer will call complete() to remove them from
the queue. The consumer doesn't need to complete all tasks, but any uncompleted tasks will be
considered abandoned. Another consumer can pick it up at the next get() call.
"""
# a function that determines the priority order (lower int is higher priority)
_order_fn: FunctionProperty[Callable[[TTask], Any]]
# batches of tasks that have been started but not completed
_in_progress: Dict[int, Tuple[TTask, ...]]
# all tasks that have been placed in the queue and have not been started
_open_queue: 'PriorityQueue[Tuple[Any, TTask]]'
# all tasks that have been placed in the queue and have not been completed
_tasks: Set[TTask]
def __init__(self,
maxsize: int = 0,
order_fn: Callable[[TTask], Any] = identity,
*,
loop: AbstractEventLoop = None) -> None:
self._maxsize = maxsize
self._full_lock = Lock(loop=loop)
self._open_queue = PriorityQueue(maxsize, loop=loop)
self._order_fn = order_fn
self._id_generator = count()
self._tasks = set()
self._in_progress = {}
async def add(self, tasks: Tuple[TTask, ...]) -> None:
"""
add() will insert as many tasks as can be inserted until the queue fills up.
Then it will pause until the queue is no longer full, and continue adding tasks.
It will finally return when all tasks have been inserted.
"""
if not isinstance(tasks, tuple):
raise ValidationError(
f"must pass a tuple of tasks to add(), but got {tasks!r}")
already_pending = self._tasks.intersection(tasks)
if already_pending:
raise ValidationError(
f"Duplicate tasks detected: {already_pending!r} are already present in the queue"
)
# make sure to insert the highest-priority items first, in case queue fills up
remaining = tuple(
sorted((self._order_fn(task), task) for task in tasks))
while remaining:
num_tasks = len(self._tasks)
if self._maxsize <= 0:
# no cap at all, immediately insert all tasks
open_slots = len(remaining)
elif num_tasks < self._maxsize:
# there is room to add at least one more task
open_slots = self._maxsize - num_tasks
else:
# wait until there is room in the queue
await self._full_lock.acquire()
# the current number of tasks has changed, can't reuse num_tasks
num_tasks = len(self._tasks)
open_slots = self._maxsize - num_tasks
queueing, remaining = remaining[:open_slots], remaining[
open_slots:]
for task in queueing:
# There will always be room in _open_queue until _maxsize is reached
try:
self._open_queue.put_nowait(task)
except QueueFull as exc:
task_idx = queueing.index(task)
qsize = self._open_queue.qsize()
raise QueueFull(
f'TaskQueue unsuccessful in adding task {task[1]!r} because qsize={qsize}, '
f'num_tasks={num_tasks}, maxsize={self._maxsize}, open_slots={open_slots}, '
f'num queueing={len(queueing)}, len(_tasks)={len(self._tasks)}, task_idx='
f'{task_idx}, queuing={queueing}, original msg: {exc}',
)
unranked_queued = tuple(task for _rank, task in queueing)
self._tasks.update(unranked_queued)
if self._full_lock.locked() and len(self._tasks) < self._maxsize:
self._full_lock.release()
def get_nowait(self,
max_results: int = None) -> Tuple[int, Tuple[TTask, ...]]:
"""
Get pending tasks. If no tasks are pending, raise an exception.
:param max_results: return up to this many pending tasks. If None, return all pending tasks.
:return: (batch_id, tasks to attempt)
:raise ~asyncio.QueueFull: if no tasks are available
"""
if self._open_queue.empty():
raise QueueFull("No tasks are available to get")
else:
pending_tasks = self._get_nowait(max_results)
# Generate a pending batch of tasks, so uncompleted tasks can be inferred
next_id = next(self._id_generator)
self._in_progress[next_id] = pending_tasks
return (next_id, pending_tasks)
async def get(self,
max_results: int = None) -> Tuple[int, Tuple[TTask, ...]]:
"""
Get pending tasks. If no tasks are pending, wait until a task is added.
:param max_results: return up to this many pending tasks. If None, return all pending tasks.
:return: (batch_id, tasks to attempt)
"""
if max_results is not None and max_results < 1:
raise ValidationError(
"Must request at least one task to process, not {max_results!r}"
)
# if the queue is empty, wait until at least one item is available
queue = self._open_queue
if queue.empty():
_rank, first_task = await queue.get()
else:
_rank, first_task = queue.get_nowait()
# In order to return from get() as soon as possible, never await again.
# Instead, take only the tasks that are already available.
if max_results is None:
remaining_count = None
else:
remaining_count = max_results - 1
remaining_tasks = self._get_nowait(remaining_count)
# Combine the first and remaining tasks
all_tasks = (first_task, ) + remaining_tasks
# Generate a pending batch of tasks, so uncompleted tasks can be inferred
next_id = next(self._id_generator)
self._in_progress[next_id] = all_tasks
return (next_id, all_tasks)
def _get_nowait(self, max_results: int = None) -> Tuple[TTask, ...]:
queue = self._open_queue
# How many results do we want?
available = queue.qsize()
if max_results is None:
num_tasks = available
else:
num_tasks = min((available, max_results))
# Combine the remaining tasks with the first task we already pulled.
ranked_tasks = tuple(queue.get_nowait() for _ in range(num_tasks))
# strip out the rank value used internally for sorting in the priority queue
return tuple(task for _rank, task in ranked_tasks)
def complete(self, batch_id: int, completed: Tuple[TTask, ...]) -> None:
if batch_id not in self._in_progress:
raise ValidationError(
f"batch id {batch_id} not recognized, with tasks {completed!r}"
)
attempted = self._in_progress.pop(batch_id)
unrecognized_tasks = set(completed).difference(attempted)
if unrecognized_tasks:
self._in_progress[batch_id] = attempted
raise ValidationError(
f"cannot complete tasks {unrecognized_tasks!r} in this batch, only {attempted!r}"
)
incomplete = set(attempted).difference(completed)
for task in incomplete:
# These tasks are already counted in the total task count, so there will be room
self._open_queue.put_nowait((self._order_fn(task), task))
self._tasks.difference_update(completed)
if self._full_lock.locked() and len(self._tasks) < self._maxsize:
self._full_lock.release()
def __contains__(self, task: TTask) -> bool:
"""Determine if a task has been added and not yet completed"""
return task in self._tasks
def __init__(self, k, nums):
self.k = k
self.pq = PriorityQueue()
for i in nums:
self.pq.put(i)
class PaymentCommunity(BamiCommunity, metaclass=ABCMeta):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# self.transfer_queue = Queue()
# self.transfer_queue_task = ensure_future(self.evaluate_transfer_queue())
# Add state db
if not kwargs.get("settings"):
self._settings = PaymentSettings()
self.state_db = PaymentState(self._settings.asset_precision)
self.context = self.state_db.context
self.reachability_cache = defaultdict(lambda: cachetools.LRUCache(100))
# Dictionary chain_id: block_dot -> block
self.tracked_blocks = defaultdict(lambda: {})
self.peer_conf = defaultdict(lambda: defaultdict(int))
self.should_witness_subcom = {}
self.counter_signing_block_queue = PriorityQueue()
self.block_sign_queue_task = ensure_future(
self.evaluate_counter_signing_blocks()
)
self.witness_delta = kwargs.get("witness_delta")
if not self.witness_delta:
self.witness_delta = self.settings.witness_block_delta
@property
def settings(self) -> PaymentSettings:
return super().settings
def join_subcommunity_gossip(self, sub_com_id: bytes) -> None:
# 0. Add master peer to the known minter group
self.state_db.add_known_minters(sub_com_id, {sub_com_id})
# 1. Main payment chain: spends and their confirmations
# - Start gossip sync task periodically on the chain updates
self.start_gossip_sync(sub_com_id)
# - Process incoming blocks on the chain in order for payments
self.subscribe_in_order_block(sub_com_id, self.receive_block_in_order)
# 2. Witness chain:
# - Gossip witness updates on the sub-chain
self.start_gossip_sync(sub_com_id, prefix=b"w")
# - Process witness block out of order
self.subscribe_out_order_block(b"w" + sub_com_id, self.process_witness_block)
# - Witness all updates on payment chain
self.should_witness_subcom[sub_com_id] = self.settings.should_witness_block
def receive_block_in_order(self, block: BamiBlock) -> None:
if block.com_dot in self.state_db.applied_dots:
raise Exception(
"Block already applied?",
block.com_dot,
self.state_db.vals_cache,
self.state_db.peer_mints,
self.state_db.applied_dots,
)
chain_id = block.com_id
dot = block.com_dot
self.state_db.applied_dots.add(dot)
# Check reachability for target block -> update risk
for blk_dot in self.tracked_blocks[chain_id]:
if self.dot_reachable(chain_id, blk_dot, dot):
self.update_risk(chain_id, block.public_key, blk_dot[0])
# Process blocks according to their type
self.logger.debug(
"Processing block %s, %s, %s", block.type, chain_id, block.hash
)
if block.type == MINT_TYPE:
self.process_mint(block)
elif block.type == SPEND_TYPE:
self.process_spend(block)
elif block.type == CONFIRM_TYPE:
self.process_confirm(block)
elif block.type == REJECT_TYPE:
self.process_reject(block)
elif block.type == WITNESS_TYPE:
raise Exception("Witness block received, while shouldn't")
# Witness block react on new block:
if (
self.should_witness_subcom.get(chain_id)
and block.type != WITNESS_TYPE
and self.should_witness_chain_point(
chain_id, self.my_pub_key_bin, block.com_seq_num
)
):
self.schedule_witness_block(chain_id, block.com_seq_num)
def process_witness_block(self, blk: BamiBlock) -> None:
"""Process witness block out of order"""
# No block is processed out of order in this community
self.logger.debug(
"Processing block %s, %s, %s", blk.type, blk.com_dot, blk.com_id
)
if blk.type != WITNESS_TYPE:
raise Exception("Received not witness block on witness sub-chain!")
self.process_witness(blk)
def should_store_store_update(self, chain_id: bytes, seq_num: int) -> bool:
"""Store the status of the chain at the seq_num for further witnessing or verification"""
# Should depend on if witnessing? - or something different
return True
# return self.should_witness_chain_point(chain_id, self.my_pub_key_bin, seq_num)
def should_witness_chain_point(
self, chain_id: bytes, peer_id: bytes, seq_num: int
) -> bool:
"""
Returns:
True if peer should witness this chain at seq_num
"""
# Based on random coin tossing?
seed = chain_id + peer_id + bytes(seq_num)
ran = Random(seed)
# Every peer should witness every K blocks?
# TODO: that should depend on the number of peers in the community - change that
# + account for the fault tolerance
if ran.random() < 1 / self.witness_delta:
return True
return False
# -------------- Mint transaction ----------------------
def verify_mint(
self, chain_id: bytes, minter: bytes, mint_transaction: Dict
) -> None:
"""
Verify that mint transaction from minter is valid:
- minter is known and acceptable
- mint if properly formatted
- mint value is withing the acceptable range
- total value minted by the minter is limited
Args:
chain_id: chain identifier
minter: id of the minter, e.g. public key
mint_transaction: transaction as a dictionary
Raises:
InvalidMintException if not valid mint
"""
# 1. Is minter known and acceptable?
if not self.state_db.known_chain_minters(
chain_id
) or minter not in self.state_db.known_chain_minters(chain_id):
raise UnknownMinterException(
"Got minting from unacceptable peer ", chain_id, minter
)
# 2. Mint if properly formatted
if not mint_transaction.get(b"value"):
raise InvalidTransactionFormatException(
"Mint transaction badly formatted ", mint_transaction, chain_id, minter
)
# 3. Minting value within the range
if not (
Decimal(self.settings.mint_value_range[0], self.context)
< mint_transaction[b"value"]
< Decimal(self.settings.mint_value_range[1], self.context)
):
raise InvalidMintRangeException(
chain_id, minter, mint_transaction.get(b"value")
)
# 4. Total value is bounded
if not (
self.state_db.peer_mints[minter]
+ Decimal(mint_transaction.get(b"value"), self.context)
< Decimal(self.settings.mint_max_value, self.context)
):
raise UnboundedMintException(
chain_id,
minter,
self.state_db.peer_mints[minter],
mint_transaction.get(b"value"),
)
def mint(self, value: Decimal = None, chain_id: bytes = None) -> None:
"""
Create mint for own reputation: Reputation & Liveness at Stake
"""
if not value:
value = self.settings.initial_mint_value
if not chain_id:
# Community id is the same as the peer id
chain_id = self.my_pub_key_bin
# Mint transaction: value
mint_tx = {b"value": float(value)}
self.verify_mint(chain_id, self.my_pub_key_bin, mint_tx)
block = self.create_signed_block(
block_type=MINT_TYPE, transaction=encode_raw(mint_tx), com_id=chain_id
)
self.share_in_community(block, chain_id)
def process_mint(self, mint_blk: BamiBlock) -> None:
"""Process received mint transaction"""
minter = mint_blk.public_key
mint_tx = decode_raw(mint_blk.transaction)
chain_id = mint_blk.com_id
mint_dot = mint_blk.com_dot
prev_links = mint_blk.links
self.verify_mint(chain_id, minter, mint_tx)
seq_num = mint_dot[0]
self.state_db.apply_mint(
chain_id,
mint_dot,
prev_links,
minter,
Decimal(mint_tx.get(b"value"), self.context),
self.should_store_store_update(chain_id, seq_num),
)
# ------ Spend transaction -----------
def spend(
self,
chain_id: bytes,
counter_party: bytes,
value: Decimal,
ignore_validation: bool = False,
) -> None:
"""
Spend tokens in the chain to the counter_party.
Args:
chain_id: identity of the chain
counter_party: identity of the counter-party
value: Decimal value to transfer
ignore_validation: if True and balance is negative - will raise an Exception
"""
bal = self.state_db.get_balance(self.my_pub_key_bin)
if ignore_validation or bal - value >= 0:
spend_tx = {
b"value": float(value),
b"to_peer": counter_party,
b"prev_pairwise_link": self.state_db.get_last_pairwise_links(
self.my_pub_key_bin, counter_party
),
}
self.verify_spend(self.my_pub_key_bin, spend_tx)
block = self.create_signed_block(
block_type=SPEND_TYPE, transaction=encode_raw(spend_tx), com_id=chain_id
)
self.logger.info("Created spend block %s", block.com_dot)
counter_peer = self.get_peer_by_key(counter_party, chain_id)
if counter_peer:
self.send_block(block, [counter_peer])
self.share_in_community(block, chain_id)
else:
raise InsufficientBalanceException("Not enough balance for spend")
def verify_spend(self, spender: bytes, spend_transaction: Dict) -> None:
"""Verify the spend transaction:
- spend formatted correctly
Raises:
InvalidTransactionFormat
"""
# 1. Verify the spend format
if (
not spend_transaction.get(b"value")
or not spend_transaction.get(b"to_peer")
or not spend_transaction.get(b"prev_pairwise_link")
):
raise InvalidTransactionFormatException(
"Spend transaction badly formatted ", spender, spend_transaction
)
# 2. Verify the spend value in range
if not (
self.settings.spend_value_range[0]
< spend_transaction.get(b"value")
< self.settings.spend_value_range[1]
):
raise InvalidSpendRangeException(
"Spend value out of range", spender, spend_transaction.get(b"value")
)
def process_spend(self, spend_block: BamiBlock) -> None:
# Store spend in the database
spend_tx = decode_raw(spend_block.transaction)
spender = spend_block.public_key
self.verify_spend(spender, spend_tx)
chain_id = spend_block.com_id
spend_dot = spend_block.com_dot
pers_links = spend_block.links
prev_spend_links = spend_tx.get(b"prev_pairwise_link")
value = Decimal(spend_tx.get(b"value"), self.context)
to_peer = spend_tx.get(b"to_peer")
seq_num = spend_dot[0]
self.state_db.apply_spend(
chain_id,
prev_spend_links,
pers_links,
spend_dot,
spender,
to_peer,
value,
self.should_store_store_update(chain_id, seq_num),
)
# Is this block related to my peer?
if to_peer == self.my_pub_key_bin:
self.add_block_to_response_processing(spend_block)
# ------------ Block Response processing ---------
def add_block_to_response_processing(self, block: BamiBlock) -> None:
self.tracked_blocks[block.com_id][block.com_dot] = block
self.counter_signing_block_queue.put_nowait((block.com_seq_num, (0, block)))
def process_counter_signing_block(
self, block: BamiBlock, time_passed: float = None, num_block_passed: int = None,
) -> bool:
"""
Process block that should be counter-signed and return True if the block should be delayed more.
Args:
block: Processed block
time_passed: time passed since first added
num_block_passed: number of blocks passed since first added
Returns:
Should add to queue again.
"""
res = self.block_response(block, time_passed, num_block_passed)
if res == BlockResponse.CONFIRM:
self.confirm(
block,
extra_data={b"value": decode_raw(block.transaction).get(b"value")},
)
return False
elif res == BlockResponse.REJECT:
self.reject(block)
return False
return True
async def evaluate_counter_signing_blocks(self, delta: float = None):
while True:
_delta = delta if delta else self.settings.block_sign_delta
priority, block_info = await self.counter_signing_block_queue.get()
process_time, block = block_info
should_delay = self.process_counter_signing_block(block, process_time)
self.logger.debug(
"Processing counter signing block. Delayed: %s", should_delay
)
if should_delay:
self.counter_signing_block_queue.put_nowait(
(priority, (process_time + _delta, block))
)
await sleep(_delta)
else:
self.tracked_blocks[block.com_id].pop(block.com_dot)
await sleep(0.001)
def block_response(
self, block: BamiBlock, wait_time: float = None, wait_blocks: int = None
) -> BlockResponse:
# Analyze the risk of accepting this block
stat = self.state_db.get_closest_peers_status(block.com_id, block.com_seq_num)
# If there is no information or chain is forked or
peer_id = shorten(block.public_key)
if not stat or not stat[1].get(peer_id):
# Check that it is not infinite
if (wait_time and wait_time > self.settings.max_wait_time) or (
wait_blocks and wait_blocks > self.settings.max_wait_block
):
return BlockResponse.REJECT
return BlockResponse.DELAY
if not stat[1][peer_id][1] or not stat[1][peer_id][0]:
# If chain is forked or negative balance => reject
return BlockResponse.REJECT
# Verify the risk of missing some information:
# - There is diverse peers building upon the block
# TODO: revisit that - number should depend on total number of peers in community.
# 1. Diversity on the block building
f = self.settings.diversity_confirm
if len(self.peer_conf[(block.com_id, block.com_seq_num)]) >= f:
return BlockResponse.CONFIRM
else:
return BlockResponse.DELAY
def dot_reachable(self, chain_id: bytes, target_dot: Dot, block_dot: Dot):
val = self.reachability_cache[(chain_id, target_dot)].get(block_dot)
if val is not None:
return val
res = self.persistence.get_chain(chain_id).get_prev_links(block_dot)
if target_dot in res:
return True
if max(res)[0] < target_dot[0]:
return False
else:
# Need to take more step
for prev_dot in res:
new_val = self.dot_reachable(chain_id, target_dot, prev_dot)
if new_val:
self.reachability_cache[(chain_id, target_dot)][block_dot] = True
return True
self.reachability_cache[(chain_id, target_dot)][block_dot] = False
return False
def update_risk(self, chain_id: bytes, conf_peer_id: bytes, target_seq_num: int):
print("Risk update: ", shorten(conf_peer_id), target_seq_num)
self.peer_conf[(chain_id, target_seq_num)][conf_peer_id] += 1
# ----------- Witness transactions --------------
def schedule_witness_block(
self, chain_id: bytes, seq_num: int, delay: float = None
):
# Schedule witness transaction
name_prefix = str(hex_to_int(chain_id + bytes(seq_num)))
if self.is_pending_task_active(name_prefix):
self.replace_task(
name_prefix,
self.witness,
chain_id,
seq_num,
delay=self.settings.witness_delta_time,
)
else:
self.register_task(
name_prefix,
self.witness,
chain_id,
seq_num,
delay=self.settings.witness_delta_time,
)
def witness_tx_well_formatted(self, witness_tx: Any) -> bool:
return len(witness_tx) == 2 and witness_tx[0] > 0 and len(witness_tx[1]) > 0
def build_witness_blob(self, chain_id: bytes, seq_num: int) -> Optional[bytes]:
chain_state = self.state_db.get_closest_peers_status(chain_id, seq_num)
if not chain_state:
return None
return encode_raw(chain_state)
def apply_witness_tx(
self, block: BamiBlock, witness_tx: Tuple[int, ChainState]
) -> None:
state = witness_tx[1]
state_hash = take_hash(state)
seq_num = witness_tx[0]
if not self.should_witness_chain_point(block.com_id, block.public_key, seq_num):
# This is invalid witnessing - react
raise InvalidWitnessTransactionException(
"Received invalid witness transaction",
block.com_id,
block.public_key,
seq_num,
)
self.state_db.add_witness_vote(
block.com_id, seq_num, state_hash, block.public_key
)
self.state_db.add_chain_state(block.com_id, seq_num, state_hash, state)
chain_id = block.com_id
if self.tracked_blocks.get(chain_id):
for block_dot, tracked_block in self.tracked_blocks[chain_id].items():
if (
block_dot[0] <= seq_num
and state.get(shorten(tracked_block.public_key))
and state.get(shorten(tracked_block.public_key)) == (True, True)
):
self.update_risk(chain_id, block.public_key, block_dot[0])
# ------ Confirm and reject transactions -------
def apply_confirm_tx(self, block: BamiBlock, confirm_tx: Dict) -> None:
claim_dot = block.com_dot
chain_id = block.com_id
claimer = block.public_key
com_links = block.links
seq_num = claim_dot[0]
self.state_db.apply_confirm(
chain_id,
claimer,
com_links,
claim_dot,
confirm_tx[b"initiator"],
confirm_tx[b"dot"],
Decimal(confirm_tx[b"value"], self.context),
self.should_store_store_update(chain_id, seq_num),
)
def apply_reject_tx(self, block: BamiBlock, reject_tx: Dict) -> None:
self.state_db.apply_reject(
block.com_id,
block.public_key,
block.links,
block.com_dot,
reject_tx[b"initiator"],
reject_tx[b"dot"],
self.should_store_store_update(block.com_id, block.com_seq_num),
)
async def unload(self):
if not self.block_sign_queue_task.done():
self.block_sign_queue_task.cancel()
await super().unload()
def __init__(self, response_command_type: Type[Command]) -> None:
self._waiting_peers = PriorityQueue()
self._response_command_type = response_command_type
self._peer_wrapper = SortableTask.orderable_by_func(self._ranked_peer)
class BrokerHandler(BrokerHandlerSetup):
"""Broker Handler class."""
__slots__ = "_handlers", "_records", "_retry", "_queue", "_consumers", "_consumer_concurrency"
def __init__(
self,
records: int,
handlers: dict[str, Optional[Callable]],
retry: int,
publisher: BrokerPublisher,
consumer_concurrency: int = 15,
**kwargs: Any,
):
super().__init__(**kwargs)
self._handlers = handlers
self._records = records
self._retry = retry
self._queue = PriorityQueue(maxsize=self._records)
self._consumers: list[Task] = list()
self._consumer_concurrency = consumer_concurrency
self._publisher = publisher
@classmethod
def _from_config(cls, config: MinosConfig, **kwargs) -> BrokerHandler:
kwargs["handlers"] = cls._get_handlers(config, **kwargs)
kwargs["publisher"] = cls._get_publisher(**kwargs)
# noinspection PyProtectedMember
return cls(**config.broker.queue._asdict(), **kwargs)
@staticmethod
def _get_handlers(
config: MinosConfig, handlers: dict[str, Optional[Callable]] = None, **kwargs
) -> dict[str, Callable[[BrokerRequest], Awaitable[Optional[BrokerResponse]]]]:
if handlers is None:
builder = EnrouteBuilder(*config.services, middleware=config.middleware)
decorators = builder.get_broker_command_query_event(config=config, **kwargs)
handlers = {decorator.topic: fn for decorator, fn in decorators.items()}
return handlers
# noinspection PyUnusedLocal
@staticmethod
@inject
def _get_publisher(
publisher: Optional[BrokerPublisher] = None,
broker_publisher: BrokerPublisher = Provide["broker_publisher"],
**kwargs,
) -> BrokerPublisher:
if publisher is None:
publisher = broker_publisher
if publisher is None or isinstance(publisher, Provide):
raise NotProvidedException(f"A {BrokerPublisher!r} object must be provided.")
return publisher
async def _setup(self) -> None:
await super()._setup()
await self._create_consumers()
async def _destroy(self) -> None:
await self._destroy_consumers()
await super()._destroy()
async def _create_consumers(self):
while len(self._consumers) < self._consumer_concurrency:
self._consumers.append(create_task(self._consume()))
async def _destroy_consumers(self):
for consumer in self._consumers:
consumer.cancel()
await gather(*self._consumers, return_exceptions=True)
self._consumers = list()
while not self._queue.empty():
entry = self._queue.get_nowait()
await self.submit_query(self._queries["update_not_processed"], (entry.id,))
async def _consume(self) -> None:
while True:
await self._consume_one()
async def _consume_one(self) -> None:
entry = await self._queue.get()
try:
await self._dispatch_one(entry)
finally:
self._queue.task_done()
@property
def publisher(self) -> BrokerPublisher:
"""Get the publisher instance.
:return: A ``BrokerPublisher`` instance.
"""
return self._publisher
@property
def consumers(self) -> list[Task]:
"""Get the consumers.
:return: A list of ``Task`` instances.
"""
return self._consumers
@property
def handlers(self) -> dict[str, Optional[Callable]]:
"""Handlers getter.
:return: A dictionary in which the keys are topics and the values are the handler.
"""
return self._handlers
@property
def topics(self) -> KeysView[str]:
"""Get an iterable containing the topic names.
:return: An ``Iterable`` of ``str``.
"""
return self.handlers.keys()
async def dispatch_forever(self, max_wait: Optional[float] = 60.0) -> NoReturn:
"""Dispatch the items in the consuming queue forever.
:param max_wait: Maximum seconds to wait for notifications. If ``None`` the wait is performed until infinity.
:return: This method does not return anything.
"""
async with self.cursor() as cursor:
await self._listen_entries(cursor)
try:
while True:
await self._wait_for_entries(cursor, max_wait)
await self.dispatch(cursor, background_mode=True)
finally:
await self._unlisten_entries(cursor)
async def _listen_entries(self, cursor: Cursor):
for topic in self.topics:
# noinspection PyTypeChecker
await cursor.execute(_LISTEN_QUERY.format(Identifier(topic)))
async def _unlisten_entries(self, cursor: Cursor) -> None:
for topic in self.topics:
# noinspection PyTypeChecker
await cursor.execute(_UNLISTEN_QUERY.format(Identifier(topic)))
async def _wait_for_entries(self, cursor: Cursor, max_wait: Optional[float]) -> None:
if await self._get_count(cursor):
return
while True:
try:
return await wait_for(consume_queue(cursor.connection.notifies, self._records), max_wait)
except TimeoutError:
if await self._get_count(cursor):
return
async def _get_count(self, cursor) -> int:
if not len(self.topics):
return 0
await cursor.execute(_COUNT_NOT_PROCESSED_QUERY, (self._retry, tuple(self.topics)))
count = (await cursor.fetchone())[0]
return count
async def dispatch(self, cursor: Optional[Cursor] = None, background_mode: bool = False) -> None:
"""Dispatch a batch of ``HandlerEntry`` instances from the database's queue.
:param cursor: The cursor to interact with the database. If ``None`` is provided a new one is acquired.
:param background_mode: If ``True`` the entries dispatching waits until every entry is processed. Otherwise,
the dispatching is performed on background.
:return: This method does not return anything.
"""
is_external_cursor = cursor is not None
if not is_external_cursor:
cursor = await self.cursor().__aenter__()
async with cursor.begin():
await cursor.execute(
self._queries["select_not_processed"], (self._retry, tuple(self.topics), self._records)
)
result = await cursor.fetchall()
if len(result):
entries = self._build_entries(result)
await cursor.execute(self._queries["mark_processing"], (tuple(e.id for e in entries),))
for entry in entries:
await self._queue.put(entry)
if not is_external_cursor:
await cursor.__aexit__(None, None, None)
if not background_mode:
await self._queue.join()
def _build_entries(self, rows: list[tuple]) -> list[BrokerHandlerEntry]:
kwargs = {"callback_lookup": self.get_action}
return [BrokerHandlerEntry(*row, **kwargs) for row in rows]
async def _dispatch_one(self, entry: BrokerHandlerEntry) -> None:
logger.debug(f"Dispatching '{entry!r}'...")
try:
await self.dispatch_one(entry)
except (CancelledError, Exception) as exc:
logger.warning(f"Raised an exception while dispatching {entry!r}: {exc!r}")
entry.exception = exc
if isinstance(exc, CancelledError):
raise exc
finally:
query_id = "delete_processed" if entry.success else "update_not_processed"
await self.submit_query(self._queries[query_id], (entry.id,))
async def dispatch_one(self, entry: BrokerHandlerEntry) -> None:
"""Dispatch one row.
:param entry: Entry to be dispatched.
:return: This method does not return anything.
"""
logger.info(f"Dispatching '{entry!s}'...")
fn = self.get_callback(entry.callback)
message = entry.data
data, status, headers = await fn(message)
if message.reply_topic is not None:
await self.publisher.send(
data,
topic=message.reply_topic,
identifier=message.identifier,
status=status,
user=message.user,
headers=headers,
)
@staticmethod
def get_callback(
fn: Callable[[BrokerRequest], Union[Optional[BrokerRequest], Awaitable[Optional[BrokerRequest]]]]
) -> Callable[[BrokerMessage], Awaitable[tuple[Any, BrokerMessageStatus, dict[str, str]]]]:
"""Get the handler function to be used by the Broker Handler.
:param fn: The action function.
:return: A wrapper function around the given one that is compatible with the Broker Handler API.
"""
@wraps(fn)
async def _wrapper(raw: BrokerMessage) -> tuple[Any, BrokerMessageStatus, dict[str, str]]:
request = BrokerRequest(raw)
user_token = REQUEST_USER_CONTEXT_VAR.set(request.user)
headers_token = REQUEST_HEADERS_CONTEXT_VAR.set(raw.headers)
try:
response = fn(request)
if isawaitable(response):
response = await response
if isinstance(response, Response):
response = await response.content()
return response, BrokerMessageStatus.SUCCESS, REQUEST_HEADERS_CONTEXT_VAR.get()
except ResponseException as exc:
logger.warning(f"Raised an application exception: {exc!s}")
return repr(exc), BrokerMessageStatus.ERROR, REQUEST_HEADERS_CONTEXT_VAR.get()
except Exception as exc:
logger.exception(f"Raised a system exception: {exc!r}")
return repr(exc), BrokerMessageStatus.SYSTEM_ERROR, REQUEST_HEADERS_CONTEXT_VAR.get()
finally:
REQUEST_USER_CONTEXT_VAR.reset(user_token)
REQUEST_HEADERS_CONTEXT_VAR.reset(headers_token)
return _wrapper
def get_action(self, topic: str) -> Optional[Callable]:
"""Get handling function to be called.
Gets the instance of the class and method to call.
Args:
topic: Kafka topic. Example: "TicketAdded"
Raises:
MinosNetworkException: topic TicketAdded have no controller/action configured, please review th
configuration file.
"""
if topic not in self._handlers:
raise MinosActionNotFoundException(
f"topic {topic} have no controller/action configured, " f"please review th configuration file"
)
handler = self._handlers[topic]
logger.debug(f"Loaded {handler!r} action!")
return handler
@cached_property
def _queries(self) -> dict[str, str]:
# noinspection PyTypeChecker
return {
"count_not_processed": _COUNT_NOT_PROCESSED_QUERY,
"select_not_processed": _SELECT_NOT_PROCESSED_QUERY,
"mark_processing": _MARK_PROCESSING_QUERY,
"delete_processed": _DELETE_PROCESSED_QUERY,
"update_not_processed": _UPDATE_NOT_PROCESSED_QUERY,
}
