class QueueMailBox(MailBox):
def __init__(self, name, *args, **kwargs):
super(QueueMailBox, self).__init__(*args, **kwargs)
self._name = name
self._queue = Queue()
self._ready = True
async def prepare(self):
self._ready = True
async def put(self, msg=None):
if await self.policy():
self._queue.put_nowait(msg)
async def size(self):
return self._queue.qsize()
async def empty(self):
return self._queue.empty()
async def get(self):
result = None
result = await self._queue.get()
return result
async def policy(self):
mem_percent = psutil.virtual_memory().percent
if mem_percent > 80:
logger.warning("memory usage is gt than 80")
return True
async def test_commit_concurrency(aconn):
# Check the condition reported in psycopg2#103
# Because of bad status check, we commit even when a commit is already on
# its way. We can detect this condition by the warnings.
notices = Queue()
aconn.add_notice_handler(
lambda diag: notices.put_nowait(diag.message_primary))
stop = False
async def committer():
nonlocal stop
while not stop:
await aconn.commit()
await asyncio.sleep(0) # Allow the other worker to work
async def runner():
nonlocal stop
cur = aconn.cursor()
for i in range(1000):
await cur.execute("select %s;", (i, ))
await aconn.commit()
# Stop the committer thread
stop = True
await asyncio.gather(committer(), runner())
assert notices.empty(), "%d notices raised" % notices.qsize()
async def consumer_log_response(self, res_queue: queues.Queue):
while True:
print(res_queue.qsize())
item: requests.Response = await res_queue.get()
if item is None:
break
print(item.url)
res_queue.task_done()
async def astart(self, input_queue: Queue):
while not self.stop_flag or (self.stop_flag
and input_queue.qsize() > 0):
kwargs = await input_queue.get()
resp = await requests_future("post",
"https://api.github.com/graphql",
**kwargs)
if resp.status_code == 200:
await self.output_queue.put(resp.content)
class Core(object):
def __init__(self, bot):
self.bot = bot
self.timeout = int(self.bot.config.get('timeout'))
self.ping_queue = Queue(loop=bot.loop)
def connection_made(self):
self.bot.loop.call_later(self.timeout, self.check_ping)
self.ping_queue.put_nowait(self.bot.loop.time())
def check_ping(self): # pragma: no cover
# check if we received a ping
# reconnect if queue is empty
self.bot.log.debug(
'Ping queue size: {}'.format(self.ping_queue.qsize()))
if self.ping_queue.empty():
self.bot.loop.call_soon(self.bot.protocol.transport.close)
else:
self.bot.loop.call_later(self.timeout, self.check_ping)
while not self.ping_queue.empty():
self.ping_queue.get_nowait()
@event(rfc.PING)
def pong(self, data):
"""PING reply"""
self.ping_queue.put_nowait(self.bot.loop.time())
self.bot.send('PONG ' + data)
@event(rfc.NEW_NICK)
def recompile(self, nick=None, new_nick=None, **kw):
"""recompile regexp on new nick"""
if self.bot.nick == nick.nick:
self.bot.config['nick'] = new_nick
self.bot.recompile()
@event(rfc.ERR_NICK)
def badnick(self, me=None, nick=None, **kw):
"""Use alt nick on nick error"""
if me == '*':
self.bot.set_nick(self.bot.nick + '_')
self.bot.log.debug('Trying to regain nickname in 30s...')
self.bot.loop.call_later(30, self.bot.set_nick, self.bot.original_nick)
@event(rfc.RPL_ENDOFMOTD)
def autojoin(self, **kw):
"""autojoin at the end of MOTD"""
self.bot.config['nick'] = kw['me']
self.bot.recompile()
channels = utils.as_list(self.bot.config.get('autojoins', []))
for channel in channels:
channel = utils.as_channel(channel)
self.bot.log.info('Trying to join %s', channel)
self.bot.join(channel)
def as_completed(
self,
coroutines: typing.Iterable[typing.Coroutine],
limit: typing.Optional[int] = None
) -> typing.Generator[typing.Coroutine, None, None]:
"""Like "asyncio.as_completed",
run and iter coroutines out of the pool.
:param limit: set to "self.concurrent_limit" by default,
this "limit" is not shared with pool`s limit
"""
limit = self.concurrent_limit if limit is None else limit
coroutines = iter(coroutines)
queue = Queue(loop=self.loop)
todo = []
def _done_callback(f):
queue.put_nowait(f)
todo.remove(f)
try:
nf = asyncio.ensure_future(next(coroutines))
nf.add_done_callback(_done_callback)
todo.append(nf)
except StopIteration:
pass
async def _wait_for_one():
f = await queue.get()
return f.result()
if limit <= 0:
fs = {
asyncio.ensure_future(cor, loop=self.loop)
for cor in coroutines
}
else:
fs = {
asyncio.ensure_future(cor, loop=self.loop)
for cor in islice(coroutines, 0, limit)
}
for f in fs:
f.add_done_callback(_done_callback)
todo.append(f)
while len(todo) > 0 or queue.qsize() > 0:
yield _wait_for_one()
def _inspect_queue(queue: Queue):
"""
Log queue size.
"""
queue_size = queue.qsize()
if queue_size < 2:
return
if queue_size > 15:
logging_level = logging.ERROR
elif queue_size > 10:
logging_level = logging.WARNING
elif queue_size > 5:
logging_level = logging.INFO
else:
logging_level = logging.DEBUG
logger.log(logging_level, 'External rates queue size: %s', queue_size)
class OthelloPlayer:
def __init__(self,
config: Config,
client,
mode="gui",
weight_table=0,
c=10,
mc=False):
"""
:param config:
:param agent.model.OthelloModel|None model:
:param TreeNode mtcs_info:
:parameter OthelloModelAPI api:
"""
self.config = config
self.client = client
self.mode = mode
self.play_config = self.config.play
self.weight_table = weight_table
self.c = c
self.mc = mc
# mc_tree
self.num_tree, self.win_tree, self.policy_tree = createTrees()
# expanded
self.expanded = set() #expanded存p（dict）的set形式
self.now_expanding = set()
# threads
self.prediction_queue = Queue(
self.play_config.prediction_queue_size) #并行计算的信息队列queue大小
self.sem = asyncio.Semaphore(
self.play_config.parallel_search_num) #限制并行搜索的线程数
self.loop = asyncio.get_event_loop()
# for gui
if self.mode == 'gui':
self.thinking_history = None # for fun
self.avalable = None
self.allow_resign = False
elif self.mode == 'self_play':
self.moves = []
self.allow_resign = True
self.test_mode = False
# params
self.running_simulation_num = 0
# solver
self.solver = OthelloSolver() #引入minmax树类
def win_rate(self, node):
return self.win_tree[node] / (self.num_tree[node] + 1e-5)
# think_and_play
def think_and_play(self, own, enemy):
"""play tmd：方案：50步以前使用深度學習mctree，若tree到達50步深度后再用minmaxtree； 50步以後直接用minmaxtree
若搜不到/超時再用之前構建的樹"""
# renew env
self.start_time = time.time()
env = OthelloEnv().update(own, enemy, next_to_play=Stone.black)
node = create_node(env)
#五十步之后直接minmax树搜索，若搜索不到，再用深度學習
if env.epoch >= self.play_config.use_solver_turn:
logger.warning(f"Entering minmax_tree process")
ret = self._solver(node)
if ret: # not save move as play data
return ret
else: # 五十步之前直接用深度學習
for t1 in range(self.play_config.thinking_loop
): # search moves for 3 times
logger.warning(f"Entering {t1} thinking_loop")
self._expand_tree(env, node)
policy, action, value_diff = self._calc_policy_and_action(node)
# if action 足够大 + n足够大 \ turn 很小
if env.epoch <= self.play_config.start_rethinking_turn or \
(value_diff > -0.01 and self.num_tree[node][action] >= self.play_config.required_visit_to_decide_action):
break
# record or return
if self.mode == 'gui':
self._update_thinking_history(own, enemy, action, policy)
self._update_avalable(own, enemy, action, policy)
elif self.mode == 'self_play':
if self.allow_resign: # resign win_rate 太小没有胜率。
if self.play_config.resign_threshold is not None and\
np.max(self.win_rate(node)-(self.num_tree[node]==0)*10) <= self.play_config.resign_threshold:
if env.epoch >= self.config.play.allowed_resign_turn:
return AcNQ(None, 0, 0) # means resign
else:
logger.debug(
f"Want to resign but disallowed turn {env.epoch} < {self.config.play.allowed_resign_turn}"
)
# save fuckers
saved_policy = self.__calc_policy_by_prob(
node
) if self.config.play_data.save_policy_of_tau_1 else policy
self.__save_data_to_moves(own, enemy, saved_policy)
return AcNQ(action=action,
n=self.num_tree[node][action],
q=self.win_rate(node)[action])
def _solver(self, node):
# use solver to do minmax搜索
action, point = self.solver.solve(node.black,
node.white,
Stone(node.next_to_play),
exactly=True)
if action is None: #如果沒搜索到是不可以返回None的因爲要
return None
else:
policy = np.zeros(64)
policy[action] = 1
update_num_tree_with_one_or_moresides(self.num_tree, node, action,
["set"], [999])
update_win_tree_with_one_or_moresides(self.win_tree, node, action,
["set"],
[np.sign(point) * 999])
update_policy_tree_with_one_or_moresides(self.policy_tree, node,
["set"], [policy])
self._update_thinking_history(node.black, node.white, action,
policy)
return AcNQ(action=action, n=999, q=np.sign(point))
def _expand_tree(self, env, node):
if env.epoch > 0: # 对树进行拓展
self._expand_tree_2(env.chessboard.black, env.chessboard.white)
else:
self._set_first_move(node)
def _expand_tree_2(self, own, enemy):
# params
loop = self.loop
self.running_simulation_num = 0
# n simulation/move
coroutine_list = []
# 200 simulations
for it in range(self.play_config.simulation_num_per_move):
coroutine_list.append(self.__start_search_my_move(own, enemy))
coroutine_list.append(self.__prediction_worker())
loop.run_until_complete(asyncio.gather(*coroutine_list))
async def __start_search_my_move(self, own, enemy):
# set parmas
self.running_simulation_num += 1
# wait sems
with await self.sem: # 8綫程
env = OthelloEnv().update(own, enemy, Stone.black)
leaf_v = await self.___recursive_simulation(env, is_root_node=True)
self.running_simulation_num -= 1
return leaf_v
async def ___recursive_simulation(self,
env: OthelloEnv,
is_root_node=False):
"fertilize tree process"
# get both keys
node, another_side_node = create_both_nodes(env)
if self.test_mode:
if (node not in map.keys()):
map[node] = env.epoch
# return condition 1
if env.done:
if env.result == Result.black:
return 1
elif env.result == Result.white:
return -1
else:
return 0
# return condition 2 : get solver（大于50步，minmax）
if env.epoch >= self.config.play.use_solver_turn_in_simulation:
action, point = self.solver.solve(node.black,
node.white,
Stone(node.next_to_play),
exactly=False)
if action:
point = point if env.next_to_play == Stone.black else -point
leaf_v = np.sign(point)
leaf_p = np.zeros(64)
leaf_p[action] = 1
# update tree
update_num_tree_with_one_or_moresides(self.num_tree, node,
action, ["plus", "plus"],
[1, 1]) #走过的位置+1
update_win_tree_with_one_or_moresides(
self.win_tree, node, action, ["plus", "minus"],
[leaf_v, leaf_v]) #走此步赢的次数+-1（win）
update_policy_tree_with_one_or_moresides(
self.policy_tree, node, ["set", "set"],
[leaf_p, leaf_p]) #此节点应该走的位置（position）
return np.sign(point)
if time.time() - self.start_time >= 55:
return 0
#return condition 3 : expand tree（小於等於50步，用深度學習）
while node in self.now_expanding: # 兩個搜索綫程遇到同一個node，會有衝突的問題
await asyncio.sleep(self.config.play.wait_for_expanding_sleep_sec)
# is leaf
if node not in self.expanded: # reach leaf node
leaf_v = await self.____expand_leaf_node(env)
if env.next_to_play == Stone.black:
return leaf_v # Value for black
else:
return -leaf_v # Value for white == -Value for black
else: # not leaf do
virtual_loss_for_w = self.config.play.virtual_loss if env.next_to_play == Stone.black else -self.config.play.virtual_loss
action_t = self.____decide_action(env, is_root_node) #UCB公式
update_num_tree_with_one_or_moresides(
self.num_tree, node, action_t, ["plus"],
[self.config.play.virtual_loss])
update_win_tree_with_one_or_moresides(self.win_tree, node,
action_t, ["minus"],
[virtual_loss_for_w])
env.do(action_t)
leaf_v = await self.___recursive_simulation(env) # next move
# on returning search path
update_num_tree_with_one_or_moresides(
self.num_tree, node, action_t, ["plus", "plus"],
[-self.config.play.virtual_loss + 1, 1])
update_win_tree_with_one_or_moresides(
self.win_tree, node, action_t, ["plus", "minus"],
[virtual_loss_for_w + leaf_v, leaf_v])
if self.test_mode:
logger.warning(map[node], leaf_v)
return leaf_v
async def ____expand_leaf_node(self, env):
"use to expand new leaf"
node, another_side_node = create_both_nodes(env)
self.now_expanding.add(node)
# flip + rotate
rotate_right_num, is_flip_vertical, black_ary, white_ary = flip_and_rotate_board_to_array(
env.chessboard.black, env.chessboard.white)
# predict
state = [
white_ary, black_ary
] if env.next_to_play == Stone.white else [black_ary, white_ary]
future = await self.predict(np.array(state)) # type: Future
await future
leaf_p, leaf_v = future.result()
# reverse rotate and flip about leaf_p
leaf_p = flip_and_rotate_result(leaf_p, rotate_right_num,
is_flip_vertical)
if self.mc:
black = env.chessboard.black
leaf_v += np.sum(bit_to_array(black, 64) * self.weight_table)
# update
update_policy_tree_with_one_or_moresides(self.policy_tree, node,
["set", "set"],
[leaf_p, leaf_p])
self.expanded.add(node)
self.now_expanding.remove(node)
return leaf_v
def ____decide_action(self, env, is_root_node):
# find correct moves
node = create_node(env)
legal_moves = find_correct_moves(
node.black, node.white
) if env.next_to_play == Stone.black else find_correct_moves(
node.white, node.black)
# vn = formula here
vn = max(np.sqrt(np.sum(self.num_tree[node])),
1) # SQRT of sum(N(s, b); for all b)
# p = formula here re-normalize in legal moves
vp = self.policy_tree[node]
vp = vp * bit_to_array(legal_moves, 64)
temperature = 1
if np.sum(vp) > 0:
temperature = min(
np.exp(1 -
np.power(env.epoch / self.config.play.policy_decay_turn,
self.config.play.policy_decay_power)), 1)
vp = normalize(vp, temperature)
# add noise 0.75*p + 0.25*noise
if is_root_node and self.play_config.noise_eps > 0: # Is it correct?? -> (1-e)p + e*Dir(alpha)
noise = dirichlet_noise_of_mask(legal_moves,
self.play_config.dirichlet_alpha)
vp = (1 - self.play_config.noise_eps
) * vp + self.play_config.noise_eps * noise
# u_ = formula here
vpn = vp * vn / (1 + self.num_tree[node])
if env.next_to_play == Stone.black:
vpn_with_weight = (self.win_rate(node) * self.c + vpn + 1000 +
self.weight_table) * bit_to_array(
legal_moves, 64)
else:
vpn_with_weight = (-self.win_rate(node) * self.c + vpn + 1000 +
self.weight_table) * bit_to_array(
legal_moves, 64)
action_t = int(np.argmax(vpn_with_weight))
return action_t
async def __prediction_worker(self):
" do prediction in this worker"
margin = 10 # wait for at most 10 epochs x 0.0001
while self.running_simulation_num > 0 or margin > 0:
if self.prediction_queue.empty():
if margin > 0:
margin -= 1
await asyncio.sleep(
self.config.play.prediction_worker_sleep_sec)
continue
item_list = [
self.prediction_queue.get_nowait()
for _ in range(self.prediction_queue.qsize())
] # type: list[QItem]
data = np.array([x.state for x in item_list])
policy_ary, value_ary = self.client.forward(
data) # shape=(N, 2, 8, 8)
for p, v, item in zip(policy_ary, value_ary, item_list):
item.future.set_result((p, v))
def _set_first_move(self, node):
# chose the random num_tree = [1] policy_tree = [每个可能的地方都是1/n]
legal_array = bit_to_array(find_correct_moves(node.black, node.white),
64)
action = np.argmax(legal_array)
update_num_tree_with_one_or_moresides(self.num_tree, node, action,
["set"], [1])
update_win_tree_with_one_or_moresides(self.win_tree, node, action,
["set"], [0])
update_policy_tree_with_one_or_moresides(
self.policy_tree, node, ["set"],
[legal_array / np.sum(legal_array)])
def _calc_policy_and_action(self, node):
policy = self._calc_policy(
node.black, node.white
) # 先验 最大的n, 前四步p[n]=num_tree[key][n]，后面p矩阵只有var_numb最大位置为1，其余为0.
action = int(np.random.choice(range(64),
p=policy)) #随机走一个点，p为随机取的各点概率（先验）
action_by_value = int(
np.argmax(self.win_rate(node) +
(self.num_tree[node] > 0) * 100)) #选走过的、q（胜率）最大的那个位置
value_diff = self.win_rate(node)[action] - self.win_rate(node)[
action_by_value] #
return policy, action, value_diff
def _calc_policy(self, own, enemy):
env = OthelloEnv().update(own, enemy, Stone.black)
node = create_node(env)
# if turn < 4
if env.epoch < self.play_config.change_tau_turn:
return self.__calc_policy_by_prob(node) # p value
else:
return self.__calc_policy_by_max(node)
def __calc_policy_by_prob(self, node):
return self.num_tree[node] / np.sum(self.num_tree[node]) # tau = 1
def __calc_policy_by_max(self, node):
action = np.argmax(self.num_tree[node]) # tau = 0
ret = np.zeros(64) # one hot
ret[action] = 1
return ret
def _update_thinking_history(self, black, white, action, policy):
node = TreeNode(black, white, Stone.black.value)
next_key = self.__get_next_key(black, white, action)
self.thinking_history = \
LastAcNQ(action, policy, list(self.win_rate(node)), list(self.num_tree[node]),
list(self.win_rate(next_key)), list(self.num_tree[next_key]))
def _update_avalable(self, black, white, action, policy):
node = TreeNode(black, white, Stone.black.value)
next_key = self.__get_next_key(black, white, action)
self.avalable = LastAva(
find_correct_moves(node.black, node.white),
find_correct_moves(next_key.white, next_key.black))
def __get_next_key(self, own, enemy, action):
env = OthelloEnv().update(own, enemy, Stone.black)
env.do(action)
return create_node(env)
def __save_data_to_moves(self, own, enemy, policy):
for flip in [False, True]:
for rot_right in range(4):
self.moves.append(
flip_and_rotate_right(flip, rot_right, own, enemy, policy))
async def predict(self, x):
future = self.loop.create_future()
await self.prediction_queue.put(QItem(x, future))
return future
class DownloadingService(metaclass=Metaclass):
def __init__(self, crawler_manager: CrawlerManager,
downloader_manager: DownloaderManager):
self.crawler_manager = crawler_manager
self.downloader_manager = downloader_manager
self.running_crawler = {}
self.start_time = int(time.time())
self.schedule_queue = Queue()
self.QUIT = False
def get_running_messages(self):
msgs = []
for crawler_name, downloader in self.downloader_manager.downloaders.items(
):
status = "Running" if downloader.is_running else "Not Running"
msg = "Status: {}" \
" | Crawler Name: {}, " \
" | Concurrent Number: {}," \
" | Request Number: {}," \
" | Finished Number: {}," \
" | Failed Number: {}".\
format(status, crawler_name, downloader.concurrent_number, downloader.pushed_request_count,
downloader.finished_request_count, downloader.failed_request_count)
msgs.append(msg)
msgs.append("运行时长：{}s".format(time.time() - self.start_time))
return "\n".join(msgs)
async def status_print_loop(self):
while True:
running_time = int(time.time()) - self.start_time
if running_time > 0 and running_time % 60 == 0:
log.info(self.get_running_messages())
await asyncio.sleep(1)
if self.QUIT is True:
log.info(self.get_running_messages())
break
log.debug("Quit status print loop")
async def append_new_crawling_mission(self, crawler_name, crawler_params):
await self.schedule_queue.put((crawler_name, crawler_params))
async def task_scheduling_loop(self):
while True:
try:
crawler_name, crawler_params = self.schedule_queue.get_nowait()
except QueueEmpty:
await asyncio.sleep(1)
else:
crawler = self.crawler_manager.create_crawler(
crawler_name, crawler_params)
downloader = self.downloader_manager.get_or_create_downloader(
crawler)
await self.downloader_manager.schedule(crawler, downloader)
finally:
if self.QUIT and self.schedule_queue.qsize() == 0:
break
log.debug("Quit task scheduling loop")
async def start(self, forever=True):
self.start_time = int(time.time())
f1 = asyncio.ensure_future(self.task_scheduling_loop())
f2 = asyncio.ensure_future(
self.downloader_manager.reschedule_downloaders())
f3 = asyncio.ensure_future(self.status_print_loop())
if forever == False:
def callback(_):
log.debug("Downloading Mission Done!")
self.QUIT = True
f2.add_done_callback(callback)
await asyncio.gather(f1, f2, f3)
class Ant(abc.ABC):
response_pipelines: typing.List[Pipeline] = []
request_pipelines: typing.List[Pipeline] = []
item_pipelines: typing.List[Pipeline] = []
request_cls = Request
response_cls = Response
request_timeout = DEFAULT_TIMEOUT.total
request_retries = 3
request_retry_delay = 5
request_proxies: typing.List[typing.Union[str, URL]] = []
request_max_redirects = 10
request_allow_redirects = True
response_in_stream = False
connection_limit = 100 # see "TCPConnector" in "aiohttp"
connection_limit_per_host = 0
concurrent_limit = 100
def __init__(self,
loop: typing.Optional[asyncio.AbstractEventLoop] = None):
self.loop = loop if loop is not None else asyncio.get_event_loop()
self.logger = logging.getLogger(self.__class__.__name__)
self.session: aiohttp.ClientSession = ClientSession(
response_class=self.response_cls,
connector=aiohttp.TCPConnector(
limit=self.connection_limit,
enable_cleanup_closed=True,
limit_per_host=self.connection_limit_per_host))
# coroutine`s concurrency support
self._queue = Queue(loop=self.loop)
self._done_queue = Queue(loop=self.loop)
self._running_count = 0
self._is_closed = False
# report var
self._reports: typing.DefaultDict[str, typing.List[
int, int]] = defaultdict(lambda: [0, 0])
self._drop_reports: typing.DefaultDict[str, typing.List[
int, int]] = defaultdict(lambda: [0, 0])
self._start_time = time.time()
self._last_time = self._start_time
self._report_slot = 60 # report once after one minute by default
@property
def name(self):
return self.__class__.__name__
@property
def is_running(self) -> bool:
return self._running_count > 0
async def request(
self,
url: typing.Union[str, URL],
method: str = aiohttp.hdrs.METH_GET,
params: typing.Optional[dict] = None,
headers: typing.Optional[dict] = None,
cookies: typing.Optional[dict] = None,
data: typing.Optional[typing.Union[typing.AnyStr, typing.Dict,
typing.IO]] = None,
proxy: typing.Optional[typing.Union[str, URL]] = None,
timeout: typing.Optional[typing.Union[int, float]] = None,
retries: typing.Optional[int] = None,
response_in_stream: typing.Optional[bool] = None) -> Response:
if not isinstance(url, URL):
url = URL(url)
if proxy and not isinstance(proxy, URL):
proxy = URL(proxy)
elif proxy is None:
proxy = self.get_proxy()
if timeout is None:
timeout = self.request_timeout
if retries is None:
retries = self.request_retries
if response_in_stream is None:
response_in_stream = self.response_in_stream
req = self.request_cls(method,
url,
timeout=timeout,
params=params,
headers=headers,
cookies=cookies,
data=data,
proxy=proxy,
response_in_stream=response_in_stream)
req = await self._handle_thing_with_pipelines(req,
self.request_pipelines)
self.report(req)
if retries > 0:
res = await self.make_retry_decorator(
retries, self.request_retry_delay)(self._request)(req)
else:
res = await self._request(req)
res = await self._handle_thing_with_pipelines(res,
self.response_pipelines)
self.report(res)
return res
async def collect(self, item: Item) -> None:
self.logger.debug('Collect item: ' + str(item))
await self._handle_thing_with_pipelines(item, self.item_pipelines)
self.report(item)
async def open(self) -> None:
self.logger.info('Opening')
for pipeline in itertools.chain(self.item_pipelines,
self.response_pipelines,
self.request_pipelines):
obj = pipeline.on_spider_open()
if asyncio.iscoroutine(obj):
await obj
async def close(self) -> None:
await self.wait_scheduled_coroutines()
for pipeline in itertools.chain(self.item_pipelines,
self.response_pipelines,
self.request_pipelines):
obj = pipeline.on_spider_close()
if asyncio.iscoroutine(obj):
await obj
await self.session.close()
self._is_closed = True
self.logger.info('Closed')
@abc.abstractmethod
async def run(self) -> None:
"""App custom entrance"""
async def main(self) -> None:
try:
await self.open()
await self.run()
except Exception as e:
self.logger.exception('Run ant with ' + e.__class__.__name__)
try:
await self.close()
except Exception as e:
self.logger.exception('Close ant with ' + e.__class__.__name__)
# total report
for name, counts in self._reports.items():
self.logger.info('Get {:d} {:s} in total'.format(counts[1], name))
for name, counts in self._drop_reports.items():
self.logger.info('Drop {:d} {:s} in total'.format(counts[1], name))
self.logger.info('Run {:s} in {:f} seconds'.format(
self.__class__.__name__,
time.time() - self._start_time))
@staticmethod
def make_retry_decorator(
retries: int, delay: float
) -> typing.Callable[[typing.Callable], typing.Callable]:
return retry(wait=wait_fixed(delay),
retry=(retry_if_result(lambda res: res.status >= 500)
| retry_if_exception_type(
exception_types=aiohttp.ClientError)),
stop=stop_after_attempt(retries + 1))
def get_proxy(self) -> typing.Optional[URL]:
"""Chose a proxy, default by random"""
try:
return URL(random.choice(self.request_proxies))
except IndexError:
return None
def schedule_coroutine(self, coroutine: typing.Coroutine) -> None:
"""Like "asyncio.ensure_future", it schedule coroutine in event loop
and return immediately.
Call "self.wait_scheduled_coroutines" make sure all coroutine has been
done.
"""
def _done_callback(f):
self._running_count -= 1
self._done_queue.put_nowait(f)
try:
if (self.concurrent_limit == -1
or self._running_count < self.concurrent_limit):
next_coroutine = self._queue.get_nowait()
self._running_count += 1
asyncio.ensure_future(
next_coroutine,
loop=self.loop).add_done_callback(_done_callback)
except QueueEmpty:
pass
if self._is_closed:
self.logger.warning('This pool has be closed!')
return
if (self.concurrent_limit == -1
or self._running_count < self.concurrent_limit):
self._running_count += 1
asyncio.ensure_future(
coroutine, loop=self.loop).add_done_callback(_done_callback)
else:
self._queue.put_nowait(coroutine)
def schedule_coroutines(
self, coroutines: typing.Iterable[typing.Coroutine]) -> None:
"""A short way to schedule many coroutines.
"""
for coroutine in coroutines:
self.schedule_coroutine(coroutine)
async def wait_scheduled_coroutines(self):
"""Wait scheduled coroutines to be done, can be called many times.
"""
while self._running_count > 0 or self._done_queue.qsize() > 0:
await self._done_queue.get()
def as_completed(
self,
coroutines: typing.Iterable[typing.Coroutine],
limit: typing.Optional[int] = None
) -> typing.Generator[typing.Coroutine, None, None]:
"""Like "asyncio.as_completed",
run and iter coroutines out of the pool.
:param limit: set to "self.concurrent_limit" by default,
this "limit" is not shared with pool`s limit
"""
limit = self.concurrent_limit if limit is None else limit
coroutines = iter(coroutines)
queue = Queue(loop=self.loop)
todo = []
def _done_callback(f):
queue.put_nowait(f)
todo.remove(f)
try:
nf = asyncio.ensure_future(next(coroutines))
nf.add_done_callback(_done_callback)
todo.append(nf)
except StopIteration:
pass
async def _wait_for_one():
f = await queue.get()
return f.result()
if limit <= 0:
fs = {
asyncio.ensure_future(cor, loop=self.loop)
for cor in coroutines
}
else:
fs = {
asyncio.ensure_future(cor, loop=self.loop)
for cor in islice(coroutines, 0, limit)
}
for f in fs:
f.add_done_callback(_done_callback)
todo.append(f)
while len(todo) > 0 or queue.qsize() > 0:
yield _wait_for_one()
async def as_completed_with_async(
self,
coroutines: typing.Iterable[typing.Coroutine],
limit: typing.Optional[int] = None,
raise_exception: bool = True,
) -> typing.AsyncGenerator[typing.Any, None]:
"""as_completed`s async version, can catch and log exception inside.
"""
for coro in self.as_completed(coroutines, limit=limit):
try:
yield await coro
except Exception as e:
if raise_exception:
raise e
else:
self.logger.exception('Get exception {:s} in '
'"as_completed_with_async"'.format(
str(e)))
def report(self, thing: Things, dropped: bool = False) -> None:
now_time = time.time()
if now_time - self._last_time > self._report_slot:
self._last_time = now_time
for name, counts in self._reports.items():
count = counts[1] - counts[0]
counts[0] = counts[1]
self.logger.info(
'Get {:d} {:s} in total with {:d}/{:d}s rate'.format(
counts[1], name, count, self._report_slot))
for name, counts in self._drop_reports.items():
count = counts[1] - counts[0]
counts[0] = counts[1]
self.logger.info(
'Drop {:d} {:s} in total with {:d}/{:d} rate'.format(
counts[1], name, count, self._report_slot))
report_type = thing.__class__.__name__
if dropped:
reports = self._drop_reports
else:
reports = self._reports
counts = reports[report_type]
counts[1] += 1
async def _handle_thing_with_pipelines(
self, thing: Things, pipelines: typing.List[Pipeline]) -> Things:
"""Process thing one by one, break the process chain when get
exception.
"""
self.logger.debug('Process thing: ' + str(thing))
raw_thing = thing
for pipeline in pipelines:
try:
thing = pipeline.process(thing)
if asyncio.iscoroutine(thing):
thing = await thing
except Exception as e:
if isinstance(e, ThingDropped):
self.report(raw_thing, dropped=True)
raise e
return thing
async def _request(self, req: Request) -> Response:
if req.proxy is not None:
# proxy auth not work in one session with many requests,
# add auth header to fix it
if req.proxy.scheme == 'http' and req.proxy.user is not None:
req.headers[aiohttp.hdrs.PROXY_AUTHORIZATION] = \
aiohttp.BasicAuth.from_url(req.proxy).encode()
# cookies in headers, params in url
req_kwargs = dict(headers=req.headers,
data=req.data,
timeout=req.timeout,
proxy=req.proxy,
max_redirects=self.request_max_redirects,
allow_redirects=self.request_allow_redirects)
response = await self.session._request(req.method, req.url,
**req_kwargs)
if not req.response_in_stream:
await response.read()
response.close()
await response.wait_for_close()
return response
async def radar(radar_Q: Queue = None):
if radar_Q is None:
print("Radar task is terminating. The radar_Q is None. Nothing to do")
return
try:
q = Queue()
while True:
async with async_timeout.timeout(5) as tm:
while True:
try:
incoming_action:INCOMING_ACTION_TYPE = await radar_Q.get()
q.put_nowait(incoming_action)
STATS['number_rx'] += 1
STATS['max_incoming_q_size'] = max(radar_Q.qsize(), STATS['max_incoming_q_size'])
except Exception as x:
break
if not tm.expired:
continue
time_now = datetime.utcnow()
while not q.empty():
item = q.get_nowait()
icao_rec = item.actionMsg
# print(F"DB_WORKER_RXED: {action_types} {icao_rec}")
# await db_process_sbs1_msg2(database, sbs1_msg)
id = item.aircraftId
last_seen = icao_rec['last_seen']
data = icao_rec['current']
delta_sec = (time_now - last_seen).total_seconds()
if delta_sec > 300:
print(F"@Radar: Got an old one: {id} {last_seen} {delta_sec}")
STATS['rx_expired'] += 1
remove_track(id)
del history[id]
continue
history[id] = last_seen
STATS['max_history_size'] = max(STATS['max_history_size'],len(history))
if data['lon'] is not None and data['lat'] is not None:
update_track(id,data['lat'], data['lon'])
STATS['updates'] += 1
update_table_store(id, last_seen, data)
render_table_store()
# endwhile
# cleanup any lingering tracks:
flush()
refresh()
except CancelledError:
print("Cancellling radar task")
except Exception as x:
print(F"Exception {x}")
async def dump1090TCPListener(incomingQ: Queue, host: str, port: int):
LOG.info(F"Starting: Opening TCP Stream to Dump Server at: {host}:{port}")
while True:
try:
stream_reader, _ = await open_connection(host, port)
STATS['connection_closed'] = False
STATS['connections'] += 1
except Exception as x:
print(F"@dump1090TCPListener: Got exception {x}")
LOG.exception(x)
STATS['connections_exception'] += 1
continue
LOG.info("Got stream_reader")
try:
BUFSIZE = 1024
data = ""
while True:
new_data = await stream_reader.read(BUFSIZE)
if not new_data:
LOG.info("receiver: connection closed")
STATS['connection_closed'] = True
return
# break it into lines
data += new_data.decode('utf-8')
#print(F"#######\n{data}######")
cur_pos = 0
while True:
# print(F"DATA\n{data}\n")
cr_index = data.find('\n')
if cr_index > 0:
# print(F"Slice {cur_pos} : {cr_index} Data: {data[cur_pos:cr_index]}")
l = data[cur_pos:cr_index].strip()
#print(F"Q<-{l}\n", end='')
incomingQ.put_nowait(l)
STATS["msg_rx"] += 1
STATS["max_incomingQ_size"] = max(
incomingQ.qsize(), STATS["max_incomingQ_size"])
# print(F"done")
if STATS["msg_rx"] % 100 == 0:
LOG.info(
F"Rxed {STATS['msg_rx']} max_q size {STATS['max_incomingQ_size']}"
)
cur_pos = cr_index + 1
data = data[cur_pos:]
cur_pos = 0
else:
# print(F"Slice {cur_pos} : {cr_index} Data: {data[cur_pos:cr_index]}")
break
except CancelledError:
LOG.info("Cancelling")
break
except Exception:
LOG.exception("General exception")
finally:
pass
LOG.info(F"Exiting")
class BackendQueue:
def start(self, loop: asyncio.AbstractEventLoop, multiprocess=False):
if hasattr(self, 'started') and self.started:
# prevent a backend callback from starting more than 1 writer and creating more than 1 queue
return
self.multiprocess = multiprocess
if self.multiprocess:
self.queue = Pipe(duplex=False)
self.worker = Process(target=BackendQueue.worker, args=(self.writer,), daemon=True)
self.worker.start()
else:
self.queue = Queue()
self.worker = loop.create_task(self.writer())
self.started = True
async def stop(self):
if self.multiprocess:
self.queue[1].send(SHUTDOWN_SENTINEL)
self.worker.join()
else:
await self.queue.put(SHUTDOWN_SENTINEL)
self.running = False
@staticmethod
def worker(writer):
try:
loop = asyncio.new_event_loop()
loop.run_until_complete(writer())
except KeyboardInterrupt:
pass
async def writer(self):
raise NotImplementedError
async def write(self, data):
if self.multiprocess:
self.queue[1].send(data)
else:
await self.queue.put(data)
@asynccontextmanager
async def read_queue(self) -> list:
if self.multiprocess:
msg = self.queue[0].recv()
if msg == SHUTDOWN_SENTINEL:
self.running = False
yield []
else:
yield [msg]
else:
current_depth = self.queue.qsize()
if current_depth == 0:
update = await self.queue.get()
if update == SHUTDOWN_SENTINEL:
yield []
else:
yield [update]
self.queue.task_done()
else:
ret = []
count = 0
while current_depth > count:
update = await self.queue.get()
count += 1
if update == SHUTDOWN_SENTINEL:
self.running = False
break
ret.append(update)
yield ret
for _ in range(count):
self.queue.task_done()
