class Common():
id_queue = Queue()
request_queue = Queue()
picture_queue = Queue()
request_and_parse_queue = Queue()
CITY_LIST = ['武汉','宜昌','黄石','十堰','襄阳','鄂州','荆州','荆门','黄冈','咸宁','孝感','随州','恩施','神农架','潜江', '天门', '仙桃']
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}
def __init__(self, limit: int = 50):
self._loop = asyncio.get_event_loop()
self._limit = limit
self.logger = logging.getLogger(self.__class__.__name__)
self._pending_queue: Queue = Queue()
self._done_queue: Queue = Queue()
self._running_count = 0
self._closed = False
async def add_task(self, *urls):
"""添加新任务"""
for url in urls:
# 域名不在允许列表内
if not self._check_domain(url):
continue
hash_value = URL.md5_value(url)
# 重复url
if hash_value in self._handled_urls:
continue
self._handled_urls.add(hash_value)
u = URL(url)
if self._url_queue.get(u.host):
await self._url_queue[u.host].put(url)
else:
# 为新的域名创建url队列
self._url_queue[u.host] = Queue()
await self._url_queue[u.host].put(url)
# 初始化工作者
self._worker[u.host] = asyncio.Semaphore(self.WORKER_DOMAIN)
# 为新的域名创建一个调度任务
asyncio.create_task(
self._task_monitor(self._url_queue[u.host],
self._worker[u.host]))
async def add_client(self, id: str, key: str) -> None:
# create if none
read = self._key_to_readqueue.get(key)
if read and read.target == id and read.live:
pass
else:
_queue: Queue[str] = Queue()
if self._file:
_record = await self._file.get_record(id)
if _record:
await _queue.put(_record)
await _queue.put("-------------- loading -------------\n")
read = ReadQueue(key=key, target=id, queue=_queue)
self._key_to_readqueue[key] = read
# create if none
log = self._id_to_logqueue.get(id)
if log and log.live:
log.targets.add(key)
else:
await self.handler.add(id)
handler_queues = self.handler._queues
queue = handler_queues.get(id)
if queue:
log = LogQueue(
id=id,
targets={key},
queue=queue,
)
self._id_to_logqueue[id] = log
loop = asyncio.get_event_loop()
log.task = loop.create_task(self.entry_point(log))
self._reflesh()
async def test_commit_concurrency(aconn):
# Check the condition reported in psycopg2#103
# Because of bad status check, we commit even when a commit is already on
# its way. We can detect this condition by the warnings.
notices = Queue()
aconn.add_notice_handler(
lambda diag: notices.put_nowait(diag.message_primary))
stop = False
async def committer():
nonlocal stop
while not stop:
await aconn.commit()
await asyncio.sleep(0) # Allow the other worker to work
async def runner():
nonlocal stop
cur = aconn.cursor()
for i in range(1000):
await cur.execute("select %s;", (i, ))
await aconn.commit()
# Stop the committer thread
stop = True
await asyncio.gather(committer(), runner())
assert notices.empty(), "%d notices raised" % notices.qsize()
def __init__(self, index: int, model: AlphaZeroModel, env: Environment):
super().__init__(index)
self.prediction_queue_size = 16
self.parallel_search_num = 2
self.thinking_loop = 2
self.logging_thinking = False
self.change_tau_turn = 5
self.simulation_num_per_move = 50
self.prediction_worker_sleep_sec = 0.0001
self.wait_for_expanding_sleep_sec = 0.00001
self.virtual_loss = 3
self.noise_eps = 0.1
self.dirichlet_alpha = 0.03
self.c_puct = 1.5
self.model: AlphaZeroModel = model
self.env: Environment = env
self.labels_n = self.env.observation.nb_actions()
self.var_n = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_w = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_q = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_u = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_p = defaultdict(lambda: np.zeros((self.labels_n, )))
self.expanded = set()
self.now_expanding = set()
self.prediction_queue = Queue(self.prediction_queue_size)
self.sem = asyncio.Semaphore(self.parallel_search_num)
self.moves: List[List] = []
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.thinking_history = {} # for fun
def start(self, loop: asyncio.AbstractEventLoop):
if hasattr(self, 'started') and self.started:
# prevent a backend callback from starting more than 1 writer and creating more than 1 queue
return
self.queue = Queue()
loop.create_task(self.writer())
self.started = True
def __init__(self, loop, num_workers):
self.loop = loop
self.tasks = Queue(loop=self.loop)
self.workers = []
for _ in range(num_workers):
worker = asyncio.ensure_future(self.worker(), loop=self.loop)
self.workers.append(worker)
async def shuffle(cmd, message, args):
if message.author.voice:
same_bound = True
if message.guild.voice_client:
if message.guild.voice_client.channel.id != message.author.voice.channel.id:
same_bound = False
if same_bound:
if message.guild.voice_client:
queue = cmd.bot.music.get_queue(message.guild.id)
if queue:
queue_list = await cmd.bot.music.listify_queue(queue)
queue_count = len(queue_list)
new_queue = Queue()
while queue_list:
await new_queue.put(queue_list.pop(secrets.randbelow(len(queue_list))))
cmd.bot.music.queues.update({message.guild.id: new_queue})
response = discord.Embed(color=0x3B88C3, title=f'🔀 Shuffled {queue_count} songs.')
requester = f'{message.author.name}#{message.author.discriminator}'
response.set_author(name=requester, icon_url=user_avatar(message.author))
else:
response = discord.Embed(color=0xBE1931, title='❗ The queue is empty.')
else:
response = discord.Embed(color=0xBE1931, title='❗ I am not connected to any channel.')
else:
response = discord.Embed(color=0xBE1931, title='❗ You are not in my voice channel.')
else:
response = discord.Embed(color=0xBE1931, title='❗ You are not in a voice channel.')
await message.channel.send(embed=response)
async def main():
loop = asyncio.get_event_loop()
q = Queue()
feeder = Feeder(q, loop=loop)
suprvisor = Supervisor(loop=loop)
consumer = Consumer(q, loop=loop) # todo: be async iterator
async def spawn(uid) -> Process:
code = "import time; [(print(i) or time.sleep(0.3)) for i in range(10)]"
p = await asyncio.create_subprocess_exec(
sys.executable,
"-u",
"-c",
code,
stdout=asyncio.subprocess.PIPE,
)
suprvisor.watch(p)
feeder.feed(p, uid=uid)
await p.wait()
return p
consumer.consume() # todo: async context manager
await asyncio.wait([spawn(i) for i in range(7)])
print("end..")
await consumer.wait()
print("end....")
async def main():
pictures_queue = Queue()
workers_count = 300
connection = {
'user': '******', # input your postgres username
'database': 'your database name', # input your database name here
'host': '127.0.0.1', # change your host if it's not local
'password': '******' # input your password for this database
}
dsn = 'postgresql://{user}:{password}@{host}/{database}'.format(**connection)
engine = create_engine(dsn)
result = engine.execute('''select picture from "your_table_name"''')
res_list = []
for row in result:
clean_jpg = row['picture'].split("\n")
for i in clean_jpg:
res_list.append(i)
print(len(res_list))
for pic in res_list:
pictures_queue.put_nowait(pic)
session = AsyncHTMLSession()
tasks = []
for num in range(workers_count):
task = worker(pictures_queue, num, session)
tasks.append(task)
await asyncio.gather(*tasks)
def __init__(self, timeout=10, loop=None):
self.timeout = timeout
super().__init__(asyncio.StreamReader(), self.client_connected, loop)
self.close_code = None
self.close_reason = ''
# Futures tracking steps in the connection's lifecycle.
self.opening_handshake = asyncio.Future()
self.closing_handshake = asyncio.Future()
self.connection_closed = asyncio.Future()
# Queue of received messages.
self.messages = Queue()
# Mapping of ping IDs to waiters, in chronological order.
self.pings = collections.OrderedDict()
# Task managing the connection.
self.worker = asyncio.async(self.run())
# In a subclass implementing the opening handshake, the state will be
# CONNECTING at this point.
if self.state == 'OPEN':
self.opening_handshake.set_result(True)
async def _run(self):
self.logger.debug("Spider Task Start")
self.proxy = await self.proxy_gener.__anext__()
self.url_task_queue = Queue(30)
start_time = datetime.datetime.now()
tasks = []
print_log = asyncio.ensure_future(self._auto_print_log())
self.logger.debug("Create Crawl Tasks")
crawl_task = asyncio.ensure_future(self._run_crawler(0))
await self._add_url_to_queue()
await asyncio.sleep(5)
while not self.url_task_queue.empty() or self.count != 0:
await asyncio.sleep(5)
self.finished = True
await crawl_task
self.logger.critical("Simpyder任务执行完毕")
end_time = datetime.datetime.now()
delta_time = end_time - start_time
self.logger.critical('累计消耗时间：% s' % str(delta_time))
self.logger.critical('累计爬取链接：% s' % str(self._url_count))
self.logger.critical('累计生成对象：% s' % str(self._item_count))
await print_log
await self.session.close()
def __init__(self,
model: ReversiModel,
config: Config,
mcts_info=None,
enable_resign=True,
play_config=None):
self.model = model
self.play_config = play_config or self.config.play
self.enable_resign = enable_resign
self.config = config
mcts_info = mcts_info if mcts_info else ReversiPlayer.create_mcts_infomation(
)
self.N, self.W, self.P = mcts_info
# N(s, a): visit times
# W(s, a): value of action
# P(s, a): possibility of policy
# Q(s, a): average value of action
self.expanded = set(self.P.keys())
self.now_expanding = set()
self.requested_stop_thinking = False
self.resigned = False
self.moves = []
self.loop = asyncio.get_event_loop()
self.prediction_queue = Queue(self.play_config.prediction_queue_size)
self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)
def __init__(self, config):
self.config = config
self._loop = asyncio.new_event_loop()
config.set('loop', self._loop)
self._proxy_db = ProxyDb(
join(config.get("data_dir"), "freehp-agent.db"))
self._proxy_db.create_table()
self._checker = self._load_checker(config.get("checker_cls"))
self._checker_clients = config.getint("checker_clients")
self._check_interval = config.getint("check_interval")
self._block_time = config.getint("block_time")
self._proxy_queue = ProxyQueue(
config.getint("queue_size"),
max_fail_times=config.getint("max_fail_times"))
self._spider = ProxySpider(
self._get_spider_config(config.get('spider_config')))
self._agent_listen = config.get("agent_listen")
self._futures = []
self._wait_queue = Queue(loop=self._loop)
self._is_running = False
def __init__(self, config: Config, model, play_config=None):
self.config = config
self.model = model
self.play_config = play_config or self.config.play
self.api = ChessModelAPI(self.config, self.model)
self.move_lookup = {
k: v
for k, v in zip((
chess.Move.from_uci(mov)
for mov in self.config.labels), range(len(self.config.labels)))
}
self.labels_n = config.n_labels
self.var_n = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_w = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_q = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_u = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_p = defaultdict(lambda: np.zeros((self.labels_n, )))
self.expanded = set()
self.now_expanding = set()
self.prediction_queue = Queue(self.play_config.prediction_queue_size)
self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)
self.moves = []
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.thinking_history = {} # for fun
def __init__(self,
side,
network,
debugging=True,
n_playout=800,
search_threads=16,
virtual_loss=0.02,
policy_loop_arg=True,
c_puct=5,
dnoise=False,
temp_round=conf.train_temp_round,
can_surrender=False,
surrender_threshold=-0.99,
allow_legacy=False,
repeat_noise=True):
super(NetworkPlayer, self).__init__(side)
self.network = network
self.debugging = debugging
self.queue = Queue(400)
self.temp_round = temp_round
self.can_surrender = can_surrender
self.allow_legacy = allow_legacy
self.surrender_threshold = surrender_threshold
self.repeat_noise = repeat_noise
self.mcts_policy = mcts_async.MCTS(self.policy_value_fn_queue,
n_playout=n_playout,
search_threads=search_threads,
virtual_loss=virtual_loss,
policy_loop_arg=policy_loop_arg,
c_puct=c_puct,
dnoise=dnoise)
def __init__(self, config: Config, model, play_config=None, enable_resign=True, mtcs_info=None, api=None):
"""
:param config:
:param reversi_zero.agent.model.ReversiModel|None model:
:param MCTSInfo mtcs_info:
:parameter ReversiModelAPI api:
"""
self.config = config
self.model = model
self.play_config = play_config or self.config.play
self.enable_resign = enable_resign
self.api = api or ReversiModelAPI(self.config, self.model)
# key=(own, enemy, action)
mtcs_info = mtcs_info or self.create_mtcs_info()
self.var_n, self.var_w, self.var_p = mtcs_info
self.expanded = set(self.var_p.keys())
self.now_expanding = set()
self.prediction_queue = Queue(self.play_config.prediction_queue_size)
self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)
self.moves = []
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.callback_in_mtcs = None
self.thinking_history = {} # for fun
self.resigned = False
self.requested_stop_thinking = False
def get_queue(self, guild_id):
if guild_id in self.queues:
queue = self.queues[guild_id]
else:
queue = Queue()
self.queues.update({guild_id: queue})
return queue
def sending_loop_clients(websocket):
# create sending-queue
loop = asyncio.get_event_loop()
sending_queue_sensors = Queue()
logger.info('websockets .... smartHome Queue startet')
def changed(tmp):
loop.call_soon_threadsafe(sending_queue_sensors.put_nowait, tmp)
try:
consumers_clients.append(changed)
logger.info(
'websockets .... ein neuer smartHome-Client wurde in die Queue aufgenommen: %s '
% changed)
while True:
tmp_data = yield from sending_queue_sensors.get()
yield from websocket.send(tmp_data)
logger.debug(
'websockets .... Sende json Daten -> smartHome-Client : %s' %
tmp_data)
finally:
consumers_clients.remove(changed)
logger.info(
'websockets .... ein smartHome-Client wurde aus der Queue entfernt: %s '
% changed)
def __init__(self, num_workers):
self.loop = asyncio.get_event_loop()
self.tasks = Queue(loop=self.loop)
self.workers = []
for _ in range(num_workers):
worker = asyncio.create_task(self.worker())
self.workers.append(worker)
def __init__(self, net, num_playouts=1600):
self.net = net
self.now_expanding = set()
self.expanded = set()
# queue size should be >= the number of semmphores
# in order to maxout the coroutines
# There is not rule of thumbs to choose optimal semmphores
# And keep in mind: the more coroutines, the less (?) quality (?)
# of the Monte Carlo Tree obtains. As my searching is less deep
# w.r.t a sequential MCTS. However, since MCTS is a randomnized
# algorithm that tries to approximate a value by averaging over run_many
# random processes, the quality of the search tree is hard to define.
# It's a trade off among time, accuracy, and the frequency of NN updates.
self.sem = asyncio.Semaphore(16)
self.queue = Queue(16)
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.playouts = num_playouts # the more playouts the better
self.position = None
self.lookup = {v: k for k, v in enumerate(['W', 'U', 'N', 'Q', 'P'])}
self.hash_table = defaultdict(lambda: np.zeros([5, go.N**2 + 1]))
# see super in gtp warpper as 'GtpInterface'
super().__init__()
def __init__(self, config: Config, model, play_config=None, mem=None):
self.config = config
self.model = model
self.play_config = play_config or self.config.play
self.api = GomokuModelAPI(self.config, self.model)
self.labels_n = config.n_labels
self.var_n = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_w = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_q = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_u = defaultdict(lambda: np.zeros((self.labels_n, )))
self.var_p = defaultdict(lambda: np.zeros((self.labels_n, )))
self.expanded = set()
self.now_expanding = set()
self.prediction_queue = Queue(self.play_config.prediction_queue_size)
self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)
self.moves = []
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.thinking_history = {} # for fun
'''@Harvey Add'''
self.mem = None
if mem is not None:
print("Using MCTS memory")
self.mem = mem
#TODO: this should be a schedule or in config
self.beta = 0.5
def __init__(self,
config: Config,
model,
play_config=None,
enable_resign=True):
"""
:param config:
:param reversi_zero.agent.model.ReversiModel model:
"""
self.config = config
self.model = model
self.play_config = play_config or self.config.play
self.enable_resign = enable_resign
self.api = ReversiModelAPI(self.config, self.model)
# key=(own, enemy, action)
self.var_n = defaultdict(lambda: np.zeros((64, )))
self.var_w = defaultdict(lambda: np.zeros((64, )))
self.var_q = defaultdict(lambda: np.zeros((64, )))
self.var_p = defaultdict(lambda: np.zeros((64, )))
self.expanded = set()
self.now_expanding = set()
self.prediction_queue = Queue(self.play_config.prediction_queue_size)
self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)
self.moves = []
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.thinking_history = {} # for fun
self.resigned = False
def __init__(self):
global _loop, _queue
_loop = _loop or get_loop()
_queue = _queue or Queue(loop=_loop)
self.loop = _loop
self.queue = _queue
consumer.run()
def __init__(self, bot: discord.Client, cog, cog_name: str,
channel: discord.Channel, *args, **kwargs):
self.cog = cog
self.cog_name = cog_name
self.queue = Queue()
self.channel = channel
self.bot = bot
super(ChannelHandler, self).__init__(*args, **kwargs)
def __init__(self):
self.writables = Queue()
self.unit = ''
self.opened = False
self.closed = False
self.line_width = 0.4
self.color = None
self.linestyle = None
self.fill = False
async def unqueue(cmd, message, args):
if args:
if message.author.voice:
same_bound = True
if message.guild.voice_client:
if message.guild.voice_client.channel.id != message.author.voice.channel.id:
same_bound = False
if same_bound:
if message.guild.voice_client:
queue = cmd.bot.music.get_queue(message.guild.id)
if not queue.empty():
try:
order_num = int(args[0])
if order_num >= 1:
order_num -= 1
queue_list = await cmd.bot.music.listify_queue(
queue)
queue_size = len(queue_list)
if order_num <= queue_size - 1:
item = queue_list[order_num]
queue_list.remove(item)
new_queue = Queue()
for list_item in queue_list:
await new_queue.put(list_item)
cmd.bot.music.queues.update(
{message.guild.id: new_queue})
response = discord.Embed(
color=0x66CC66,
title=f'✅ Removed {item.title}.')
requester = f'{message.author.name}#{message.author.discriminator}'
response.set_author(name=requester,
icon_url=user_avatar(
message.author))
else:
response = discord.Embed(
color=0xBE1931,
title='❗ Input out of range.')
except ValueError:
response = discord.Embed(
color=0xBE1931,
title='❗ Invalid input. Numbers only.')
else:
response = discord.Embed(color=0xBE1931,
title='❗ The queue is empty.')
else:
response = discord.Embed(
color=0xBE1931,
title='❗ I am not connected to any channel.')
else:
response = discord.Embed(
color=0xBE1931, title='❗ You are not in my voice channel.')
else:
response = discord.Embed(color=0xBE1931,
title='❗ You are not in a voice channel.')
else:
response = discord.Embed(color=0xBE1931, title='❗ Nothing inputted.')
await message.channel.send(embed=response)
def subscribe_to_convo(user_id, channel_id):
if user_id not in BotData.convo_subscribers:
BotData.convo_subscribers[user_id] = {}
queue = Queue(loop=client.loop)
if channel_id in BotData.convo_subscribers[user_id]:
raise RuntimeError(
'User is already subscribed to that conversation')
BotData.convo_subscribers[user_id][channel_id] = queue
return queue
def __init__(self,
loop: typing.Optional[asyncio.AbstractEventLoop] = None):
self.loop = loop if loop is not None else asyncio.get_event_loop()
self.logger = logging.getLogger(self.__class__.__name__)
self.session: typing.Optional[aiohttp.ClientSession] = None
# coroutine`s concurrency support
self._queue: Queue = Queue(loop=self.loop)
self._done_queue: Queue = Queue(loop=self.loop)
self._running_count = 0
self._is_closed = False
# report var
self._reports: typing.DefaultDict[str, typing.List[int]] = defaultdict(
lambda: [0, 0])
self._drop_reports: typing.DefaultDict[
str, typing.List[int]] = defaultdict(lambda: [0, 0])
self._start_time = time.time()
self._last_time = self._start_time
self._report_slot = 60 # report once after one minute by default
