def __init__(self, room_address, db_name='./MusicBotDB.json'):
self.loop = asyncio.get_event_loop()
self.action_queue = asyncio.JoinableQueue()
self.packet_queue = asyncio.JoinableQueue()
self.ws_queue = asyncio.JoinableQueue()
self.action_task = None
self.recv_task = None
self.running_actions = []
self.room_address = room_address
self.ws = None
self.db = TinyDB(db_name)
self.mid = 0
self.next_ping_time = int(time())
self.last_latency = None
self.last_ping_log = int(time())
self.reset_mid()
### middleware
self.middleware = {}
self.queues = {}
self.packet_queues = []
self.add_middleware(LoggerMiddleware())
self.log_queue = self.get_input_queue(LoggerMiddleware.TAG)
signal.signal(signal.SIGINT, self.sigint_handler())
self.closing = False
def start(self):
start = time.time()
logger.info("Starting pipeline")
requester = self.requester or Requester(
error_contents=self.error_contents,
not_found_contents=self.not_found_contents)
futures, processes = [], []
if isinstance(self.input, asyncio.Queue):
input_q = self.input
else:
input_q = asyncio.JoinableQueue(maxsize=10)
process_queues = [input_q] + [
asyncio.JoinableQueue(5) for _ in range(len(self.processes))
]
logger.info("Created {0} queues".format(len(process_queues)))
for idx, process_cls in enumerate(self.processes):
process = process_cls(process_queues[idx], process_queues[idx + 1],
requester, self)
if isinstance(process, (Scraper, Pipeline)) and self.prepend_host:
process.set_host(self.prepend_host)
logger.info("Starting process {0}".format(process))
processes.append(process)
futures.append(process.start())
if not isinstance(self.input, asyncio.Queue):
logger.info("Using iterable input queue")
@asyncio.coroutine
def _input_func():
for item in self.input:
yield from input_q.put(item)
logger.info("Input queue drained")
yield from input_q.put(QueueDone)
futures.append(_input_func())
@asyncio.coroutine
def status_task():
nonlocal process_queues
while True:
yield from asyncio.sleep(1)
for p in processes:
print("{0:<20s}: {1:>6d}: {2}".format(
p.__class__.__name__, p.processed,
{k: v
for k, v in p.errors.items() if v != 0}))
print(" ")
if "display" in self.plugins:
asyncio. async (status_task())
[asyncio. async (f) for f in futures]
def test_task_done(self):
q = asyncio.JoinableQueue(loop=self.loop)
for i in range(100):
q.put_nowait(i)
accumulator = 0
# Two workers get items from the queue and call task_done after each.
# Join the queue and assert all items have been processed.
running = True
@asyncio.coroutine
def worker():
nonlocal accumulator
while running:
item = yield from q.get()
accumulator += item
q.task_done()
@asyncio.coroutine
def test():
for _ in range(2):
asyncio.Task(worker(), loop=self.loop)
yield from q.join()
self.loop.run_until_complete(test())
self.assertEqual(sum(range(100)), accumulator)
# close running generators
running = False
for i in range(2):
q.put_nowait(0)
def __init__(self, loop):
try:
self.queue = asyncio.JoinableQueue(loop=loop)
except AttributeError:
self.queue = asyncio.Queue(loop=loop)
self.closed = False
def __init__(self, conn_factory, logger, staging_tbl, proxy=None):
logger.info('Initializing AsyncScraper')
self._conn_factory = conn_factory
self._logger = logger
self._staging_tbl = staging_tbl
self._proxy = proxy
if hasattr(asyncio.Queue, 'join'):
jobsq = asyncio.Queue()
else:
jobsq = asyncio.JoinableQueue()
self._jobsq = jobsq
errors = {}
# Set up the queue and the listener to stage the records
logger.debug('Initializing Queue')
self._db_queue = queue.Queue()
Worker = RecordWorker(self._db_queue, conn_factory,
models.staging_table, models.log_table, True)
work_thread = Thread(target=Worker.record_inserter)
self._dbworker = Worker
work_thread.start()
logger.debug('Queue initialized')
def __init__(self):
super(BotMiddleware, self).__init__()
self.input = asyncio.JoinableQueue()
self._output = {}
self.task = None
self.closing = False
self._recv_functions = {}
self._exception_handler = None
self._add_routes()
def __init__(self):
super(PlayQueuedSongsMiddleware, self).__init__()
self.message_queue = asyncio.JoinableQueue()
self.song_queue = []
self.current_song = None
self.play_callback = None
# queued a song, waiting to see if it turns up
self.expecting_song = False
self.in_backlog = False
UsesCommands.set_handler(self, self.handle_event)
def test_join_empty_queue(self):
q = asyncio.JoinableQueue(loop=self.loop)
# Test that a queue join()s successfully, and before anything else
# (done twice for insurance).
@asyncio.coroutine
def join():
yield from q.join()
yield from q.join()
self.loop.run_until_complete(join())
def __init__(self, seed,
loop=None,
concurrency=10):
self.seed = seed
self.loop = loop or asyncio.get_event_loop()
self.concurrency = concurrency
self.q = asyncio.JoinableQueue()
self.visited = set()
self.done = set()
self.fp = open("produtos.csv", "w")
self.csv = csv.DictWriter(self.fp,
fieldnames=Page._fields)
self.csv.writeheader()
self.add_link(seed)
def __init__(
self,
roots,
exclude=None,
strict=True, # What to crawl.
max_redirect=10,
max_tries=4, # Per-url limits.
max_tasks=10,
*,
loop=None):
self.loop = loop or asyncio.get_event_loop()
self.roots = roots
self.exclude = exclude
self.strict = strict
self.max_redirect = max_redirect
self.max_tries = max_tries
self.max_tasks = max_tasks
self.q = asyncio.JoinableQueue(loop=self.loop)
self.urls = set()
self.done = []
self.connector = aiohttp.TCPConnector(loop=self.loop)
self.root_domains = set()
for root in roots:
parts = urllib.parse.urlparse(root)
host, port = urllib.parse.splitport(parts.netloc)
if not host:
continue
if re.match(r'\A[\d\.]*\Z', host):
self.root_domains.add(host)
else:
host = host.lower()
if self.strict:
self.root_domains.add(host)
else:
self.root_domains.add(lenient_host(host))
for root in roots:
self.add_url(root)
self.t0 = time.time()
self.t1 = None
def test_format(self):
q = asyncio.JoinableQueue(loop=self.loop)
self.assertEqual(q._format(), 'maxsize=0')
q._unfinished_tasks = 2
self.assertEqual(q._format(), 'maxsize=0 tasks=2')
def test_task_done_underflow(self):
q = asyncio.JoinableQueue(loop=self.loop)
self.assertRaises(ValueError, q.task_done)
def __init__(self, loop):
self._loop = loop
try:
self.queue = asyncio.JoinableQueue(loop=loop)
except AttributeError:
self.queue = asyncio.Queue(loop=loop)
def __init__(self, pipelines):
self._pipelines = pipelines
self._queue = asyncio.JoinableQueue()
def JoinableQueue(loop):
"""Create a JoinableQueue, even in new Python where it is called Queue."""
try:
return asyncio.JoinableQueue(loop=loop)
except AttributeError:
return asyncio.Queue(loop=loop)
