Example #1
0
class Converse:
    """Responsible for communication
    between client and server
    """
    def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
        self.reader = reader
        self.writer = writer
        self.message_queue = Queue(maxsize=maxsize)
        self.frame = Frames(self.reader, self.writer)

    async def send(self, message, fin: bool = True, mask: bool = True):
        """Send message to server """

        if isinstance(message, str):
            message = message.encode()
        code = DataFrames.text.value
        await self.frame.write(fin=fin, code=code, message=message, mask=mask)

    async def receive(self, text=False, mask=False):
        """Get a message
        If there is no message in the message queue,
        try to read it or pop up directly from the message queue
        """
        if not self.message_queue.qsize():
            single_message = await self.frame.read(text, mask)
            await self.message_queue.put(single_message)
        message = await self.message_queue.get()
        return message or None

    @property
    def get_queue_size(self):
        return self.message_queue.qsize()
Example #2
0
class Converse:
    """ 司职通信交流 """
    def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
        self.reader = reader
        self.writer = writer
        self.message_queue = Queue(maxsize=maxsize)
        self.frame = Frames(self.reader, self.writer)

    async def send(self, message: bytes, fin: bool = True,
                   code: int = OperationCode.binary.value):
        """ 向服务端发送消息 """
        if isinstance(message, str):
            message = message.encode('utf-8')
        if isinstance(message, bytes):
            message = message
        else:
            raise ValueError('Message must be str or bytes,not {mst}'.format(mst=type(message)))
        await self.frame.write(fin=fin, code=code, message=message)

    async def receive(self):
        """ 获取一条消息
        如果消息队列中没有消息则尝试读取,否则直接从消息队列中弹出一条
        """
        if not self.message_queue.qsize():
            single_message = await self.frame.read()
            await self.message_queue.put(single_message)
        message = await self.message_queue.get()
        return message or None

    @property
    def get_queue_size(self):
        return self.message_queue.qsize()
Example #3
0
class Converse:
    """Responsible for communication
    between client and server
    """
    def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
        self.reader = reader
        self.writer = writer
        self.message_queue = Queue(maxsize=maxsize)
        self.frame = Frames(self.reader, self.writer)

    async def send(self, message: bytes, fin: bool = True,
                   code: int = DataFrames.binary.value):
        """Send message to server """
        if isinstance(message, str):
            message = message.encode('utf-8')
        if isinstance(message, bytes):
            message = message
        else:
            raise ValueError('Message must be str or bytes,not {mst}'.format(mst=type(message)))
        await self.frame.write(fin=fin, code=code, message=message)

    async def receive(self):
        """Get a message
        If there is no message in the message queue,
        try to read it or pop up directly from the message queue
        """
        if not self.message_queue.qsize():
            single_message = await self.frame.read()
            await self.message_queue.put(single_message)
        message = await self.message_queue.get()
        return message or None

    @property
    def get_queue_size(self):
        return self.message_queue.qsize()
Example #4
0
class CyclicQueuePool:
    """Cyclic queue pool of connections."""
    def __init__(self, connector, pool_size, connection_cls):
        self.pool_size = pool_size
        self.pool = Queue(pool_size)

        for _ in range(pool_size):
            self.pool.put_nowait(connection_cls(connector))

    async def acquire(self, _urlparsed: ParseResult = None):
        """Acquire connection."""
        return await self.pool.get()

    async def release(self, conn):
        """Release connection."""
        return self.pool.put_nowait(conn)

    def is_all_free(self):
        """Indicates if all pool is free."""
        return self.pool_size == self.pool.qsize()

    def free_conns(self) -> int:
        return self.pool.qsize()

    async def cleanup(self):
        """Get all conn and close them, this method let this pool unusable."""
        for _ in range(self.pool_size):
            conn = self.pool.get()
            conn.close()
Example #5
0
async def _progress_reporter(progress_interval: Union[float, int],
                             queue: asyncio.Queue, total_tests: int) -> None:
    while queue.qsize() > 0:
        done_count = total_tests - queue.qsize()
        LOG.info("{} / {} test suites ran ({}%)".format(
            done_count, total_tests, int((done_count / total_tests) * 100)))
        await asyncio.sleep(progress_interval)

    LOG.debug("progress_reporter finished")
 async def __scan_many(self, targets: asyncio.Queue, scan_result):
     port_tasks = asyncio.Queue(self.queue_size)
     is_able = True
     while targets.qsize():
         while is_able and targets.qsize():
             await port_tasks.put(asyncio.create_task(self.check_port(*await targets.get())))
             targets.task_done()
             if port_tasks.full():
                 is_able = await self.__do_port_tasks(port_tasks, scan_result)
         while port_tasks.qsize():
             is_able = await self.__do_port_tasks(port_tasks, scan_result)
     return scan_result
Example #7
0
def desktop_changer(changes_num: int = 5):

    global leave
    photos_queue = Queue(maxsize=changes_num)
    logging.basicConfig(level=logging.INFO)

    logging.info(
        "I will now go fetch %s random pictures then set them as your background",
        changes_num)

    while leave != "Yes":

        for j in range(changes_num):
            jason_data = requests.get(API_LINK).json()
            urllib.request.urlretrieve(jason_data['urls']['full'],
                                       "BackGround")
            background = Image.open("Background")
            random_photo = storage_path.joinpath(f'CBG{j}.jpg')
            background.save(random_photo)
            photos_queue.put_nowait(random_photo)
            logging.info(
                f"Loading background number %s"
                f" when i reach %s the magic will begin!",
                photos_queue.qsize(), changes_num)

        sleep(0.5)
        logging.info("Starting the Desktop change")
        sleep(1)

        if photos_queue.full():

            while not photos_queue.empty():
                logging.info("Current BG is number %s in line",
                             photos_queue.qsize())
                ctypes.windll.user32.SystemParametersInfoW(
                    20, 0, str(photos_queue.get_nowait()), 0)
                sleep(2)

        logging.info(
            "That's all for now , i shall now delete the backgrounds from your memory"
        )
        for n in range(changes_num):
            os.remove(storage_path.joinpath(f"CBG{n}.jpg"))

        leave = input(
            "Would you like to quit ? Type Yes if so , Enter anything else if you wanna go again!"
        )
    else:
        logging.info("Thanks for using me!")
        sleep(2)
Example #8
0
class AsynHttpProcessorAsynQueue:
    def __init__(self, header: str = None):
        from asyncio import Queue
        self._url_queue = Queue()
        print(139)
        self._header = header
        self._num_of_cor = self._url_queue.qsize()
        self._loop = asyncio.get_event_loop()
        self._tasks = []
        self._exit_flag = False

    async def http_get(self, url):
        response = requests.get(url, headers=self._header)
        self.process_rsp(url, response)

    def process_rsp(self, rsp_content):
        raise NotImplementedError

    async def add_url(self, url):
        await self._url_queue.put(url)

    def _get_exit_flag(self):
        return self._exit_flag

    def set_exit_flag(self):
        self._exit_flag = True

    async def start_forever(self, num_of_cor: int = None):
        if num_of_cor is not None:
            self._num_of_cor = num_of_cor
        i = 0
        while not self._get_exit_flag():
            url = await self._url_queue.get()
            if i < self._num_of_cor and self._url_queue.qsize() > 0:
                i += 1
                self._tasks.append(asyncio.ensure_future(self.http_get(url)))
                print(f'add {url}')
                continue
            #if self._url_queue.qsize() > 0:
            print(174, len(self._tasks))
            if len(self._tasks) > 0:
                self._loop.run_until_complete(asyncio.wait(self._tasks))
            print(
                f'len of url queue: {self._url_queue.qsize()}, len of task:{len(self._tasks)}'
            )
            i = 0
            self._tasks = []
        self._loop.run_until_complete(asyncio.wait(self._tasks))
        self._loop.close()
async def read_input_stdin(queue_input: asyncio.Queue,
                           settings: dict,
                           path_to_file: str = None) -> None:
    """
        посредством модуля aioconsole функция "асинхронно" читает из stdin записи, представляющие собой
        обязательно или ip адрес или запись подсети в ipv4
        из данной записи формируется экзэмпляр NamedTuple - Target, который отправляется в Очередь
        TODO: использовать один модуль - или aioconsole или aiofiles
        :param queue_results:
        :param settings:
        :param path_to_file:
        :return:
        """
    global count_input
    while True:
        try:
            _tmp_input = await ainput()  # read str from stdin
            linein = _tmp_input.strip()
            if any([check_ip(linein), check_network(linein)]):
                targets = create_targets_ssh_protocol(linein, settings)
                if targets:
                    for target in targets:
                        check_queue = True
                        while check_queue:
                            size_queue = queue_input.qsize()
                            if size_queue < queue_limit_targets - 1:
                                count_input += 1  # statistics
                                queue_input.put_nowait(target)
                                check_queue = False
                            else:
                                await asyncio.sleep(sleep_duration_queue_full)
        except EOFError:
            await queue_input.put(b"check for end")
            break
Example #10
0
async def acquire_images(queue: asyncio.Queue, cam: PySpin.Camera):
    """
    A coroutine that captures `NUM_IMAGES` images from `cam` and puts them along
    with the camera serial number as a tuple into the `queue`.
    """
    # Set up camera
    cam_id = cam.GetUniqueID()
    cam.Init()
    cam.BeginAcquisition()
    prev_frame_ID = 0

    # Acquisition loop
    for i in range(NUM_IMAGES):
        img = cam.GetNextImage()
        frame_ID = img.GetFrameID()
        if img.IsIncomplete():
            print('WARNING: img incomplete', frame_ID, 'with status',
                  PySpin.Image_GetImageStatusDescription(img.GetImageStatus()))
            prev_frame_ID = frame_ID
            continue
        if frame_ID != prev_frame_ID + 1:
            print('WARNING: skipped frames', frame_ID)
        prev_frame_ID = frame_ID
        queue.put_nowait((img, cam_id))
        print('Queue size:', queue.qsize())
        print('[{}] Acquired image {}'.format(cam_id, frame_ID))
        await asyncio.sleep(0)  # This is necessary for context switches

    # Clean up
    await queue.join()  # Wait for all images to be saved before EndAcquisition
    cam.EndAcquisition()
    cam.DeInit()
    del cam
Example #11
0
async def serve(parser_server_queue: asyncio.Queue,
                server_crawler_queue: asyncio.Queue):

    # server being consumer in parser-server linkage,
    # consumes url from the queue.
    while True:
        print(
            "SSSSeeeeSSSSSSSSSSSSSSSssseeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeSSSSSSSSSSSSSSSSSSSSSseeeeeeeeeeeeeeeeeeeee"
        )
        url = await parser_server_queue.get()

        # handle_urls method is called to check if the url is already crawled or not.
        # answer = YES ====> Not Crawled
        # answer = NO =====> Already Crawled

        answer = await server.handle_urls(url, server_mongodb)
        parser_server_queue.task_done()
        print(answer)
        if answer == "YES":

            # insert tuple into the server_crawler_queue.
            # first entry is url to crawl, second entry to show if the url is to be
            # deleted from "Not Crawled" collection of compressor database.
            # if No ====> Do not delete, Yes ====> Delete
            # every server task will insert No into the queue as these urls are not
            # present in "Not Crawled" collection and are fresh urls and not the ones
            # giving 404 due to blocking of requests.

            await server_crawler_queue.put((url, "No"))
            print("S E R V E R  S I Z E        ------------------>        ",
                  server_crawler_queue.qsize())
        #return ""
        # server task recursively calls itself.
        await asyncio.sleep(15)
Example #12
0
async def db_worker(name: str, queue: asyncio.Queue, *args, **kwargs) -> None:
    while True:
        try:
            # job will be a query to Gmail API from Gmail interface
            job = await queue.get()
        except queue.Empty:
            print(f'{name} sleeping for 5')
            await asyncio.sleep(5)
            continue

        size = queue.qsize()

        try:
            # executes the db statement by passing args and kwargs
            # eg query=insert_statement or values=values
            await job(*args, **kwargs)
        except Exception as e:
            print(f'Error saving data to DB: {e}')
            break

        print(
            f"{name} finished a job with {queue}. {size} remaining and sleeping 3 seconds... \n\n"
        )
        await asyncio.sleep(3)

    return
Example #13
0
async def acquire_images(queue: asyncio.Queue, cam: PySpin.Camera):
    """
    A coroutine that captures `NUM_IMAGES` images from `cam` and puts them along
    with the camera serial number as a tuple into the `queue`.
    """
    # Set up camera
    cam_id = cam.GetUniqueID()

    # Acquisition loop
    try:
        while True:
            img = cam.GetNextImage()
            frame_ID = img.GetFrameID()
            queue.put_nowait((img, cam_id))
            print('Queue size:', queue.qsize())
            print('[{}] Acquired image {}'.format(cam_id, frame_ID))
            await asyncio.sleep(0)  # This is necessary for context switches
    except KeyboardInterrupt:
        print("Pressed Ctrl-C to terminate while loop")

    # Clean up
    await queue.join()  # Wait for all images to be saved before EndAcquisition
    cam.EndAcquisition()
    cam.DeInit()
    del cam
class Domain(object):
    def __init__(self, domain):
        self.domain = domain
        self.requests = Queue()

        loop = asyncio.get_event_loop()
        self.fetcher = loop.create_task(self.fetch_loop())

    async def fetch_loop(self):
        print('Domain({}): Fetch loop started'.format(self.domain))
        while True:
            addr, future = await self.requests.get()
            # Start an asynchronous sleep of crawl delay in length
            print('Domain({}): Fetching {}'.format(self.domain, addr))
            await asyncio.sleep(1)
            # Wait until the async sleep with crawl delay is complete
            future.set_result(42)

    async def get(self, addr):
        future = loop.create_future()
        await self.requests.put((addr, future))
        print('Q:', self.requests.qsize())
        print('Domain({}): Queued {}'.format(self.domain, addr))
        return future

    async def close(self):
        # Cancelling only starts the process to cancel
        self.fetcher.cancel()
        # We must wait for the task itself to complete
        await self.fetcher
async def consumer(q_args: asyncio.Queue) -> None:
    while q_args.qsize():
        delay = await q_args.get()
        await func(delay)
        print("sleeping for 1 sec")
        await asyncio.sleep(1)
        q_args.task_done()
async def read_input_file(queue_input: asyncio.Queue, settings: dict,
                          path_to_file: str) -> None:
    """
    посредством модуля aiofiles функция "асинхронно" читает из файла записи, представляющие собой
    обязательно или ip адрес или запись подсети в ipv4
    из данной записи формируется экзэмпляр NamedTuple - Target, который отправляется в Очередь
    :param queue_results:
    :param settings:
    :param path_to_file:
    :return:
    """
    global count_input
    async with aiofiles_open(path_to_file, mode='rt') as f:  # read str
        async for line in f:
            linein = line.strip()
            if any([check_ip(linein), check_network(linein)]):
                targets = create_targets_ssh_protocol(linein,
                                                      settings)  # generator
                if targets:
                    for target in targets:
                        check_queue = True
                        while check_queue:
                            size_queue = queue_input.qsize()
                            if size_queue < queue_limit_targets - 1:
                                count_input += 1  # statistics
                                queue_input.put_nowait(target)
                                check_queue = False
                            else:
                                await asyncio.sleep(sleep_duration_queue_full)

    await queue_input.put(b"check for end")
Example #17
0
class ProgressStream(Future):

    def __init__(self):
        super().__init__()
        self._progress = Queue()
        self._progress_task = create_task(self._progress.get())
        self._complete_task = create_task(self)

    def __aiter__(self):
        return self

    async def __anext__(self):

        if self.done():
            raise StopAsyncIteration

        done, pending = await wait({ self._complete_task, self._progress_task }, return_when=FIRST_COMPLETED)

        if self._complete_task in done:
            self._progress_task.cancel()
            self._complete_task.result()  # throws an exception (when necessary)
            raise StopAsyncIteration
        else:
            progress = self._progress_task.result()
            self._progress_task = create_task(self._progress.get())
            return progress

    def write(self, item):
        if self.done():
            raise InvalidStateError
        if self._progress.qsize() > 0:
            self._progress.get_nowait()
        self._progress.put_nowait(item)
Example #18
0
class _MessageWriter:
    def __init__(self, bus):
        self.messages = Queue()
        self.negotiate_unix_fd = bus._negotiate_unix_fd
        self.bus = bus
        self.sock = bus._sock
        self.loop = bus._loop
        self.buf = None
        self.fd = bus._fd
        self.offset = 0
        self.unix_fds = None

    def write_callback(self):
        try:
            while True:
                if self.buf is None:
                    if self.messages.qsize() == 0:
                        # nothing more to write
                        self.loop.remove_writer(self.fd)
                        return
                    buf, unix_fds, fut = self.messages.get_nowait()
                    self.unix_fds = unix_fds
                    self.buf = memoryview(buf)
                    self.offset = 0
                    self.fut = fut

                if self.unix_fds and self.negotiate_unix_fd:
                    ancdata = [(socket.SOL_SOCKET, socket.SCM_RIGHTS,
                                array.array("i", self.unix_fds))]
                    self.offset += self.sock.sendmsg([self.buf[self.offset:]],
                                                     ancdata)
                    self.unix_fds = None
                else:
                    self.offset += self.sock.send(self.buf[self.offset:])

                if self.offset >= len(self.buf):
                    # finished writing
                    self.buf = None
                    if self.fut is not None:
                        self.fut.set_result(None)
                else:
                    # wait for writable
                    return
        except Exception as e:
            if self.fut is not None:
                self.fut.set_exception(e)
            self.bus._finalize(e)

    def buffer_message(self, msg: Message, future=None):
        self.messages.put_nowait(
            (msg._marshall(negotiate_unix_fd=self.negotiate_unix_fd),
             copy(msg.unix_fds), future))

    def schedule_write(self, msg: Message = None, future=None):
        if msg is not None:
            self.buffer_message(msg, future)
        if self.bus.unique_name:
            # don't run the writer until the bus is ready to send messages
            self.loop.add_writer(self.fd, self.write_callback)
Example #19
0
 async def _log_discarded_items_from_the_queue(
         self, connection_id: int, queue: asyncio.Queue) -> None:
     while queue.qsize() > 0:
         item: ResponseQueueItem = await queue.get(
         )  # log all info about not yet retrieved items
         self.logger.info(f"Dropped message: {type(item.message)}, "
                          f"Concent request ID: {item.concent_request_id}, "
                          f"connection ID: {connection_id}")
Example #20
0
def subscriber1(q: asyncio.Queue, *args):
    print('subscriber 1 received event:')
    time.sleep(2)
    print('sub 1: ' + args[0])

    print(q.empty())
    print(q.qsize())
    msg = q.get_nowait()
    print(msg)
Example #21
0
class Buffer:
    def __init__(self, ack_callback, *, loop):
        self._ack_callback = ack_callback
        self._eof = False
        self._unacked = Queue(loop=loop)
        self._acked = deque()
        self._acked_size = 0

    def add(self, data, ack_size):
        self._unacked.put_nowait(UnackedData(data, len(data), ack_size))

    def eof(self):
        self._unacked.put_nowait(UnackedData(b'', 0, 0))
        self._eof = True

    async def read(self, size):
        assert size >= 0, 'Size can not be negative'
        if size == 0:
            return b''

        if not self._eof or not self._unacked.empty():
            while self._acked_size < size:
                data, data_size, ack_size = await self._unacked.get()
                if not ack_size:
                    break
                self._acked.append(AckedData(memoryview(data), data_size))
                self._acked_size += data_size
                self._ack_callback(ack_size)

        if self._eof and self._acked_size == 0:
            return b''

        if self._acked_size < size:
            raise AssertionError('Received less data than expected')

        chunks = []
        chunks_size = 0
        while chunks_size < size:
            next_chunk, next_chunk_size = self._acked[0]
            if chunks_size + next_chunk_size <= size:
                chunks.append(next_chunk)
                chunks_size += next_chunk_size
                self._acked.popleft()
            else:
                offset = size - chunks_size
                chunks.append(next_chunk[:offset])
                chunks_size += offset
                self._acked[0] = (next_chunk[offset:],
                                  next_chunk_size - offset)
        self._acked_size -= size
        assert chunks_size == size
        return b''.join(chunks)

    def unacked_size(self):
        return sum(self._unacked.get_nowait().ack_size
                   for _ in range(self._unacked.qsize()))
Example #22
0
def tree_debug_travl(tree):
	root=tree.root
	print('Done')
	if(root is None):
		return
	q=Queue()
	q.put_nowait(root)
	assert(q.qsize() != 0)
	while(not q.empty()):
		size=q.qsize()
		for i in range(0,size):
			tmp=q.get_nowait()
			if(tmp is None):
				sys.stdout.write('((-,-),(-,-)) ')
			else:
				sys.stdout.write(str(tmp.aabb)+' ')
				q.put_nowait(tmp.right)
				q.put_nowait(tmp.left)
		sys.stdout.write('\n')
Example #23
0
async def work_queue_monitor(work_deque: deque, parse_que: asyncio.Queue,
                             total_block_size, ct_url):
    while True:
        print(
            "{}: Parse Queue Size: {}, Downloaded blocks: {}/{} ({:.4f}%)".
            format(ct_url, parse_que.qsize(),
                   total_block_size - len(work_deque), total_block_size,
                   ((total_block_size - len(work_deque)) / total_block_size) *
                   100.0))
        # print(len(work_deque))
        await asyncio.sleep(2)
Example #24
0
async def process_download_asset(queue: asyncio.Queue, factories: Dict[str,
                                                                       Any],
                                 downloaded_ids: set[str],
                                 logger: DownloadLogger):
    while True:
        url, context, headers = await queue.get()

        raw_id = url.split('/api/assets-gateway/raw/')[1].split('/')[1]
        asset_id = encode_id(raw_id)
        remote_gtw_client = await RemoteClients.get_assets_gateway_client(
            context=context)
        try:
            asset = await remote_gtw_client.get_asset_metadata(
                asset_id=asset_id, headers=headers)
            raw_id = decode_id(asset_id)
            process_id = str(uuid.uuid4())
            await logger.info(
                process_id=process_id,
                title=
                f"Lookup for eventual download of asset {asset['kind']} of id {raw_id}",
                url=url,
                raw_id=raw_id)

            task = factories[asset['kind']](process_id=process_id,
                                            raw_id=raw_id,
                                            asset_id=asset_id,
                                            url=url,
                                            context=context)
            download_id = task.download_id()
            if download_id not in downloaded_ids and not await task.is_local_up_to_date(
            ):
                downloaded_ids.add(download_id)
                await task.create_local_asset()

        except YouWolException as e:
            print(e)
        queue.task_done()
        print("remaining asset to download in the queue:", queue.qsize())
        if queue.qsize() == 0:
            logger.dumps()
Example #25
0
class Converse:
    def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
        self.reader = reader
        self.writer = writer
        self.message_queue = Queue(maxsize=maxsize)  # 接收消息直接从这里get就可以了
        self.frame = Frames(self.reader, self.writer)  # 帧  写/发送消息的操作在这里

    async def send(self, message, fin: bool = True, mask: bool = True):  # 发送消息
        if isinstance(message, str):
            message = message.encode()
        code = DataFrames.text.value  # 0x01
        await self.frame.write(fin=fin, code=code, message=message, mask=mask)

    async def receive(self, text=False, mask=False):  # 接收消息
        if not self.message_queue.qsize():
            single_message = await self.frame.read(text, mask)
            await self.message_queue.put(single_message)
        message = await self.message_queue.get()  # 当异步队列存在的时候,使用get方法获取其中的消息
        return message or None

    @property
    def get_queue_size(self):  # 获取异步队列的大小
        return self.message_queue.qsize()
Example #26
0
 async def queue_to_kafka(self, queue: asyncio.Queue):
     async with self:
         try:
             await self._get_from_queue_and_produce(queue)
         except asyncio.CancelledError:
             self.exiting = True
             if queue.qsize() > 0:
                 self.logger.warning(
                     f"Cancellation> There are {queue.qsize()} items"
                     " in producer queue. Waiting to produce them.")
             # Send termination signal
             await queue.put(None)
             await self._get_from_queue_and_produce(queue)
             raise
Example #27
0
async def compress(compressor_queue: asyncio.Queue):

    while True:
        # checks if any web page has been downloaded. If not, the task waits for its
        # turn in the loop.
        global compressor_doc_id
        if os.path.getsize(path) == 0 or compressor_doc_id > crawler_doc_id:
            await compressor.wait()

        # acting as consumer, fetch tuple from the queue. All the tuple elements
        # are explained in web_page_downloader method below.
        print(
            "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCcccc"
        )
        url, page_status, content_type, extension = await compressor_queue.get(
        )
        print(compressor_queue.qsize())

        if page_status == "200":

            # Web page document id to be used as document identifier for the local directory.
            #global compressor_doc_id
            compressor_doc_id += 1

            # 'read_file' method  reads corresponding web page from the local directory.
            doc = await compressor.read_file(compressor_doc_id, extension)

            # Web page Id to be used as _id in MongoDb.
            global mongodb_doc_id
            mongodb_doc_id += 1

            # for specifications of  'create_mongodb_doc', 'insert_mongodb_doc' methods, refer compressor module.

            await compressor.create_mongodb_doc(doc, content_type, extension,
                                                mongodb_doc_id,
                                                compressor_mongodb)
            print(
                "44444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444"
            )
        elif page_status == "404":
            await compressor.insert_mongodb_doc("Not Crawled", {'_id': url},
                                                compressor_mongodb)
            print(
                "    IIIIIIIIINNNNNSEEERTTTTTTEEEE   DDDDDDD              404")
        elif page_status == "100":
            await compressor.insert_mongodb_doc("Not Text", {
                'url': url,
                'content_type': content_type
            }, compressor_mongodb)
            print("IIIIIIIIIIIIIIINNNMSSSSSSSSerted          Content")
Example #28
0
async def consume(name: int, requester: Requester,
                  queue: asyncio.Queue) -> None:

    while True:
        url_data = await queue.get()

        # Periodical log
        if queue.qsize() % 500 == 0:
            LOGGER.info(f"Queue size: {queue.qsize()}")
            # LOGGER.info(f"Consumer {name} got {url_data} from queue.")

        if queue.lock is False and queue.qsize() < settings.QUEUE_LOW_LIMIT:
            queue.lock = True
            await update_queue(requester.db_connector, queue)
            queue.lock = False

        LOGGER.debug('Parsing: {}'.format(url_data))

        try:
            await requester.url_handler(url_data)
        except Exception:
            LOGGER.error(f'\n Error during parsing: {url_data}', exc_info=True)
        finally:
            queue.task_done()
Example #29
0
async def worker(async_queue: Queue,
                 out_dir: str,
                 gmt_dir: str,
                 collection: pymongo.collection.Collection,
                 processes: int = 4):
    while True:
        changes = list()
        changes.append(await async_queue.get())
        while len(changes) <= processes:
            if async_queue.qsize() > 0:
                changes.append(await async_queue.get())
            else:
                break

        functions = map(lambda x: partial(FUNCTION_MAPPING[x.type], path_to_dataset = x.file), changes)
        await asyncio.gather(*map(lambda fun: fun(out_dir=out_dir, gmt_dir=gmt_dir, collection=collection), functions))
Example #30
0
async def worker(q: asyncio.Queue, se: asyncio.Event):
    print('[WORKER] Starting worker...')

    while True:
        print('[WORKER] Waiting for task...')
        task: str = await q.get()

        complexity = len(task)
        print('[WORKER] Processing task <{}>...'
              'Complexity: {}'.format(task, complexity))
        await asyncio.sleep(complexity)
        print('[WORKER] Task <{}> done...'.format(task))

        if se.is_set() and not q.qsize():
            print('[WORKER] Stopping...'.format(task))
            break
Example #31
0
class Session:
    states = ['new', 'connected', 'disconnected']

    def __init__(self):
        self._init_states()
        self.remote_address = None
        self.remote_port = None
        self.client_id = None
        self.clean_session = None
        self.will_flag = False
        self.will_message = None
        self.will_qos = None
        self.will_retain = None
        self.will_topic = None
        self.keep_alive = 0
        self.publish_retry_delay = 0
        self.broker_uri = None
        self.username = None
        self.password = None
        self.cafile = None
        self.capath = None
        self.cadata = None
        self._packet_id = 0
        self.parent = 0

        # Used to store outgoing ApplicationMessage while publish protocol flows
        self.inflight_out = OrderedDict()

        # Used to store incoming ApplicationMessage while publish protocol flows
        self.inflight_in = OrderedDict()

        # Stores messages retained for this session
        self.retained_messages = Queue()

        # Stores PUBLISH messages ID received in order and ready for application process
        self.delivered_message_queue = Queue()

    def _init_states(self):
        self.transitions = Machine(states=Session.states, initial='new')
        self.transitions.add_transition(trigger='connect', source='new', dest='connected')
        self.transitions.add_transition(trigger='connect', source='disconnected', dest='connected')
        self.transitions.add_transition(trigger='disconnect', source='connected', dest='disconnected')
        self.transitions.add_transition(trigger='disconnect', source='new', dest='disconnected')
        self.transitions.add_transition(trigger='disconnect', source='disconnected', dest='disconnected')

    @property
    def next_packet_id(self):
        self._packet_id += 1
        return self._packet_id

    @property
    def inflight_in_count(self):
        return len(self.inflight_in)

    @property
    def inflight_out_count(self):
        return len(self.inflight_out)

    @property
    def retained_messages_count(self):
        return self.retained_messages.qsize()

    def __repr__(self):
        return type(self).__name__ + '(clientId={0}, state={1})'.format(self.client_id, self.transitions.state)

    def __getstate__(self):
        state = self.__dict__.copy()
        # Remove the unpicklable entries.
        # del state['transitions']
        del state['retained_messages']
        del state['delivered_message_queue']
        return state

    def __setstate(self, state):
        self.__dict__.update(state)
        self.retained_messages = Queue()
        self.delivered_message_queue = Queue()

    def __eq__(self, other):
        return self.client_id == other.client_id