class Converse:
"""Responsible for communication
between client and server
"""
def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
self.reader = reader
self.writer = writer
self.message_queue = Queue(maxsize=maxsize)
self.frame = Frames(self.reader, self.writer)
async def send(self, message, fin: bool = True, mask: bool = True):
"""Send message to server """
if isinstance(message, str):
message = message.encode()
code = DataFrames.text.value
await self.frame.write(fin=fin, code=code, message=message, mask=mask)
async def receive(self, text=False, mask=False):
"""Get a message
If there is no message in the message queue,
try to read it or pop up directly from the message queue
"""
if not self.message_queue.qsize():
single_message = await self.frame.read(text, mask)
await self.message_queue.put(single_message)
message = await self.message_queue.get()
return message or None
@property
def get_queue_size(self):
return self.message_queue.qsize()
class Converse:
""" 司职通信交流 """
def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
self.reader = reader
self.writer = writer
self.message_queue = Queue(maxsize=maxsize)
self.frame = Frames(self.reader, self.writer)
async def send(self, message: bytes, fin: bool = True,
code: int = OperationCode.binary.value):
""" 向服务端发送消息 """
if isinstance(message, str):
message = message.encode('utf-8')
if isinstance(message, bytes):
message = message
else:
raise ValueError('Message must be str or bytes,not {mst}'.format(mst=type(message)))
await self.frame.write(fin=fin, code=code, message=message)
async def receive(self):
""" 获取一条消息
如果消息队列中没有消息则尝试读取，否则直接从消息队列中弹出一条
"""
if not self.message_queue.qsize():
single_message = await self.frame.read()
await self.message_queue.put(single_message)
message = await self.message_queue.get()
return message or None
@property
def get_queue_size(self):
return self.message_queue.qsize()
class Converse:
"""Responsible for communication
between client and server
"""
def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
self.reader = reader
self.writer = writer
self.message_queue = Queue(maxsize=maxsize)
self.frame = Frames(self.reader, self.writer)
async def send(self, message: bytes, fin: bool = True,
code: int = DataFrames.binary.value):
"""Send message to server """
if isinstance(message, str):
message = message.encode('utf-8')
if isinstance(message, bytes):
message = message
else:
raise ValueError('Message must be str or bytes,not {mst}'.format(mst=type(message)))
await self.frame.write(fin=fin, code=code, message=message)
async def receive(self):
"""Get a message
If there is no message in the message queue,
try to read it or pop up directly from the message queue
"""
if not self.message_queue.qsize():
single_message = await self.frame.read()
await self.message_queue.put(single_message)
message = await self.message_queue.get()
return message or None
@property
def get_queue_size(self):
return self.message_queue.qsize()
class CyclicQueuePool:
"""Cyclic queue pool of connections."""
def __init__(self, connector, pool_size, connection_cls):
self.pool_size = pool_size
self.pool = Queue(pool_size)
for _ in range(pool_size):
self.pool.put_nowait(connection_cls(connector))
async def acquire(self, _urlparsed: ParseResult = None):
"""Acquire connection."""
return await self.pool.get()
async def release(self, conn):
"""Release connection."""
return self.pool.put_nowait(conn)
def is_all_free(self):
"""Indicates if all pool is free."""
return self.pool_size == self.pool.qsize()
def free_conns(self) -> int:
return self.pool.qsize()
async def cleanup(self):
"""Get all conn and close them, this method let this pool unusable."""
for _ in range(self.pool_size):
conn = self.pool.get()
conn.close()
async def _progress_reporter(progress_interval: Union[float, int],
queue: asyncio.Queue, total_tests: int) -> None:
while queue.qsize() > 0:
done_count = total_tests - queue.qsize()
LOG.info("{} / {} test suites ran ({}%)".format(
done_count, total_tests, int((done_count / total_tests) * 100)))
await asyncio.sleep(progress_interval)
LOG.debug("progress_reporter finished")
async def __scan_many(self, targets: asyncio.Queue, scan_result):
port_tasks = asyncio.Queue(self.queue_size)
is_able = True
while targets.qsize():
while is_able and targets.qsize():
await port_tasks.put(asyncio.create_task(self.check_port(*await targets.get())))
targets.task_done()
if port_tasks.full():
is_able = await self.__do_port_tasks(port_tasks, scan_result)
while port_tasks.qsize():
is_able = await self.__do_port_tasks(port_tasks, scan_result)
return scan_result
def desktop_changer(changes_num: int = 5):
global leave
photos_queue = Queue(maxsize=changes_num)
logging.basicConfig(level=logging.INFO)
logging.info(
"I will now go fetch %s random pictures then set them as your background",
changes_num)
while leave != "Yes":
for j in range(changes_num):
jason_data = requests.get(API_LINK).json()
urllib.request.urlretrieve(jason_data['urls']['full'],
"BackGround")
background = Image.open("Background")
random_photo = storage_path.joinpath(f'CBG{j}.jpg')
background.save(random_photo)
photos_queue.put_nowait(random_photo)
logging.info(
f"Loading background number %s"
f" when i reach %s the magic will begin!",
photos_queue.qsize(), changes_num)
sleep(0.5)
logging.info("Starting the Desktop change")
sleep(1)
if photos_queue.full():
while not photos_queue.empty():
logging.info("Current BG is number %s in line",
photos_queue.qsize())
ctypes.windll.user32.SystemParametersInfoW(
20, 0, str(photos_queue.get_nowait()), 0)
sleep(2)
logging.info(
"That's all for now , i shall now delete the backgrounds from your memory"
)
for n in range(changes_num):
os.remove(storage_path.joinpath(f"CBG{n}.jpg"))
leave = input(
"Would you like to quit ? Type Yes if so , Enter anything else if you wanna go again!"
)
else:
logging.info("Thanks for using me!")
sleep(2)
class AsynHttpProcessorAsynQueue:
def __init__(self, header: str = None):
from asyncio import Queue
self._url_queue = Queue()
print(139)
self._header = header
self._num_of_cor = self._url_queue.qsize()
self._loop = asyncio.get_event_loop()
self._tasks = []
self._exit_flag = False
async def http_get(self, url):
response = requests.get(url, headers=self._header)
self.process_rsp(url, response)
def process_rsp(self, rsp_content):
raise NotImplementedError
async def add_url(self, url):
await self._url_queue.put(url)
def _get_exit_flag(self):
return self._exit_flag
def set_exit_flag(self):
self._exit_flag = True
async def start_forever(self, num_of_cor: int = None):
if num_of_cor is not None:
self._num_of_cor = num_of_cor
i = 0
while not self._get_exit_flag():
url = await self._url_queue.get()
if i < self._num_of_cor and self._url_queue.qsize() > 0:
i += 1
self._tasks.append(asyncio.ensure_future(self.http_get(url)))
print(f'add {url}')
continue
#if self._url_queue.qsize() > 0:
print(174, len(self._tasks))
if len(self._tasks) > 0:
self._loop.run_until_complete(asyncio.wait(self._tasks))
print(
f'len of url queue: {self._url_queue.qsize()}, len of task:{len(self._tasks)}'
)
i = 0
self._tasks = []
self._loop.run_until_complete(asyncio.wait(self._tasks))
self._loop.close()
async def read_input_stdin(queue_input: asyncio.Queue,
settings: dict,
path_to_file: str = None) -> None:
"""
посредством модуля aioconsole функция "асинхронно" читает из stdin записи, представляющие собой
обязательно или ip адрес или запись подсети в ipv4
из данной записи формируется экзэмпляр NamedTuple - Target, который отправляется в Очередь
TODO: использовать один модуль - или aioconsole или aiofiles
:param queue_results:
:param settings:
:param path_to_file:
:return:
"""
global count_input
while True:
try:
_tmp_input = await ainput() # read str from stdin
linein = _tmp_input.strip()
if any([check_ip(linein), check_network(linein)]):
targets = create_targets_ssh_protocol(linein, settings)
if targets:
for target in targets:
check_queue = True
while check_queue:
size_queue = queue_input.qsize()
if size_queue < queue_limit_targets - 1:
count_input += 1 # statistics
queue_input.put_nowait(target)
check_queue = False
else:
await asyncio.sleep(sleep_duration_queue_full)
except EOFError:
await queue_input.put(b"check for end")
break
async def acquire_images(queue: asyncio.Queue, cam: PySpin.Camera):
"""
A coroutine that captures `NUM_IMAGES` images from `cam` and puts them along
with the camera serial number as a tuple into the `queue`.
"""
# Set up camera
cam_id = cam.GetUniqueID()
cam.Init()
cam.BeginAcquisition()
prev_frame_ID = 0
# Acquisition loop
for i in range(NUM_IMAGES):
img = cam.GetNextImage()
frame_ID = img.GetFrameID()
if img.IsIncomplete():
print('WARNING: img incomplete', frame_ID, 'with status',
PySpin.Image_GetImageStatusDescription(img.GetImageStatus()))
prev_frame_ID = frame_ID
continue
if frame_ID != prev_frame_ID + 1:
print('WARNING: skipped frames', frame_ID)
prev_frame_ID = frame_ID
queue.put_nowait((img, cam_id))
print('Queue size:', queue.qsize())
print('[{}] Acquired image {}'.format(cam_id, frame_ID))
await asyncio.sleep(0) # This is necessary for context switches
# Clean up
await queue.join() # Wait for all images to be saved before EndAcquisition
cam.EndAcquisition()
cam.DeInit()
del cam
async def serve(parser_server_queue: asyncio.Queue,
server_crawler_queue: asyncio.Queue):
# server being consumer in parser-server linkage,
# consumes url from the queue.
while True:
print(
"SSSSeeeeSSSSSSSSSSSSSSSssseeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeSSSSSSSSSSSSSSSSSSSSSseeeeeeeeeeeeeeeeeeeee"
)
url = await parser_server_queue.get()
# handle_urls method is called to check if the url is already crawled or not.
# answer = YES ====> Not Crawled
# answer = NO =====> Already Crawled
answer = await server.handle_urls(url, server_mongodb)
parser_server_queue.task_done()
print(answer)
if answer == "YES":
# insert tuple into the server_crawler_queue.
# first entry is url to crawl, second entry to show if the url is to be
# deleted from "Not Crawled" collection of compressor database.
# if No ====> Do not delete, Yes ====> Delete
# every server task will insert No into the queue as these urls are not
# present in "Not Crawled" collection and are fresh urls and not the ones
# giving 404 due to blocking of requests.
await server_crawler_queue.put((url, "No"))
print("S E R V E R S I Z E ------------------> ",
server_crawler_queue.qsize())
#return ""
# server task recursively calls itself.
await asyncio.sleep(15)
async def db_worker(name: str, queue: asyncio.Queue, *args, **kwargs) -> None:
while True:
try:
# job will be a query to Gmail API from Gmail interface
job = await queue.get()
except queue.Empty:
print(f'{name} sleeping for 5')
await asyncio.sleep(5)
continue
size = queue.qsize()
try:
# executes the db statement by passing args and kwargs
# eg query=insert_statement or values=values
await job(*args, **kwargs)
except Exception as e:
print(f'Error saving data to DB: {e}')
break
print(
f"{name} finished a job with {queue}. {size} remaining and sleeping 3 seconds... \n\n"
)
await asyncio.sleep(3)
return
async def acquire_images(queue: asyncio.Queue, cam: PySpin.Camera):
"""
A coroutine that captures `NUM_IMAGES` images from `cam` and puts them along
with the camera serial number as a tuple into the `queue`.
"""
# Set up camera
cam_id = cam.GetUniqueID()
# Acquisition loop
try:
while True:
img = cam.GetNextImage()
frame_ID = img.GetFrameID()
queue.put_nowait((img, cam_id))
print('Queue size:', queue.qsize())
print('[{}] Acquired image {}'.format(cam_id, frame_ID))
await asyncio.sleep(0) # This is necessary for context switches
except KeyboardInterrupt:
print("Pressed Ctrl-C to terminate while loop")
# Clean up
await queue.join() # Wait for all images to be saved before EndAcquisition
cam.EndAcquisition()
cam.DeInit()
del cam
class Domain(object):
def __init__(self, domain):
self.domain = domain
self.requests = Queue()
loop = asyncio.get_event_loop()
self.fetcher = loop.create_task(self.fetch_loop())
async def fetch_loop(self):
print('Domain({}): Fetch loop started'.format(self.domain))
while True:
addr, future = await self.requests.get()
# Start an asynchronous sleep of crawl delay in length
print('Domain({}): Fetching {}'.format(self.domain, addr))
await asyncio.sleep(1)
# Wait until the async sleep with crawl delay is complete
future.set_result(42)
async def get(self, addr):
future = loop.create_future()
await self.requests.put((addr, future))
print('Q:', self.requests.qsize())
print('Domain({}): Queued {}'.format(self.domain, addr))
return future
async def close(self):
# Cancelling only starts the process to cancel
self.fetcher.cancel()
# We must wait for the task itself to complete
await self.fetcher
async def consumer(q_args: asyncio.Queue) -> None:
while q_args.qsize():
delay = await q_args.get()
await func(delay)
print("sleeping for 1 sec")
await asyncio.sleep(1)
q_args.task_done()
async def read_input_file(queue_input: asyncio.Queue, settings: dict,
path_to_file: str) -> None:
"""
посредством модуля aiofiles функция "асинхронно" читает из файла записи, представляющие собой
обязательно или ip адрес или запись подсети в ipv4
из данной записи формируется экзэмпляр NamedTuple - Target, который отправляется в Очередь
:param queue_results:
:param settings:
:param path_to_file:
:return:
"""
global count_input
async with aiofiles_open(path_to_file, mode='rt') as f: # read str
async for line in f:
linein = line.strip()
if any([check_ip(linein), check_network(linein)]):
targets = create_targets_ssh_protocol(linein,
settings) # generator
if targets:
for target in targets:
check_queue = True
while check_queue:
size_queue = queue_input.qsize()
if size_queue < queue_limit_targets - 1:
count_input += 1 # statistics
queue_input.put_nowait(target)
check_queue = False
else:
await asyncio.sleep(sleep_duration_queue_full)
await queue_input.put(b"check for end")
class ProgressStream(Future):
def __init__(self):
super().__init__()
self._progress = Queue()
self._progress_task = create_task(self._progress.get())
self._complete_task = create_task(self)
def __aiter__(self):
return self
async def __anext__(self):
if self.done():
raise StopAsyncIteration
done, pending = await wait({ self._complete_task, self._progress_task }, return_when=FIRST_COMPLETED)
if self._complete_task in done:
self._progress_task.cancel()
self._complete_task.result() # throws an exception (when necessary)
raise StopAsyncIteration
else:
progress = self._progress_task.result()
self._progress_task = create_task(self._progress.get())
return progress
def write(self, item):
if self.done():
raise InvalidStateError
if self._progress.qsize() > 0:
self._progress.get_nowait()
self._progress.put_nowait(item)
class _MessageWriter:
def __init__(self, bus):
self.messages = Queue()
self.negotiate_unix_fd = bus._negotiate_unix_fd
self.bus = bus
self.sock = bus._sock
self.loop = bus._loop
self.buf = None
self.fd = bus._fd
self.offset = 0
self.unix_fds = None
def write_callback(self):
try:
while True:
if self.buf is None:
if self.messages.qsize() == 0:
# nothing more to write
self.loop.remove_writer(self.fd)
return
buf, unix_fds, fut = self.messages.get_nowait()
self.unix_fds = unix_fds
self.buf = memoryview(buf)
self.offset = 0
self.fut = fut
if self.unix_fds and self.negotiate_unix_fd:
ancdata = [(socket.SOL_SOCKET, socket.SCM_RIGHTS,
array.array("i", self.unix_fds))]
self.offset += self.sock.sendmsg([self.buf[self.offset:]],
ancdata)
self.unix_fds = None
else:
self.offset += self.sock.send(self.buf[self.offset:])
if self.offset >= len(self.buf):
# finished writing
self.buf = None
if self.fut is not None:
self.fut.set_result(None)
else:
# wait for writable
return
except Exception as e:
if self.fut is not None:
self.fut.set_exception(e)
self.bus._finalize(e)
def buffer_message(self, msg: Message, future=None):
self.messages.put_nowait(
(msg._marshall(negotiate_unix_fd=self.negotiate_unix_fd),
copy(msg.unix_fds), future))
def schedule_write(self, msg: Message = None, future=None):
if msg is not None:
self.buffer_message(msg, future)
if self.bus.unique_name:
# don't run the writer until the bus is ready to send messages
self.loop.add_writer(self.fd, self.write_callback)
async def _log_discarded_items_from_the_queue(
self, connection_id: int, queue: asyncio.Queue) -> None:
while queue.qsize() > 0:
item: ResponseQueueItem = await queue.get(
) # log all info about not yet retrieved items
self.logger.info(f"Dropped message: {type(item.message)}, "
f"Concent request ID: {item.concent_request_id}, "
f"connection ID: {connection_id}")
def subscriber1(q: asyncio.Queue, *args):
print('subscriber 1 received event:')
time.sleep(2)
print('sub 1: ' + args[0])
print(q.empty())
print(q.qsize())
msg = q.get_nowait()
print(msg)
class Buffer:
def __init__(self, ack_callback, *, loop):
self._ack_callback = ack_callback
self._eof = False
self._unacked = Queue(loop=loop)
self._acked = deque()
self._acked_size = 0
def add(self, data, ack_size):
self._unacked.put_nowait(UnackedData(data, len(data), ack_size))
def eof(self):
self._unacked.put_nowait(UnackedData(b'', 0, 0))
self._eof = True
async def read(self, size):
assert size >= 0, 'Size can not be negative'
if size == 0:
return b''
if not self._eof or not self._unacked.empty():
while self._acked_size < size:
data, data_size, ack_size = await self._unacked.get()
if not ack_size:
break
self._acked.append(AckedData(memoryview(data), data_size))
self._acked_size += data_size
self._ack_callback(ack_size)
if self._eof and self._acked_size == 0:
return b''
if self._acked_size < size:
raise AssertionError('Received less data than expected')
chunks = []
chunks_size = 0
while chunks_size < size:
next_chunk, next_chunk_size = self._acked[0]
if chunks_size + next_chunk_size <= size:
chunks.append(next_chunk)
chunks_size += next_chunk_size
self._acked.popleft()
else:
offset = size - chunks_size
chunks.append(next_chunk[:offset])
chunks_size += offset
self._acked[0] = (next_chunk[offset:],
next_chunk_size - offset)
self._acked_size -= size
assert chunks_size == size
return b''.join(chunks)
def unacked_size(self):
return sum(self._unacked.get_nowait().ack_size
for _ in range(self._unacked.qsize()))
def tree_debug_travl(tree):
root=tree.root
print('Done')
if(root is None):
return
q=Queue()
q.put_nowait(root)
assert(q.qsize() != 0)
while(not q.empty()):
size=q.qsize()
for i in range(0,size):
tmp=q.get_nowait()
if(tmp is None):
sys.stdout.write('((-,-),(-,-)) ')
else:
sys.stdout.write(str(tmp.aabb)+' ')
q.put_nowait(tmp.right)
q.put_nowait(tmp.left)
sys.stdout.write('\n')
async def work_queue_monitor(work_deque: deque, parse_que: asyncio.Queue,
total_block_size, ct_url):
while True:
print(
"{}: Parse Queue Size: {}, Downloaded blocks: {}/{} ({:.4f}%)".
format(ct_url, parse_que.qsize(),
total_block_size - len(work_deque), total_block_size,
((total_block_size - len(work_deque)) / total_block_size) *
100.0))
# print(len(work_deque))
await asyncio.sleep(2)
async def process_download_asset(queue: asyncio.Queue, factories: Dict[str,
Any],
downloaded_ids: set[str],
logger: DownloadLogger):
while True:
url, context, headers = await queue.get()
raw_id = url.split('/api/assets-gateway/raw/')[1].split('/')[1]
asset_id = encode_id(raw_id)
remote_gtw_client = await RemoteClients.get_assets_gateway_client(
context=context)
try:
asset = await remote_gtw_client.get_asset_metadata(
asset_id=asset_id, headers=headers)
raw_id = decode_id(asset_id)
process_id = str(uuid.uuid4())
await logger.info(
process_id=process_id,
title=
f"Lookup for eventual download of asset {asset['kind']} of id {raw_id}",
url=url,
raw_id=raw_id)
task = factories[asset['kind']](process_id=process_id,
raw_id=raw_id,
asset_id=asset_id,
url=url,
context=context)
download_id = task.download_id()
if download_id not in downloaded_ids and not await task.is_local_up_to_date(
):
downloaded_ids.add(download_id)
await task.create_local_asset()
except YouWolException as e:
print(e)
queue.task_done()
print("remaining asset to download in the queue:", queue.qsize())
if queue.qsize() == 0:
logger.dumps()
class Converse:
def __init__(self, reader: object, writer: object, maxsize: int = 2**16):
self.reader = reader
self.writer = writer
self.message_queue = Queue(maxsize=maxsize) # 接收消息直接从这里get就可以了
self.frame = Frames(self.reader, self.writer) # 帧 写/发送消息的操作在这里
async def send(self, message, fin: bool = True, mask: bool = True): # 发送消息
if isinstance(message, str):
message = message.encode()
code = DataFrames.text.value # 0x01
await self.frame.write(fin=fin, code=code, message=message, mask=mask)
async def receive(self, text=False, mask=False): # 接收消息
if not self.message_queue.qsize():
single_message = await self.frame.read(text, mask)
await self.message_queue.put(single_message)
message = await self.message_queue.get() # 当异步队列存在的时候，使用get方法获取其中的消息
return message or None
@property
def get_queue_size(self): # 获取异步队列的大小
return self.message_queue.qsize()
async def queue_to_kafka(self, queue: asyncio.Queue):
async with self:
try:
await self._get_from_queue_and_produce(queue)
except asyncio.CancelledError:
self.exiting = True
if queue.qsize() > 0:
self.logger.warning(
f"Cancellation> There are {queue.qsize()} items"
" in producer queue. Waiting to produce them.")
# Send termination signal
await queue.put(None)
await self._get_from_queue_and_produce(queue)
raise
async def compress(compressor_queue: asyncio.Queue):
while True:
# checks if any web page has been downloaded. If not, the task waits for its
# turn in the loop.
global compressor_doc_id
if os.path.getsize(path) == 0 or compressor_doc_id > crawler_doc_id:
await compressor.wait()
# acting as consumer, fetch tuple from the queue. All the tuple elements
# are explained in web_page_downloader method below.
print(
"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCcccc"
)
url, page_status, content_type, extension = await compressor_queue.get(
)
print(compressor_queue.qsize())
if page_status == "200":
# Web page document id to be used as document identifier for the local directory.
#global compressor_doc_id
compressor_doc_id += 1
# 'read_file' method reads corresponding web page from the local directory.
doc = await compressor.read_file(compressor_doc_id, extension)
# Web page Id to be used as _id in MongoDb.
global mongodb_doc_id
mongodb_doc_id += 1
# for specifications of 'create_mongodb_doc', 'insert_mongodb_doc' methods, refer compressor module.
await compressor.create_mongodb_doc(doc, content_type, extension,
mongodb_doc_id,
compressor_mongodb)
print(
"44444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444"
)
elif page_status == "404":
await compressor.insert_mongodb_doc("Not Crawled", {'_id': url},
compressor_mongodb)
print(
" IIIIIIIIINNNNNSEEERTTTTTTEEEE DDDDDDD 404")
elif page_status == "100":
await compressor.insert_mongodb_doc("Not Text", {
'url': url,
'content_type': content_type
}, compressor_mongodb)
print("IIIIIIIIIIIIIIINNNMSSSSSSSSerted Content")
async def consume(name: int, requester: Requester,
queue: asyncio.Queue) -> None:
while True:
url_data = await queue.get()
# Periodical log
if queue.qsize() % 500 == 0:
LOGGER.info(f"Queue size: {queue.qsize()}")
# LOGGER.info(f"Consumer {name} got {url_data} from queue.")
if queue.lock is False and queue.qsize() < settings.QUEUE_LOW_LIMIT:
queue.lock = True
await update_queue(requester.db_connector, queue)
queue.lock = False
LOGGER.debug('Parsing: {}'.format(url_data))
try:
await requester.url_handler(url_data)
except Exception:
LOGGER.error(f'\n Error during parsing: {url_data}', exc_info=True)
finally:
queue.task_done()
async def worker(async_queue: Queue,
out_dir: str,
gmt_dir: str,
collection: pymongo.collection.Collection,
processes: int = 4):
while True:
changes = list()
changes.append(await async_queue.get())
while len(changes) <= processes:
if async_queue.qsize() > 0:
changes.append(await async_queue.get())
else:
break
functions = map(lambda x: partial(FUNCTION_MAPPING[x.type], path_to_dataset = x.file), changes)
await asyncio.gather(*map(lambda fun: fun(out_dir=out_dir, gmt_dir=gmt_dir, collection=collection), functions))
async def worker(q: asyncio.Queue, se: asyncio.Event):
print('[WORKER] Starting worker...')
while True:
print('[WORKER] Waiting for task...')
task: str = await q.get()
complexity = len(task)
print('[WORKER] Processing task <{}>...'
'Complexity: {}'.format(task, complexity))
await asyncio.sleep(complexity)
print('[WORKER] Task <{}> done...'.format(task))
if se.is_set() and not q.qsize():
print('[WORKER] Stopping...'.format(task))
break
class Session:
states = ['new', 'connected', 'disconnected']
def __init__(self):
self._init_states()
self.remote_address = None
self.remote_port = None
self.client_id = None
self.clean_session = None
self.will_flag = False
self.will_message = None
self.will_qos = None
self.will_retain = None
self.will_topic = None
self.keep_alive = 0
self.publish_retry_delay = 0
self.broker_uri = None
self.username = None
self.password = None
self.cafile = None
self.capath = None
self.cadata = None
self._packet_id = 0
self.parent = 0
# Used to store outgoing ApplicationMessage while publish protocol flows
self.inflight_out = OrderedDict()
# Used to store incoming ApplicationMessage while publish protocol flows
self.inflight_in = OrderedDict()
# Stores messages retained for this session
self.retained_messages = Queue()
# Stores PUBLISH messages ID received in order and ready for application process
self.delivered_message_queue = Queue()
def _init_states(self):
self.transitions = Machine(states=Session.states, initial='new')
self.transitions.add_transition(trigger='connect', source='new', dest='connected')
self.transitions.add_transition(trigger='connect', source='disconnected', dest='connected')
self.transitions.add_transition(trigger='disconnect', source='connected', dest='disconnected')
self.transitions.add_transition(trigger='disconnect', source='new', dest='disconnected')
self.transitions.add_transition(trigger='disconnect', source='disconnected', dest='disconnected')
@property
def next_packet_id(self):
self._packet_id += 1
return self._packet_id
@property
def inflight_in_count(self):
return len(self.inflight_in)
@property
def inflight_out_count(self):
return len(self.inflight_out)
@property
def retained_messages_count(self):
return self.retained_messages.qsize()
def __repr__(self):
return type(self).__name__ + '(clientId={0}, state={1})'.format(self.client_id, self.transitions.state)
def __getstate__(self):
state = self.__dict__.copy()
# Remove the unpicklable entries.
# del state['transitions']
del state['retained_messages']
del state['delivered_message_queue']
return state
def __setstate(self, state):
self.__dict__.update(state)
self.retained_messages = Queue()
self.delivered_message_queue = Queue()
def __eq__(self, other):
return self.client_id == other.client_id
