async def work_with_queue_result(queue_out: asyncio.Queue, filename,
mode_write) -> None:
"""
:param queue_out:
:param filename:
:param mode_write:
:return:
"""
if mode_write == 'a':
method_write_result = write_to_file
else:
method_write_result = write_to_stdout
async with aiofiles_open(filename, mode=mode_write) as file_with_results:
while True:
line = await queue_out.get()
if line == b"check for end":
break
if line:
await method_write_result(file_with_results, line)
await asyncio.sleep(0.5)
# region dev
if args.statistics:
stop_time = datetime.datetime.now()
_delta_time = stop_time - start_time
duration_time_sec = _delta_time.total_seconds()
statistics = {
'duration': duration_time_sec,
'valid targets': count_input,
'success': count_good,
'fails': count_error
}
async with aiofiles_open('/dev/stdout', mode='wb') as stats:
await stats.write(ujson_dumps(statistics).encode('utf-8') + b'\n')
async def _write_tag(
self, name: str, body: str, tag_id: Optional[str] = None
) -> Tag:
if tag_id is None:
while True:
tag_id = "".join(choice(ascii_letters) for _ in range(6))
if tag_id in self._tails:
await sleep(0)
continue
self._tails[tag_id] = name
unserialized = {"ident": tag_id, "body": body}
async with aiofiles_open(self._repository_path / name, "w") as entry:
await entry.write(json_dumps(unserialized))
async with aiofiles_open(self._repository_path / "head.jsonl", "a") as header:
await header.write(json_dumps({"aliased": name, "aliases": []}) + "\n")
self._head[tag_id] = name
# TODO: Commit and push changes.
return Tag(**unserialized)
async def json2tsv(cls, ori_path, path):
cls.logger.debug('Start to decode SMR JSON')
async with aiofiles_open(ori_path) as inFile:
try:
data = json.loads(await inFile.read())
except Exception as e:
cls.logger.error(f"Error in '{ori_path}'")
raise e
res = Dict2Tabular.pyexcel_io(cls.yieldSMR(data))
if res is not None:
if isinstance(res, Generator):
count = 0
for r in res:
if r is not None:
await pipe_out(df=r,
path=path,
format='tsv',
mode='a' if count else 'w')
count += 1
if not count:
cls.logger.debug(
f"Without Expected Data (swissmodel/repository/uniprot/): {data}"
)
return None
else:
await pipe_out(df=res, path=path, format='tsv', mode='w')
cls.logger.debug(f"Decoded file in '{path}'")
return path
else:
cls.logger.warning(
f"Without Expected Data (swissmodel/repository/uniprot/): {data}"
)
return None
async def main():
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
arguments = parse_args()
target_settings, config = parse_settings(arguments)
queue_input = asyncio.Queue()
queue_tasks = asyncio.Queue()
queue_prints = asyncio.Queue()
task_semaphore = asyncio.Semaphore(config.senders)
statistics = Stats() if config.statistics else None
async with aiofiles_open(config.output_file,
mode=config.write_mode) as file_with_results:
writer_coroutine = get_async_writer(config)
target_worker = TargetWorker(statistics, task_semaphore, queue_prints,
config.show_only_success)
input_reader: TargetReader = create_io_reader(statistics, queue_input,
target_settings, config)
task_producer = TaskProducer(statistics, queue_input, queue_tasks,
target_worker)
executor = Executor(statistics, queue_tasks, queue_prints)
printer = OutputPrinter(statistics, queue_prints, file_with_results,
writer_coroutine)
await asyncio.wait([
worker.run()
for worker in [input_reader, task_producer, executor, printer]
])
async def read_input_file(queue_input: asyncio.Queue, settings: dict,
path_to_file: str) -> None:
"""
посредством модуля aiofiles функция "асинхронно" читает из файла записи, представляющие собой
обязательно или ip адрес или запись подсети в ipv4
из данной записи формируется экзэмпляр NamedTuple - Target, который отправляется в Очередь
:param queue_results:
:param settings:
:param path_to_file:
:return:
"""
global count_input
async with aiofiles_open(path_to_file, mode='rt') as f: # read str
async for line in f:
linein = line.strip()
if any([check_ip(linein), check_network(linein)]):
targets = create_targets_ssh_protocol(linein,
settings) # generator
if targets:
for target in targets:
check_queue = True
while check_queue:
size_queue = queue_input.qsize()
if size_queue < queue_limit_targets - 1:
count_input += 1 # statistics
queue_input.put_nowait(target)
check_queue = False
else:
await asyncio.sleep(sleep_duration_queue_full)
await queue_input.put(b"check for end")
async def main():
arguments = parse_args()
target_settings, config = parse_settings(arguments)
queue_input = asyncio.Queue()
queue_tasks = asyncio.Queue()
queue_prints = asyncio.Queue()
#
task_semaphore = asyncio.Semaphore(config.senders)
statistics = Stats() if config.statistics else None
#
async with aiofiles_open(config.output_file,
mode=config.write_mode) as file_with_results:
writer_coroutine = get_async_writer(config)
target_worker = TargetWorker(statistics, task_semaphore, queue_prints,
config)
input_reader: TargetReader = create_io_reader(statistics, queue_input,
target_settings, config)
task_producer = TaskProducer(statistics, queue_input, queue_tasks,
target_worker)
executor = Executor(statistics, queue_tasks, queue_prints)
printer = OutputPrinter(config.output_file, statistics, queue_prints,
file_with_results, writer_coroutine)
running_tasks = [
asyncio.create_task(worker.run())
for worker in [input_reader, task_producer, executor, printer]
]
await asyncio.wait(running_tasks)
async def run(self):
async with aiofiles_open(self.file_path, mode='rt') as f:
async for line in f:
linein = line.strip()
await self.producer.send(linein)
await self.producer.send_stop()
async def pipe_out(df, path, **kwargs):
if not isinstance(df, Sheet):
raise TypeError(f"Invalid Object for pipe_out(): {type(df)}")
if len(df) == 0:
raise ValueError("Zero record!")
path = Path(path)
mode = kwargs.get('mode', 'a')
clear_headers: bool = path.exists() and mode.startswith('a')
var_format = kwargs.get('format', 'tsv').lower()
async with aiofiles_open(path, mode) as file_io:
"""
if isinstance(df, DataFrame):
sorted_col = sorted(df.columns)
if clear_headers:
headers = None
else:
headers = sorted_col
dataset = Dataset(headers=headers)
dataset.extend(df[sorted_col].to_records(index=False))
to_write = dataset.export(var_format, lineterminator='\n')
elif isinstance(df, Dataset):
if clear_headers:
df.headers = None
to_write = df.export(var_format, lineterminator='\n')
"""
df = df.project(sorted(df.colnames))
if clear_headers:
df.colnames = []
to_write = getattr(df, f"get_{var_format}")(lineterminator='\n')
await file_io.write(to_write)
async def json2tsv(cls, suffix: str, ori_path: Union[str, Path],
path: Union[str, Path]):
cls.logger.debug('Start to decode')
async with aiofiles_open(ori_path) as handle:
try:
data = json.loads(await handle.read())
except Exception as e:
cls.logger.error(f"Error in '{ori_path}'")
raise e
res = Dict2Tabular.pyexcel_io(traverseSuffixes(suffix, data))
if res is not None:
if isinstance(res, Generator):
count = 0
for r in res:
if r is not None:
await pipe_out(df=r,
path=path,
format='tsv',
mode='a' if count else 'w')
count += 1
if not count:
cls.logger.debug(
f"Without Expected Data ({suffix}): {data}")
return None
else:
await pipe_out(df=res, path=path, format='tsv', mode='w')
cls.logger.debug(f"Decoded file in '{path}'")
return path
else:
cls.logger.debug(f"Without Expected Data ({suffix}): {data}")
return None
async def _get_tag(self, tag_id: str) -> Tag:
try:
name = self._tails[tag_id]
except KeyError:
raise TagLookupError(f"Tag not found with id {tag_id!r}")
async with aiofiles_open(self._repository_path / name) as entry:
return Tag(**json_loads(await entry.read()))
async def a_seq_reader(path: Union[Unfuture, Union[Path, str]]):
if isinstance(path, Unfuture):
path = await path
async with aiofiles_open(path, 'rt') as handle:
header, content = fasta_pat.match(await handle.read()).groups()
content = content.replace('\n', '')
assert content != '', str(path)
return header, content
async def a_load_json(path):
try:
if isawaitable(path):
path = await path
if path is None:
return None
async with aiofiles_open(path) as inFile:
return json.loads(await inFile.read())
except Exception:
raise ValueError(str(path))
async def as_txt2tsv(cls, identifier: str, **kwargs):
path = await cls.UniProtTXT.stream_retrieve_txt(identifier, **kwargs)
if path is None:
return identifier, None
else:
async with aiofiles_open(path, 'rt') as handle:
txt = await handle.read()
raw = txt.replace('FT ', '').replace(
'\n', '').split('FT VAR_SEQ ')[1:]
return identifier, '; '.join('VAR_SEQ ' + i.replace(' /', '; /')
for i in raw)
async def writer(handle, path, header: bytes, start_key: bytes):
start = False
async with aiofiles_open(path, 'wb') as fileOb:
if header:
await fileOb.write(header)
async for line in handle:
if line.startswith(start_key):
start = True
elif start and line.startswith(b'#'):
await fileOb.write(line)
return path
if start:
await fileOb.write(line)
async def a_seq_parser(path: Union[Unfuture, Coroutine, Path, str]):
if isawaitable(path):
path = await path
async with aiofiles_open(path, 'rt') as handle:
header, content = None, ''
async for line in handle:
if line.startswith('>'):
if header is not None:
yield header.strip(), content.replace('\n', '')
header, content = line, ''
else:
content += line
assert header is not None, f"\npath: {path}\ncur_content: {content}"
yield header.strip(), content.replace('\n', '')
async def run(self):
while True:
line = await self.in_queue.get()
if line == STOP_SIGNAL:
break
if line:
await self.async_writer(self.io, line)
await asyncio.sleep(0.5)
if self.stats:
statistics = self.stats.dict()
async with aiofiles_open('/dev/stdout', mode='wb') as stats:
await stats.write(
ujson_dumps(statistics).encode('utf-8') + b'\n')
async def a_read_csv(path, read_mode='r', **kwargs):
'''
only suitable for small dataframe
'''
try:
if isinstance(path, (Coroutine, Unfuture)):
path = await path
if isinstance(path, (Path, str)):
async with aiofiles_open(path, read_mode) as file_io:
with StringIO(await file_io.read()) as text_io:
return read_csv(text_io, **kwargs)
elif isinstance(path, DataFrame):
return path
else:
return DataFrame(path, columns=kwargs.get('columns', None))
except Exception:
raise ValueError(f'{path}')
async def ftp_download(cls, semaphore, method: str, info: Dict, path,
rate: float):
url = urlparse(info['url'])
cls.logger.debug("ftp_download: Start to download file: {}".format(
info['url']))
async with semaphore:
async with aioftp_Client.context(url.netloc) as client:
if await client.exists(url.path):
async with aiofiles_open(
path, 'wb') as file_out, client.download_stream(
url.path) as stream:
async for block in stream.iter_by_block():
await file_out.write(block)
cls.logger.debug(f"File has been saved in: {path}")
await asyncio.sleep(rate)
return path
return None
async def main():
arguments = parse_args()
target_settings, config = parse_settings(arguments)
queue_input = asyncio.Queue()
queue_tasks = asyncio.Queue()
queue_prints = asyncio.Queue()
task_semaphore = asyncio.Semaphore(config.senders)
statistics = Stats() if config.statistics else None
async with aiofiles_open(config.output_file,
mode=config.write_mode) as file_with_results:
writer_coroutine = get_async_writer(config)
if config.custom_module == 'default':
target_worker = TargetWorker(statistics, task_semaphore,
queue_prints,
config.show_only_success,
config.body_not_empty)
else:
try:
module_name = f'lib.modules.{config.custom_module}.{config.custom_module}'
_mod = importlib.import_module(module_name)
CustomWorker = getattr(_mod, 'CustomWorker')
target_worker = CustomWorker(statistics, task_semaphore,
queue_prints,
config.show_only_success,
config.body_not_empty)
except Exception as e:
print(f'exit, error: {e}')
exit(1)
input_reader: TargetReader = create_io_reader(statistics, queue_input,
target_settings, config)
task_producer = TaskProducer(statistics, queue_input, queue_tasks,
target_worker)
executor = Executor(statistics, queue_tasks, queue_prints)
printer = OutputPrinter(config.output_file, statistics, queue_prints,
file_with_results, writer_coroutine)
await asyncio.wait([
worker.run()
for worker in [input_reader, task_producer, executor, printer]
])
async def http_download(cls, semaphore, method: str, info: Dict, path: str,
rate: float):
'''
possible exceptions:
RemoteServerError,
aiohttp.client_exceptions.ServerDisconnectedError,
aiohttp.client_exceptions.ClientConnectorError,
aiohttp.client_exceptions.ClientPayloadError
'''
cls.logger.debug(
f"http_download: Start to download file: {info['url']}")
try:
async with semaphore:
async with aiohttp.ClientSession(
connector=aiohttp.TCPConnector(ssl=False),
trust_env=True) as session:
async_func = getattr(session, method)
async with async_func(**info) as resp:
if resp.status == 200:
async with aiofiles_open(path, 'wb') as file_out:
# Asynchronous iterator implementation of readany()
async for chunk in resp.content.iter_any():
await file_out.write(chunk)
cls.logger.debug(
f"File has been saved in: '{path}'")
await asyncio.sleep(rate)
return path
elif resp.status in (204, 300, 400, 403, 404, 405,
406):
cls.logger.debug(f"{resp.status} for: {info}")
return None
else:
mes = "code={resp.status}, message={resp.reason}, headers={resp.headers}".format(
resp=resp)
cls.logger.error(f"{info} -> {mes}")
raise RemoteServerError(mes)
except Exception as e:
cls.logger.error(f"{info} -> {e}")
raise e
async def fasta_load(cls, path):
async with aiofiles_open(path, 'rt') as handle:
data = await handle.read()
assert bool(cls.fasta_pat.fullmatch(data))
async def cif_load(path):
async with aiofiles_open(path, 'rt') as handle:
data = await handle.read()
assert data.endswith('\n#\n')
async def json_load(path):
async with aiofiles_open(path, 'rt') as handle:
data = await handle.read()
json.loads(data)
class Tagging(Cog):
"""Tag and access tagged resouces."""
_base: str = r"!(t|tag)"
_tag_id: str = r"(?P<tag_id>\d+|[A-Za-z]+)"
_tag_body: str = r"(?P<tag_body>.{0,512})"
_commands = Cog.group()
_repository_path: Path
_tails: Set[str]
_head: Dict[str, str]
_repository_url: str = ConfigValue("default", "tagging", "repository")
@Cog.task
async def __ainit__(self):
repository_path: str = mkdtemp()
path = await clone_repository(self._repository_url, repository_path)
self.logger.info("Cloned tagging repository into %s", repr(path))
await self._hook_repository(path)
@Cog.destructor
def _cleanup_path(self) -> None:
if self._repository_path is None:
return
assert self._repository_path.exists()
assert self._repository_path != Path(
"."
) # Unless you like to cry, don't remove this assertion.
rmtree(self._repository_path)
# Internals
async def _hook_repository(self, path: Path) -> None:
assert isinstance(path, Path)
assert path.is_dir()
self._repository_path = path
# Index repository
# ep.tagging/head.jsonl
# ep.tagging/{CHANNEL_ID}.{MESSAGE_ID}
self._tails = tails = {}
self._head = head = {}
if not (header := (path / "head.jsonl")).exists():
header.touch()
async with aiofiles_open(path / "head.jsonl") as header:
async for entry in header:
serialized = json_loads(entry)
aliased = serialized["aliased"]
async with aiofiles_open(path / aliased) as file:
data = json_loads(await file.read())
tails[data["ident"]] = aliased
head[data["ident"]] = aliased
for alias in serialized["aliases"]:
head[alias] = aliased
async def tsv_load(cls, path):
async with aiofiles_open(path, 'rt') as handle:
data = await handle.readline()
assert data.strip() != ''
