async def work_with_queue_result(queue_out: asyncio.Queue, filename, mode_write) -> None: """ :param queue_out: :param filename: :param mode_write: :return: """ if mode_write == 'a': method_write_result = write_to_file else: method_write_result = write_to_stdout async with aiofiles_open(filename, mode=mode_write) as file_with_results: while True: line = await queue_out.get() if line == b"check for end": break if line: await method_write_result(file_with_results, line) await asyncio.sleep(0.5) # region dev if args.statistics: stop_time = datetime.datetime.now() _delta_time = stop_time - start_time duration_time_sec = _delta_time.total_seconds() statistics = { 'duration': duration_time_sec, 'valid targets': count_input, 'success': count_good, 'fails': count_error } async with aiofiles_open('/dev/stdout', mode='wb') as stats: await stats.write(ujson_dumps(statistics).encode('utf-8') + b'\n')
async def _write_tag( self, name: str, body: str, tag_id: Optional[str] = None ) -> Tag: if tag_id is None: while True: tag_id = "".join(choice(ascii_letters) for _ in range(6)) if tag_id in self._tails: await sleep(0) continue self._tails[tag_id] = name unserialized = {"ident": tag_id, "body": body} async with aiofiles_open(self._repository_path / name, "w") as entry: await entry.write(json_dumps(unserialized)) async with aiofiles_open(self._repository_path / "head.jsonl", "a") as header: await header.write(json_dumps({"aliased": name, "aliases": []}) + "\n") self._head[tag_id] = name # TODO: Commit and push changes. return Tag(**unserialized)
async def json2tsv(cls, ori_path, path): cls.logger.debug('Start to decode SMR JSON') async with aiofiles_open(ori_path) as inFile: try: data = json.loads(await inFile.read()) except Exception as e: cls.logger.error(f"Error in '{ori_path}'") raise e res = Dict2Tabular.pyexcel_io(cls.yieldSMR(data)) if res is not None: if isinstance(res, Generator): count = 0 for r in res: if r is not None: await pipe_out(df=r, path=path, format='tsv', mode='a' if count else 'w') count += 1 if not count: cls.logger.debug( f"Without Expected Data (swissmodel/repository/uniprot/): {data}" ) return None else: await pipe_out(df=res, path=path, format='tsv', mode='w') cls.logger.debug(f"Decoded file in '{path}'") return path else: cls.logger.warning( f"Without Expected Data (swissmodel/repository/uniprot/): {data}" ) return None
async def main(): asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) arguments = parse_args() target_settings, config = parse_settings(arguments) queue_input = asyncio.Queue() queue_tasks = asyncio.Queue() queue_prints = asyncio.Queue() task_semaphore = asyncio.Semaphore(config.senders) statistics = Stats() if config.statistics else None async with aiofiles_open(config.output_file, mode=config.write_mode) as file_with_results: writer_coroutine = get_async_writer(config) target_worker = TargetWorker(statistics, task_semaphore, queue_prints, config.show_only_success) input_reader: TargetReader = create_io_reader(statistics, queue_input, target_settings, config) task_producer = TaskProducer(statistics, queue_input, queue_tasks, target_worker) executor = Executor(statistics, queue_tasks, queue_prints) printer = OutputPrinter(statistics, queue_prints, file_with_results, writer_coroutine) await asyncio.wait([ worker.run() for worker in [input_reader, task_producer, executor, printer] ])
async def read_input_file(queue_input: asyncio.Queue, settings: dict, path_to_file: str) -> None: """ посредством модуля aiofiles функция "асинхронно" читает из файла записи, представляющие собой обязательно или ip адрес или запись подсети в ipv4 из данной записи формируется экзэмпляр NamedTuple - Target, который отправляется в Очередь :param queue_results: :param settings: :param path_to_file: :return: """ global count_input async with aiofiles_open(path_to_file, mode='rt') as f: # read str async for line in f: linein = line.strip() if any([check_ip(linein), check_network(linein)]): targets = create_targets_ssh_protocol(linein, settings) # generator if targets: for target in targets: check_queue = True while check_queue: size_queue = queue_input.qsize() if size_queue < queue_limit_targets - 1: count_input += 1 # statistics queue_input.put_nowait(target) check_queue = False else: await asyncio.sleep(sleep_duration_queue_full) await queue_input.put(b"check for end")
async def main(): arguments = parse_args() target_settings, config = parse_settings(arguments) queue_input = asyncio.Queue() queue_tasks = asyncio.Queue() queue_prints = asyncio.Queue() # task_semaphore = asyncio.Semaphore(config.senders) statistics = Stats() if config.statistics else None # async with aiofiles_open(config.output_file, mode=config.write_mode) as file_with_results: writer_coroutine = get_async_writer(config) target_worker = TargetWorker(statistics, task_semaphore, queue_prints, config) input_reader: TargetReader = create_io_reader(statistics, queue_input, target_settings, config) task_producer = TaskProducer(statistics, queue_input, queue_tasks, target_worker) executor = Executor(statistics, queue_tasks, queue_prints) printer = OutputPrinter(config.output_file, statistics, queue_prints, file_with_results, writer_coroutine) running_tasks = [ asyncio.create_task(worker.run()) for worker in [input_reader, task_producer, executor, printer] ] await asyncio.wait(running_tasks)
async def run(self): async with aiofiles_open(self.file_path, mode='rt') as f: async for line in f: linein = line.strip() await self.producer.send(linein) await self.producer.send_stop()
async def pipe_out(df, path, **kwargs): if not isinstance(df, Sheet): raise TypeError(f"Invalid Object for pipe_out(): {type(df)}") if len(df) == 0: raise ValueError("Zero record!") path = Path(path) mode = kwargs.get('mode', 'a') clear_headers: bool = path.exists() and mode.startswith('a') var_format = kwargs.get('format', 'tsv').lower() async with aiofiles_open(path, mode) as file_io: """ if isinstance(df, DataFrame): sorted_col = sorted(df.columns) if clear_headers: headers = None else: headers = sorted_col dataset = Dataset(headers=headers) dataset.extend(df[sorted_col].to_records(index=False)) to_write = dataset.export(var_format, lineterminator='\n') elif isinstance(df, Dataset): if clear_headers: df.headers = None to_write = df.export(var_format, lineterminator='\n') """ df = df.project(sorted(df.colnames)) if clear_headers: df.colnames = [] to_write = getattr(df, f"get_{var_format}")(lineterminator='\n') await file_io.write(to_write)
async def json2tsv(cls, suffix: str, ori_path: Union[str, Path], path: Union[str, Path]): cls.logger.debug('Start to decode') async with aiofiles_open(ori_path) as handle: try: data = json.loads(await handle.read()) except Exception as e: cls.logger.error(f"Error in '{ori_path}'") raise e res = Dict2Tabular.pyexcel_io(traverseSuffixes(suffix, data)) if res is not None: if isinstance(res, Generator): count = 0 for r in res: if r is not None: await pipe_out(df=r, path=path, format='tsv', mode='a' if count else 'w') count += 1 if not count: cls.logger.debug( f"Without Expected Data ({suffix}): {data}") return None else: await pipe_out(df=res, path=path, format='tsv', mode='w') cls.logger.debug(f"Decoded file in '{path}'") return path else: cls.logger.debug(f"Without Expected Data ({suffix}): {data}") return None
async def _get_tag(self, tag_id: str) -> Tag: try: name = self._tails[tag_id] except KeyError: raise TagLookupError(f"Tag not found with id {tag_id!r}") async with aiofiles_open(self._repository_path / name) as entry: return Tag(**json_loads(await entry.read()))
async def a_seq_reader(path: Union[Unfuture, Union[Path, str]]): if isinstance(path, Unfuture): path = await path async with aiofiles_open(path, 'rt') as handle: header, content = fasta_pat.match(await handle.read()).groups() content = content.replace('\n', '') assert content != '', str(path) return header, content
async def a_load_json(path): try: if isawaitable(path): path = await path if path is None: return None async with aiofiles_open(path) as inFile: return json.loads(await inFile.read()) except Exception: raise ValueError(str(path))
async def as_txt2tsv(cls, identifier: str, **kwargs): path = await cls.UniProtTXT.stream_retrieve_txt(identifier, **kwargs) if path is None: return identifier, None else: async with aiofiles_open(path, 'rt') as handle: txt = await handle.read() raw = txt.replace('FT ', '').replace( '\n', '').split('FT VAR_SEQ ')[1:] return identifier, '; '.join('VAR_SEQ ' + i.replace(' /', '; /') for i in raw)
async def writer(handle, path, header: bytes, start_key: bytes): start = False async with aiofiles_open(path, 'wb') as fileOb: if header: await fileOb.write(header) async for line in handle: if line.startswith(start_key): start = True elif start and line.startswith(b'#'): await fileOb.write(line) return path if start: await fileOb.write(line)
async def a_seq_parser(path: Union[Unfuture, Coroutine, Path, str]): if isawaitable(path): path = await path async with aiofiles_open(path, 'rt') as handle: header, content = None, '' async for line in handle: if line.startswith('>'): if header is not None: yield header.strip(), content.replace('\n', '') header, content = line, '' else: content += line assert header is not None, f"\npath: {path}\ncur_content: {content}" yield header.strip(), content.replace('\n', '')
async def run(self): while True: line = await self.in_queue.get() if line == STOP_SIGNAL: break if line: await self.async_writer(self.io, line) await asyncio.sleep(0.5) if self.stats: statistics = self.stats.dict() async with aiofiles_open('/dev/stdout', mode='wb') as stats: await stats.write( ujson_dumps(statistics).encode('utf-8') + b'\n')
async def a_read_csv(path, read_mode='r', **kwargs): ''' only suitable for small dataframe ''' try: if isinstance(path, (Coroutine, Unfuture)): path = await path if isinstance(path, (Path, str)): async with aiofiles_open(path, read_mode) as file_io: with StringIO(await file_io.read()) as text_io: return read_csv(text_io, **kwargs) elif isinstance(path, DataFrame): return path else: return DataFrame(path, columns=kwargs.get('columns', None)) except Exception: raise ValueError(f'{path}')
async def ftp_download(cls, semaphore, method: str, info: Dict, path, rate: float): url = urlparse(info['url']) cls.logger.debug("ftp_download: Start to download file: {}".format( info['url'])) async with semaphore: async with aioftp_Client.context(url.netloc) as client: if await client.exists(url.path): async with aiofiles_open( path, 'wb') as file_out, client.download_stream( url.path) as stream: async for block in stream.iter_by_block(): await file_out.write(block) cls.logger.debug(f"File has been saved in: {path}") await asyncio.sleep(rate) return path return None
async def main(): arguments = parse_args() target_settings, config = parse_settings(arguments) queue_input = asyncio.Queue() queue_tasks = asyncio.Queue() queue_prints = asyncio.Queue() task_semaphore = asyncio.Semaphore(config.senders) statistics = Stats() if config.statistics else None async with aiofiles_open(config.output_file, mode=config.write_mode) as file_with_results: writer_coroutine = get_async_writer(config) if config.custom_module == 'default': target_worker = TargetWorker(statistics, task_semaphore, queue_prints, config.show_only_success, config.body_not_empty) else: try: module_name = f'lib.modules.{config.custom_module}.{config.custom_module}' _mod = importlib.import_module(module_name) CustomWorker = getattr(_mod, 'CustomWorker') target_worker = CustomWorker(statistics, task_semaphore, queue_prints, config.show_only_success, config.body_not_empty) except Exception as e: print(f'exit, error: {e}') exit(1) input_reader: TargetReader = create_io_reader(statistics, queue_input, target_settings, config) task_producer = TaskProducer(statistics, queue_input, queue_tasks, target_worker) executor = Executor(statistics, queue_tasks, queue_prints) printer = OutputPrinter(config.output_file, statistics, queue_prints, file_with_results, writer_coroutine) await asyncio.wait([ worker.run() for worker in [input_reader, task_producer, executor, printer] ])
async def http_download(cls, semaphore, method: str, info: Dict, path: str, rate: float): ''' possible exceptions: RemoteServerError, aiohttp.client_exceptions.ServerDisconnectedError, aiohttp.client_exceptions.ClientConnectorError, aiohttp.client_exceptions.ClientPayloadError ''' cls.logger.debug( f"http_download: Start to download file: {info['url']}") try: async with semaphore: async with aiohttp.ClientSession( connector=aiohttp.TCPConnector(ssl=False), trust_env=True) as session: async_func = getattr(session, method) async with async_func(**info) as resp: if resp.status == 200: async with aiofiles_open(path, 'wb') as file_out: # Asynchronous iterator implementation of readany() async for chunk in resp.content.iter_any(): await file_out.write(chunk) cls.logger.debug( f"File has been saved in: '{path}'") await asyncio.sleep(rate) return path elif resp.status in (204, 300, 400, 403, 404, 405, 406): cls.logger.debug(f"{resp.status} for: {info}") return None else: mes = "code={resp.status}, message={resp.reason}, headers={resp.headers}".format( resp=resp) cls.logger.error(f"{info} -> {mes}") raise RemoteServerError(mes) except Exception as e: cls.logger.error(f"{info} -> {e}") raise e
async def fasta_load(cls, path): async with aiofiles_open(path, 'rt') as handle: data = await handle.read() assert bool(cls.fasta_pat.fullmatch(data))
async def cif_load(path): async with aiofiles_open(path, 'rt') as handle: data = await handle.read() assert data.endswith('\n#\n')
async def json_load(path): async with aiofiles_open(path, 'rt') as handle: data = await handle.read() json.loads(data)
class Tagging(Cog): """Tag and access tagged resouces.""" _base: str = r"!(t|tag)" _tag_id: str = r"(?P<tag_id>\d+|[A-Za-z]+)" _tag_body: str = r"(?P<tag_body>.{0,512})" _commands = Cog.group() _repository_path: Path _tails: Set[str] _head: Dict[str, str] _repository_url: str = ConfigValue("default", "tagging", "repository") @Cog.task async def __ainit__(self): repository_path: str = mkdtemp() path = await clone_repository(self._repository_url, repository_path) self.logger.info("Cloned tagging repository into %s", repr(path)) await self._hook_repository(path) @Cog.destructor def _cleanup_path(self) -> None: if self._repository_path is None: return assert self._repository_path.exists() assert self._repository_path != Path( "." ) # Unless you like to cry, don't remove this assertion. rmtree(self._repository_path) # Internals async def _hook_repository(self, path: Path) -> None: assert isinstance(path, Path) assert path.is_dir() self._repository_path = path # Index repository # ep.tagging/head.jsonl # ep.tagging/{CHANNEL_ID}.{MESSAGE_ID} self._tails = tails = {} self._head = head = {} if not (header := (path / "head.jsonl")).exists(): header.touch() async with aiofiles_open(path / "head.jsonl") as header: async for entry in header: serialized = json_loads(entry) aliased = serialized["aliased"] async with aiofiles_open(path / aliased) as file: data = json_loads(await file.read()) tails[data["ident"]] = aliased head[data["ident"]] = aliased for alias in serialized["aliases"]: head[alias] = aliased
async def tsv_load(cls, path): async with aiofiles_open(path, 'rt') as handle: data = await handle.readline() assert data.strip() != ''