def table_rowindexes_colspan2trimmed( cls, table, rowindex_list, colspan, ): logger = FoxylibLogger.func_level2logger(cls.table_colspan2filtered, logging.DEBUG) table_02 = cls.table_rowindexes2filtered(table, rowindex_list) table_03 = cls.table_colspan2filtered(table_02, colspan) logger.debug({ "colspan": colspan, "# table": len(table), "# table_02": len(table_02), "# table_03": len(table_03), }) return table_03
async def dequeue_n_nowait(cls, queue, n): logger = FoxylibLogger.func_level2logger(cls.dequeue_n_nowait, logging.DEBUG) # logger.debug({"queue.qsize()":queue.qsize()}) item_first = await queue.get() items = [item_first] # logger.debug({"items":items}) try: # logger.debug({"n": n}) for i in range(n - 1): # logger.debug({"i": i}) item = queue.get_nowait() items.append(item) except asyncio.QueueEmpty: pass # logger.debug({"return items": items}) return items
async def produce_consume(cls, queue, produce_coros, consume_coros): logger = FoxylibLogger.func_level2logger(cls.produce_consume, logging.DEBUG) producer_tasks = lmap(asyncio.create_task, produce_coros) consumer_tasks = lmap(asyncio.create_task, consume_coros) # with both producers and consumers running, wait for # the producers to finish await asyncio.gather(*producer_tasks) logger.debug('producers done') # wait for the remaining tasks to be processed await queue.join() logger.debug('consumers done') # cancel the consumers, which are now idle for c in consumer_tasks: c.cancel() logger.debug('cancelled')
def cachedmethod( cls, func=None, self2cache=None, reader=None, writer=None, lock=None, ): logger = FoxylibLogger.func_level2logger(cls.cachedmethod, logging.DEBUG) assert_is_not_none(self2cache) def wrapper(f): """ CLASSMETHOD can used cached() too. But for simplicity of the system, CLASSMETHOD is forced to use cachedmethod """ types_valid = { CallableTool.Type.FUNCTION, } assert_in( CallableTool.callable2type(f), types_valid, "For instancemethod, use attach_cachedmethod() instead") @wraps(f) def wrapped(self, *_, **__): cache = self2cache(self) f_lazy = partial(f, self, *_, **__) v = AsymmetricCache._read_or_write(f_lazy, cache, reader, writer, *_, lock=lock, *__) return v return wrapped return wrapper(func) if func else wrapper
def text2entity_list(cls, str_in): logger = FoxylibLogger.func_level2logger(cls.text2entity_list, logging.DEBUG) entity_list_single = DayofweekEntityKoSingle.text2entity_list(str_in) entity_list_concat = DayofweekEntityKoConcat.text2entity_list(str_in) ll = [ entity_list_single, entity_list_concat, ] l = sorted(luniq(chain(*ll), idfun=FoxylibEntity.entity2span), key=FoxylibEntity.entity2span) logger.debug({ "entity_list_single": entity_list_single, "entity_list_concat": entity_list_concat, "ll": ll, "l": l, }) return l
def cached( cls, func=None, cache=None, key=None, f_pivot=None, lock=None, ): logger = FoxylibLogger.func_level2logger(cls.cached, logging.DEBUG) assert_is_not_none(cache) assert_is_not_none(f_pivot) key = key or hashkey reader = DatetimedCache.key_pivot2reader(key, f_pivot) writer = DatetimedCache.key_pivot2writer(key, f_pivot) return AsymmetricCache.Decorator.cached(func=func, cache=cache, reader=reader, writer=writer, lock=lock)
def filepath2text(cls, filepath_hwp): logger = FoxylibLogger.func_level2logger(cls.filepath2text, logging.DEBUG) tt = cls._text_transform() try: with closing(Hwp5File(filepath_hwp)) as hwp5file: with BytesIO() as bytes_io: tt.transform_hwp5_to_text(hwp5file, bytes_io) # https://stackoverflow.com/questions/26879981/writing-then-reading-in-memory-bytes-bytesio-gives-a-blank-result bytes_io.seek(0) bytes = bytes_io.read() return bytes.decode('UTF-8') except ParseError as e: e.print_to_logger(logger) raise except InvalidHwp5FileError as e: logger.error('%s', e) raise
def table2percentage(cls, table, v): logger = FoxylibLogger.func_level2logger(cls.table2percentage, logging.DEBUG) count_col = iter2singleton(map(len, table)) h_j2col_sum = { j: sum(map(int, filter(bool, map(ig(j), table[1:])))) for j in range(cls.COUNT_COLHEAD, count_col) } for i, l in enumerate(table): if i == 0: yield l continue # logger.debug({"l":l}) l_head = l[:cls.COUNT_COLHEAD] l_right = [ "{:.02f}".format(int(l[j]) * 100 / h_j2col_sum[j]) if l[j] else l[j] for j in range(cls.COUNT_COLHEAD, count_col) ] yield lchain(l_head, l_right)
def attach_cached( cls, func=None, cached=None, cache=None, key=None, lock=None, ): logger = FoxylibLogger.func_level2logger(cls.attach_cached, logging.DEBUG) assert_is_not_none(cache) cached = cached or cachetools.cached config = DictTool.filter( lambda k, v: v is not None, { cls.Config.Field.CACHE: cache, cls.Config.Field.KEY: key or hashkey, cls.Config.Field.LOCK: lock, }) kwargs = cls.Config.config2kwargs(config) def wrapper(f): """ CLASSMETHOD can used cached() too. But for simplicity of the system, CLASSMETHOD is forced to use cachedmethod """ # types_valid = {CallableTool.Type.FUNCTION, } # assert_in(CallableTool.callable2type(f), types_valid, # "For instancemethod, use attach_cachedmethod() instead") assert_false(hasattr(f, cls.Constant.ATTRIBUTE_NAME)) setattr(f, cls.Constant.ATTRIBUTE_NAME, config) f_cached = cached(cache, **kwargs)(f) return f_cached return wrapper(func) if func else wrapper
async def coros_list2pipelined(cls, coros_list, queue_list): logger = FoxylibLogger.func_level2logger(cls.coros_list2pipelined, logging.DEBUG) n = len(coros_list) assert_greater_equal(n, 1) assert_equal(len(queue_list), n - 1) for q in queue_list: assert_true(AioQueueTool.queue2is_valid_loop(q)) tasks_list = [lmap(asyncio.create_task, coros) for coros in coros_list] await asyncio.gather(*tasks_list[0]) logger.debug('{}/{} coros done'.format(1, n)) for i in range(1, n): await queue_list[i - 1].join() logger.debug('{}/{} queue empty'.format(i, n - 1)) for c in tasks_list[i]: c.cancel() logger.debug('{}/{} coros done'.format(i + 1, n))
def test_08(self): logger = FoxylibLogger.func_level2logger(self.test_08, logging.DEBUG) produced = [] consumed = [] producer_list = [partial(P2C.producer, produced) for _ in range(5)] piper_batch_list_1 = [FunctionTool.func2batch(P2C.piper) for _ in range(7)] piper_batch_list_2 = [FunctionTool.func2batch(P2C.piper) for _ in range(1)] piper_batch_list_3 = [FunctionTool.func2batch(P2C.piper) for _ in range(4)] consumer_batch_list = [FunctionTool.func2batch(partial(P2C.consumer, consumed)) for x in range(10)] batches_list = [producer_list, piper_batch_list_1, piper_batch_list_2, piper_batch_list_3, consumer_batch_list, ] pipeline_coro = AioPipeline.batches_list2pipelined(batches_list) AioTool.awaitable2result(pipeline_coro) self.assertEqual(len(produced), len(consumed)) self.assertEqual(sorted(produced), sorted(consumed))
def j_pair_list2upsert( cls, collection, j_pair_list, ): logger = FoxylibLogger.func_level2logger(cls.j_pair_list2upsert, logging.DEBUG) # def j_pair2operation_upsertone(j_pair, ): # j_filter, j_update = j_pair # return UpdateOne(j_filter, {"$set": j_update}, upsert=True, ) op_list = lmap(lambda j_pair: cls.pair2operation_upsert(*j_pair), j_pair_list) logger.debug({"op_list": op_list}) # op_list = lmap(j_pair2operation_upsertone, j_pair_list) # bulk_write = ErrorTool.log_when_error(collection.bulk_write, logger) try: result = collection.bulk_write(op_list) except BulkWriteError as e: print(e.details) raise e return result
def str_ll2interpolated_list(cls, str_ll_in, i_pivot, multiple): logger = FoxylibLogger.func_level2logger(cls.str_ll2interpolated_list, logging.DEBUG) n_row = len(str_ll_in) n_col_raw = iter2singleton(map(len, str_ll_in)) n_col_extended = cls.col_count_raw2extended(n_col_raw, multiple) logger.debug({ "n_col_raw": n_col_raw, "n_col_extended": n_col_extended, "multiple": multiple, }) # raise Exception() ll_tr = [ cls._col_index_extended2str_list(str_ll_in, k, i_pivot, multiple) for k in range(n_col_extended) ] ll_out = [[ll_tr[k][i] for k in range(n_col_extended)] for i in range(n_row)] return ll_out
def parse2nullboolean(cls, s): logger = FoxylibLogger.func_level2logger(cls.parse2nullboolean, logging.DEBUG) logger.debug({"s":s}) if any(map(lambda x: s is x, [None, True, False])): return s # logger.debug({"s": s, "s is False": s is False}) if s is None: return None s_lower = s.lower() if s_lower.isdecimal(): v = int(s_lower) return bool(v) if s_lower in {"true", "t", "yes", "y",}: return True if s_lower in {"false", "f", "no", "n", ""}: return False return None
def table_colspan2fresh_rows_filtered(cls, table, colspan): logger = FoxylibLogger.func_level2logger( cls.table_colspan2fresh_rows_filtered, logging.DEBUG) i_list_starting = cls.table_colspan2rowindex_list_starting( table, colspan) table_row_filtered = cls.table_rowindexes2filtered( table, i_list_starting) logger.debug({ "colspan": colspan, "len(table)": len(table), "iter2singleton(map(len,table))": iter2singleton(map(len, table)), "len(i_list_starting)": len(i_list_starting), "len(table_row_filtered)": len(table_row_filtered), "iter2singleton(map(len,table_row_filtered))": iter2singleton(map(len, table_row_filtered)), }) return table_row_filtered
def test_02(self): logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG) c = MongodbToolCollection.collection_default() c.delete_many({}) doc_in = {'a': 'a', 'b': 'b'} op_list = [ InsertOne(doc_in), ] c.bulk_write(op_list) doc_found = iter2singleton(c.find({})) # hyp1 = DictTool.keys2excluded(doc_found, [MongoDBTool.Field._ID]) logger.debug({"doc_found": doc_found, "doc_in": doc_in}) self.assertEqual(doc_found, doc_in) self.assertTrue(isinstance(doc_in[MongoDBTool.Field._ID], ObjectId)) # doc_in updated!! self.assertTrue(isinstance(doc_found[MongoDBTool.Field._ID], ObjectId)) hyp3 = set(doc_found.keys()) ref3 = {'a', '_id', 'b'} self.assertEqual(hyp3, ref3)
def text2entity_list(cls, str_in): logger = FoxylibLogger.func_level2logger(cls.text2entity_list, logging.DEBUG) p = cls.pattern() m_list = list(p.finditer(str_in)) logger.debug({ "m_list": m_list, }) def match2entity_list(m): s, e = m.span() text = m.group() n = len(text) l = [{ FoxylibEntity.Field.SPAN: (s + i, s + i + 1), FoxylibEntity.Field.VALUE: DayofweekEntityKo.str2value(text[i]), FoxylibEntity.Field.TEXT: text[i], FoxylibEntity.Field.FULLTEXT: str_in, FoxylibEntity.Field.TYPE: DayofweekEntity.entity_type(), } for i in range(n) if text[i] != "," and not text[i].isspace()] # logger.debug({"s": s, # "e": e, # "m": m, # "text": text, # "n": n, # "l": l, # }) return l return lchain(*lmap(match2entity_list, m_list))
def create(cls, es_client, index, alias): logger = FoxylibLogger.func_level2logger(cls.create, logging.DEBUG) logger.debug({"index": index, "alias": alias}) j_result = es_client.indices.put_alias(index, alias) return j_result
import logging import os import sys from foxylib.tools.jinja2.jinja2_tool import Jinja2Renderer from foxylib.tools.log.foxylib_logger import FoxylibLogger def main(): logger = FoxylibLogger.func_level2logger(main, logging.DEBUG) str_in = sys.stdin.read() h_env = dict(os.environ) str_out = Jinja2Renderer.text2text(str_in, h_env) logger.debug({ "str_in": str_in, "str_out": str_out, }) print(str_out) if __name__ == "__main__": FoxylibLogger.attach_stderr2loggers(logging.DEBUG) main()
def node_list2groupname(cls, node_list): logger = FoxylibLogger.func_level2logger(cls.node_list2groupname, logging.DEBUG) # logger.debug({"node_list": node_list}) name_list = lmap(cls2name, node_list) return "__".join(name_list)
def test_01(self): logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG) self.assertEqual(MyObject.myclassmethod.a, "a") self.assertEqual(getattr(getattr(MyObject, "myclassmethod"), "a"), "a")
def env(cls): logger = FoxylibLogger.func_level2logger(cls.env, logging.DEBUG) env_raw = EnvTool.env_raw() # logger.debug({"env_raw":env_raw}) return cls.env2norm(env_raw) or "local"
def test_01(self): logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG) web_client = FoxylibSlack.web_client() channel = FoxylibChannel.V.FOXYLIB filepath = os.path.join(FILE_DIR, "test_01.txt") response = FilesUploadMethod.invoke(web_client, channel, filepath) self.assertTrue(SlackResponseTool.response2is_ok(response)) j_response = SlackResponseTool.response2j_resopnse(response) hyp = j_response ref = { "ok": True, "file": { # "id": "FSB2C7L4F", # "created": 1578114293, # "timestamp": 1578114293, "name": "test_01.txt", "title": "test 01", "mimetype": "text/plain", "filetype": "text", "pretty_type": "Plain Text", "user": "******", "editable": True, "size": 30, "mode": "snippet", "is_external": False, "external_type": "", "is_public": True, "public_url_shared": False, "display_as_bot": False, "username": "", # "url_private": "https://files.slack.com/files-pri/TB2V2S5L6-FSB2C7L4F/test_01.txt", # "url_private_download": "https://files.slack.com/files-pri/TB2V2S5L6-FSB2C7L4F/download/test_01.txt", # "permalink": "https://foxytrixy.slack.com/files/US8SU4H8W/FSB2C7L4F/test_01.txt", # "permalink_public": "https://slack-files.com/TB2V2S5L6-FSB2C7L4F-1846b53c2e", # "edit_link": "https://foxytrixy.slack.com/files/US8SU4H8W/FSB2C7L4F/test_01.txt/edit", # "preview": "\ud14c\uc2a4\ud2b8\uc6a9 \ud30c\uc77c\n\u314e\u314e\u314e\n", # "preview_highlight": "<div class=\"CodeMirror cm-s-default CodeMirrorServer\" oncopy=\"if(event.clipboardData){event.clipboardData.setData('text/plain',window.getSelection().toString().replace(/\\u200b/g,''));event.preventDefault();event.stopPropagation();}\">\n<div class=\"CodeMirror-code\">\n<div><pre>\ud14c\uc2a4\ud2b8\uc6a9 \ud30c\uc77c</pre></div>\n<div><pre>\u314e\u314e\u314e</pre></div>\n<div><pre></pre></div>\n</div>\n</div>\n", "lines": 3, "lines_more": 0, "preview_is_truncated": False, "comments_count": 0, "is_starred": False, "shares": { # "public": { # "CS7V80KGE": [ # { # "reply_users": [], # "reply_users_count": 0, # "reply_count": 0, # "ts": "1578114293.000400", # "channel_name": "foxylib", # "team_id": "TB2V2S5L6" # } # ] # } }, "channels": [ "CS7V80KGE" ], "groups": [], "ims": [], "has_rich_preview": False } } # logger.debug(json.dumps({"j_response":j_response}, indent=2)) self.assertEqual(FileUploadMethod.j_response2norm(hyp), ref) # cleanup j_file = FilesUploadMethod.j_response2j_file(j_response) file_id = SlackFile.j_file2id(j_file) web_client.files_delete(**{"file": file_id})
def test_01(self): logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG) self.assertEqual(NumberTool.num2ordinal(2), "2nd") self.assertEqual(NumberTool.num2ordinal(13), "13th") self.assertEqual(NumberTool.num2ordinal(101), "101st")
def test_09(self): logger = FoxylibLogger.func_level2logger(self.test_09, logging.DEBUG) aiter = AioTool.iterable2aiter(range(5)) self.assertEqual(asyncio.run(AioTool.aiter2list(aiter)), list(range(5)))
def main(): FoxylibLogger.attach_stderr2loggers(logging.DEBUG) app = FoxylibFlask.app() app.run()
def dict2normalized(cls, dict_value2texts, normalizer): logger = FoxylibLogger.func_level2logger(cls.dict2normalized, logging.DEBUG) # logger.debug({"dict_value2texts":dict_value2texts}) return {value: lmap(normalizer, texts) for value, texts in dict_value2texts.items()}
def _load_urls2app(cls, app): logger = FoxylibLogger.func_level2logger(cls._load_urls2app, logging.DEBUG)
def setUpClass(cls): FoxylibLogger.attach_stderr2loggers(logging.DEBUG)
def test_02(self): logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG) hyp = OwapiTool.battletag2exists("yeri#11212", ) self.assertFalse(hyp)