def table_rowindexes_colspan2trimmed(
cls,
table,
rowindex_list,
colspan,
):
logger = FoxylibLogger.func_level2logger(cls.table_colspan2filtered,
logging.DEBUG)
table_02 = cls.table_rowindexes2filtered(table, rowindex_list)
table_03 = cls.table_colspan2filtered(table_02, colspan)
logger.debug({
"colspan": colspan,
"# table": len(table),
"# table_02": len(table_02),
"# table_03": len(table_03),
})
return table_03
async def dequeue_n_nowait(cls, queue, n):
logger = FoxylibLogger.func_level2logger(cls.dequeue_n_nowait,
logging.DEBUG)
# logger.debug({"queue.qsize()":queue.qsize()})
item_first = await queue.get()
items = [item_first]
# logger.debug({"items":items})
try:
# logger.debug({"n": n})
for i in range(n - 1):
# logger.debug({"i": i})
item = queue.get_nowait()
items.append(item)
except asyncio.QueueEmpty:
pass
# logger.debug({"return items": items})
return items
async def produce_consume(cls, queue, produce_coros, consume_coros):
logger = FoxylibLogger.func_level2logger(cls.produce_consume,
logging.DEBUG)
producer_tasks = lmap(asyncio.create_task, produce_coros)
consumer_tasks = lmap(asyncio.create_task, consume_coros)
# with both producers and consumers running, wait for
# the producers to finish
await asyncio.gather(*producer_tasks)
logger.debug('producers done')
# wait for the remaining tasks to be processed
await queue.join()
logger.debug('consumers done')
# cancel the consumers, which are now idle
for c in consumer_tasks:
c.cancel()
logger.debug('cancelled')
def cachedmethod(
cls,
func=None,
self2cache=None,
reader=None,
writer=None,
lock=None,
):
logger = FoxylibLogger.func_level2logger(cls.cachedmethod,
logging.DEBUG)
assert_is_not_none(self2cache)
def wrapper(f):
"""
CLASSMETHOD can used cached() too.
But for simplicity of the system, CLASSMETHOD is forced to use cachedmethod
"""
types_valid = {
CallableTool.Type.FUNCTION,
}
assert_in(
CallableTool.callable2type(f), types_valid,
"For instancemethod, use attach_cachedmethod() instead")
@wraps(f)
def wrapped(self, *_, **__):
cache = self2cache(self)
f_lazy = partial(f, self, *_, **__)
v = AsymmetricCache._read_or_write(f_lazy,
cache,
reader,
writer,
*_,
lock=lock,
*__)
return v
return wrapped
return wrapper(func) if func else wrapper
def text2entity_list(cls, str_in):
logger = FoxylibLogger.func_level2logger(cls.text2entity_list,
logging.DEBUG)
entity_list_single = DayofweekEntityKoSingle.text2entity_list(str_in)
entity_list_concat = DayofweekEntityKoConcat.text2entity_list(str_in)
ll = [
entity_list_single,
entity_list_concat,
]
l = sorted(luniq(chain(*ll), idfun=FoxylibEntity.entity2span),
key=FoxylibEntity.entity2span)
logger.debug({
"entity_list_single": entity_list_single,
"entity_list_concat": entity_list_concat,
"ll": ll,
"l": l,
})
return l
def cached(
cls,
func=None,
cache=None,
key=None,
f_pivot=None,
lock=None,
):
logger = FoxylibLogger.func_level2logger(cls.cached, logging.DEBUG)
assert_is_not_none(cache)
assert_is_not_none(f_pivot)
key = key or hashkey
reader = DatetimedCache.key_pivot2reader(key, f_pivot)
writer = DatetimedCache.key_pivot2writer(key, f_pivot)
return AsymmetricCache.Decorator.cached(func=func,
cache=cache,
reader=reader,
writer=writer,
lock=lock)
def filepath2text(cls, filepath_hwp):
logger = FoxylibLogger.func_level2logger(cls.filepath2text,
logging.DEBUG)
tt = cls._text_transform()
try:
with closing(Hwp5File(filepath_hwp)) as hwp5file:
with BytesIO() as bytes_io:
tt.transform_hwp5_to_text(hwp5file, bytes_io)
# https://stackoverflow.com/questions/26879981/writing-then-reading-in-memory-bytes-bytesio-gives-a-blank-result
bytes_io.seek(0)
bytes = bytes_io.read()
return bytes.decode('UTF-8')
except ParseError as e:
e.print_to_logger(logger)
raise
except InvalidHwp5FileError as e:
logger.error('%s', e)
raise
def table2percentage(cls, table, v):
logger = FoxylibLogger.func_level2logger(cls.table2percentage,
logging.DEBUG)
count_col = iter2singleton(map(len, table))
h_j2col_sum = {
j: sum(map(int, filter(bool, map(ig(j), table[1:]))))
for j in range(cls.COUNT_COLHEAD, count_col)
}
for i, l in enumerate(table):
if i == 0:
yield l
continue
# logger.debug({"l":l})
l_head = l[:cls.COUNT_COLHEAD]
l_right = [
"{:.02f}".format(int(l[j]) * 100 /
h_j2col_sum[j]) if l[j] else l[j]
for j in range(cls.COUNT_COLHEAD, count_col)
]
yield lchain(l_head, l_right)
def attach_cached(
cls,
func=None,
cached=None,
cache=None,
key=None,
lock=None,
):
logger = FoxylibLogger.func_level2logger(cls.attach_cached,
logging.DEBUG)
assert_is_not_none(cache)
cached = cached or cachetools.cached
config = DictTool.filter(
lambda k, v: v is not None, {
cls.Config.Field.CACHE: cache,
cls.Config.Field.KEY: key or hashkey,
cls.Config.Field.LOCK: lock,
})
kwargs = cls.Config.config2kwargs(config)
def wrapper(f):
"""
CLASSMETHOD can used cached() too.
But for simplicity of the system, CLASSMETHOD is forced to use cachedmethod
"""
# types_valid = {CallableTool.Type.FUNCTION, }
# assert_in(CallableTool.callable2type(f), types_valid,
# "For instancemethod, use attach_cachedmethod() instead")
assert_false(hasattr(f, cls.Constant.ATTRIBUTE_NAME))
setattr(f, cls.Constant.ATTRIBUTE_NAME, config)
f_cached = cached(cache, **kwargs)(f)
return f_cached
return wrapper(func) if func else wrapper
async def coros_list2pipelined(cls, coros_list, queue_list):
logger = FoxylibLogger.func_level2logger(cls.coros_list2pipelined,
logging.DEBUG)
n = len(coros_list)
assert_greater_equal(n, 1)
assert_equal(len(queue_list), n - 1)
for q in queue_list:
assert_true(AioQueueTool.queue2is_valid_loop(q))
tasks_list = [lmap(asyncio.create_task, coros) for coros in coros_list]
await asyncio.gather(*tasks_list[0])
logger.debug('{}/{} coros done'.format(1, n))
for i in range(1, n):
await queue_list[i - 1].join()
logger.debug('{}/{} queue empty'.format(i, n - 1))
for c in tasks_list[i]:
c.cancel()
logger.debug('{}/{} coros done'.format(i + 1, n))
def test_08(self):
logger = FoxylibLogger.func_level2logger(self.test_08, logging.DEBUG)
produced = []
consumed = []
producer_list = [partial(P2C.producer, produced) for _ in range(5)]
piper_batch_list_1 = [FunctionTool.func2batch(P2C.piper) for _ in range(7)]
piper_batch_list_2 = [FunctionTool.func2batch(P2C.piper) for _ in range(1)]
piper_batch_list_3 = [FunctionTool.func2batch(P2C.piper) for _ in range(4)]
consumer_batch_list = [FunctionTool.func2batch(partial(P2C.consumer, consumed)) for x in range(10)]
batches_list = [producer_list,
piper_batch_list_1,
piper_batch_list_2,
piper_batch_list_3,
consumer_batch_list,
]
pipeline_coro = AioPipeline.batches_list2pipelined(batches_list)
AioTool.awaitable2result(pipeline_coro)
self.assertEqual(len(produced), len(consumed))
self.assertEqual(sorted(produced), sorted(consumed))
def j_pair_list2upsert(
cls,
collection,
j_pair_list,
):
logger = FoxylibLogger.func_level2logger(cls.j_pair_list2upsert,
logging.DEBUG)
# def j_pair2operation_upsertone(j_pair, ):
# j_filter, j_update = j_pair
# return UpdateOne(j_filter, {"$set": j_update}, upsert=True, )
op_list = lmap(lambda j_pair: cls.pair2operation_upsert(*j_pair),
j_pair_list)
logger.debug({"op_list": op_list})
# op_list = lmap(j_pair2operation_upsertone, j_pair_list)
# bulk_write = ErrorTool.log_when_error(collection.bulk_write, logger)
try:
result = collection.bulk_write(op_list)
except BulkWriteError as e:
print(e.details)
raise e
return result
def str_ll2interpolated_list(cls, str_ll_in, i_pivot, multiple):
logger = FoxylibLogger.func_level2logger(cls.str_ll2interpolated_list,
logging.DEBUG)
n_row = len(str_ll_in)
n_col_raw = iter2singleton(map(len, str_ll_in))
n_col_extended = cls.col_count_raw2extended(n_col_raw, multiple)
logger.debug({
"n_col_raw": n_col_raw,
"n_col_extended": n_col_extended,
"multiple": multiple,
})
# raise Exception()
ll_tr = [
cls._col_index_extended2str_list(str_ll_in, k, i_pivot, multiple)
for k in range(n_col_extended)
]
ll_out = [[ll_tr[k][i] for k in range(n_col_extended)]
for i in range(n_row)]
return ll_out
def parse2nullboolean(cls, s):
logger = FoxylibLogger.func_level2logger(cls.parse2nullboolean, logging.DEBUG)
logger.debug({"s":s})
if any(map(lambda x: s is x, [None, True, False])):
return s
# logger.debug({"s": s, "s is False": s is False})
if s is None:
return None
s_lower = s.lower()
if s_lower.isdecimal():
v = int(s_lower)
return bool(v)
if s_lower in {"true", "t", "yes", "y",}:
return True
if s_lower in {"false", "f", "no", "n", ""}:
return False
return None
def table_colspan2fresh_rows_filtered(cls, table, colspan):
logger = FoxylibLogger.func_level2logger(
cls.table_colspan2fresh_rows_filtered, logging.DEBUG)
i_list_starting = cls.table_colspan2rowindex_list_starting(
table, colspan)
table_row_filtered = cls.table_rowindexes2filtered(
table, i_list_starting)
logger.debug({
"colspan":
colspan,
"len(table)":
len(table),
"iter2singleton(map(len,table))":
iter2singleton(map(len, table)),
"len(i_list_starting)":
len(i_list_starting),
"len(table_row_filtered)":
len(table_row_filtered),
"iter2singleton(map(len,table_row_filtered))":
iter2singleton(map(len, table_row_filtered)),
})
return table_row_filtered
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
c = MongodbToolCollection.collection_default()
c.delete_many({})
doc_in = {'a': 'a', 'b': 'b'}
op_list = [
InsertOne(doc_in),
]
c.bulk_write(op_list)
doc_found = iter2singleton(c.find({}))
# hyp1 = DictTool.keys2excluded(doc_found, [MongoDBTool.Field._ID])
logger.debug({"doc_found": doc_found, "doc_in": doc_in})
self.assertEqual(doc_found, doc_in)
self.assertTrue(isinstance(doc_in[MongoDBTool.Field._ID],
ObjectId)) # doc_in updated!!
self.assertTrue(isinstance(doc_found[MongoDBTool.Field._ID], ObjectId))
hyp3 = set(doc_found.keys())
ref3 = {'a', '_id', 'b'}
self.assertEqual(hyp3, ref3)
def text2entity_list(cls, str_in):
logger = FoxylibLogger.func_level2logger(cls.text2entity_list,
logging.DEBUG)
p = cls.pattern()
m_list = list(p.finditer(str_in))
logger.debug({
"m_list": m_list,
})
def match2entity_list(m):
s, e = m.span()
text = m.group()
n = len(text)
l = [{
FoxylibEntity.Field.SPAN: (s + i, s + i + 1),
FoxylibEntity.Field.VALUE:
DayofweekEntityKo.str2value(text[i]),
FoxylibEntity.Field.TEXT:
text[i],
FoxylibEntity.Field.FULLTEXT:
str_in,
FoxylibEntity.Field.TYPE:
DayofweekEntity.entity_type(),
} for i in range(n) if text[i] != "," and not text[i].isspace()]
# logger.debug({"s": s,
# "e": e,
# "m": m,
# "text": text,
# "n": n,
# "l": l,
# })
return l
return lchain(*lmap(match2entity_list, m_list))
def create(cls, es_client, index, alias):
logger = FoxylibLogger.func_level2logger(cls.create, logging.DEBUG)
logger.debug({"index": index, "alias": alias})
j_result = es_client.indices.put_alias(index, alias)
return j_result
import logging
import os
import sys
from foxylib.tools.jinja2.jinja2_tool import Jinja2Renderer
from foxylib.tools.log.foxylib_logger import FoxylibLogger
def main():
logger = FoxylibLogger.func_level2logger(main, logging.DEBUG)
str_in = sys.stdin.read()
h_env = dict(os.environ)
str_out = Jinja2Renderer.text2text(str_in, h_env)
logger.debug({
"str_in": str_in,
"str_out": str_out,
})
print(str_out)
if __name__ == "__main__":
FoxylibLogger.attach_stderr2loggers(logging.DEBUG)
main()
def node_list2groupname(cls, node_list):
logger = FoxylibLogger.func_level2logger(cls.node_list2groupname,
logging.DEBUG)
# logger.debug({"node_list": node_list})
name_list = lmap(cls2name, node_list)
return "__".join(name_list)
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
self.assertEqual(MyObject.myclassmethod.a, "a")
self.assertEqual(getattr(getattr(MyObject, "myclassmethod"), "a"), "a")
def env(cls):
logger = FoxylibLogger.func_level2logger(cls.env, logging.DEBUG)
env_raw = EnvTool.env_raw()
# logger.debug({"env_raw":env_raw})
return cls.env2norm(env_raw) or "local"
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
web_client = FoxylibSlack.web_client()
channel = FoxylibChannel.V.FOXYLIB
filepath = os.path.join(FILE_DIR, "test_01.txt")
response = FilesUploadMethod.invoke(web_client, channel, filepath)
self.assertTrue(SlackResponseTool.response2is_ok(response))
j_response = SlackResponseTool.response2j_resopnse(response)
hyp = j_response
ref = {
"ok": True,
"file": {
# "id": "FSB2C7L4F",
# "created": 1578114293,
# "timestamp": 1578114293,
"name": "test_01.txt",
"title": "test 01",
"mimetype": "text/plain",
"filetype": "text",
"pretty_type": "Plain Text",
"user": "******",
"editable": True,
"size": 30,
"mode": "snippet",
"is_external": False,
"external_type": "",
"is_public": True,
"public_url_shared": False,
"display_as_bot": False,
"username": "",
# "url_private": "https://files.slack.com/files-pri/TB2V2S5L6-FSB2C7L4F/test_01.txt",
# "url_private_download": "https://files.slack.com/files-pri/TB2V2S5L6-FSB2C7L4F/download/test_01.txt",
# "permalink": "https://foxytrixy.slack.com/files/US8SU4H8W/FSB2C7L4F/test_01.txt",
# "permalink_public": "https://slack-files.com/TB2V2S5L6-FSB2C7L4F-1846b53c2e",
# "edit_link": "https://foxytrixy.slack.com/files/US8SU4H8W/FSB2C7L4F/test_01.txt/edit",
# "preview": "\ud14c\uc2a4\ud2b8\uc6a9 \ud30c\uc77c\n\u314e\u314e\u314e\n",
# "preview_highlight": "<div class=\"CodeMirror cm-s-default CodeMirrorServer\" oncopy=\"if(event.clipboardData){event.clipboardData.setData('text/plain',window.getSelection().toString().replace(/\\u200b/g,''));event.preventDefault();event.stopPropagation();}\">\n<div class=\"CodeMirror-code\">\n<div><pre>\ud14c\uc2a4\ud2b8\uc6a9 \ud30c\uc77c</pre></div>\n<div><pre>\u314e\u314e\u314e</pre></div>\n<div><pre></pre></div>\n</div>\n</div>\n",
"lines": 3,
"lines_more": 0,
"preview_is_truncated": False,
"comments_count": 0,
"is_starred": False,
"shares": {
# "public": {
# "CS7V80KGE": [
# {
# "reply_users": [],
# "reply_users_count": 0,
# "reply_count": 0,
# "ts": "1578114293.000400",
# "channel_name": "foxylib",
# "team_id": "TB2V2S5L6"
# }
# ]
# }
},
"channels": [
"CS7V80KGE"
],
"groups": [],
"ims": [],
"has_rich_preview": False
}
}
# logger.debug(json.dumps({"j_response":j_response}, indent=2))
self.assertEqual(FileUploadMethod.j_response2norm(hyp), ref)
# cleanup
j_file = FilesUploadMethod.j_response2j_file(j_response)
file_id = SlackFile.j_file2id(j_file)
web_client.files_delete(**{"file": file_id})
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
self.assertEqual(NumberTool.num2ordinal(2), "2nd")
self.assertEqual(NumberTool.num2ordinal(13), "13th")
self.assertEqual(NumberTool.num2ordinal(101), "101st")
def test_09(self):
logger = FoxylibLogger.func_level2logger(self.test_09, logging.DEBUG)
aiter = AioTool.iterable2aiter(range(5))
self.assertEqual(asyncio.run(AioTool.aiter2list(aiter)), list(range(5)))
def main():
FoxylibLogger.attach_stderr2loggers(logging.DEBUG)
app = FoxylibFlask.app()
app.run()
def dict2normalized(cls, dict_value2texts, normalizer):
logger = FoxylibLogger.func_level2logger(cls.dict2normalized, logging.DEBUG)
# logger.debug({"dict_value2texts":dict_value2texts})
return {value: lmap(normalizer, texts)
for value, texts in dict_value2texts.items()}
def _load_urls2app(cls, app):
logger = FoxylibLogger.func_level2logger(cls._load_urls2app,
logging.DEBUG)
def setUpClass(cls):
FoxylibLogger.attach_stderr2loggers(logging.DEBUG)
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
hyp = OwapiTool.battletag2exists("yeri#11212", )
self.assertFalse(hyp)
