def iter_batch2yoo(cls, iter, f_batch, chunk_size):
logger = FoxylibLogger.func_level2logger(cls.iter_batch2yoo,
logging.DEBUG)
for x_list_chunk in cls.chunk_size2chunks(iter, chunk_size):
f_batch(x_list_chunk)
yield from x_list_chunk
def bulk(
cls,
es_client,
j_action_list,
run_bulk=True,
es_kwargs=None,
):
logger = FoxylibLogger.func_level2logger(cls.bulk, logging.DEBUG)
n = len(j_action_list)
count_list = [n * i // 100 for i in range(100)]
_run_bulk = run_bulk and n > 1
if _run_bulk:
return bulk(es_client, j_action_list, **es_kwargs)
else:
result_list = []
for i, j_action in enumerate(j_action_list):
if i in count_list:
logger.debug({
"i/n": "{}/{}".format(i + 1, n),
# "j_action":j_action,
})
# raise Exception()
op_type = cls.j_action2op_type(j_action)
if op_type == "index":
result = cls._j_action2op_index(es_client,
j_action,
es_kwargs=es_kwargs)
result_list.append(result)
else:
raise NotImplementedError()
return result_list
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
def v_pair2j_analysis(v_charged, v_rawprice):
v_vat_removed = KoreaTaxTool.charged2vat_removed(v_charged)
j_analysis = StripeAnalysis.charged_rawprice2j_analysis(
v_vat_removed, v_rawprice)
return j_analysis
hyp1 = v_pair2j_analysis(Decimal("1.50"), Decimal("1.00"))
ref1 = {'profit_absolute': '0.03', 'profit_ratio': '0.0337'}
logger.debug({"hyp1": hyp1})
self.assertEqual(hyp1, ref1)
hyp2 = v_pair2j_analysis(Decimal("6.20"), Decimal("5.00"))
ref2 = {'profit_absolute': '0.19', 'profit_ratio': '0.0372'}
logger.debug({"hyp2": hyp2})
self.assertEqual(hyp2, ref2)
hyp3 = v_pair2j_analysis(Decimal("12.00"), Decimal("10.00"))
ref3 = {'profit_absolute': '0.31', 'profit_ratio': '0.0310'}
logger.debug({"hyp3": hyp3})
self.assertEqual(hyp3, ref3)
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
cls = self.__class__
obj1 = cls.TestClass()
obj1.func1(3)
logger.debug({"obj1": obj1, "obj1.func1": obj1.func1})
cache1 = CacheManager.callable2cache(obj1.func1)
self.assertEqual(len(cache1), 1)
self.assertIn((3, ), cache1)
obj2 = cls.TestClass()
obj2.func1(2)
cache2 = CacheManager.callable2cache(obj2.func1)
self.assertEqual(len(cache2), 1)
self.assertIn((2, ), cache2)
self.assertNotIn((3, ), cache2)
self.assertEqual(len(cache1), 1)
self.assertIn((3, ), cache1)
self.assertNotIn((2, ), cache1)
CacheManager.add2cache(obj2.func1, 5, args=[-1])
self.assertEqual(obj2.func1(-1), 5)
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
tz = pytz.timezone("America/Los_Angeles")
dt_tz = datetime.now(tz=tz)
hours_1 = timedelta(seconds=60 * 60)
hours_23 = timedelta(seconds=60 * 60 * 23)
hours_24 = timedelta(seconds=60 * 60 * 24)
# hours_25 = timedelta(seconds=60 * 60 * 25)
time_past = (dt_tz - timedelta(seconds=60 * 5)).timetz()
dt_coming_of_past = TimeTool.time2datetime_nearest(
dt_tz, time_past, timedelta(days=1), Nearest.COMING)
self.assertGreater(dt_coming_of_past, dt_tz + hours_23)
self.assertLess(dt_coming_of_past, dt_tz + hours_24)
time_future = (dt_tz + timedelta(seconds=60 * 5)).timetz()
dt_coming_of_future = TimeTool.time2datetime_nearest(
dt_tz, time_future, timedelta(days=1), Nearest.COMING)
self.assertGreater(dt_coming_of_future, dt_tz)
# raise Exception({"dt_tz + hours_24": dt_tz + hours_24,
# "dt_future_of_future": dt_future_of_future,
# })
self.assertLess(dt_coming_of_future, dt_tz + hours_1)
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
a1 = A()
a2 = A()
self.assertTrue(ClassTool.cls_name2has_variable(A, "_h"))
self.assertFalse(ObjectTool.obj_name2has_variable(a1, "_h"))
A.put("0", "0")
self.assertIn("0", A._h)
self.assertIn("0", a1._h)
a1._h = {}
self.assertNotIn("0", a1._h)
a1.put("1", "one")
self.assertIn("1", A._h)
self.assertNotIn("1", a1._h)
a1._h["2"] = "two"
# self.assertNotIn("2", A._h) # system-dependent ?
self.assertIn("2", a1._h)
self.assertIsNotNone(a1.lookup("2"))
def wait_all(cls, f_list, sec_timeout, sec_interval):
logger = FoxylibLogger.func_level2logger(cls.wait_all, logging.DEBUG)
time_end = time.time(
) + sec_timeout if sec_timeout is not None else None
n = len(f_list)
status_list = [None] * n
logger.debug(
format_str("waiting for {} process for {} secs", len(f_list),
sec_timeout))
while (time_end is None) or (time.time() < time_end):
for i in range(n):
if status_list[i] is True:
continue
status_list[i] = f_list[i]()
if all(status_list):
break
logger.debug(
format_str(
"waiting for {}/{} processes for {} secs with {} sec interval",
len(lfilter(lambda x: not x, status_list)),
len(f_list),
"{:.3f}".format(time_end - time.time()),
sec_interval,
))
time.sleep(sec_interval)
return status_list
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
cls = self.__class__
filepath_hwp = os.path.join(FILE_DIR, "hwp샘플.hwp")
hyp = HWPTool.filepath2text(filepath_hwp)
self.assertEqual(hyp, cls.ref())
def func2threaded(
cls,
func=None,
max_workers=None,
):
logger = FoxylibLogger.func_level2logger(cls.func2threaded,
logging.DEBUG)
def wrapper(f):
@wraps(f)
def wrapped(*args, **kwargs):
executor = ThreadPoolExecutor(
max_workers=max_workers) # non-blocking
def f_new(*args, **kwargs):
rv = f(*args, **kwargs)
logger.info({"message": "func2thread", "value": rv})
LoggerTool.logger2flush_handlers(logger)
return rv
future = executor.submit(f_new, *args, **kwargs)
# future.add_done_callback(lambda x: rs.setex(name, time, x.result()))
return future
return wrapped
return wrapper(func) if func else wrapper
def _index_strs_beam2index_iter(cls, str_list_in, i_pivot_in, beam):
logger = FoxylibLogger.func_level2logger(
cls._index_strs_beam2index_iter, logging.DEBUG)
offset = 1
i_pivot = i_pivot_in - offset
str_list = str_list_in[offset:]
count_value = len(str_list)
v_list = lmap(
lambda i: (-float(str_list[i])
if str_list[i] else AbsoluteOrder.MAX, -1
if i == i_pivot_in else 0),
range(count_value),
)
i_list = SpanTool.index_values_beam2neighbor_indexes(
i_pivot, v_list, beam)
# logger.debug({"i_list":i_list, "v_list":lmap(lambda i:v_list[i], i_list)})
assert_equal(len(i_list), sum(beam) + 1)
i_list_out = ListTool.value2front(i_list, i_pivot)
for i in i_list_out:
i_out = i + offset
yield i_out
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
app, auth0 = FoxylibAuth0.app_auth0()
c = app.test_client()
response_login = c.get("/auth0/login/", follow_redirects=False)
# logger.debug({"response.data": response.data,
# "response.location": response.location,
# })
self.assertEqual(response_login.status_code, 302)
url_auth0 = response_login.location
self.assertTrue(
url_auth0.startswith("https://dev-8gnjw0rn.auth0.com/authorize"))
h_next = URLTool.url2h_query(url_auth0)
redirect_uri = l_singleton2obj(h_next.get("redirect_uri"))
self.assertEqual(redirect_uri, "http://localhost:5000/auth0/callback/")
response_auth0 = requests.get(url_auth0)
logger.debug({
"response_auth0": response_auth0,
# "response_auth0.text": response_auth0.text,
# "response_auth0.content": response_auth0.content,
})
self.assertEqual(response_auth0.status_code, 200)
self.assertIn("Log in to foxylib", response_auth0.text)
self.assertIn("Log in to dev-8gnjw0rn to continue to foxylib",
response_auth0.text)
def creds_sheet2data_ll(cls, creds, gss_info):
gss_id, sheet_range = GSSInfo.info2gss_id_sheet_range(gss_info)
logger = FoxylibLogger.func_level2logger(cls.creds_sheet2data_ll,
logging.DEBUG)
logger.info({"gss_id": gss_id, "sheet_range": sheet_range})
# username_FXTRX = FoxytrixyBot.USERNAME
# str_SCOPE = cls.SCOPE_READONLY
# creds = username_scope2creds(username_FXTRX, str_SCOPE)
f_build = LoggerTool.SEWrapper.info(func2logger=partial(
FoxylibLogger.func_level2logger, level=logging.DEBUG))(build)
service = f_build('sheets', 'v4', http=creds.authorize(Http()))
h = {
"spreadsheetId": gss_id,
"range": sheet_range,
}
result = service.spreadsheets().values().get(**h).execute()
values = result.get('values', [])
logger.info("values(# of lines: {0})".format(len(values)))
return values
def table_ll2j_pair(cls, ll_IN):
logger = FoxylibLogger.func_level2logger(cls.table_ll2j_pair,
logging.DEBUG)
logger.info({"# rows": len(ll_IN)})
ll_RECT = cls._table_ll2rectangle(ll_IN)
str_list_HEAD, str_COL_list_ROW_list = ll_RECT[0], ll_RECT[1:]
cls.ColHead.str_list2check_unique(str_list_HEAD)
col_count = len(str_list_HEAD)
j_colhead_list = [
cls.ColHead.parse_str2j_colhead(str_list_HEAD[k])
for k in range(col_count)
]
cls.data2check_unique(j_colhead_list, str_COL_list_ROW_list)
j_row_list_raw = [
cls.str_list2j_row(j_colhead_list, str_COL_list_ROW)
for str_COL_list_ROW in str_COL_list_ROW_list
]
j_row_list = lfilter(bool, j_row_list_raw)
logger.info({"j_row_list[0]": j_row_list[0]})
return j_colhead_list, j_row_list
def _load_urls2app(cls, app, auth0):
logger = FoxylibLogger.func_level2logger(cls._load_urls2app,
logging.DEBUG)
# callback_url = "/auth0/callback"
FlaskTool.add_url2app(app,
cls.V.URL_CALLBACK,
partial_n_wraps(cls.auth02callback, auth0),
methods=[
"GET",
])
FlaskTool.add_url2app(app,
cls.V.URL_LOGIN,
partial_n_wraps(
Auth0Tool.auth0_callback_url2login,
auth0,
cls.abspath2url(cls.V.URL_CALLBACK),
),
methods=[
"GET",
])
FlaskTool.add_url2app(app,
cls.V.URL_DASHBOARD,
cls.dashboard,
methods=[
"GET",
])
def batchrun(cls, f_batch, args, kwargs, cache, indexes_each, key, lock=None):
logger = FoxylibLogger.func_level2logger(cls.batchrun, logging.DEBUG)
# key = key if key else cachetools.keys.hashkey
def args_kwargs2k_list(_key, _indexes_each, _args, _kwargs):
_args_list = FunctionTool.args2split(_args, _indexes_each)
return [key(*_args_each, **_kwargs) for _args_each in _args_list]
k_list = args_kwargs2k_list(key, indexes_each, args, kwargs)
n = len(k_list)
h_i2v_missing = cls.batchrun_missing(f_batch, args, kwargs, cache, indexes_each, k_list, lock=lock)
def index2value(index):
# raise Exception({"index": index, "k_list":k_list, "h_i2v_missing": h_i2v_missing})
k = k_list[index]
if index not in h_i2v_missing:
return CacheTool.k2get(cache, k, lock=lock)
else:
v = h_i2v_missing[index]
CacheTool.k2set(cache, k, v, lock=lock)
return v
v_list = lmap(index2value, range(n))
# logger.debug({"hex(id(cache))":hex(id(cache)), "cache":cache, "h_i2v_missing":h_i2v_missing})
# raise Exception({"h_i2v_missing":h_i2v_missing,"v_list":v_list})
return v_list
def test_07(self):
logger = FoxylibLogger.func_level2logger(self.test_07, logging.DEBUG)
produced = []
consumed = []
producer_list = [partial(P2C.producer, produced) for _ in range(5)]
piper_batch_list_1 = [FunctionTool.func2batch(P2C.piper) for _ in range(7)]
piper_batch_list_2 = [FunctionTool.func2batch(P2C.piper) for _ in range(1)]
piper_batch_list_3 = [FunctionTool.func2batch(P2C.piper) for _ in range(4)]
consumer_batch_list = [FunctionTool.func2batch(partial(P2C.consumer, consumed)) for x in range(10)]
batches_list = [producer_list,
piper_batch_list_1,
piper_batch_list_2,
piper_batch_list_3,
consumer_batch_list,
]
config_list = [None,
AioPipeline.Config(dequeue_chunksize=4, dequeue_timeout=1.0),
None,
AioPipeline.Config(dequeue_chunksize=5, dequeue_timeout=1.0),
]
pipeline_coro = AioPipeline.batches_list2pipelined(batches_list, config_list=config_list)
AioTool.awaitable2result(pipeline_coro)
self.assertEqual(len(produced), len(consumed))
self.assertEqual(sorted(produced), sorted(consumed))
def test_03(self):
logger = FoxylibLogger.func_level2logger(self.test_03, logging.DEBUG)
c = MongodbToolCollection.collection_default()
c.delete_many({})
doc_in = {'a': 'a', 'b': 'b'}
doc_upsert = {'b': 'c'}
op_list = [
InsertOne(doc_in),
UpdateOne({'a': 'a'}, {"$set": doc_upsert}, upsert=True),
]
c.bulk_write(op_list)
self.assertEqual(c.count({}), 1)
doc_found = iter2singleton(c.find({}))
logger.debug({
"doc_found": doc_found,
"doc_in": doc_in,
"doc_upsert": doc_upsert,
})
self.assertNotEqual(doc_found, doc_in)
self.assertEqual(DocumentTool.doc2meta_keys_removed(doc_found), {
"a": "a",
"b": "c"
})
def cachedmethod_each(cls, self2cache, indexes_each, key=cachetools.keys.hashkey, lock=None):
logger = FoxylibLogger.func_level2logger(cls.cachedmethod_each, logging.DEBUG)
"""Decorator to wrap a function with a memoizing callable that saves
results in a cache for each obj given a list of objects.
Motivated by cachetools.cached
"""
assert_is_not_none(self2cache)
assert_is_not_none(indexes_each)
indexes_each_no_self = lmap(lambda x:x-1, indexes_each)
def wrapper(f_batch):
@wraps(f_batch)
def wrapped(self, *args, **kwargs):
cache = self2cache(self)
# logger.debug({"hex(id(cache))": hex(id(cache))})
result = CacheBatchTool.batchrun(partial(f_batch, self), args, kwargs, cache,
indexes_each_no_self, key, lock)
return result
return wrapped
return wrapper
def j_event2j_file_list(cls, j_event):
logger = FoxylibLogger.func_level2logger(cls.j_event2j_file_list,
logging.DEBUG)
logger.debug({"j_event": j_event})
j_file_list = j_event.get("files", [])
return j_file_list
def index_init_unittest(cls, settings=None, mappings=None):
logger = FoxylibLogger.func_level2logger(cls.index_init_unittest, logging.DEBUG)
client = FoxylibElasticsearch.client()
index = UnittestIndex.env2name(FoxylibEnv.env())
logger.debug({"index_unittest": index})
# ElasticsearchTool.delete_or_skip(client, index_unittest)
# settings = UnittestIndex.settings()
# mappings = UnittestIndex.mappings()
body = DictTool.filter(lambda k, v: bool(v),
{"settings": settings if settings else None,
"mappings": mappings if mappings else None,
}
)
ElasticsearchTool.create_or_skip(client, index, body=body)
query_all = {"match_all": {}}
client.delete_by_query(index, {"query": query_all}, refresh=True)
result = client.search(index, body={"query": query_all, })
assert_false(ElasticsearchTool.j_result2j_hit_list(result))
return client, index
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
loop = asyncio.get_event_loop()
rtm_client = RTMClient(token=FoxylibSlack.xoxb_token(),
run_async=True,
loop=loop)
async def inf_loop():
logger = logging.getLogger()
while 1:
try:
logger.info("Ping Pong! I'm alive")
await asyncio.sleep(900)
except asyncio.CancelledError:
break
tasks = asyncio.gather(rtm_client.start(), inf_loop())
def callback(signum, frame):
tasks.cancel()
logger.warning("Cancelling tasks...")
# loop.add_signal_handler(signal.SIGINT, callback)
signal.signal(signal.SIGINT, callback)
signal.signal(signal.SIGTERM, callback)
try:
loop.run_until_complete(tasks)
except asyncio.CancelledError as e:
logger.error(e)
finally:
logger.info("Quitting... Bye!")
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
web_client = FoxylibSlack.web_client()
channel = FoxylibChannel.V.FOXYLIB
filepath = os.path.join(FILE_DIR, "test_01.txt")
response = FilesUploadMethod.invoke(web_client, channel, filepath)
self.assertTrue(SlackResponseTool.response2is_ok(response))
j_response = SlackResponseTool.response2j_resopnse(response)
j_file = FilesUploadMethod.j_response2j_file(j_response)
logger.debug(json.dumps({"j_file":j_file}, indent=2))
self.assertEqual(SlackFile.j_file2mimetype(j_file), MimetypeTool.V.TEXT_PLAIN)
# download
url_private = SlackFile.j_file2url_private(j_file)
self.assertEqual(MimetypeTool.url2mimetype(url_private), MimetypeTool.V.TEXT_PLAIN)
token = FoxylibSlack.xoxp_token()
logger.debug({"url_private":url_private,
"token":token,
})
utf8 = SlackTool.fileurl_token2utf8(url_private, token)
logger.debug({"utf8": utf8})
# FileTool.utf82file(utf8, "/tmp/t.html")
self.assertEqual(utf8, FileTool.filepath2utf8(filepath))
# cleanup
file_id = SlackFile.j_file2id(j_file)
web_client.files_delete(**{"file":file_id})
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
cls = self.__class__
obj1 = cls.TestClass()
obj1.func2()
logger.debug({"obj1": obj1, "obj1.func2": obj1.func2})
cache1 = CacheManager.callable2cache(obj1.func2)
self.assertEqual(len(cache1), 1)
self.assertIn(tuple([]), cache1)
obj2 = cls.TestClass()
obj2.func2()
cache2 = CacheManager.callable2cache(obj2.func2)
self.assertEqual(len(cache2), 1)
self.assertIn(tuple([]), cache2)
self.assertEqual(len(cache1), 1)
self.assertIn(tuple([]), cache1)
CacheManager.add2cache(obj2.func2, 5, args=[])
self.assertEqual(obj2.func2(), 5)
def test_04(self):
logger = FoxylibLogger.func_level2logger(self.test_04, logging.DEBUG)
channel = "C014DTPM24V"
headers = {"Content-type": 'application/json',
"Authorization": "Bearer {}".format(FoxylibSlack.xoxb_token()),
}
json1 = {"channel": channel,
"text": "Hello world :tada:",
}
response1 = requests.post("https://slack.com/api/chat.postMessage",
headers=headers,
json=json1)
self.assertEqual(response1.status_code, requests.codes.ok)
j1 = response1.json()
self.assertTrue(j1.get("ok"), j1)
ts = JsonTool.down(j1, ["message", "ts"])
json2 = {"channel": channel,
"ts": ts,
}
response2 = requests.post("https://slack.com/api/chat.delete",
headers=headers,
json=json2)
self.assertEqual(response2.status_code, requests.codes.ok)
j2 = response2.json()
self.assertTrue(j2.get("ok"), j2)
def client_index_query2j_result(cls, es_client, index, j_query):
logger = FoxylibLogger.func_level2logger(
cls.client_index_query2j_result, logging.DEBUG)
logger.debug({"index": index, "j_query": j_query})
j_result = es_client.search(index, j_query)
return j_result
def test_01(self):
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
# a1 = A.f()
# a2 = A.f()
logger.debug({"START":"a1"})
a1 = next(A.f())
logger.debug({"START": "a2"})
a2 = A()
logger.debug({"START": "a3"})
a3 = next(A.f())
logger.debug({"a1":a1, "a2":a2})
self.assertEqual(a1.id, a3.id)
self.assertNotEqual(a2.id, a3.id)
self.assertEqual(a1.enter_counter, 1)
self.assertEqual(a2.enter_counter, 0)
self.assertEqual(a3.enter_counter, 1)
self.assertEqual(a1.exit_counter, 0)
self.assertEqual(a2.exit_counter, 0)
self.assertEqual(a3.exit_counter, 0)
def test_01(self): # charge
logger = FoxylibLogger.func_level2logger(self.test_01, logging.DEBUG)
secret_key = FoxylibStripe.secret_key()
# logger.debug({"secret_key": secret_key})
cardnumber = StripeCardSample.Number.VISA
j_card_test = StripeCardSample.cardnumber2j_card_test(cardnumber)
j_token_config = {
StripeTool.Config.API_KEY: secret_key,
StripeToken.Config.CARD: j_card_test,
}
token = stripe.Token.create(**j_token_config)
logger.debug({"token": token})
self.assertFalse(StripeToken.token2livemode(token))
self.assertEqual(StripeToken.token2type(token), StripeTokenType.V.CARD)
card_token = StripeToken.token2card(token)
self.assertEqual(StripeCard.card2last4(card_token), cardnumber[-4:])
amount = 100
currency = "usd"
j_charge_config = {
StripeTool.Config.API_KEY: secret_key,
StripeCharge.Config.AMOUNT: 100,
StripeCharge.Config.CURRENCY: currency,
StripeCharge.Config.SOURCE: token,
}
charge = stripe.Charge.create(**j_charge_config)
logger.debug({"charge": charge})
self.assertEqual(StripeCharge.charge2amount(charge), amount)
self.assertEqual(StripeCharge.charge2currency(charge), currency)
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
j_tier_list = OverwatchTier.j_list_all()
for j_tier in j_tier_list:
self.assertTrue(
URLTool.url2is_accessible(OverwatchTier.doc2image_url(j_tier)))
def video_id2live_streaming_data(cls, video_id, credentials):
logger = FoxylibLogger.func_level2logger(
cls.video_id2live_streaming_data, logging.DEBUG)
# https://stackoverflow.com/questions/36683878/youtube-api-how-do-i-get-the-livechatid
# https://developers.google.com/youtube/v3/docs/videos/list
from foxylib.tools.googleapi.foxylib_googleapi import FoxylibGoogleapi
# credentials = FoxylibGoogleapi.ServiceAccount.credentials()
service = YoutubeapiTool.credentials2service(credentials)
request = service.videos().list(
id=video_id,
part='liveStreamingDetails',
)
response = request.execute()
logger.debug({"response": response})
items = response.get("items")
if not items:
return None
item = l_singleton2obj(items)
return item.get("liveStreamingDetails")
def xoxp_token(cls):
logger = FoxylibLogger.func_level2logger(cls.xoxp_token, logging.DEBUG)
token = FoxylibEnv.key2value("SLACK_OAUTH_ACCESS_TOKEN")
# logger.debug({"token": token})
return token
