def func_iter2buffered_result_iter(cls, func_iter, buffer_size):
logger = FoxylibLogger.func2logger(cls.func_iter2buffered_result_iter)
# be careful because Pool() object should be able to see the target function definition
# https://stackoverflow.com/questions/2782961/yet-another-confusion-with-multiprocessing-error-module-object-has-no-attribu
with Pool() as pool:
yield from cls.pool_func_iter2buffered_result_iter(pool, func_iter, buffer_size)
def func2threaded(
cls,
func=None,
max_workers=None,
):
logger = FoxylibLogger.func2logger(cls.func2threaded)
def wrapper(f):
@wraps(f)
def wrapped(*args, **kwargs):
executor = ThreadPoolExecutor(
max_workers=max_workers) # non-blocking
def f_new(*args, **kwargs):
rv = f(*args, **kwargs)
logger.info({"message": "func2thread", "value": rv})
LoggerTool.logger2flush_handlers(logger)
return rv
future = executor.submit(f_new, *args, **kwargs)
# future.add_done_callback(lambda x: rs.setex(name, time, x.result()))
return future
return wrapped
return wrapper(func) if func else wrapper
def delete(cls, es_client, alias):
logger = FoxylibLogger.func2logger(cls.create)
index_list = cls.alias2indexes(es_client, alias)
if index_list is None: return
return es_client.indices.delete_alias(index=",".join(index_list),
name=alias)
def iter_batches2yoo(cls, iter, batch_chunksize_list):
logger = FoxylibLogger.func_level2logger(cls.iter_batches2yoo, logging.DEBUG)
iter_out = reduce(lambda x_iter, fp: cls.iter_batch2yoo(x_iter, fp[0], fp[1]),
batch_chunksize_list,
iter)
yield from iter_out
def wrapped(*args, **kwargs):
logger = FoxylibLogger.func2logger(wrapped)
# logger.debug({"func": func, "args": args, "kwargs": kwargs, })
_args = tuple([f_serialize(arg) for arg in args])
_kwargs = {k: f_serialize(v) for k, v in kwargs.items()}
return cached_func(*_args, **_kwargs)
def _node2rstr_unnamed(
cls,
node,
ancestors,
args=None,
kwargs=None,
):
logger = FoxylibLogger.func2logger(cls._node2rstr_unnamed)
_args = args or []
_kwargs = kwargs or {}
# logger.debug({"node": node, "args": args, "kwargs": kwargs, "type":cls.node2type(node),
# "h_node2ak": h_node2ak,
# })
if cls.node2type(node) == cls.Type.RSTR_NODE:
rstr = node.rstr(*_args, **_kwargs)
return rstr
subnode_list = node.subnode_list()
ancestors_and_me = lchain(ancestors, [node])
rstr_list_subnode = [
cls._node2rstr_named(sn,
ancestors_and_me,
args=args,
kwargs=kwargs) for sn in subnode_list
]
str_format = node.rformat(*_args, **_kwargs)
rstr = format_str(str_format, *rstr_list_subnode)
return rstr
def wait_all(cls, f_list, sec_timeout, sec_interval):
logger = FoxylibLogger.func_level2logger(cls.wait_all, logging.DEBUG)
time_end = time.time() + sec_timeout if sec_timeout is not None else None
n = len(f_list)
status_list = [None] * n
logger.debug(format_str("waiting for {} process for {} secs", len(f_list), sec_timeout))
while (time_end is None) or (time.time() < time_end):
for i in range(n):
if status_list[i] is True:
continue
status_list[i] = f_list[i]()
if all(status_list):
break
logger.debug(format_str("waiting for {}/{} processes for {} secs with {} sec interval",
len(lfilter(lambda x: not x, status_list)),
len(f_list),
"{:.3f}".format(time_end - time.time()),
sec_interval,
))
time.sleep(sec_interval)
return status_list
def bulk(
cls,
es_client,
j_action_list,
run_bulk=True,
es_kwargs=None,
):
logger = FoxylibLogger.func2logger(cls.bulk)
n = len(j_action_list)
count_list = [n * i // 100 for i in range(100)]
_run_bulk = run_bulk and n > 1
if _run_bulk:
return bulk(es_client, j_action_list, **es_kwargs)
else:
result_list = []
for i, j_action in enumerate(j_action_list):
if i in count_list:
logger.debug({
"i/n": "{}/{}".format(i + 1, n),
# "j_action":j_action,
})
# raise Exception()
op_type = cls.j_action2op_type(j_action)
if op_type == "index":
result = cls._j_action2op_index(es_client,
j_action,
es_kwargs=es_kwargs)
result_list.append(result)
else:
raise NotImplementedError()
return result_list
def wrapped(*_, **__):
_logger = logger if logger else FoxylibLogger.func2logger(f)
try:
return f(*_, **__)
except Exception as e:
_logger.exception(err2msg(e))
raise
def filepath2j(cls, filepath):
logger = FoxylibLogger.func2logger(cls.filepath2j)
utf8 = FileTool.filepath2utf8(filepath)
# logger.info({"utf8": utf8})
j = yaml.load(utf8)
return j
def func_list2result_list_OLD(cls, func_list):
logger = FoxylibLogger.func2logger(cls.func_list2result_list)
with Pool() as pool:
ar_list = [pool.apply_async(f) for f in func_list]
output_list = [ar.get() for ar in ar_list]
return output_list
def func_list2result_list(cls, func_list):
logger = FoxylibLogger.func2logger(cls.func_list2result_list)
logger.debug({"# func_list": len(func_list)})
output_iter = cls.func_list2buffered_result_iter(func_list, len(func_list))
result_list = list(output_iter)
logger.debug({"# result_list":len(result_list)})
return result_list
def filepath2j(cls, filepath):
logger = FoxylibLogger.func_level2logger(cls.filepath2j, logging.DEBUG)
from foxylib.tools.file.file_tool import FileTool
utf8 = FileTool.filepath2utf8(filepath)
if not utf8: return None
j = json.loads(utf8)
return j
def f_or_file2iter(cls, f, filepath):
logger = FoxylibLogger.func2logger(cls.f_or_file2iter)
# logger.debug({"filepath": filepath, "f": f})
f_str = lambda: "\n".join(list(f()))
utf8 = cls.f_or_file2utf8(f_str, filepath)
if utf8 is None: return None
return utf8.splitlines()
def _h_node2args_kwargs(cls, h, node):
if not h: return [], {}
logger = FoxylibLogger.func2logger(cls._h_node2args_kwargs)
args_kwargs = h.get(node)
# logger.debug({"h": h, "node": node, "args_kwargs":args_kwargs})
if not args_kwargs: return [], {}
return args_kwargs
def merge_list(cls, j_list):
logger = FoxylibLogger.func2logger(cls.merge_list)
#logger.info("j_list({0})".format(json.dumps(j_list, ensure_ascii=False)))
if not j_list: return None
j1 = copy.deepcopy(j_list[0])
j_MERGED = reduce(lambda j_BASE, j: cls.merge2(j_BASE, j), j_list[1:],
j1)
return j_MERGED
def deserialize_and_func(*args, **kwargs):
logger = FoxylibLogger.func2logger(deserialize_and_func)
_args = tuple([f_deserialize(arg) for arg in args])
_kwargs = {
k: f_deserialize(v)
for k, v in six.viewitems(kwargs)
}
# logger.debug({"func": func, "args": args, "_args": _args, "kwargs": kwargs, "_kwargs": _kwargs, })
return func(*_args, **_kwargs)
def _node2rstr_named(cls, node, ancestors, args=None, kwargs=None):
logger = FoxylibLogger.func2logger(cls._node2rstr_named)
# logger.debug({"node":node, "ancestors": ancestors, })
rstr_unnamed = cls._node2rstr_unnamed(node,
ancestors,
args=args,
kwargs=kwargs)
rstr_named = RegexTool.name_rstr2named(
cls._node_parents2name(node, ancestors), rstr_unnamed)
return rstr_named
def join_html(cls, delim, l):
logger = FoxylibLogger.func_level2logger(cls.join_html, logging.DEBUG)
# delim_safe = cls.escape(delim)
# html_delim = Markup(delim)
logger.debug({
"delim": delim,
"l": l,
})
html = cls.escape(delim).join(lmap(cls.escape, l))
return Markup(html)
def alias2indexes(cls, es_client, alias):
logger = FoxylibLogger.func2logger(cls.alias2indexes)
try:
j_result = es_client.indices.get_alias(name=alias)
except NotFoundError:
return None
index_list = list(j_result.keys())
return index_list
def node2rstr(
cls,
node,
named=True,
args=None,
kwargs=None,
):
logger = FoxylibLogger.func2logger(cls.node2rstr)
# logger.debug({"node":node, "ancestors": ancestors, })
if named:
return cls._node2rstr_named(node, [], args=args, kwargs=kwargs)
else:
return cls._node2rstr_unnamed(node, [], args=args, kwargs=kwargs)
def env2client(cls, *_, **__):
logger = FoxylibLogger.func2logger(cls.env2client)
auth = cls.env2auth()
host = cls.env2host()
logger.info({"auth": auth, "host": host})
if auth:
return Elasticsearch([auth], *_, **__)
if host:
return Elasticsearch([host], *_, **__)
raise Exception("ELASTICSEARCH_HOST not defined")
def f_or_file2utf8(cls, f, filepath):
logger = FoxylibLogger.func2logger(cls.f_or_file2utf8)
# logger.debug({"filepath": filepath, "f": f})
FileTool.dirpath2mkdirs(os.path.dirname(filepath))
utf8 = FileTool.filepath2utf8(filepath)
if utf8:
return utf8
utf8 = f()
if utf8 is not None:
FileTool.utf82file(utf8, filepath)
return utf8
def parse_str2reldelta(cls, s):
logger = FoxylibLogger.func2logger(cls.parse_str2reldelta)
p = cls.pattern_timedelta()
m_list = list(p.finditer(s))
if not m_list: return None
m = l_singleton2obj(m_list)
int_sign = IntToolkit.parse_sign2int(m.group("sign"))
kv_list = [(k, int_sign*int(m.group(k)))
for k in cls.reldelta_name_list() if m.group(k)]
logger.debug({"kv_list":kv_list})
reldelta = relativedelta.relativedelta(**dict(kv_list))
return reldelta
def append_query2url(cls, url, h_query_in=None):
logger = FoxylibLogger.func2logger(cls.append_query2url)
if not h_query_in:
return url
url_parts = list(urllib.parse.urlparse(url))
h_query_ori = dict(urllib.parse.parse_qsl(url_parts[4]))
h_query = merge_dicts([h_query_ori, h_query_in,], vwrite=vwrite_overwrite)
# logger.debug({"h_query":h_query, "h_query_ori":h_query_ori, "url_parts":url_parts,})
url_parts[4] = urllib.parse.urlencode(h_query)
url_paramed = urllib.parse.urlunparse(url_parts)
url_utf8 = cls.url2utf8_safe(url_paramed)
return url_utf8
def iter_batch_cond2processed(cls, x_iter, f_batch, f_cond, size_minimax,):
logger = FoxylibLogger.func_level2logger(cls.iter_batch_cond2processed, logging.DEBUG)
def inputs_indexes2y_list(x_list, i_list):
x_list_target = lmap(lambda i: x_list[i], i_list)
y_list_target = f_batch(x_list_target)
h_i2k = {i: k for k, i in enumerate(i_list)}
y_list = [x if i not in h_i2k else y_list_target[h_i2k[i]]
for i, x in enumerate(x_list)]
return y_list
size_min, size_max = size_minimax
x_list_buffer = []
i_list_target = []
for x in x_iter:
is_buffer_empty = list2singleton([not x_list_buffer, not i_list_target])
is_target = (f_cond is None) or f_cond(x)
if (not is_target) and is_buffer_empty:
yield x
continue
if is_target:
i_list_target.append(len(x_list_buffer))
x_list_buffer.append(x)
run_batch = len(x_list_buffer)>=size_max or len(i_list_target)>=size_min
if not run_batch:
continue
if not is_buffer_empty:
y_list_buffer = inputs_indexes2y_list(x_list_buffer, i_list_target)
logger.debug({"# i_list_target": len(i_list_target),
"# y_list_buffer": len(y_list_buffer),})
yield from y_list_buffer
x_list_buffer = []
i_list_target = []
is_buffer_empty = list2singleton([not x_list_buffer, not i_list_target])
if not is_buffer_empty:
yield from inputs_indexes2y_list(x_list_buffer, i_list_target)
def iter2buffered(cls, iter, buffer_size):
logger = FoxylibLogger.func2logger(cls.iter2buffered)
if not buffer_size:
yield from iter
else:
l = deque()
for x in iter:
l.append(x)
# logger.debug({"len(l)":len(l), "buffer_size":buffer_size,})
while len(l) > buffer_size:
yield l.popleft()
while l:
yield l.popleft()
def utf82file(
cls,
utf8,
filepath,
encoding="utf-8",
f_open=None,
):
logger = FoxylibLogger.func2logger(cls.utf82file)
if f_open is None:
f_open = lambda filepath: codecs.open(
filepath, "w", encoding=encoding)
OUT_DIR = os.path.dirname(filepath)
if not os.path.exists(OUT_DIR): os.makedirs(OUT_DIR)
if os.path.islink(filepath): os.unlink(filepath)
with f_open(filepath) as f:
if utf8:
print(utf8, file=f)
def pattern_timedelta(cls):
logger = FoxylibLogger.func2logger(cls.pattern_timedelta)
j_yaml = cls.yaml()
reldalta_name_list = cls.reldelta_name_list()
j_reldelta = j_yaml["relativedelta"]
j_name2strs = lambda j: lchain.from_iterable(j.values())
rstr_reldelta_list = [format_str(r"(?:(?P<{0}>\d+)\s*(?:{1}))?",
k,
r"|".join(lmap(re.escape, j_name2strs(j_reldelta[k]))),
)
for k in reldalta_name_list]
rstr_reldeltas = r"\s*".join([r"(?:{0})".format(rstr) for rstr in rstr_reldelta_list])
rstr = r"\s*".join([r"(?P<sign>[+-])", rstr_reldeltas])
logger.debug({"rstr":rstr})
pattern = re.compile(rstr, re.IGNORECASE)
return pattern
def es2j_hit_iter_scroll(
cls,
es_client,
es_index,
jq,
scroll,
):
logger = FoxylibLogger.func2logger(cls.es2j_hit_iter_scroll)
j_result = es_client.search(es_index, jq, scroll=scroll)
scroll_id = cls.j_result2scroll_id(j_result)
j_hit_list_this = ESToolkit.j_result2j_hit_list(j_result)
# count_result = len(j_hit_list_this)
yield from j_hit_list_this
while j_hit_list_this:
j_result = es_client.scroll(scroll_id=scroll_id, scroll=scroll)
scroll_id = cls.j_result2scroll_id(j_result)
j_hit_list_this = ESToolkit.j_result2j_hit_list(j_result)
yield from j_hit_list_this