def wait_all(cls, f_list, sec_timeout, sec_interval):
logger = FoxylibLogger.func_level2logger(cls.wait_all, logging.DEBUG)
time_end = time.time() + sec_timeout if sec_timeout is not None else None
n = len(f_list)
status_list = [None] * n
logger.debug(format_str("waiting for {} process for {} secs", len(f_list), sec_timeout))
while (time_end is None) or (time.time() < time_end):
for i in range(n):
if status_list[i] is True:
continue
status_list[i] = f_list[i]()
if all(status_list):
break
logger.debug(format_str("waiting for {}/{} processes for {} secs with {} sec interval",
len(lfilter(lambda x: not x, status_list)),
len(f_list),
"{:.3f}".format(time_end - time.time()),
sec_interval,
))
time.sleep(sec_interval)
return status_list
def iter_batches2yoo(cls, iter, batch_chunksize_list):
logger = FoxylibLogger.func_level2logger(cls.iter_batches2yoo, logging.DEBUG)
iter_out = reduce(lambda x_iter, fp: cls.iter_batch2yoo(x_iter, fp[0], fp[1]),
batch_chunksize_list,
iter)
yield from iter_out
def filepath2j(cls, filepath):
logger = FoxylibLogger.func_level2logger(cls.filepath2j, logging.DEBUG)
from foxylib.tools.file.file_tool import FileTool
utf8 = FileTool.filepath2utf8(filepath)
if not utf8: return None
j = json.loads(utf8)
return j
def join_html(cls, delim, l):
logger = FoxylibLogger.func_level2logger(cls.join_html, logging.DEBUG)
# delim_safe = cls.escape(delim)
# html_delim = Markup(delim)
logger.debug({
"delim": delim,
"l": l,
})
html = cls.escape(delim).join(lmap(cls.escape, l))
return Markup(html)
def iter_batch_cond2processed(cls, x_iter, f_batch, f_cond, size_minimax,):
logger = FoxylibLogger.func_level2logger(cls.iter_batch_cond2processed, logging.DEBUG)
def inputs_indexes2y_list(x_list, i_list):
x_list_target = lmap(lambda i: x_list[i], i_list)
y_list_target = f_batch(x_list_target)
h_i2k = {i: k for k, i in enumerate(i_list)}
y_list = [x if i not in h_i2k else y_list_target[h_i2k[i]]
for i, x in enumerate(x_list)]
return y_list
size_min, size_max = size_minimax
x_list_buffer = []
i_list_target = []
for x in x_iter:
is_buffer_empty = list2singleton([not x_list_buffer, not i_list_target])
is_target = (f_cond is None) or f_cond(x)
if (not is_target) and is_buffer_empty:
yield x
continue
if is_target:
i_list_target.append(len(x_list_buffer))
x_list_buffer.append(x)
run_batch = len(x_list_buffer)>=size_max or len(i_list_target)>=size_min
if not run_batch:
continue
if not is_buffer_empty:
y_list_buffer = inputs_indexes2y_list(x_list_buffer, i_list_target)
logger.debug({"# i_list_target": len(i_list_target),
"# y_list_buffer": len(y_list_buffer),})
yield from y_list_buffer
x_list_buffer = []
i_list_target = []
is_buffer_empty = list2singleton([not x_list_buffer, not i_list_target])
if not is_buffer_empty:
yield from inputs_indexes2y_list(x_list_buffer, i_list_target)
def tmplt_file2str(cls, filepath, data=None, autoescape=None):
logger = FoxylibLogger.func_level2logger(cls.tmplt_file2str,
logging.DEBUG)
str_tmplt = FileTool.filepath2utf8(filepath)
# logger.debug({"filepath":filepath,"str_tmplt": str_tmplt})
return cls.tmplt_str2str(str_tmplt, data=data, autoescape=autoescape)
def iter_batch2yoo(cls, iter, f_batch, chunk_size):
logger = FoxylibLogger.func_level2logger(cls.iter_batch2yoo, logging.DEBUG)
for x_list_chunk in cls.chunk_size2chunks(iter, chunk_size):
f_batch(x_list_chunk)
yield from x_list_chunk
def iter2backoff_batches(cls, iter, batch_chunksize_list, buffer_size):
logger = FoxylibLogger.func_level2logger(cls.iter2backoff_batches, logging.DEBUG)
n = buffer_size
m = len(batch_chunksize_list)
queue_list = [[] for _ in range(m)]
h_i2j = {}
h_i2out = {}
buffer_in = [None] * (n+1)
# f_batch_list = lmap(ig(0), batch_chunksize_list)
# chunksize_list = lmap(ig(1), batch_chunksize_list)
def i2next(i): return (i+1) % (n+1)
def append2j(i,j):
h_i2j[i] = j
queue_list[j].append(i)
def iy2out(i,y):
h_i2out[i] = y
h_i2j.pop(i)
def i2j_new(i, j_old):
j_new = j_old +1
h_i2j[i] = j_new
queue_list[j_new].append(i)
def j2batch(j):
if not queue_list[j]:
return
f_batch, chunksize = batch_chunksize_list[j]
i_list = queue_list[j][:chunksize]
queue_list[j] = queue_list[j][chunksize:]
x_list = [buffer_in[i] for i in i_list]
by_list = f_batch(x_list)
if j == m-1:
ix_list = [(i, buffer_in[i])
for i, (b, y) in zip_strict(i_list, by_list)
if not b]
assert_false(ix_list)
for i,(b,y) in zip_strict(i_list,by_list):
if b:
iy2out(i,y)
else:
i2j_new(i,j)
logger.debug({"step": "j2batch", "j": j, "h_i2out":h_i2out, "h_i2j":h_i2j,})
def i2yield(i):
assert_in(i, h_i2out)
assert_not_in(i, h_i2j)
y = h_i2out.pop(i)
logger.debug({"step": "i2yield", "i": i, "y":y})
return y
i_head = -1
i_tail = -1
for x in iter:
i_head = i2next(i_head)
logger.debug({"i_head":i_head})
assert_not_in(i_head, h_i2j)
buffer_in[i_head] = x
append2j(i_head, 0)
if len(h_i2j) + len(h_i2out)<n:
continue
i_tail = i2next(i_tail)
logger.debug({"i_tail": i_tail, "h_i2j":h_i2j, "h_i2out":h_i2out,})
assert_not_equal(i_tail in h_i2j, i_tail in h_i2out)
if i_tail in h_i2out:
yield i2yield(i_tail)
continue
j_tail = h_i2j[i_tail]
for j in range(j_tail,m):
j2batch(j)
if i_tail not in h_i2out: # not done
continue
yield i2yield(i_tail)
break
for j in range(m):
while queue_list[j]:
j2batch(j)
assert_false(h_i2j)
while h_i2out:
i_tail = i2next(i_tail)
yield i2yield(i_tail)