def table_pagesize2split_OLD(cls, table, ncol_per_page):
buffer = cls.COUNT_COLHEAD
ncol_overlap = 1 # overlapping column 1
n_row = len(table)
divider = ncol_per_page - buffer - ncol_overlap
ncol_table = iter2singleton(map(len, table))
ncol_data = ncol_table - buffer
n_page = (ncol_data - ncol_overlap) // divider + (
1 if (ncol_data - ncol_overlap) % divider else 0)
cols_header = lmap(lambda l: l[:cls.COUNT_COLHEAD], table)
for i in range(n_page):
start = buffer + i * divider
end = buffer + min((i + 1) * divider + 1, ncol_data)
cols_body = lmap(
lambda l: SpanTool.list_span2sublist(l, (start, end)), table)
table_partial = [
cols_header[i] + cols_body[i] for i in range(n_row)
if any(cols_body[i])
]
yield table_partial
def test_03(self):
logger = FoxylibLogger.func_level2logger(self.test_03, logging.DEBUG)
c = MongodbToolCollection.collection_default()
c.delete_many({})
doc_in = {'a': 'a', 'b': 'b'}
doc_upsert = {'b': 'c'}
op_list = [
InsertOne(doc_in),
UpdateOne({'a': 'a'}, {"$set": doc_upsert}, upsert=True),
]
c.bulk_write(op_list)
self.assertEqual(c.count({}), 1)
doc_found = iter2singleton(c.find({}))
logger.debug({
"doc_found": doc_found,
"doc_in": doc_in,
"doc_upsert": doc_upsert,
})
self.assertNotEqual(doc_found, doc_in)
self.assertEqual(DocumentTool.doc2meta_keys_removed(doc_found), {
"a": "a",
"b": "c"
})
def table2col_trimmed(cls, table):
m = iter2singleton(map(len, table))
colindex_list_valid = lfilter(
lambda j: not cls.table_colindex2is_empty(table, j),
range(cls.COUNT_COLHEAD, m))
return cls.table_colindexes2filtered(table, colindex_list_valid)
def _col_index_extended2str_list(cls, str_ll, k_col, i_pivot, multiple):
logger = FoxylibLogger.func_level2logger(
cls._col_index_extended2str_list, logging.DEBUG)
n_row = len(str_ll)
n_col_raw = iter2singleton(map(len, str_ll))
j_col = cls._col_index_extended2raw(k_col, multiple)
assert_less(j_col, n_col_raw)
n_col_extended = cls.col_count_raw2extended(n_col_raw, multiple)
assert_less(k_col, n_col_extended)
if not cls._index2is_data(j_col):
return [str_ll[i][j_col] for i in range(n_row)]
offset = cls._col_index_extended2offset(k_col, multiple)
def interpolate(str_ll, i, j_col, p):
assert_less(j_col, n_col_extended)
s1 = str_ll[i][j_col]
if not s1: return s1
v1 = float(s1)
if not p: return v1
# logger.debug({"n_col_raw":n_col_raw,
# "n_col_extended":n_col_extended,
# "multiple": multiple,
# "i":i,
# "j_col":j_col,
# "p":p,
# "k_col":k_col,
# "offset":offset,
# })
s2 = str_ll[i][j_col + 1]
if not s2: return s2
v2 = float(s2)
return v1 + (v2 - v1) * p
p = offset / multiple
v_list_col = [
interpolate(str_ll, i, j_col, p) for i in range(1, n_row)
]
str_time = cls._colindex_extended2str_time(str_ll[0], k_col, multiple)
v_pivot = v_list_col[i_pivot - 1]
count_before = IterTool.count(
filter(lambda v: v and v > v_pivot, v_list_col))
rank = count_before + 1
count_total = IterTool.count(filter(bool, v_list_col))
str_title = "{} (#{}/{})".format(str_time, rank, count_total)
return [str_title] + v_list_col
def table_titles2subbed(cls, table, title_list):
n_col = iter2singleton(map(len, table))
assert_equal(n_col, len(title_list) + cls.COUNT_COLHEAD)
l_row_top = lchain(
table[0][:cls.COUNT_COLHEAD],
title_list,
)
return lchain([l_row_top], table[1:])
def index_values_beam2neighbor_indexes(cls, i_pivot, v_list, beam):
v_count = len(v_list)
i_list_sorted = sorted(range(v_count), key=lambda i: v_list[i])
k_pivot = iter2singleton(
filter(lambda k: i_list_sorted[k] == i_pivot, range(v_count)))
k_span = cls.index_total_beam2span(k_pivot, v_count, beam)
i_sublist = cls.list_span2sublist(i_list_sorted, k_span)
return i_sublist
def clique2type(cls, clique):
has_rate = iter2singleton(
map(is_not_none, [
cls.clique2rate(clique),
cls.clique2trend(clique),
]))
if has_rate:
return cls.Type.UPDATE
return cls.Type.LOOKUP
def str_ll2h_list(cls, str_ll):
assert_greater_equal(len(str_ll), 1)
m = iter2singleton(map(len, str_ll))
key_list = str_ll[0]
h_list = [
merge_dicts([{
key_list[j]: l[j]
} for j in range(m)],
vwrite=vwrite_no_duplicate_key) for l in str_ll[1:]
]
return h_list
def table_colspan2fresh_rows_filtered(cls, table, colspan):
logger = FoxylibLogger.func_level2logger(
cls.table_colspan2fresh_rows_filtered, logging.DEBUG)
i_list_starting = cls.table_colspan2rowindex_list_starting(
table, colspan)
table_row_filtered = cls.table_rowindexes2filtered(
table, i_list_starting)
logger.debug({
"colspan":
colspan,
"len(table)":
len(table),
"iter2singleton(map(len,table))":
iter2singleton(map(len, table)),
"len(i_list_starting)":
len(i_list_starting),
"len(table_row_filtered)":
len(table_row_filtered),
"iter2singleton(map(len,table_row_filtered))":
iter2singleton(map(len, table_row_filtered)),
})
return table_row_filtered
def _str_ll_i2ij_iter(cls, str_ll, i_pivot, beam):
logger = FoxylibLogger.func_level2logger(cls._str_ll_i2ij_iter,
logging.DEBUG)
# raise Exception({"lmap(len, str_ll)":lmap(len, str_ll)})
col_count = iter2singleton(map(len, str_ll))
for j in range(cls.COUNT_COLHEAD, col_count):
logger.debug({"j": j})
if not str_ll[i_pivot][j]: # empty column
continue
str_col_list = lmap(lambda l: l[j], str_ll)
i_iter = cls._index_strs_beam2index_iter(str_col_list, i_pivot,
beam)
for i in i_iter:
yield (i, j)
async def funcs_list2pipelined(cls, funcs_list, queue_list=None):
assert_true(funcs_list)
n = len(funcs_list)
if queue_list:
assert_equal(len(queue_list), n - 1)
batches_producer = funcs_list[0]
batches_list_after = [[lambda l: f(iter2singleton(l)) for f in funcs]
for funcs in funcs_list[1:]]
batches_list = [batches_producer, *batches_list_after]
config_list = [
AioPipeline.Config(
f_queue=lambda: queue_list[i] if queue_list else None)
for i in range(n - 1)
]
return await cls.batches_list2pipelined(batches_list,
config_list=config_list)
def table2percentage(cls, table, v):
logger = FoxylibLogger.func_level2logger(cls.table2percentage,
logging.DEBUG)
count_col = iter2singleton(map(len, table))
h_j2col_sum = {
j: sum(map(int, filter(bool, map(ig(j), table[1:]))))
for j in range(cls.COUNT_COLHEAD, count_col)
}
for i, l in enumerate(table):
if i == 0:
yield l
continue
# logger.debug({"l":l})
l_head = l[:cls.COUNT_COLHEAD]
l_right = [
"{:.02f}".format(int(l[j]) * 100 /
h_j2col_sum[j]) if l[j] else l[j]
for j in range(cls.COUNT_COLHEAD, count_col)
]
yield lchain(l_head, l_right)
def test_02(self):
logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG)
c = MongodbToolCollection.collection_default()
c.delete_many({})
doc_in = {'a': 'a', 'b': 'b'}
op_list = [
InsertOne(doc_in),
]
c.bulk_write(op_list)
doc_found = iter2singleton(c.find({}))
# hyp1 = DictTool.keys2excluded(doc_found, [MongoDBTool.Field._ID])
logger.debug({"doc_found": doc_found, "doc_in": doc_in})
self.assertEqual(doc_found, doc_in)
self.assertTrue(isinstance(doc_in[MongoDBTool.Field._ID],
ObjectId)) # doc_in updated!!
self.assertTrue(isinstance(doc_found[MongoDBTool.Field._ID], ObjectId))
hyp3 = set(doc_found.keys())
ref3 = {'a', '_id', 'b'}
self.assertEqual(hyp3, ref3)
def str_ll2interpolated_list(cls, str_ll_in, i_pivot, multiple):
logger = FoxylibLogger.func_level2logger(cls.str_ll2interpolated_list,
logging.DEBUG)
n_row = len(str_ll_in)
n_col_raw = iter2singleton(map(len, str_ll_in))
n_col_extended = cls.col_count_raw2extended(n_col_raw, multiple)
logger.debug({
"n_col_raw": n_col_raw,
"n_col_extended": n_col_extended,
"multiple": multiple,
})
# raise Exception()
ll_tr = [
cls._col_index_extended2str_list(str_ll_in, k, i_pivot, multiple)
for k in range(n_col_extended)
]
ll_out = [[ll_tr[k][i] for k in range(n_col_extended)]
for i in range(n_row)]
return ll_out
def entity_group2parameter_type(cls, entity_list):
entity_types = map(FoxylibEntity.entity2type, entity_list)
return iter2singleton(
map(cls.entity_type2parameter_type, entity_types))
def codenames(cls):
codename_sets = [set(h_codename2aliases.keys())
for lang, h_codename2aliases in cls.dict_lang2codename2aliases().items()]
return iter2singleton(codename_sets)
def table2col_count(cls, ll):
return iter2singleton(map(len, ll))