def table_pagesize2split_OLD(cls, table, ncol_per_page): buffer = cls.COUNT_COLHEAD ncol_overlap = 1 # overlapping column 1 n_row = len(table) divider = ncol_per_page - buffer - ncol_overlap ncol_table = iter2singleton(map(len, table)) ncol_data = ncol_table - buffer n_page = (ncol_data - ncol_overlap) // divider + ( 1 if (ncol_data - ncol_overlap) % divider else 0) cols_header = lmap(lambda l: l[:cls.COUNT_COLHEAD], table) for i in range(n_page): start = buffer + i * divider end = buffer + min((i + 1) * divider + 1, ncol_data) cols_body = lmap( lambda l: SpanTool.list_span2sublist(l, (start, end)), table) table_partial = [ cols_header[i] + cols_body[i] for i in range(n_row) if any(cols_body[i]) ] yield table_partial
def test_03(self): logger = FoxylibLogger.func_level2logger(self.test_03, logging.DEBUG) c = MongodbToolCollection.collection_default() c.delete_many({}) doc_in = {'a': 'a', 'b': 'b'} doc_upsert = {'b': 'c'} op_list = [ InsertOne(doc_in), UpdateOne({'a': 'a'}, {"$set": doc_upsert}, upsert=True), ] c.bulk_write(op_list) self.assertEqual(c.count({}), 1) doc_found = iter2singleton(c.find({})) logger.debug({ "doc_found": doc_found, "doc_in": doc_in, "doc_upsert": doc_upsert, }) self.assertNotEqual(doc_found, doc_in) self.assertEqual(DocumentTool.doc2meta_keys_removed(doc_found), { "a": "a", "b": "c" })
def table2col_trimmed(cls, table): m = iter2singleton(map(len, table)) colindex_list_valid = lfilter( lambda j: not cls.table_colindex2is_empty(table, j), range(cls.COUNT_COLHEAD, m)) return cls.table_colindexes2filtered(table, colindex_list_valid)
def _col_index_extended2str_list(cls, str_ll, k_col, i_pivot, multiple): logger = FoxylibLogger.func_level2logger( cls._col_index_extended2str_list, logging.DEBUG) n_row = len(str_ll) n_col_raw = iter2singleton(map(len, str_ll)) j_col = cls._col_index_extended2raw(k_col, multiple) assert_less(j_col, n_col_raw) n_col_extended = cls.col_count_raw2extended(n_col_raw, multiple) assert_less(k_col, n_col_extended) if not cls._index2is_data(j_col): return [str_ll[i][j_col] for i in range(n_row)] offset = cls._col_index_extended2offset(k_col, multiple) def interpolate(str_ll, i, j_col, p): assert_less(j_col, n_col_extended) s1 = str_ll[i][j_col] if not s1: return s1 v1 = float(s1) if not p: return v1 # logger.debug({"n_col_raw":n_col_raw, # "n_col_extended":n_col_extended, # "multiple": multiple, # "i":i, # "j_col":j_col, # "p":p, # "k_col":k_col, # "offset":offset, # }) s2 = str_ll[i][j_col + 1] if not s2: return s2 v2 = float(s2) return v1 + (v2 - v1) * p p = offset / multiple v_list_col = [ interpolate(str_ll, i, j_col, p) for i in range(1, n_row) ] str_time = cls._colindex_extended2str_time(str_ll[0], k_col, multiple) v_pivot = v_list_col[i_pivot - 1] count_before = IterTool.count( filter(lambda v: v and v > v_pivot, v_list_col)) rank = count_before + 1 count_total = IterTool.count(filter(bool, v_list_col)) str_title = "{} (#{}/{})".format(str_time, rank, count_total) return [str_title] + v_list_col
def table_titles2subbed(cls, table, title_list): n_col = iter2singleton(map(len, table)) assert_equal(n_col, len(title_list) + cls.COUNT_COLHEAD) l_row_top = lchain( table[0][:cls.COUNT_COLHEAD], title_list, ) return lchain([l_row_top], table[1:])
def index_values_beam2neighbor_indexes(cls, i_pivot, v_list, beam): v_count = len(v_list) i_list_sorted = sorted(range(v_count), key=lambda i: v_list[i]) k_pivot = iter2singleton( filter(lambda k: i_list_sorted[k] == i_pivot, range(v_count))) k_span = cls.index_total_beam2span(k_pivot, v_count, beam) i_sublist = cls.list_span2sublist(i_list_sorted, k_span) return i_sublist
def clique2type(cls, clique): has_rate = iter2singleton( map(is_not_none, [ cls.clique2rate(clique), cls.clique2trend(clique), ])) if has_rate: return cls.Type.UPDATE return cls.Type.LOOKUP
def str_ll2h_list(cls, str_ll): assert_greater_equal(len(str_ll), 1) m = iter2singleton(map(len, str_ll)) key_list = str_ll[0] h_list = [ merge_dicts([{ key_list[j]: l[j] } for j in range(m)], vwrite=vwrite_no_duplicate_key) for l in str_ll[1:] ] return h_list
def table_colspan2fresh_rows_filtered(cls, table, colspan): logger = FoxylibLogger.func_level2logger( cls.table_colspan2fresh_rows_filtered, logging.DEBUG) i_list_starting = cls.table_colspan2rowindex_list_starting( table, colspan) table_row_filtered = cls.table_rowindexes2filtered( table, i_list_starting) logger.debug({ "colspan": colspan, "len(table)": len(table), "iter2singleton(map(len,table))": iter2singleton(map(len, table)), "len(i_list_starting)": len(i_list_starting), "len(table_row_filtered)": len(table_row_filtered), "iter2singleton(map(len,table_row_filtered))": iter2singleton(map(len, table_row_filtered)), }) return table_row_filtered
def _str_ll_i2ij_iter(cls, str_ll, i_pivot, beam): logger = FoxylibLogger.func_level2logger(cls._str_ll_i2ij_iter, logging.DEBUG) # raise Exception({"lmap(len, str_ll)":lmap(len, str_ll)}) col_count = iter2singleton(map(len, str_ll)) for j in range(cls.COUNT_COLHEAD, col_count): logger.debug({"j": j}) if not str_ll[i_pivot][j]: # empty column continue str_col_list = lmap(lambda l: l[j], str_ll) i_iter = cls._index_strs_beam2index_iter(str_col_list, i_pivot, beam) for i in i_iter: yield (i, j)
async def funcs_list2pipelined(cls, funcs_list, queue_list=None): assert_true(funcs_list) n = len(funcs_list) if queue_list: assert_equal(len(queue_list), n - 1) batches_producer = funcs_list[0] batches_list_after = [[lambda l: f(iter2singleton(l)) for f in funcs] for funcs in funcs_list[1:]] batches_list = [batches_producer, *batches_list_after] config_list = [ AioPipeline.Config( f_queue=lambda: queue_list[i] if queue_list else None) for i in range(n - 1) ] return await cls.batches_list2pipelined(batches_list, config_list=config_list)
def table2percentage(cls, table, v): logger = FoxylibLogger.func_level2logger(cls.table2percentage, logging.DEBUG) count_col = iter2singleton(map(len, table)) h_j2col_sum = { j: sum(map(int, filter(bool, map(ig(j), table[1:])))) for j in range(cls.COUNT_COLHEAD, count_col) } for i, l in enumerate(table): if i == 0: yield l continue # logger.debug({"l":l}) l_head = l[:cls.COUNT_COLHEAD] l_right = [ "{:.02f}".format(int(l[j]) * 100 / h_j2col_sum[j]) if l[j] else l[j] for j in range(cls.COUNT_COLHEAD, count_col) ] yield lchain(l_head, l_right)
def test_02(self): logger = FoxylibLogger.func_level2logger(self.test_02, logging.DEBUG) c = MongodbToolCollection.collection_default() c.delete_many({}) doc_in = {'a': 'a', 'b': 'b'} op_list = [ InsertOne(doc_in), ] c.bulk_write(op_list) doc_found = iter2singleton(c.find({})) # hyp1 = DictTool.keys2excluded(doc_found, [MongoDBTool.Field._ID]) logger.debug({"doc_found": doc_found, "doc_in": doc_in}) self.assertEqual(doc_found, doc_in) self.assertTrue(isinstance(doc_in[MongoDBTool.Field._ID], ObjectId)) # doc_in updated!! self.assertTrue(isinstance(doc_found[MongoDBTool.Field._ID], ObjectId)) hyp3 = set(doc_found.keys()) ref3 = {'a', '_id', 'b'} self.assertEqual(hyp3, ref3)
def str_ll2interpolated_list(cls, str_ll_in, i_pivot, multiple): logger = FoxylibLogger.func_level2logger(cls.str_ll2interpolated_list, logging.DEBUG) n_row = len(str_ll_in) n_col_raw = iter2singleton(map(len, str_ll_in)) n_col_extended = cls.col_count_raw2extended(n_col_raw, multiple) logger.debug({ "n_col_raw": n_col_raw, "n_col_extended": n_col_extended, "multiple": multiple, }) # raise Exception() ll_tr = [ cls._col_index_extended2str_list(str_ll_in, k, i_pivot, multiple) for k in range(n_col_extended) ] ll_out = [[ll_tr[k][i] for k in range(n_col_extended)] for i in range(n_row)] return ll_out
def entity_group2parameter_type(cls, entity_list): entity_types = map(FoxylibEntity.entity2type, entity_list) return iter2singleton( map(cls.entity_type2parameter_type, entity_types))
def codenames(cls): codename_sets = [set(h_codename2aliases.keys()) for lang, h_codename2aliases in cls.dict_lang2codename2aliases().items()] return iter2singleton(codename_sets)
def table2col_count(cls, ll): return iter2singleton(map(len, ll))