def intersection(self, hashes):
"""Iterate over values from `hashes` which exist in the index."""
# sqlite has a compile time limit of 999, see:
# https://www.sqlite.org/c3ref/c_limit_attached.html#sqlitelimitvariablenumber
for chunk in lchunks(999, hashes):
cmd = "SELECT checksum FROM {} WHERE checksum IN ({})".format(
self.INDEX_TABLE, ",".join("?" for hash_ in chunk))
for (hash_, ) in self._execute(cmd, chunk):
yield hash_
def create_chunks():
some_list = [1, 2, 3, 4, 5]
print(lchunks(3, some_list))
# sub.loc[sub.index[i]] = w,h,row.strip(' ')
else:
for bb, seg, label in zip(bbs, segms, labels):
prob = "{:.8f}".format(
bb[4]) if args.eight_digit else "{:.6f}".format(
bb[4])
mask = maskUtils.decode(seg).astype(np.bool)
rle = encode_binary_mask(mask).decode("utf-8")
row += (label + ' ' + prob + ' ' + rle + ' ')
if args.expand:
for parent in all_keyed_child[label]:
row += (parent + ' ' + prob + ' ' + rle + ' ')
df.loc[sub.index.values[i]] = w, h, row.strip(' ')
return df
chunks = funcy.lchunks(int(len(i_lst) / 100), i_lst)
p = Pool(processes=num_processes)
df_list = list(
tqdm(p.imap(process_img, chunks, chunksize=1), total=len(chunks)))
p.close()
p.join()
print(f"len(df_list) = " + str(len(df_list)))
df_total = pd.concat(df_list)
print(f"df_total.shape = " + str(df_total.shape))
df_total.ImageWidth = df_total.ImageWidth.astype('int')
df_total.ImageHeight = df_total.ImageHeight.astype('int')
df_total.reset_index().to_csv(pkl_path.replace('.pkl', '.csv').replace(
0.5).astype('uint8')
if LB_flag:
# compress and base64 encoding --
binary_str = zlib.compress(
mask.encode(wt_avg_msk)['counts'],
zlib.Z_BEST_COMPRESSION)
d_updated_msk[i1] = base64.b64encode(
binary_str).decode()
else:
d_updated_msk[i1] = mask.encode(
wt_avg_msk)['counts'].decode()
if mask_voting: return suppresed, d_updated_msk
else: return suppresed
chunks = funcy.lchunks(int(len(df.ImageID.unique()) / 100),
df.ImageID.unique())
num_processes = 12
p = Pool(processes=num_processes)
if mask_voting:
tuple_lst = list(
tqdm(p.imap(partial(process_img_lst, df=df), chunks, chunksize=1),
total=len(chunks)))
else:
suppresed_lst = list(
tqdm(p.imap(partial(process_img_lst, df=df), chunks, chunksize=1),
total=len(chunks)))
p.close()
p.join()
if mask_voting:
suppresed_lst = [x[0] for x in tuple_lst]