def worker(dbid_fpath_pairs): start_time = time.time() dbids = [tup[0] for tup in dbid_fpath_pairs] fpaths = [tup[1] for tup in dbid_fpath_pairs] dhashes = list(map(get_dhash, fpaths)) indexer = Indexer() im_tensors = list(map(get_tensor(indexer), fpaths)) muvar_pairs = indexer.batch_get_feat_stacked(im_tensors) mus = map(lambda t: t[0].unsqueeze(0), muvar_pairs) cids = list(map(indexer.get_cluster_id, mus)) muvar_bufs = list(map(serialize, muvar_pairs)) lastupdates = [indexer_start_time] * len(im_tensors) feature_rows = list(zip(dbids, dhashes, muvar_bufs, lastupdates, cids)) insert_fpaths_stmt = 'INSERT OR REPLACE INTO filepaths VALUES (?,?)' insert_feats_stmt = 'INSERT OR REPLACE INTO features VALUES (?,?,?,?,?)' with sqlite3.connect('deeprelevance.db', timeout=60 * 5) as conn: c = conn.cursor() c.executemany(insert_fpaths_stmt, dbid_fpath_pairs) c.executemany(insert_feats_stmt, feature_rows) conn.commit() run_time = time.time() - start_time print("{} Runtime: {}".format(indexer_start_time, run_time))