def load_codevecs(vec_path, chunk_size=2000000):
logger.debug(f'Loading code vectors (chunk size={chunk_size})..')
"""read vectors (2D numpy array) from a hdf5 file"""
codevecs = []
chunk_id = 0
reprs = load_vecs(vec_path)
chunk_path = f"{vec_path[:-3]}_part{chunk_id}.h5"
while os.path.exists(chunk_path):
reprs = load_vecs(chunk_path)
codevecs.append(reprs)
chunk_id += 1
chunk_path = f"{vec_path[:-3]}_part{chunk_id}.h5"
return codevecs
def load_codevecs(self):
logger.debug('Loading code vectors..')
if not self.codevecs: # empty
"""read vectors (2D numpy array) from a hdf5 file"""
reprs=load_vecs(self.path+self.conf['use_codevecs'])
for i in range(0,reprs.shape[0], self.codebase_chunksize):
self.codevecs.append(reprs[i:i+self.codebase_chunksize])
def load_codevecs(vec_path, chunk_size=2000000):
logger.debug('Loading code vectors..')
"""read vectors (2D numpy array) from a hdf5 file"""
codevecs = []
reprs = load_vecs(vec_path)
for i in range(0, reprs.shape[0], chunk_size):
codevecs.append(reprs[i:i + chunk_size])
return codevecs
