def sorted_iterator(self): self.container.sort() fileslist = [open(filename) for filename in self.diskdumps] sequences = [spickle.s_load(file) for file in fileslist] sequences.append(iter(self.container)) return heapq.merge(*sequences) map(lambda x:x.close(), fileslist)
def read_results_spickle(path): """Reads a resultset from a streaming pickle file. This allows for iterative file reading instead of loading the whole file first. Parameters ---------- path : str The file path from which results are read Returns ------- results : ResultSet The read result set """ return spickle.s_load(open(path, 'rb'))