def commit(self, **kwargs): try: for task in self.tasks: self.jobqueue.put(None) for task in self.tasks: task.join() for task in self.tasks: taskseg = self.resultqueue.get() assert isinstance(taskseg, Segment), type(taskseg) self.segments.append(taskseg) self.jobqueue.close() self.resultqueue.close() from whoosh.filedb.fileindex import _write_toc, _clean_files _write_toc(self.storage, self.schema, self.index.indexname, self.generation, self.segment_number, self.segments) readlock = self.index.lock("READLOCK") readlock.acquire(True) try: _clean_files(self.storage, self.index.indexname, self.generation, self.segments) finally: readlock.release() finally: self.writelock.release()
def commit(self, **kwargs): try: # index the remaining stuff in self.docbuffer self._enqueue() for task in self.tasks: self.jobqueue.put(None) for task in self.tasks: task.join() for task in self.tasks: taskseg = self.resultqueue.get() assert isinstance(taskseg, Segment), type(taskseg) self.segments.append(taskseg) self.jobqueue.close() self.resultqueue.close() from whoosh.filedb.fileindex import _write_toc, _clean_files _write_toc(self.storage, self.schema, self.index.indexname, self.generation, self.segment_number, self.segments) # Delete leftover files _clean_files(self.storage, self.index.indexname, self.generation, self.segments) finally: self.writelock.release()
def commit(self, mergetype=None, optimize=False, merge=True): """Finishes writing and saves all additions and changes to disk. There are four possible ways to use this method:: # Merge small segments but leave large segments, trying to # balance fast commits with fast searching: writer.commit() # Merge all segments into a single segment: writer.commit(optimize=True) # Don't merge any existing segments: writer.commit(merge=False) # Use a custom merge function writer.commit(mergetype=my_merge_function) :param mergetype: a custom merge function taking a Writer object and segment list as arguments, and returning a new segment list. If you supply a ``mergetype`` function, the values of the ``optimize`` and ``merge`` arguments are ignored. :param optimize: if True, all existing segments are merged with the documents you've added to this writer (and the value of the ``merge`` argument is ignored). :param merge: if False, do not merge small segments. """ self._check_state() try: if mergetype: pass elif optimize: mergetype = OPTIMIZE elif not merge: mergetype = NO_MERGE else: mergetype = MERGE_SMALL # Call the merge policy function. The policy may choose to merge other # segments into this writer's pool new_segments = mergetype(self, self.segments) # Tell the pool we're finished adding information, it should add its # accumulated data to the lengths, terms index, and posting files. if self._added: self.pool.finish(self.termswriter, self.docnum, self.lengthfile) # Create a Segment object for the segment created by this writer and # add it to the list of remaining segments returned by the merge policy # function new_segments.append(self._getsegment()) else: self.pool.cleanup() # Close all files, write a new TOC with the new segment list, and # release the lock. self._close_all() from whoosh.filedb.fileindex import _write_toc, _clean_files _write_toc(self.storage, self.schema, self.indexname, self.generation, self.segment_number, new_segments) self.readlock.acquire(True) try: _clean_files(self.storage, self.indexname, self.generation, new_segments) finally: self.readlock.release() finally: if self.writelock: self.writelock.release()