Beispiel #1
0
 def doc_op(self, doc, action='insert'):
     if action == 'insert':
         doc['_id'] = str(uuid.uuid1()).decode()
     doc = self.integrity(doc, action)
     path = self.gen_path(doc)
     ext = ExtHandler.get_ext(path)
     # file operation
     self.fhdict[ext].operate(path, doc, action)
     # index operation
     with self.ix.writer() as writer:
         self.index_doc(doc, writer, action)
Beispiel #2
0
 def init(self, clean=False):
     with self.ix.writer() as writer:
         count = 0
         for root, dirnames, filenames in os.walk(self.dir):
             for filename in filenames:
                 ext = ExtHandler.get_ext(filename)
                 if ext not in self.ext:
                     continue
                 self.index_file(os.path.join(root, filename), writer)
                 count  += 1
                 if count % 1000 == 0:
                     utils.logger.info('create index for %d files' % count)
         writer.mergetype = writing.CLEAR # re-index without disturbing current users
Beispiel #3
0
 def index_file(self, path, writer):
     ext = ExtHandler.get_ext(path)
     for doc in self.fhdict[ext].iter(path):
         self.index_doc(doc, writer, 'insert')