class WhooshManager(models.Manager): def __init__(self, *args, **kwargs): self.default = kwargs.pop("default",None) self.parser = None self.fields = kwargs.pop('fields', []) + ['id'] self.real_time = kwargs.pop('real_time', True) if not os.path.lexists(STORAGE_DIR): os.makedirs(STORAGE_DIR) self.storage = filestore.FileStorage(STORAGE_DIR) try: self.index = FileIndex(self.storage) except (IndexError, EmptyIndexError): self.index = None super(WhooshManager, self).__init__(*args, **kwargs) def contribute_to_class(self, model, name): super(WhooshManager, self).contribute_to_class(model, name) class_prepared.connect(self.class_prepared_callback, sender=self.model) def class_prepared_callback(self, sender, **kwargs): schema_dict = {} for field_name in self.fields: field = self.model._meta.get_field_by_name(field_name)[0] schema_dict[field.name] = field_mapping[field.__class__] self.schema = Schema(**schema_dict) if self.index is None: self.index = FileIndex.create(self.storage, self.schema) self.searcher = self.index.searcher() if self.real_time: post_save.connect(self.post_save_callback, sender=self.model) post_delete.connect(self.post_delete_callback, sender=self.model) def post_save_callback(self, sender, instance, created, **kwargs): dct = dict([(f, unicode(getattr(instance, f))) for f in self.fields]) self.index = self.index.refresh() writer = self.index.writer() if created: writer.add_document(**dct) else: writer.update_document(**dct) writer.commit() def post_delete_callback(self, sender, instance, **kwargs): pass def query(self, q): if self.parser is None: self.parser = QueryParser(self.default, schema=self.schema) results = self.searcher.search(self.parser.parse(q)) return self.filter(id__in=[r['id'] for r in results])
class WhooshManager(models.Manager): def __init__(self, *args, **kwargs): self.default = kwargs.pop("default", None) self.parser = None self.fields = kwargs.pop('fields', []) + ['id'] self.real_time = kwargs.pop('real_time', True) if not os.path.lexists(STORAGE_DIR): os.makedirs(STORAGE_DIR) self.storage = filestore.FileStorage(STORAGE_DIR) try: self.index = FileIndex(self.storage) except (IndexError, EmptyIndexError): self.index = None super(WhooshManager, self).__init__(*args, **kwargs) def contribute_to_class(self, model, name): super(WhooshManager, self).contribute_to_class(model, name) class_prepared.connect(self.class_prepared_callback, sender=self.model) def class_prepared_callback(self, sender, **kwargs): schema_dict = {} for field_name in self.fields: field = self.model._meta.get_field_by_name(field_name)[0] schema_dict[field.name] = field_mapping[field.__class__] self.schema = Schema(**schema_dict) if self.index is None: self.index = FileIndex.create(self.storage, self.schema) self.searcher = self.index.searcher() if self.real_time: post_save.connect(self.post_save_callback, sender=self.model) post_delete.connect(self.post_delete_callback, sender=self.model) def post_save_callback(self, sender, instance, created, **kwargs): dct = dict([(f, unicode(getattr(instance, f))) for f in self.fields]) self.index = self.index.refresh() writer = self.index.writer() if created: writer.add_document(**dct) else: writer.update_document(**dct) writer.commit() def post_delete_callback(self, sender, instance, **kwargs): pass def query(self, q): if self.parser is None: self.parser = QueryParser(self.default, schema=self.schema) results = self.searcher.search(self.parser.parse(q)) return self.filter(id__in=[r['id'] for r in results])
def update_index(index: FileIndex, cards: List[Card]) -> None: writer = index.writer() # We exclude tokens here because they can have the exact same name as cards. # We exclude emblems here to stop them showing up as cards = [c for c in cards if multiverse.is_playable_layout(c.layout)] for card in cards: names = card.names if card.name not in names: names.append(card.name) # Split and aftermath cards for name in names: document = {} document['id'] = card.id document['name'] = name document['canonical_name'] = card.name document['name_tokenized'] = name document['name_stemmed'] = name document['name_normalized'] = name writer.update_document(**document) writer.commit()
def create_writer(index: FileIndex) -> IndexWriter: w = index.writer(limitmb=16000, procs=1) return w