class ISolrFields(model.Schema): """ Additional fields to control Solr integration """ directives.fieldset("categorization", fields=["showinsearch", "searchwords"]) showinsearch = schema.Bool( required=False, default=True, missing_value=True, title=_("label_showinsearch", default=u"Show in search"), description=_("help_showinsearch", default=""), ) searchwords = schema.List( required=False, default=[], missing_value=[], title=_("label_searchwords", default=u"Search words"), value_type=schema.TextLine(), description=_( "help_searchwords", u"Specify words for which this item will show up " u"as the first search result. Multiple words can be " u"specified on new lines.", ), )
def elevate_invariant(data): schema = json.loads(data.elevate_schema) request = getRequest() words_mapping = [x["text"] for x in schema] for i, schema_item in enumerate(schema): elevate_text = schema_item.get("text", []) if not elevate_text: raise Invalid( translate( _( "text_required_label", default= "Text field must be filled for Group ${id}.", mapping=dict(id=i + 1), ), context=request, )) for text in elevate_text: for words_i, words in enumerate(words_mapping): if i == words_i: # it's the current config continue if text in words: raise Invalid( translate( _( "text_duplicated_label", default= '"${text}" is used in several groups.', mapping=dict(id=i, text=text), ), context=request, ))
def search(**kwargs): solr = get_solr_connection() if not solr: msg = u"Unable to search using solr. Configuration is incomplete." logger.error(msg) return { "error": True, "message": translate( _("solr_configuration_error_label", default=msg), context=api.portal.get().REQUEST, ), } solr_query = generate_query(**kwargs) try: return solr.search(**solr_query) except SolrError as e: logger.exception(e) return { "error": True, "message": translate( _( "search_error_label", default=u"Unable to perform a search with SOLR." u" Please contact the site administrator or wait some" u" minutes.", ), context=api.portal.get().REQUEST, ), }
class FormSearch(group.Group): label = _("settings_search_label", default=u"Search") description = _( "settings_search_help", default=u"Use these settings to tweak search results.", ) fields = field.Fields(IRerSolrpushSearchConf)
class ReindexSolrView(ReactView): label = _("maintenance_reindex_label", default="Reindex SOLR") description = _( "maintenance_reindex_help", default="Get all Plone contents and reindex them on SOLR.", ) action = "do-reindex"
class SyncSolrView(ReactView): label = _("maintenance_sync_label", default="Sync SOLR") description = _( "maintenance_sync_help", default= "Remove no more existing contents from SOLR and sync with Plone.", # noqa ) action = "do-sync"
class RerSolrpushEditForm(RegistryEditForm): """Nel form del pannello di controllo mettiamo anche le logiche per il caricamento """ schema = IRerSolrpushSettings groups = (FormDefault, FormSearch) label = _(u"Solr Push Configuration") formErrorsMessage = _( "settings_form_error", default= u"Sono presenti degli errori, si prega di ricontrollare i dati inseriti", ) def updateFields(self): super(RerSolrpushEditForm, self).updateFields() for form_group in self.groups: if "ready" in form_group.fields: form_group.fields["ready"].mode = HIDDEN_MODE @button.buttonAndHandler(_("Save"), name=None) def handleSave(self, action): data, errors = self.extractData() if errors: self.status = self.formErrorsMessage return self.applyChanges(data) api.portal.show_message(_(u"Changes saved."), request=self.request) init_error = init_solr_push() if init_error: api.portal.show_message(init_error, type="error", request=self.request) else: api.portal.show_message(_(u"Loaded schema.xml from SOLR"), request=self.request) self.request.response.redirect(self.request.getURL()) @button.buttonAndHandler(_("Cancel"), name="cancel") def handleCancel(self, action): super(RerSolrpushEditForm, self).handleCancel(self, action) @button.buttonAndHandler(_("Reload schema.xml"), name="reload") def handleReload(self, action): data, errors = self.extractData() init_error = init_solr_push() if init_error: api.portal.show_message(init_error, type="error", request=self.request) else: api.portal.show_message(_(u"Reloaded schema.xml from SOLR"), request=self.request) self.request.response.redirect(self.request.getURL())
def search( query, fl=None, facets=False, facet_fields=["Subject", "portal_type"], **kwargs ): """[summary] TODO Args: query ([type]): [description] TODO fl (str, optional): [description]. Defaults to "". facets (bool, optional): [description]. Defaults to False. facet_fields (list, optional): [description]. Defaults to ["Subject", "portal_type"]. Returns: [type]: [description] """ solr = get_solr_connection() if not solr: msg = u"Unable to search using solr. Configuration is incomplete." logger.error(msg) return { "error": True, "message": translate( _("solr_configuration_error_label", default=msg), context=api.portal.get().REQUEST, ), } solr_query = generate_query( query, fl=fl, facets=facets, facet_fields=facet_fields, ) try: _set_query_debug(solr=solr, params=solr_query) res = solr.search(**solr_query) return res except Exception as e: logger.exception(e) return { "error": True, "message": translate( _( "search_error_label", default=u"Unable to perform a search with SOLR." u" Please contact the site administrator or wait some" u" minutes.", ), context=api.portal.get().REQUEST, ), }
def init_solr_push(): """Inizializza la voce di registro 'index_fields' Lo fa leggendo il file xml di SOLR. :param solr_url: [required] L'url a cui richiedere il file xml :type solr_url: string :returns: Empty String if everything's good :rtype: String """ solr_url = get_setting(field="solr_url") if solr_url: if not solr_url.endswith("/"): solr_url = solr_url + "/" try: respo = requests.get(solr_url + "admin/file?file=schema.xml") except requests.exceptions.RequestException as err: ErrorMessage = "Connection problem:\n{0}".format(err) return ErrorMessage if respo.status_code != 200: ErrorMessage = "Problems fetching schema:\n{0}\n{1}".format( respo.status_code, respo.reason) return ErrorMessage root = etree.fromstring(respo.content) chosen_fields = json.dumps( extract_fields(nodes=root.findall(".//field"))) if six.PY2: chosen_fields = chosen_fields.decode("utf-8") set_setting(field="index_fields", value=chosen_fields) set_setting(field="ready", value=True) return return _("No SOLR url provided")
def remove_from_solr(uid): """ Perform remove item from solr """ if not is_solr_active(): return solr = get_solr_connection() portal = api.portal.get() if not solr: logger.error("Unable to push to solr. Configuration is incomplete.") return try: solr.delete( q="UID:{}".format(uid), commit=should_force_commit(), ) except (pysolr.SolrError, TypeError) as err: logger.error(err) message = _( "content_remove_error", default=u"There was a problem removing this content from SOLR. " " Please contact site administrator.", ) api.portal.show_message( message=message, request=portal.REQUEST, type="error" )
def handleSave(self, action): data, errors = self.extractData() if errors: self.status = self.formErrorsMessage return self.applyChanges(data) api.portal.show_message(_(u"Changes saved."), request=self.request) init_error = init_solr_push() if init_error: api.portal.show_message(init_error, type="error", request=self.request) else: api.portal.show_message(_(u"Loaded schema.xml from SOLR"), request=self.request) self.request.response.redirect(self.request.getURL())
class IElevateSettings(model.Schema): """ """ elevate_schema = schema.SourceText( title=_(u"elevate_schema_label", default=u"Elevate configuration"), description=_( u"elevate_schema_help", default=u"Insert a list of values for elevate.", ), required=False, ) @invariant def elevate_invariant(data): schema = json.loads(data.elevate_schema) request = getRequest() words_mapping = [x["text"] for x in schema] for i, schema_item in enumerate(schema): elevate_text = schema_item.get("text", []) if not elevate_text: raise Invalid( translate( _( "text_required_label", default= "Text field must be filled for Group ${id}.", mapping=dict(id=i + 1), ), context=request, )) for text in elevate_text: for words_i, words in enumerate(words_mapping): if i == words_i: # it's the current config continue if text in words: raise Invalid( translate( _( "text_duplicated_label", default= '"${text}" is used in several groups.', mapping=dict(id=i, text=text), ), context=request, ))
def do_action(self): reset_solr() msg_label = _("maintenance_reset_success", default="SOLR index dropped") logger.info("##### SOLR Index dropped #####") api.portal.show_message(message=msg_label, request=self.request) return self.request.response.redirect("{}/@@solrpush-conf".format( api.portal.get().absolute_url()))
def solr_error_message(self): return translate( _( 'solr_error_connection', default=u'There have been problems connecting to SOLR. ' u'Contact site administrator.', ), context=self.request, )
class ResetSolr(SolrMaintenanceBaseForm): """ Reset solr index """ label = _("maintenance_reset_solr_label", default="Reset SOLR index") description = _( "maintenance_reset_solr_description", default="Drop all items in SOLR index.", ) def do_action(self): reset_solr() msg_label = _("maintenance_reset_success", default="SOLR index dropped") logger.info("##### SOLR Index dropped #####") api.portal.show_message(message=msg_label, request=self.request) return self.request.response.redirect("{}/@@solrpush-conf".format( api.portal.get().absolute_url()))
class IElevateRowSchema(model.Schema): text = schema.TextLine( title=_("elevate_row_schema_text_label", default=u"Text"), description=_( "elevate_row_schema_text_help", default=u"The word that should match in the search.", ), required=True, ) uid = schema.List( title=_("elevate_row_schema_uid_label", u"Elements"), description=_( "elevate_row_schema_uid_help", u"Select a list of elements to elevate for that search word.", ), value_type=schema.Choice(source=CatalogSource()), required=True, ) form.widget("uid", RelatedItemsFieldWidget)
class SolrMaintenanceBaseForm(form.Form): # template = ViewPageTemplateFile('templates/reindex_solr.pt') ignoreContext = True @button.buttonAndHandler(_("start_label", default=u"Start")) def handleApply(self, action): data, errors = self.extractData() if errors: self.status = self.formErrorsMessage return self.do_action() @button.buttonAndHandler(_("cancel_label", default=u"Cancel")) def handleCancel(self, action): msg_label = _("maintenance_cancel_action", default="Action cancelled") api.portal.show_message(message=msg_label, request=self.request) return self.request.response.redirect("{}/@@solrpush-conf".format( api.portal.get().absolute_url()))
def handleReload(self, action): data, errors = self.extractData() init_error = init_solr_push() if init_error: api.portal.show_message(init_error, type="error", request=self.request) else: api.portal.show_message(_(u"Reloaded schema.xml from SOLR"), request=self.request) self.request.response.redirect(self.request.getURL())
class ElevateSettingsEditForm(RegistryEditForm): schema = IElevateSettings label = _( "solr_elevate_configuration_label", default=u"Solr Push Elevate Configuration", ) fields = field.Fields(IElevateSettings) fields["elevate_schema"].widgetFactory = JSONFieldWidget def updateWidgets(self): """ """ super(ElevateSettingsEditForm, self).updateWidgets() self.widgets["elevate_schema"].schema = IElevateRowSchema
def commit(self, wait=None): # TODO: è possibile con sol anche mandare un set di comandi (add+delete) in un # unica volta, anzichè uno alla volta, valutare le due opzioni for action, obj, args in self.queue: try: if action == INDEX: push_to_solr(obj) elif action == UNINDEX: remove_from_solr(obj.UID()) except SolrError as err: logger.exception(err) message = _( 'content_indexed_error', default=u'There was a problem indexing this content. Please ' 'contact site administrator.', ) api.portal.show_message(message=message, request=obj.REQUEST, type='error') self.queue = []
def commit(self, wait=None): if self.active and self.queue: # TODO: è possibile con sol anche mandare un set di comandi (add+delete) in un # unica volta, anzichè uno alla volta, valutare le due opzioni # Sort so unindex operations come first # for iop, obj in sorted(res.values(), key=itemgetter(0)): for uid, iop, data, args in self.queue: try: if iop in (INDEX, REINDEX): push_to_solr(data) elif iop == UNINDEX: remove_from_solr(uid) except SolrError: logger.exception("error indexing %s %s", iop, uid) message = _( "content_indexed_error", default=u"There was a problem indexing or unindexing " u"this content. Please take note of this address and " u"contact site administrator.", ) api.portal.show_message( message=message, request=getRequest(), type="error" ) self.queue = []
class FormDefault(group.Group): label = _("settings_default_label", default=u"Settings") fields = field.Fields(IRerSolrpushConf)
class IRerSolrpushSearchConf(model.Schema): query_debug = schema.Bool( title=_(u"Query debug"), description=_( u"If enabled, when a search to SOLR is performed (for " u"example in Collection), the query will be showed in the page for " u"debug. Only visible to Managers."), required=False, default=False, ) remote_elevate_schema = schema.TextLine( title=_(u"remote_elevate_label", default=u"Remote elevate"), description=_( u"remote_elevate_help", default=u'If this field is set and no "site_name" is ' u"passed in query, elevate schema is taken from an external " u"source. This is useful if you index several sites and handle " u"elevate configuration in one single site. This should be an url " u'that points to "@elevate-schema" view.' u"For example: http://my-site/@elevate-schema.", ), default=u"", required=False, ) qf = schema.TextLine( title=_("qf_label", default=u"qf (query fields)"), description=_( "qf_help", default=u"Set a list of fields, each of which is assigned a boost " u"factor to increase or decrease that particular field’s " u"importance in the query. " u"For example: fieldOne^1000.0 fieldTwo fieldThree^10.0", ), required=False, default=u"", ) bq = schema.TextLine( title=_("bq_label", default=u"bq (boost query)"), description=_( "bq_help", default=u"Set a list query clauses that will be added to the main " u"query to influence the score. For example if we want to boost " u'results that have a specific "searchwords" term: ' u"searchwords:something^1000", ), required=False, default=u"", ) bf = schema.TextLine( title=_("bf_label", default=u"bf (boost functions)"), description=_( "bf_help", default=u"Set a list of functions (with optional boosts) that " u"will be used to construct FunctionQueries which will be added " u"to the main query as optional clauses that will influence the " u"score. Any function supported natively by Solr can be used, " u"along with a boost value. " u"For example if we want to give less relevance to " u"items deeper in the tree we can set something like this: " u"recip(path_depth,10,100,1)", ), required=False, default=u"", )
class IRerSolrpushConf(model.Schema): """ """ active = schema.Bool( title=_(u"Active"), description=_(u"Enable SOLR indexing on this site."), required=False, default=False, ) solr_url = schema.TextLine( title=_(u"SOLR url"), description=_(u"The SOLR core to connect to."), required=True, ) frontend_url = schema.TextLine( title=_(u"Frontend url"), description=_(u"If the website has different URL for frontend users."), required=False, ) enabled_types = schema.List( title=_(u'enabled_types_label', default=u'Enabled portal types'), description=_( u'enabled_types_help', default=u'Select a list of portal types to index in solr. ' u'Empty list means that all portal types will be indexed.', ), required=False, default=[], missing_value=[], value_type=schema.Choice( vocabulary='plone.app.vocabularies.PortalTypes' ), ) # elevate_xml = schema.Text( # title=u'Configurazione elevate', # description=u'Inserisci una configurazione per l\'elevate come ' # u'se fosse un file xml.', # required=False, # constraint=elevateConstraint, # ) # enable_query_debug = schema.Bool( # title=u'Abilita il debug delle query solr', # description=u'Se selezionato, mostra la query effettuata su solr, ' # u'per il debug. Solo per gli amministratori del sito.', # required=False, # ) # directives.widget(index_fields=SolrFieldsFieldWidget) index_fields = schema.SourceText( title=_( 'index_fields_label', default=u'List of fields loaded from SOLR that we use for indexing.', ), description=_( u'index_fields_help', default=u'We store this list for performance' u' reasons. If the configuration changes, you need to click on' u' Reload button', ), required=False, ) # NASCOSTO DAL PANNELLO DI CONTROLLO (vedi: browser/controlpanel.py) ready = schema.Bool( title=_(u"Ready"), description=_(u"SOLR push is ready to be used."), required=False, default=False, )
class IRerSolrpushConf(model.Schema): """""" active = schema.Bool( title=_(u"Active"), description=_(u"Enable SOLR indexing on this site."), required=False, default=False, ) force_commit = schema.Bool( title=_(u"Force commit"), description= _(u"Force commits on CRUD operations. If enabled, each indexing " u"operation to SOLR will be immediately committed and persisted. " u"This means that updates are immediately available on SOLR queries." # noqa u"If you are using SolrCloud with ZooKeeper, immediate commits " u"will slow down response performances when indexing, so it's " u"better to turn it off. In this case updates will be available " u"when SOLR periodically commit changes."), required=False, default=True, ) solr_url = schema.TextLine( title=_(u"SOLR url"), description=_(u"The SOLR core to connect to."), required=True, ) frontend_url = schema.TextLine( title=_(u"Frontend url"), description=_(u"If the website has different URL for frontend users."), required=False, ) enabled_types = schema.List( title=_(u"enabled_types_label", default=u"Enabled portal types"), description=_( u"enabled_types_help", default=u"Select a list of portal types to index in solr. " u"Empty list means that all portal types will be indexed.", ), required=False, default=[], missing_value=[], value_type=schema.Choice( vocabulary="plone.app.vocabularies.PortalTypes"), ) index_fields = schema.SourceText( title=_( "index_fields_label", default=u"List of fields loaded from SOLR that we use for " u"indexing.", ), description=_( u"index_fields_help", default=u"We store this list for performance" u" reasons. If the configuration changes, you need to click on" u" Reload button", ), required=False, ) # NASCOSTO DAL PANNELLO DI CONTROLLO (vedi: browser/controlpanel.py) ready = schema.Bool( title=_(u"Ready"), description=_(u"SOLR push is ready to be used."), required=False, default=False, )
def handleCancel(self, action): msg_label = _("maintenance_cancel_action", default="Action cancelled") api.portal.show_message(message=msg_label, request=self.request) return self.request.response.redirect("{}/@@solrpush-conf".format( api.portal.get().absolute_url()))