class ISolrFields(model.Schema):
""" Additional fields to control Solr integration
"""
directives.fieldset("categorization",
fields=["showinsearch", "searchwords"])
showinsearch = schema.Bool(
required=False,
default=True,
missing_value=True,
title=_("label_showinsearch", default=u"Show in search"),
description=_("help_showinsearch", default=""),
)
searchwords = schema.List(
required=False,
default=[],
missing_value=[],
title=_("label_searchwords", default=u"Search words"),
value_type=schema.TextLine(),
description=_(
"help_searchwords",
u"Specify words for which this item will show up "
u"as the first search result. Multiple words can be "
u"specified on new lines.",
),
)
def elevate_invariant(data):
schema = json.loads(data.elevate_schema)
request = getRequest()
words_mapping = [x["text"] for x in schema]
for i, schema_item in enumerate(schema):
elevate_text = schema_item.get("text", [])
if not elevate_text:
raise Invalid(
translate(
_(
"text_required_label",
default=
"Text field must be filled for Group ${id}.",
mapping=dict(id=i + 1),
),
context=request,
))
for text in elevate_text:
for words_i, words in enumerate(words_mapping):
if i == words_i:
# it's the current config
continue
if text in words:
raise Invalid(
translate(
_(
"text_duplicated_label",
default=
'"${text}" is used in several groups.',
mapping=dict(id=i, text=text),
),
context=request,
))
def search(**kwargs):
solr = get_solr_connection()
if not solr:
msg = u"Unable to search using solr. Configuration is incomplete."
logger.error(msg)
return {
"error":
True,
"message":
translate(
_("solr_configuration_error_label", default=msg),
context=api.portal.get().REQUEST,
),
}
solr_query = generate_query(**kwargs)
try:
return solr.search(**solr_query)
except SolrError as e:
logger.exception(e)
return {
"error":
True,
"message":
translate(
_(
"search_error_label",
default=u"Unable to perform a search with SOLR."
u" Please contact the site administrator or wait some"
u" minutes.",
),
context=api.portal.get().REQUEST,
),
}
class FormSearch(group.Group):
label = _("settings_search_label", default=u"Search")
description = _(
"settings_search_help",
default=u"Use these settings to tweak search results.",
)
fields = field.Fields(IRerSolrpushSearchConf)
class ReindexSolrView(ReactView):
label = _("maintenance_reindex_label", default="Reindex SOLR")
description = _(
"maintenance_reindex_help",
default="Get all Plone contents and reindex them on SOLR.",
)
action = "do-reindex"
class SyncSolrView(ReactView):
label = _("maintenance_sync_label", default="Sync SOLR")
description = _(
"maintenance_sync_help",
default=
"Remove no more existing contents from SOLR and sync with Plone.", # noqa
)
action = "do-sync"
class RerSolrpushEditForm(RegistryEditForm):
"""Nel form del pannello di controllo mettiamo anche le logiche per
il caricamento
"""
schema = IRerSolrpushSettings
groups = (FormDefault, FormSearch)
label = _(u"Solr Push Configuration")
formErrorsMessage = _(
"settings_form_error",
default=
u"Sono presenti degli errori, si prega di ricontrollare i dati inseriti",
)
def updateFields(self):
super(RerSolrpushEditForm, self).updateFields()
for form_group in self.groups:
if "ready" in form_group.fields:
form_group.fields["ready"].mode = HIDDEN_MODE
@button.buttonAndHandler(_("Save"), name=None)
def handleSave(self, action):
data, errors = self.extractData()
if errors:
self.status = self.formErrorsMessage
return
self.applyChanges(data)
api.portal.show_message(_(u"Changes saved."), request=self.request)
init_error = init_solr_push()
if init_error:
api.portal.show_message(init_error,
type="error",
request=self.request)
else:
api.portal.show_message(_(u"Loaded schema.xml from SOLR"),
request=self.request)
self.request.response.redirect(self.request.getURL())
@button.buttonAndHandler(_("Cancel"), name="cancel")
def handleCancel(self, action):
super(RerSolrpushEditForm, self).handleCancel(self, action)
@button.buttonAndHandler(_("Reload schema.xml"), name="reload")
def handleReload(self, action):
data, errors = self.extractData()
init_error = init_solr_push()
if init_error:
api.portal.show_message(init_error,
type="error",
request=self.request)
else:
api.portal.show_message(_(u"Reloaded schema.xml from SOLR"),
request=self.request)
self.request.response.redirect(self.request.getURL())
def search(
query,
fl=None,
facets=False,
facet_fields=["Subject", "portal_type"],
**kwargs
):
"""[summary] TODO
Args:
query ([type]): [description] TODO
fl (str, optional): [description]. Defaults to "".
facets (bool, optional): [description]. Defaults to False.
facet_fields (list, optional): [description].
Defaults to ["Subject", "portal_type"].
Returns:
[type]: [description]
"""
solr = get_solr_connection()
if not solr:
msg = u"Unable to search using solr. Configuration is incomplete."
logger.error(msg)
return {
"error": True,
"message": translate(
_("solr_configuration_error_label", default=msg),
context=api.portal.get().REQUEST,
),
}
solr_query = generate_query(
query,
fl=fl,
facets=facets,
facet_fields=facet_fields,
)
try:
_set_query_debug(solr=solr, params=solr_query)
res = solr.search(**solr_query)
return res
except Exception as e:
logger.exception(e)
return {
"error": True,
"message": translate(
_(
"search_error_label",
default=u"Unable to perform a search with SOLR."
u" Please contact the site administrator or wait some"
u" minutes.",
),
context=api.portal.get().REQUEST,
),
}
def init_solr_push():
"""Inizializza la voce di registro 'index_fields'
Lo fa leggendo il file xml di SOLR.
:param solr_url: [required] L'url a cui richiedere il file xml
:type solr_url: string
:returns: Empty String if everything's good
:rtype: String
"""
solr_url = get_setting(field="solr_url")
if solr_url:
if not solr_url.endswith("/"):
solr_url = solr_url + "/"
try:
respo = requests.get(solr_url + "admin/file?file=schema.xml")
except requests.exceptions.RequestException as err:
ErrorMessage = "Connection problem:\n{0}".format(err)
return ErrorMessage
if respo.status_code != 200:
ErrorMessage = "Problems fetching schema:\n{0}\n{1}".format(
respo.status_code, respo.reason)
return ErrorMessage
root = etree.fromstring(respo.content)
chosen_fields = json.dumps(
extract_fields(nodes=root.findall(".//field")))
if six.PY2:
chosen_fields = chosen_fields.decode("utf-8")
set_setting(field="index_fields", value=chosen_fields)
set_setting(field="ready", value=True)
return
return _("No SOLR url provided")
def remove_from_solr(uid):
"""
Perform remove item from solr
"""
if not is_solr_active():
return
solr = get_solr_connection()
portal = api.portal.get()
if not solr:
logger.error("Unable to push to solr. Configuration is incomplete.")
return
try:
solr.delete(
q="UID:{}".format(uid),
commit=should_force_commit(),
)
except (pysolr.SolrError, TypeError) as err:
logger.error(err)
message = _(
"content_remove_error",
default=u"There was a problem removing this content from SOLR. "
" Please contact site administrator.",
)
api.portal.show_message(
message=message, request=portal.REQUEST, type="error"
)
def handleSave(self, action):
data, errors = self.extractData()
if errors:
self.status = self.formErrorsMessage
return
self.applyChanges(data)
api.portal.show_message(_(u"Changes saved."), request=self.request)
init_error = init_solr_push()
if init_error:
api.portal.show_message(init_error,
type="error",
request=self.request)
else:
api.portal.show_message(_(u"Loaded schema.xml from SOLR"),
request=self.request)
self.request.response.redirect(self.request.getURL())
class IElevateSettings(model.Schema):
""" """
elevate_schema = schema.SourceText(
title=_(u"elevate_schema_label", default=u"Elevate configuration"),
description=_(
u"elevate_schema_help",
default=u"Insert a list of values for elevate.",
),
required=False,
)
@invariant
def elevate_invariant(data):
schema = json.loads(data.elevate_schema)
request = getRequest()
words_mapping = [x["text"] for x in schema]
for i, schema_item in enumerate(schema):
elevate_text = schema_item.get("text", [])
if not elevate_text:
raise Invalid(
translate(
_(
"text_required_label",
default=
"Text field must be filled for Group ${id}.",
mapping=dict(id=i + 1),
),
context=request,
))
for text in elevate_text:
for words_i, words in enumerate(words_mapping):
if i == words_i:
# it's the current config
continue
if text in words:
raise Invalid(
translate(
_(
"text_duplicated_label",
default=
'"${text}" is used in several groups.',
mapping=dict(id=i, text=text),
),
context=request,
))
def do_action(self):
reset_solr()
msg_label = _("maintenance_reset_success",
default="SOLR index dropped")
logger.info("##### SOLR Index dropped #####")
api.portal.show_message(message=msg_label, request=self.request)
return self.request.response.redirect("{}/@@solrpush-conf".format(
api.portal.get().absolute_url()))
def solr_error_message(self):
return translate(
_(
'solr_error_connection',
default=u'There have been problems connecting to SOLR. '
u'Contact site administrator.',
),
context=self.request,
)
class ResetSolr(SolrMaintenanceBaseForm):
"""
Reset solr index
"""
label = _("maintenance_reset_solr_label", default="Reset SOLR index")
description = _(
"maintenance_reset_solr_description",
default="Drop all items in SOLR index.",
)
def do_action(self):
reset_solr()
msg_label = _("maintenance_reset_success",
default="SOLR index dropped")
logger.info("##### SOLR Index dropped #####")
api.portal.show_message(message=msg_label, request=self.request)
return self.request.response.redirect("{}/@@solrpush-conf".format(
api.portal.get().absolute_url()))
class IElevateRowSchema(model.Schema):
text = schema.TextLine(
title=_("elevate_row_schema_text_label", default=u"Text"),
description=_(
"elevate_row_schema_text_help",
default=u"The word that should match in the search.",
),
required=True,
)
uid = schema.List(
title=_("elevate_row_schema_uid_label", u"Elements"),
description=_(
"elevate_row_schema_uid_help",
u"Select a list of elements to elevate for that search word.",
),
value_type=schema.Choice(source=CatalogSource()),
required=True,
)
form.widget("uid", RelatedItemsFieldWidget)
class SolrMaintenanceBaseForm(form.Form):
# template = ViewPageTemplateFile('templates/reindex_solr.pt')
ignoreContext = True
@button.buttonAndHandler(_("start_label", default=u"Start"))
def handleApply(self, action):
data, errors = self.extractData()
if errors:
self.status = self.formErrorsMessage
return
self.do_action()
@button.buttonAndHandler(_("cancel_label", default=u"Cancel"))
def handleCancel(self, action):
msg_label = _("maintenance_cancel_action", default="Action cancelled")
api.portal.show_message(message=msg_label, request=self.request)
return self.request.response.redirect("{}/@@solrpush-conf".format(
api.portal.get().absolute_url()))
def handleReload(self, action):
data, errors = self.extractData()
init_error = init_solr_push()
if init_error:
api.portal.show_message(init_error,
type="error",
request=self.request)
else:
api.portal.show_message(_(u"Reloaded schema.xml from SOLR"),
request=self.request)
self.request.response.redirect(self.request.getURL())
class ElevateSettingsEditForm(RegistryEditForm):
schema = IElevateSettings
label = _(
"solr_elevate_configuration_label",
default=u"Solr Push Elevate Configuration",
)
fields = field.Fields(IElevateSettings)
fields["elevate_schema"].widgetFactory = JSONFieldWidget
def updateWidgets(self):
"""
"""
super(ElevateSettingsEditForm, self).updateWidgets()
self.widgets["elevate_schema"].schema = IElevateRowSchema
def commit(self, wait=None):
# TODO: è possibile con sol anche mandare un set di comandi (add+delete) in un
# unica volta, anzichè uno alla volta, valutare le due opzioni
for action, obj, args in self.queue:
try:
if action == INDEX:
push_to_solr(obj)
elif action == UNINDEX:
remove_from_solr(obj.UID())
except SolrError as err:
logger.exception(err)
message = _(
'content_indexed_error',
default=u'There was a problem indexing this content. Please '
'contact site administrator.',
)
api.portal.show_message(message=message,
request=obj.REQUEST,
type='error')
self.queue = []
def commit(self, wait=None):
if self.active and self.queue:
# TODO: è possibile con sol anche mandare un set di comandi (add+delete) in un
# unica volta, anzichè uno alla volta, valutare le due opzioni
# Sort so unindex operations come first
# for iop, obj in sorted(res.values(), key=itemgetter(0)):
for uid, iop, data, args in self.queue:
try:
if iop in (INDEX, REINDEX):
push_to_solr(data)
elif iop == UNINDEX:
remove_from_solr(uid)
except SolrError:
logger.exception("error indexing %s %s", iop, uid)
message = _(
"content_indexed_error",
default=u"There was a problem indexing or unindexing "
u"this content. Please take note of this address and "
u"contact site administrator.",
)
api.portal.show_message(
message=message, request=getRequest(), type="error"
)
self.queue = []
class FormDefault(group.Group):
label = _("settings_default_label", default=u"Settings")
fields = field.Fields(IRerSolrpushConf)
class IRerSolrpushSearchConf(model.Schema):
query_debug = schema.Bool(
title=_(u"Query debug"),
description=_(
u"If enabled, when a search to SOLR is performed (for "
u"example in Collection), the query will be showed in the page for "
u"debug. Only visible to Managers."),
required=False,
default=False,
)
remote_elevate_schema = schema.TextLine(
title=_(u"remote_elevate_label", default=u"Remote elevate"),
description=_(
u"remote_elevate_help",
default=u'If this field is set and no "site_name" is '
u"passed in query, elevate schema is taken from an external "
u"source. This is useful if you index several sites and handle "
u"elevate configuration in one single site. This should be an url "
u'that points to "@elevate-schema" view.'
u"For example: http://my-site/@elevate-schema.",
),
default=u"",
required=False,
)
qf = schema.TextLine(
title=_("qf_label", default=u"qf (query fields)"),
description=_(
"qf_help",
default=u"Set a list of fields, each of which is assigned a boost "
u"factor to increase or decrease that particular field’s "
u"importance in the query. "
u"For example: fieldOne^1000.0 fieldTwo fieldThree^10.0",
),
required=False,
default=u"",
)
bq = schema.TextLine(
title=_("bq_label", default=u"bq (boost query)"),
description=_(
"bq_help",
default=u"Set a list query clauses that will be added to the main "
u"query to influence the score. For example if we want to boost "
u'results that have a specific "searchwords" term: '
u"searchwords:something^1000",
),
required=False,
default=u"",
)
bf = schema.TextLine(
title=_("bf_label", default=u"bf (boost functions)"),
description=_(
"bf_help",
default=u"Set a list of functions (with optional boosts) that "
u"will be used to construct FunctionQueries which will be added "
u"to the main query as optional clauses that will influence the "
u"score. Any function supported natively by Solr can be used, "
u"along with a boost value. "
u"For example if we want to give less relevance to "
u"items deeper in the tree we can set something like this: "
u"recip(path_depth,10,100,1)",
),
required=False,
default=u"",
)
class IRerSolrpushConf(model.Schema):
"""
"""
active = schema.Bool(
title=_(u"Active"),
description=_(u"Enable SOLR indexing on this site."),
required=False,
default=False,
)
solr_url = schema.TextLine(
title=_(u"SOLR url"),
description=_(u"The SOLR core to connect to."),
required=True,
)
frontend_url = schema.TextLine(
title=_(u"Frontend url"),
description=_(u"If the website has different URL for frontend users."),
required=False,
)
enabled_types = schema.List(
title=_(u'enabled_types_label', default=u'Enabled portal types'),
description=_(
u'enabled_types_help',
default=u'Select a list of portal types to index in solr. '
u'Empty list means that all portal types will be indexed.',
),
required=False,
default=[],
missing_value=[],
value_type=schema.Choice(
vocabulary='plone.app.vocabularies.PortalTypes'
),
)
# elevate_xml = schema.Text(
# title=u'Configurazione elevate',
# description=u'Inserisci una configurazione per l\'elevate come '
# u'se fosse un file xml.',
# required=False,
# constraint=elevateConstraint,
# )
# enable_query_debug = schema.Bool(
# title=u'Abilita il debug delle query solr',
# description=u'Se selezionato, mostra la query effettuata su solr, '
# u'per il debug. Solo per gli amministratori del sito.',
# required=False,
# )
# directives.widget(index_fields=SolrFieldsFieldWidget)
index_fields = schema.SourceText(
title=_(
'index_fields_label',
default=u'List of fields loaded from SOLR that we use for indexing.',
),
description=_(
u'index_fields_help',
default=u'We store this list for performance'
u' reasons. If the configuration changes, you need to click on'
u' Reload button',
),
required=False,
)
# NASCOSTO DAL PANNELLO DI CONTROLLO (vedi: browser/controlpanel.py)
ready = schema.Bool(
title=_(u"Ready"),
description=_(u"SOLR push is ready to be used."),
required=False,
default=False,
)
class IRerSolrpushConf(model.Schema):
""""""
active = schema.Bool(
title=_(u"Active"),
description=_(u"Enable SOLR indexing on this site."),
required=False,
default=False,
)
force_commit = schema.Bool(
title=_(u"Force commit"),
description=
_(u"Force commits on CRUD operations. If enabled, each indexing "
u"operation to SOLR will be immediately committed and persisted. "
u"This means that updates are immediately available on SOLR queries." # noqa
u"If you are using SolrCloud with ZooKeeper, immediate commits "
u"will slow down response performances when indexing, so it's "
u"better to turn it off. In this case updates will be available "
u"when SOLR periodically commit changes."),
required=False,
default=True,
)
solr_url = schema.TextLine(
title=_(u"SOLR url"),
description=_(u"The SOLR core to connect to."),
required=True,
)
frontend_url = schema.TextLine(
title=_(u"Frontend url"),
description=_(u"If the website has different URL for frontend users."),
required=False,
)
enabled_types = schema.List(
title=_(u"enabled_types_label", default=u"Enabled portal types"),
description=_(
u"enabled_types_help",
default=u"Select a list of portal types to index in solr. "
u"Empty list means that all portal types will be indexed.",
),
required=False,
default=[],
missing_value=[],
value_type=schema.Choice(
vocabulary="plone.app.vocabularies.PortalTypes"),
)
index_fields = schema.SourceText(
title=_(
"index_fields_label",
default=u"List of fields loaded from SOLR that we use for "
u"indexing.",
),
description=_(
u"index_fields_help",
default=u"We store this list for performance"
u" reasons. If the configuration changes, you need to click on"
u" Reload button",
),
required=False,
)
# NASCOSTO DAL PANNELLO DI CONTROLLO (vedi: browser/controlpanel.py)
ready = schema.Bool(
title=_(u"Ready"),
description=_(u"SOLR push is ready to be used."),
required=False,
default=False,
)
def handleCancel(self, action):
msg_label = _("maintenance_cancel_action", default="Action cancelled")
api.portal.show_message(message=msg_label, request=self.request)
return self.request.response.redirect("{}/@@solrpush-conf".format(
api.portal.get().absolute_url()))
