def get_index(self, app_id, namespace, name):
""" Gets an index from SOLR.
Performs a JSON request to the SOLR schema API to get the list of defined
fields. Extracts the fields that match the naming convention
appid_[namespace]_index_name.
Args:
app_id: A str, the application identifier.
namespace: A str, the application namespace.
name: A str, the index name.
Raises:
search_exceptions.InternalError: Bad response from SOLR server.
Returns:
An index item.
"""
index_name = self.__get_index_name(app_id, namespace, name)
solr_url = "http://{0}:{1}/solr/schema/fields".format(self._search_location,
self.SOLR_SERVER_PORT)
logging.debug("URL: {0}".format(solr_url))
try:
conn = urllib2.urlopen(solr_url)
if conn.getcode() != HTTP_OK:
raise search_exceptions.InternalError("Malformed response from SOLR.")
response = json_load_byteified(conn)
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError("Malformed response from SOLR.")
def __execute_query(self, solr_query):
""" Executes query string on SOLR.
Args:
solr_query: A str, the query to run.
Returns:
The results from the query executing.
Raises:
search_exceptions.InternalError on internal SOLR error.
"""
solr_url = "http://{0}:{1}/solr/select/?wt=json&{2}"\
.format(self._search_location, self.SOLR_SERVER_PORT,
solr_query)
logging.debug("SOLR URL: {0}".format(solr_url))
try:
req = urllib2.Request(solr_url)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
logging.error("Got code {0} with URL {1}.".format(
conn.getcode(), solr_url))
raise search_exceptions.InternalError("Bad request sent to SOLR.")
response = json_load_byteified(conn)
status = response['responseHeader']['status']
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError("Malformed response from SOLR.")
def delete_doc(self, doc_id):
""" Deletes a document by doc ID.
Args:
doc_id: A list of document IDs.
Raises:
search_exceptions.InternalError on internal errors.
"""
solr_request = {"delete": {"id": doc_id}}
solr_url = "http://{0}:{1}/solr/update?commit=true".format(self._search_location,
self.SOLR_SERVER_PORT)
logging.debug("SOLR URL: {0}".format(solr_url))
json_request = json.dumps(solr_request)
logging.debug("SOLR JSON: {0}".format(json_request))
try:
req = urllib2.Request(solr_url, data=json_request)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
raise search_exceptions.InternalError("Malformed response from SOLR.")
response = json_load_byteified(conn)
status = response['responseHeader']['status']
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError("Malformed response from SOLR.")
def commit_update(self, hash_map):
""" Commits field/value changes to SOLR.
Args:
hash_map: A dictionary to send to SOLR.
Raises:
search_exceptions.InternalError: On failure.
"""
docs = []
docs.append(hash_map)
json_payload = json.dumps(docs)
solr_url = "http://{0}:{1}/solr/update/json?commit=true".format(
self._search_location, self.SOLR_SERVER_PORT)
try:
req = urllib2.Request(solr_url, data=json_payload)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
logging.error("Got code {0} with URL {1} and payload {2}".format(
conn.getcode(), solr_url, json_payload))
raise search_exceptions.InternalError("Bad request sent to SOLR.")
response = json_load_byteified(conn)
status = response['responseHeader']['status']
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError("Malformed response from SOLR.")
def update_schema(self, updates):
""" Updates the schema of a document.
Args:
updates: A list of updates to apply.
Raises:
search_exceptions.InternalError on internal errors from SOLR.
"""
field_list = []
for update in updates:
field_list.append({'name': update['name'], 'type': update['type'],
'stored': 'true', 'indexed': 'true', 'multiValued': 'false'})
solr_url = "http://{0}:{1}/solr/schema/fields".format(
self._search_location, self.SOLR_SERVER_PORT)
json_request = json.dumps(field_list)
try:
req = urllib2.Request(solr_url, data=json_request)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
raise search_exceptions.InternalError("Malformed response from SOLR.")
response = json_load_byteified(conn)
status = response['responseHeader']['status']
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError("Malformed response from SOLR.")
def __execute_query(self, solr_query_params):
""" Executes query on SOLR.
Args:
solr_query_params: a dict containing query params to send in request.
Returns:
The results from the query executing.
Raises:
search_exceptions.InternalError on internal SOLR error.
"""
solr_query_params['wt'] = 'json'
solr_url = "{}/solr/select/?{}".format(
self._search_location, urllib.urlencode(solr_query_params))
try:
req = urllib2.Request(solr_url)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
logging.error("Got code {0} with URL {1}.".format(
conn.getcode(), solr_url))
raise search_exceptions.InternalError(
"Bad request sent to SOLR.")
response = json_load_byteified(conn)
status = response['responseHeader']['status']
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError(
"Malformed response from SOLR.")
class Solr():
""" Class for doing solar operations. """
# The port SOLR is running on.
SOLR_SERVER_PORT = 8983
def __init__(self):
""" Constructor for solr interface. """
self._search_location = appscale_info.get_search_location()
def __get_index_name(self, app_id, namespace, name):
""" Gets the internal index name.
Args:
app_id: A str, the application identifier.
namespace: A str, the application namespace.
name: A str, the index name.
Returns:
A str, the internal name of the index.
"""
return app_id + "_" + namespace + "_" + name
def delete_doc(self, doc_id):
""" Deletes a document by doc ID.
Args:
doc_id: A list of document IDs.
Raises:
search_exceptions.InternalError on internal errors.
"""
solr_request = {"delete": {"id": doc_id}}
solr_url = "http://{0}:{1}/solr/update?commit=true".format(
self._search_location, self.SOLR_SERVER_PORT)
logging.debug("SOLR URL: {0}".format(solr_url))
json_request = json.dumps(solr_request)
logging.debug("SOLR JSON: {0}".format(json_request))
try:
req = urllib2.Request(solr_url, data=json_request)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
raise search_exceptions.InternalError(
"Malformed response from SOLR.")
response = json.load(conn)
status = response['responseHeader']['status']
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError(
"Malformed response from SOLR.")
if status != 0:
raise search_exceptions.InternalError(
"SOLR response status of {0}".format(status))
def to_solr_doc(self, doc):
""" Converts to an internal SOLR document.
Args:
doc: A document_pb.Document type.
Returns:
A converted Document type.
Raises:
search_exceptions.InternalError if field type is not valid.
"""
fields = []
for field in doc.field_list():
value = field.value().string_value()
field_type = field.value().type()
if field_type == FieldValue.TEXT:
lang = field.value().language()
name = field.name()
new_field = Field(name, Field.TEXT_ + lang, value=value)
elif field_type == FieldValue.HTML:
lang = field.value().language()
name = field.name()
new_field = Field(name, Field.HTML, value=value)
elif field_type == FieldValue.ATOM:
name = field.name()
new_field = Field(name, Field.ATOM, value=value.lower())
elif field_type == FieldValue.DATE:
name = field.name()
new_field = Field(name, Field.DATE, value=value)
elif field_type == FieldValue.NUMBER:
name = field.name()
new_field = Field(name, Field.NUMBER, value=value)
elif field_type == FieldValue.GEO:
geo = field.value.geo()
name = field.name()
new_field = Field(name, Field.GEO, geo.lat + "," + geo.lng())
else:
logging.error("Unknown field type {0}".format(field_type))
raise search_exceptions.InternalError(
"Unknown or unimplemented field type!")
fields.append(new_field)
return Document(doc.id(), doc.language(), fields)
class Solr(object):
""" Class for doing solr operations. """
def __init__(self):
""" Constructor for solr interface. """
self._search_location = 'http://{}:{}'.format(
appscale_info.get_search_location(), SOLR_SERVER_PORT)
def delete_doc(self, doc_id):
""" Deletes a document by doc ID.
Args:
doc_id: A list of document IDs.
Raises:
search_exceptions.InternalError on internal errors.
"""
solr_request = {"delete": {"id": doc_id}}
solr_url = "{}/solr/update?commit=true".format(self._search_location)
logging.debug("SOLR URL: {0}".format(solr_url))
json_request = json.dumps(solr_request)
logging.debug("SOLR JSON: {0}".format(json_request))
try:
req = urllib2.Request(solr_url, data=json_request)
req.add_header('Content-Type', 'application/json')
conn = urllib2.urlopen(req)
if conn.getcode() != HTTP_OK:
raise search_exceptions.InternalError(
"Malformed response from SOLR.")
response = json_load_byteified(conn)
status = response['responseHeader']['status']
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError(
"Malformed response from SOLR.")
if status != 0:
raise search_exceptions.InternalError(
"SOLR response status of {0}".format(status))
logging.debug("URL: {0}".format(solr_url))
try:
conn = urllib2.urlopen(solr_url)
if conn.getcode() != HTTP_OK:
raise search_exceptions.InternalError("Malformed response from SOLR.")
response = json_load_byteified(conn)
logging.debug("Response: {0}".format(response))
except ValueError, exception:
logging.error("Unable to decode json from SOLR server: {0}".format(
exception))
raise search_exceptions.InternalError("Malformed response from SOLR.")
# Make sure the response we got from SOLR was with a good status.
status = response['responseHeader']['status']
if status != 0:
raise search_exceptions.InternalError(
"SOLR response status of {0}".format(status))
# Get only fields which match the index name prefix.
filtered_fields = []
for field in response['fields']:
if field['name'].startswith("{0}_".format(index_name)):
filtered_fields.append(field)
schema = Schema(filtered_fields, response['responseHeader'])
return Index(index_name, schema)
def update_schema(self, updates):
""" Updates the schema of a document.
Args:
updates: A list of updates to apply.
Raises:
