def get_index(self, app_id, namespace, name): """ Gets an index from SOLR. Performs a JSON request to the SOLR schema API to get the list of defined fields. Extracts the fields that match the naming convention appid_[namespace]_index_name. Args: app_id: A str, the application identifier. namespace: A str, the application namespace. name: A str, the index name. Raises: search_exceptions.InternalError: Bad response from SOLR server. Returns: An index item. """ index_name = self.__get_index_name(app_id, namespace, name) solr_url = "http://{0}:{1}/solr/schema/fields".format(self._search_location, self.SOLR_SERVER_PORT) logging.debug("URL: {0}".format(solr_url)) try: conn = urllib2.urlopen(solr_url) if conn.getcode() != HTTP_OK: raise search_exceptions.InternalError("Malformed response from SOLR.") response = json_load_byteified(conn) logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError("Malformed response from SOLR.")
def __execute_query(self, solr_query): """ Executes query string on SOLR. Args: solr_query: A str, the query to run. Returns: The results from the query executing. Raises: search_exceptions.InternalError on internal SOLR error. """ solr_url = "http://{0}:{1}/solr/select/?wt=json&{2}"\ .format(self._search_location, self.SOLR_SERVER_PORT, solr_query) logging.debug("SOLR URL: {0}".format(solr_url)) try: req = urllib2.Request(solr_url) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: logging.error("Got code {0} with URL {1}.".format( conn.getcode(), solr_url)) raise search_exceptions.InternalError("Bad request sent to SOLR.") response = json_load_byteified(conn) status = response['responseHeader']['status'] logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError("Malformed response from SOLR.")
def delete_doc(self, doc_id): """ Deletes a document by doc ID. Args: doc_id: A list of document IDs. Raises: search_exceptions.InternalError on internal errors. """ solr_request = {"delete": {"id": doc_id}} solr_url = "http://{0}:{1}/solr/update?commit=true".format(self._search_location, self.SOLR_SERVER_PORT) logging.debug("SOLR URL: {0}".format(solr_url)) json_request = json.dumps(solr_request) logging.debug("SOLR JSON: {0}".format(json_request)) try: req = urllib2.Request(solr_url, data=json_request) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: raise search_exceptions.InternalError("Malformed response from SOLR.") response = json_load_byteified(conn) status = response['responseHeader']['status'] logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError("Malformed response from SOLR.")
def commit_update(self, hash_map): """ Commits field/value changes to SOLR. Args: hash_map: A dictionary to send to SOLR. Raises: search_exceptions.InternalError: On failure. """ docs = [] docs.append(hash_map) json_payload = json.dumps(docs) solr_url = "http://{0}:{1}/solr/update/json?commit=true".format( self._search_location, self.SOLR_SERVER_PORT) try: req = urllib2.Request(solr_url, data=json_payload) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: logging.error("Got code {0} with URL {1} and payload {2}".format( conn.getcode(), solr_url, json_payload)) raise search_exceptions.InternalError("Bad request sent to SOLR.") response = json_load_byteified(conn) status = response['responseHeader']['status'] logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError("Malformed response from SOLR.")
def update_schema(self, updates): """ Updates the schema of a document. Args: updates: A list of updates to apply. Raises: search_exceptions.InternalError on internal errors from SOLR. """ field_list = [] for update in updates: field_list.append({'name': update['name'], 'type': update['type'], 'stored': 'true', 'indexed': 'true', 'multiValued': 'false'}) solr_url = "http://{0}:{1}/solr/schema/fields".format( self._search_location, self.SOLR_SERVER_PORT) json_request = json.dumps(field_list) try: req = urllib2.Request(solr_url, data=json_request) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: raise search_exceptions.InternalError("Malformed response from SOLR.") response = json_load_byteified(conn) status = response['responseHeader']['status'] logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError("Malformed response from SOLR.")
def __execute_query(self, solr_query_params): """ Executes query on SOLR. Args: solr_query_params: a dict containing query params to send in request. Returns: The results from the query executing. Raises: search_exceptions.InternalError on internal SOLR error. """ solr_query_params['wt'] = 'json' solr_url = "{}/solr/select/?{}".format( self._search_location, urllib.urlencode(solr_query_params)) try: req = urllib2.Request(solr_url) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: logging.error("Got code {0} with URL {1}.".format( conn.getcode(), solr_url)) raise search_exceptions.InternalError( "Bad request sent to SOLR.") response = json_load_byteified(conn) status = response['responseHeader']['status'] except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError( "Malformed response from SOLR.")
class Solr(): """ Class for doing solar operations. """ # The port SOLR is running on. SOLR_SERVER_PORT = 8983 def __init__(self): """ Constructor for solr interface. """ self._search_location = appscale_info.get_search_location() def __get_index_name(self, app_id, namespace, name): """ Gets the internal index name. Args: app_id: A str, the application identifier. namespace: A str, the application namespace. name: A str, the index name. Returns: A str, the internal name of the index. """ return app_id + "_" + namespace + "_" + name def delete_doc(self, doc_id): """ Deletes a document by doc ID. Args: doc_id: A list of document IDs. Raises: search_exceptions.InternalError on internal errors. """ solr_request = {"delete": {"id": doc_id}} solr_url = "http://{0}:{1}/solr/update?commit=true".format( self._search_location, self.SOLR_SERVER_PORT) logging.debug("SOLR URL: {0}".format(solr_url)) json_request = json.dumps(solr_request) logging.debug("SOLR JSON: {0}".format(json_request)) try: req = urllib2.Request(solr_url, data=json_request) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: raise search_exceptions.InternalError( "Malformed response from SOLR.") response = json.load(conn) status = response['responseHeader']['status'] logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError( "Malformed response from SOLR.") if status != 0: raise search_exceptions.InternalError( "SOLR response status of {0}".format(status))
def to_solr_doc(self, doc): """ Converts to an internal SOLR document. Args: doc: A document_pb.Document type. Returns: A converted Document type. Raises: search_exceptions.InternalError if field type is not valid. """ fields = [] for field in doc.field_list(): value = field.value().string_value() field_type = field.value().type() if field_type == FieldValue.TEXT: lang = field.value().language() name = field.name() new_field = Field(name, Field.TEXT_ + lang, value=value) elif field_type == FieldValue.HTML: lang = field.value().language() name = field.name() new_field = Field(name, Field.HTML, value=value) elif field_type == FieldValue.ATOM: name = field.name() new_field = Field(name, Field.ATOM, value=value.lower()) elif field_type == FieldValue.DATE: name = field.name() new_field = Field(name, Field.DATE, value=value) elif field_type == FieldValue.NUMBER: name = field.name() new_field = Field(name, Field.NUMBER, value=value) elif field_type == FieldValue.GEO: geo = field.value.geo() name = field.name() new_field = Field(name, Field.GEO, geo.lat + "," + geo.lng()) else: logging.error("Unknown field type {0}".format(field_type)) raise search_exceptions.InternalError( "Unknown or unimplemented field type!") fields.append(new_field) return Document(doc.id(), doc.language(), fields)
class Solr(object): """ Class for doing solr operations. """ def __init__(self): """ Constructor for solr interface. """ self._search_location = 'http://{}:{}'.format( appscale_info.get_search_location(), SOLR_SERVER_PORT) def delete_doc(self, doc_id): """ Deletes a document by doc ID. Args: doc_id: A list of document IDs. Raises: search_exceptions.InternalError on internal errors. """ solr_request = {"delete": {"id": doc_id}} solr_url = "{}/solr/update?commit=true".format(self._search_location) logging.debug("SOLR URL: {0}".format(solr_url)) json_request = json.dumps(solr_request) logging.debug("SOLR JSON: {0}".format(json_request)) try: req = urllib2.Request(solr_url, data=json_request) req.add_header('Content-Type', 'application/json') conn = urllib2.urlopen(req) if conn.getcode() != HTTP_OK: raise search_exceptions.InternalError( "Malformed response from SOLR.") response = json_load_byteified(conn) status = response['responseHeader']['status'] logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError( "Malformed response from SOLR.") if status != 0: raise search_exceptions.InternalError( "SOLR response status of {0}".format(status))
logging.debug("URL: {0}".format(solr_url)) try: conn = urllib2.urlopen(solr_url) if conn.getcode() != HTTP_OK: raise search_exceptions.InternalError("Malformed response from SOLR.") response = json_load_byteified(conn) logging.debug("Response: {0}".format(response)) except ValueError, exception: logging.error("Unable to decode json from SOLR server: {0}".format( exception)) raise search_exceptions.InternalError("Malformed response from SOLR.") # Make sure the response we got from SOLR was with a good status. status = response['responseHeader']['status'] if status != 0: raise search_exceptions.InternalError( "SOLR response status of {0}".format(status)) # Get only fields which match the index name prefix. filtered_fields = [] for field in response['fields']: if field['name'].startswith("{0}_".format(index_name)): filtered_fields.append(field) schema = Schema(filtered_fields, response['responseHeader']) return Index(index_name, schema) def update_schema(self, updates): """ Updates the schema of a document. Args: updates: A list of updates to apply. Raises: