def delete_items(context, request): """Delete the given items from the index """ # If the request isn't an RSS feed, bail out if request.content_type not in ALLOWED_CONTENT: body_msg = ( "The content-type of the request must be one of the " "following: %s" ) % ", ".join(ALLOWED_CONTENT) return HTTPBadRequest(body=body_msg) solr_uri = request.registry.settings.get('push.solr_uri', None) if solr_uri is None: raise AttributeError(u'A push.solr_uri is required') # XXX: We are importing solr here to be able to mock it in the tests from mysolr import Solr solr = Solr(solr_uri) shared_content = feedparser.parse(request.body) missing = [] removed = 0 for item in shared_content.entries: uid = item['id'] uid = normalize_uid(uid) logger.debug('Deleting %s' % uid) if uid not in context.shared: missing.append(uid) solr.delete_by_key(uid) continue del context.shared[uid] solr.delete_by_key(uid) removed += 1 body_msg = "Removed %s items." % removed if missing: msg_str = " %s items could not be found for deletion: %s" args = (len(missing), ', '.join(missing)) msg = msg_str % args logger.warn(msg) body_msg += msg return HTTPOk(body=body_msg)
def delete_items(context, request): """Delete the given items from the index """ # If the request isn't an RSS feed, bail out if request.content_type not in ALLOWED_CONTENT: body_msg = ( "The content-type of the request must be one of the " "following: %s" ) % ", ".join(ALLOWED_CONTENT) return HTTPBadRequest(body=body_msg) solr_uri = request.registry.settings.get('push.solr_uri', None) if solr_uri is None: raise AttributeError(u'A push.solr_uri is required') # XXX: We are importing solr here to be able to mock it in the tests from mysolr import Solr solr = Solr(solr_uri) shared_content = feedparser.parse(request.body) missing = [] removed = 0 for item in shared_content.entries: uid = item['id'] uid = normalize_uid(uid) logger.debug('Deleting %s' % uid) if uid not in context.shared: missing.append(uid) solr.delete_by_key(uid) continue del context.shared[uid] solr.delete_by_key(uid) removed += 1 body_msg = "Removed %s items." % removed if missing: msg_str = " %s items could not be found for deletion: %s" args = (len(missing), ', '.join(missing)) msg = msg_str % args logger.warn(msg) body_msg += msg return HTTPOk(body=body_msg)
class QueryResultTestCase(unittest.TestCase): def setUp(self): self.solr = Solr('http://localhost:8983/solr') def test_search(self): response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) self.assertEqual(response.total_results, 4) self.assertEqual(len(response.documents), 4) def test_search_cursor(self): cursor = self.solr.search_cursor(q='*:*') i = 0 for response in cursor.fetch(1): self.assertEqual(response.status, 200) i += 1 self.assertEqual(i, 4) cursor = self.solr.search_cursor(q='*:*') i = 0 for response in cursor.fetch(4): self.assertEqual(response.status, 200) i += 1 self.assertEqual(i, 1) def test_commit(self): response = self.solr.commit() self.assertEqual(response.status, 200) def test_optimize(self): response = self.solr.optimize() self.assertEqual(response.status, 200) def test_ping(self): response = self.solr.ping() self.assertEqual(response.status, 200) def test_is_up(self): response = self.solr.is_up() self.assertEqual(response, True) def test_update_delete(self): # Get total results response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) total_results = response.total_results # Post one document using json documents = [{'id' : 1}] response = self.solr.update(documents, input_type='json') self.assertEqual(response.status, 200) # Post anoter document using xml documents = [{'id' : 2}] response = self.solr.update(documents, input_type='xml') self.assertEqual(response.status, 200) # Compare total results response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) self.assertEqual(response.total_results, total_results + 2) # Now delete the two document posted above query = 'id:1' key = 2 response = self.solr.delete_by_query(query) self.assertEqual(response.status, 200) response = self.solr.delete_by_key(key) self.assertEqual(response.status, 200) response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) self.assertEqual(response.total_results, total_results) def tearDown(self): pass def test_query(self): pass
class eBsolr: cursor = None def __init__(self, urls, config, version=4): self.cursor = Solr(urls, version=version) def update(self, documents, input_type='json', commit=False): self.cursor.update(documents, input_type, commit) def deleteById(self, tid, commit=False): return self.cursor.delete_by_key(tid, commit=commit) def deleteByQuery(self, query, commit=False): return self.cursor.delete_by_query(query=query, commit=commit) def deleteAll(self, commit=False): return self.cursor.delete_by_query("*:*", commit=commit) def getResponse(self, search, fields=None, start=0, rows=None, sort=None, fq=None): query = {'q': search} if fields: if isinstance(fields, basestring): query['fl'] = fields else: query['fl'] = ",".join(fields) if sort: query['sort'] = sort if fq: query['fq'] = fq # Default to 10000 rows limit = rows if rows is None: limit = _MAXROWS query['start'] = start query['rows'] = limit response = self.cursor.search(**query) if int(response.status) >= 400: raise Exception('Error Solr {}: {}'.format(response.status, response.extract_errmessage())) if rows is None and response.total_results > limit: # query['start'] = response.total_results query['rows'] = response.total_results response = self.cursor.search(**query) return response def get_language_query(self, language): q_temp = None if language is not None and language != "": langArray = language.split(';') if len(langArray) > 0: lang = langArray[0] q_temp = "language:%s" % lang for lang in langArray[1:]: q_temp = "%s OR language:%s" % (q_temp, lang) return q_temp def getDocs(self, search, fields=None, start=0, rows=None, sort=None, fq=None): """search: query sintaks ex: "field:keys,field2:keys2" fields: field yg di ambil (list) ex: ['field', 'field2'] start: start row rows: max / limit row sort: order rows ex: field asc, field2 desc""" # Get documents response = self.getResponse(search, fields, start, rows, sort, fq) return {"docs": response.documents, "count": response.total_results} def getFacetList(self, facets, facetField): ff = {} if not isinstance(facetField, list): facetField = facetField.split(",") for facet in facetField: if facet: ff[facet] = facets['facet_fields'][facet] return ff def getFacetPivotGeneral(self, query, facetField, pivotField, limit=None, fq=None): try: url = "{0}select?q={1}&rows=1&wt=json&indent=true&facet=true&facet.pivot={2},{3}".format( self.cursor.base_url, query.replace("+", "%2B"), facetField, pivotField) url = '{}select'.format(self.cursor.base_url) params = {'q': query, 'rows': 0, 'wt': 'json', 'indent': 'true', 'facet': 'true', 'facet.pivot': '{},{}'.format(facetField, pivotField)} if limit: params['facet.limit'] = limit if fq: params['fq'] = fq # url = "%s&facet.limit=%d" % (url, limit) http_response = requests.get(url, params=params) # print url # http_response = requests.get(url) return http_response.json()['facet_counts']['facet_pivot']['{0},{1}'.format(facetField, pivotField)] except Exception, e: print("Error parsing facet pivot...") print e return None
class QueryResultTestCase(unittest.TestCase): def setUp(self): self.solr = Solr('http://localhost:8983/solr') def test_search(self): response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) self.assertEqual(response.total_results, 4) self.assertEqual(len(response.documents), 4) def test_search_cursor(self): cursor = self.solr.search_cursor(q='*:*') i = 0 for response in cursor.fetch(1): self.assertEqual(response.status, 200) i += 1 self.assertEqual(i, 4) cursor = self.solr.search_cursor(q='*:*') i = 0 for response in cursor.fetch(4): self.assertEqual(response.status, 200) i += 1 self.assertEqual(i, 1) def test_commit(self): response = self.solr.commit() self.assertEqual(response.status, 200) def test_optimize(self): response = self.solr.optimize() self.assertEqual(response.status, 200) def test_ping(self): response = self.solr.ping() self.assertEqual(response.status, 200) def test_is_up(self): response = self.solr.is_up() self.assertEqual(response, True) def test_update_delete(self): # Get total results response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) total_results = response.total_results # Post one document using json documents = [{'id': 1}] response = self.solr.update(documents, input_type='json') self.assertEqual(response.status, 200) # Post anoter document using xml documents = [{'id': 2}] response = self.solr.update(documents, input_type='xml') self.assertEqual(response.status, 200) # Compare total results response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) self.assertEqual(response.total_results, total_results + 2) # Now delete the two document posted above query = 'id:1' key = 2 response = self.solr.delete_by_query(query) self.assertEqual(response.status, 200) response = self.solr.delete_by_key(key) self.assertEqual(response.status, 200) response = self.solr.search(q='*:*') self.assertEqual(response.status, 200) self.assertEqual(response.total_results, total_results) def tearDown(self): pass def test_query(self): pass
# delete from mysolr import Solr #update lixiaoyao #db.baike.update({_id:"http://baike.baidu.com/view/4488.htm"},{$set:{gameURL:["http://baike.baidu.com/subview/2188/5215542.htm" ]}}) solr_url = 'http://10.76.0.137:8983/solr/' iden = [ "http://baike.baidu.com/view/760101.htm", "http://baike.baidu.com/subview/10941/5236539.htm", "http://baike.baidu.com/subview/3049782/11262117.htm", "http://baike.baidu.com/subview/10786/6081536.htm", "http://baike.baidu.com/subview/533601/8190340.htm", "http://baike.baidu.com/view/1016334.htm", "http://baike.baidu.com/view/2106174.htm", "http://baike.baidu.com/subview/187895/6353044.htm", "http://baike.baidu.com/subview/187895/6353038.htm", "http://baike.baidu.com/view/123796.htm", "http://baike.baidu.com/subview/187895/6353044.htm", "http://baike.baidu.com/subview/168512/7010262.htm", ] solr = Solr(solr_url) for x in iden: solr.delete_by_key(x,commit=True) ''' query = {'q':'*:*'} solr.delete_by_query(query,commit=True) '''
class QueryResultTestCase(unittest.TestCase): def setUp(self): self.solr = Solr(os.getenv("SOLR_URL")) def test_search(self): response = self.solr.search(q="*:*") self.assertEqual(response.status, 200) self.assertEqual(response.total_results, 4) self.assertEqual(len(response.documents), 4) def test_search_cursor(self): cursor = self.solr.search_cursor(q="*:*") i = 0 for response in cursor.fetch(1): self.assertEqual(response.status, 200) i += 1 self.assertEqual(i, 4) cursor = self.solr.search_cursor(q="*:*") i = 0 for response in cursor.fetch(4): self.assertEqual(response.status, 200) i += 1 self.assertEqual(i, 1) def test_commit(self): response = self.solr.commit() self.assertEqual(response.status, 200) def test_optimize(self): response = self.solr.optimize() self.assertEqual(response.status, 200) def test_ping(self): response = self.solr.ping() self.assertEqual(response.status, 200) def test_is_up(self): response = self.solr.is_up() self.assertEqual(response, True) def test_update_delete(self): # Get total results response = self.solr.search(q="*:*") self.assertEqual(response.status, 200) total_results = response.total_results # Post one document using json documents = [{"id": 1}] response = self.solr.update(documents, input_type="json") self.assertEqual(response.status, 200) # Post anoter document using xml documents = [{"id": 2}] response = self.solr.update(documents, input_type="xml") self.assertEqual(response.status, 200) # Compare total results response = self.solr.search(q="*:*") self.assertEqual(response.status, 200) self.assertEqual(response.total_results, total_results + 2) # Now delete the two document posted above query = "id:1" key = 2 response = self.solr.delete_by_query(query) self.assertEqual(response.status, 200) response = self.solr.delete_by_key(key) self.assertEqual(response.status, 200) response = self.solr.search(q="*:*") self.assertEqual(response.status, 200) self.assertEqual(response.total_results, total_results) def tearDown(self): pass def test_query(self): pass