def setUp(self): # Generate a fake solr response data to mock perform_solr_query function self.NUM_RESULTS = 15 sound_ids = list( Sound.objects.filter(moderation_state="OK", processing_state="OK").values_list( 'id', flat=True)[:self.NUM_RESULTS]) solr_select_returned_data['grouped']['grouping_pack']['groups'] = [{ 'doclist': { 'docs': [{ 'id': sound_id }], 'numFound': 1, 'start': 0 }, 'groupValue': str(sound_id) } for sound_id in sound_ids] results = SolrResponseInterpreter( copy.deepcopy(solr_select_returned_data)) # NOTE: in the line abve, we need to deepcopy the dictionary of results because SolrResponseInterpreter changes # it and makes it break when run a second time. Ideally SolrResponseInterpreter should be fixed so that it does # not change its input parameter. paginator = SolrResponseInterpreterPaginator(results, self.NUM_RESULTS) page = paginator.page(1) # Get first page self.perform_solr_query_response = \ (results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs)
def perform_solr_query(q, current_page): """ This util function performs the query to SOLR and returns needed parameters to continue with the view. The main reason to have this util function is to facilitate mocking in unit tests for this view. """ solr = Solr(settings.SOLR_URL) results = SolrResponseInterpreter(solr.select(unicode(q))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) page = paginator.page(current_page) return results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs
def tags(request, multiple_tags=None): if multiple_tags: multiple_tags = multiple_tags.split('/') else: multiple_tags = [] multiple_tags = sorted(filter(lambda x:x, multiple_tags)) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 solr = Solr(settings.SOLR_URL) query = SolrQuery() if multiple_tags: query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags)) else: query.set_query("*:*") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"]) query.add_facet_fields("tag") query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) page = paginator.page(current_page) error = False tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]] except SolrException, e: error = True search_logger.error("SOLR ERROR - %s" % e)
def tags(request, multiple_tags=None): if multiple_tags: multiple_tags = multiple_tags.split('/') else: multiple_tags = [] multiple_tags = sorted(filter(lambda x: x, multiple_tags)) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 solr = Solr(settings.SOLR_URL) query = SolrQuery() if multiple_tags: query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags)) else: query.set_query("*:*") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"]) query.add_facet_fields("tag") query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False) query.set_group_field(group_field="grouping_pack") query.set_group_options( group_func=None, group_query=None, group_rows=10, group_start=0, group_limit=1, group_offset=0, group_sort=None, group_sort_ingroup=None, group_format='grouped', group_main=False, group_num_groups=True, group_cache_percent=0, group_truncate=True ) # Sets how many results from the same grup are taken into account for computing the facets try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]] docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s for d in docs: d["sound"] = allsounds[d["id"]] except SolrException as e: error = True search_logger.error("SOLR ERROR - %s" % e) except: error = True slash_tag = "/".join(multiple_tags) space_tag = " ".join(multiple_tags) if slash_tag: follow_tags_url = reverse('follow-tags', args=[slash_tag]) unfollow_tags_url = reverse('unfollow-tags', args=[slash_tag]) show_unfollow_button = False if request.user.is_authenticated: show_unfollow_button = follow_utils.is_user_following_tag( request.user, slash_tag) return render(request, 'sounds/tags.html', locals())
def search_forum(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 current_forum_name_slug = request.GET.get( "forum", "").strip() # for context sensitive search if current_forum_name_slug: current_forum = get_object_or_404(forum.models.Forum.objects, name_slug=current_forum_name_slug) else: current_forum = None sort = ["thread_created desc"] # Parse advanced search options advanced_search = request.GET.get("advanced_search", "") date_from = request.GET.get("dt_from", "") try: df_parsed = datetime.datetime.strptime(date_from, "%Y-%m-%d") date_from_display = df_parsed.strftime("%d-%m-%Y") except ValueError: date_from = "" date_from_display = "Choose a Date" date_to = request.GET.get("dt_to", "") try: dt_parsed = datetime.datetime.strptime(date_to, "%Y-%m-%d") date_to_display = dt_parsed.strftime("%d-%m-%Y") except ValueError: date_to = "" date_to_display = "Choose a Date" if search_query.startswith("search in"): search_query = "" error = False error_text = "" paginator = None num_results = None page = None results = [] if search_query.strip() != "" or filter_query: # add current forum if current_forum: filter_query += "forum_name_slug:" + current_forum.name_slug # add date range if advanced_search == "1" and date_from != "" or date_to != "": filter_query = __add_date_range(filter_query, date_from, date_to) query = SolrQuery() query.set_dismax_query(search_query, query_fields=[("thread_title", 4), ("post_body", 3), ("thread_author", 3), ("post_author", 3), ("forum_name", 2)]) query.set_highlighting_options_default( field_list=["post_body"], fragment_size=200, alternate_field="post_body", # TODO: revise this param require_field_match=False, pre="<strong>", post="</strong>") query.set_query_options( start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=[ "id", "forum_name", "forum_name_slug", "thread_id", "thread_title", "thread_author", "thread_created", "post_body", "post_author", "post_created", "num_posts" ], filter_query=filter_query, sort=sort) query.set_group_field("thread_title_grouped") query.set_group_options(group_limit=30) solr = Solr(settings.SOLR_FORUM_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator( results, settings.SOUNDS_PER_PAGE) num_results = paginator.count page = paginator.page(current_page) error = False except SolrException as e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?' except Exception as e: logger.error("Could probably not connect to Solr - %s" % e) error = True error_text = 'The search server could not be reached, please try again later.' tvars = { 'advanced_search': advanced_search, 'current_forum': current_forum, 'current_page': current_page, 'date_from': date_from, 'date_from_display': date_from_display, 'date_to': date_to, 'date_to_display': date_to_display, 'error': error, 'error_text': error_text, 'filter_query': filter_query, 'num_results': num_results, 'page': page, 'paginator': paginator, 'search_query': search_query, 'sort': sort, 'results': results, } return render(request, 'search/search_forum.html', tvars)
def search_forum(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 current_forum_name_slug = request.GET.get("current_forum_name_slug", "").strip() # for context sensitive search current_forum_name = request.GET.get("current_forum_name", "").strip() # used in breadcrumb sort = ["thread_created desc"] # Parse advanced search options advanced_search = request.GET.get("advanced_search", "") date_from = request.GET.get("dt_from", "") date_to = request.GET.get("dt_to", "") # TEMPORAL WORKAROUND!!! to prevent using watermark as the query for forum search... # It only happens in some situations. if "search in " in search_query: invalid = 1 if search_query.strip() != "" or filter_query: # add current forum if current_forum_name_slug.strip() != "": filter_query += "forum_name_slug:" + current_forum_name_slug # add date range if advanced_search == "1" and date_from != "" or date_to != "": filter_query = __add_date_range(filter_query, date_from, date_to) query = SolrQuery() query.set_dismax_query(search_query, query_fields=[("thread_title", 4), ("post_body", 3), ("thread_author", 3), ("post_author", 3), ("forum_name", 2)]) query.set_highlighting_options_default(field_list=["post_body"], fragment_size=200, alternate_field="post_body", # TODO: revise this param require_field_match=False, pre="<strong>", post="</strong>") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id", "forum_name", "forum_name_slug", "thread_id", "thread_title", "thread_author", "thread_created", "post_body", "post_author", "post_created", "num_posts"], filter_query=filter_query, sort=sort) query.set_group_field("thread_title_grouped") query.set_group_options(group_limit=30) solr = Solr(settings.SOLR_FORUM_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count page = paginator.page(current_page) error = False except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?' except Exception, e: logger.error("Could probably not connect to Solr - %s" % e) error = True error_text = 'The search server could not be reached, please try again later.'
def search(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") filter_query_link_more_when_grouping_packs = filter_query.replace(' ','+') try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 sort = request.GET.get("s", None) sort_options = forms.SEARCH_SORT_OPTIONS_WEB grouping = request.GET.get("g", "1") # Group by default actual_groupnig = grouping # If the query is filtered by pack, do not collapse sounds of the same pack (makes no sense) # If the query is thourhg ajax (for sources remix editing), do not collapse if "pack" in filter_query or request.GET.get("ajax", "") == "1": actual_groupnig = "" # Set default values id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # Parse advanced search options advanced = request.GET.get("advanced", "") # if advanced search if advanced == "1" : a_tag = request.GET.get("a_tag", "") a_filename = request.GET.get("a_filename", "") a_description = request.GET.get("a_description", "") a_packname = request.GET.get("a_packname", "") a_soundid = request.GET.get("a_soundid", "") a_username = request.GET.get("a_username", "") # If none is selected use all (so other filter can be appleid) if a_tag or a_filename or a_description or a_packname or a_soundid or a_username != "" : # Initialize all weights to 0 id_weight = 0 tag_weight = 0 description_weight = 0 username_weight = 0 pack_tokenized_weight = 0 original_filename_weight = 0 # Set the weights of selected checkboxes if a_soundid != "" : id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] if a_tag != "" : tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] if a_description != "" : description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] if a_username != "" : username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] if a_packname != "" : pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] if a_filename != "" : original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # ALLOW "q" empty queries #if search_query.strip() == "" sort = search_prepare_sort(sort, forms.SEARCH_SORT_OPTIONS_WEB) query = search_prepare_query(search_query, filter_query, sort, current_page, settings.SOUNDS_PER_PAGE, id_weight, tag_weight, description_weight, username_weight, pack_tokenized_weight, original_filename_weight, grouping = actual_groupnig ) solr = Solr(settings.SOLR_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s for d in docs: d["sound"] = allsounds[d["id"]] # clickusage tracking if settings.LOG_CLICKTHROUGH_DATA: request_full_path = request.get_full_path() # The session id of an unauthenticated user is different from the session id of the same user when # authenticated. request.session["searchtime_session_key"] = request.session.session_key if results.docs is not None: ids = [] for item in results.docs: ids.append(item["id"]) logger_click.info("QUERY : %s : %s : %s : %s" % (unicode(request_full_path).encode('utf-8'), request.session.session_key, unicode(ids).encode('utf-8'), unicode(current_page).encode('utf-8'))) except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?'
def search(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") filter_query_link_more_when_grouping_packs = filter_query.replace(' ', '+') # Generate array with information of filters filter_query_split = [] if filter_query != "": for filter_str in re.findall(r'[\w-]+:\"[^\"]+', filter_query): filter_str = filter_str + '"' filter_display = filter_str.replace('"', '') filter_name = filter_str.split(":")[0] if filter_name != "duration" and filter_name != "is_geotagged": if filter_name == "grouping_pack": val = filter_display.split(":")[1] filter_display = "pack:" + val.split("_")[1] filter = { 'name': filter_display, 'remove_url': filter_query.replace(filter_str, ''), } filter_query_split.append(filter) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 sort = request.GET.get("s", None) sort_options = forms.SEARCH_SORT_OPTIONS_WEB grouping = request.GET.get("g", "1") # Group by default actual_groupnig = grouping # If the query is filtered by pack, do not collapse sounds of the same pack (makes no sense) # If the query is thourhg ajax (for sources remix editing), do not collapse if "pack" in filter_query or request.GET.get("ajax", "") == "1": actual_groupnig = "" # Set default values id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'original_filename'] # Parse advanced search options advanced = request.GET.get("advanced", "") # if advanced search if advanced == "1": a_tag = request.GET.get("a_tag", "") a_filename = request.GET.get("a_filename", "") a_description = request.GET.get("a_description", "") a_packname = request.GET.get("a_packname", "") a_soundid = request.GET.get("a_soundid", "") a_username = request.GET.get("a_username", "") # If none is selected use all (so other filter can be appleid) if a_tag or a_filename or a_description or a_packname or a_soundid or a_username != "": # Initialize all weights to 0 id_weight = 0 tag_weight = 0 description_weight = 0 username_weight = 0 pack_tokenized_weight = 0 original_filename_weight = 0 # Set the weights of selected checkboxes if a_soundid != "": id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] if a_tag != "": tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] if a_description != "": description_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'description'] if a_username != "": username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] if a_packname != "": pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'pack_tokenized'] if a_filename != "": original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'original_filename'] # ALLOW "q" empty queries #if search_query.strip() == "" sort = search_prepare_sort(sort, forms.SEARCH_SORT_OPTIONS_WEB) logger.info(u'Search (%s)' % json.dumps({ 'ip': get_client_ip(request), 'query': search_query, 'filter': filter_query, 'username': request.user.username, 'page': current_page, 'sort': sort[0], 'group_by_pack': actual_groupnig, 'advanced': json.dumps({ 'search_in_tag': a_tag, 'search_in_filename': a_filename, 'search_in_description': a_description, 'search_in_packname': a_packname, 'search_in_soundid': a_soundid, 'search_in_username': a_username }) if advanced == "1" else "" })) query = search_prepare_query(search_query, filter_query, sort, current_page, settings.SOUNDS_PER_PAGE, id_weight, tag_weight, description_weight, username_weight, pack_tokenized_weight, original_filename_weight, grouping=actual_groupnig) solr = Solr(settings.SOLR_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s # allsounds will contain info from all the sounds returned by bulk_query_id. This should # be all sounds in docs, but if solr and db are not synchronised, it might happen that there # are ids in docs which are not found in bulk_query_id. To avoid problems we remove elements # in docs that have not been loaded in allsounds. docs = [doc for doc in docs if doc["id"] in allsounds] for d in docs: d["sound"] = allsounds[d["id"]] except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?'