def setUp(self): # Generate a fake solr response data to mock perform_solr_query function self.NUM_RESULTS = 15 sound_ids = list( Sound.objects.filter(moderation_state="OK", processing_state="OK").values_list( 'id', flat=True)[:self.NUM_RESULTS]) solr_select_returned_data['grouped']['grouping_pack']['groups'] = [{ 'doclist': { 'docs': [{ 'id': sound_id }], 'numFound': 1, 'start': 0 }, 'groupValue': str(sound_id) } for sound_id in sound_ids] results = SolrResponseInterpreter( copy.deepcopy(solr_select_returned_data)) # NOTE: in the line abve, we need to deepcopy the dictionary of results because SolrResponseInterpreter changes # it and makes it break when run a second time. Ideally SolrResponseInterpreter should be fixed so that it does # not change its input parameter. paginator = SolrResponseInterpreterPaginator(results, self.NUM_RESULTS) page = paginator.page(1) # Get first page self.perform_solr_query_response = \ (results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs)
def tags(request, multiple_tags=None): if multiple_tags: multiple_tags = multiple_tags.split('/') else: multiple_tags = [] multiple_tags = sorted(filter(lambda x: x, multiple_tags)) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 solr = Solr(settings.SOLR_URL) query = SolrQuery() if multiple_tags: query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags)) else: query.set_query("*:*") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"]) query.add_facet_fields("tag") query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False) query.set_group_field(group_field="grouping_pack") query.set_group_options(group_func=None, group_query=None, group_rows=10, group_start=0, group_limit=1, group_offset=0, group_sort=None, group_sort_ingroup=None, group_format='grouped', group_main=False, group_num_groups=True, group_cache_percent=0, group_truncate=True) # Sets how many results from the same grup are taken into account for computing the facets try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]] docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s for d in docs: d["sound"] = allsounds[d["id"]] except SolrException, e: error = True search_logger.error("SOLR ERROR - %s" % e)
def perform_solr_query(q, current_page): """ This util function performs the query to Solr and returns needed parameters to continue with the view. The main reason to have this util function is to facilitate mocking in unit tests for this view. """ solr = Solr(settings.SOLR_URL) results = SolrResponseInterpreter(solr.select(unicode(q))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) page = paginator.page(current_page) return results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs
def perform_solr_query(q, current_page): """ This util function performs the query to SOLR and returns needed parameters to continue with the view. The main reason to have this util function is to facilitate mocking in unit tests for this view. """ solr = Solr(settings.SOLR_URL) results = SolrResponseInterpreter(solr.select(unicode(q))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) page = paginator.page(current_page) return results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs
def setUp(self): # Generate a fake solr response data to mock perform_solr_query function self.NUM_RESULTS = 15 sound_ids = list(Sound.objects.filter( moderation_state="OK", processing_state="OK").values_list('id', flat=True)[:self.NUM_RESULTS]) solr_select_returned_data['grouped']['grouping_pack']['groups'] = [ {'doclist': {'docs': [{'id': sound_id}], 'numFound': 1, 'start': 0}, 'groupValue': str(sound_id)} for sound_id in sound_ids ] results = SolrResponseInterpreter(copy.deepcopy(solr_select_returned_data)) # NOTE: in the line abve, we need to deepcopy the dictionary of results because SolrResponseInterpreter changes # it and makes it break when run a second time. Ideally SolrResponseInterpreter should be fixed so that it does # not change its input parameter. paginator = SolrResponseInterpreterPaginator(results, self.NUM_RESULTS) page = paginator.page(1) # Get first page self.perform_solr_query_response = \ (results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs)
def tags(request, multiple_tags=None): if multiple_tags: multiple_tags = multiple_tags.split('/') else: multiple_tags = [] multiple_tags = sorted(filter(lambda x:x, multiple_tags)) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 solr = Solr(settings.SOLR_URL) query = SolrQuery() if multiple_tags: query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags)) else: query.set_query("*:*") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"]) query.add_facet_fields("tag") query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) page = paginator.page(current_page) error = False tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]] except SolrException, e: error = True search_logger.error("SOLR ERROR - %s" % e)
def search(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") filter_query_link_more_when_grouping_packs = filter_query.replace(' ','+') try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 sort = request.GET.get("s", None) sort_options = forms.SEARCH_SORT_OPTIONS_WEB grouping = request.GET.get("g", "1") # Group by default actual_groupnig = grouping # If the query is filtered by pack, do not collapse sounds of the same pack (makes no sense) # If the query is thourhg ajax (for sources remix editing), do not collapse if "pack" in filter_query or request.GET.get("ajax", "") == "1": actual_groupnig = "" # Set default values id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # Parse advanced search options advanced = request.GET.get("advanced", "") # if advanced search if advanced == "1" : a_tag = request.GET.get("a_tag", "") a_filename = request.GET.get("a_filename", "") a_description = request.GET.get("a_description", "") a_packname = request.GET.get("a_packname", "") a_soundid = request.GET.get("a_soundid", "") a_username = request.GET.get("a_username", "") # If none is selected use all (so other filter can be appleid) if a_tag or a_filename or a_description or a_packname or a_soundid or a_username != "" : # Initialize all weights to 0 id_weight = 0 tag_weight = 0 description_weight = 0 username_weight = 0 pack_tokenized_weight = 0 original_filename_weight = 0 # Set the weights of selected checkboxes if a_soundid != "" : id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] if a_tag != "" : tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] if a_description != "" : description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] if a_username != "" : username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] if a_packname != "" : pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] if a_filename != "" : original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # ALLOW "q" empty queries #if search_query.strip() == "" sort = search_prepare_sort(sort, forms.SEARCH_SORT_OPTIONS_WEB) logger.info(u'Search (%s)' % json.dumps({ 'ip': get_client_ip(request), 'query': search_query, 'filter': filter_query, 'username': request.user.username, 'page': current_page, 'sort': sort[0], 'group_by_pack' : actual_groupnig, 'advanced': json.dumps({ 'search_in_tag': a_tag, 'search_in_filename': a_filename, 'search_in_description': a_description, 'search_in_packname': a_packname, 'search_in_soundid': a_soundid, 'search_in_username': a_username }) if advanced == "1" else "" })) query = search_prepare_query(search_query, filter_query, sort, current_page, settings.SOUNDS_PER_PAGE, id_weight, tag_weight, description_weight, username_weight, pack_tokenized_weight, original_filename_weight, grouping = actual_groupnig ) solr = Solr(settings.SOLR_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s # allsounds will contain info from all the sounds returned by bulk_query_id. This should # be all sounds in docs, but if solr and db are not synchronised, it might happen that there # are ids in docs which are not found in bulk_query_id. To avoid problems we remove elements # in docs that have not been loaded in allsounds. docs = [doc for doc in docs if doc["id"] in allsounds] for d in docs: d["sound"] = allsounds[d["id"]] # clickusage tracking if settings.LOG_CLICKTHROUGH_DATA: request_full_path = request.get_full_path() # The session id of an unauthenticated user is different from the session id of the same user when # authenticated. request.session["searchtime_session_key"] = request.session.session_key if results.docs is not None: ids = [] for item in results.docs: ids.append(item["id"]) logger_click.info("QUERY : %s : %s : %s : %s" % (unicode(request_full_path).encode('utf-8'), request.session.session_key, unicode(ids).encode('utf-8'), unicode(current_page).encode('utf-8'))) except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?'
def search(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") filter_query_link_more_when_grouping_packs = filter_query.replace(' ','+') try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 sort = request.GET.get("s", None) sort_options = forms.SEARCH_SORT_OPTIONS_WEB grouping = request.GET.get("g", "1") # Group by default actual_groupnig = grouping # If the query is filtered by pack, do not collapse sounds of the same pack (makes no sense) # If the query is thourhg ajax (for sources remix editing), do not collapse if "pack" in filter_query or request.GET.get("ajax", "") == "1": actual_groupnig = "" # Set default values id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # Parse advanced search options advanced = request.GET.get("advanced", "") # if advanced search if advanced == "1" : a_tag = request.GET.get("a_tag", "") a_filename = request.GET.get("a_filename", "") a_description = request.GET.get("a_description", "") a_packname = request.GET.get("a_packname", "") a_soundid = request.GET.get("a_soundid", "") a_username = request.GET.get("a_username", "") # If none is selected use all (so other filter can be appleid) if a_tag or a_filename or a_description or a_packname or a_soundid or a_username != "" : # Initialize all weights to 0 id_weight = 0 tag_weight = 0 description_weight = 0 username_weight = 0 pack_tokenized_weight = 0 original_filename_weight = 0 # Set the weights of selected checkboxes if a_soundid != "" : id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] if a_tag != "" : tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] if a_description != "" : description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] if a_username != "" : username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] if a_packname != "" : pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] if a_filename != "" : original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # ALLOW "q" empty queries #if search_query.strip() == "" sort = search_prepare_sort(sort, forms.SEARCH_SORT_OPTIONS_WEB) query = search_prepare_query(search_query, filter_query, sort, current_page, settings.SOUNDS_PER_PAGE, id_weight, tag_weight, description_weight, username_weight, pack_tokenized_weight, original_filename_weight, grouping = actual_groupnig ) solr = Solr(settings.SOLR_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False # clickusage tracking if settings.LOG_CLICKTHROUGH_DATA: request_full_path = request.get_full_path() # The session id of an unauthenticated user is different from the session id of the same user when # authenticated. request.session["searchtime_session_key"] = request.session.session_key if results.docs is not None: ids = [] for item in results.docs: ids.append(item["id"]) logger_click.info("QUERY : %s : %s : %s : %s" % (unicode(request_full_path).encode('utf-8'), request.session.session_key, unicode(ids).encode('utf-8'), unicode(current_page).encode('utf-8'))) except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?'
def search_forum(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 current_forum_name_slug = request.GET.get( "forum", "").strip() # for context sensitive search if current_forum_name_slug: current_forum = get_object_or_404(forum.models.Forum.objects, name_slug=current_forum_name_slug) else: current_forum = None sort = ["thread_created desc"] # Parse advanced search options advanced_search = request.GET.get("advanced_search", "") date_from = request.GET.get("dt_from", "") try: df_parsed = datetime.datetime.strptime(date_from, "%Y-%m-%d") date_from_display = df_parsed.strftime("%d-%m-%Y") except ValueError: date_from = "" date_from_display = "Choose a Date" date_to = request.GET.get("dt_to", "") try: dt_parsed = datetime.datetime.strptime(date_to, "%Y-%m-%d") date_to_display = dt_parsed.strftime("%d-%m-%Y") except ValueError: date_to = "" date_to_display = "Choose a Date" if search_query.startswith("search in"): search_query = "" error = False error_text = "" paginator = None num_results = None page = None results = [] if search_query.strip() != "" or filter_query: # add current forum if current_forum: filter_query += "forum_name_slug:" + current_forum.name_slug # add date range if advanced_search == "1" and date_from != "" or date_to != "": filter_query = __add_date_range(filter_query, date_from, date_to) query = SolrQuery() query.set_dismax_query(search_query, query_fields=[("thread_title", 4), ("post_body", 3), ("thread_author", 3), ("post_author", 3), ("forum_name", 2)]) query.set_highlighting_options_default( field_list=["post_body"], fragment_size=200, alternate_field="post_body", # TODO: revise this param require_field_match=False, pre="<strong>", post="</strong>") query.set_query_options( start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=[ "id", "forum_name", "forum_name_slug", "thread_id", "thread_title", "thread_author", "thread_created", "post_body", "post_author", "post_created", "num_posts" ], filter_query=filter_query, sort=sort) query.set_group_field("thread_title_grouped") query.set_group_options(group_limit=30) solr = Solr(settings.SOLR_FORUM_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator( results, settings.SOUNDS_PER_PAGE) num_results = paginator.count page = paginator.page(current_page) error = False except SolrException as e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?' except Exception as e: logger.error("Could probably not connect to Solr - %s" % e) error = True error_text = 'The search server could not be reached, please try again later.' tvars = { 'advanced_search': advanced_search, 'current_forum': current_forum, 'current_page': current_page, 'date_from': date_from, 'date_from_display': date_from_display, 'date_to': date_to, 'date_to_display': date_to_display, 'error': error, 'error_text': error_text, 'filter_query': filter_query, 'num_results': num_results, 'page': page, 'paginator': paginator, 'search_query': search_query, 'sort': sort, 'results': results, } return render(request, 'search/search_forum.html', tvars)
def search_forum(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 current_forum_name_slug = request.GET.get("current_forum_name_slug", "").strip() # for context sensitive search current_forum_name = request.GET.get("current_forum_name", "").strip() # used in breadcrumb sort = ["thread_created desc"] # Parse advanced search options advanced_search = request.GET.get("advanced_search", "") date_from = request.GET.get("dt_from", "") date_to = request.GET.get("dt_to", "") # TEMPORAL WORKAROUND!!! to prevent using watermark as the query for forum search... # It only happens in some situations. if "search in " in search_query: invalid = 1 if search_query.strip() != "" or filter_query: # add current forum if current_forum_name_slug.strip() != "": filter_query += "forum_name_slug:" + current_forum_name_slug # add date range if advanced_search == "1" and date_from != "" or date_to != "": filter_query = __add_date_range(filter_query, date_from, date_to) query = SolrQuery() query.set_dismax_query(search_query, query_fields=[("thread_title", 4), ("post_body", 3), ("thread_author", 3), ("post_author", 3), ("forum_name", 2)]) query.set_highlighting_options_default(field_list=["post_body"], fragment_size=200, alternate_field="post_body", # TODO: revise this param require_field_match=False, pre="<strong>", post="</strong>") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id", "forum_name", "forum_name_slug", "thread_id", "thread_title", "thread_author", "thread_created", "post_body", "post_author", "post_created", "num_posts"], filter_query=filter_query, sort=sort) query.set_group_field("thread_title_grouped") query.set_group_options(group_limit=30) solr = Solr(settings.SOLR_FORUM_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count page = paginator.page(current_page) error = False except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?' except Exception, e: logger.error("Could probably not connect to Solr - %s" % e) error = True error_text = 'The search server could not be reached, please try again later.'
def search(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") filter_query_link_more_when_grouping_packs = filter_query.replace(' ', '+') # Generate array with information of filters filter_query_split = [] if filter_query != "": for filter_str in re.findall(r'[\w-]+:\"[^\"]+', filter_query): filter_str = filter_str + '"' filter_display = filter_str.replace('"', '') filter_name = filter_str.split(":")[0] if filter_name != "duration" and filter_name != "is_geotagged": if filter_name == "grouping_pack": val = filter_display.split(":")[1] filter_display = "pack:" + val.split("_")[1] filter = { 'name': filter_display, 'remove_url': filter_query.replace(filter_str, ''), } filter_query_split.append(filter) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 sort = request.GET.get("s", None) sort_options = forms.SEARCH_SORT_OPTIONS_WEB grouping = request.GET.get("g", "1") # Group by default actual_groupnig = grouping # If the query is filtered by pack, do not collapse sounds of the same pack (makes no sense) # If the query is thourhg ajax (for sources remix editing), do not collapse if "pack" in filter_query or request.GET.get("ajax", "") == "1": actual_groupnig = "" # Set default values id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'original_filename'] # Parse advanced search options advanced = request.GET.get("advanced", "") # if advanced search if advanced == "1": a_tag = request.GET.get("a_tag", "") a_filename = request.GET.get("a_filename", "") a_description = request.GET.get("a_description", "") a_packname = request.GET.get("a_packname", "") a_soundid = request.GET.get("a_soundid", "") a_username = request.GET.get("a_username", "") # If none is selected use all (so other filter can be appleid) if a_tag or a_filename or a_description or a_packname or a_soundid or a_username != "": # Initialize all weights to 0 id_weight = 0 tag_weight = 0 description_weight = 0 username_weight = 0 pack_tokenized_weight = 0 original_filename_weight = 0 # Set the weights of selected checkboxes if a_soundid != "": id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] if a_tag != "": tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] if a_description != "": description_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'description'] if a_username != "": username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] if a_packname != "": pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'pack_tokenized'] if a_filename != "": original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS[ 'original_filename'] # ALLOW "q" empty queries #if search_query.strip() == "" sort = search_prepare_sort(sort, forms.SEARCH_SORT_OPTIONS_WEB) logger.info(u'Search (%s)' % json.dumps({ 'ip': get_client_ip(request), 'query': search_query, 'filter': filter_query, 'username': request.user.username, 'page': current_page, 'sort': sort[0], 'group_by_pack': actual_groupnig, 'advanced': json.dumps({ 'search_in_tag': a_tag, 'search_in_filename': a_filename, 'search_in_description': a_description, 'search_in_packname': a_packname, 'search_in_soundid': a_soundid, 'search_in_username': a_username }) if advanced == "1" else "" })) query = search_prepare_query(search_query, filter_query, sort, current_page, settings.SOUNDS_PER_PAGE, id_weight, tag_weight, description_weight, username_weight, pack_tokenized_weight, original_filename_weight, grouping=actual_groupnig) solr = Solr(settings.SOLR_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s # allsounds will contain info from all the sounds returned by bulk_query_id. This should # be all sounds in docs, but if solr and db are not synchronised, it might happen that there # are ids in docs which are not found in bulk_query_id. To avoid problems we remove elements # in docs that have not been loaded in allsounds. docs = [doc for doc in docs if doc["id"] in allsounds] for d in docs: d["sound"] = allsounds[d["id"]] except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?'
def read(self, request): ip = get_client_ip(request) form = SoundSearchForm(SEARCH_SORT_OPTIONS_API, request.GET) if not form.is_valid(): resp = rc.BAD_REQUEST resp.content = form.errors return resp cd = form.cleaned_data grouping = request.GET.get("g", "") if grouping == "0": grouping = "" solr = Solr(settings.SOLR_URL) sounds_per_page = min(int(request.GET.get('sounds_per_page', settings.SOUNDS_PER_API_RESPONSE)),settings.MAX_SOUNDS_PER_API_RESPONSE) query = search_prepare_query(cd['q'], cd['f'], search_prepare_sort(cd['s'], SEARCH_SORT_OPTIONS_API), cd['p'], sounds_per_page, grouping = grouping) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results,sounds_per_page) page = paginator.page(form.cleaned_data['p']) sounds = [] bad_results = 0 for object in page['object_list'] : try: sound = prepare_collection_sound(Sound.objects.select_related('user').get(id=object['id']), custom_fields = request.GET.get('fields', False)) if 'more_from_pack' in object.keys(): if object['more_from_pack'] > 0: link = prepend_base(reverse('api-search')+'?q=%s&f=pack:"%s" %s&s=%s&g=%s' % (my_quote(cd['q']),object['pack_name'],my_quote(cd['f']),cd['s'],"")) if request.GET.get('sounds_per_page', None): link += "&sounds_per_page=" + str(request.GET.get('sounds_per_page', None)) if request.GET.get('fields', False): link += "&fields=" + str(request.GET.get('fields', False)) sound['results_from_the_same_pack'] = link sound['n_results_from_the_same_pack'] = object['more_from_pack'] sounds.append(sound) except: # This will happen if there are synchronization errors between solr index and the database. In that case sounds are ommited and both num_results and results per page might become inacurate pass result = {'sounds': sounds, 'num_results': paginator.count - bad_results, 'num_pages': paginator.num_pages} # construct previous and next urls if page['has_other_pages']: if page['has_previous']: result['previous'] = self.__construct_pagination_link(cd['q'], page['previous_page_number'], cd['f'], find_api_option(cd['s']), request.GET.get('sounds_per_page', None), request.GET.get('fields', False), grouping) if page['has_next']: result['next'] = self.__construct_pagination_link(cd['q'], page['next_page_number'], cd['f'], find_api_option(cd['s']), request.GET.get('sounds_per_page',None), request.GET.get('fields', False), grouping) add_request_id(request,result) logger.info("Searching,q=" + cd['q'] + ",f=" + cd['f'] + ",p=" + str(cd['p']) + ",sounds_per_page=" + str(sounds_per_page) + ",api_key=" + request.GET.get("api_key", False) + ",api_key_username="******",ip=" + ip) return result except SolrException, e: error = "search_query %s filter_query %s sort %s error %s" \ % (cd['s'], cd['f'], cd['s'], e) raise ReturnError(500, "SearchError", {"explanation": error})
def tags(request, multiple_tags=None): if multiple_tags: multiple_tags = multiple_tags.split('/') else: multiple_tags = [] multiple_tags = sorted(filter(lambda x: x, multiple_tags)) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 solr = Solr(settings.SOLR_URL) query = SolrQuery() if multiple_tags: query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags)) else: query.set_query("*:*") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"]) query.add_facet_fields("tag") query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False) query.set_group_field(group_field="grouping_pack") query.set_group_options(group_func=None, group_query=None, group_rows=10, group_start=0, group_limit=1, group_offset=0, group_sort=None, group_sort_ingroup=None, group_format='grouped', group_main=False, group_num_groups=True, group_cache_percent=0, group_truncate=True) # Sets how many results from the same group are taken into account for computing the facets page = None num_results = 0 tags = [] error = False docs = {} non_grouped_number_of_results = 0 paginator = None try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]] docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s for d in docs: d["sound"] = allsounds[d["id"]] except SolrException as e: error = True search_logger.error("SOLR ERROR - %s" % e) except: error = True slash_tag = "/".join(multiple_tags) follow_tags_url = '' unfollow_tags_url = '' show_unfollow_button = False if slash_tag: follow_tags_url = reverse('follow-tags', args=[slash_tag]) unfollow_tags_url = reverse('unfollow-tags', args=[slash_tag]) show_unfollow_button = False if request.user.is_authenticated: show_unfollow_button = follow_utils.is_user_following_tag(request.user, slash_tag) tvars = {'show_unfollow_button': show_unfollow_button, 'multiple_tags': multiple_tags, 'follow_tags_url': follow_tags_url, 'unfollow_tags_url': unfollow_tags_url, 'error': error, 'tags': tags, 'slash_tag': slash_tag, 'num_results': num_results, 'non_grouped_number_of_results': non_grouped_number_of_results, 'docs': docs, 'paginator': paginator, 'page': page, 'current_page': current_page } return render(request, 'sounds/tags.html', tvars)
def search_forum(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 current_forum_name_slug = request.GET.get("forum", "").strip() # for context sensitive search if current_forum_name_slug: current_forum = get_object_or_404(forum.models.Forum.objects, name_slug=current_forum_name_slug) else: current_forum = None sort = ["thread_created desc"] # Parse advanced search options advanced_search = request.GET.get("advanced_search", "") date_from = request.GET.get("dt_from", "") try: df_parsed = datetime.datetime.strptime(date_from, "%Y-%m-%d") date_from_display = df_parsed.strftime("%d-%m-%Y") except ValueError: date_from = "" date_from_display = "Choose a Date" date_to = request.GET.get("dt_to", "") try: dt_parsed = datetime.datetime.strptime(date_to, "%Y-%m-%d") date_to_display = dt_parsed.strftime("%d-%m-%Y") except ValueError: date_to = "" date_to_display = "Choose a Date" if search_query.startswith("search in"): search_query = "" error = False error_text = "" paginator = None num_results = None page = None results = [] if search_query.strip() != "" or filter_query: # add current forum if current_forum: filter_query += "forum_name_slug:" + current_forum.name_slug # add date range if advanced_search == "1" and date_from != "" or date_to != "": filter_query = __add_date_range(filter_query, date_from, date_to) query = SolrQuery() query.set_dismax_query(search_query, query_fields=[("thread_title", 4), ("post_body", 3), ("thread_author", 3), ("post_author", 3), ("forum_name", 2)]) query.set_highlighting_options_default(field_list=["post_body"], fragment_size=200, alternate_field="post_body", # TODO: revise this param require_field_match=False, pre="<strong>", post="</strong>") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id", "forum_name", "forum_name_slug", "thread_id", "thread_title", "thread_author", "thread_created", "post_body", "post_author", "post_created", "num_posts"], filter_query=filter_query, sort=sort) query.set_group_field("thread_title_grouped") query.set_group_options(group_limit=30) solr = Solr(settings.SOLR_FORUM_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count page = paginator.page(current_page) error = False except SolrException as e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?' except Exception as e: logger.error("Could probably not connect to Solr - %s" % e) error = True error_text = 'The search server could not be reached, please try again later.' tvars = { 'advanced_search': advanced_search, 'current_forum': current_forum, 'current_page': current_page, 'date_from': date_from, 'date_from_display': date_from_display, 'date_to': date_to, 'date_to_display': date_to_display, 'error': error, 'error_text': error_text, 'filter_query': filter_query, 'num_results': num_results, 'page': page, 'paginator': paginator, 'search_query': search_query, 'sort': sort, 'results': results, } return render(request, 'search/search_forum.html', tvars)
def tags(request, multiple_tags=None): if multiple_tags: multiple_tags = multiple_tags.split('/') else: multiple_tags = [] multiple_tags = sorted(filter(lambda x: x, multiple_tags)) try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 solr = Solr(settings.SOLR_URL) query = SolrQuery() if multiple_tags: query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags)) else: query.set_query("*:*") query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"]) query.add_facet_fields("tag") query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False) query.set_group_field(group_field="grouping_pack") query.set_group_options( group_func=None, group_query=None, group_rows=10, group_start=0, group_limit=1, group_offset=0, group_sort=None, group_sort_ingroup=None, group_format='grouped', group_main=False, group_num_groups=True, group_cache_percent=0, group_truncate=True ) # Sets how many results from the same grup are taken into account for computing the facets try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]] docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s for d in docs: d["sound"] = allsounds[d["id"]] except SolrException as e: error = True search_logger.error("SOLR ERROR - %s" % e) except: error = True slash_tag = "/".join(multiple_tags) space_tag = " ".join(multiple_tags) if slash_tag: follow_tags_url = reverse('follow-tags', args=[slash_tag]) unfollow_tags_url = reverse('unfollow-tags', args=[slash_tag]) show_unfollow_button = False if request.user.is_authenticated: show_unfollow_button = follow_utils.is_user_following_tag( request.user, slash_tag) return render(request, 'sounds/tags.html', locals())
def search(request): search_query = request.GET.get("q", "") filter_query = request.GET.get("f", "") filter_query_link_more_when_grouping_packs = filter_query.replace(' ','+') try: current_page = int(request.GET.get("page", 1)) except ValueError: current_page = 1 sort = request.GET.get("s", None) sort_options = forms.SEARCH_SORT_OPTIONS_WEB grouping = request.GET.get("g", "1") # Group by default actual_groupnig = grouping # If the query is filtered by pack, do not collapse sounds of the same pack (makes no sense) # If the query is thourhg ajax (for sources remix editing), do not collapse if "pack" in filter_query or request.GET.get("ajax", "") == "1": actual_groupnig = "" # Set default values id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # Parse advanced search options advanced = request.GET.get("advanced", "") # if advanced search if advanced == "1" : a_tag = request.GET.get("a_tag", "") a_filename = request.GET.get("a_filename", "") a_description = request.GET.get("a_description", "") a_packname = request.GET.get("a_packname", "") a_soundid = request.GET.get("a_soundid", "") a_username = request.GET.get("a_username", "") # If none is selected use all (so other filter can be appleid) if a_tag or a_filename or a_description or a_packname or a_soundid or a_username != "" : # Initialize all weights to 0 id_weight = 0 tag_weight = 0 description_weight = 0 username_weight = 0 pack_tokenized_weight = 0 original_filename_weight = 0 # Set the weights of selected checkboxes if a_soundid != "" : id_weight = settings.DEFAULT_SEARCH_WEIGHTS['id'] if a_tag != "" : tag_weight = settings.DEFAULT_SEARCH_WEIGHTS['tag'] if a_description != "" : description_weight = settings.DEFAULT_SEARCH_WEIGHTS['description'] if a_username != "" : username_weight = settings.DEFAULT_SEARCH_WEIGHTS['username'] if a_packname != "" : pack_tokenized_weight = settings.DEFAULT_SEARCH_WEIGHTS['pack_tokenized'] if a_filename != "" : original_filename_weight = settings.DEFAULT_SEARCH_WEIGHTS['original_filename'] # ALLOW "q" empty queries #if search_query.strip() == "" sort = search_prepare_sort(sort, forms.SEARCH_SORT_OPTIONS_WEB) query = search_prepare_query(search_query, filter_query, sort, current_page, settings.SOUNDS_PER_PAGE, id_weight, tag_weight, description_weight, username_weight, pack_tokenized_weight, original_filename_weight, grouping = actual_groupnig ) solr = Solr(settings.SOLR_URL) try: results = SolrResponseInterpreter(solr.select(unicode(query))) paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE) num_results = paginator.count non_grouped_number_of_results = results.non_grouped_number_of_matches page = paginator.page(current_page) error = False docs = results.docs resultids = [d.get("id") for d in docs] resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids) allsounds = {} for s in resultsounds: allsounds[s.id] = s for d in docs: d["sound"] = allsounds[d["id"]] # clickusage tracking if settings.LOG_CLICKTHROUGH_DATA: request_full_path = request.get_full_path() # The session id of an unauthenticated user is different from the session id of the same user when # authenticated. request.session["searchtime_session_key"] = request.session.session_key if results.docs is not None: ids = [] for item in results.docs: ids.append(item["id"]) logger_click.info("QUERY : %s : %s : %s : %s" % (unicode(request_full_path).encode('utf-8'), request.session.session_key, unicode(ids).encode('utf-8'), unicode(current_page).encode('utf-8'))) except SolrException, e: logger.warning("search error: query: %s error %s" % (query, e)) error = True error_text = 'There was an error while searching, is your query correct?'