class Search(search_views.SearchView): def get_redirect_search_entity(self): return next(s for s in self.search_entities if s.name == 'product') # ignore CPDBear search_entities = [ search_engine.Search( name='category', qs=Category.objects.active(), fields=['name'], # Ignore CPDBear min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='product', qs=Product.objects.active(), fields=['name'], redirect_field='vendor_code', min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='page', # Ignore CPDBear qs=Page.objects.filter(is_active=True).exclude( type=Page.MODEL_TYPE), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ) ] redirect_field = 'vendor_code'
class Search(search_views.SearchView): def get_redirect_search_entity(self): return next(s for s in self.search_entities if s.name == 'product') # ignore CPDBear search_entities = [ search_engine.Search( name='category', qs=Category.objects.all(), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='product', qs=Product.objects.all(), fields=['name'], redirect_field='vendor_code', min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='page', qs=ExcludedModelTPage.objects.all(), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ) ] redirect_field = 'vendor_code'
class Autocomplete(search_views.AutocompleteView): # ignore CPDBear search_entities = [ search_engine.Search( name='category', qs=Category.objects.filter(page__is_active=True), fields=['name', 'id'], template_fields=['name', 'url'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='product', qs=Product.objects.active(), fields=['name', 'id', 'vendor_code'], template_fields=['name', 'price', 'url'], # Ignore CPDBear min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='pages', qs=Page.objects.filter(is_active=True).exclude( type=Page.MODEL_TYPE), fields=['name'], template_fields=['name', 'url'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ) ] see_all_label = settings.SEARCH_SEE_ALL_LABEL
class Autocomplete(search_views.AutocompleteView): # ignore CPDBear search_entities = [ search_engine.Search( name='category', qs=Category.objects.all(), fields=['name', 'id'], template_fields=['name', 'url'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='product', qs=Product.objects.all(), fields=['name', 'id', 'vendor_code'], template_fields=['name', 'price', 'url'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='pages', qs=ExcludedModelTPage.objects.all(), fields=['name'], template_fields=['name', 'url'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ) ] see_all_label = settings.SEARCH_SEE_ALL_LABEL
class MockSearchView(search.SearchView): search_entities = [ search_engine.Search( name='category', qs=MockCategory.objects.all(), fields=['name', 'id'], ), search_engine.Search( name='product', qs=MockProduct.objects.all(), fields=['name', 'id'], ) ]
class MockAdminAutocompleteView(search.AdminAutocompleteView): search_entities = [ search_engine.Search( name='category', qs=MockCategory.objects.all(), fields=['name', 'id'], template_fields=['name', 'url'], ), search_engine.Search( name='product', qs=MockProduct.objects.all(), fields=['name', 'id'], template_fields=['name', 'price', 'url'], ) ]
class AdminAutocomplete(search_views.AdminAutocompleteView): # ignore CPDBear search_entities = [ search_engine.Search( name='category', qs=Category.objects.all(), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='product', qs=Product.objects.all(), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='pages', qs=ExcludedModelTPage.objects.all(), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ) ]
class AdminAutocomplete(search_views.AdminAutocompleteView): # ignore CPDBear search_entities = [ search_engine.Search( name='category', qs=Category.objects.filter(page__is_active=True), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='product', qs=Product.objects.active(), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ), search_engine.Search( name='pages', qs=Page.objects.filter(is_active=True).exclude(type=Page.MODEL_TYPE), fields=['name'], min_similarity=settings.TRIGRAM_MIN_SIMILARITY, ) ]
def _generate_results_for(city, week_start): start_time = week_start end_time = start_time + datetime.timedelta(days=8) latlng_bounds = ((city.latitude, city.longitude), (city.latitude, city.longitude)) city_bounds = math.expand_bounds(latlng_bounds, cities.NEARBY_DISTANCE_KM) search_query = search_base.SearchQuery( time_period=search_base.TIME_ALL_FUTURE, start_date=start_time, end_date=end_time, bounds=city_bounds) searcher = search.Search(search_query) search_results = searcher.get_search_results(full_event=True) return search_results
def promote_events_to_user(user): # TODO: Adjust when we have iphone notifications if not android.can_notify(user): return logging.info("Promoting new events to user %s", user.fb_uid) # Only send notifications for Mike for now user = users.User.get_by_id(user.fb_uid) if not user: logging.error("No user found: %s", user.fb_uid) return if user.expired_oauth_token: logging.info("User has expired token, aborting: %s", user.fb_uid) return user_location = user.location if not user_location: return distance_in_km = user.distance_in_km() min_attendees = user.min_attendees # search for relevant events geocode = gmaps_api.get_geocode(address=user_location) if not geocode: return None bounds = math.expand_bounds(geocode.latlng_bounds(), distance_in_km) query = search_base.SearchQuery(time_period=search_base.TIME_UPCOMING, bounds=bounds, min_attendees=min_attendees) one_day_ago = time.mktime( (datetime.datetime.now() - datetime.timedelta(hours=24)).timetuple()) search_query = search.Search(query) search_query.extra_fields = ['creation_time'] search_results = search_query._get_candidate_doc_events() # TODO: can we move this filter into the search query itself?? recent_events = [ x.doc_id for x in search_results if x.field('creation_time').value > one_day_ago ] logging.info("Found %s search_results, %s new events", len(search_results), len(recent_events)) for event_id in recent_events: if android.add_notify(user, event_id): logging.info("Sent notification!")
def get(self): self.finish_preload() form = search_base.SearchForm( formdata=self.request.GET, data=self.user.dict_for_form() if self.user else None) if not form.validate(): logging.warning("Form errors: %s", form.errors) self.write_json_response([]) return search_query = form.build_query(start_end_query=True) search_results = search.Search(search_query).get_search_results() if 'class' in form.deb.data: from classes import class_index class_results = class_index.ClassSearch( search_query).get_search_results() search_results += class_results search_results.sort(key=lambda x: x.start_time) json_results = [] for result in search_results: start_time = result.start_time end_time = result.fake_end_time duration = end_time - start_time if duration > datetime.timedelta(days=5): end_time = start_time elif duration <= datetime.timedelta(days=1): end_time = start_time all_day = False title = '@ %s\n\n%s' % (result.actual_city_name, result.name) json_results.append( dict( id=result.event_id, title=title, start=start_time.strftime('%Y-%m-%dT%H:%M:%SZ'), end=end_time.strftime('%Y-%m-%dT%H:%M:%SZ'), url=urls.dd_event_url(result.event_id), allDay=all_day, )) self.write_json_response(json_results)
def get(self, name): topics = topic_db.Topic.query(topic_db.Topic.url_path == name).fetch(1) if not topics: self.response.set_status(404) return topic = topics[0] topic.init() def prefilter(doc_event): """Function for fitlering doc results, before we spend the energy to load the corresponding DBEvents. We only want on-topic events here: - Must contain keyword in the title - Must contain keyword on a line where it makes up >10% of the text (for judges, workshops, etc). We want to hide the resume-includes-classes-from-X people """ logging.info("Prefiltering event %s", doc_event.doc_id) name = doc_event.field('name').value.lower() description = doc_event.field('description').value.lower() description_lines = description.split('\n') for keyword in topic.search_keywords: keyword_word_re = re.compile(r'\b%s\b' % keyword) if keyword_word_re.search(name): return True for line in description_lines: result = keyword_word_re.search(line) # If the keyword is more than 10% of the text in the line: # Examples: # "- HOUSE - KAPELA (Serial Stepperz/Wanted Posse)" # "5th November : EVENT Judged by HIRO :" if result: if 1.0 * len(keyword) / len(line) > 0.1: return True else: logging.info( "Found keyword %r on line, but not long enough: %r", keyword, line) logging.info("Prefilter dropping event %s with name: %r" % (doc_event.doc_id, name)) return False keywords = ' OR '.join('"%s"' % x for x in topic.search_keywords) search_query = search_base.SearchQuery(keywords=keywords) # Need these fields for the prefilter search_query.extra_fields = ['name', 'description'] # TODO: query needs to include the 'all time' bits somehow, so we can grab all events for our topic pages searcher = search.Search(search_query) searcher.search_index = search.AllEventsIndex search_results = searcher.get_search_results(prefilter=prefilter) json_search_response = api.build_search_results_api( None, search_query, search_results, (2, 0), need_full_event=False, geocode=None, distance=None) videos = get_videos_for(topic.youtube_query) if topic.social().get('instagram'): instagrams = get_instagrams_for(topic.social()['instagram']) else: instagrams = {'items': []} topic_json = { 'title': topic.title(), 'description': topic.description(), 'image_url': topic.image_url(), 'social': topic.social(), } props = dict( response=json_search_response, videos=videos, instagrams=instagrams, topic=topic_json, ) self.setup_react_template('topic.js', props) self.render_template('topic')
def get(self, name): topics = topic_db.Topic.query(topic_db.Topic.url_path == name).fetch(1) if not topics: self.response.set_status(404) return topic = topics[0] if topic.graph_id: # We shouldn't need any tokens to access pages fbl = fb_api.FBLookup(None, None) fb_source = fbl.get(topic_db.LookupTopicPage, topic.graph_id) else: fb_source = None def prefilter(doc_event): """Function for fitlering doc results, before we spend the energy to load the corresponding DBEvents. We only want on-topic events here: - Must contain keyword in the title - Must contain keyword on a line where it makes up >10% of the text (for judges, workshops, etc). We want to hide the resume-includes-classes-from-X people """ logging.info("Prefiltering event %s", doc_event.doc_id) name = doc_event.field('name').value.lower() description = doc_event.field('description').value.lower() description_lines = description.split('\n') for keyword in topic.search_keywords: keyword_word_re = re.compile(r'\b%s\b' % keyword) if keyword_word_re.search(name): return True for line in description_lines: result = keyword_word_re.search(line) # If the keyword is more than 10% of the text in the line: # Examples: # "- HOUSE - KAPELA (Serial Stepperz/Wanted Posse)" # "5th November : EVENT Judged by HIRO :" if result: if 1.0 * len(keyword) / len(line) > 0.1: return True else: logging.info( "Found keyword %r on line, but not long enough: %r", keyword, line) logging.info("Prefilter dropping event %s with name: %r" % (doc_event.doc_id, name)) return False keywords = ' OR '.join('"%s"' % x for x in topic.search_keywords) search_query = search_base.SearchQuery(keywords=keywords) # Need these fields for the prefilter search_query.extra_fields = ['name', 'description'] search_results = search.Search(search_query).get_search_results( prefilter=prefilter) self.display['topic_title'] = topic.override_title or ( fb_source and fb_source['info']['name']) self.display['topic_image'] = topic.override_image or ( fb_source and fb_source['picture']['data']['url']) self.display['topic_description'] = topic.override_description or ( fb_source and fb_source['info'].get('about')) or '' self.display['all_results'] = search_results by_year = [] for year, month_events in sorted( grouping.group_results_by_date(search_results).items()): by_year.append((year, sorted(month_events.items()))) self.display['group_by_date'] = by_year by_country = sorted( grouping.group_results_by_location(search_results).items(), key=lambda x: (-len(x[1]), x[0])) self.display['group_by_location'] = by_country # TODO: # show points on map (future and past?) # show future events # show past events # show high quality and low quality events (most viable with 'past') # have an ajax filter on the page that lets me filter by location? self.display['fb_page'] = fb_source self.render_template('topic')
def get(self): data = { 'location': self.request.get('location'), 'keywords': self.request.get('keywords'), 'locale': self.request.get('locale'), } # If it's 1.0 clients, or web clients, then grab all data if self.version == (1, 0): time_period = search_base.TIME_UPCOMING else: time_period = self.request.get('time_period') data['time_period'] = time_period form = search_base.SearchForm(data=data) if not form.validate(): for field, errors in form.errors.items(): for error in errors: self.add_error(u"%s error: %s" % (getattr(form, field).label.text, error)) if not form.location.data: city_name = None southwest = None northeast = None if not form.keywords.data: if self.version == (1, 0): self.write_json_success({'results': []}) return else: self.add_error('Please enter a location or keywords') else: place = gmaps_api.fetch_place_as_json(query=form.location.data, language=form.locale.data) if place['status'] == 'OK' and place['results']: geocode = gmaps_api.GMapsGeocode(place['results'][0]) southwest, northeast = math.expand_bounds( geocode.latlng_bounds(), form.distance_in_km()) city_name = place['results'][0]['formatted_address'] # This will fail on a bad location, so let's verify the location is geocodable above first. else: if self.version == (1, 0): self.write_json_success({'results': []}) return else: self.add_error('Could not geocode location') self.errors_are_fatal() search_results = [] distances = [50, 100, 170, 300] distance_index = 0 while not search_results: form.distance.data = distances[distance_index] form.distance_units.data = 'miles' search_query = form.build_query() searcher = search.Search(search_query) # TODO(lambert): Increase the size limit when our clients can handle it. And improve our result sorting to return the 'best' results. searcher.limit = 500 search_results = searcher.get_search_results(full_event=True) # Increase our search distance in the hopes of finding something distance_index += 1 if distance_index == len(distances): # If we searched the world, then break break logging.info("Found %r events within %s %s of %s", form.keywords.data, form.distance.data, form.distance_units.data, form.location.data) onebox_links = onebox.get_links_for_query(search_query) json_results = [] for result in search_results: try: json_result = canonicalize_event_data(result.db_event, result.event_keywords) json_results.append(json_result) except Exception as e: logging.exception("Error processing event %s: %s" % (result.event_id, e)) title = self._get_title(city_name, form.keywords.data) json_response = { 'results': json_results, 'onebox_links': onebox_links, 'title': title, 'location': city_name, 'query': data, } if southwest and northeast: json_response['location_box'] = { 'southwest': { 'latitude': southwest[0], 'longitude': southwest[1], }, 'northeast': { 'latitude': northeast[0], 'longitude': northeast[1], }, } self.write_json_success(json_response)