def create_instances(self, data): created_count = 0 for i, row in enumerate(data): name = self.clean_string(row["Name"]) if not name: continue try: url = self.clean_string(row["URL"]) website = Website.objects.get(url=url) print "Game with URL %s already exists. Continuing." % url continue except Website.DoesNotExist: pass with atomic(): try: website = Website( categories=["games"], devices=[DEVICE_DESKTOP.id], description=self.clean_string(row["Description"]), name=name, status=STATUS_PUBLIC, ) self.set_url(website, row) website.save() # Keywords use a M2M, so do that once the website is saved. self.set_tags(website, row) # Launch task to fetch imgs once we know everything is OK. try: self.set_icon(website, row) self.set_promo_imgs(website, row) WebsiteIndexer.index_ids([website.id], no_delay=True) except Exception as e: print e WebsiteIndexer.refresh_index() website.delete() raise e created_count += 1 except ParsingError as e: print e.message return created_count
def create_instances(self, data): created_count = 0 for i, row in enumerate(data): name = self.clean_string(row['Name']) if not name: continue try: url = self.clean_string(row['URL']) website = Website.objects.get(url=url) print 'Game with URL %s already exists. Continuing.' % url continue except Website.DoesNotExist: pass with atomic(): try: website = Website( categories=['games'], devices=[DEVICE_DESKTOP.id], description=self.clean_string(row['Description']), name=name, status=STATUS_PUBLIC, ) self.set_url(website, row) website.save() # Keywords use a M2M, so do that once the website is saved. self.set_tags(website, row) # Launch task to fetch imgs once we know everything is OK. try: self.set_icon(website, row) self.set_promo_imgs(website, row) WebsiteIndexer.index_ids([website.id], no_delay=True) except Exception as e: print e WebsiteIndexer.refresh_index() website.delete() raise e created_count += 1 except ParsingError as e: print e.message return created_count
def _get_indices(self): # Check if we are filtering by a doc_type (e.g., apps, sites). # Default to all content types. doc_type = self.request.GET.get('doc_type', 'all') app_index = WebappIndexer.get_index() site_index = WebsiteIndexer.get_index() if doc_type == 'webapp': return [app_index] elif doc_type == 'website': return [site_index] return [app_index, site_index]
def _get_doc_types(self): # Check if we are filtering by a doc_type (e.g., apps, sites). # Default to all content types. doc_type = self.request.GET.get('doc_type', 'all') app_doc = WebappIndexer.get_mapping_type_name() site_doc = WebsiteIndexer.get_mapping_type_name() if doc_type == 'webapp': return [app_doc] elif doc_type == 'website': return [site_doc] return [app_doc, site_doc]
def get_featured_websites(self): """ Get up to 11 featured MOWs for the request's region. If less than 11 are available, make up the difference with globally-featured MOWs. """ REGION_TAG = "featured-website-%s" % self.request.REGION.slug region_filter = es_filter.Term(tags=REGION_TAG) GLOBAL_TAG = "featured-website" global_filter = es_filter.Term(tags=GLOBAL_TAG) mow_query = query.Q( "function_score", filter=es_filter.Bool(should=[region_filter, global_filter]), functions=[ SF("random_score", seed=self._get_daily_seed()), es_function.BoostFactor(value=100.0, filter=region_filter), ], ) es = Search(using=WebsiteIndexer.get_es())[:11] results = es.query(mow_query).execute().hits return ESWebsiteSerializer(results, many=True).data
def get_featured_websites(self): """ Get up to 11 featured MOWs for the request's region. If less than 11 are available, make up the difference with globally-featured MOWs. """ REGION_TAG = 'featured-website-%s' % self.request.REGION.slug region_filter = es_filter.Term(tags=REGION_TAG) GLOBAL_TAG = 'featured-website' global_filter = es_filter.Term(tags=GLOBAL_TAG) mow_query = query.Q( 'function_score', filter=es_filter.Bool(should=[region_filter, global_filter]), functions=[ SF('random_score', seed=self._get_daily_seed()), es_function.BoostFactor(value=100.0, filter=region_filter) ], ) es = Search(using=WebsiteIndexer.get_es())[:11] results = es.query(mow_query).execute().hits return ESWebsiteSerializer(results, many=True).data
def test_excluded_fields(self): ok_(WebsiteIndexer.hidden_fields) data = WebsiteIndexer.search().execute().hits eq_(len(data), 1) obj = data[0] ok_('trending_2' not in obj) ok_('popularity_2' not in obj) ok_('description_translations' in obj) ok_('description' not in obj) ok_('description_l10n_english' not in obj) ok_('name_translations' in obj) ok_('name' not in obj) ok_('name_l10n_english' not in obj) ok_('name_sort' not in obj) ok_('name.raw' not in obj) ok_('short_name_translations' in obj) ok_('short_name' not in obj) ok_('short_name_l10n_english' not in obj)
def test_excluded_fields(self): ok_(WebsiteIndexer.hidden_fields) data = WebsiteIndexer.search().execute().hits eq_(len(data), 1) obj = data[0] ok_("trending_2" not in obj) ok_("popularity_2" not in obj) ok_("description_translations" in obj) ok_("description" not in obj) ok_("description_l10n_english" not in obj) ok_("name_translations" in obj) ok_("name" not in obj) ok_("name_l10n_english" not in obj) ok_("name_sort" not in obj) ok_("name.raw" not in obj) ok_("short_name_translations" in obj) ok_("short_name" not in obj) ok_("short_name_l10n_english" not in obj)
def test_mapping(self): mapping = WebsiteIndexer.get_mapping() eq_(mapping.keys(), ["website"]) eq_(mapping["website"]["_all"], {"enabled": False})
def test_mapping(self): mapping = WebsiteIndexer.get_mapping() eq_(mapping.keys(), ['website']) eq_(mapping['website']['_all'], {'enabled': False})
def test_index(self): with self.settings(ES_INDEXES={'website': 'websites'}): eq_(WebsiteIndexer.get_index(), 'websites')
def test_get_mapping_ok(self): eq_(WebsiteIndexer.get_mapping_type_name(), 'website') ok_(isinstance(self.indexer.get_mapping(), dict))
def serialize(self, site, profile=None): data = WebsiteIndexer.search().filter('term', id=site.pk).execute().hits[0] a = TVESWebsiteSerializer(instance=data, context={'request': self.request}) return a.data
def serialize(self, site, profile=None): data = WebsiteIndexer.search().filter("term", id=site.pk).execute().hits[0] a = TVESWebsiteSerializer(instance=data, context={"request": self.request}) return a.data
def get_queryset(self): return Search( using=BaseIndexer.get_es(), index=[WebappIndexer.get_index(), WebsiteIndexer.get_index()], doc_type=[WebappIndexer.get_mapping_type_name(), WebsiteIndexer.get_mapping_type_name()])
def get_obj(self): return WebsiteIndexer.search().filter( 'term', id=self.obj.pk).execute().hits[0]
def serialize(self): obj = WebsiteIndexer.search().filter( 'term', id=self.website.pk).execute().hits[0] return WebsiteLookupSerializer(obj).data
def test_index(self): with self.settings(ES_INDEXES={"website": "websites"}): eq_(WebsiteIndexer.get_index(), "websites")
def get_queryset(self): return WebsiteIndexer.search()
def get_obj(self): return WebsiteIndexer.search().filter('term', id=self.obj.pk).execute().hits[0]