def create_instances(self, data):
created_count = 0
for i, row in enumerate(data):
name = self.clean_string(row["Name"])
if not name:
continue
try:
url = self.clean_string(row["URL"])
website = Website.objects.get(url=url)
print "Game with URL %s already exists. Continuing." % url
continue
except Website.DoesNotExist:
pass
with atomic():
try:
website = Website(
categories=["games"],
devices=[DEVICE_DESKTOP.id],
description=self.clean_string(row["Description"]),
name=name,
status=STATUS_PUBLIC,
)
self.set_url(website, row)
website.save()
# Keywords use a M2M, so do that once the website is saved.
self.set_tags(website, row)
# Launch task to fetch imgs once we know everything is OK.
try:
self.set_icon(website, row)
self.set_promo_imgs(website, row)
WebsiteIndexer.index_ids([website.id], no_delay=True)
except Exception as e:
print e
WebsiteIndexer.refresh_index()
website.delete()
raise e
created_count += 1
except ParsingError as e:
print e.message
return created_count
def create_instances(self, data):
created_count = 0
for i, row in enumerate(data):
name = self.clean_string(row['Name'])
if not name:
continue
try:
url = self.clean_string(row['URL'])
website = Website.objects.get(url=url)
print 'Game with URL %s already exists. Continuing.' % url
continue
except Website.DoesNotExist:
pass
with atomic():
try:
website = Website(
categories=['games'],
devices=[DEVICE_DESKTOP.id],
description=self.clean_string(row['Description']),
name=name,
status=STATUS_PUBLIC,
)
self.set_url(website, row)
website.save()
# Keywords use a M2M, so do that once the website is saved.
self.set_tags(website, row)
# Launch task to fetch imgs once we know everything is OK.
try:
self.set_icon(website, row)
self.set_promo_imgs(website, row)
WebsiteIndexer.index_ids([website.id], no_delay=True)
except Exception as e:
print e
WebsiteIndexer.refresh_index()
website.delete()
raise e
created_count += 1
except ParsingError as e:
print e.message
return created_count
def _get_indices(self):
# Check if we are filtering by a doc_type (e.g., apps, sites).
# Default to all content types.
doc_type = self.request.GET.get('doc_type', 'all')
app_index = WebappIndexer.get_index()
site_index = WebsiteIndexer.get_index()
if doc_type == 'webapp':
return [app_index]
elif doc_type == 'website':
return [site_index]
return [app_index, site_index]
def _get_doc_types(self):
# Check if we are filtering by a doc_type (e.g., apps, sites).
# Default to all content types.
doc_type = self.request.GET.get('doc_type', 'all')
app_doc = WebappIndexer.get_mapping_type_name()
site_doc = WebsiteIndexer.get_mapping_type_name()
if doc_type == 'webapp':
return [app_doc]
elif doc_type == 'website':
return [site_doc]
return [app_doc, site_doc]
def _get_doc_types(self):
# Check if we are filtering by a doc_type (e.g., apps, sites).
# Default to all content types.
doc_type = self.request.GET.get('doc_type', 'all')
app_doc = WebappIndexer.get_mapping_type_name()
site_doc = WebsiteIndexer.get_mapping_type_name()
if doc_type == 'webapp':
return [app_doc]
elif doc_type == 'website':
return [site_doc]
return [app_doc, site_doc]
def _get_indices(self):
# Check if we are filtering by a doc_type (e.g., apps, sites).
# Default to all content types.
doc_type = self.request.GET.get('doc_type', 'all')
app_index = WebappIndexer.get_index()
site_index = WebsiteIndexer.get_index()
if doc_type == 'webapp':
return [app_index]
elif doc_type == 'website':
return [site_index]
return [app_index, site_index]
def get_featured_websites(self):
"""
Get up to 11 featured MOWs for the request's region. If less than 11
are available, make up the difference with globally-featured MOWs.
"""
REGION_TAG = "featured-website-%s" % self.request.REGION.slug
region_filter = es_filter.Term(tags=REGION_TAG)
GLOBAL_TAG = "featured-website"
global_filter = es_filter.Term(tags=GLOBAL_TAG)
mow_query = query.Q(
"function_score",
filter=es_filter.Bool(should=[region_filter, global_filter]),
functions=[
SF("random_score", seed=self._get_daily_seed()),
es_function.BoostFactor(value=100.0, filter=region_filter),
],
)
es = Search(using=WebsiteIndexer.get_es())[:11]
results = es.query(mow_query).execute().hits
return ESWebsiteSerializer(results, many=True).data
def get_featured_websites(self):
"""
Get up to 11 featured MOWs for the request's region. If less than 11
are available, make up the difference with globally-featured MOWs.
"""
REGION_TAG = 'featured-website-%s' % self.request.REGION.slug
region_filter = es_filter.Term(tags=REGION_TAG)
GLOBAL_TAG = 'featured-website'
global_filter = es_filter.Term(tags=GLOBAL_TAG)
mow_query = query.Q(
'function_score',
filter=es_filter.Bool(should=[region_filter, global_filter]),
functions=[
SF('random_score', seed=self._get_daily_seed()),
es_function.BoostFactor(value=100.0, filter=region_filter)
],
)
es = Search(using=WebsiteIndexer.get_es())[:11]
results = es.query(mow_query).execute().hits
return ESWebsiteSerializer(results, many=True).data
def test_excluded_fields(self):
ok_(WebsiteIndexer.hidden_fields)
data = WebsiteIndexer.search().execute().hits
eq_(len(data), 1)
obj = data[0]
ok_('trending_2' not in obj)
ok_('popularity_2' not in obj)
ok_('description_translations' in obj)
ok_('description' not in obj)
ok_('description_l10n_english' not in obj)
ok_('name_translations' in obj)
ok_('name' not in obj)
ok_('name_l10n_english' not in obj)
ok_('name_sort' not in obj)
ok_('name.raw' not in obj)
ok_('short_name_translations' in obj)
ok_('short_name' not in obj)
ok_('short_name_l10n_english' not in obj)
def test_excluded_fields(self):
ok_(WebsiteIndexer.hidden_fields)
data = WebsiteIndexer.search().execute().hits
eq_(len(data), 1)
obj = data[0]
ok_("trending_2" not in obj)
ok_("popularity_2" not in obj)
ok_("description_translations" in obj)
ok_("description" not in obj)
ok_("description_l10n_english" not in obj)
ok_("name_translations" in obj)
ok_("name" not in obj)
ok_("name_l10n_english" not in obj)
ok_("name_sort" not in obj)
ok_("name.raw" not in obj)
ok_("short_name_translations" in obj)
ok_("short_name" not in obj)
ok_("short_name_l10n_english" not in obj)
def test_excluded_fields(self):
ok_(WebsiteIndexer.hidden_fields)
data = WebsiteIndexer.search().execute().hits
eq_(len(data), 1)
obj = data[0]
ok_('trending_2' not in obj)
ok_('popularity_2' not in obj)
ok_('description_translations' in obj)
ok_('description' not in obj)
ok_('description_l10n_english' not in obj)
ok_('name_translations' in obj)
ok_('name' not in obj)
ok_('name_l10n_english' not in obj)
ok_('name_sort' not in obj)
ok_('name.raw' not in obj)
ok_('short_name_translations' in obj)
ok_('short_name' not in obj)
ok_('short_name_l10n_english' not in obj)
def test_mapping(self):
mapping = WebsiteIndexer.get_mapping()
eq_(mapping.keys(), ["website"])
eq_(mapping["website"]["_all"], {"enabled": False})
def test_mapping(self):
mapping = WebsiteIndexer.get_mapping()
eq_(mapping.keys(), ['website'])
eq_(mapping['website']['_all'], {'enabled': False})
def test_index(self):
with self.settings(ES_INDEXES={'website': 'websites'}):
eq_(WebsiteIndexer.get_index(), 'websites')
def test_get_mapping_ok(self):
eq_(WebsiteIndexer.get_mapping_type_name(), 'website')
ok_(isinstance(self.indexer.get_mapping(), dict))
def serialize(self, site, profile=None):
data = WebsiteIndexer.search().filter('term',
id=site.pk).execute().hits[0]
a = TVESWebsiteSerializer(instance=data,
context={'request': self.request})
return a.data
def serialize(self, site, profile=None):
data = WebsiteIndexer.search().filter("term", id=site.pk).execute().hits[0]
a = TVESWebsiteSerializer(instance=data, context={"request": self.request})
return a.data
def get_queryset(self):
return Search(
using=BaseIndexer.get_es(),
index=[WebappIndexer.get_index(), WebsiteIndexer.get_index()],
doc_type=[WebappIndexer.get_mapping_type_name(),
WebsiteIndexer.get_mapping_type_name()])
def get_obj(self):
return WebsiteIndexer.search().filter(
'term', id=self.obj.pk).execute().hits[0]
def serialize(self):
obj = WebsiteIndexer.search().filter(
'term', id=self.website.pk).execute().hits[0]
return WebsiteLookupSerializer(obj).data
def test_mapping(self):
mapping = WebsiteIndexer.get_mapping()
eq_(mapping.keys(), ['website'])
eq_(mapping['website']['_all'], {'enabled': False})
def test_get_mapping_ok(self):
eq_(WebsiteIndexer.get_mapping_type_name(), 'website')
ok_(isinstance(self.indexer.get_mapping(), dict))
def test_index(self):
with self.settings(ES_INDEXES={"website": "websites"}):
eq_(WebsiteIndexer.get_index(), "websites")
def serialize(self):
obj = WebsiteIndexer.search().filter(
'term', id=self.website.pk).execute().hits[0]
return WebsiteLookupSerializer(obj).data
def test_index(self):
with self.settings(ES_INDEXES={'website': 'websites'}):
eq_(WebsiteIndexer.get_index(), 'websites')
def get_queryset(self):
return WebsiteIndexer.search()
def get_queryset(self):
return WebsiteIndexer.search()
def get_obj(self):
return WebsiteIndexer.search().filter('term',
id=self.obj.pk).execute().hits[0]
