def fill_db_with_items(self, objects):
"""
Put items in db
"""
objs = []
for obj in objects:
if self.item_file_identifier_field in obj:
obj[self.item_field] = str(obj[self.item_field])
objs.append(obj)
objects = objs
json_items = {json_item[self.item_field]: json_item for json_item in objects} # Map items for easy treatment
items = {item.external_id: item for item in Item.objects.filter(external_id__in=json_items.keys())}
new_items = {}
categories = set([])
locales = set([])
regions = {}
with click.progressbar(json_items.items(), label="Loading items") as bar:
for item_eid, json_item in bar:
if item_eid not in items:
try:
name = json_item["name"][json_item["default_locale"]]
except (KeyError, TypeError):
name = json_item.get("name", "NO NAME")
except UnicodeEncodeError as e:
logging.error(e)
name = "NO NAME"
new_items[item_eid] = Item(external_id=item_eid, name=name)
# In case of json[self.item_genres_field] = None
json_categories = json_item.get(self.item_genres_field, None) or ()
if isinstance(json_categories, basestring):
categories.add(json_categories)
else:
categories = categories.union(json_item[self.item_genres_field])
# In case of json[self.item_locales_field] = None
json_locales = json_item.get(self.item_locales_field, None) or ()
if isinstance(json_locales, basestring):
locales.add(json_locales)
else:
locales = locales.union(json_locales)
for region in json_item.get("regions", None) or ():
try:
regions[region["name"]]["items"].append(item_eid)
except KeyError:
regions[region["name"]] = {"slug": region["slug"], "items": [item_eid]}
#logging.debug("Items ready to be saved")
Item.objects.bulk_create(new_items.values())
#logging.debug("%d new items saved with bulk_create" % len(new_items))
items.update({item.external_id: item for item in Item.objects.filter(external_id__in=json_items.keys())})
assert len(items) == len(objects), \
"Size of items and size of objects are different (%d != %d)" % (len(items), len(objects))
if "recommendation.diversity" in settings.INSTALLED_APPS:
#logging.debug("Preparing genres")
db_categories = self.get_genres(categories)
self.fill_item_genre(objects, items, db_categories)
#logging.debug("Genres loaded")
if "recommendation.language" in settings.INSTALLED_APPS:
#logging.debug("Preparing languages")
db_locales = self.get_locales(locales)
db_regions = self.get_regions(regions)
self.fill_item_locale(objects, items, db_locales)
self.fill_item_region(regions, items, db_regions)
