def create_instances(self, data): created_count = 0 for i, row in enumerate(data): if (i + 1) % 100 == 0: print 'Processing row %d... (%d websites created)' % ( i + 1, created_count) if self.limit and created_count >= self.limit: print 'Limit (%d) was hit, stopping the import' % self.limit break id_ = int(self.clean_string(row['Unique Moz ID'])) rank = int(self.clean_string(row['Rank'])) try: website = Website.objects.get(moz_id=id_) if self.overwrite: # Existing website and we were asked to overwrite: delete # it! website.delete() else: # Existing website and we were not asked to overwrite: skip # it, storing its ranking first to set popularity later. if self.set_popularity: self.remember_website_ranking(website, rank) continue except Website.DoesNotExist: pass with atomic(): try: devices = [] if row['Mobile URL']: devices += [DEVICE_GAIA.id, DEVICE_MOBILE.id, DEVICE_TABLET.id] if row['TV URL']: devices.append(DEVICE_TV.id) website = Website(moz_id=id_, status=STATUS_PUBLIC, devices=devices) self.set_default_locale(website, row) self.set_automatic_properties(website, row) self.set_categories(website, row) self.set_preferred_regions(website, row) self.set_url(website, row) website.save() if self.set_popularity: # Remember ranking to set popularity later. self.remember_website_ranking(website, rank) # Keywords use a M2M, so do that once the website is saved. self.set_tags(website, row) # Launch task to fetch icon once we know everything is OK. self.set_icon(website, row) created_count += 1 except ParsingError as e: print e.message return created_count
def create_instances(self, data): created_count = 0 for i, row in enumerate(data): name = self.clean_string(row['Name']) if not name: continue try: url = self.clean_string(row['URL']) website = Website.objects.get(url=url) print 'Game with URL %s already exists. Continuing.' % url continue except Website.DoesNotExist: pass with atomic(): try: website = Website( categories=['games'], devices=[DEVICE_DESKTOP.id], description=self.clean_string(row['Description']), name=name, status=STATUS_PUBLIC, ) self.set_url(website, row) website.save() # Keywords use a M2M, so do that once the website is saved. self.set_tags(website, row) # Launch task to fetch imgs once we know everything is OK. try: self.set_icon(website, row) self.set_promo_imgs(website, row) WebsiteIndexer.index_ids([website.id], no_delay=True) except Exception as e: print e WebsiteIndexer.refresh_index() website.delete() raise e created_count += 1 except ParsingError as e: print e.message return created_count
def create_instances(self, data): created_count = 0 for i, row in enumerate(data): name = self.clean_string(row["Name"]) if not name: continue try: url = self.clean_string(row["URL"]) website = Website.objects.get(url=url) print "Game with URL %s already exists. Continuing." % url continue except Website.DoesNotExist: pass with atomic(): try: website = Website( categories=["games"], devices=[DEVICE_DESKTOP.id], description=self.clean_string(row["Description"]), name=name, status=STATUS_PUBLIC, ) self.set_url(website, row) website.save() # Keywords use a M2M, so do that once the website is saved. self.set_tags(website, row) # Launch task to fetch imgs once we know everything is OK. try: self.set_icon(website, row) self.set_promo_imgs(website, row) WebsiteIndexer.index_ids([website.id], no_delay=True) except Exception as e: print e WebsiteIndexer.refresh_index() website.delete() raise e created_count += 1 except ParsingError as e: print e.message return created_count