def update(self, force=False): if force: self.model_class.remove_except([]) news_list = self.parse_news_list() # add new items for news in news_list: # Use news url as the key if self.model_class.query.get(news['url']): logger.info('Updating %s', news['url']) # We need the url so we can't pop it here _news = news.copy() self.model_class.update(_news.pop('url'), **_news) else: logger.info("Fetching %s", news['url']) try: parser = legendary_parser_factory(news['url']) news['summary'] = parser.get_summary(summary_length) tm = parser.get_top_image() if tm: img_id = models.Image.add(content_type=tm.content_type, raw_data=tm.raw_data) news['img_id'] = img_id except Exception as e: logger.exception('Failed to fetch %s, %s', news['url'], e) self.model_class.add(**news) if not force: # clean up old items self.model_class.remove_except([n['url'] for n in news_list])
def insert_news(self, news, stats): try: logger.info("Fetching %s", news['url']) parser = legendary_parser_factory(news['url']) news['summary'] = parser.get_summary(summary_length) news['favicon'] = parser.get_favicon_url() tm = parser.get_illustration() if tm: img_id = models.Image.add(url=tm.url, content_type=tm.content_type, raw_data=tm.raw_data) news['img_id'] = img_id except Exception as e: logger.exception('Failed to fetch %s, %s', news['url'], e) stats['errors'].append(str(e)) finally: self.model_class.add(**news) stats['added'] += 1
def insert_news(self, news, stats): try: logger.info("Fetching %s", news['url']) parser = legendary_parser_factory(news['url']) news['summary'] = parser.get_summary(summary_length) news['favicon'] = parser.get_favicon_url() tm = parser.get_illustration() if tm: img_id = models.Image.add( url=tm.url, content_type=tm.content_type, raw_data=tm.raw_data) news['img_id'] = img_id except Exception as e: logger.exception('Failed to fetch %s, %s', news['url'], e) stats['errors'].append(str(e)) finally: self.model_class.add(**news) stats['added'] += 1
def update(self, force=False): stats = {'updated': 0, 'added': 0, 'errors': []} if force: self.model_class.remove_except([]) news_list = self.parse_news_list() # add new items for news in news_list: try: # Use news url as the key if self.model_class.query.get(news['url']): logger.info('Updating %s', news['url']) stats['updated'] += 1 # We need the url so we can't pop it here _news = news.copy() self.model_class.update(_news.pop('url'), **_news) else: logger.info("Fetching %s", news['url']) try: parser = legendary_parser_factory(news['url']) news['summary'] = parser.get_summary(summary_length) news['favicon'] = parser.get_favicon_url() tm = parser.get_top_image() if tm: img_id = models.Image.add( content_type=tm.content_type, raw_data=tm.raw_data) news['img_id'] = img_id except Exception as e: logger.exception('Failed to fetch %s, %s', news['url'], e) stats['errors'].append(str(e)) self.model_class.add(**news) stats['added'] += 1 except Exception as e: logger.exception(e) stats['errors'].append(str(e)) if not force: # clean up old items self.model_class.remove_except([n['url'] for n in news_list]) return stats
def update(self, force=False): stats = {'updated': 0, 'added': 0, 'errors': []} if force: self.model_class.remove_except([]) news_list = self.parse_news_list() # add new items for news in news_list: try: # Use news url as the key if self.model_class.query.get(news['url']): logger.info('Updating %s', news['url']) stats['updated'] += 1 # We need the url so we can't pop it here _news = news.copy() self.model_class.update(_news.pop('url'), **_news) else: logger.info("Fetching %s", news['url']) try: parser = legendary_parser_factory(news['url']) news['summary'] = parser.get_summary(summary_length) news['favicon'] = parser.get_favicon_url() tm = parser.get_illustration() if tm: img_id = models.Image.add(content_type=tm.content_type, raw_data=tm.raw_data) news['img_id'] = img_id except Exception as e: logger.exception('Failed to fetch %s, %s', news['url'], e) stats['errors'].append(str(e)) self.model_class.add(**news) stats['added'] += 1 except Exception as e: logger.exception(e) stats['errors'].append(str(e)) if not force: # clean up old items self.model_class.remove_except([n['url'] for n in news_list]) return stats