def get(self, feed_id, order_type='normal'): res = [] for entry in storage.get_stories(feed_id): res.append({ "title": entry['title'], "_id": entry['_id'], "read": entry['read'], 'last_update': entry['last_update']}) # Must implement different order_type # Readed readed = [] for entry in res: if entry['read']: readed.append(entry) readed.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) # Unread unreaded = [] for entry in res: if not entry['read']: unreaded.append(entry) unreaded.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) return unreaded + readed
def get(self, feed_id, order_type='normal'): res = [] for entry in backend.get_stories(feed_id): res.append({ "title": entry['title'], "_id": entry['_id'], "read": entry['read'], 'last_update': entry['last_update'] }) # Must implement different order_type # Readed readed = [] for entry in res: if entry['read']: readed.append(entry) readed.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) # Unread unreaded = [] for entry in res: if not entry['read']: unreaded.append(entry) unreaded.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) return unreaded + readed
def run(self): self.data = feedparser.parse(self.feed['url']) # Update title if it change (yes some guys change it...) if self.data.feed['title'] != self.feed['title']: self.feed['title'] = self.data.feed['title'] storage.update_feed(self.feed['_id'], copy.copy(self.feed)) local_update = get_datetime(self.feed['last_update']) remote_update = False if self.data.feed.get('updated_parsed'): remote_update = get_datetime(self.data.feed.updated_parsed) remote_update_raw = get_dicttime(self.data.feed.updated_parsed) if self.data.get('updated_parsed'): if remote_update: if get_datetime(self.data.updated_parsed) > remote_update: remote_update = get_datetime(self.data.updated_parsed) remote_update_raw = get_dicttime(self.data.updated_parsed) if self.data.feed.get('published_parsed'): if remote_update: if get_datetime(self.data.feed.published_parsed) > remote_update: remote_update = get_datetime(self.data.feed.published_parsed) remote_update_raw = get_dicttime(self.data.feed.published_parsed) if self.data.get('published_parsed'): if remote_update: if get_datetime(self.data.published_parsed > remote_update): remote_update = get_datetime(self.data.published_parsed) remote_update_raw = get_dicttime(self.data.published_parsed) if not remote_update: remote_update = datetime.datetime.now() remote_update_raw = get_dicttime(remote_update.timetuple()) if remote_update > local_update: print('!! %s is outdated.' % self.feed['title'].encode('utf-8')) readed = [] for entry in storage.get_stories(self.feed['_id']): if entry['read']: readed.append(entry['title']) if len(self.data.entries) <= int(config.get('worker', 'story_before_retention')): do_retention = False else: do_retention = True retriever = Retriever(self.data, self.feed, do_retention=do_retention) retriever.start() retriever.join() for entry in readed: if storage.get_story_by_title(entry): entry = storage.get_story_by_title(entry) entry['read'] = True storage.update_story(entry['_id'], copy.copy(entry)) self.feed['last_update'] = remote_update_raw storage.update_feed(self.feed_id, self.feed) else: print('=> %s is up-to-date.' % self.feed['title'].encode('utf-8'))
def add(self, url): feed_guesser = self.get_feed(url) if feed_guesser['success']: feed, url = feed_guesser['output'] else: return feed_guesser # Bad feed if feed.version == '' or not feed.feed.get('title'): return {'success': False, 'output': 'Bad feed'} title = feed.feed['title'] link = feed.feed['link'] feed_id = storage.get_feed_by_title(title) if not feed_id: if feed.feed.get('updated_parsed'): feed_update = get_datetime(feed.feed.updated_parsed) elif feed.get('updated_parsed'): feed_update = get_datetime(feed.updated_parsed) elif feed.feed.get('published_parsed'): feed_update = get_datetime(feed.feed.published_parsed) elif feed.get('published_parsed'): feed_update = get_datetime(feed.published_parsed) else: feed_update = datetime.datetime.now() feed_id = storage.add_feed({ 'url': url, 'title': title, 'link': link, 'last_update': feed_update }) else: return {'success': False, 'output': 'Feed already exists'} retriever = Retriever(feed, do_retention=False) retriever.start() return { 'success': True, 'title': title, 'url': url, 'link': link, 'feed_id': feed_id, 'output': 'Feed added', 'counter': len(feed['entries']) }
def add(self, url): feed_guesser = self.get_feed(url) if feed_guesser['success']: feed, url = feed_guesser['output'] else: return feed_guesser # Bad feed if feed.version == '' or not feed.feed.get('title'): return {'success': False, 'output': 'Bad feed'} title = feed.feed['title'] link = feed.feed['link'] feed_id = storage.get_feed_by_title(title) if not feed_id: if feed.feed.get('updated_parsed'): feed_update = get_datetime(feed.feed.updated_parsed) elif feed.get('updated_parsed'): feed_update = get_datetime(feed.updated_parsed) elif feed.feed.get('published_parsed'): feed_update = get_datetime(feed.feed.published_parsed) elif feed.get('published_parsed'): feed_update = get_datetime(feed.published_parsed) else: feed_update = datetime.datetime.now() feed_id = storage.add_feed({'url': url, 'title': title, 'link': link, 'last_update': feed_update}) else: return {'success': False, 'output': 'Feed already exists'} retriever = Retriever(feed, do_retention=False) retriever.start() return { 'success': True, 'title': title, 'url': url, 'link': link, 'feed_id': feed_id, 'output': 'Feed added', 'counter': len(feed['entries'])}
def run(self): # This feed comes from database feed = storage.get_feed_by_title(self.title) feed_id = feed.get('_id') for entry in self.data: title = entry.get('title') link = entry.get('link') guid = entry.get('guid') or entry.get('id') or title if storage.get_story_by_guid(feed_id, guid): storage.remove_story( storage.get_story_by_guid(feed_id, guid).get('_id')) try: description = entry['content'][0]['value'] except KeyError: description = entry['summary'] if entry.get('updated_parsed'): last_update = get_datetime(entry.updated_parsed) else: last_update = datetime.datetime.now() if entry.get('published_parsed', False): published = get_datetime(entry.published_parsed) else: published = datetime.datetime.now() if self.do_retention: delta = datetime.datetime.now() - published if delta.days > int(config.get('worker', 'retention')): continue storage.add_story({ 'title': title, 'guid': guid, 'link': link, 'description': description, 'published': published, 'last_update': last_update, 'feed_id': feed_id, 'read': False })
def run(self): # This feed comes from database feed = storage.get_feed_by_title(self.title) feed_id = feed.get('_id') for entry in self.data: title = entry.get('title') link = entry.get('link') guid = entry.get('guid') or entry.get('id') or title if storage.get_story_by_guid(feed_id, guid): storage.remove_story(storage.get_story_by_guid(feed_id, guid).get('_id')) try: description = entry['content'][0]['value'] except KeyError: description = entry['summary'] if entry.get('updated_parsed'): last_update = get_datetime(entry.updated_parsed) else: last_update = datetime.datetime.now() if entry.get('published_parsed', False): published = get_datetime(entry.published_parsed) else: published = datetime.datetime.now() if self.do_retention: delta = datetime.datetime.now() - published if delta.days > int(config.get('worker', 'retention')): continue storage.add_story({ 'title': title, 'guid': guid, 'link': link, 'description': description, 'published': published, 'last_update': last_update, 'feed_id': feed_id, 'read': False})
def run(self): self.data = feedparser.parse(self.feed['url']) local_update = get_datetime(self.feed['last_update']) if self.data.feed.get('updated_parsed'): remote_update = get_datetime(self.data.feed.updated_parsed) remote_update_raw = get_dicttime(self.data.feed.updated_parsed) elif self.data.get('updated_parsed'): remote_update = get_datetime(self.data.updated_parsed) remote_update_raw = get_dicttime(self.data.updated_parsed) elif self.data.feed.get('published_parsed'): remote_update = get_datetime(self.data.feed.published_parsed) remote_update_raw = get_dicttime(self.data.feed.published_parsed) elif self.data.get('published_parsed'): remote_update = get_datetime(self.data.published_parsed) remote_update_raw = get_dicttime(self.data.published_parsed) else: remote_update = datetime.datetime.now() remote_update_raw = get_dicttime(remote_update.timetuple()) if remote_update > local_update: print(':: %s is outdated' % self.feed['title']) readed = [] for entry in storage.get_stories(self.feed['_id']): if entry['read']: readed.append(entry['title']) storage.remove_story(entry['_id']) retriever = Retriever(self.data) retriever.start() retriever.join() for entry in readed: if storage.get_story_by_title(entry): entry = storage.get_story_by_title(entry) entry['read'] = True storage.update_story(entry['_id'], copy.copy(entry)) self.feed['last_update'] = remote_update_raw storage.update_feed(self.feed_id, self.feed)
def run(self): self.data = feedparser.parse(self.feed['url']) local_update = get_datetime(self.feed['last_update']) if self.data.feed.get('updated_parsed'): remote_update = get_datetime(self.data.feed.updated_parsed) remote_update_raw = get_dicttime(self.data.feed.updated_parsed) elif self.data.get('updated_parsed'): remote_update = get_datetime(self.data.updated_parsed) remote_update_raw = get_dicttime(self.data.updated_parsed) elif self.data.feed.get('published_parsed'): remote_update = get_datetime(self.data.feed.published_parsed) remote_update_raw = get_dicttime(self.data.feed.published_parsed) elif self.data.get('published_parsed'): remote_update = get_datetime(self.data.published_parsed) remote_update_raw = get_dicttime(self.data.published_parsed) else: return if remote_update > local_update: print(':: %s is outdated' % self.feed['title']) readed = [] for entry in backend.get_stories(self.feed['_id']): if entry['read']: readed.append(entry['title']) backend.remove_story(entry['_id']) retriever = Retriever(self.data) retriever.start() retriever.join() for entry in readed: if backend.get_story_by_title(entry): entry = backend.get_story_by_title(entry) entry['read'] = True backend.update_story(entry['_id'], copy.copy(entry)) self.feed['last_update'] = remote_update_raw backend.update_feed(self.feed_id, self.feed)
def run(self): # This feed comes from database feed = storage.get_feed_by_title(self.title) for entry in self.data: title = entry['title'] link = entry['link'] try: description = entry['content'][0]['value'] except KeyError: description = entry['summary'] if entry.get('updated_parsed'): last_update = get_dicttime(entry.updated_parsed) else: last_update = get_dicttime(datetime.datetime.now().timetuple()) if entry.get('published_parsed', False): published = get_dicttime(entry.published_parsed) published_datetime = get_datetime(entry.published_parsed) else: published = get_dicttime(datetime.datetime.now().timetuple()) published_datetime = datetime.datetime.now() if self.do_retention: delta = published_datetime - datetime.datetime.now() if delta.days < -int(config.get('worker', 'retention')): break storage.add_story({ 'title': title, 'link': link, 'description': description, 'published': published, 'last_update': last_update, 'feed_id': feed['_id'], 'read': False})
def run(self): self.data = feedparser.parse(self.feed.get('url')) if self.data.get('bozo_exception', False): print( "!! Can't retrieve %s feed (%s)" % (self.feed_title.encode('utf-8'), self.data['bozo_exception'])) return need_update = False # Update title if it change if self.data.feed.get('title') != self.feed_title: self.feed['title'] = self.data.feed.get('title') self.feed_title = self.feed['title'] need_update = True # Add website url if not setted if self.data.feed.get('link') != self.feed.get('link'): self.feed['link'] = self.data.feed.get('link') self.feed_link = self.feed['link'] need_update = True if need_update: storage.update_feed(self.feed_id, copy.copy(self.feed)) local_update = self.feed.get('last_update') remote_update = False if self.data.feed.get('updated_parsed'): remote_update = get_datetime(self.data.feed.updated_parsed) if self.data.get('updated_parsed'): if remote_update: if get_datetime(self.data.updated_parsed) > remote_update: remote_update = get_datetime(self.data.updated_parsed) else: remote_update = get_datetime(self.data.updated_parsed) if self.data.feed.get('published_parsed'): if remote_update: if get_datetime( self.data.feed.published_parsed) > remote_update: remote_update = get_datetime( self.data.feed.published_parsed) else: remote_update = get_datetime(self.data.feed.published_parsed) if self.data.get('published_parsed'): if remote_update: if get_datetime(self.data.published_parsed) > remote_update: remote_update = get_datetime(self.data.published_parsed) else: remote_update = get_datetime(self.data.published_parsed) if not remote_update: remote_update = datetime.datetime.now() if remote_update > local_update: print('!! %s is outdated.' % self.feed_title.encode('utf-8')) readed = [] for entry in storage.get_stories(self.feed_id, "published", 0, 0): if entry.get('read'): readed.append(entry.get('guid')) if len(self.data.entries) <= int( config.get('worker', 'story_before_retention')): do_retention = False else: do_retention = True retriever = Retriever(self.data, self.feed, do_retention=do_retention) retriever.start() retriever.join() for entry_guid in readed: entry = storage.get_story_by_guid(self.feed_id, entry_guid) if entry: # print(' _ update: %s' % entry['title'].encode('utf-8')) entry['read'] = True storage.update_story(entry['_id'], copy.copy(entry)) self.feed['last_update'] = remote_update storage.update_feed(self.feed_id, self.feed) else: print('=> %s is up-to-date.' % self.feed_title.encode('utf-8'))
def get(self, feed_id=False, order_type="user", start=0, end=100): if not feed_id: if order_type == "user" or order_type not in ['unreaded', 'published']: order_type = storage.get_setting('all_items_ordering') if not order_type: storage.set_setting('all_items_ordering', 'unreaded') order_type = storage.get_setting('all_items_ordering') else: if order_type == "user" or order_type not in ['unreaded', 'published']: setting_order_type = storage.get_feed_setting(feed_id, 'ordering') if not setting_order_type: storage.set_feed_setting(feed_id, 'ordering', 'unreaded') order_type = 'unreaded' else: order_type = setting_order_type['value'] if not feed_id: stories = storage.all_stories() else: stories = storage.get_stories(feed_id) res = [] entries = [] if order_type == 'unreaded': for entry in stories: story = { "title": entry['title'], "_id": entry['_id'], "read": entry['read'], 'last_update': entry['last_update']} if not feed_id: story['feed_id'] = entry['feed_id'] res.append(story) # Readed readed = [] for entry in res: if entry['read']: readed.append(entry) readed.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) # Unread unreaded = [] for entry in res: if not entry['read']: unreaded.append(entry) unreaded.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) entries = unreaded + readed elif order_type == 'published': for entry in stories: story = { "title": entry['title'], "_id": entry['_id'], "read": entry['read'], 'last_update': entry['published']} if not feed_id: story['feed_id'] = entry['feed_id'] res.append(story) res.sort(key=lambda r: get_datetime(r['last_update']), reverse=True) entries = res return {'entries': entries, 'ordering': order_type}
def run(self): self.data = feedparser.parse(self.feed.get('url')) if self.data.get('bozo_exception', False): print("!! Can't retrieve %s feed (%s)" % (self.feed_title.encode('utf-8'), self.data['bozo_exception'])) return need_update = False # Update title if it change if self.data.feed.get('title') != self.feed_title: self.feed['title'] = self.data.feed.get('title') self.feed_title = self.feed['title'] need_update = True # Add website url if not setted if self.data.feed.get('link') != self.feed.get('link'): self.feed['link'] = self.data.feed.get('link') self.feed_link = self.feed['link'] need_update = True if need_update: storage.update_feed(self.feed_id, copy.copy(self.feed)) local_update = self.feed.get('last_update') remote_update = False if self.data.feed.get('updated_parsed'): remote_update = get_datetime(self.data.feed.updated_parsed) if self.data.get('updated_parsed'): if remote_update: if get_datetime(self.data.updated_parsed) > remote_update: remote_update = get_datetime(self.data.updated_parsed) else: remote_update = get_datetime(self.data.updated_parsed) if self.data.feed.get('published_parsed'): if remote_update: if get_datetime(self.data.feed.published_parsed) > remote_update: remote_update = get_datetime(self.data.feed.published_parsed) else: remote_update = get_datetime(self.data.feed.published_parsed) if self.data.get('published_parsed'): if remote_update: if get_datetime(self.data.published_parsed) > remote_update: remote_update = get_datetime(self.data.published_parsed) else: remote_update = get_datetime(self.data.published_parsed) if not remote_update: remote_update = datetime.datetime.now() if remote_update > local_update: print('!! %s is outdated.' % self.feed_title.encode('utf-8')) readed = [] for entry in storage.get_stories(self.feed_id, "published", 0, 0): if entry.get('read'): readed.append(entry.get('guid')) if len(self.data.entries) <= int(config.get('worker', 'story_before_retention')): do_retention = False else: do_retention = True retriever = Retriever(self.data, self.feed, do_retention=do_retention) retriever.start() retriever.join() for entry_guid in readed: entry = storage.get_story_by_guid(self.feed_id, entry_guid) if entry: # print(' _ update: %s' % entry['title'].encode('utf-8')) entry['read'] = True storage.update_story(entry['_id'], copy.copy(entry)) self.feed['last_update'] = remote_update storage.update_feed(self.feed_id, self.feed) else: print('=> %s is up-to-date.' % self.feed_title.encode('utf-8'))